1//===--- CGDeclCXX.cpp - Emit LLVM Code for C++ declarations --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code dealing with code generation of C++ declarations
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGDebugInfo.h"
15#include "CGHLSLRuntime.h"
16#include "CGObjCRuntime.h"
17#include "CGOpenMPRuntime.h"
18#include "CodeGenFunction.h"
19#include "TargetInfo.h"
20#include "clang/AST/Attr.h"
21#include "clang/Basic/LangOptions.h"
22#include "clang/CodeGenUtils/ModuleUtils.h"
23#include "llvm/ADT/StringExtras.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/MDBuilder.h"
26#include "llvm/Support/Path.h"
27
28using namespace clang;
29using namespace CodeGen;
30
31static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D,
32 ConstantAddress DeclPtr) {
33 assert(
34 (D.hasGlobalStorage() ||
35 (D.hasLocalStorage() && CGF.getContext().getLangOpts().OpenCLCPlusPlus)) &&
36 "VarDecl must have global or local (in the case of OpenCL) storage!");
37 assert(!D.getType()->isReferenceType() &&
38 "Should not call EmitDeclInit on a reference!");
39
40 QualType type = D.getType();
41 LValue lv = CGF.MakeAddrLValue(Addr: DeclPtr, T: type);
42
43 const Expr *Init = D.getInit();
44 switch (CGF.getEvaluationKind(T: type)) {
45 case TEK_Scalar: {
46 CodeGenModule &CGM = CGF.CGM;
47 if (lv.isObjCStrong())
48 CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF, src: CGF.EmitScalarExpr(E: Init),
49 dest: DeclPtr, threadlocal: D.getTLSKind());
50 else if (lv.isObjCWeak())
51 CGM.getObjCRuntime().EmitObjCWeakAssign(CGF, src: CGF.EmitScalarExpr(E: Init),
52 dest: DeclPtr);
53 else
54 CGF.EmitScalarInit(init: Init, D: &D, lvalue: lv, capturedByInit: false);
55 return;
56 }
57 case TEK_Complex:
58 CGF.EmitComplexExprIntoLValue(E: Init, dest: lv, /*isInit*/ true);
59 return;
60 case TEK_Aggregate:
61 CGF.EmitAggExpr(E: Init,
62 AS: AggValueSlot::forLValue(LV: lv, isDestructed: AggValueSlot::IsDestructed,
63 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
64 isAliased: AggValueSlot::IsNotAliased,
65 mayOverlap: AggValueSlot::DoesNotOverlap));
66 return;
67 }
68 llvm_unreachable("bad evaluation kind");
69}
70
71/// Emit code to cause the destruction of the given variable with
72/// static storage duration.
73static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
74 ConstantAddress Addr) {
75 // Honor __attribute__((no_destroy)) and bail instead of attempting
76 // to emit a reference to a possibly nonexistent destructor, which
77 // in turn can cause a crash. This will result in a global constructor
78 // that isn't balanced out by a destructor call as intended by the
79 // attribute. This also checks for -fno-c++-static-destructors and
80 // bails even if the attribute is not present.
81 QualType::DestructionKind DtorKind = D.needsDestruction(Ctx: CGF.getContext());
82
83 // FIXME: __attribute__((cleanup)) ?
84
85 switch (DtorKind) {
86 case QualType::DK_none:
87 return;
88
89 case QualType::DK_cxx_destructor:
90 break;
91
92 case QualType::DK_objc_strong_lifetime:
93 case QualType::DK_objc_weak_lifetime:
94 case QualType::DK_nontrivial_c_struct:
95 // We don't care about releasing objects during process teardown.
96 assert(!D.getTLSKind() && "should have rejected this");
97 return;
98 }
99
100 llvm::FunctionCallee Func;
101 llvm::Constant *Argument;
102
103 CodeGenModule &CGM = CGF.CGM;
104 QualType Type = D.getType();
105
106 // Special-case non-array C++ destructors, if they have the right signature.
107 // Under some ABIs, destructors return this instead of void, and cannot be
108 // passed directly to __cxa_atexit if the target does not allow this
109 // mismatch.
110 const CXXRecordDecl *Record = Type->getAsCXXRecordDecl();
111 bool CanRegisterDestructor =
112 Record && (!CGM.getCXXABI().HasThisReturn(
113 GD: GlobalDecl(Record->getDestructor(), Dtor_Complete)) ||
114 CGM.getCXXABI().canCallMismatchedFunctionType());
115 // If __cxa_atexit is disabled via a flag, a different helper function is
116 // generated elsewhere which uses atexit instead, and it takes the destructor
117 // directly.
118 bool UsingExternalHelper = !CGM.getCodeGenOpts().CXAAtExit;
119 if (Record && (CanRegisterDestructor || UsingExternalHelper)) {
120 assert(!Record->hasTrivialDestructor());
121 CXXDestructorDecl *Dtor = Record->getDestructor();
122
123 Func = CGM.getAddrAndTypeOfCXXStructor(GD: GlobalDecl(Dtor, Dtor_Complete));
124 if (CGF.getContext().getLangOpts().OpenCL) {
125 auto DestAS =
126 CGM.getTargetCodeGenInfo().getAddrSpaceOfCxaAtexitPtrParam();
127 auto DestTy = llvm::PointerType::get(
128 C&: CGM.getLLVMContext(), AddressSpace: CGM.getContext().getTargetAddressSpace(AS: DestAS));
129 auto SrcAS = D.getType().getQualifiers().getAddressSpace();
130 if (DestAS == SrcAS)
131 Argument = Addr.getPointer();
132 else
133 // FIXME: On addr space mismatch we are passing NULL. The generation
134 // of the global destructor function should be adjusted accordingly.
135 Argument = llvm::ConstantPointerNull::get(T: DestTy);
136 } else {
137 Argument = Addr.getPointer();
138 }
139 // Otherwise, the standard logic requires a helper function.
140 } else {
141 Addr = Addr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: Type));
142 Func = CodeGenFunction(CGM)
143 .generateDestroyHelper(addr: Addr, type: Type, destroyer: CGF.getDestroyer(destructionKind: DtorKind),
144 useEHCleanupForArray: CGF.needsEHCleanup(kind: DtorKind), VD: &D);
145 Argument = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
146 }
147
148 CGM.getCXXABI().registerGlobalDtor(CGF, D, Dtor: Func, Addr: Argument);
149}
150
151/// Emit code to cause the variable at the given address to be considered as
152/// constant from this point onwards.
153static void EmitDeclInvariant(CodeGenFunction &CGF, const VarDecl &D,
154 llvm::Constant *Addr) {
155 return CGF.EmitInvariantStart(
156 Addr, Size: CGF.getContext().getTypeSizeInChars(T: D.getType()));
157}
158
159void CodeGenFunction::EmitInvariantStart(llvm::Constant *Addr, CharUnits Size) {
160 // Do not emit the intrinsic if we're not optimizing.
161 if (!CGM.getCodeGenOpts().OptimizationLevel)
162 return;
163
164 // Grab the llvm.invariant.start intrinsic.
165 llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
166 // Overloaded address space type.
167 assert(Addr->getType()->isPointerTy() && "Address must be a pointer");
168 llvm::Type *ObjectPtr[1] = {Addr->getType()};
169 llvm::Function *InvariantStart = CGM.getIntrinsic(IID: InvStartID, Tys: ObjectPtr);
170
171 // Emit a call with the size in bytes of the object.
172 uint64_t Width = Size.getQuantity();
173 llvm::Value *Args[2] = {llvm::ConstantInt::getSigned(Ty: Int64Ty, V: Width), Addr};
174 Builder.CreateCall(Callee: InvariantStart, Args);
175}
176
177void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
178 llvm::GlobalVariable *GV,
179 bool PerformInit) {
180
181 const Expr *Init = D.getInit();
182 QualType T = D.getType();
183
184 // The address space of a static local variable (DeclPtr) may be different
185 // from the address space of the "this" argument of the constructor. In that
186 // case, we need an addrspacecast before calling the constructor.
187 //
188 // struct StructWithCtor {
189 // __device__ StructWithCtor() {...}
190 // };
191 // __device__ void foo() {
192 // __shared__ StructWithCtor s;
193 // ...
194 // }
195 //
196 // For example, in the above CUDA code, the static local variable s has a
197 // "shared" address space qualifier, but the constructor of StructWithCtor
198 // expects "this" in the "generic" address space.
199 unsigned ExpectedAddrSpace = getTypes().getTargetAddressSpace(T);
200 unsigned ActualAddrSpace = GV->getAddressSpace();
201 llvm::Constant *DeclPtr = GV;
202 if (ActualAddrSpace != ExpectedAddrSpace) {
203 llvm::PointerType *PTy =
204 llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: ExpectedAddrSpace);
205 DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(C: DeclPtr, Ty: PTy);
206 }
207
208 ConstantAddress DeclAddr(
209 DeclPtr, GV->getValueType(), getContext().getDeclAlign(D: &D));
210
211 if (!T->isReferenceType()) {
212 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
213 D.hasAttr<OMPThreadPrivateDeclAttr>()) {
214 (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition(
215 VD: &D, VDAddr: DeclAddr, Loc: D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
216 PerformInit, CGF: this);
217 }
218 bool NeedsDtor =
219 D.needsDestruction(Ctx: getContext()) == QualType::DK_cxx_destructor;
220 if (PerformInit)
221 EmitDeclInit(CGF&: *this, D, DeclPtr: DeclAddr);
222 if (D.getType().isConstantStorage(Ctx: getContext(), ExcludeCtor: true, ExcludeDtor: !NeedsDtor))
223 EmitDeclInvariant(CGF&: *this, D, Addr: DeclPtr);
224 else
225 EmitDeclDestroy(CGF&: *this, D, Addr: DeclAddr);
226 return;
227 }
228
229 assert(PerformInit && "cannot have constant initializer which needs "
230 "destruction for reference");
231 RValue RV = EmitReferenceBindingToExpr(E: Init);
232 EmitStoreOfScalar(Value: RV.getScalarVal(), Addr: DeclAddr, Volatile: false, Ty: T);
233}
234
235/// Create a stub function, suitable for being passed to atexit,
236/// which passes the given address to the given destructor function.
237llvm::Constant *CodeGenFunction::createAtExitStub(const VarDecl &VD,
238 llvm::FunctionCallee dtor,
239 llvm::Constant *addr) {
240 // Get the destructor function type, void(*)(void).
241 llvm::FunctionType *ty = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
242 SmallString<256> FnName;
243 {
244 llvm::raw_svector_ostream Out(FnName);
245 CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(D: &VD, Out);
246 }
247
248 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
249 llvm::Function *fn = CGM.CreateGlobalInitOrCleanUpFunction(
250 ty, name: FnName.str(), FI, Loc: VD.getLocation());
251
252 CodeGenFunction CGF(CGM);
253
254 CGF.StartFunction(GD: GlobalDecl(&VD, DynamicInitKind::AtExit),
255 RetTy: CGM.getContext().VoidTy, Fn: fn, FnInfo: FI, Args: FunctionArgList(),
256 Loc: VD.getLocation(), StartLoc: VD.getInit()->getExprLoc());
257 // Emit an artificial location for this function.
258 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
259
260 llvm::CallInst *call = CGF.Builder.CreateCall(Callee: dtor, Args: addr);
261
262 // Make sure the call and the callee agree on calling convention.
263 if (auto *dtorFn = dyn_cast<llvm::Function>(
264 Val: dtor.getCallee()->stripPointerCastsAndAliases()))
265 call->setCallingConv(dtorFn->getCallingConv());
266
267 CGF.FinishFunction();
268
269 // Get a proper function pointer.
270 FunctionProtoType::ExtProtoInfo EPI(getContext().getDefaultCallingConvention(
271 /*IsVariadic=*/false, /*IsCXXMethod=*/false));
272 QualType fnType = getContext().getFunctionType(ResultTy: getContext().VoidTy,
273 Args: {getContext().VoidPtrTy}, EPI);
274 return CGM.getFunctionPointer(Pointer: fn, FunctionType: fnType);
275}
276
277/// Create a stub function, suitable for being passed to __pt_atexit_np,
278/// which passes the given address to the given destructor function.
279llvm::Function *CodeGenFunction::createTLSAtExitStub(
280 const VarDecl &D, llvm::FunctionCallee Dtor, llvm::Constant *Addr,
281 llvm::FunctionCallee &AtExit) {
282 SmallString<256> FnName;
283 {
284 llvm::raw_svector_ostream Out(FnName);
285 CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(D: &D, Out);
286 }
287
288 const CGFunctionInfo &FI = CGM.getTypes().arrangeLLVMFunctionInfo(
289 returnType: getContext().IntTy, opts: FnInfoOpts::None, argTypes: {getContext().IntTy},
290 info: FunctionType::ExtInfo(), paramInfos: {}, args: RequiredArgs::All,
291 /*ABIInfoFD=*/nullptr);
292
293 // Get the stub function type, int(*)(int,...).
294 llvm::FunctionType *StubTy =
295 llvm::FunctionType::get(Result: CGM.IntTy, Params: {CGM.IntTy}, isVarArg: true);
296
297 llvm::Function *DtorStub = CGM.CreateGlobalInitOrCleanUpFunction(
298 ty: StubTy, name: FnName.str(), FI, Loc: D.getLocation());
299
300 CodeGenFunction CGF(CGM);
301
302 auto *IPD = ImplicitParamDecl::Create(
303 C&: CGM.getContext(), T: CGM.getContext().IntTy, ParamKind: ImplicitParamKind::Other);
304 QualType ResTy = CGM.getContext().IntTy;
305
306 FunctionArgList Args{IPD};
307 CGF.StartFunction(GD: GlobalDecl(&D, DynamicInitKind::AtExit), RetTy: ResTy, Fn: DtorStub,
308 FnInfo: FI, Args, Loc: D.getLocation(), StartLoc: D.getInit()->getExprLoc());
309
310 // Emit an artificial location for this function.
311 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
312
313 llvm::CallInst *call = CGF.Builder.CreateCall(Callee: Dtor, Args: Addr);
314
315 // Make sure the call and the callee agree on calling convention.
316 if (auto *DtorFn = dyn_cast<llvm::Function>(
317 Val: Dtor.getCallee()->stripPointerCastsAndAliases()))
318 call->setCallingConv(DtorFn->getCallingConv());
319
320 // Return 0 from function
321 CGF.Builder.CreateStore(Val: llvm::Constant::getNullValue(Ty: CGM.IntTy),
322 Addr: CGF.ReturnValue);
323
324 CGF.FinishFunction();
325
326 return DtorStub;
327}
328
329/// Register a global destructor using the C atexit runtime function.
330void CodeGenFunction::registerGlobalDtorWithAtExit(const VarDecl &VD,
331 llvm::FunctionCallee dtor,
332 llvm::Constant *addr) {
333 // Create a function which calls the destructor.
334 llvm::Constant *dtorStub = createAtExitStub(VD, dtor, addr);
335 registerGlobalDtorWithAtExit(dtorStub);
336}
337
338/// Register a global destructor using the LLVM 'llvm.global_dtors' global.
339void CodeGenFunction::registerGlobalDtorWithLLVM(const VarDecl &VD,
340 llvm::FunctionCallee Dtor,
341 llvm::Constant *Addr) {
342 // Create a function which calls the destructor.
343 llvm::Function *dtorStub =
344 cast<llvm::Function>(Val: createAtExitStub(VD, dtor: Dtor, addr: Addr));
345 CGM.AddGlobalDtor(Dtor: dtorStub);
346}
347
348void CodeGenFunction::registerGlobalDtorWithAtExit(llvm::Constant *dtorStub) {
349 // extern "C" int atexit(void (*f)(void));
350 assert(dtorStub->getType()->isPointerTy() &&
351 "Argument to atexit has a wrong type.");
352
353 llvm::FunctionType *atexitTy =
354 llvm::FunctionType::get(Result: IntTy, Params: dtorStub->getType(), isVarArg: false);
355
356 llvm::FunctionCallee atexit =
357 CGM.CreateRuntimeFunction(Ty: atexitTy, Name: "atexit", ExtraAttrs: llvm::AttributeList(),
358 /*Local=*/true);
359 if (llvm::Function *atexitFn = dyn_cast<llvm::Function>(Val: atexit.getCallee()))
360 atexitFn->setDoesNotThrow();
361
362 EmitNounwindRuntimeCall(callee: atexit, args: dtorStub);
363}
364
365llvm::Value *
366CodeGenFunction::unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub) {
367 // The unatexit subroutine unregisters __dtor functions that were previously
368 // registered by the atexit subroutine. If the referenced function is found,
369 // it is removed from the list of functions that are called at normal program
370 // termination and the unatexit returns a value of 0, otherwise a non-zero
371 // value is returned.
372 //
373 // extern "C" int unatexit(void (*f)(void));
374 assert(dtorStub->getType()->isPointerTy() &&
375 "Argument to unatexit has a wrong type.");
376
377 llvm::FunctionType *unatexitTy =
378 llvm::FunctionType::get(Result: IntTy, Params: {dtorStub->getType()}, /*isVarArg=*/false);
379
380 llvm::FunctionCallee unatexit =
381 CGM.CreateRuntimeFunction(Ty: unatexitTy, Name: "unatexit", ExtraAttrs: llvm::AttributeList());
382
383 cast<llvm::Function>(Val: unatexit.getCallee())->setDoesNotThrow();
384
385 return EmitNounwindRuntimeCall(callee: unatexit, args: dtorStub);
386}
387
388void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D,
389 llvm::GlobalVariable *DeclPtr,
390 bool PerformInit) {
391 // If we've been asked to forbid guard variables, emit an error now.
392 // This diagnostic is hard-coded for Darwin's use case; we can find
393 // better phrasing if someone else needs it.
394 if (CGM.getCodeGenOpts().ForbidGuardVariables)
395 CGM.Error(loc: D.getLocation(),
396 error: "this initialization requires a guard variable, which "
397 "the kernel does not support");
398
399 CGM.getCXXABI().EmitGuardedInit(CGF&: *this, D, DeclPtr, PerformInit);
400}
401
402void CodeGenFunction::EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
403 llvm::BasicBlock *InitBlock,
404 llvm::BasicBlock *NoInitBlock,
405 GuardKind Kind,
406 const VarDecl *D) {
407 assert((Kind == GuardKind::TlsGuard || D) && "no guarded variable");
408
409 // A guess at how many times we will enter the initialization of a
410 // variable, depending on the kind of variable.
411 static const uint64_t InitsPerTLSVar = 1024;
412 static const uint64_t InitsPerLocalVar = 1024 * 1024;
413
414 llvm::MDNode *Weights;
415 if (Kind == GuardKind::VariableGuard && !D->isLocalVarDecl()) {
416 // For non-local variables, don't apply any weighting for now. Due to our
417 // use of COMDATs, we expect there to be at most one initialization of the
418 // variable per DSO, but we have no way to know how many DSOs will try to
419 // initialize the variable.
420 Weights = nullptr;
421 } else {
422 uint64_t NumInits;
423 // FIXME: For the TLS case, collect and use profiling information to
424 // determine a more accurate brach weight.
425 if (Kind == GuardKind::TlsGuard || D->getTLSKind())
426 NumInits = InitsPerTLSVar;
427 else
428 NumInits = InitsPerLocalVar;
429
430 // The probability of us entering the initializer is
431 // 1 / (total number of times we attempt to initialize the variable).
432 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
433 Weights = MDHelper.createBranchWeights(TrueWeight: 1, FalseWeight: NumInits - 1);
434 }
435
436 Builder.CreateCondBr(Cond: NeedsInit, True: InitBlock, False: NoInitBlock, BranchWeights: Weights);
437}
438
439llvm::Function *CodeGenModule::CreateGlobalInitOrCleanUpFunction(
440 llvm::FunctionType *FTy, const Twine &Name, const CGFunctionInfo &FI,
441 SourceLocation Loc, bool TLS, llvm::GlobalVariable::LinkageTypes Linkage) {
442 llvm::Function *Fn = llvm::Function::Create(Ty: FTy, Linkage, N: Name, M: &getModule());
443
444 if (!getLangOpts().AppleKext && !TLS) {
445 // Set the section if needed.
446 if (const char *Section = getTarget().getStaticInitSectionSpecifier())
447 Fn->setSection(Section);
448 }
449
450 if (Linkage == llvm::GlobalVariable::InternalLinkage)
451 SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
452 else {
453 SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, IsThunk: false);
454 SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
455 getTargetCodeGenInfo().setTargetAttributes(D: nullptr, GV: Fn, M&: *this);
456 }
457
458 Fn->setCallingConv(getRuntimeCC());
459
460 if (!getLangOpts().Exceptions)
461 Fn->setDoesNotThrow();
462
463 if (getLangOpts().Sanitize.has(K: SanitizerKind::Address) &&
464 !isInNoSanitizeList(Kind: SanitizerKind::Address, Fn, Loc))
465 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
466
467 if (getLangOpts().Sanitize.has(K: SanitizerKind::KernelAddress) &&
468 !isInNoSanitizeList(Kind: SanitizerKind::KernelAddress, Fn, Loc))
469 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
470
471 if (getLangOpts().Sanitize.has(K: SanitizerKind::HWAddress) &&
472 !isInNoSanitizeList(Kind: SanitizerKind::HWAddress, Fn, Loc))
473 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
474
475 if (getLangOpts().Sanitize.has(K: SanitizerKind::KernelHWAddress) &&
476 !isInNoSanitizeList(Kind: SanitizerKind::KernelHWAddress, Fn, Loc))
477 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
478
479 if (getLangOpts().Sanitize.has(K: SanitizerKind::MemtagStack) &&
480 !isInNoSanitizeList(Kind: SanitizerKind::MemtagStack, Fn, Loc))
481 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemTag);
482
483 if (getLangOpts().Sanitize.has(K: SanitizerKind::Type) &&
484 !isInNoSanitizeList(Kind: SanitizerKind::Type, Fn, Loc))
485 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeType);
486
487 if (getLangOpts().Sanitize.has(K: SanitizerKind::Thread) &&
488 !isInNoSanitizeList(Kind: SanitizerKind::Thread, Fn, Loc))
489 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeThread);
490
491 if (getLangOpts().Sanitize.has(K: SanitizerKind::NumericalStability) &&
492 !isInNoSanitizeList(Kind: SanitizerKind::NumericalStability, Fn, Loc))
493 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeNumericalStability);
494
495 if (getLangOpts().Sanitize.has(K: SanitizerKind::Memory) &&
496 !isInNoSanitizeList(Kind: SanitizerKind::Memory, Fn, Loc))
497 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
498
499 if (getLangOpts().Sanitize.has(K: SanitizerKind::KernelMemory) &&
500 !isInNoSanitizeList(Kind: SanitizerKind::KernelMemory, Fn, Loc))
501 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
502
503 if (getLangOpts().Sanitize.has(K: SanitizerKind::SafeStack) &&
504 !isInNoSanitizeList(Kind: SanitizerKind::SafeStack, Fn, Loc))
505 Fn->addFnAttr(Kind: llvm::Attribute::SafeStack);
506
507 if (getLangOpts().Sanitize.has(K: SanitizerKind::ShadowCallStack) &&
508 !isInNoSanitizeList(Kind: SanitizerKind::ShadowCallStack, Fn, Loc))
509 Fn->addFnAttr(Kind: llvm::Attribute::ShadowCallStack);
510
511 return Fn;
512}
513
514/// Create a global pointer to a function that will initialize a global
515/// variable. The user has requested that this pointer be emitted in a specific
516/// section.
517void CodeGenModule::EmitPointerToInitFunc(const VarDecl *D,
518 llvm::GlobalVariable *GV,
519 llvm::Function *InitFunc,
520 InitSegAttr *ISA) {
521 llvm::GlobalVariable *PtrArray = new llvm::GlobalVariable(
522 TheModule, InitFunc->getType(), /*isConstant=*/true,
523 llvm::GlobalValue::PrivateLinkage, InitFunc, "__cxx_init_fn_ptr");
524 PtrArray->setSection(ISA->getSection());
525 addUsedGlobal(GV: PtrArray);
526
527 // If the GV is already in a comdat group, then we have to join it.
528 if (llvm::Comdat *C = GV->getComdat())
529 PtrArray->setComdat(C);
530}
531
532void
533CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
534 llvm::GlobalVariable *Addr,
535 bool PerformInit) {
536
537 // According to E.2.3.1 in CUDA-7.5 Programming guide: __device__,
538 // __constant__ and __shared__ variables defined in namespace scope,
539 // that are of class type, cannot have a non-empty constructor. All
540 // the checks have been done in Sema by now. Whatever initializers
541 // are allowed are empty and we just need to ignore them here.
542 if (getLangOpts().CUDAIsDevice && !getLangOpts().GPUAllowDeviceInit &&
543 (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
544 D->hasAttr<CUDASharedAttr>()))
545 return;
546
547 // Check if we've already initialized this decl.
548 auto I = DelayedCXXInitPosition.find(Val: D);
549 if (I != DelayedCXXInitPosition.end() && I->second == ~0U)
550 return;
551
552 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
553 SmallString<256> FnName;
554 {
555 llvm::raw_svector_ostream Out(FnName);
556 getCXXABI().getMangleContext().mangleDynamicInitializer(D, Out);
557 }
558
559 // Create a variable initialization function.
560 llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
561 FTy, Name: FnName.str(), FI: getTypes().arrangeNullaryFunction(), Loc: D->getLocation());
562
563 auto *ISA = D->getAttr<InitSegAttr>();
564 CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D, Addr,
565 PerformInit);
566
567 llvm::GlobalVariable *COMDATKey =
568 supportsCOMDAT() && D->isExternallyVisible() ? Addr : nullptr;
569
570 if (D->getTLSKind()) {
571 // FIXME: Should we support init_priority for thread_local?
572 // FIXME: We only need to register one __cxa_thread_atexit function for the
573 // entire TU.
574 CXXThreadLocalInits.push_back(x: Fn);
575 CXXThreadLocalInitVars.push_back(x: D);
576 } else if (PerformInit && ISA) {
577 // Contract with backend that "init_seg(compiler)" corresponds to priority
578 // 200 and "init_seg(lib)" corresponds to priority 400.
579 int Priority = -1;
580 if (ISA->getSection() == ".CRT$XCC")
581 Priority = 200;
582 else if (ISA->getSection() == ".CRT$XCL")
583 Priority = 400;
584
585 if (Priority != -1)
586 AddGlobalCtor(Ctor: Fn, Priority, LexOrder: ~0U, AssociatedData: COMDATKey);
587 else
588 EmitPointerToInitFunc(D, GV: Addr, InitFunc: Fn, ISA);
589 } else if (auto *IPA = D->getAttr<InitPriorityAttr>()) {
590 OrderGlobalInitsOrStermFinalizers Key(IPA->getPriority(),
591 PrioritizedCXXGlobalInits.size());
592 PrioritizedCXXGlobalInits.push_back(Elt: std::make_pair(x&: Key, y&: Fn));
593 } else if (isTemplateInstantiation(Kind: D->getTemplateSpecializationKind()) ||
594 !isUniqueGVALinkage(L: getContext().GetGVALinkageForVariable(VD: D)) ||
595 D->hasAttr<SelectAnyAttr>()) {
596 // For vague linkage globals, put the initializer into its own global_ctors
597 // entry with the global as a comdat key. This ensures at most one
598 // initializer per DSO runs during DSO dynamic initialization.
599 //
600 // For ELF platforms, this is an important code size and startup time
601 // optimization. For dynamic, non-hidden symbols, the weak guard variable
602 // remains to ensure that other DSOs do not re-initialize the global.
603 //
604 // For PE-COFF platforms, there is no guard variable, and COMDAT
605 // associativity is the only way to ensure vauge linkage globals are
606 // initialized exactly once.
607 //
608 // MachO is the only remaining platform with no comdats that doesn't
609 // benefit from this optimization. The rest are mainly modeled on ELF
610 // behavior.
611 //
612 // C++ requires that inline global variables are initialized in source
613 // order, but this requirement does not exist for templated entities.
614 // llvm.global_ctors does not guarantee initialization order, so in
615 // general, Clang does not fully conform to the ordering requirement.
616 // However, in practice, LLVM emits global_ctors in the provided order, and
617 // users typically don't rely on ordering between inline globals in
618 // different headers which are then transitively included in varying order.
619 // Clang's current behavior is a practical tradeoff, since dropping the
620 // comdat would lead to unacceptable impact on code size and startup time.
621 //
622 // FIXME: Find a solution to guarantee source-order initialization of
623 // inline variables.
624 //
625 // C++ [basic.start.init]p2:
626 // Definitions of explicitly specialized class template static data
627 // members have ordered initialization. Other class template static data
628 // members (i.e., implicitly or explicitly instantiated specializations)
629 // have unordered initialization.
630 //
631 // CXXGlobalInits.size() is the lex order number for the next deferred
632 // VarDecl. Use it when the current VarDecl is non-deferred. Although this
633 // lex order number is shared between current VarDecl and some following
634 // VarDecls, their order of insertion into `llvm.global_ctors` is the same
635 // as the lexing order and the following stable sort would preserve such
636 // order.
637 I = DelayedCXXInitPosition.find(Val: D);
638 unsigned LexOrder =
639 I == DelayedCXXInitPosition.end() ? CXXGlobalInits.size() : I->second;
640 AddGlobalCtor(Ctor: Fn, Priority: 65535, LexOrder, AssociatedData: COMDATKey);
641 if (COMDATKey && (getTriple().isOSBinFormatELF() ||
642 getTarget().getCXXABI().isMicrosoft())) {
643 // When COMDAT is used on ELF or in the MS C++ ABI, the key must be in
644 // llvm.used to prevent linker GC.
645 addUsedGlobal(GV: COMDATKey);
646 }
647
648 // If we used a COMDAT key for the global ctor, the init function can be
649 // discarded if the global ctor entry is discarded.
650 // FIXME: Do we need to restrict this to ELF and Wasm?
651 llvm::Comdat *C = Addr->getComdat();
652 if (COMDATKey && C &&
653 (getTarget().getTriple().isOSBinFormatELF() ||
654 getTarget().getTriple().isOSBinFormatWasm())) {
655 Fn->setComdat(C);
656 }
657 } else {
658 I = DelayedCXXInitPosition.find(Val: D); // Re-do lookup in case of re-hash.
659 if (I == DelayedCXXInitPosition.end()) {
660 CXXGlobalInits.push_back(x: Fn);
661 } else if (I->second != ~0U) {
662 assert(I->second < CXXGlobalInits.size() &&
663 CXXGlobalInits[I->second] == nullptr);
664 CXXGlobalInits[I->second] = Fn;
665 }
666 }
667
668 // Remember that we already emitted the initializer for this global.
669 DelayedCXXInitPosition[D] = ~0U;
670}
671
672void CodeGenModule::EmitCXXThreadLocalInitFunc() {
673 getCXXABI().EmitThreadLocalInitFuncs(
674 CGM&: *this, CXXThreadLocals, CXXThreadLocalInits, CXXThreadLocalInitVars);
675
676 CXXThreadLocalInits.clear();
677 CXXThreadLocalInitVars.clear();
678 CXXThreadLocals.clear();
679}
680
681/* Build the initializer for a C++20 module:
682 This is arranged to be run only once regardless of how many times the module
683 might be included transitively. This arranged by using a guard variable.
684
685 If there are no initializers at all (and also no imported modules) we reduce
686 this to an empty function (since the Itanium ABI requires that this function
687 be available to a caller, which might be produced by a different
688 implementation).
689
690 First we call any initializers for imported modules.
691 We then call initializers for the Global Module Fragment (if present)
692 We then call initializers for the current module.
693 We then call initializers for the Private Module Fragment (if present)
694*/
695
696void CodeGenModule::EmitCXXModuleInitFunc(Module *Primary) {
697 assert(Primary->isInterfaceOrPartition() &&
698 "The function should only be called for C++20 named module interface"
699 " or partition.");
700
701 while (!CXXGlobalInits.empty() && !CXXGlobalInits.back())
702 CXXGlobalInits.pop_back();
703
704 // As noted above, we create the function, even if it is empty.
705 // Module initializers for imported modules are emitted first.
706
707 SmallVector<llvm::Function *, 8> ModuleInits;
708 for (Module *M : CodeGenUtils::importedModulesToInitialize(Primary, Imported: {})) {
709 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
710 SmallString<256> FnName;
711 {
712 llvm::raw_svector_ostream Out(FnName);
713 cast<ItaniumMangleContext>(Val&: getCXXABI().getMangleContext())
714 .mangleModuleInitializer(Module: M, Out);
715 }
716 assert(!GetGlobalValue(FnName.str()) &&
717 "We should only have one use of the initializer call");
718 llvm::Function *Fn = llvm::Function::Create(
719 Ty: FTy, Linkage: llvm::Function::ExternalLinkage, N: FnName.str(), M: &getModule());
720 ModuleInits.push_back(Elt: Fn);
721 }
722
723 // Add any initializers with specified priority; this uses the same approach
724 // as EmitCXXGlobalInitFunc().
725 if (!PrioritizedCXXGlobalInits.empty()) {
726 llvm::array_pod_sort(Start: PrioritizedCXXGlobalInits.begin(),
727 End: PrioritizedCXXGlobalInits.end());
728 for (SmallVectorImpl<GlobalInitData>::iterator
729 I = PrioritizedCXXGlobalInits.begin(),
730 E = PrioritizedCXXGlobalInits.end();
731 I != E;) {
732 SmallVectorImpl<GlobalInitData>::iterator PrioE =
733 std::upper_bound(first: I + 1, last: E, val: *I, comp: GlobalInitPriorityCmp());
734
735 for (; I < PrioE; ++I)
736 ModuleInits.push_back(Elt: I->second);
737 }
738 }
739
740 // Now append the ones without specified priority.
741 for (auto *F : CXXGlobalInits)
742 ModuleInits.push_back(Elt: F);
743
744 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
745 const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
746
747 // We now build the initializer for this module, which has a mangled name
748 // as per the Itanium ABI . The action of the initializer is guarded so that
749 // each init is run just once (even though a module might be imported
750 // multiple times via nested use).
751 llvm::Function *Fn;
752 {
753 SmallString<256> InitFnName;
754 llvm::raw_svector_ostream Out(InitFnName);
755 cast<ItaniumMangleContext>(Val&: getCXXABI().getMangleContext())
756 .mangleModuleInitializer(Module: Primary, Out);
757 Fn = CreateGlobalInitOrCleanUpFunction(
758 FTy, Name: llvm::Twine(InitFnName), FI, Loc: SourceLocation(), TLS: false,
759 Linkage: llvm::GlobalVariable::ExternalLinkage);
760
761 // If we have a completely empty initializer then we do not want to create
762 // the guard variable.
763 ConstantAddress GuardAddr = ConstantAddress::invalid();
764 if (!ModuleInits.empty()) {
765 // Create the guard var.
766 llvm::GlobalVariable *Guard = new llvm::GlobalVariable(
767 getModule(), Int8Ty, /*isConstant=*/false,
768 llvm::GlobalVariable::InternalLinkage,
769 llvm::ConstantInt::get(Ty: Int8Ty, V: 0), InitFnName.str() + "__in_chrg");
770 CharUnits GuardAlign = CharUnits::One();
771 Guard->setAlignment(GuardAlign.getAsAlign());
772 GuardAddr = ConstantAddress(Guard, Int8Ty, GuardAlign);
773 }
774 CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXThreadLocals: ModuleInits,
775 Guard: GuardAddr);
776 }
777
778 // We allow for the case that a module object is added to a linked binary
779 // without a specific call to the the initializer. This also ensures that
780 // implementation partition initializers are called when the partition
781 // is not imported as an interface.
782 AddGlobalCtor(Ctor: Fn);
783
784 // See the comment in EmitCXXGlobalInitFunc about OpenCL global init
785 // functions.
786 if (getLangOpts().OpenCL) {
787 GenKernelArgMetadata(FN: Fn);
788 Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
789 }
790
791 assert(!getLangOpts().CUDA || !getLangOpts().CUDAIsDevice ||
792 getLangOpts().GPUAllowDeviceInit);
793 if (getLangOpts().HIP && getLangOpts().CUDAIsDevice) {
794 if (getTriple().isSPIRV())
795 Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
796 else
797 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
798 Fn->addFnAttr(Kind: "device-init");
799 }
800
801 // We are done with the inits.
802 PrioritizedCXXGlobalInits.clear();
803 CXXGlobalInits.clear();
804 ModuleInits.clear();
805}
806
807static SmallString<128> getTransformedFileName(llvm::Module &M) {
808 SmallString<128> FileName = llvm::sys::path::filename(path: M.getName());
809
810 if (FileName.empty())
811 FileName = "<null>";
812
813 for (size_t i = 0; i < FileName.size(); ++i) {
814 // Replace everything that's not [a-zA-Z0-9._] with a _. This set happens
815 // to be the set of C preprocessing numbers.
816 if (!isPreprocessingNumberBody(c: FileName[i]))
817 FileName[i] = '_';
818 }
819
820 return FileName;
821}
822
823static std::string getPrioritySuffix(unsigned int Priority) {
824 assert(Priority <= 65535 && "Priority should always be <= 65535.");
825
826 // Compute the function suffix from priority. Prepend with zeroes to make
827 // sure the function names are also ordered as priorities.
828 std::string PrioritySuffix = llvm::utostr(X: Priority);
829 PrioritySuffix = std::string(6 - PrioritySuffix.size(), '0') + PrioritySuffix;
830
831 return PrioritySuffix;
832}
833
834void
835CodeGenModule::EmitCXXGlobalInitFunc() {
836 while (!CXXGlobalInits.empty() && !CXXGlobalInits.back())
837 CXXGlobalInits.pop_back();
838
839 // When we import C++20 modules, we must run their initializers first.
840 SmallVector<llvm::Function *, 8> ModuleInits;
841 if (CXX20ModuleInits)
842 for (Module *M : CodeGenUtils::importedModulesToInitialize(
843 Primary: getContext().getCurrentNamedModule(),
844 Imported: ImportedModules.getArrayRef())) {
845 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
846 SmallString<256> FnName;
847 {
848 llvm::raw_svector_ostream Out(FnName);
849 cast<ItaniumMangleContext>(Val&: getCXXABI().getMangleContext())
850 .mangleModuleInitializer(Module: M, Out);
851 }
852 assert(!GetGlobalValue(FnName.str()) &&
853 "We should only have one use of the initializer call");
854 llvm::Function *Fn = llvm::Function::Create(
855 Ty: FTy, Linkage: llvm::Function::ExternalLinkage, N: FnName.str(), M: &getModule());
856 ModuleInits.push_back(Elt: Fn);
857 }
858
859 if (ModuleInits.empty() && CXXGlobalInits.empty() &&
860 PrioritizedCXXGlobalInits.empty())
861 return;
862
863 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
864 const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
865
866 // Create our global prioritized initialization function.
867 if (!PrioritizedCXXGlobalInits.empty()) {
868 SmallVector<llvm::Function *, 8> LocalCXXGlobalInits;
869 llvm::array_pod_sort(Start: PrioritizedCXXGlobalInits.begin(),
870 End: PrioritizedCXXGlobalInits.end());
871 // Iterate over "chunks" of ctors with same priority and emit each chunk
872 // into separate function. Note - everything is sorted first by priority,
873 // second - by lex order, so we emit ctor functions in proper order.
874 for (SmallVectorImpl<GlobalInitData >::iterator
875 I = PrioritizedCXXGlobalInits.begin(),
876 E = PrioritizedCXXGlobalInits.end(); I != E; ) {
877 SmallVectorImpl<GlobalInitData >::iterator
878 PrioE = std::upper_bound(first: I + 1, last: E, val: *I, comp: GlobalInitPriorityCmp());
879
880 LocalCXXGlobalInits.clear();
881
882 unsigned int Priority = I->first.priority;
883 llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
884 FTy, Name: "_GLOBAL__I_" + getPrioritySuffix(Priority), FI);
885
886 // Prepend the module inits to the highest priority set.
887 if (!ModuleInits.empty()) {
888 for (auto *F : ModuleInits)
889 LocalCXXGlobalInits.push_back(Elt: F);
890 ModuleInits.clear();
891 }
892
893 for (; I < PrioE; ++I)
894 LocalCXXGlobalInits.push_back(Elt: I->second);
895
896 CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXThreadLocals: LocalCXXGlobalInits);
897 AddGlobalCtor(Ctor: Fn, Priority);
898 }
899 PrioritizedCXXGlobalInits.clear();
900 }
901
902 if (getCXXABI().useSinitAndSterm() && ModuleInits.empty() &&
903 CXXGlobalInits.empty())
904 return;
905
906 for (auto *F : CXXGlobalInits)
907 ModuleInits.push_back(Elt: F);
908 CXXGlobalInits.clear();
909
910 // Include the filename in the symbol name. Including "sub_" matches gcc
911 // and makes sure these symbols appear lexicographically behind the symbols
912 // with priority emitted above. Module implementation units behave the same
913 // way as a non-modular TU with imports.
914 llvm::Function *Fn;
915 if (CXX20ModuleInits && getContext().getCurrentNamedModule() &&
916 !getContext().getCurrentNamedModule()->isModuleImplementation()) {
917 SmallString<256> InitFnName;
918 llvm::raw_svector_ostream Out(InitFnName);
919 cast<ItaniumMangleContext>(Val&: getCXXABI().getMangleContext())
920 .mangleModuleInitializer(Module: getContext().getCurrentNamedModule(), Out);
921 Fn = CreateGlobalInitOrCleanUpFunction(
922 FTy, Name: llvm::Twine(InitFnName), FI, Loc: SourceLocation(), TLS: false,
923 Linkage: llvm::GlobalVariable::ExternalLinkage);
924 } else
925 Fn = CreateGlobalInitOrCleanUpFunction(
926 FTy,
927 Name: llvm::Twine("_GLOBAL__sub_I_", getTransformedFileName(M&: getModule())),
928 FI);
929
930 CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXThreadLocals: ModuleInits);
931 AddGlobalCtor(Ctor: Fn);
932
933 // In OpenCL global init functions must be converted to kernels in order to
934 // be able to launch them from the host.
935 // FIXME: Some more work might be needed to handle destructors correctly.
936 // Current initialization function makes use of function pointers callbacks.
937 // We can't support function pointers especially between host and device.
938 // However it seems global destruction has little meaning without any
939 // dynamic resource allocation on the device and program scope variables are
940 // destroyed by the runtime when program is released.
941 if (getLangOpts().OpenCL) {
942 GenKernelArgMetadata(FN: Fn);
943 Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
944 }
945
946 assert(!getLangOpts().CUDA || !getLangOpts().CUDAIsDevice ||
947 getLangOpts().GPUAllowDeviceInit);
948 if (getLangOpts().HIP && getLangOpts().CUDAIsDevice) {
949 if (getTriple().isSPIRV())
950 Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
951 else
952 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
953 Fn->addFnAttr(Kind: "device-init");
954 }
955
956 ModuleInits.clear();
957}
958
959void CodeGenModule::EmitCXXGlobalCleanUpFunc() {
960 if (CXXGlobalDtorsOrStermFinalizers.empty() &&
961 PrioritizedCXXStermFinalizers.empty())
962 return;
963
964 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
965 const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
966
967 // Create our global prioritized cleanup function.
968 if (!PrioritizedCXXStermFinalizers.empty()) {
969 SmallVector<CXXGlobalDtorsOrStermFinalizer_t, 8> LocalCXXStermFinalizers;
970 llvm::array_pod_sort(Start: PrioritizedCXXStermFinalizers.begin(),
971 End: PrioritizedCXXStermFinalizers.end());
972 // Iterate over "chunks" of dtors with same priority and emit each chunk
973 // into separate function. Note - everything is sorted first by priority,
974 // second - by lex order, so we emit dtor functions in proper order.
975 for (SmallVectorImpl<StermFinalizerData>::iterator
976 I = PrioritizedCXXStermFinalizers.begin(),
977 E = PrioritizedCXXStermFinalizers.end();
978 I != E;) {
979 SmallVectorImpl<StermFinalizerData>::iterator PrioE =
980 std::upper_bound(first: I + 1, last: E, val: *I, comp: StermFinalizerPriorityCmp());
981
982 LocalCXXStermFinalizers.clear();
983
984 unsigned int Priority = I->first.priority;
985 llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
986 FTy, Name: "_GLOBAL__a_" + getPrioritySuffix(Priority), FI);
987
988 for (; I < PrioE; ++I) {
989 llvm::FunctionCallee DtorFn = I->second;
990 LocalCXXStermFinalizers.emplace_back(Args: DtorFn.getFunctionType(),
991 Args: DtorFn.getCallee(), Args: nullptr);
992 }
993
994 CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
995 Fn, DtorsOrStermFinalizers: LocalCXXStermFinalizers);
996 AddGlobalDtor(Dtor: Fn, Priority);
997 }
998 PrioritizedCXXStermFinalizers.clear();
999 }
1000
1001 if (CXXGlobalDtorsOrStermFinalizers.empty())
1002 return;
1003
1004 // Create our global cleanup function.
1005 llvm::Function *Fn =
1006 CreateGlobalInitOrCleanUpFunction(FTy, Name: "_GLOBAL__D_a", FI);
1007
1008 CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
1009 Fn, DtorsOrStermFinalizers: CXXGlobalDtorsOrStermFinalizers);
1010 AddGlobalDtor(Dtor: Fn);
1011 CXXGlobalDtorsOrStermFinalizers.clear();
1012}
1013
1014/// Emit the code necessary to initialize the given global variable.
1015void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
1016 const VarDecl *D,
1017 llvm::GlobalVariable *Addr,
1018 bool PerformInit) {
1019 // Check if we need to emit debug info for variable initializer.
1020 if (D->hasAttr<NoDebugAttr>())
1021 DebugInfo = nullptr; // disable debug info indefinitely for this function
1022
1023 CurEHLocation = D->getBeginLoc();
1024
1025 StartFunction(GD: GlobalDecl(D, DynamicInitKind::Initializer),
1026 RetTy: getContext().VoidTy, Fn, FnInfo: getTypes().arrangeNullaryFunction(),
1027 Args: FunctionArgList());
1028 // Emit an artificial location for this function.
1029 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1030
1031 // Use guarded initialization if the global variable is weak. This
1032 // occurs for, e.g., instantiated static data members and
1033 // definitions explicitly marked weak.
1034 //
1035 // Also use guarded initialization for a variable with dynamic TLS and
1036 // unordered initialization. (If the initialization is ordered, the ABI
1037 // layer will guard the whole-TU initialization for us.)
1038 if (Addr->hasWeakLinkage() || Addr->hasLinkOnceLinkage() ||
1039 (D->getTLSKind() == VarDecl::TLS_Dynamic &&
1040 isTemplateInstantiation(Kind: D->getTemplateSpecializationKind()))) {
1041 EmitCXXGuardedInit(D: *D, DeclPtr: Addr, PerformInit);
1042 } else {
1043 EmitCXXGlobalVarDeclInit(D: *D, GV: Addr, PerformInit);
1044 }
1045
1046 FinishFunction();
1047}
1048
1049void
1050CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
1051 ArrayRef<llvm::Function *> Decls,
1052 ConstantAddress Guard) {
1053 {
1054 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1055 StartFunction(GD: GlobalDecl(), RetTy: getContext().VoidTy, Fn,
1056 FnInfo: getTypes().arrangeNullaryFunction(), Args: FunctionArgList());
1057 // Emit an artificial location for this function.
1058 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1059
1060 llvm::BasicBlock *ExitBlock = nullptr;
1061 if (Guard.isValid()) {
1062 // If we have a guard variable, check whether we've already performed
1063 // these initializations. This happens for TLS initialization functions.
1064 llvm::Value *GuardVal = Builder.CreateLoad(Addr: Guard);
1065 llvm::Value *Uninit = Builder.CreateIsNull(Arg: GuardVal,
1066 Name: "guard.uninitialized");
1067 llvm::BasicBlock *InitBlock = createBasicBlock(name: "init");
1068 ExitBlock = createBasicBlock(name: "exit");
1069 EmitCXXGuardedInitBranch(NeedsInit: Uninit, InitBlock, NoInitBlock: ExitBlock,
1070 Kind: GuardKind::TlsGuard, D: nullptr);
1071 EmitBlock(BB: InitBlock);
1072 // Mark as initialized before initializing anything else. If the
1073 // initializers use previously-initialized thread_local vars, that's
1074 // probably supposed to be OK, but the standard doesn't say.
1075 Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: GuardVal->getType(),V: 1), Addr: Guard);
1076
1077 // The guard variable can't ever change again.
1078 EmitInvariantStart(
1079 Addr: Guard.getPointer(),
1080 Size: CharUnits::fromQuantity(
1081 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: GuardVal->getType())));
1082 }
1083
1084 RunCleanupsScope Scope(*this);
1085
1086 // When building in Objective-C++ ARC mode, create an autorelease pool
1087 // around the global initializers.
1088 if (getLangOpts().ObjCAutoRefCount && getLangOpts().CPlusPlus) {
1089 llvm::Value *token = EmitObjCAutoreleasePoolPush();
1090 EmitObjCAutoreleasePoolCleanup(Ptr: token);
1091 }
1092
1093 for (llvm::Function *Decl : Decls) {
1094 if (Decl)
1095 EmitRuntimeCall(callee: Decl, args: {}, name: "", IsConvergent: Decl->isConvergent());
1096 }
1097
1098 Scope.ForceCleanup();
1099
1100 if (ExitBlock) {
1101 Builder.CreateBr(Dest: ExitBlock);
1102 EmitBlock(BB: ExitBlock);
1103 }
1104 }
1105
1106 FinishFunction();
1107}
1108
1109void CodeGenFunction::GenerateCXXGlobalCleanUpFunc(
1110 llvm::Function *Fn,
1111 ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
1112 llvm::Constant *>>
1113 DtorsOrStermFinalizers) {
1114 {
1115 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1116 StartFunction(GD: GlobalDecl(), RetTy: getContext().VoidTy, Fn,
1117 FnInfo: getTypes().arrangeNullaryFunction(), Args: FunctionArgList());
1118 // Emit an artificial location for this function.
1119 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1120
1121 // Emit the cleanups, in reverse order from construction.
1122 for (unsigned i = 0, e = DtorsOrStermFinalizers.size(); i != e; ++i) {
1123 llvm::FunctionType *CalleeTy;
1124 llvm::Value *Callee;
1125 llvm::Constant *Arg;
1126 std::tie(args&: CalleeTy, args&: Callee, args&: Arg) = DtorsOrStermFinalizers[e - i - 1];
1127
1128 bool IsConvergent = getLangOpts().assumeFunctionsAreConvergent();
1129 SmallVector<llvm::OperandBundleDef> BundleList;
1130 if (IsConvergent)
1131 addConvergenceControlBundle(Bundles&: BundleList);
1132
1133 llvm::CallBase *CI = nullptr;
1134 if (Arg == nullptr) {
1135 assert(
1136 CGM.getCXXABI().useSinitAndSterm() &&
1137 "Arg could not be nullptr unless using sinit and sterm functions.");
1138 CI = Builder.CreateCall(FTy: CalleeTy, Callee, Args: {}, OpBundles: BundleList);
1139 } else {
1140 // If the object lives in a different address space, the `this` pointer
1141 // address space won't match the dtor `this` param. An addrspacecast is
1142 // required.
1143 assert(Arg->getType()->isPointerTy());
1144 assert(CalleeTy->getParamType(0)->isPointerTy());
1145 unsigned ActualAddrSpace = Arg->getType()->getPointerAddressSpace();
1146 unsigned ExpectedAddrSpace =
1147 CalleeTy->getParamType(i: 0)->getPointerAddressSpace();
1148 if (ActualAddrSpace != ExpectedAddrSpace) {
1149 llvm::PointerType *PTy =
1150 llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: ExpectedAddrSpace);
1151 Arg = llvm::ConstantExpr::getAddrSpaceCast(C: Arg, Ty: PTy);
1152 }
1153
1154 CI = Builder.CreateCall(FTy: CalleeTy, Callee, Args: Arg, OpBundles: BundleList);
1155 }
1156
1157 // Make sure the call and the callee agree on calling convention.
1158 if (llvm::Function *F = dyn_cast<llvm::Function>(Val: Callee))
1159 CI->setCallingConv(F->getCallingConv());
1160
1161 setConvergentCallSite(Call: CI, IsConvergent);
1162 }
1163 }
1164
1165 FinishFunction();
1166}
1167
1168/// generateDestroyHelper - Generates a helper function which, when
1169/// invoked, destroys the given object. The address of the object
1170/// should be in global memory.
1171llvm::Function *CodeGenFunction::generateDestroyHelper(
1172 Address addr, QualType type, Destroyer *destroyer,
1173 bool useEHCleanupForArray, const VarDecl *VD) {
1174 auto *Dst = ImplicitParamDecl::Create(C&: getContext(), T: getContext().VoidPtrTy,
1175 ParamKind: ImplicitParamKind::Other);
1176
1177 FunctionArgList args{Dst};
1178 const CGFunctionInfo &FI =
1179 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: getContext().VoidTy, args);
1180 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1181 llvm::Function *fn = CGM.CreateGlobalInitOrCleanUpFunction(
1182 FTy, Name: "__cxx_global_array_dtor", FI, Loc: VD->getLocation());
1183
1184 CurEHLocation = VD->getBeginLoc();
1185
1186 StartFunction(GD: GlobalDecl(VD, DynamicInitKind::GlobalArrayDestructor),
1187 RetTy: getContext().VoidTy, Fn: fn, FnInfo: FI, Args: args);
1188 // Emit an artificial location for this function.
1189 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1190
1191 emitDestroy(addr, type, destroyer, useEHCleanupForArray);
1192
1193 FinishFunction();
1194
1195 return fn;
1196}
1197