1//===--- CGDeclCXX.cpp - Emit LLVM Code for C++ declarations --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code dealing with code generation of C++ declarations
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGDebugInfo.h"
15#include "CGHLSLRuntime.h"
16#include "CGObjCRuntime.h"
17#include "CGOpenMPRuntime.h"
18#include "CodeGenFunction.h"
19#include "TargetInfo.h"
20#include "clang/AST/Attr.h"
21#include "clang/Basic/LangOptions.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/MDBuilder.h"
25#include "llvm/Support/Path.h"
26
27using namespace clang;
28using namespace CodeGen;
29
30static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D,
31 ConstantAddress DeclPtr) {
32 assert(
33 (D.hasGlobalStorage() ||
34 (D.hasLocalStorage() && CGF.getContext().getLangOpts().OpenCLCPlusPlus)) &&
35 "VarDecl must have global or local (in the case of OpenCL) storage!");
36 assert(!D.getType()->isReferenceType() &&
37 "Should not call EmitDeclInit on a reference!");
38
39 QualType type = D.getType();
40 LValue lv = CGF.MakeAddrLValue(Addr: DeclPtr, T: type);
41
42 const Expr *Init = D.getInit();
43 switch (CGF.getEvaluationKind(T: type)) {
44 case TEK_Scalar: {
45 CodeGenModule &CGM = CGF.CGM;
46 if (lv.isObjCStrong())
47 CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF, src: CGF.EmitScalarExpr(E: Init),
48 dest: DeclPtr, threadlocal: D.getTLSKind());
49 else if (lv.isObjCWeak())
50 CGM.getObjCRuntime().EmitObjCWeakAssign(CGF, src: CGF.EmitScalarExpr(E: Init),
51 dest: DeclPtr);
52 else
53 CGF.EmitScalarInit(init: Init, D: &D, lvalue: lv, capturedByInit: false);
54 return;
55 }
56 case TEK_Complex:
57 CGF.EmitComplexExprIntoLValue(E: Init, dest: lv, /*isInit*/ true);
58 return;
59 case TEK_Aggregate:
60 CGF.EmitAggExpr(E: Init,
61 AS: AggValueSlot::forLValue(LV: lv, isDestructed: AggValueSlot::IsDestructed,
62 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
63 isAliased: AggValueSlot::IsNotAliased,
64 mayOverlap: AggValueSlot::DoesNotOverlap));
65 return;
66 }
67 llvm_unreachable("bad evaluation kind");
68}
69
70/// Emit code to cause the destruction of the given variable with
71/// static storage duration.
72static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
73 ConstantAddress Addr) {
74 // Honor __attribute__((no_destroy)) and bail instead of attempting
75 // to emit a reference to a possibly nonexistent destructor, which
76 // in turn can cause a crash. This will result in a global constructor
77 // that isn't balanced out by a destructor call as intended by the
78 // attribute. This also checks for -fno-c++-static-destructors and
79 // bails even if the attribute is not present.
80 QualType::DestructionKind DtorKind = D.needsDestruction(Ctx: CGF.getContext());
81
82 // FIXME: __attribute__((cleanup)) ?
83
84 switch (DtorKind) {
85 case QualType::DK_none:
86 return;
87
88 case QualType::DK_cxx_destructor:
89 break;
90
91 case QualType::DK_objc_strong_lifetime:
92 case QualType::DK_objc_weak_lifetime:
93 case QualType::DK_nontrivial_c_struct:
94 // We don't care about releasing objects during process teardown.
95 assert(!D.getTLSKind() && "should have rejected this");
96 return;
97 }
98
99 llvm::FunctionCallee Func;
100 llvm::Constant *Argument;
101
102 CodeGenModule &CGM = CGF.CGM;
103 QualType Type = D.getType();
104
105 // Special-case non-array C++ destructors, if they have the right signature.
106 // Under some ABIs, destructors return this instead of void, and cannot be
107 // passed directly to __cxa_atexit if the target does not allow this
108 // mismatch.
109 const CXXRecordDecl *Record = Type->getAsCXXRecordDecl();
110 bool CanRegisterDestructor =
111 Record && (!CGM.getCXXABI().HasThisReturn(
112 GD: GlobalDecl(Record->getDestructor(), Dtor_Complete)) ||
113 CGM.getCXXABI().canCallMismatchedFunctionType());
114 // If __cxa_atexit is disabled via a flag, a different helper function is
115 // generated elsewhere which uses atexit instead, and it takes the destructor
116 // directly.
117 bool UsingExternalHelper = !CGM.getCodeGenOpts().CXAAtExit;
118 if (Record && (CanRegisterDestructor || UsingExternalHelper)) {
119 assert(!Record->hasTrivialDestructor());
120 CXXDestructorDecl *Dtor = Record->getDestructor();
121
122 Func = CGM.getAddrAndTypeOfCXXStructor(GD: GlobalDecl(Dtor, Dtor_Complete));
123 if (CGF.getContext().getLangOpts().OpenCL) {
124 auto DestAS =
125 CGM.getTargetCodeGenInfo().getAddrSpaceOfCxaAtexitPtrParam();
126 auto DestTy = llvm::PointerType::get(
127 C&: CGM.getLLVMContext(), AddressSpace: CGM.getContext().getTargetAddressSpace(AS: DestAS));
128 auto SrcAS = D.getType().getQualifiers().getAddressSpace();
129 if (DestAS == SrcAS)
130 Argument = Addr.getPointer();
131 else
132 // FIXME: On addr space mismatch we are passing NULL. The generation
133 // of the global destructor function should be adjusted accordingly.
134 Argument = llvm::ConstantPointerNull::get(T: DestTy);
135 } else {
136 Argument = Addr.getPointer();
137 }
138 // Otherwise, the standard logic requires a helper function.
139 } else {
140 Addr = Addr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: Type));
141 Func = CodeGenFunction(CGM)
142 .generateDestroyHelper(addr: Addr, type: Type, destroyer: CGF.getDestroyer(destructionKind: DtorKind),
143 useEHCleanupForArray: CGF.needsEHCleanup(kind: DtorKind), VD: &D);
144 Argument = llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy);
145 }
146
147 CGM.getCXXABI().registerGlobalDtor(CGF, D, Dtor: Func, Addr: Argument);
148}
149
150/// Emit code to cause the variable at the given address to be considered as
151/// constant from this point onwards.
152static void EmitDeclInvariant(CodeGenFunction &CGF, const VarDecl &D,
153 llvm::Constant *Addr) {
154 return CGF.EmitInvariantStart(
155 Addr, Size: CGF.getContext().getTypeSizeInChars(T: D.getType()));
156}
157
158void CodeGenFunction::EmitInvariantStart(llvm::Constant *Addr, CharUnits Size) {
159 // Do not emit the intrinsic if we're not optimizing.
160 if (!CGM.getCodeGenOpts().OptimizationLevel)
161 return;
162
163 // Grab the llvm.invariant.start intrinsic.
164 llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
165 // Overloaded address space type.
166 assert(Addr->getType()->isPointerTy() && "Address must be a pointer");
167 llvm::Type *ObjectPtr[1] = {Addr->getType()};
168 llvm::Function *InvariantStart = CGM.getIntrinsic(IID: InvStartID, Tys: ObjectPtr);
169
170 // Emit a call with the size in bytes of the object.
171 uint64_t Width = Size.getQuantity();
172 llvm::Value *Args[2] = {llvm::ConstantInt::getSigned(Ty: Int64Ty, V: Width), Addr};
173 Builder.CreateCall(Callee: InvariantStart, Args);
174}
175
176void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
177 llvm::GlobalVariable *GV,
178 bool PerformInit) {
179
180 const Expr *Init = D.getInit();
181 QualType T = D.getType();
182
183 // The address space of a static local variable (DeclPtr) may be different
184 // from the address space of the "this" argument of the constructor. In that
185 // case, we need an addrspacecast before calling the constructor.
186 //
187 // struct StructWithCtor {
188 // __device__ StructWithCtor() {...}
189 // };
190 // __device__ void foo() {
191 // __shared__ StructWithCtor s;
192 // ...
193 // }
194 //
195 // For example, in the above CUDA code, the static local variable s has a
196 // "shared" address space qualifier, but the constructor of StructWithCtor
197 // expects "this" in the "generic" address space.
198 unsigned ExpectedAddrSpace = getTypes().getTargetAddressSpace(T);
199 unsigned ActualAddrSpace = GV->getAddressSpace();
200 llvm::Constant *DeclPtr = GV;
201 if (ActualAddrSpace != ExpectedAddrSpace) {
202 llvm::PointerType *PTy =
203 llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: ExpectedAddrSpace);
204 DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(C: DeclPtr, Ty: PTy);
205 }
206
207 ConstantAddress DeclAddr(
208 DeclPtr, GV->getValueType(), getContext().getDeclAlign(D: &D));
209
210 if (!T->isReferenceType()) {
211 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
212 D.hasAttr<OMPThreadPrivateDeclAttr>()) {
213 (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition(
214 VD: &D, VDAddr: DeclAddr, Loc: D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
215 PerformInit, CGF: this);
216 }
217 bool NeedsDtor =
218 D.needsDestruction(Ctx: getContext()) == QualType::DK_cxx_destructor;
219 if (PerformInit)
220 EmitDeclInit(CGF&: *this, D, DeclPtr: DeclAddr);
221 if (D.getType().isConstantStorage(Ctx: getContext(), ExcludeCtor: true, ExcludeDtor: !NeedsDtor))
222 EmitDeclInvariant(CGF&: *this, D, Addr: DeclPtr);
223 else
224 EmitDeclDestroy(CGF&: *this, D, Addr: DeclAddr);
225 return;
226 }
227
228 assert(PerformInit && "cannot have constant initializer which needs "
229 "destruction for reference");
230 RValue RV = EmitReferenceBindingToExpr(E: Init);
231 EmitStoreOfScalar(Value: RV.getScalarVal(), Addr: DeclAddr, Volatile: false, Ty: T);
232}
233
234/// Create a stub function, suitable for being passed to atexit,
235/// which passes the given address to the given destructor function.
236llvm::Constant *CodeGenFunction::createAtExitStub(const VarDecl &VD,
237 llvm::FunctionCallee dtor,
238 llvm::Constant *addr) {
239 // Get the destructor function type, void(*)(void).
240 llvm::FunctionType *ty = llvm::FunctionType::get(Result: CGM.VoidTy, isVarArg: false);
241 SmallString<256> FnName;
242 {
243 llvm::raw_svector_ostream Out(FnName);
244 CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(D: &VD, Out);
245 }
246
247 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
248 llvm::Function *fn = CGM.CreateGlobalInitOrCleanUpFunction(
249 ty, name: FnName.str(), FI, Loc: VD.getLocation());
250
251 CodeGenFunction CGF(CGM);
252
253 CGF.StartFunction(GD: GlobalDecl(&VD, DynamicInitKind::AtExit),
254 RetTy: CGM.getContext().VoidTy, Fn: fn, FnInfo: FI, Args: FunctionArgList(),
255 Loc: VD.getLocation(), StartLoc: VD.getInit()->getExprLoc());
256 // Emit an artificial location for this function.
257 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
258
259 llvm::CallInst *call = CGF.Builder.CreateCall(Callee: dtor, Args: addr);
260
261 // Make sure the call and the callee agree on calling convention.
262 if (auto *dtorFn = dyn_cast<llvm::Function>(
263 Val: dtor.getCallee()->stripPointerCastsAndAliases()))
264 call->setCallingConv(dtorFn->getCallingConv());
265
266 CGF.FinishFunction();
267
268 // Get a proper function pointer.
269 FunctionProtoType::ExtProtoInfo EPI(getContext().getDefaultCallingConvention(
270 /*IsVariadic=*/false, /*IsCXXMethod=*/false));
271 QualType fnType = getContext().getFunctionType(ResultTy: getContext().VoidTy,
272 Args: {getContext().VoidPtrTy}, EPI);
273 return CGM.getFunctionPointer(Pointer: fn, FunctionType: fnType);
274}
275
276/// Create a stub function, suitable for being passed to __pt_atexit_np,
277/// which passes the given address to the given destructor function.
278llvm::Function *CodeGenFunction::createTLSAtExitStub(
279 const VarDecl &D, llvm::FunctionCallee Dtor, llvm::Constant *Addr,
280 llvm::FunctionCallee &AtExit) {
281 SmallString<256> FnName;
282 {
283 llvm::raw_svector_ostream Out(FnName);
284 CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(D: &D, Out);
285 }
286
287 const CGFunctionInfo &FI = CGM.getTypes().arrangeLLVMFunctionInfo(
288 returnType: getContext().IntTy, opts: FnInfoOpts::None, argTypes: {getContext().IntTy},
289 info: FunctionType::ExtInfo(), paramInfos: {}, args: RequiredArgs::All);
290
291 // Get the stub function type, int(*)(int,...).
292 llvm::FunctionType *StubTy =
293 llvm::FunctionType::get(Result: CGM.IntTy, Params: {CGM.IntTy}, isVarArg: true);
294
295 llvm::Function *DtorStub = CGM.CreateGlobalInitOrCleanUpFunction(
296 ty: StubTy, name: FnName.str(), FI, Loc: D.getLocation());
297
298 CodeGenFunction CGF(CGM);
299
300 auto *IPD = ImplicitParamDecl::Create(
301 C&: CGM.getContext(), T: CGM.getContext().IntTy, ParamKind: ImplicitParamKind::Other);
302 QualType ResTy = CGM.getContext().IntTy;
303
304 FunctionArgList Args{IPD};
305 CGF.StartFunction(GD: GlobalDecl(&D, DynamicInitKind::AtExit), RetTy: ResTy, Fn: DtorStub,
306 FnInfo: FI, Args, Loc: D.getLocation(), StartLoc: D.getInit()->getExprLoc());
307
308 // Emit an artificial location for this function.
309 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
310
311 llvm::CallInst *call = CGF.Builder.CreateCall(Callee: Dtor, Args: Addr);
312
313 // Make sure the call and the callee agree on calling convention.
314 if (auto *DtorFn = dyn_cast<llvm::Function>(
315 Val: Dtor.getCallee()->stripPointerCastsAndAliases()))
316 call->setCallingConv(DtorFn->getCallingConv());
317
318 // Return 0 from function
319 CGF.Builder.CreateStore(Val: llvm::Constant::getNullValue(Ty: CGM.IntTy),
320 Addr: CGF.ReturnValue);
321
322 CGF.FinishFunction();
323
324 return DtorStub;
325}
326
327/// Register a global destructor using the C atexit runtime function.
328void CodeGenFunction::registerGlobalDtorWithAtExit(const VarDecl &VD,
329 llvm::FunctionCallee dtor,
330 llvm::Constant *addr) {
331 // Create a function which calls the destructor.
332 llvm::Constant *dtorStub = createAtExitStub(VD, dtor, addr);
333 registerGlobalDtorWithAtExit(dtorStub);
334}
335
336/// Register a global destructor using the LLVM 'llvm.global_dtors' global.
337void CodeGenFunction::registerGlobalDtorWithLLVM(const VarDecl &VD,
338 llvm::FunctionCallee Dtor,
339 llvm::Constant *Addr) {
340 // Create a function which calls the destructor.
341 llvm::Function *dtorStub =
342 cast<llvm::Function>(Val: createAtExitStub(VD, dtor: Dtor, addr: Addr));
343 CGM.AddGlobalDtor(Dtor: dtorStub);
344}
345
346void CodeGenFunction::registerGlobalDtorWithAtExit(llvm::Constant *dtorStub) {
347 // extern "C" int atexit(void (*f)(void));
348 assert(dtorStub->getType()->isPointerTy() &&
349 "Argument to atexit has a wrong type.");
350
351 llvm::FunctionType *atexitTy =
352 llvm::FunctionType::get(Result: IntTy, Params: dtorStub->getType(), isVarArg: false);
353
354 llvm::FunctionCallee atexit =
355 CGM.CreateRuntimeFunction(Ty: atexitTy, Name: "atexit", ExtraAttrs: llvm::AttributeList(),
356 /*Local=*/true);
357 if (llvm::Function *atexitFn = dyn_cast<llvm::Function>(Val: atexit.getCallee()))
358 atexitFn->setDoesNotThrow();
359
360 EmitNounwindRuntimeCall(callee: atexit, args: dtorStub);
361}
362
363llvm::Value *
364CodeGenFunction::unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub) {
365 // The unatexit subroutine unregisters __dtor functions that were previously
366 // registered by the atexit subroutine. If the referenced function is found,
367 // it is removed from the list of functions that are called at normal program
368 // termination and the unatexit returns a value of 0, otherwise a non-zero
369 // value is returned.
370 //
371 // extern "C" int unatexit(void (*f)(void));
372 assert(dtorStub->getType()->isPointerTy() &&
373 "Argument to unatexit has a wrong type.");
374
375 llvm::FunctionType *unatexitTy =
376 llvm::FunctionType::get(Result: IntTy, Params: {dtorStub->getType()}, /*isVarArg=*/false);
377
378 llvm::FunctionCallee unatexit =
379 CGM.CreateRuntimeFunction(Ty: unatexitTy, Name: "unatexit", ExtraAttrs: llvm::AttributeList());
380
381 cast<llvm::Function>(Val: unatexit.getCallee())->setDoesNotThrow();
382
383 return EmitNounwindRuntimeCall(callee: unatexit, args: dtorStub);
384}
385
386void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D,
387 llvm::GlobalVariable *DeclPtr,
388 bool PerformInit) {
389 // If we've been asked to forbid guard variables, emit an error now.
390 // This diagnostic is hard-coded for Darwin's use case; we can find
391 // better phrasing if someone else needs it.
392 if (CGM.getCodeGenOpts().ForbidGuardVariables)
393 CGM.Error(loc: D.getLocation(),
394 error: "this initialization requires a guard variable, which "
395 "the kernel does not support");
396
397 CGM.getCXXABI().EmitGuardedInit(CGF&: *this, D, DeclPtr, PerformInit);
398}
399
400void CodeGenFunction::EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
401 llvm::BasicBlock *InitBlock,
402 llvm::BasicBlock *NoInitBlock,
403 GuardKind Kind,
404 const VarDecl *D) {
405 assert((Kind == GuardKind::TlsGuard || D) && "no guarded variable");
406
407 // A guess at how many times we will enter the initialization of a
408 // variable, depending on the kind of variable.
409 static const uint64_t InitsPerTLSVar = 1024;
410 static const uint64_t InitsPerLocalVar = 1024 * 1024;
411
412 llvm::MDNode *Weights;
413 if (Kind == GuardKind::VariableGuard && !D->isLocalVarDecl()) {
414 // For non-local variables, don't apply any weighting for now. Due to our
415 // use of COMDATs, we expect there to be at most one initialization of the
416 // variable per DSO, but we have no way to know how many DSOs will try to
417 // initialize the variable.
418 Weights = nullptr;
419 } else {
420 uint64_t NumInits;
421 // FIXME: For the TLS case, collect and use profiling information to
422 // determine a more accurate brach weight.
423 if (Kind == GuardKind::TlsGuard || D->getTLSKind())
424 NumInits = InitsPerTLSVar;
425 else
426 NumInits = InitsPerLocalVar;
427
428 // The probability of us entering the initializer is
429 // 1 / (total number of times we attempt to initialize the variable).
430 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
431 Weights = MDHelper.createBranchWeights(TrueWeight: 1, FalseWeight: NumInits - 1);
432 }
433
434 Builder.CreateCondBr(Cond: NeedsInit, True: InitBlock, False: NoInitBlock, BranchWeights: Weights);
435}
436
437llvm::Function *CodeGenModule::CreateGlobalInitOrCleanUpFunction(
438 llvm::FunctionType *FTy, const Twine &Name, const CGFunctionInfo &FI,
439 SourceLocation Loc, bool TLS, llvm::GlobalVariable::LinkageTypes Linkage) {
440 llvm::Function *Fn = llvm::Function::Create(Ty: FTy, Linkage, N: Name, M: &getModule());
441
442 if (!getLangOpts().AppleKext && !TLS) {
443 // Set the section if needed.
444 if (const char *Section = getTarget().getStaticInitSectionSpecifier())
445 Fn->setSection(Section);
446 }
447
448 if (Linkage == llvm::GlobalVariable::InternalLinkage)
449 SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
450 else {
451 SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, IsThunk: false);
452 SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
453 getTargetCodeGenInfo().setTargetAttributes(D: nullptr, GV: Fn, M&: *this);
454 }
455
456 Fn->setCallingConv(getRuntimeCC());
457
458 if (!getLangOpts().Exceptions)
459 Fn->setDoesNotThrow();
460
461 if (getLangOpts().Sanitize.has(K: SanitizerKind::Address) &&
462 !isInNoSanitizeList(Kind: SanitizerKind::Address, Fn, Loc))
463 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
464
465 if (getLangOpts().Sanitize.has(K: SanitizerKind::KernelAddress) &&
466 !isInNoSanitizeList(Kind: SanitizerKind::KernelAddress, Fn, Loc))
467 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
468
469 if (getLangOpts().Sanitize.has(K: SanitizerKind::HWAddress) &&
470 !isInNoSanitizeList(Kind: SanitizerKind::HWAddress, Fn, Loc))
471 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
472
473 if (getLangOpts().Sanitize.has(K: SanitizerKind::KernelHWAddress) &&
474 !isInNoSanitizeList(Kind: SanitizerKind::KernelHWAddress, Fn, Loc))
475 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
476
477 if (getLangOpts().Sanitize.has(K: SanitizerKind::MemtagStack) &&
478 !isInNoSanitizeList(Kind: SanitizerKind::MemtagStack, Fn, Loc))
479 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemTag);
480
481 if (getLangOpts().Sanitize.has(K: SanitizerKind::Type) &&
482 !isInNoSanitizeList(Kind: SanitizerKind::Type, Fn, Loc))
483 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeType);
484
485 if (getLangOpts().Sanitize.has(K: SanitizerKind::Thread) &&
486 !isInNoSanitizeList(Kind: SanitizerKind::Thread, Fn, Loc))
487 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeThread);
488
489 if (getLangOpts().Sanitize.has(K: SanitizerKind::NumericalStability) &&
490 !isInNoSanitizeList(Kind: SanitizerKind::NumericalStability, Fn, Loc))
491 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeNumericalStability);
492
493 if (getLangOpts().Sanitize.has(K: SanitizerKind::Memory) &&
494 !isInNoSanitizeList(Kind: SanitizerKind::Memory, Fn, Loc))
495 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
496
497 if (getLangOpts().Sanitize.has(K: SanitizerKind::KernelMemory) &&
498 !isInNoSanitizeList(Kind: SanitizerKind::KernelMemory, Fn, Loc))
499 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
500
501 if (getLangOpts().Sanitize.has(K: SanitizerKind::SafeStack) &&
502 !isInNoSanitizeList(Kind: SanitizerKind::SafeStack, Fn, Loc))
503 Fn->addFnAttr(Kind: llvm::Attribute::SafeStack);
504
505 if (getLangOpts().Sanitize.has(K: SanitizerKind::ShadowCallStack) &&
506 !isInNoSanitizeList(Kind: SanitizerKind::ShadowCallStack, Fn, Loc))
507 Fn->addFnAttr(Kind: llvm::Attribute::ShadowCallStack);
508
509 return Fn;
510}
511
512/// Create a global pointer to a function that will initialize a global
513/// variable. The user has requested that this pointer be emitted in a specific
514/// section.
515void CodeGenModule::EmitPointerToInitFunc(const VarDecl *D,
516 llvm::GlobalVariable *GV,
517 llvm::Function *InitFunc,
518 InitSegAttr *ISA) {
519 llvm::GlobalVariable *PtrArray = new llvm::GlobalVariable(
520 TheModule, InitFunc->getType(), /*isConstant=*/true,
521 llvm::GlobalValue::PrivateLinkage, InitFunc, "__cxx_init_fn_ptr");
522 PtrArray->setSection(ISA->getSection());
523 addUsedGlobal(GV: PtrArray);
524
525 // If the GV is already in a comdat group, then we have to join it.
526 if (llvm::Comdat *C = GV->getComdat())
527 PtrArray->setComdat(C);
528}
529
530void
531CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
532 llvm::GlobalVariable *Addr,
533 bool PerformInit) {
534
535 // According to E.2.3.1 in CUDA-7.5 Programming guide: __device__,
536 // __constant__ and __shared__ variables defined in namespace scope,
537 // that are of class type, cannot have a non-empty constructor. All
538 // the checks have been done in Sema by now. Whatever initializers
539 // are allowed are empty and we just need to ignore them here.
540 if (getLangOpts().CUDAIsDevice && !getLangOpts().GPUAllowDeviceInit &&
541 (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
542 D->hasAttr<CUDASharedAttr>()))
543 return;
544
545 // Check if we've already initialized this decl.
546 auto I = DelayedCXXInitPosition.find(Val: D);
547 if (I != DelayedCXXInitPosition.end() && I->second == ~0U)
548 return;
549
550 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
551 SmallString<256> FnName;
552 {
553 llvm::raw_svector_ostream Out(FnName);
554 getCXXABI().getMangleContext().mangleDynamicInitializer(D, Out);
555 }
556
557 // Create a variable initialization function.
558 llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
559 FTy, Name: FnName.str(), FI: getTypes().arrangeNullaryFunction(), Loc: D->getLocation());
560
561 auto *ISA = D->getAttr<InitSegAttr>();
562 CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D, Addr,
563 PerformInit);
564
565 llvm::GlobalVariable *COMDATKey =
566 supportsCOMDAT() && D->isExternallyVisible() ? Addr : nullptr;
567
568 if (D->getTLSKind()) {
569 // FIXME: Should we support init_priority for thread_local?
570 // FIXME: We only need to register one __cxa_thread_atexit function for the
571 // entire TU.
572 CXXThreadLocalInits.push_back(x: Fn);
573 CXXThreadLocalInitVars.push_back(x: D);
574 } else if (PerformInit && ISA) {
575 // Contract with backend that "init_seg(compiler)" corresponds to priority
576 // 200 and "init_seg(lib)" corresponds to priority 400.
577 int Priority = -1;
578 if (ISA->getSection() == ".CRT$XCC")
579 Priority = 200;
580 else if (ISA->getSection() == ".CRT$XCL")
581 Priority = 400;
582
583 if (Priority != -1)
584 AddGlobalCtor(Ctor: Fn, Priority, LexOrder: ~0U, AssociatedData: COMDATKey);
585 else
586 EmitPointerToInitFunc(D, GV: Addr, InitFunc: Fn, ISA);
587 } else if (auto *IPA = D->getAttr<InitPriorityAttr>()) {
588 OrderGlobalInitsOrStermFinalizers Key(IPA->getPriority(),
589 PrioritizedCXXGlobalInits.size());
590 PrioritizedCXXGlobalInits.push_back(Elt: std::make_pair(x&: Key, y&: Fn));
591 } else if (isTemplateInstantiation(Kind: D->getTemplateSpecializationKind()) ||
592 !isUniqueGVALinkage(L: getContext().GetGVALinkageForVariable(VD: D)) ||
593 D->hasAttr<SelectAnyAttr>()) {
594 // For vague linkage globals, put the initializer into its own global_ctors
595 // entry with the global as a comdat key. This ensures at most one
596 // initializer per DSO runs during DSO dynamic initialization.
597 //
598 // For ELF platforms, this is an important code size and startup time
599 // optimization. For dynamic, non-hidden symbols, the weak guard variable
600 // remains to ensure that other DSOs do not re-initialize the global.
601 //
602 // For PE-COFF platforms, there is no guard variable, and COMDAT
603 // associativity is the only way to ensure vauge linkage globals are
604 // initialized exactly once.
605 //
606 // MachO is the only remaining platform with no comdats that doesn't
607 // benefit from this optimization. The rest are mainly modeled on ELF
608 // behavior.
609 //
610 // C++ requires that inline global variables are initialized in source
611 // order, but this requirement does not exist for templated entities.
612 // llvm.global_ctors does not guarantee initialization order, so in
613 // general, Clang does not fully conform to the ordering requirement.
614 // However, in practice, LLVM emits global_ctors in the provided order, and
615 // users typically don't rely on ordering between inline globals in
616 // different headers which are then transitively included in varying order.
617 // Clang's current behavior is a practical tradeoff, since dropping the
618 // comdat would lead to unacceptable impact on code size and startup time.
619 //
620 // FIXME: Find a solution to guarantee source-order initialization of
621 // inline variables.
622 //
623 // C++ [basic.start.init]p2:
624 // Definitions of explicitly specialized class template static data
625 // members have ordered initialization. Other class template static data
626 // members (i.e., implicitly or explicitly instantiated specializations)
627 // have unordered initialization.
628 //
629 // CXXGlobalInits.size() is the lex order number for the next deferred
630 // VarDecl. Use it when the current VarDecl is non-deferred. Although this
631 // lex order number is shared between current VarDecl and some following
632 // VarDecls, their order of insertion into `llvm.global_ctors` is the same
633 // as the lexing order and the following stable sort would preserve such
634 // order.
635 I = DelayedCXXInitPosition.find(Val: D);
636 unsigned LexOrder =
637 I == DelayedCXXInitPosition.end() ? CXXGlobalInits.size() : I->second;
638 AddGlobalCtor(Ctor: Fn, Priority: 65535, LexOrder, AssociatedData: COMDATKey);
639 if (COMDATKey && (getTriple().isOSBinFormatELF() ||
640 getTarget().getCXXABI().isMicrosoft())) {
641 // When COMDAT is used on ELF or in the MS C++ ABI, the key must be in
642 // llvm.used to prevent linker GC.
643 addUsedGlobal(GV: COMDATKey);
644 }
645
646 // If we used a COMDAT key for the global ctor, the init function can be
647 // discarded if the global ctor entry is discarded.
648 // FIXME: Do we need to restrict this to ELF and Wasm?
649 llvm::Comdat *C = Addr->getComdat();
650 if (COMDATKey && C &&
651 (getTarget().getTriple().isOSBinFormatELF() ||
652 getTarget().getTriple().isOSBinFormatWasm())) {
653 Fn->setComdat(C);
654 }
655 } else {
656 I = DelayedCXXInitPosition.find(Val: D); // Re-do lookup in case of re-hash.
657 if (I == DelayedCXXInitPosition.end()) {
658 CXXGlobalInits.push_back(x: Fn);
659 } else if (I->second != ~0U) {
660 assert(I->second < CXXGlobalInits.size() &&
661 CXXGlobalInits[I->second] == nullptr);
662 CXXGlobalInits[I->second] = Fn;
663 }
664 }
665
666 // Remember that we already emitted the initializer for this global.
667 DelayedCXXInitPosition[D] = ~0U;
668}
669
670void CodeGenModule::EmitCXXThreadLocalInitFunc() {
671 getCXXABI().EmitThreadLocalInitFuncs(
672 CGM&: *this, CXXThreadLocals, CXXThreadLocalInits, CXXThreadLocalInitVars);
673
674 CXXThreadLocalInits.clear();
675 CXXThreadLocalInitVars.clear();
676 CXXThreadLocals.clear();
677}
678
679/* Build the initializer for a C++20 module:
680 This is arranged to be run only once regardless of how many times the module
681 might be included transitively. This arranged by using a guard variable.
682
683 If there are no initializers at all (and also no imported modules) we reduce
684 this to an empty function (since the Itanium ABI requires that this function
685 be available to a caller, which might be produced by a different
686 implementation).
687
688 First we call any initializers for imported modules.
689 We then call initializers for the Global Module Fragment (if present)
690 We then call initializers for the current module.
691 We then call initializers for the Private Module Fragment (if present)
692*/
693
694void CodeGenModule::EmitCXXModuleInitFunc(Module *Primary) {
695 assert(Primary->isInterfaceOrPartition() &&
696 "The function should only be called for C++20 named module interface"
697 " or partition.");
698
699 while (!CXXGlobalInits.empty() && !CXXGlobalInits.back())
700 CXXGlobalInits.pop_back();
701
702 // As noted above, we create the function, even if it is empty.
703 // Module initializers for imported modules are emitted first.
704
705 // Collect all the modules that we import
706 llvm::SmallSetVector<Module *, 8> AllImports;
707 // Ones that we export
708 for (auto I : Primary->Exports)
709 AllImports.insert(X: I.first);
710 // Ones that we only import.
711 AllImports.insert_range(R&: Primary->Imports);
712 // Ones that we import in the global module fragment or the private module
713 // fragment.
714 for (Module *SubM : Primary->submodules()) {
715 assert((SubM->isGlobalModule() || SubM->isPrivateModule()) &&
716 "The sub modules of C++20 module unit should only be global module "
717 "fragments or private module framents.");
718 assert(SubM->Exports.empty() &&
719 "The global mdoule fragments and the private module fragments are "
720 "not allowed to export import modules.");
721 AllImports.insert_range(R&: SubM->Imports);
722 }
723
724 SmallVector<llvm::Function *, 8> ModuleInits;
725 for (Module *M : AllImports) {
726 // No Itanium initializer in header like modules.
727 if (M->isHeaderLikeModule())
728 continue; // TODO: warn of mixed use of module map modules and C++20?
729 // We're allowed to skip the initialization if we are sure it doesn't
730 // do any thing.
731 if (!M->isNamedModuleInterfaceHasInit())
732 continue;
733 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
734 SmallString<256> FnName;
735 {
736 llvm::raw_svector_ostream Out(FnName);
737 cast<ItaniumMangleContext>(Val&: getCXXABI().getMangleContext())
738 .mangleModuleInitializer(Module: M, Out);
739 }
740 assert(!GetGlobalValue(FnName.str()) &&
741 "We should only have one use of the initializer call");
742 llvm::Function *Fn = llvm::Function::Create(
743 Ty: FTy, Linkage: llvm::Function::ExternalLinkage, N: FnName.str(), M: &getModule());
744 ModuleInits.push_back(Elt: Fn);
745 }
746
747 // Add any initializers with specified priority; this uses the same approach
748 // as EmitCXXGlobalInitFunc().
749 if (!PrioritizedCXXGlobalInits.empty()) {
750 llvm::array_pod_sort(Start: PrioritizedCXXGlobalInits.begin(),
751 End: PrioritizedCXXGlobalInits.end());
752 for (SmallVectorImpl<GlobalInitData>::iterator
753 I = PrioritizedCXXGlobalInits.begin(),
754 E = PrioritizedCXXGlobalInits.end();
755 I != E;) {
756 SmallVectorImpl<GlobalInitData>::iterator PrioE =
757 std::upper_bound(first: I + 1, last: E, val: *I, comp: GlobalInitPriorityCmp());
758
759 for (; I < PrioE; ++I)
760 ModuleInits.push_back(Elt: I->second);
761 }
762 }
763
764 // Now append the ones without specified priority.
765 for (auto *F : CXXGlobalInits)
766 ModuleInits.push_back(Elt: F);
767
768 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
769 const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
770
771 // We now build the initializer for this module, which has a mangled name
772 // as per the Itanium ABI . The action of the initializer is guarded so that
773 // each init is run just once (even though a module might be imported
774 // multiple times via nested use).
775 llvm::Function *Fn;
776 {
777 SmallString<256> InitFnName;
778 llvm::raw_svector_ostream Out(InitFnName);
779 cast<ItaniumMangleContext>(Val&: getCXXABI().getMangleContext())
780 .mangleModuleInitializer(Module: Primary, Out);
781 Fn = CreateGlobalInitOrCleanUpFunction(
782 FTy, Name: llvm::Twine(InitFnName), FI, Loc: SourceLocation(), TLS: false,
783 Linkage: llvm::GlobalVariable::ExternalLinkage);
784
785 // If we have a completely empty initializer then we do not want to create
786 // the guard variable.
787 ConstantAddress GuardAddr = ConstantAddress::invalid();
788 if (!ModuleInits.empty()) {
789 // Create the guard var.
790 llvm::GlobalVariable *Guard = new llvm::GlobalVariable(
791 getModule(), Int8Ty, /*isConstant=*/false,
792 llvm::GlobalVariable::InternalLinkage,
793 llvm::ConstantInt::get(Ty: Int8Ty, V: 0), InitFnName.str() + "__in_chrg");
794 CharUnits GuardAlign = CharUnits::One();
795 Guard->setAlignment(GuardAlign.getAsAlign());
796 GuardAddr = ConstantAddress(Guard, Int8Ty, GuardAlign);
797 }
798 CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXThreadLocals: ModuleInits,
799 Guard: GuardAddr);
800 }
801
802 // We allow for the case that a module object is added to a linked binary
803 // without a specific call to the the initializer. This also ensures that
804 // implementation partition initializers are called when the partition
805 // is not imported as an interface.
806 AddGlobalCtor(Ctor: Fn);
807
808 // See the comment in EmitCXXGlobalInitFunc about OpenCL global init
809 // functions.
810 if (getLangOpts().OpenCL) {
811 GenKernelArgMetadata(FN: Fn);
812 Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
813 }
814
815 assert(!getLangOpts().CUDA || !getLangOpts().CUDAIsDevice ||
816 getLangOpts().GPUAllowDeviceInit);
817 if (getLangOpts().HIP && getLangOpts().CUDAIsDevice) {
818 if (getTriple().isSPIRV())
819 Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
820 else
821 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
822 Fn->addFnAttr(Kind: "device-init");
823 }
824
825 // We are done with the inits.
826 AllImports.clear();
827 PrioritizedCXXGlobalInits.clear();
828 CXXGlobalInits.clear();
829 ModuleInits.clear();
830}
831
832static SmallString<128> getTransformedFileName(llvm::Module &M) {
833 SmallString<128> FileName = llvm::sys::path::filename(path: M.getName());
834
835 if (FileName.empty())
836 FileName = "<null>";
837
838 for (size_t i = 0; i < FileName.size(); ++i) {
839 // Replace everything that's not [a-zA-Z0-9._] with a _. This set happens
840 // to be the set of C preprocessing numbers.
841 if (!isPreprocessingNumberBody(c: FileName[i]))
842 FileName[i] = '_';
843 }
844
845 return FileName;
846}
847
848static std::string getPrioritySuffix(unsigned int Priority) {
849 assert(Priority <= 65535 && "Priority should always be <= 65535.");
850
851 // Compute the function suffix from priority. Prepend with zeroes to make
852 // sure the function names are also ordered as priorities.
853 std::string PrioritySuffix = llvm::utostr(X: Priority);
854 PrioritySuffix = std::string(6 - PrioritySuffix.size(), '0') + PrioritySuffix;
855
856 return PrioritySuffix;
857}
858
859void
860CodeGenModule::EmitCXXGlobalInitFunc() {
861 while (!CXXGlobalInits.empty() && !CXXGlobalInits.back())
862 CXXGlobalInits.pop_back();
863
864 // When we import C++20 modules, we must run their initializers first.
865 SmallVector<llvm::Function *, 8> ModuleInits;
866 if (CXX20ModuleInits)
867 for (Module *M : ImportedModules) {
868 // No Itanium initializer in header like modules.
869 if (M->isHeaderLikeModule())
870 continue;
871 // We're allowed to skip the initialization if we are sure it doesn't
872 // do any thing.
873 if (!M->isNamedModuleInterfaceHasInit())
874 continue;
875 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
876 SmallString<256> FnName;
877 {
878 llvm::raw_svector_ostream Out(FnName);
879 cast<ItaniumMangleContext>(Val&: getCXXABI().getMangleContext())
880 .mangleModuleInitializer(Module: M, Out);
881 }
882 assert(!GetGlobalValue(FnName.str()) &&
883 "We should only have one use of the initializer call");
884 llvm::Function *Fn = llvm::Function::Create(
885 Ty: FTy, Linkage: llvm::Function::ExternalLinkage, N: FnName.str(), M: &getModule());
886 ModuleInits.push_back(Elt: Fn);
887 }
888
889 if (ModuleInits.empty() && CXXGlobalInits.empty() &&
890 PrioritizedCXXGlobalInits.empty())
891 return;
892
893 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
894 const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
895
896 // Create our global prioritized initialization function.
897 if (!PrioritizedCXXGlobalInits.empty()) {
898 SmallVector<llvm::Function *, 8> LocalCXXGlobalInits;
899 llvm::array_pod_sort(Start: PrioritizedCXXGlobalInits.begin(),
900 End: PrioritizedCXXGlobalInits.end());
901 // Iterate over "chunks" of ctors with same priority and emit each chunk
902 // into separate function. Note - everything is sorted first by priority,
903 // second - by lex order, so we emit ctor functions in proper order.
904 for (SmallVectorImpl<GlobalInitData >::iterator
905 I = PrioritizedCXXGlobalInits.begin(),
906 E = PrioritizedCXXGlobalInits.end(); I != E; ) {
907 SmallVectorImpl<GlobalInitData >::iterator
908 PrioE = std::upper_bound(first: I + 1, last: E, val: *I, comp: GlobalInitPriorityCmp());
909
910 LocalCXXGlobalInits.clear();
911
912 unsigned int Priority = I->first.priority;
913 llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
914 FTy, Name: "_GLOBAL__I_" + getPrioritySuffix(Priority), FI);
915
916 // Prepend the module inits to the highest priority set.
917 if (!ModuleInits.empty()) {
918 for (auto *F : ModuleInits)
919 LocalCXXGlobalInits.push_back(Elt: F);
920 ModuleInits.clear();
921 }
922
923 for (; I < PrioE; ++I)
924 LocalCXXGlobalInits.push_back(Elt: I->second);
925
926 CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXThreadLocals: LocalCXXGlobalInits);
927 AddGlobalCtor(Ctor: Fn, Priority);
928 }
929 PrioritizedCXXGlobalInits.clear();
930 }
931
932 if (getCXXABI().useSinitAndSterm() && ModuleInits.empty() &&
933 CXXGlobalInits.empty())
934 return;
935
936 for (auto *F : CXXGlobalInits)
937 ModuleInits.push_back(Elt: F);
938 CXXGlobalInits.clear();
939
940 // Include the filename in the symbol name. Including "sub_" matches gcc
941 // and makes sure these symbols appear lexicographically behind the symbols
942 // with priority emitted above. Module implementation units behave the same
943 // way as a non-modular TU with imports.
944 llvm::Function *Fn;
945 if (CXX20ModuleInits && getContext().getCurrentNamedModule() &&
946 !getContext().getCurrentNamedModule()->isModuleImplementation()) {
947 SmallString<256> InitFnName;
948 llvm::raw_svector_ostream Out(InitFnName);
949 cast<ItaniumMangleContext>(Val&: getCXXABI().getMangleContext())
950 .mangleModuleInitializer(Module: getContext().getCurrentNamedModule(), Out);
951 Fn = CreateGlobalInitOrCleanUpFunction(
952 FTy, Name: llvm::Twine(InitFnName), FI, Loc: SourceLocation(), TLS: false,
953 Linkage: llvm::GlobalVariable::ExternalLinkage);
954 } else
955 Fn = CreateGlobalInitOrCleanUpFunction(
956 FTy,
957 Name: llvm::Twine("_GLOBAL__sub_I_", getTransformedFileName(M&: getModule())),
958 FI);
959
960 CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXThreadLocals: ModuleInits);
961 AddGlobalCtor(Ctor: Fn);
962
963 // In OpenCL global init functions must be converted to kernels in order to
964 // be able to launch them from the host.
965 // FIXME: Some more work might be needed to handle destructors correctly.
966 // Current initialization function makes use of function pointers callbacks.
967 // We can't support function pointers especially between host and device.
968 // However it seems global destruction has little meaning without any
969 // dynamic resource allocation on the device and program scope variables are
970 // destroyed by the runtime when program is released.
971 if (getLangOpts().OpenCL) {
972 GenKernelArgMetadata(FN: Fn);
973 Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
974 }
975
976 assert(!getLangOpts().CUDA || !getLangOpts().CUDAIsDevice ||
977 getLangOpts().GPUAllowDeviceInit);
978 if (getLangOpts().HIP && getLangOpts().CUDAIsDevice) {
979 if (getTriple().isSPIRV())
980 Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
981 else
982 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
983 Fn->addFnAttr(Kind: "device-init");
984 }
985
986 ModuleInits.clear();
987}
988
989void CodeGenModule::EmitCXXGlobalCleanUpFunc() {
990 if (CXXGlobalDtorsOrStermFinalizers.empty() &&
991 PrioritizedCXXStermFinalizers.empty())
992 return;
993
994 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, isVarArg: false);
995 const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
996
997 // Create our global prioritized cleanup function.
998 if (!PrioritizedCXXStermFinalizers.empty()) {
999 SmallVector<CXXGlobalDtorsOrStermFinalizer_t, 8> LocalCXXStermFinalizers;
1000 llvm::array_pod_sort(Start: PrioritizedCXXStermFinalizers.begin(),
1001 End: PrioritizedCXXStermFinalizers.end());
1002 // Iterate over "chunks" of dtors with same priority and emit each chunk
1003 // into separate function. Note - everything is sorted first by priority,
1004 // second - by lex order, so we emit dtor functions in proper order.
1005 for (SmallVectorImpl<StermFinalizerData>::iterator
1006 I = PrioritizedCXXStermFinalizers.begin(),
1007 E = PrioritizedCXXStermFinalizers.end();
1008 I != E;) {
1009 SmallVectorImpl<StermFinalizerData>::iterator PrioE =
1010 std::upper_bound(first: I + 1, last: E, val: *I, comp: StermFinalizerPriorityCmp());
1011
1012 LocalCXXStermFinalizers.clear();
1013
1014 unsigned int Priority = I->first.priority;
1015 llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
1016 FTy, Name: "_GLOBAL__a_" + getPrioritySuffix(Priority), FI);
1017
1018 for (; I < PrioE; ++I) {
1019 llvm::FunctionCallee DtorFn = I->second;
1020 LocalCXXStermFinalizers.emplace_back(Args: DtorFn.getFunctionType(),
1021 Args: DtorFn.getCallee(), Args: nullptr);
1022 }
1023
1024 CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
1025 Fn, DtorsOrStermFinalizers: LocalCXXStermFinalizers);
1026 AddGlobalDtor(Dtor: Fn, Priority);
1027 }
1028 PrioritizedCXXStermFinalizers.clear();
1029 }
1030
1031 if (CXXGlobalDtorsOrStermFinalizers.empty())
1032 return;
1033
1034 // Create our global cleanup function.
1035 llvm::Function *Fn =
1036 CreateGlobalInitOrCleanUpFunction(FTy, Name: "_GLOBAL__D_a", FI);
1037
1038 CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
1039 Fn, DtorsOrStermFinalizers: CXXGlobalDtorsOrStermFinalizers);
1040 AddGlobalDtor(Dtor: Fn);
1041 CXXGlobalDtorsOrStermFinalizers.clear();
1042}
1043
1044/// Emit the code necessary to initialize the given global variable.
1045void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
1046 const VarDecl *D,
1047 llvm::GlobalVariable *Addr,
1048 bool PerformInit) {
1049 // Check if we need to emit debug info for variable initializer.
1050 if (D->hasAttr<NoDebugAttr>())
1051 DebugInfo = nullptr; // disable debug info indefinitely for this function
1052
1053 CurEHLocation = D->getBeginLoc();
1054
1055 StartFunction(GD: GlobalDecl(D, DynamicInitKind::Initializer),
1056 RetTy: getContext().VoidTy, Fn, FnInfo: getTypes().arrangeNullaryFunction(),
1057 Args: FunctionArgList());
1058 // Emit an artificial location for this function.
1059 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1060
1061 // Use guarded initialization if the global variable is weak. This
1062 // occurs for, e.g., instantiated static data members and
1063 // definitions explicitly marked weak.
1064 //
1065 // Also use guarded initialization for a variable with dynamic TLS and
1066 // unordered initialization. (If the initialization is ordered, the ABI
1067 // layer will guard the whole-TU initialization for us.)
1068 if (Addr->hasWeakLinkage() || Addr->hasLinkOnceLinkage() ||
1069 (D->getTLSKind() == VarDecl::TLS_Dynamic &&
1070 isTemplateInstantiation(Kind: D->getTemplateSpecializationKind()))) {
1071 EmitCXXGuardedInit(D: *D, DeclPtr: Addr, PerformInit);
1072 } else {
1073 EmitCXXGlobalVarDeclInit(D: *D, GV: Addr, PerformInit);
1074 }
1075
1076 FinishFunction();
1077}
1078
1079void
1080CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
1081 ArrayRef<llvm::Function *> Decls,
1082 ConstantAddress Guard) {
1083 {
1084 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1085 StartFunction(GD: GlobalDecl(), RetTy: getContext().VoidTy, Fn,
1086 FnInfo: getTypes().arrangeNullaryFunction(), Args: FunctionArgList());
1087 // Emit an artificial location for this function.
1088 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1089
1090 llvm::BasicBlock *ExitBlock = nullptr;
1091 if (Guard.isValid()) {
1092 // If we have a guard variable, check whether we've already performed
1093 // these initializations. This happens for TLS initialization functions.
1094 llvm::Value *GuardVal = Builder.CreateLoad(Addr: Guard);
1095 llvm::Value *Uninit = Builder.CreateIsNull(Arg: GuardVal,
1096 Name: "guard.uninitialized");
1097 llvm::BasicBlock *InitBlock = createBasicBlock(name: "init");
1098 ExitBlock = createBasicBlock(name: "exit");
1099 EmitCXXGuardedInitBranch(NeedsInit: Uninit, InitBlock, NoInitBlock: ExitBlock,
1100 Kind: GuardKind::TlsGuard, D: nullptr);
1101 EmitBlock(BB: InitBlock);
1102 // Mark as initialized before initializing anything else. If the
1103 // initializers use previously-initialized thread_local vars, that's
1104 // probably supposed to be OK, but the standard doesn't say.
1105 Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: GuardVal->getType(),V: 1), Addr: Guard);
1106
1107 // The guard variable can't ever change again.
1108 EmitInvariantStart(
1109 Addr: Guard.getPointer(),
1110 Size: CharUnits::fromQuantity(
1111 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: GuardVal->getType())));
1112 }
1113
1114 RunCleanupsScope Scope(*this);
1115
1116 // When building in Objective-C++ ARC mode, create an autorelease pool
1117 // around the global initializers.
1118 if (getLangOpts().ObjCAutoRefCount && getLangOpts().CPlusPlus) {
1119 llvm::Value *token = EmitObjCAutoreleasePoolPush();
1120 EmitObjCAutoreleasePoolCleanup(Ptr: token);
1121 }
1122
1123 for (llvm::Function *Decl : Decls)
1124 if (Decl)
1125 EmitRuntimeCall(callee: Decl);
1126
1127 Scope.ForceCleanup();
1128
1129 if (ExitBlock) {
1130 Builder.CreateBr(Dest: ExitBlock);
1131 EmitBlock(BB: ExitBlock);
1132 }
1133 }
1134
1135 FinishFunction();
1136}
1137
1138void CodeGenFunction::GenerateCXXGlobalCleanUpFunc(
1139 llvm::Function *Fn,
1140 ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
1141 llvm::Constant *>>
1142 DtorsOrStermFinalizers) {
1143 {
1144 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1145 StartFunction(GD: GlobalDecl(), RetTy: getContext().VoidTy, Fn,
1146 FnInfo: getTypes().arrangeNullaryFunction(), Args: FunctionArgList());
1147 // Emit an artificial location for this function.
1148 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1149
1150 // Emit the cleanups, in reverse order from construction.
1151 for (unsigned i = 0, e = DtorsOrStermFinalizers.size(); i != e; ++i) {
1152 llvm::FunctionType *CalleeTy;
1153 llvm::Value *Callee;
1154 llvm::Constant *Arg;
1155 std::tie(args&: CalleeTy, args&: Callee, args&: Arg) = DtorsOrStermFinalizers[e - i - 1];
1156
1157 llvm::CallBase *CI = nullptr;
1158 if (Arg == nullptr) {
1159 assert(
1160 CGM.getCXXABI().useSinitAndSterm() &&
1161 "Arg could not be nullptr unless using sinit and sterm functions.");
1162 CI = Builder.CreateCall(FTy: CalleeTy, Callee);
1163 } else {
1164 // If the object lives in a different address space, the `this` pointer
1165 // address space won't match the dtor `this` param. An addrspacecast is
1166 // required.
1167 assert(Arg->getType()->isPointerTy());
1168 assert(CalleeTy->getParamType(0)->isPointerTy());
1169 unsigned ActualAddrSpace = Arg->getType()->getPointerAddressSpace();
1170 unsigned ExpectedAddrSpace =
1171 CalleeTy->getParamType(i: 0)->getPointerAddressSpace();
1172 if (ActualAddrSpace != ExpectedAddrSpace) {
1173 llvm::PointerType *PTy =
1174 llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: ExpectedAddrSpace);
1175 Arg = llvm::ConstantExpr::getAddrSpaceCast(C: Arg, Ty: PTy);
1176 }
1177 CI = Builder.CreateCall(FTy: CalleeTy, Callee, Args: Arg);
1178 }
1179
1180 // Make sure the call and the callee agree on calling convention.
1181 if (llvm::Function *F = dyn_cast<llvm::Function>(Val: Callee))
1182 CI->setCallingConv(F->getCallingConv());
1183
1184 if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
1185 CI = addConvergenceControlToken(Input: CI);
1186 }
1187 }
1188
1189 FinishFunction();
1190}
1191
1192/// generateDestroyHelper - Generates a helper function which, when
1193/// invoked, destroys the given object. The address of the object
1194/// should be in global memory.
1195llvm::Function *CodeGenFunction::generateDestroyHelper(
1196 Address addr, QualType type, Destroyer *destroyer,
1197 bool useEHCleanupForArray, const VarDecl *VD) {
1198 auto *Dst = ImplicitParamDecl::Create(C&: getContext(), T: getContext().VoidPtrTy,
1199 ParamKind: ImplicitParamKind::Other);
1200
1201 FunctionArgList args{Dst};
1202 const CGFunctionInfo &FI =
1203 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: getContext().VoidTy, args);
1204 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1205 llvm::Function *fn = CGM.CreateGlobalInitOrCleanUpFunction(
1206 FTy, Name: "__cxx_global_array_dtor", FI, Loc: VD->getLocation());
1207
1208 CurEHLocation = VD->getBeginLoc();
1209
1210 StartFunction(GD: GlobalDecl(VD, DynamicInitKind::GlobalArrayDestructor),
1211 RetTy: getContext().VoidTy, Fn: fn, FnInfo: FI, Args: args);
1212 // Emit an artificial location for this function.
1213 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1214
1215 emitDestroy(addr, type, destroyer, useEHCleanupForArray);
1216
1217 FinishFunction();
1218
1219 return fn;
1220}
1221