1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code dealing with C++ code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenCXXABI.h"
14#include "CIRGenFunction.h"
15#include "CIRGenModule.h"
16
17#include "clang/AST/GlobalDecl.h"
18#include "clang/CIR/Dialect/IR/CIRAttrs.h"
19#include "clang/CIR/MissingFeatures.h"
20#include "llvm/Support/SaveAndRestore.h"
21
22using namespace clang;
23using namespace clang::CIRGen;
24
25/// Emit code to cause the variable at the given address to be considered as
26/// constant from this point onwards.
27static void emitDeclInvariant(CIRGenFunction &cgf, const VarDecl *d) {
28 mlir::Value addr = cgf.cgm.getAddrOfGlobalVar(d);
29 cgf.emitInvariantStart(size: cgf.getContext().getTypeSizeInChars(T: d->getType()),
30 addr, loc: cgf.getLoc(srcLoc: d->getSourceRange()));
31}
32
33void CIRGenFunction::emitInvariantStart(CharUnits size, mlir::Value addr,
34 mlir::Location loc) {
35 // Do not emit the intrinsic if we're not optimizing.
36 if (!cgm.getCodeGenOpts().OptimizationLevel)
37 return;
38
39 CIRGenBuilderTy &builder = getBuilder();
40
41 // Create the size constant as i64
42 uint64_t width = size.getQuantity();
43 mlir::Value sizeValue = builder.getConstInt(loc, t: builder.getSInt64Ty(),
44 c: static_cast<int64_t>(width));
45
46 // Create the intrinsic call. The llvm.invariant.start intrinsic returns a
47 // token, but we don't need to capture it. The address space will be
48 // automatically handled when the intrinsic is lowered to LLVM IR.
49 cir::LLVMIntrinsicCallOp::create(
50 builder, location: loc, intrinsic_name: builder.getStringAttr(bytes: "invariant.start"), resType: addr.getType(),
51 operands: mlir::ValueRange{sizeValue, addr});
52}
53
54/// The ctor and dtor regions of a global (or static local) hold a single block,
55/// but an initializer can have control flow: a `throw` or a trap ends the block
56/// it is in, and the rest of the initializer goes in a new, unreachable one.
57/// If emitting into \p region produced more than one block, move them all into
58/// a cir.scope, leaving the region with one block that holds the scope.
59static void wrapMultiBlockRegionInScope(CIRGenBuilderTy &builder,
60 mlir::Region &region,
61 mlir::Location loc) {
62 if (region.empty() || region.hasOneBlock())
63 return;
64
65 mlir::OpBuilder::InsertionGuard guard(builder);
66 mlir::Block *wrapper = builder.createBlock(parent: &region, insertPt: region.begin());
67 builder.setInsertionPointToStart(wrapper);
68 auto scope = cir::ScopeOp::create(builder, location: loc,
69 scopeBuilder: [](mlir::OpBuilder &, mlir::Location) {});
70
71 // Swap the scope's empty block for the original blocks, which already end in
72 // the cir.yield that terminates the scope.
73 mlir::Region &scopeRegion = scope.getRegion();
74 scopeRegion.getBlocks().clear();
75 scopeRegion.getBlocks().splice(where: scopeRegion.end(), L2&: region.getBlocks(),
76 first: std::next(x: region.begin()), last: region.end());
77
78 builder.setInsertionPointToEnd(wrapper);
79 cir::YieldOp::create(builder, location: loc);
80}
81
82static void emitDeclInit(CIRGenFunction &cgf, const VarDecl *varDecl,
83 cir::GlobalOp globalOp, mlir::Region &ctorRegion) {
84 assert((varDecl->hasGlobalStorage() ||
85 (varDecl->hasLocalStorage() &&
86 cgf.getContext().getLangOpts().OpenCLCPlusPlus)) &&
87 "VarDecl must have global or local (in the case of OpenCL) storage!");
88 assert(!varDecl->getType()->isReferenceType() &&
89 "Should not call emitDeclInit on a reference!");
90
91 CIRGenBuilderTy &builder = cgf.getBuilder();
92
93 // Set up the ctor region.
94 mlir::OpBuilder::InsertionGuard guard(builder);
95 mlir::Block *block = builder.createBlock(parent: &ctorRegion);
96 CIRGenFunction::LexicalScope lexScope{cgf, globalOp.getLoc(),
97 builder.getInsertionBlock()};
98 lexScope.setAsGlobalInit();
99 builder.setInsertionPointToStart(block);
100
101 Address declAddr(cgf.cgm.getAddrOfGlobalVar(d: varDecl),
102 cgf.cgm.getASTContext().getDeclAlign(D: varDecl));
103
104 QualType type = varDecl->getType();
105 LValue lv = cgf.makeAddrLValue(addr: declAddr, ty: type);
106
107 const Expr *init = varDecl->getInit();
108 switch (CIRGenFunction::getEvaluationKind(type)) {
109 case cir::TEK_Scalar:
110 assert(!cir::MissingFeatures::objCGC());
111 cgf.emitScalarInit(init, lvalue: lv, capturedByInit: false);
112 break;
113 case cir::TEK_Complex:
114 cgf.emitComplexExprIntoLValue(e: init, dest: lv, /*isInit=*/true);
115 break;
116 case cir::TEK_Aggregate:
117 assert(!cir::MissingFeatures::aggValueSlotGC());
118 cgf.emitAggExpr(e: init,
119 slot: AggValueSlot::forLValue(LV: lv, isDestructed: AggValueSlot::IsDestructed,
120 isAliased: AggValueSlot::IsNotAliased,
121 mayOverlap: AggValueSlot::DoesNotOverlap));
122 break;
123 }
124
125 // Finish the ctor region.
126 builder.setInsertionPointToEnd(builder.getInsertionBlock());
127 cir::YieldOp::create(builder, location: globalOp.getLoc());
128}
129
130static void emitDeclDestroy(CIRGenFunction &cgf, const VarDecl *vd,
131 cir::GlobalOp addr, mlir::Region &dtorRegion) {
132 // Honor __attribute__((no_destroy)) and bail instead of attempting
133 // to emit a reference to a possibly nonexistent destructor, which
134 // in turn can cause a crash. This will result in a global constructor
135 // that isn't balanced out by a destructor call as intended by the
136 // attribute. This also checks for -fno-c++-static-destructors and
137 // bails even if the attribute is not present.
138 QualType::DestructionKind dtorKind = vd->needsDestruction(Ctx: cgf.getContext());
139
140 // FIXME: __attribute__((cleanup)) ?
141
142 switch (dtorKind) {
143 case QualType::DK_none:
144 return;
145
146 case QualType::DK_cxx_destructor:
147 break;
148
149 case QualType::DK_objc_strong_lifetime:
150 case QualType::DK_objc_weak_lifetime:
151 case QualType::DK_nontrivial_c_struct:
152 // We don't care about releasing objects during process teardown.
153 assert(!vd->getTLSKind() && "should have rejected this");
154 return;
155 }
156
157 // If not constant storage we'll emit this regardless of NeedsDtor value.
158 CIRGenBuilderTy &builder = cgf.getBuilder();
159
160 // Prepare the dtor region.
161 mlir::OpBuilder::InsertionGuard guard(builder);
162 // Lifetime extended temporaries might have already created a block, so use
163 // that if it exists.
164 mlir::Block *block = dtorRegion.empty() ? builder.createBlock(parent: &dtorRegion)
165 : &dtorRegion.front();
166 CIRGenFunction::LexicalScope lexScope{cgf, addr.getLoc(), block};
167 lexScope.setAsGlobalInit();
168 builder.setInsertionPointToStart(block);
169
170 CIRGenModule &cgm = cgf.cgm;
171 QualType type = vd->getType();
172
173 // Special-case non-array C++ destructors, if they have the right signature.
174 // Under some ABIs, destructors return this instead of void, and cannot be
175 // passed directly to __cxa_atexit if the target does not allow this
176 // mismatch.
177 const CXXRecordDecl *record = type->getAsCXXRecordDecl();
178 bool canRegisterDestructor =
179 record && (!cgm.getCXXABI().hasThisReturn(
180 gd: GlobalDecl(record->getDestructor(), Dtor_Complete)) ||
181 cgm.getCXXABI().canCallMismatchedFunctionType());
182
183 // If __cxa_atexit is disabled via a flag, a different helper function is
184 // generated elsewhere which uses atexit instead, and it takes the destructor
185 // directly.
186 cir::FuncOp fnOp;
187 if (record && (canRegisterDestructor || cgm.getCodeGenOpts().CXAAtExit)) {
188 assert(!record->hasTrivialDestructor());
189 assert(!cir::MissingFeatures::openCL());
190 CXXDestructorDecl *dtor = record->getDestructor();
191 // In LLVM OG codegen this is done in registerGlobalDtor, but CIRGen
192 // relies on LoweringPrepare for further decoupling, so build the
193 // call right here.
194 auto gd = GlobalDecl(dtor, Dtor_Complete);
195 fnOp = cgm.getAddrAndTypeOfCXXStructor(gd).second;
196 // When a global has a constant initializer that fixes the active member
197 // of a union (e.g. an SSO short variant), CIR creates the global with
198 // the initializer's narrowed record type, so `getAddrOfGlobalVar` returns
199 // a pointer to the narrowed type rather than the variable's declared
200 // type. Mirror the cast pattern from `emitGlobalVarDeclLValue` so the
201 // destructor receives a `this` pointer typed as the declared class.
202 mlir::Value thisAddr = cgm.getAddrOfGlobalVar(d: vd);
203 mlir::Type realVarTy = cgm.getTypes().convertTypeForMem(type);
204 cir::PointerType realPtrTy = cir::PointerType::get(
205 pointee: realVarTy,
206 addrSpace: mlir::cast<cir::PointerType>(Val: thisAddr.getType()).getAddrSpace());
207 if (realPtrTy != thisAddr.getType())
208 thisAddr = builder.createBitcast(loc: thisAddr.getLoc(), src: thisAddr, newTy: realPtrTy);
209 // Make sure the call and the callee agree on calling convention.
210 builder.createCallOp(loc: cgf.getLoc(srcLoc: vd->getSourceRange()), callee: fnOp,
211 operands: mlir::ValueRange{thisAddr}, /*attrs=*/{},
212 /*argAttrs=*/{}, /*resAttrs=*/{},
213 callingConv: fnOp.getCallingConv());
214 assert(fnOp && "expected cir.func");
215 // TODO(cir): This doesn't do anything but check for unhandled conditions.
216 // What it is meant to do should really be happening in LoweringPrepare.
217 cgm.getCXXABI().registerGlobalDtor(vd, dtor: fnOp, addr: nullptr);
218 } else {
219 // Otherwise, a custom destroyed is needed. Classic codegen creates a helper
220 // function here and emits the destroy into the helper function, which is
221 // called from __cxa_atexit.
222 // In CIR, we just emit the destroy into the dtor region. It will be moved
223 // into a separate function during the LoweringPrepare pass.
224 // FIXME(cir): We should create a new operation here to explicitly get the
225 // address of the global into whose dtor region we are emiiting the destroy.
226 // The same applies to code above where it is calling getAddrOfGlobalVar.
227 mlir::Value globalVal = builder.createGetGlobal(global: addr);
228 globalVal.getDefiningOp<cir::GetGlobalOp>().setStaticLocal(
229 addr.getDynamicInitGuard().has_value() && vd->isLocalVarDecl());
230 CharUnits alignment = cgf.getContext().getDeclAlign(D: vd);
231 Address globalAddr{globalVal, cgf.convertTypeForMem(t: type), alignment};
232 cgf.emitDestroy(addr: globalAddr, type, destroyer: cgf.getDestroyer(kind: dtorKind));
233 }
234
235 builder.setInsertionPointToEnd(block);
236 if (block->empty()) {
237 block->erase();
238 // Don't confuse lexical cleanup.
239 builder.clearInsertionPoint();
240 } else if (!block->mightHaveTerminator()) {
241 // Temporary cleanup might have created a yield already: if not, create one.
242 cir::YieldOp::create(builder, location: addr.getLoc());
243 }
244}
245
246cir::FuncOp CIRGenModule::codegenCXXStructor(GlobalDecl gd) {
247 const CIRGenFunctionInfo &fnInfo =
248 getTypes().arrangeCXXStructorDeclaration(gd);
249 cir::FuncType funcType = getTypes().getFunctionType(info: fnInfo);
250 cir::FuncOp fn = getAddrOfCXXStructor(gd, fnInfo: &fnInfo, /*FnType=*/fnType: nullptr,
251 /*DontDefer=*/dontDefer: true, isForDefinition: ForDefinition);
252 setFunctionLinkage(gd, f: fn);
253 CIRGenFunction cgf{*this, builder};
254 curCGF = &cgf;
255 {
256 mlir::OpBuilder::InsertionGuard guard(builder);
257 cgf.generateCode(gd, fn, funcType);
258 }
259 curCGF = nullptr;
260
261 setNonAliasAttributes(gd, op: fn);
262 setCIRFunctionAttributesForDefinition(decl: mlir::cast<FunctionDecl>(Val: gd.getDecl()),
263 f: fn);
264 return fn;
265}
266
267// Global variables requiring non-trivial initialization are handled
268// differently in CIR than in classic codegen. Classic codegen emits
269// a global init function (__cxx_global_var_init) and inserts
270// initialization for each global there. In CIR, we attach a ctor
271// region to the global variable and insert the initialization code
272// into the ctor region. This will be moved into the
273// Map clang's VarDecl::TLSKind onto the CIR-native enum.
274static cir::TLSKind getCIRTLSKind(clang::VarDecl::TLSKind kind) {
275 switch (kind) {
276 case clang::VarDecl::TLS_None:
277 return cir::TLSKind::None;
278 case clang::VarDecl::TLS_Static:
279 return cir::TLSKind::Static;
280 case clang::VarDecl::TLS_Dynamic:
281 return cir::TLSKind::Dynamic;
282 }
283 llvm_unreachable("unknown TLSKind");
284}
285
286// Map clang's TemplateSpecializationKind onto the CIR-native enum.
287static cir::TemplateSpecializationKind
288getCIRTemplateSpecializationKind(clang::TemplateSpecializationKind kind) {
289 switch (kind) {
290 case clang::TSK_Undeclared:
291 return cir::TemplateSpecializationKind::Undeclared;
292 case clang::TSK_ImplicitInstantiation:
293 return cir::TemplateSpecializationKind::ImplicitInstantiation;
294 case clang::TSK_ExplicitSpecialization:
295 return cir::TemplateSpecializationKind::ExplicitSpecialization;
296 case clang::TSK_ExplicitInstantiationDeclaration:
297 return cir::TemplateSpecializationKind::ExplicitInstantiationDeclaration;
298 case clang::TSK_ExplicitInstantiationDefinition:
299 return cir::TemplateSpecializationKind::ExplicitInstantiationDefinition;
300 }
301 llvm_unreachable("unknown clang::TemplateSpecializationKind");
302}
303
304// __cxx_global_var_init function during the LoweringPrepare pass.
305void CIRGenModule::emitCXXSpecialVarDeclInit(const VarDecl *varDecl,
306 cir::GlobalOp addr,
307 bool performInit,
308 mlir::Region &ctorRegion,
309 mlir::Region &dtorRegion) {
310 QualType ty = varDecl->getType();
311 assert(curCGF && "Special var init only available inside of a function");
312 CIRGenFunction &cgf = *curCGF;
313
314 // TODO: handle address space
315 // The address space of a static local variable (addr) may be different
316 // from the address space of the "this" argument of the constructor. In that
317 // case, we need an addrspacecast before calling the constructor.
318 //
319 // struct StructWithCtor {
320 // __device__ StructWithCtor() {...}
321 // };
322 // __device__ void foo() {
323 // __shared__ StructWithCtor s;
324 // ...
325 // }
326 //
327 // For example, in the above CUDA code, the static local variable s has a
328 // "shared" address space qualifier, but the constructor of StructWithCtor
329 // expects "this" in the "generic" address space.
330 assert(!cir::MissingFeatures::addressSpace());
331
332 // Attach the AST handle for consumers that need arbitrary AST properties.
333 addr.setAstAttr(cir::ASTVarDeclAttr::get(context: &getMLIRContext(), ast: varDecl));
334
335 // For dynamic-init-guarded globals, also materialize the specific facts
336 // LoweringPrepare needs into a serializable attribute, so that lowering can
337 // run without a live ASTContext (e.g. on serialized CIR in split-compilation
338 // flows). This is orthogonal to the AST handle above.
339 if (addr.getDynamicInitGuard().has_value())
340 addr.setDynamicInitInfoAttr(cir::DynamicInitInfoAttr::get(
341 context: &getMLIRContext(), local: varDecl->isLocalVarDecl(),
342 tls: getCIRTLSKind(kind: varDecl->getTLSKind()), is_inline: varDecl->isInline(),
343 tsk: getCIRTemplateSpecializationKind(
344 kind: varDecl->getTemplateSpecializationKind())));
345
346 if (!ty->isReferenceType()) {
347 assert(!cir::MissingFeatures::openMP());
348
349 bool needsDtor = varDecl->needsDestruction(Ctx: getASTContext()) ==
350 QualType::DK_cxx_destructor;
351 bool isConstantStorage =
352 varDecl->getType().isConstantStorage(Ctx: getASTContext(), ExcludeCtor: true, ExcludeDtor: !needsDtor);
353 // PerformInit, constant store invariant / destroy handled below.
354 if (performInit) {
355 emitDeclInit(cgf, varDecl, globalOp: addr, ctorRegion);
356 // Do this before anything else is added to the end of the region.
357 wrapMultiBlockRegionInScope(builder&: cgf.getBuilder(), region&: ctorRegion, loc: addr.getLoc());
358 // For constant storage, emit invariant.start in the ctor region after
359 // initialization but before the yield.
360 if (isConstantStorage) {
361 CIRGenBuilderTy &builder = cgf.getBuilder();
362 mlir::OpBuilder::InsertionGuard guard(builder);
363 // Set insertion point to end of ctor region (before yield)
364 if (!ctorRegion.empty()) {
365 mlir::Block *block = &ctorRegion.back();
366 // Find the yield op and insert before it
367 mlir::Operation *yieldOp = block->getTerminator();
368 if (yieldOp) {
369 builder.setInsertionPoint(yieldOp);
370 emitDeclInvariant(cgf, d: varDecl);
371 }
372 }
373 }
374 } else if (isConstantStorage) {
375 emitDeclInvariant(cgf, d: varDecl);
376 }
377
378 if (!isConstantStorage) {
379 emitDeclDestroy(cgf, vd: varDecl, addr, dtorRegion);
380 wrapMultiBlockRegionInScope(builder&: cgf.getBuilder(), region&: dtorRegion, loc: addr.getLoc());
381 }
382 return;
383 }
384
385 mlir::OpBuilder::InsertionGuard guard(builder);
386 auto *block = builder.createBlock(parent: &ctorRegion);
387 CIRGenFunction::LexicalScope scope{*curCGF, addr.getLoc(),
388 builder.getInsertionBlock()};
389 scope.setAsGlobalInit();
390 builder.setInsertionPointToStart(block);
391 mlir::Value getGlobal = builder.createGetGlobal(global: addr, threadLocal: varDecl->getTLSKind());
392 // If we're initializing a guarded local static, set the flag that
393 // indicates that. Namespace-scope vague-linkage globals also get a
394 // dynamic-init guard, but aren't "static locals".
395 getGlobal.getDefiningOp<cir::GetGlobalOp>().setStaticLocal(
396 addr.getDynamicInitGuard().has_value() && varDecl->isLocalVarDecl());
397
398 Address declAddr(getGlobal, getASTContext().getDeclAlign(D: varDecl));
399 assert(performInit && "cannot have a constant initializer which needs "
400 "destruction for reference");
401 RValue rv = cgf.emitReferenceBindingToExpr(e: varDecl->getInit());
402 {
403 mlir::OpBuilder::InsertionGuard guard(builder);
404 mlir::Operation *rvalDefOp = rv.getValue().getDefiningOp();
405 if (rvalDefOp && rvalDefOp->getBlock()) {
406 mlir::Block *rvalSrcBlock = rvalDefOp->getBlock();
407
408 if (!rvalSrcBlock->empty() && isa<cir::YieldOp>(Val: rvalSrcBlock->back())) {
409 mlir::Operation &front = rvalSrcBlock->front();
410 getGlobal.getDefiningOp()->moveBefore(existingOp: &front);
411 builder.setInsertionPoint(cast<cir::YieldOp>(Val&: rvalSrcBlock->back()));
412 }
413 }
414 cgf.emitStoreOfScalar(value: rv.getValue(), addr: declAddr, /*isVolatile=*/false, ty,
415 baseInfo: LValueBaseInfo{});
416 }
417
418 builder.setInsertionPointToEnd(builder.getInsertionBlock());
419 cir::YieldOp::create(builder, location: addr->getLoc());
420 wrapMultiBlockRegionInScope(builder, region&: ctorRegion, loc: addr.getLoc());
421}
422
423void CIRGenModule::emitCXXGlobalVarDeclInit(const VarDecl *varDecl,
424 cir::GlobalOp addr,
425 bool performInit) {
426 assert(!varDecl->isStaticLocal());
427
428 // Create a CIRGenFunction to emit the initializer. While this isn't a true
429 // function, the handling works the same way.
430 CIRGenFunction cgf{*this, builder, true};
431 llvm::SaveAndRestore<CIRGenFunction *> savedCGF(curCGF, &cgf);
432 curCGF->curFn = addr;
433
434 CIRGenFunction::SourceLocRAIIObject fnLoc{cgf, varDecl->getSourceRange()};
435
436 // Set up the constrained FP environment for the dynamic initializer.
437 llvm::RoundingMode rm = getLangOpts().getDefaultRoundingMode();
438 LangOptions::FPExceptionModeKind eb = getLangOpts().getDefaultExceptionMode();
439 builder.setDefaultConstrainedRounding(rm);
440 builder.setDefaultConstrainedExcept(eb);
441 builder.setIsFPConstrained(false);
442 if (eb != LangOptions::FPE_Ignore ||
443 rm != llvm::RoundingMode::NearestTiesToEven) {
444 builder.setIsFPConstrained(true);
445 addr.setStrictfp(true);
446 }
447
448 if (const auto *ipa = varDecl->getAttr<InitPriorityAttr>())
449 addr.setInitPriority(ipa->getPriority());
450
451 emitCXXSpecialVarDeclInit(varDecl, addr, performInit, ctorRegion&: addr.getCtorRegion(),
452 dtorRegion&: addr.getDtorRegion());
453}
454
455void CIRGenModule::emitCXXStaticLocalVarDeclInit(const VarDecl *varDecl,
456 cir::GlobalOp addr,
457 bool performInit) {
458 assert(varDecl->isStaticLocal());
459
460 auto initOp =
461 cir::LocalInitOp::create(builder, location: addr->getLoc(), globalName: addr.getSymNameAttr(),
462 tls: varDecl->getTLSKind() != VarDecl::TLS_None);
463
464 emitCXXSpecialVarDeclInit(varDecl, addr, performInit, ctorRegion&: initOp.getCtorRegion(),
465 dtorRegion&: initOp.getDtorRegion());
466}
467