1//===--- CIRGenCleanup.cpp - Bookkeeping and code emission for cleanups ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains code dealing with the IR generation for cleanups
10// and related information.
11//
12// A "cleanup" is a piece of code which needs to be executed whenever
13// control transfers out of a particular scope. This can be
14// conditionalized to occur only on exceptional control flow, only on
15// normal control flow, or both.
16//
17//===----------------------------------------------------------------------===//
18
19#include "CIRGenCleanup.h"
20#include "CIRGenFunction.h"
21
22#include "clang/AST/RecursiveASTVisitor.h"
23#include "llvm/Support/MathExtras.h"
24#include "llvm/Support/SaveAndRestore.h"
25
26using namespace clang;
27using namespace clang::CIRGen;
28
29namespace {
30/// Return true if the expression tree contains a construct that causes cleanups
31/// to be deferred via pushFullExprCleanup.
32class ConditionalEvaluationFinder
33 : public RecursiveASTVisitor<ConditionalEvaluationFinder> {
34 bool foundConditional = false;
35 bool inResumeExpr = false;
36
37 bool traverseSuspendExpr(CoroutineSuspendExpr *e) {
38 return TraverseStmt(S: e->getCommonExpr()) && traverseResumeExpr(e);
39 }
40
41 /// Traverse the resume expression of \p e, except for the binding of its
42 /// result, which emitSuspendExpression destroys after the cir.await.
43 bool traverseResumeExpr(CoroutineSuspendExpr *e) {
44 Expr *resume = e->getResumeExpr();
45 if (auto *bind = dyn_cast_if_present<CXXBindTemporaryExpr>(Val: resume))
46 resume = bind->getSubExpr();
47 llvm::SaveAndRestore inResume(inResumeExpr, true);
48 return TraverseStmt(S: resume);
49 }
50
51public:
52 static bool needsCleanupScope(const Stmt *s) {
53 ConditionalEvaluationFinder finder;
54 finder.TraverseStmt(S: const_cast<Stmt *>(s));
55 return finder.foundConditional;
56 }
57
58 static bool resumeExprDefersCleanups(const CoroutineSuspendExpr &s) {
59 ConditionalEvaluationFinder finder;
60 finder.traverseResumeExpr(e: const_cast<CoroutineSuspendExpr *>(&s));
61 return finder.foundConditional;
62 }
63
64 // Default arguments and default member initializers can contain conditional
65 // temporaries whose cleanups belong to the enclosing full-expression.
66 bool shouldVisitImplicitCode() const { return true; }
67
68 bool VisitAbstractConditionalOperator(AbstractConditionalOperator *) {
69 foundConditional = true;
70 return false;
71 }
72
73 bool VisitBinaryOperator(BinaryOperator *e) {
74 if (e->isLogicalOp()) {
75 foundConditional = true;
76 return false;
77 }
78 return true;
79 }
80
81 bool VisitCXXNewExpr(CXXNewExpr *e) {
82 // If the new expression has an initializer, the initializer may contain a
83 // a temporary expression that requires deferred cleanup. If we're emitting
84 // a null check, we need to make this cleanup conditional.
85 if (e->hasInitializer() && e->shouldNullCheckAllocation()) {
86 foundConditional = true;
87 return false;
88 }
89 return true;
90 }
91
92 // Don't cross evaluation-context boundaries. Only the initializers of a
93 // lambda's captures are part of the enclosing full-expression.
94 bool TraverseLambdaExpr(LambdaExpr *e) {
95 for (Expr *init : e->capture_inits())
96 if (init && !TraverseStmt(S: init))
97 return false;
98 return true;
99 }
100 bool TraverseBlockExpr(BlockExpr *) { return true; }
101 bool TraverseStmtExpr(StmtExpr *) { return true; }
102
103 bool TraverseGCCAsmStmt(GCCAsmStmt *s) {
104 for (Stmt *operand : s->children())
105 if (!TraverseStmt(S: operand))
106 return false;
107 return true;
108 }
109
110 // Traverse the common and resume expressions, whose temporaries belong to
111 // the enclosing full-expression. Ready and suspend have their own cleanup
112 // boundaries. The resume result's cleanup is handled separately.
113 bool TraverseCoawaitExpr(CoawaitExpr *e) { return traverseSuspendExpr(e); }
114 bool TraverseCoyieldExpr(CoyieldExpr *e) { return traverseSuspendExpr(e); }
115
116 bool VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *) {
117 if (!inResumeExpr)
118 return true;
119 foundConditional = true;
120 return false;
121 }
122
123 bool VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *e) {
124 if (!inResumeExpr || !e->getType().isDestructedType())
125 return true;
126 foundConditional = true;
127 return false;
128 }
129};
130
131/// Owns a copy of a cleanup and its metadata so they remain valid after the EH
132/// stack entry is popped.
133///
134/// Every popped cleanup is copied, so the copy must not allocate memory in the
135/// common case. It goes into \c data, which holds cleanups of up to 8 pointers
136/// on the stack. A SavedCleanup is normally a local variable, so its inline
137/// storage, and with it the copy, lives on the stack. Only a larger cleanup
138/// falls back to the heap. SavedCleanups are trivially copyable, meaning a
139/// bitwise copy will be performed.
140class SavedCleanup {
141 struct alignas(EHScopeStack::ScopeStackAlignment) StorageUnit {
142 char bytes[EHScopeStack::ScopeStackAlignment];
143 };
144 llvm::SmallVector<StorageUnit, 8 * sizeof(void *) / sizeof(StorageUnit)> data;
145 size_t size;
146 CleanupKind kind;
147 Address activeFlag;
148
149public:
150 explicit SavedCleanup(EHCleanupScope &scope)
151 : size(scope.getCleanupSize()), activeFlag(scope.getActiveFlag()) {
152 unsigned cleanupKind = 0;
153 if (scope.isNormalCleanup())
154 cleanupKind |= NormalCleanup;
155 if (scope.isEHCleanup())
156 cleanupKind |= EHCleanup;
157 if (scope.isLifetimeMarker())
158 cleanupKind |= LifetimeMarker;
159 kind = static_cast<CleanupKind>(cleanupKind);
160
161 data.resize_for_overwrite(N: llvm::divideCeil(Numerator: size, Denominator: sizeof(StorageUnit)));
162 memcpy(dest: data.data(), src: scope.getCleanupBuffer(), n: size);
163 }
164
165 const void *getData() const { return data.data(); }
166 size_t getSize() const { return size; }
167 CleanupKind getKind() const { return kind; }
168 Address getActiveFlag() const { return activeFlag; }
169
170 EHScopeStack::Cleanup *getCleanup() {
171 return reinterpret_cast<EHScopeStack::Cleanup *>(data.data());
172 }
173
174 EHScopeStack::Cleanup::Flags getFlags() const {
175 EHScopeStack::Cleanup::Flags flags;
176 if (kind & NormalCleanup)
177 flags.setIsNormalCleanupKind();
178 if (kind & EHCleanup)
179 flags.setIsEHCleanupKind();
180 return flags;
181 }
182};
183} // namespace
184
185//===----------------------------------------------------------------------===//
186// CIRGenFunction cleanup related
187//===----------------------------------------------------------------------===//
188
189bool CIRGenFunction::resumeExprDefersCleanups(const CoroutineSuspendExpr &s) {
190 return ConditionalEvaluationFinder::resumeExprDefersCleanups(s);
191}
192
193/// Emits all the code to cause the given temporary to be cleaned up.
194void CIRGenFunction::emitCXXTemporary(const CXXTemporary *temporary,
195 QualType tempType, Address ptr) {
196 pushDestroy(kind: NormalAndEHCleanup, addr: ptr, type: tempType, destroyer: destroyCXXObject);
197}
198
199Address CIRGenFunction::createCleanupActiveFlag() {
200 assert(isInConditionalBranch());
201 mlir::Location loc = builder.getUnknownLoc();
202
203 // Place the alloca in the function entry block so it dominates everything,
204 // including both regions of any enclosing cir.cleanup.scope. We can't rely
205 // on the default curLexScope path because we may be inside a ternary branch
206 // whose LexicalScope would capture the alloca.
207 Address active = createTempAllocaWithoutCast(
208 ty: builder.getBoolTy(), align: CharUnits::One(), loc, name: "cleanup.cond",
209 /*arraySize=*/nullptr,
210 ip: builder.getBestAllocaInsertPoint(block: getCurFunctionEntryBlock()));
211
212 // Initialize to false before the outermost conditional.
213 {
214 mlir::OpBuilder::InsertionGuard guard(builder);
215 builder.restoreInsertionPoint(ip: outermostConditional->getInsertPoint());
216 cir::StoreOp store =
217 builder.createFlagStore(loc, val: false, dst: active.getPointer());
218 outermostConditional->advanceInsertPoint(op: store);
219 }
220
221 // Set to true at the current location (inside the conditional branch).
222 builder.createFlagStore(loc, val: true, dst: active.getPointer());
223
224 return active;
225}
226
227void CIRGenFunction::initFullExprCleanup() {
228 initFullExprCleanupWithFlag(activeFlag: createCleanupActiveFlag());
229}
230
231void CIRGenFunction::initFullExprCleanupWithFlag(Address activeFlag) {
232 EHCleanupScope &cleanup = cast<EHCleanupScope>(Val&: *ehStack.begin());
233 assert(!cleanup.hasActiveFlag() && "cleanup already has active flag?");
234 cleanup.setActiveFlag(activeFlag);
235
236 cleanup.setTestFlagInNormalCleanup(cleanup.isNormalCleanup());
237 cleanup.setTestFlagInEHCleanup(cleanup.isEHCleanup());
238}
239
240CIRGenFunction::FullExprCleanupScope::FullExprCleanupScope(CIRGenFunction &cgf,
241 const Stmt *s)
242 : cgf(cgf), cleanups(cgf), scope(nullptr),
243 oldFullExprCleanupScope(cgf.currentFullExprCleanupScope),
244 oldOutermostConditional(cgf.outermostConditional),
245 oldHiddenOutermostConditional(cgf.hiddenOutermostConditional),
246 deferredCleanupStackSize(cgf.deferredConditionalCleanupStack.size()),
247 conditionalScopeDepth(cgf.conditionalCleanupScopes.size()) {
248
249 assert(s && "expected an AST root for a cleanup boundary");
250 if (ConditionalEvaluationFinder::needsCleanupScope(s)) {
251 mlir::Location loc = cgf.builder.getUnknownLoc();
252 cir::CleanupKind cleanupKind = cgf.getLangOpts().Exceptions
253 ? cir::CleanupKind::All
254 : cir::CleanupKind::Normal;
255 scope = cir::CleanupScopeOp::create(
256 builder&: cgf.builder, location: loc, cleanupKind,
257 /*bodyBuilder=*/
258 [&](mlir::OpBuilder &b, mlir::Location loc) {},
259 /*cleanupBuilder=*/
260 [&](mlir::OpBuilder &b, mlir::Location loc) {});
261 cgf.builder.setInsertionPointToEnd(&scope.getBodyRegion().front());
262 }
263
264 // Conditional cleanup scopes are only opened while a full-expression scope
265 // is active to close them.
266 cgf.currentFullExprCleanupScope = scope;
267
268 // This full-expression owns its cleanups, even inside a conditional branch.
269 // Hide the outer conditional to keep those cleanups here, but remember it
270 // for lifetime-extended temporaries whose cleanups belong to an outer scope.
271 if (cgf.outermostConditional)
272 cgf.hiddenOutermostConditional = cgf.outermostConditional;
273 cgf.outermostConditional = nullptr;
274}
275
276/// If the alloca that backs \p addr is currently nested inside the body
277/// region of \p scope, hoist it, and any cast chain leading to it, out of the
278// scope so the alloca dominates the scope's sibling cleanup region.
279static void hoistAllocaOutOfCleanupScope(CIRGenFunction &cgf, Address addr,
280 cir::CleanupScopeOp scope) {
281 cir::AllocaOp alloca = addr.getUnderlyingAllocaOp();
282 if (!alloca)
283 return;
284
285 // If the alloca is not contained within the cleanup scope we're currently
286 // proccessing we don't need to hoist it.
287 auto cur = alloca->getParentOfType<cir::CleanupScopeOp>();
288 while (cur && cur != scope)
289 cur = cur->getParentOfType<cir::CleanupScopeOp>();
290 if (cur != scope)
291 return;
292
293 // Place the alloca at the canonical alloca insertion point of the block
294 // containing the cleanup scope op, so it groups with any preceding
295 // allocas / labels and dominates both the body and cleanup regions.
296 mlir::Block *parentBlock = scope->getBlock();
297 mlir::OpBuilder::InsertPoint ip =
298 CIRGenBuilderTy::getBestAllocaInsertPoint(block: parentBlock);
299 alloca->moveBefore(block: parentBlock, iterator: ip.getPoint());
300
301 // Move any cast chain that consumes the alloca's result to immediately after
302 // the alloca, so the address used by the deferred cleanup also dominates the
303 // cleanup region. We walk down the chain starting from the alloca's user
304 // that the Address was built from. This is very conservative. In practice,
305 // we should only ever see alloca or address_space(alloca) operations here.
306 mlir::Value ptr = addr.getPointer();
307 llvm::SmallVector<cir::CastOp> casts;
308 for (mlir::Operation *cur = ptr.getDefiningOp(); cur && cur != alloca;) {
309 auto cast = mlir::dyn_cast<cir::CastOp>(Val: cur);
310 if (!cast)
311 break;
312 casts.push_back(Elt: cast);
313 cur = cast.getSrc().getDefiningOp();
314 }
315 // Move casts in source order (closest to the alloca first).
316 mlir::Operation *prev = alloca;
317 for (cir::CastOp cast : llvm::reverse(C&: casts)) {
318 cast->moveAfter(existingOp: prev);
319 prev = cast;
320 }
321}
322
323/// Close a cleanup scope opened by a ConditionalEvaluation. Terminate its body
324/// region and emit this scope's slice of deferredConditionalCleanupStack into
325/// its cleanup region, in reverse of push order. An insertion point that was
326/// inside the scope's body is moved to immediately after the scope op.
327static void closeConditionalCleanupScope(
328 CIRGenFunction &cgf,
329 const CIRGenFunction::ConditionalCleanupScope &condScope) {
330 CIRGenBuilderTy &builder = cgf.getBuilder();
331 cir::CleanupScopeOp scope = condScope.scope;
332 size_t base = condScope.deferredCleanupStackSize;
333 auto &stack = cgf.deferredConditionalCleanupStack;
334 bool hasDeferredCleanups = stack.size() > base;
335
336 // Make sure the body region has a terminator.
337 {
338 mlir::OpBuilder::InsertionGuard guard(builder);
339 mlir::Block &lastBodyBlock = scope.getBodyRegion().back();
340 builder.setInsertionPointToEnd(&lastBodyBlock);
341 if (lastBodyBlock.empty() ||
342 !lastBodyBlock.back().hasTrait<mlir::OpTrait::IsTerminator>())
343 builder.createYield(loc: scope.getLoc());
344 }
345
346 // Each deferred cleanup references its addr from the sibling cleanup region
347 // we are about to fill. If the alloca that backs that addr was created
348 // inside this scope's body region, hoist it out so it dominates the cleanup
349 // region.
350 if (hasDeferredCleanups) {
351 for (const CIRGenFunction::PendingCleanupEntry &entry :
352 llvm::make_range(x: stack.begin() + base, y: stack.end()))
353 hoistAllocaOutOfCleanupScope(cgf, addr: entry.addr, scope);
354 }
355
356 {
357 mlir::OpBuilder::InsertionGuard guard(builder);
358 mlir::Block &cleanupBlock = scope.getCleanupRegion().front();
359 builder.setInsertionPointToEnd(&cleanupBlock);
360
361 for (const CIRGenFunction::PendingCleanupEntry &entry :
362 llvm::reverse(C: llvm::make_range(x: stack.begin() + base, y: stack.end()))) {
363 if (entry.activeFlag.isValid()) {
364 // We may have hoisted this alloca out of the cleanup scope. If so, we
365 // will have also hoisted any casts between it and the address that we
366 // stored in the deferredConditionalCleanupStack. While I can't find a
367 // case where this actually happens, there is a theoretical
368 // possibility that we could have a second address that uses an alloca
369 // that has already been hoisted but a different cast chain. This
370 // assert guards against that possibility.
371 assert(entry.addr.getUnderlyingAllocaOp() &&
372 (entry.addr.getUnderlyingAllocaOp()->getBlock() ==
373 entry.addr.getPointer().getDefiningOp()->getBlock()) &&
374 "alloca and cast are in different blocks");
375 mlir::Value flag = builder.createLoad(loc: scope.getLoc(), addr: entry.activeFlag);
376 cir::IfOp::create(builder, location: scope.getLoc(), cond: flag,
377 /*withElseRegion=*/false,
378 thenBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) {
379 cgf.emitDestroy(addr: entry.addr, type: entry.type,
380 destroyer: entry.destroyer);
381 builder.createYield(loc);
382 });
383 } else {
384 cgf.emitDestroy(addr: entry.addr, type: entry.type, destroyer: entry.destroyer);
385 }
386 }
387 builder.createYield(loc: scope.getLoc());
388 }
389
390 stack.truncate(N: base);
391
392 // Move the insertion point out of the scope just closed, but only if it was
393 // inside it. Moving it unconditionally can strand it outside the region it
394 // belonged to, leaving an enclosing scope unterminated. popCleanup guards
395 // this the same way.
396 mlir::Block *insertBlock = builder.getInsertionBlock();
397 if (insertBlock &&
398 scope.getBodyRegion().findAncestorBlockInRegion(block&: *insertBlock))
399 builder.setInsertionPointAfter(scope);
400}
401
402void CIRGenFunction::FullExprCleanupScope::exit(
403 ArrayRef<mlir::Value *> valuesToReload) {
404 assert(!exited && "FullExprCleanupScope::exit called twice");
405 exited = true;
406
407 cgf.currentFullExprCleanupScope = oldFullExprCleanupScope;
408 cgf.outermostConditional = oldOutermostConditional;
409 cgf.hiddenOutermostConditional = oldHiddenOutermostConditional;
410
411 size_t oldSize = deferredCleanupStackSize;
412
413 if (!scope) {
414 assert(cgf.conditionalCleanupScopes.size() == conditionalScopeDepth &&
415 "conditional cleanup scope opened without one to close it");
416 cgf.deferredConditionalCleanupStack.truncate(N: oldSize);
417 cleanups.forceCleanup(valuesToReload);
418 return;
419 }
420
421 // Spill any values that callers need after the scope is closed.
422 SmallVector<Address> tempAllocas;
423 for (mlir::Value *valPtr : valuesToReload) {
424 mlir::Value val = *valPtr;
425 if (!val) {
426 tempAllocas.push_back(Elt: Address::invalid());
427 continue;
428 }
429 Address temp = cgf.createDefaultAlignTempAlloca(ty: val.getType(), loc: val.getLoc(),
430 name: "tmp.exprcleanup");
431 tempAllocas.push_back(Elt: temp);
432 cgf.builder.createStore(loc: val.getLoc(), val, dst: temp);
433 }
434
435 // Close the cleanup scopes opened by conditionals in this full expression,
436 // innermost first. The EH cleanups popped below own the scopes enclosing
437 // these, so they are closed after.
438 while (cgf.conditionalCleanupScopes.size() > conditionalScopeDepth)
439 closeConditionalCleanupScope(cgf,
440 condScope: cgf.conditionalCleanupScopes.pop_back_val());
441
442 // Pop any EH cleanups that were pushed during the expression but leave
443 // any lifetime-extended cleanups so that they can be promoted to the EH
444 // stack after we've finished emitting any deferred cleanups.
445 cleanups.forceCleanupExceptLifetimeExtended();
446
447 // Make sure the cleanup scope body region has a terminator.
448 {
449 mlir::OpBuilder::InsertionGuard guard(cgf.builder);
450 mlir::Block &lastBodyBlock = scope.getBodyRegion().back();
451 cgf.builder.setInsertionPointToEnd(&lastBodyBlock);
452 if (lastBodyBlock.empty() ||
453 !lastBodyBlock.back().hasTrait<mlir::OpTrait::IsTerminator>())
454 cgf.builder.createYield(loc: scope.getLoc());
455 }
456
457 // Deferred cleanups are emitted into the conditional scopes closed above,
458 // so this scope's cleanup region is empty and canonicalization will inline
459 // the scope away.
460 {
461 mlir::OpBuilder::InsertionGuard guard(cgf.builder);
462 mlir::Block &cleanupBlock = scope.getCleanupRegion().front();
463 cgf.builder.setInsertionPointToEnd(&cleanupBlock);
464 cgf.builder.createYield(loc: scope.getLoc());
465 }
466
467 assert(cgf.deferredConditionalCleanupStack.size() == oldSize &&
468 "deferred cleanups were not consumed by a conditional scope");
469 cgf.builder.setInsertionPointAfter(scope);
470
471 // Promote any lifetime-extended cleanups onto the EH scope stack. The new
472 // cir.cleanup.scope ops created here will wrap any code in the enclosing
473 // scope, including reloads of any spilled values below, so the
474 // lifetime-extended destructors run at the correct point.
475 cleanups.forceLifetimeExtendedCleanups();
476
477 // Reload spilled values now that the builder is after the closed scope.
478 for (auto [addr, valPtr] : llvm::zip(t&: tempAllocas, u&: valuesToReload)) {
479 if (!addr.isValid())
480 continue;
481 *valPtr = cgf.builder.createLoad(loc: valPtr->getLoc(), addr);
482 }
483}
484
485//===----------------------------------------------------------------------===//
486// EHScopeStack
487//===----------------------------------------------------------------------===//
488
489void EHScopeStack::Cleanup::anchor() {}
490
491EHScopeStack::stable_iterator
492EHScopeStack::getInnermostActiveNormalCleanup() const {
493 stable_iterator si = getInnermostNormalCleanup();
494 stable_iterator se = stable_end();
495 while (si != se) {
496 EHCleanupScope &cleanup = llvm::cast<EHCleanupScope>(Val&: *find(savePoint: si));
497 if (cleanup.isActive())
498 return si;
499 si = cleanup.getEnclosingNormalCleanup();
500 }
501 return stable_end();
502}
503
504/// Push an entry of the given size onto this protected-scope stack.
505char *EHScopeStack::allocate(size_t size) {
506 size = llvm::alignTo(Value: size, Align: ScopeStackAlignment);
507 if (!startOfBuffer) {
508 unsigned capacity = llvm::PowerOf2Ceil(A: std::max<size_t>(a: size, b: 1024ul));
509 startOfBuffer = std::make_unique<char[]>(num: capacity);
510 startOfData = endOfBuffer = startOfBuffer.get() + capacity;
511 } else if (static_cast<size_t>(startOfData - startOfBuffer.get()) < size) {
512 unsigned currentCapacity = endOfBuffer - startOfBuffer.get();
513 unsigned usedCapacity =
514 currentCapacity - (startOfData - startOfBuffer.get());
515 unsigned requiredCapacity = usedCapacity + size;
516 // We know from the 'else if' condition that requiredCapacity is greater
517 // than currentCapacity.
518 unsigned newCapacity = llvm::PowerOf2Ceil(A: requiredCapacity);
519
520 std::unique_ptr<char[]> newStartOfBuffer =
521 std::make_unique<char[]>(num: newCapacity);
522 char *newEndOfBuffer = newStartOfBuffer.get() + newCapacity;
523 char *newStartOfData = newEndOfBuffer - usedCapacity;
524 memcpy(dest: newStartOfData, src: startOfData, n: usedCapacity);
525 startOfBuffer.swap(u&: newStartOfBuffer);
526 endOfBuffer = newEndOfBuffer;
527 startOfData = newStartOfData;
528 }
529
530 assert(startOfBuffer.get() + size <= startOfData);
531 startOfData -= size;
532 return startOfData;
533}
534
535void EHScopeStack::deallocate(size_t size) {
536 startOfData += llvm::alignTo(Value: size, Align: ScopeStackAlignment);
537}
538
539void *EHScopeStack::pushCleanup(CleanupKind kind, size_t size) {
540 char *buffer = allocate(size: EHCleanupScope::getSizeForCleanupSize(size));
541 bool isNormalCleanup = kind & NormalCleanup;
542 bool isEHCleanup = kind & EHCleanup;
543 bool isLifetimeMarker = kind & LifetimeMarker;
544 bool skipCleanupScope = false;
545
546 cir::CleanupKind cleanupKind = cir::CleanupKind::All;
547 if (isEHCleanup && cgf->getLangOpts().Exceptions) {
548 cleanupKind =
549 isNormalCleanup ? cir::CleanupKind::All : cir::CleanupKind::EH;
550 } else {
551 // Exceptions are disabled (or no EH flag was requested). Drop the EH
552 // flag so the scope entry stays consistent with the op's cleanup kind.
553 isEHCleanup = false;
554 if (isNormalCleanup)
555 cleanupKind = cir::CleanupKind::Normal;
556 else
557 skipCleanupScope = true;
558 }
559
560 // While emitting a variable with captured cleanups, suppress
561 // cir.cleanup.scope creation. The variable's cleanups are captured on the EH
562 // stack and later emitted into the cleanup region of the op that owns them.
563 if (capturingCleanups)
564 skipCleanupScope = true;
565
566 cir::CleanupScopeOp cleanupScope = nullptr;
567 if (!skipCleanupScope) {
568 CIRGenBuilderTy &builder = cgf->getBuilder();
569 mlir::Location loc = builder.getUnknownLoc();
570 cleanupScope = cir::CleanupScopeOp::create(
571 builder, location: loc, cleanupKind,
572 /*bodyBuilder=*/
573 [&](mlir::OpBuilder &b, mlir::Location loc) {
574 // Terminations will be handled in popCleanup
575 },
576 /*cleanupBuilder=*/
577 [&](mlir::OpBuilder &b, mlir::Location loc) {
578 // Terminations will be handled after emiting cleanup
579 });
580
581 builder.setInsertionPointToEnd(&cleanupScope.getBodyRegion().back());
582 }
583
584 // Per C++ [except.terminate], it is implementation-defined whether none,
585 // some, or all cleanups are called before std::terminate. Thus, when
586 // terminate is the current EH scope, we may skip adding any EH cleanup
587 // scopes.
588 if (innermostEHScope != stable_end() &&
589 find(savePoint: innermostEHScope)->getKind() == EHScope::Terminate)
590 isEHCleanup = false;
591
592 EHCleanupScope *scope = new (buffer)
593 EHCleanupScope(isNormalCleanup, isEHCleanup, size, cleanupScope,
594 innermostNormalCleanup, innermostEHScope);
595
596 if (isNormalCleanup)
597 innermostNormalCleanup = stable_begin();
598
599 if (isEHCleanup)
600 innermostEHScope = stable_begin();
601
602 if (isLifetimeMarker)
603 scope->setLifetimeMarker();
604
605 // With Windows -EHa, Invoke llvm.seh.scope.begin() for EHCleanup
606 if (cgf->getLangOpts().EHAsynch && isEHCleanup && !isLifetimeMarker &&
607 cgf->getTarget().getCXXABI().isMicrosoft())
608 cgf->cgm.errorNYI(feature: "push seh cleanup");
609
610 return scope->getCleanupBuffer();
611}
612
613void EHScopeStack::popCleanup() {
614 assert(!empty() && "popping exception stack when not empty");
615
616 assert(isa<EHCleanupScope>(*begin()));
617 EHCleanupScope &cleanup = cast<EHCleanupScope>(Val&: *begin());
618 innermostNormalCleanup = cleanup.getEnclosingNormalCleanup();
619 innermostEHScope = cleanup.getEnclosingEHScope();
620 deallocate(size: cleanup.getAllocatedSize());
621
622 cir::CleanupScopeOp cleanupScope = cleanup.getCleanupScopeOp();
623 if (cleanupScope) {
624 auto *block = &cleanupScope.getBodyRegion().back();
625 if (!block->mightHaveTerminator()) {
626 mlir::OpBuilder::InsertionGuard guard(cgf->getBuilder());
627 cgf->getBuilder().setInsertionPointToEnd(block);
628 cir::YieldOp::create(builder&: cgf->getBuilder(),
629 location: cgf->getBuilder().getUnknownLoc());
630 }
631 // If the insertion point was inside the cleanup scope we just closed, move
632 // it to immediate after the scope.
633 mlir::Block *insertBlock = cgf->getBuilder().getInsertionBlock();
634 if (insertBlock &&
635 cleanupScope.getBodyRegion().findAncestorBlockInRegion(block&: *insertBlock))
636 cgf->getBuilder().setInsertionPointAfter(cleanupScope);
637 }
638
639 // Destroy the cleanup.
640 cleanup.destroy();
641}
642
643bool EHScopeStack::requiresCatchOrCleanup() const {
644 for (stable_iterator si = getInnermostEHScope(); si != stable_end();) {
645 if (auto *cleanup = dyn_cast<EHCleanupScope>(Val: &*find(savePoint: si))) {
646 if (cleanup->isLifetimeMarker()) {
647 // Skip lifetime markers and continue from the enclosing EH scope
648 si = cleanup->getEnclosingEHScope();
649 continue;
650 }
651 }
652 return true;
653 }
654 return false;
655}
656
657/// The given cleanup block is being deactivated. Configure a cleanup variable
658/// if necessary.
659static void setupCleanupBlockDeactivation(CIRGenFunction &cgf,
660 EHScopeStack::stable_iterator c,
661 mlir::Operation *dominatingIP) {
662 EHCleanupScope &scope = cast<EHCleanupScope>(Val&: *cgf.ehStack.find(savePoint: c));
663
664 // An EH-only cleanup has both flags clear when pushCleanup dropped
665 // the EH flag. That happens when exceptions are disabled, and when
666 // the innermost EH scope is a terminate scope. The cleanup never
667 // runs, so there is no active flag to clear.
668 if (!scope.isNormalCleanup() && !scope.isEHCleanup())
669 return;
670
671 scope.setTestFlagInNormalCleanup(scope.isNormalCleanup());
672 scope.setTestFlagInEHCleanup(scope.isEHCleanup());
673
674 CIRGenBuilderTy &builder = cgf.getBuilder();
675
676 // If the cleanup block doesn't exist yet, create it and set its initial
677 // value to `true`. If we are inside a conditional branch, the value must be
678 // initialized before the conditional branch begins.
679 Address var = scope.getActiveFlag();
680 if (!var.isValid()) {
681 mlir::Location loc = builder.getUnknownLoc();
682
683 var = cgf.createTempAllocaWithoutCast(ty: builder.getBoolTy(), align: CharUnits::One(),
684 loc, name: "cleanup.isactive");
685 scope.setActiveFlag(var);
686
687 assert(dominatingIP && "no existing variable and no dominating IP!");
688
689 if (cgf.isInConditionalBranch()) {
690 mlir::Value val = builder.getBool(state: true, loc);
691 cgf.setBeforeOutermostConditional(value: val, addr: var);
692 } else {
693 mlir::OpBuilder::InsertionGuard guard(builder);
694 builder.setInsertionPoint(dominatingIP);
695 builder.createFlagStore(loc, val: true, dst: var.getPointer());
696 }
697 }
698
699 // The code above sets the `isActive` flag to `true` as its initial state
700 // at the point where the variable is created. The code below sets it to
701 // `false` at the point where the cleanup is deactivated.
702 mlir::Location loc = builder.getUnknownLoc();
703 builder.createFlagStore(loc, val: false, dst: var.getPointer());
704}
705
706static bool enclosesOpenConditionalCleanupScope(CIRGenFunction &cgf,
707 EHCleanupScope &scope) {
708 cir::CleanupScopeOp op = scope.getCleanupScopeOp();
709 return op &&
710 llvm::any_of(Range&: cgf.conditionalCleanupScopes, P: [&](const auto &condScope) {
711 return op->isProperAncestor(other: condScope.scope);
712 });
713}
714
715/// Deactivate a cleanup that was created in an active state.
716void CIRGenFunction::deactivateCleanupBlock(EHScopeStack::stable_iterator c,
717 mlir::Operation *dominatingIP) {
718 assert(c != ehStack.stable_end() && "deactivating bottom of stack?");
719 EHCleanupScope &scope = cast<EHCleanupScope>(Val&: *ehStack.find(savePoint: c));
720 assert(scope.isActive() && "double deactivation");
721
722 // If it's the top of the stack, just pop it, but do so only if it belongs
723 // to the current RunCleanupsScope. If a conditional opened a cleanup scope
724 // keep it open till the end of the full-expression.
725 if (c == ehStack.stable_begin() &&
726 currentCleanupStackDepth.strictlyEncloses(I: c) &&
727 !enclosesOpenConditionalCleanupScope(cgf&: *this, scope)) {
728 popCleanupBlock(/*forDeactivation=*/true);
729 return;
730 }
731
732 // Otherwise, follow the general case.
733 setupCleanupBlockDeactivation(cgf&: *this, c, dominatingIP);
734
735 scope.setActive(false);
736}
737
738static void emitCleanupBody(CIRGenFunction &cgf, EHScopeStack::Cleanup *cleanup,
739 EHScopeStack::Cleanup::Flags flags,
740 Address activeFlag, mlir::Location loc) {
741 CIRGenBuilderTy &builder = cgf.getBuilder();
742
743 // Ask the cleanup to emit itself.
744 assert(cgf.haveInsertPoint() && "expected insertion point");
745
746 if (activeFlag.isValid()) {
747 mlir::Value isActive = builder.createFlagLoad(loc, addr: activeFlag.getPointer());
748 cir::IfOp::create(builder, location: loc, cond: isActive,
749 /*withElseRegion=*/false,
750 /*thenBuilder=*/
751 [&](mlir::OpBuilder &, mlir::Location) {
752 cleanup->emit(cgf, flags);
753 assert(cgf.haveInsertPoint() &&
754 "cleanup ended with no insertion point?");
755 builder.createYield(loc);
756 });
757 } else {
758 cleanup->emit(cgf, flags);
759 assert(cgf.haveInsertPoint() && "cleanup ended with no insertion point?");
760 }
761}
762
763static void emitCleanup(CIRGenFunction &cgf, cir::CleanupScopeOp cleanupScope,
764 EHScopeStack::Cleanup *cleanup,
765 EHScopeStack::Cleanup::Flags flags,
766 Address activeFlag) {
767 CIRGenBuilderTy &builder = cgf.getBuilder();
768 mlir::Block &block = cleanupScope.getCleanupRegion().back();
769
770 mlir::OpBuilder::InsertionGuard guard(builder);
771 builder.setInsertionPointToStart(&block);
772
773 emitCleanupBody(cgf, cleanup, flags, activeFlag, loc: cleanupScope.getLoc());
774
775 mlir::Block &cleanupRegionLastBlock = cleanupScope.getCleanupRegion().back();
776 if (cleanupRegionLastBlock.empty() ||
777 !cleanupRegionLastBlock.back().hasTrait<mlir::OpTrait::IsTerminator>()) {
778 mlir::OpBuilder::InsertionGuard guardCase(builder);
779 builder.setInsertionPointToEnd(&cleanupRegionLastBlock);
780 builder.createYield(loc: cleanupScope.getLoc());
781 }
782}
783
784/// Check whether a cleanup scope body contains any non-yield exits that branch
785/// through the cleanup. These exits branch through the cleanup and require
786/// the normal cleanup to be executed even when the cleanup has been
787/// deactivated.
788static bool bodyHasBranchThroughExits(mlir::Region &bodyRegion) {
789 return bodyRegion
790 .walk(callback: [&](mlir::Operation *op) {
791 if (isa<cir::ReturnOp, cir::GotoOp>(Val: op))
792 return mlir::WalkResult::interrupt();
793 return mlir::WalkResult::advance();
794 })
795 .wasInterrupted();
796}
797
798/// Pop a cleanup block from the stack.
799///
800/// \param forDeactivation - When true, this indicates that the cleanup block
801/// is being popped because it was deactivated while at the top of the stack.
802void CIRGenFunction::popCleanupBlock(bool forDeactivation) {
803 assert(!ehStack.empty() && "cleanup stack is empty!");
804 assert(isa<EHCleanupScope>(*ehStack.begin()) && "top not a cleanup!");
805 EHCleanupScope &scope = cast<EHCleanupScope>(Val&: *ehStack.begin());
806
807 // If we pushed an EH-only cleanup but exceptions are disabled, it will leave
808 // an effectively empty cleanup on the EH stack. In that case, there is
809 // nothing to do here except pop the cleanup.
810 cir::CleanupScopeOp cleanupScope = scope.getCleanupScopeOp();
811 if (!cleanupScope) {
812 assert(!scope.isNormalCleanup() && !scope.isEHCleanup() &&
813 "missing cir.cleanup.scope for active cleanup");
814 ehStack.popCleanup();
815 return;
816 }
817
818 bool requiresNormalCleanup = scope.isNormalCleanup();
819 bool requiresEHCleanup = scope.isEHCleanup();
820
821 // When we're popping a cleanup to deactivate it, we need to know if anything
822 // in the cleanup scope body region branches through the cleanup handler
823 // before the entire cleanup scope body has executed. If the cleanup scope
824 // body falls through, we don't want to emit normal cleanup code. However,
825 // if the cleanup body region contains early exits (return or goto), we do
826 // need to execute the normal cleanup when the early exit is taken. To handle
827 // that case, we guard the cleanup with an "active" flag so that it executes
828 // conditionally and set the flag to false when the cleanup body falls
829 // through. Classic codegen tracks this state with "hasBranches" and
830 // "getFixupDepth" on the cleanup scope, but because CIR uses structured
831 // control flow, we need to check for early exits and insert the active
832 // flag handling here. Note that when a cleanup is deactivated while not at
833 // the top of the stack, the active flag gets created in
834 // setupCleanupBlockDeactivation.
835 if (forDeactivation && requiresNormalCleanup) {
836 if (bodyHasBranchThroughExits(bodyRegion&: cleanupScope.getBodyRegion())) {
837 // The active flag shouldn't exist if the scope was at the top of the
838 // stack when it was deactivated.
839 assert(!scope.getActiveFlag().isValid() && "active flag already set");
840
841 // Create the flag.
842 mlir::Location loc = builder.getUnknownLoc();
843 Address activeFlag = createTempAllocaWithoutCast(
844 ty: builder.getBoolTy(), align: CharUnits::One(), loc, name: "cleanup.isactive");
845
846 // Initialize the flag to true before the cleanup scope (the point where
847 // the cleanup becomes active).
848 {
849 mlir::OpBuilder::InsertionGuard guard(builder);
850 builder.setInsertionPoint(cleanupScope);
851 builder.createFlagStore(loc, val: true, dst: activeFlag.getPointer());
852 }
853
854 // Set the flag to false at the end of the cleanup scope body region.
855 assert(builder.getInsertionBlock() ==
856 &cleanupScope.getBodyRegion().back() &&
857 "expected insertion point in cleanup body");
858 builder.createFlagStore(loc, val: false, dst: activeFlag.getPointer());
859
860 scope.setActiveFlag(activeFlag);
861 scope.setTestFlagInNormalCleanup(true);
862 } else {
863 // If the cleanup was pushed on the stack as normal+eh, downgrade it to
864 // eh-only.
865 if (requiresEHCleanup)
866 cleanupScope.setCleanupKind(cir::CleanupKind::EH);
867 requiresNormalCleanup = false;
868 }
869 }
870
871 Address normalActiveFlag = scope.shouldTestFlagInNormalCleanup()
872 ? scope.getActiveFlag()
873 : Address::invalid();
874 Address ehActiveFlag = scope.shouldTestFlagInEHCleanup()
875 ? scope.getActiveFlag()
876 : Address::invalid();
877
878 // If we don't need the cleanup at all, we're done.
879 if (!requiresNormalCleanup && !requiresEHCleanup) {
880 // If we get here, the cleanup scope isn't needed. Rather than try to move
881 // the contents of its body region out of the cleanup and erase it, we just
882 // add a yield to the cleanup region to make it valid but no-op. It will be
883 // erased during canonicalization.
884 mlir::Block &cleanupBlock = cleanupScope.getCleanupRegion().back();
885 if (!cleanupBlock.mightHaveTerminator()) {
886 mlir::OpBuilder::InsertionGuard guard(builder);
887 builder.setInsertionPointToEnd(&cleanupBlock);
888 cir::YieldOp::create(builder, location: builder.getUnknownLoc());
889 }
890 ehStack.popCleanup();
891 return;
892 }
893
894 SavedCleanup cleanup(scope);
895
896 // Determine the active flag for the cleanup handler.
897 Address cleanupActiveFlag = normalActiveFlag.isValid() ? normalActiveFlag
898 : ehActiveFlag.isValid() ? ehActiveFlag
899 : Address::invalid();
900
901 // In CIR, the cleanup code is emitted into the cleanup region of the
902 // cir.cleanup.scope op. There is no CFG threading needed — the FlattenCFG
903 // pass handles lowering the structured cleanup scope.
904 ehStack.popCleanup();
905 scope.markEmitted();
906 emitCleanup(cgf&: *this, cleanupScope, cleanup: cleanup.getCleanup(), flags: cleanup.getFlags(),
907 activeFlag: cleanupActiveFlag);
908}
909
910void CIRGenFunction::emitCapturedCleanups(EHScopeStack::stable_iterator depth,
911 mlir::Location loc) {
912 // The captured cleanups were pushed while EHScopeStack was capturing
913 // cleanups, so they own no cir.cleanup.scope. Emit them directly into the
914 // owning op's cleanup region (the current insertion point), popping each off
915 // the EH stack.
916 while (ehStack.stable_begin() != depth) {
917 assert(isa<EHCleanupScope>(*ehStack.begin()) && "top not a cleanup!");
918 EHCleanupScope &scope = cast<EHCleanupScope>(Val&: *ehStack.begin());
919 assert(!scope.getCleanupScopeOp() &&
920 "captured cleanup should not own a cleanup scope");
921
922 // A captured cleanup may be guarded by an active flag. The destructor
923 // cleanups of a loop condition variable and of the temporaries that its
924 // initializer lifetime-extends have one that is false while its
925 // initializer runs, so a throwing initializer does not destroy the
926 // not-yet-constructed variables. The lifetime-end cleanup has no flag
927 // because its lifetime starts before initialization. For a loop, each
928 // emission serves both the normal per-iteration exit and the EH unwind
929 // path.
930 SavedCleanup cleanup(scope);
931 ehStack.popCleanup();
932 emitCleanupBody(cgf&: *this, cleanup: cleanup.getCleanup(), flags: cleanup.getFlags(),
933 activeFlag: cleanup.getActiveFlag(), loc);
934 }
935}
936
937void CIRGenFunction::emitCapturedCleanupsAsScopes(
938 EHScopeStack::stable_iterator depth, mlir::Location loc) {
939 // Copy the captured cleanups off the EH stack, the last captured one first.
940 // Unlike popCleanupBlock, this is not on a hot path, so the copies may live
941 // on the heap.
942 llvm::SmallVector<SavedCleanup> captured;
943 while (ehStack.stable_begin() != depth) {
944 assert(isa<EHCleanupScope>(*ehStack.begin()) && "top not a cleanup!");
945 EHCleanupScope &scope = cast<EHCleanupScope>(Val&: *ehStack.begin());
946 assert(!scope.getCleanupScopeOp() &&
947 "captured cleanup should not own a cleanup scope");
948 assert(!scope.hasActiveFlag() && "captured cleanup with an active flag");
949 captured.emplace_back(Args&: scope);
950 ehStack.popCleanup();
951 }
952
953 // Push them again in the order in which they were captured, each one
954 // opening a cir.cleanup.scope, and emit them.
955 for (const SavedCleanup &cleanup : llvm::reverse(C&: captured)) {
956 ehStack.pushCopyOfCleanup(kind: cleanup.getKind(), cleanup: cleanup.getData(),
957 size: cleanup.getSize());
958 if (auto scope = cast<EHCleanupScope>(Val&: *ehStack.begin()).getCleanupScopeOp())
959 scope->setLoc(loc);
960 }
961 popCleanupBlocks(oldCleanupStackDepth: depth);
962}
963
964/// Pops cleanup blocks until the given savepoint is reached.
965void CIRGenFunction::popCleanupBlocks(
966 EHScopeStack::stable_iterator oldCleanupStackDepth,
967 ArrayRef<mlir::Value *> valuesToReload) {
968 // If the current stack depth is the same as the cleanup stack depth,
969 // we won't be exiting any cleanup scopes, so we don't need to reload
970 // any values.
971 bool requiresCleanup = false;
972 for (auto it = ehStack.begin(), ie = ehStack.find(savePoint: oldCleanupStackDepth);
973 it != ie; ++it) {
974 if (isa<EHCleanupScope>(Val: &*it)) {
975 requiresCleanup = true;
976 break;
977 }
978 }
979
980 // If there are values that we need to keep live, spill them now before
981 // we pop the cleanup blocks. These are passed as pointers to mlir::Value
982 // because we're going to replace them with the reloaded value.
983 SmallVector<Address> tempAllocas;
984 if (requiresCleanup) {
985 for (mlir::Value *valPtr : valuesToReload) {
986 mlir::Value val = *valPtr;
987 if (!val)
988 continue;
989
990 // TODO(cir): Check for static allocas.
991
992 Address temp = createDefaultAlignTempAlloca(ty: val.getType(), loc: val.getLoc(),
993 name: "tmp.exprcleanup");
994 tempAllocas.push_back(Elt: temp);
995 builder.createStore(loc: val.getLoc(), val, dst: temp);
996 }
997 }
998
999 // Pop cleanup blocks until we reach the base stack depth for the
1000 // current scope.
1001 while (ehStack.stable_begin() != oldCleanupStackDepth)
1002 popCleanupBlock();
1003
1004 // Reload the values that we spilled, if necessary.
1005 if (requiresCleanup) {
1006 for (auto [addr, valPtr] : llvm::zip(t&: tempAllocas, u&: valuesToReload)) {
1007 mlir::Location loc = valPtr->getLoc();
1008 *valPtr = builder.createLoad(loc, addr);
1009 }
1010 }
1011}
1012
1013/// Pops cleanup blocks until the given savepoint is reached, then add the
1014/// cleanups from the given savepoint in the lifetime-extended cleanups stack.
1015void CIRGenFunction::popCleanupBlocks(
1016 EHScopeStack::stable_iterator oldCleanupStackDepth,
1017 size_t oldLifetimeExtendedSize, ArrayRef<mlir::Value *> valuesToReload) {
1018 popCleanupBlocks(oldCleanupStackDepth, valuesToReload);
1019
1020 // Promote deferred lifetime-extended cleanups onto the EH scope stack.
1021 for (const PendingCleanupEntry &cleanup : llvm::make_range(
1022 x: lifetimeExtendedCleanupStack.begin() + oldLifetimeExtendedSize,
1023 y: lifetimeExtendedCleanupStack.end()))
1024 pushPendingCleanupToEHStack(entry: cleanup);
1025 lifetimeExtendedCleanupStack.truncate(N: oldLifetimeExtendedSize);
1026}
1027