| 1 | //===--- CIRGenCleanup.cpp - Bookkeeping and code emission for cleanups ---===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains code dealing with the IR generation for cleanups |
| 10 | // and related information. |
| 11 | // |
| 12 | // A "cleanup" is a piece of code which needs to be executed whenever |
| 13 | // control transfers out of a particular scope. This can be |
| 14 | // conditionalized to occur only on exceptional control flow, only on |
| 15 | // normal control flow, or both. |
| 16 | // |
| 17 | //===----------------------------------------------------------------------===// |
| 18 | |
| 19 | #include "CIRGenCleanup.h" |
| 20 | #include "CIRGenFunction.h" |
| 21 | |
| 22 | #include "clang/AST/RecursiveASTVisitor.h" |
| 23 | #include "llvm/Support/MathExtras.h" |
| 24 | #include "llvm/Support/SaveAndRestore.h" |
| 25 | |
| 26 | using namespace clang; |
| 27 | using namespace clang::CIRGen; |
| 28 | |
| 29 | namespace { |
| 30 | /// Return true if the expression tree contains a construct that causes cleanups |
| 31 | /// to be deferred via pushFullExprCleanup. |
| 32 | class ConditionalEvaluationFinder |
| 33 | : public RecursiveASTVisitor<ConditionalEvaluationFinder> { |
| 34 | bool foundConditional = false; |
| 35 | bool inResumeExpr = false; |
| 36 | |
| 37 | bool traverseSuspendExpr(CoroutineSuspendExpr *e) { |
| 38 | return TraverseStmt(S: e->getCommonExpr()) && traverseResumeExpr(e); |
| 39 | } |
| 40 | |
| 41 | /// Traverse the resume expression of \p e, except for the binding of its |
| 42 | /// result, which emitSuspendExpression destroys after the cir.await. |
| 43 | bool traverseResumeExpr(CoroutineSuspendExpr *e) { |
| 44 | Expr *resume = e->getResumeExpr(); |
| 45 | if (auto *bind = dyn_cast_if_present<CXXBindTemporaryExpr>(Val: resume)) |
| 46 | resume = bind->getSubExpr(); |
| 47 | llvm::SaveAndRestore inResume(inResumeExpr, true); |
| 48 | return TraverseStmt(S: resume); |
| 49 | } |
| 50 | |
| 51 | public: |
| 52 | static bool needsCleanupScope(const Stmt *s) { |
| 53 | ConditionalEvaluationFinder finder; |
| 54 | finder.TraverseStmt(S: const_cast<Stmt *>(s)); |
| 55 | return finder.foundConditional; |
| 56 | } |
| 57 | |
| 58 | static bool resumeExprDefersCleanups(const CoroutineSuspendExpr &s) { |
| 59 | ConditionalEvaluationFinder finder; |
| 60 | finder.traverseResumeExpr(e: const_cast<CoroutineSuspendExpr *>(&s)); |
| 61 | return finder.foundConditional; |
| 62 | } |
| 63 | |
| 64 | // Default arguments and default member initializers can contain conditional |
| 65 | // temporaries whose cleanups belong to the enclosing full-expression. |
| 66 | bool shouldVisitImplicitCode() const { return true; } |
| 67 | |
| 68 | bool VisitAbstractConditionalOperator(AbstractConditionalOperator *) { |
| 69 | foundConditional = true; |
| 70 | return false; |
| 71 | } |
| 72 | |
| 73 | bool VisitBinaryOperator(BinaryOperator *e) { |
| 74 | if (e->isLogicalOp()) { |
| 75 | foundConditional = true; |
| 76 | return false; |
| 77 | } |
| 78 | return true; |
| 79 | } |
| 80 | |
| 81 | bool VisitCXXNewExpr(CXXNewExpr *e) { |
| 82 | // If the new expression has an initializer, the initializer may contain a |
| 83 | // a temporary expression that requires deferred cleanup. If we're emitting |
| 84 | // a null check, we need to make this cleanup conditional. |
| 85 | if (e->hasInitializer() && e->shouldNullCheckAllocation()) { |
| 86 | foundConditional = true; |
| 87 | return false; |
| 88 | } |
| 89 | return true; |
| 90 | } |
| 91 | |
| 92 | // Don't cross evaluation-context boundaries. Only the initializers of a |
| 93 | // lambda's captures are part of the enclosing full-expression. |
| 94 | bool TraverseLambdaExpr(LambdaExpr *e) { |
| 95 | for (Expr *init : e->capture_inits()) |
| 96 | if (init && !TraverseStmt(S: init)) |
| 97 | return false; |
| 98 | return true; |
| 99 | } |
| 100 | bool TraverseBlockExpr(BlockExpr *) { return true; } |
| 101 | bool TraverseStmtExpr(StmtExpr *) { return true; } |
| 102 | |
| 103 | bool TraverseGCCAsmStmt(GCCAsmStmt *s) { |
| 104 | for (Stmt *operand : s->children()) |
| 105 | if (!TraverseStmt(S: operand)) |
| 106 | return false; |
| 107 | return true; |
| 108 | } |
| 109 | |
| 110 | // Traverse the common and resume expressions, whose temporaries belong to |
| 111 | // the enclosing full-expression. Ready and suspend have their own cleanup |
| 112 | // boundaries. The resume result's cleanup is handled separately. |
| 113 | bool TraverseCoawaitExpr(CoawaitExpr *e) { return traverseSuspendExpr(e); } |
| 114 | bool TraverseCoyieldExpr(CoyieldExpr *e) { return traverseSuspendExpr(e); } |
| 115 | |
| 116 | bool VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *) { |
| 117 | if (!inResumeExpr) |
| 118 | return true; |
| 119 | foundConditional = true; |
| 120 | return false; |
| 121 | } |
| 122 | |
| 123 | bool VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *e) { |
| 124 | if (!inResumeExpr || !e->getType().isDestructedType()) |
| 125 | return true; |
| 126 | foundConditional = true; |
| 127 | return false; |
| 128 | } |
| 129 | }; |
| 130 | |
| 131 | /// Owns a copy of a cleanup and its metadata so they remain valid after the EH |
| 132 | /// stack entry is popped. |
| 133 | /// |
| 134 | /// Every popped cleanup is copied, so the copy must not allocate memory in the |
| 135 | /// common case. It goes into \c data, which holds cleanups of up to 8 pointers |
| 136 | /// on the stack. A SavedCleanup is normally a local variable, so its inline |
| 137 | /// storage, and with it the copy, lives on the stack. Only a larger cleanup |
| 138 | /// falls back to the heap. SavedCleanups are trivially copyable, meaning a |
| 139 | /// bitwise copy will be performed. |
| 140 | class SavedCleanup { |
| 141 | struct alignas(EHScopeStack::ScopeStackAlignment) StorageUnit { |
| 142 | char bytes[EHScopeStack::ScopeStackAlignment]; |
| 143 | }; |
| 144 | llvm::SmallVector<StorageUnit, 8 * sizeof(void *) / sizeof(StorageUnit)> data; |
| 145 | size_t size; |
| 146 | CleanupKind kind; |
| 147 | Address activeFlag; |
| 148 | |
| 149 | public: |
| 150 | explicit SavedCleanup(EHCleanupScope &scope) |
| 151 | : size(scope.getCleanupSize()), activeFlag(scope.getActiveFlag()) { |
| 152 | unsigned cleanupKind = 0; |
| 153 | if (scope.isNormalCleanup()) |
| 154 | cleanupKind |= NormalCleanup; |
| 155 | if (scope.isEHCleanup()) |
| 156 | cleanupKind |= EHCleanup; |
| 157 | if (scope.isLifetimeMarker()) |
| 158 | cleanupKind |= LifetimeMarker; |
| 159 | kind = static_cast<CleanupKind>(cleanupKind); |
| 160 | |
| 161 | data.resize_for_overwrite(N: llvm::divideCeil(Numerator: size, Denominator: sizeof(StorageUnit))); |
| 162 | memcpy(dest: data.data(), src: scope.getCleanupBuffer(), n: size); |
| 163 | } |
| 164 | |
| 165 | const void *getData() const { return data.data(); } |
| 166 | size_t getSize() const { return size; } |
| 167 | CleanupKind getKind() const { return kind; } |
| 168 | Address getActiveFlag() const { return activeFlag; } |
| 169 | |
| 170 | EHScopeStack::Cleanup *getCleanup() { |
| 171 | return reinterpret_cast<EHScopeStack::Cleanup *>(data.data()); |
| 172 | } |
| 173 | |
| 174 | EHScopeStack::Cleanup::Flags getFlags() const { |
| 175 | EHScopeStack::Cleanup::Flags flags; |
| 176 | if (kind & NormalCleanup) |
| 177 | flags.setIsNormalCleanupKind(); |
| 178 | if (kind & EHCleanup) |
| 179 | flags.setIsEHCleanupKind(); |
| 180 | return flags; |
| 181 | } |
| 182 | }; |
| 183 | } // namespace |
| 184 | |
| 185 | //===----------------------------------------------------------------------===// |
| 186 | // CIRGenFunction cleanup related |
| 187 | //===----------------------------------------------------------------------===// |
| 188 | |
| 189 | bool CIRGenFunction::resumeExprDefersCleanups(const CoroutineSuspendExpr &s) { |
| 190 | return ConditionalEvaluationFinder::resumeExprDefersCleanups(s); |
| 191 | } |
| 192 | |
| 193 | /// Emits all the code to cause the given temporary to be cleaned up. |
| 194 | void CIRGenFunction::emitCXXTemporary(const CXXTemporary *temporary, |
| 195 | QualType tempType, Address ptr) { |
| 196 | pushDestroy(kind: NormalAndEHCleanup, addr: ptr, type: tempType, destroyer: destroyCXXObject); |
| 197 | } |
| 198 | |
| 199 | Address CIRGenFunction::createCleanupActiveFlag() { |
| 200 | assert(isInConditionalBranch()); |
| 201 | mlir::Location loc = builder.getUnknownLoc(); |
| 202 | |
| 203 | // Place the alloca in the function entry block so it dominates everything, |
| 204 | // including both regions of any enclosing cir.cleanup.scope. We can't rely |
| 205 | // on the default curLexScope path because we may be inside a ternary branch |
| 206 | // whose LexicalScope would capture the alloca. |
| 207 | Address active = createTempAllocaWithoutCast( |
| 208 | ty: builder.getBoolTy(), align: CharUnits::One(), loc, name: "cleanup.cond" , |
| 209 | /*arraySize=*/nullptr, |
| 210 | ip: builder.getBestAllocaInsertPoint(block: getCurFunctionEntryBlock())); |
| 211 | |
| 212 | // Initialize to false before the outermost conditional. |
| 213 | { |
| 214 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 215 | builder.restoreInsertionPoint(ip: outermostConditional->getInsertPoint()); |
| 216 | cir::StoreOp store = |
| 217 | builder.createFlagStore(loc, val: false, dst: active.getPointer()); |
| 218 | outermostConditional->advanceInsertPoint(op: store); |
| 219 | } |
| 220 | |
| 221 | // Set to true at the current location (inside the conditional branch). |
| 222 | builder.createFlagStore(loc, val: true, dst: active.getPointer()); |
| 223 | |
| 224 | return active; |
| 225 | } |
| 226 | |
| 227 | void CIRGenFunction::initFullExprCleanup() { |
| 228 | initFullExprCleanupWithFlag(activeFlag: createCleanupActiveFlag()); |
| 229 | } |
| 230 | |
| 231 | void CIRGenFunction::initFullExprCleanupWithFlag(Address activeFlag) { |
| 232 | EHCleanupScope &cleanup = cast<EHCleanupScope>(Val&: *ehStack.begin()); |
| 233 | assert(!cleanup.hasActiveFlag() && "cleanup already has active flag?" ); |
| 234 | cleanup.setActiveFlag(activeFlag); |
| 235 | |
| 236 | cleanup.setTestFlagInNormalCleanup(cleanup.isNormalCleanup()); |
| 237 | cleanup.setTestFlagInEHCleanup(cleanup.isEHCleanup()); |
| 238 | } |
| 239 | |
| 240 | CIRGenFunction::FullExprCleanupScope::FullExprCleanupScope(CIRGenFunction &cgf, |
| 241 | const Stmt *s) |
| 242 | : cgf(cgf), cleanups(cgf), scope(nullptr), |
| 243 | oldFullExprCleanupScope(cgf.currentFullExprCleanupScope), |
| 244 | oldOutermostConditional(cgf.outermostConditional), |
| 245 | oldHiddenOutermostConditional(cgf.hiddenOutermostConditional), |
| 246 | deferredCleanupStackSize(cgf.deferredConditionalCleanupStack.size()), |
| 247 | conditionalScopeDepth(cgf.conditionalCleanupScopes.size()) { |
| 248 | |
| 249 | assert(s && "expected an AST root for a cleanup boundary" ); |
| 250 | if (ConditionalEvaluationFinder::needsCleanupScope(s)) { |
| 251 | mlir::Location loc = cgf.builder.getUnknownLoc(); |
| 252 | cir::CleanupKind cleanupKind = cgf.getLangOpts().Exceptions |
| 253 | ? cir::CleanupKind::All |
| 254 | : cir::CleanupKind::Normal; |
| 255 | scope = cir::CleanupScopeOp::create( |
| 256 | builder&: cgf.builder, location: loc, cleanupKind, |
| 257 | /*bodyBuilder=*/ |
| 258 | [&](mlir::OpBuilder &b, mlir::Location loc) {}, |
| 259 | /*cleanupBuilder=*/ |
| 260 | [&](mlir::OpBuilder &b, mlir::Location loc) {}); |
| 261 | cgf.builder.setInsertionPointToEnd(&scope.getBodyRegion().front()); |
| 262 | } |
| 263 | |
| 264 | // Conditional cleanup scopes are only opened while a full-expression scope |
| 265 | // is active to close them. |
| 266 | cgf.currentFullExprCleanupScope = scope; |
| 267 | |
| 268 | // This full-expression owns its cleanups, even inside a conditional branch. |
| 269 | // Hide the outer conditional to keep those cleanups here, but remember it |
| 270 | // for lifetime-extended temporaries whose cleanups belong to an outer scope. |
| 271 | if (cgf.outermostConditional) |
| 272 | cgf.hiddenOutermostConditional = cgf.outermostConditional; |
| 273 | cgf.outermostConditional = nullptr; |
| 274 | } |
| 275 | |
| 276 | /// If the alloca that backs \p addr is currently nested inside the body |
| 277 | /// region of \p scope, hoist it, and any cast chain leading to it, out of the |
| 278 | // scope so the alloca dominates the scope's sibling cleanup region. |
| 279 | static void hoistAllocaOutOfCleanupScope(CIRGenFunction &cgf, Address addr, |
| 280 | cir::CleanupScopeOp scope) { |
| 281 | cir::AllocaOp alloca = addr.getUnderlyingAllocaOp(); |
| 282 | if (!alloca) |
| 283 | return; |
| 284 | |
| 285 | // If the alloca is not contained within the cleanup scope we're currently |
| 286 | // proccessing we don't need to hoist it. |
| 287 | auto cur = alloca->getParentOfType<cir::CleanupScopeOp>(); |
| 288 | while (cur && cur != scope) |
| 289 | cur = cur->getParentOfType<cir::CleanupScopeOp>(); |
| 290 | if (cur != scope) |
| 291 | return; |
| 292 | |
| 293 | // Place the alloca at the canonical alloca insertion point of the block |
| 294 | // containing the cleanup scope op, so it groups with any preceding |
| 295 | // allocas / labels and dominates both the body and cleanup regions. |
| 296 | mlir::Block *parentBlock = scope->getBlock(); |
| 297 | mlir::OpBuilder::InsertPoint ip = |
| 298 | CIRGenBuilderTy::getBestAllocaInsertPoint(block: parentBlock); |
| 299 | alloca->moveBefore(block: parentBlock, iterator: ip.getPoint()); |
| 300 | |
| 301 | // Move any cast chain that consumes the alloca's result to immediately after |
| 302 | // the alloca, so the address used by the deferred cleanup also dominates the |
| 303 | // cleanup region. We walk down the chain starting from the alloca's user |
| 304 | // that the Address was built from. This is very conservative. In practice, |
| 305 | // we should only ever see alloca or address_space(alloca) operations here. |
| 306 | mlir::Value ptr = addr.getPointer(); |
| 307 | llvm::SmallVector<cir::CastOp> casts; |
| 308 | for (mlir::Operation *cur = ptr.getDefiningOp(); cur && cur != alloca;) { |
| 309 | auto cast = mlir::dyn_cast<cir::CastOp>(Val: cur); |
| 310 | if (!cast) |
| 311 | break; |
| 312 | casts.push_back(Elt: cast); |
| 313 | cur = cast.getSrc().getDefiningOp(); |
| 314 | } |
| 315 | // Move casts in source order (closest to the alloca first). |
| 316 | mlir::Operation *prev = alloca; |
| 317 | for (cir::CastOp cast : llvm::reverse(C&: casts)) { |
| 318 | cast->moveAfter(existingOp: prev); |
| 319 | prev = cast; |
| 320 | } |
| 321 | } |
| 322 | |
| 323 | /// Close a cleanup scope opened by a ConditionalEvaluation. Terminate its body |
| 324 | /// region and emit this scope's slice of deferredConditionalCleanupStack into |
| 325 | /// its cleanup region, in reverse of push order. An insertion point that was |
| 326 | /// inside the scope's body is moved to immediately after the scope op. |
| 327 | static void closeConditionalCleanupScope( |
| 328 | CIRGenFunction &cgf, |
| 329 | const CIRGenFunction::ConditionalCleanupScope &condScope) { |
| 330 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 331 | cir::CleanupScopeOp scope = condScope.scope; |
| 332 | size_t base = condScope.deferredCleanupStackSize; |
| 333 | auto &stack = cgf.deferredConditionalCleanupStack; |
| 334 | bool hasDeferredCleanups = stack.size() > base; |
| 335 | |
| 336 | // Make sure the body region has a terminator. |
| 337 | { |
| 338 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 339 | mlir::Block &lastBodyBlock = scope.getBodyRegion().back(); |
| 340 | builder.setInsertionPointToEnd(&lastBodyBlock); |
| 341 | if (lastBodyBlock.empty() || |
| 342 | !lastBodyBlock.back().hasTrait<mlir::OpTrait::IsTerminator>()) |
| 343 | builder.createYield(loc: scope.getLoc()); |
| 344 | } |
| 345 | |
| 346 | // Each deferred cleanup references its addr from the sibling cleanup region |
| 347 | // we are about to fill. If the alloca that backs that addr was created |
| 348 | // inside this scope's body region, hoist it out so it dominates the cleanup |
| 349 | // region. |
| 350 | if (hasDeferredCleanups) { |
| 351 | for (const CIRGenFunction::PendingCleanupEntry &entry : |
| 352 | llvm::make_range(x: stack.begin() + base, y: stack.end())) |
| 353 | hoistAllocaOutOfCleanupScope(cgf, addr: entry.addr, scope); |
| 354 | } |
| 355 | |
| 356 | { |
| 357 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 358 | mlir::Block &cleanupBlock = scope.getCleanupRegion().front(); |
| 359 | builder.setInsertionPointToEnd(&cleanupBlock); |
| 360 | |
| 361 | for (const CIRGenFunction::PendingCleanupEntry &entry : |
| 362 | llvm::reverse(C: llvm::make_range(x: stack.begin() + base, y: stack.end()))) { |
| 363 | if (entry.activeFlag.isValid()) { |
| 364 | // We may have hoisted this alloca out of the cleanup scope. If so, we |
| 365 | // will have also hoisted any casts between it and the address that we |
| 366 | // stored in the deferredConditionalCleanupStack. While I can't find a |
| 367 | // case where this actually happens, there is a theoretical |
| 368 | // possibility that we could have a second address that uses an alloca |
| 369 | // that has already been hoisted but a different cast chain. This |
| 370 | // assert guards against that possibility. |
| 371 | assert(entry.addr.getUnderlyingAllocaOp() && |
| 372 | (entry.addr.getUnderlyingAllocaOp()->getBlock() == |
| 373 | entry.addr.getPointer().getDefiningOp()->getBlock()) && |
| 374 | "alloca and cast are in different blocks" ); |
| 375 | mlir::Value flag = builder.createLoad(loc: scope.getLoc(), addr: entry.activeFlag); |
| 376 | cir::IfOp::create(builder, location: scope.getLoc(), cond: flag, |
| 377 | /*withElseRegion=*/false, |
| 378 | thenBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 379 | cgf.emitDestroy(addr: entry.addr, type: entry.type, |
| 380 | destroyer: entry.destroyer); |
| 381 | builder.createYield(loc); |
| 382 | }); |
| 383 | } else { |
| 384 | cgf.emitDestroy(addr: entry.addr, type: entry.type, destroyer: entry.destroyer); |
| 385 | } |
| 386 | } |
| 387 | builder.createYield(loc: scope.getLoc()); |
| 388 | } |
| 389 | |
| 390 | stack.truncate(N: base); |
| 391 | |
| 392 | // Move the insertion point out of the scope just closed, but only if it was |
| 393 | // inside it. Moving it unconditionally can strand it outside the region it |
| 394 | // belonged to, leaving an enclosing scope unterminated. popCleanup guards |
| 395 | // this the same way. |
| 396 | mlir::Block *insertBlock = builder.getInsertionBlock(); |
| 397 | if (insertBlock && |
| 398 | scope.getBodyRegion().findAncestorBlockInRegion(block&: *insertBlock)) |
| 399 | builder.setInsertionPointAfter(scope); |
| 400 | } |
| 401 | |
| 402 | void CIRGenFunction::FullExprCleanupScope::exit( |
| 403 | ArrayRef<mlir::Value *> valuesToReload) { |
| 404 | assert(!exited && "FullExprCleanupScope::exit called twice" ); |
| 405 | exited = true; |
| 406 | |
| 407 | cgf.currentFullExprCleanupScope = oldFullExprCleanupScope; |
| 408 | cgf.outermostConditional = oldOutermostConditional; |
| 409 | cgf.hiddenOutermostConditional = oldHiddenOutermostConditional; |
| 410 | |
| 411 | size_t oldSize = deferredCleanupStackSize; |
| 412 | |
| 413 | if (!scope) { |
| 414 | assert(cgf.conditionalCleanupScopes.size() == conditionalScopeDepth && |
| 415 | "conditional cleanup scope opened without one to close it" ); |
| 416 | cgf.deferredConditionalCleanupStack.truncate(N: oldSize); |
| 417 | cleanups.forceCleanup(valuesToReload); |
| 418 | return; |
| 419 | } |
| 420 | |
| 421 | // Spill any values that callers need after the scope is closed. |
| 422 | SmallVector<Address> tempAllocas; |
| 423 | for (mlir::Value *valPtr : valuesToReload) { |
| 424 | mlir::Value val = *valPtr; |
| 425 | if (!val) { |
| 426 | tempAllocas.push_back(Elt: Address::invalid()); |
| 427 | continue; |
| 428 | } |
| 429 | Address temp = cgf.createDefaultAlignTempAlloca(ty: val.getType(), loc: val.getLoc(), |
| 430 | name: "tmp.exprcleanup" ); |
| 431 | tempAllocas.push_back(Elt: temp); |
| 432 | cgf.builder.createStore(loc: val.getLoc(), val, dst: temp); |
| 433 | } |
| 434 | |
| 435 | // Close the cleanup scopes opened by conditionals in this full expression, |
| 436 | // innermost first. The EH cleanups popped below own the scopes enclosing |
| 437 | // these, so they are closed after. |
| 438 | while (cgf.conditionalCleanupScopes.size() > conditionalScopeDepth) |
| 439 | closeConditionalCleanupScope(cgf, |
| 440 | condScope: cgf.conditionalCleanupScopes.pop_back_val()); |
| 441 | |
| 442 | // Pop any EH cleanups that were pushed during the expression but leave |
| 443 | // any lifetime-extended cleanups so that they can be promoted to the EH |
| 444 | // stack after we've finished emitting any deferred cleanups. |
| 445 | cleanups.forceCleanupExceptLifetimeExtended(); |
| 446 | |
| 447 | // Make sure the cleanup scope body region has a terminator. |
| 448 | { |
| 449 | mlir::OpBuilder::InsertionGuard guard(cgf.builder); |
| 450 | mlir::Block &lastBodyBlock = scope.getBodyRegion().back(); |
| 451 | cgf.builder.setInsertionPointToEnd(&lastBodyBlock); |
| 452 | if (lastBodyBlock.empty() || |
| 453 | !lastBodyBlock.back().hasTrait<mlir::OpTrait::IsTerminator>()) |
| 454 | cgf.builder.createYield(loc: scope.getLoc()); |
| 455 | } |
| 456 | |
| 457 | // Deferred cleanups are emitted into the conditional scopes closed above, |
| 458 | // so this scope's cleanup region is empty and canonicalization will inline |
| 459 | // the scope away. |
| 460 | { |
| 461 | mlir::OpBuilder::InsertionGuard guard(cgf.builder); |
| 462 | mlir::Block &cleanupBlock = scope.getCleanupRegion().front(); |
| 463 | cgf.builder.setInsertionPointToEnd(&cleanupBlock); |
| 464 | cgf.builder.createYield(loc: scope.getLoc()); |
| 465 | } |
| 466 | |
| 467 | assert(cgf.deferredConditionalCleanupStack.size() == oldSize && |
| 468 | "deferred cleanups were not consumed by a conditional scope" ); |
| 469 | cgf.builder.setInsertionPointAfter(scope); |
| 470 | |
| 471 | // Promote any lifetime-extended cleanups onto the EH scope stack. The new |
| 472 | // cir.cleanup.scope ops created here will wrap any code in the enclosing |
| 473 | // scope, including reloads of any spilled values below, so the |
| 474 | // lifetime-extended destructors run at the correct point. |
| 475 | cleanups.forceLifetimeExtendedCleanups(); |
| 476 | |
| 477 | // Reload spilled values now that the builder is after the closed scope. |
| 478 | for (auto [addr, valPtr] : llvm::zip(t&: tempAllocas, u&: valuesToReload)) { |
| 479 | if (!addr.isValid()) |
| 480 | continue; |
| 481 | *valPtr = cgf.builder.createLoad(loc: valPtr->getLoc(), addr); |
| 482 | } |
| 483 | } |
| 484 | |
| 485 | //===----------------------------------------------------------------------===// |
| 486 | // EHScopeStack |
| 487 | //===----------------------------------------------------------------------===// |
| 488 | |
| 489 | void EHScopeStack::Cleanup::anchor() {} |
| 490 | |
| 491 | EHScopeStack::stable_iterator |
| 492 | EHScopeStack::getInnermostActiveNormalCleanup() const { |
| 493 | stable_iterator si = getInnermostNormalCleanup(); |
| 494 | stable_iterator se = stable_end(); |
| 495 | while (si != se) { |
| 496 | EHCleanupScope &cleanup = llvm::cast<EHCleanupScope>(Val&: *find(savePoint: si)); |
| 497 | if (cleanup.isActive()) |
| 498 | return si; |
| 499 | si = cleanup.getEnclosingNormalCleanup(); |
| 500 | } |
| 501 | return stable_end(); |
| 502 | } |
| 503 | |
| 504 | /// Push an entry of the given size onto this protected-scope stack. |
| 505 | char *EHScopeStack::allocate(size_t size) { |
| 506 | size = llvm::alignTo(Value: size, Align: ScopeStackAlignment); |
| 507 | if (!startOfBuffer) { |
| 508 | unsigned capacity = llvm::PowerOf2Ceil(A: std::max<size_t>(a: size, b: 1024ul)); |
| 509 | startOfBuffer = std::make_unique<char[]>(num: capacity); |
| 510 | startOfData = endOfBuffer = startOfBuffer.get() + capacity; |
| 511 | } else if (static_cast<size_t>(startOfData - startOfBuffer.get()) < size) { |
| 512 | unsigned currentCapacity = endOfBuffer - startOfBuffer.get(); |
| 513 | unsigned usedCapacity = |
| 514 | currentCapacity - (startOfData - startOfBuffer.get()); |
| 515 | unsigned requiredCapacity = usedCapacity + size; |
| 516 | // We know from the 'else if' condition that requiredCapacity is greater |
| 517 | // than currentCapacity. |
| 518 | unsigned newCapacity = llvm::PowerOf2Ceil(A: requiredCapacity); |
| 519 | |
| 520 | std::unique_ptr<char[]> newStartOfBuffer = |
| 521 | std::make_unique<char[]>(num: newCapacity); |
| 522 | char *newEndOfBuffer = newStartOfBuffer.get() + newCapacity; |
| 523 | char *newStartOfData = newEndOfBuffer - usedCapacity; |
| 524 | memcpy(dest: newStartOfData, src: startOfData, n: usedCapacity); |
| 525 | startOfBuffer.swap(u&: newStartOfBuffer); |
| 526 | endOfBuffer = newEndOfBuffer; |
| 527 | startOfData = newStartOfData; |
| 528 | } |
| 529 | |
| 530 | assert(startOfBuffer.get() + size <= startOfData); |
| 531 | startOfData -= size; |
| 532 | return startOfData; |
| 533 | } |
| 534 | |
| 535 | void EHScopeStack::deallocate(size_t size) { |
| 536 | startOfData += llvm::alignTo(Value: size, Align: ScopeStackAlignment); |
| 537 | } |
| 538 | |
| 539 | void *EHScopeStack::pushCleanup(CleanupKind kind, size_t size) { |
| 540 | char *buffer = allocate(size: EHCleanupScope::getSizeForCleanupSize(size)); |
| 541 | bool isNormalCleanup = kind & NormalCleanup; |
| 542 | bool isEHCleanup = kind & EHCleanup; |
| 543 | bool isLifetimeMarker = kind & LifetimeMarker; |
| 544 | bool skipCleanupScope = false; |
| 545 | |
| 546 | cir::CleanupKind cleanupKind = cir::CleanupKind::All; |
| 547 | if (isEHCleanup && cgf->getLangOpts().Exceptions) { |
| 548 | cleanupKind = |
| 549 | isNormalCleanup ? cir::CleanupKind::All : cir::CleanupKind::EH; |
| 550 | } else { |
| 551 | // Exceptions are disabled (or no EH flag was requested). Drop the EH |
| 552 | // flag so the scope entry stays consistent with the op's cleanup kind. |
| 553 | isEHCleanup = false; |
| 554 | if (isNormalCleanup) |
| 555 | cleanupKind = cir::CleanupKind::Normal; |
| 556 | else |
| 557 | skipCleanupScope = true; |
| 558 | } |
| 559 | |
| 560 | // While emitting a variable with captured cleanups, suppress |
| 561 | // cir.cleanup.scope creation. The variable's cleanups are captured on the EH |
| 562 | // stack and later emitted into the cleanup region of the op that owns them. |
| 563 | if (capturingCleanups) |
| 564 | skipCleanupScope = true; |
| 565 | |
| 566 | cir::CleanupScopeOp cleanupScope = nullptr; |
| 567 | if (!skipCleanupScope) { |
| 568 | CIRGenBuilderTy &builder = cgf->getBuilder(); |
| 569 | mlir::Location loc = builder.getUnknownLoc(); |
| 570 | cleanupScope = cir::CleanupScopeOp::create( |
| 571 | builder, location: loc, cleanupKind, |
| 572 | /*bodyBuilder=*/ |
| 573 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 574 | // Terminations will be handled in popCleanup |
| 575 | }, |
| 576 | /*cleanupBuilder=*/ |
| 577 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 578 | // Terminations will be handled after emiting cleanup |
| 579 | }); |
| 580 | |
| 581 | builder.setInsertionPointToEnd(&cleanupScope.getBodyRegion().back()); |
| 582 | } |
| 583 | |
| 584 | // Per C++ [except.terminate], it is implementation-defined whether none, |
| 585 | // some, or all cleanups are called before std::terminate. Thus, when |
| 586 | // terminate is the current EH scope, we may skip adding any EH cleanup |
| 587 | // scopes. |
| 588 | if (innermostEHScope != stable_end() && |
| 589 | find(savePoint: innermostEHScope)->getKind() == EHScope::Terminate) |
| 590 | isEHCleanup = false; |
| 591 | |
| 592 | EHCleanupScope *scope = new (buffer) |
| 593 | EHCleanupScope(isNormalCleanup, isEHCleanup, size, cleanupScope, |
| 594 | innermostNormalCleanup, innermostEHScope); |
| 595 | |
| 596 | if (isNormalCleanup) |
| 597 | innermostNormalCleanup = stable_begin(); |
| 598 | |
| 599 | if (isEHCleanup) |
| 600 | innermostEHScope = stable_begin(); |
| 601 | |
| 602 | if (isLifetimeMarker) |
| 603 | scope->setLifetimeMarker(); |
| 604 | |
| 605 | // With Windows -EHa, Invoke llvm.seh.scope.begin() for EHCleanup |
| 606 | if (cgf->getLangOpts().EHAsynch && isEHCleanup && !isLifetimeMarker && |
| 607 | cgf->getTarget().getCXXABI().isMicrosoft()) |
| 608 | cgf->cgm.errorNYI(feature: "push seh cleanup" ); |
| 609 | |
| 610 | return scope->getCleanupBuffer(); |
| 611 | } |
| 612 | |
| 613 | void EHScopeStack::popCleanup() { |
| 614 | assert(!empty() && "popping exception stack when not empty" ); |
| 615 | |
| 616 | assert(isa<EHCleanupScope>(*begin())); |
| 617 | EHCleanupScope &cleanup = cast<EHCleanupScope>(Val&: *begin()); |
| 618 | innermostNormalCleanup = cleanup.getEnclosingNormalCleanup(); |
| 619 | innermostEHScope = cleanup.getEnclosingEHScope(); |
| 620 | deallocate(size: cleanup.getAllocatedSize()); |
| 621 | |
| 622 | cir::CleanupScopeOp cleanupScope = cleanup.getCleanupScopeOp(); |
| 623 | if (cleanupScope) { |
| 624 | auto *block = &cleanupScope.getBodyRegion().back(); |
| 625 | if (!block->mightHaveTerminator()) { |
| 626 | mlir::OpBuilder::InsertionGuard guard(cgf->getBuilder()); |
| 627 | cgf->getBuilder().setInsertionPointToEnd(block); |
| 628 | cir::YieldOp::create(builder&: cgf->getBuilder(), |
| 629 | location: cgf->getBuilder().getUnknownLoc()); |
| 630 | } |
| 631 | // If the insertion point was inside the cleanup scope we just closed, move |
| 632 | // it to immediate after the scope. |
| 633 | mlir::Block *insertBlock = cgf->getBuilder().getInsertionBlock(); |
| 634 | if (insertBlock && |
| 635 | cleanupScope.getBodyRegion().findAncestorBlockInRegion(block&: *insertBlock)) |
| 636 | cgf->getBuilder().setInsertionPointAfter(cleanupScope); |
| 637 | } |
| 638 | |
| 639 | // Destroy the cleanup. |
| 640 | cleanup.destroy(); |
| 641 | } |
| 642 | |
| 643 | bool EHScopeStack::requiresCatchOrCleanup() const { |
| 644 | for (stable_iterator si = getInnermostEHScope(); si != stable_end();) { |
| 645 | if (auto *cleanup = dyn_cast<EHCleanupScope>(Val: &*find(savePoint: si))) { |
| 646 | if (cleanup->isLifetimeMarker()) { |
| 647 | // Skip lifetime markers and continue from the enclosing EH scope |
| 648 | si = cleanup->getEnclosingEHScope(); |
| 649 | continue; |
| 650 | } |
| 651 | } |
| 652 | return true; |
| 653 | } |
| 654 | return false; |
| 655 | } |
| 656 | |
| 657 | /// The given cleanup block is being deactivated. Configure a cleanup variable |
| 658 | /// if necessary. |
| 659 | static void setupCleanupBlockDeactivation(CIRGenFunction &cgf, |
| 660 | EHScopeStack::stable_iterator c, |
| 661 | mlir::Operation *dominatingIP) { |
| 662 | EHCleanupScope &scope = cast<EHCleanupScope>(Val&: *cgf.ehStack.find(savePoint: c)); |
| 663 | |
| 664 | // An EH-only cleanup has both flags clear when pushCleanup dropped |
| 665 | // the EH flag. That happens when exceptions are disabled, and when |
| 666 | // the innermost EH scope is a terminate scope. The cleanup never |
| 667 | // runs, so there is no active flag to clear. |
| 668 | if (!scope.isNormalCleanup() && !scope.isEHCleanup()) |
| 669 | return; |
| 670 | |
| 671 | scope.setTestFlagInNormalCleanup(scope.isNormalCleanup()); |
| 672 | scope.setTestFlagInEHCleanup(scope.isEHCleanup()); |
| 673 | |
| 674 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 675 | |
| 676 | // If the cleanup block doesn't exist yet, create it and set its initial |
| 677 | // value to `true`. If we are inside a conditional branch, the value must be |
| 678 | // initialized before the conditional branch begins. |
| 679 | Address var = scope.getActiveFlag(); |
| 680 | if (!var.isValid()) { |
| 681 | mlir::Location loc = builder.getUnknownLoc(); |
| 682 | |
| 683 | var = cgf.createTempAllocaWithoutCast(ty: builder.getBoolTy(), align: CharUnits::One(), |
| 684 | loc, name: "cleanup.isactive" ); |
| 685 | scope.setActiveFlag(var); |
| 686 | |
| 687 | assert(dominatingIP && "no existing variable and no dominating IP!" ); |
| 688 | |
| 689 | if (cgf.isInConditionalBranch()) { |
| 690 | mlir::Value val = builder.getBool(state: true, loc); |
| 691 | cgf.setBeforeOutermostConditional(value: val, addr: var); |
| 692 | } else { |
| 693 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 694 | builder.setInsertionPoint(dominatingIP); |
| 695 | builder.createFlagStore(loc, val: true, dst: var.getPointer()); |
| 696 | } |
| 697 | } |
| 698 | |
| 699 | // The code above sets the `isActive` flag to `true` as its initial state |
| 700 | // at the point where the variable is created. The code below sets it to |
| 701 | // `false` at the point where the cleanup is deactivated. |
| 702 | mlir::Location loc = builder.getUnknownLoc(); |
| 703 | builder.createFlagStore(loc, val: false, dst: var.getPointer()); |
| 704 | } |
| 705 | |
| 706 | static bool enclosesOpenConditionalCleanupScope(CIRGenFunction &cgf, |
| 707 | EHCleanupScope &scope) { |
| 708 | cir::CleanupScopeOp op = scope.getCleanupScopeOp(); |
| 709 | return op && |
| 710 | llvm::any_of(Range&: cgf.conditionalCleanupScopes, P: [&](const auto &condScope) { |
| 711 | return op->isProperAncestor(other: condScope.scope); |
| 712 | }); |
| 713 | } |
| 714 | |
| 715 | /// Deactivate a cleanup that was created in an active state. |
| 716 | void CIRGenFunction::deactivateCleanupBlock(EHScopeStack::stable_iterator c, |
| 717 | mlir::Operation *dominatingIP) { |
| 718 | assert(c != ehStack.stable_end() && "deactivating bottom of stack?" ); |
| 719 | EHCleanupScope &scope = cast<EHCleanupScope>(Val&: *ehStack.find(savePoint: c)); |
| 720 | assert(scope.isActive() && "double deactivation" ); |
| 721 | |
| 722 | // If it's the top of the stack, just pop it, but do so only if it belongs |
| 723 | // to the current RunCleanupsScope. If a conditional opened a cleanup scope |
| 724 | // keep it open till the end of the full-expression. |
| 725 | if (c == ehStack.stable_begin() && |
| 726 | currentCleanupStackDepth.strictlyEncloses(I: c) && |
| 727 | !enclosesOpenConditionalCleanupScope(cgf&: *this, scope)) { |
| 728 | popCleanupBlock(/*forDeactivation=*/true); |
| 729 | return; |
| 730 | } |
| 731 | |
| 732 | // Otherwise, follow the general case. |
| 733 | setupCleanupBlockDeactivation(cgf&: *this, c, dominatingIP); |
| 734 | |
| 735 | scope.setActive(false); |
| 736 | } |
| 737 | |
| 738 | static void emitCleanupBody(CIRGenFunction &cgf, EHScopeStack::Cleanup *cleanup, |
| 739 | EHScopeStack::Cleanup::Flags flags, |
| 740 | Address activeFlag, mlir::Location loc) { |
| 741 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 742 | |
| 743 | // Ask the cleanup to emit itself. |
| 744 | assert(cgf.haveInsertPoint() && "expected insertion point" ); |
| 745 | |
| 746 | if (activeFlag.isValid()) { |
| 747 | mlir::Value isActive = builder.createFlagLoad(loc, addr: activeFlag.getPointer()); |
| 748 | cir::IfOp::create(builder, location: loc, cond: isActive, |
| 749 | /*withElseRegion=*/false, |
| 750 | /*thenBuilder=*/ |
| 751 | [&](mlir::OpBuilder &, mlir::Location) { |
| 752 | cleanup->emit(cgf, flags); |
| 753 | assert(cgf.haveInsertPoint() && |
| 754 | "cleanup ended with no insertion point?" ); |
| 755 | builder.createYield(loc); |
| 756 | }); |
| 757 | } else { |
| 758 | cleanup->emit(cgf, flags); |
| 759 | assert(cgf.haveInsertPoint() && "cleanup ended with no insertion point?" ); |
| 760 | } |
| 761 | } |
| 762 | |
| 763 | static void emitCleanup(CIRGenFunction &cgf, cir::CleanupScopeOp cleanupScope, |
| 764 | EHScopeStack::Cleanup *cleanup, |
| 765 | EHScopeStack::Cleanup::Flags flags, |
| 766 | Address activeFlag) { |
| 767 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 768 | mlir::Block &block = cleanupScope.getCleanupRegion().back(); |
| 769 | |
| 770 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 771 | builder.setInsertionPointToStart(&block); |
| 772 | |
| 773 | emitCleanupBody(cgf, cleanup, flags, activeFlag, loc: cleanupScope.getLoc()); |
| 774 | |
| 775 | mlir::Block &cleanupRegionLastBlock = cleanupScope.getCleanupRegion().back(); |
| 776 | if (cleanupRegionLastBlock.empty() || |
| 777 | !cleanupRegionLastBlock.back().hasTrait<mlir::OpTrait::IsTerminator>()) { |
| 778 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 779 | builder.setInsertionPointToEnd(&cleanupRegionLastBlock); |
| 780 | builder.createYield(loc: cleanupScope.getLoc()); |
| 781 | } |
| 782 | } |
| 783 | |
| 784 | /// Check whether a cleanup scope body contains any non-yield exits that branch |
| 785 | /// through the cleanup. These exits branch through the cleanup and require |
| 786 | /// the normal cleanup to be executed even when the cleanup has been |
| 787 | /// deactivated. |
| 788 | static bool bodyHasBranchThroughExits(mlir::Region &bodyRegion) { |
| 789 | return bodyRegion |
| 790 | .walk(callback: [&](mlir::Operation *op) { |
| 791 | if (isa<cir::ReturnOp, cir::GotoOp>(Val: op)) |
| 792 | return mlir::WalkResult::interrupt(); |
| 793 | return mlir::WalkResult::advance(); |
| 794 | }) |
| 795 | .wasInterrupted(); |
| 796 | } |
| 797 | |
| 798 | /// Pop a cleanup block from the stack. |
| 799 | /// |
| 800 | /// \param forDeactivation - When true, this indicates that the cleanup block |
| 801 | /// is being popped because it was deactivated while at the top of the stack. |
| 802 | void CIRGenFunction::popCleanupBlock(bool forDeactivation) { |
| 803 | assert(!ehStack.empty() && "cleanup stack is empty!" ); |
| 804 | assert(isa<EHCleanupScope>(*ehStack.begin()) && "top not a cleanup!" ); |
| 805 | EHCleanupScope &scope = cast<EHCleanupScope>(Val&: *ehStack.begin()); |
| 806 | |
| 807 | // If we pushed an EH-only cleanup but exceptions are disabled, it will leave |
| 808 | // an effectively empty cleanup on the EH stack. In that case, there is |
| 809 | // nothing to do here except pop the cleanup. |
| 810 | cir::CleanupScopeOp cleanupScope = scope.getCleanupScopeOp(); |
| 811 | if (!cleanupScope) { |
| 812 | assert(!scope.isNormalCleanup() && !scope.isEHCleanup() && |
| 813 | "missing cir.cleanup.scope for active cleanup" ); |
| 814 | ehStack.popCleanup(); |
| 815 | return; |
| 816 | } |
| 817 | |
| 818 | bool requiresNormalCleanup = scope.isNormalCleanup(); |
| 819 | bool requiresEHCleanup = scope.isEHCleanup(); |
| 820 | |
| 821 | // When we're popping a cleanup to deactivate it, we need to know if anything |
| 822 | // in the cleanup scope body region branches through the cleanup handler |
| 823 | // before the entire cleanup scope body has executed. If the cleanup scope |
| 824 | // body falls through, we don't want to emit normal cleanup code. However, |
| 825 | // if the cleanup body region contains early exits (return or goto), we do |
| 826 | // need to execute the normal cleanup when the early exit is taken. To handle |
| 827 | // that case, we guard the cleanup with an "active" flag so that it executes |
| 828 | // conditionally and set the flag to false when the cleanup body falls |
| 829 | // through. Classic codegen tracks this state with "hasBranches" and |
| 830 | // "getFixupDepth" on the cleanup scope, but because CIR uses structured |
| 831 | // control flow, we need to check for early exits and insert the active |
| 832 | // flag handling here. Note that when a cleanup is deactivated while not at |
| 833 | // the top of the stack, the active flag gets created in |
| 834 | // setupCleanupBlockDeactivation. |
| 835 | if (forDeactivation && requiresNormalCleanup) { |
| 836 | if (bodyHasBranchThroughExits(bodyRegion&: cleanupScope.getBodyRegion())) { |
| 837 | // The active flag shouldn't exist if the scope was at the top of the |
| 838 | // stack when it was deactivated. |
| 839 | assert(!scope.getActiveFlag().isValid() && "active flag already set" ); |
| 840 | |
| 841 | // Create the flag. |
| 842 | mlir::Location loc = builder.getUnknownLoc(); |
| 843 | Address activeFlag = createTempAllocaWithoutCast( |
| 844 | ty: builder.getBoolTy(), align: CharUnits::One(), loc, name: "cleanup.isactive" ); |
| 845 | |
| 846 | // Initialize the flag to true before the cleanup scope (the point where |
| 847 | // the cleanup becomes active). |
| 848 | { |
| 849 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 850 | builder.setInsertionPoint(cleanupScope); |
| 851 | builder.createFlagStore(loc, val: true, dst: activeFlag.getPointer()); |
| 852 | } |
| 853 | |
| 854 | // Set the flag to false at the end of the cleanup scope body region. |
| 855 | assert(builder.getInsertionBlock() == |
| 856 | &cleanupScope.getBodyRegion().back() && |
| 857 | "expected insertion point in cleanup body" ); |
| 858 | builder.createFlagStore(loc, val: false, dst: activeFlag.getPointer()); |
| 859 | |
| 860 | scope.setActiveFlag(activeFlag); |
| 861 | scope.setTestFlagInNormalCleanup(true); |
| 862 | } else { |
| 863 | // If the cleanup was pushed on the stack as normal+eh, downgrade it to |
| 864 | // eh-only. |
| 865 | if (requiresEHCleanup) |
| 866 | cleanupScope.setCleanupKind(cir::CleanupKind::EH); |
| 867 | requiresNormalCleanup = false; |
| 868 | } |
| 869 | } |
| 870 | |
| 871 | Address normalActiveFlag = scope.shouldTestFlagInNormalCleanup() |
| 872 | ? scope.getActiveFlag() |
| 873 | : Address::invalid(); |
| 874 | Address ehActiveFlag = scope.shouldTestFlagInEHCleanup() |
| 875 | ? scope.getActiveFlag() |
| 876 | : Address::invalid(); |
| 877 | |
| 878 | // If we don't need the cleanup at all, we're done. |
| 879 | if (!requiresNormalCleanup && !requiresEHCleanup) { |
| 880 | // If we get here, the cleanup scope isn't needed. Rather than try to move |
| 881 | // the contents of its body region out of the cleanup and erase it, we just |
| 882 | // add a yield to the cleanup region to make it valid but no-op. It will be |
| 883 | // erased during canonicalization. |
| 884 | mlir::Block &cleanupBlock = cleanupScope.getCleanupRegion().back(); |
| 885 | if (!cleanupBlock.mightHaveTerminator()) { |
| 886 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 887 | builder.setInsertionPointToEnd(&cleanupBlock); |
| 888 | cir::YieldOp::create(builder, location: builder.getUnknownLoc()); |
| 889 | } |
| 890 | ehStack.popCleanup(); |
| 891 | return; |
| 892 | } |
| 893 | |
| 894 | SavedCleanup cleanup(scope); |
| 895 | |
| 896 | // Determine the active flag for the cleanup handler. |
| 897 | Address cleanupActiveFlag = normalActiveFlag.isValid() ? normalActiveFlag |
| 898 | : ehActiveFlag.isValid() ? ehActiveFlag |
| 899 | : Address::invalid(); |
| 900 | |
| 901 | // In CIR, the cleanup code is emitted into the cleanup region of the |
| 902 | // cir.cleanup.scope op. There is no CFG threading needed — the FlattenCFG |
| 903 | // pass handles lowering the structured cleanup scope. |
| 904 | ehStack.popCleanup(); |
| 905 | scope.markEmitted(); |
| 906 | emitCleanup(cgf&: *this, cleanupScope, cleanup: cleanup.getCleanup(), flags: cleanup.getFlags(), |
| 907 | activeFlag: cleanupActiveFlag); |
| 908 | } |
| 909 | |
| 910 | void CIRGenFunction::emitCapturedCleanups(EHScopeStack::stable_iterator depth, |
| 911 | mlir::Location loc) { |
| 912 | // The captured cleanups were pushed while EHScopeStack was capturing |
| 913 | // cleanups, so they own no cir.cleanup.scope. Emit them directly into the |
| 914 | // owning op's cleanup region (the current insertion point), popping each off |
| 915 | // the EH stack. |
| 916 | while (ehStack.stable_begin() != depth) { |
| 917 | assert(isa<EHCleanupScope>(*ehStack.begin()) && "top not a cleanup!" ); |
| 918 | EHCleanupScope &scope = cast<EHCleanupScope>(Val&: *ehStack.begin()); |
| 919 | assert(!scope.getCleanupScopeOp() && |
| 920 | "captured cleanup should not own a cleanup scope" ); |
| 921 | |
| 922 | // A captured cleanup may be guarded by an active flag. The destructor |
| 923 | // cleanups of a loop condition variable and of the temporaries that its |
| 924 | // initializer lifetime-extends have one that is false while its |
| 925 | // initializer runs, so a throwing initializer does not destroy the |
| 926 | // not-yet-constructed variables. The lifetime-end cleanup has no flag |
| 927 | // because its lifetime starts before initialization. For a loop, each |
| 928 | // emission serves both the normal per-iteration exit and the EH unwind |
| 929 | // path. |
| 930 | SavedCleanup cleanup(scope); |
| 931 | ehStack.popCleanup(); |
| 932 | emitCleanupBody(cgf&: *this, cleanup: cleanup.getCleanup(), flags: cleanup.getFlags(), |
| 933 | activeFlag: cleanup.getActiveFlag(), loc); |
| 934 | } |
| 935 | } |
| 936 | |
| 937 | void CIRGenFunction::emitCapturedCleanupsAsScopes( |
| 938 | EHScopeStack::stable_iterator depth, mlir::Location loc) { |
| 939 | // Copy the captured cleanups off the EH stack, the last captured one first. |
| 940 | // Unlike popCleanupBlock, this is not on a hot path, so the copies may live |
| 941 | // on the heap. |
| 942 | llvm::SmallVector<SavedCleanup> captured; |
| 943 | while (ehStack.stable_begin() != depth) { |
| 944 | assert(isa<EHCleanupScope>(*ehStack.begin()) && "top not a cleanup!" ); |
| 945 | EHCleanupScope &scope = cast<EHCleanupScope>(Val&: *ehStack.begin()); |
| 946 | assert(!scope.getCleanupScopeOp() && |
| 947 | "captured cleanup should not own a cleanup scope" ); |
| 948 | assert(!scope.hasActiveFlag() && "captured cleanup with an active flag" ); |
| 949 | captured.emplace_back(Args&: scope); |
| 950 | ehStack.popCleanup(); |
| 951 | } |
| 952 | |
| 953 | // Push them again in the order in which they were captured, each one |
| 954 | // opening a cir.cleanup.scope, and emit them. |
| 955 | for (const SavedCleanup &cleanup : llvm::reverse(C&: captured)) { |
| 956 | ehStack.pushCopyOfCleanup(kind: cleanup.getKind(), cleanup: cleanup.getData(), |
| 957 | size: cleanup.getSize()); |
| 958 | if (auto scope = cast<EHCleanupScope>(Val&: *ehStack.begin()).getCleanupScopeOp()) |
| 959 | scope->setLoc(loc); |
| 960 | } |
| 961 | popCleanupBlocks(oldCleanupStackDepth: depth); |
| 962 | } |
| 963 | |
| 964 | /// Pops cleanup blocks until the given savepoint is reached. |
| 965 | void CIRGenFunction::popCleanupBlocks( |
| 966 | EHScopeStack::stable_iterator oldCleanupStackDepth, |
| 967 | ArrayRef<mlir::Value *> valuesToReload) { |
| 968 | // If the current stack depth is the same as the cleanup stack depth, |
| 969 | // we won't be exiting any cleanup scopes, so we don't need to reload |
| 970 | // any values. |
| 971 | bool requiresCleanup = false; |
| 972 | for (auto it = ehStack.begin(), ie = ehStack.find(savePoint: oldCleanupStackDepth); |
| 973 | it != ie; ++it) { |
| 974 | if (isa<EHCleanupScope>(Val: &*it)) { |
| 975 | requiresCleanup = true; |
| 976 | break; |
| 977 | } |
| 978 | } |
| 979 | |
| 980 | // If there are values that we need to keep live, spill them now before |
| 981 | // we pop the cleanup blocks. These are passed as pointers to mlir::Value |
| 982 | // because we're going to replace them with the reloaded value. |
| 983 | SmallVector<Address> tempAllocas; |
| 984 | if (requiresCleanup) { |
| 985 | for (mlir::Value *valPtr : valuesToReload) { |
| 986 | mlir::Value val = *valPtr; |
| 987 | if (!val) |
| 988 | continue; |
| 989 | |
| 990 | // TODO(cir): Check for static allocas. |
| 991 | |
| 992 | Address temp = createDefaultAlignTempAlloca(ty: val.getType(), loc: val.getLoc(), |
| 993 | name: "tmp.exprcleanup" ); |
| 994 | tempAllocas.push_back(Elt: temp); |
| 995 | builder.createStore(loc: val.getLoc(), val, dst: temp); |
| 996 | } |
| 997 | } |
| 998 | |
| 999 | // Pop cleanup blocks until we reach the base stack depth for the |
| 1000 | // current scope. |
| 1001 | while (ehStack.stable_begin() != oldCleanupStackDepth) |
| 1002 | popCleanupBlock(); |
| 1003 | |
| 1004 | // Reload the values that we spilled, if necessary. |
| 1005 | if (requiresCleanup) { |
| 1006 | for (auto [addr, valPtr] : llvm::zip(t&: tempAllocas, u&: valuesToReload)) { |
| 1007 | mlir::Location loc = valPtr->getLoc(); |
| 1008 | *valPtr = builder.createLoad(loc, addr); |
| 1009 | } |
| 1010 | } |
| 1011 | } |
| 1012 | |
| 1013 | /// Pops cleanup blocks until the given savepoint is reached, then add the |
| 1014 | /// cleanups from the given savepoint in the lifetime-extended cleanups stack. |
| 1015 | void CIRGenFunction::popCleanupBlocks( |
| 1016 | EHScopeStack::stable_iterator oldCleanupStackDepth, |
| 1017 | size_t oldLifetimeExtendedSize, ArrayRef<mlir::Value *> valuesToReload) { |
| 1018 | popCleanupBlocks(oldCleanupStackDepth, valuesToReload); |
| 1019 | |
| 1020 | // Promote deferred lifetime-extended cleanups onto the EH scope stack. |
| 1021 | for (const PendingCleanupEntry &cleanup : llvm::make_range( |
| 1022 | x: lifetimeExtendedCleanupStack.begin() + oldLifetimeExtendedSize, |
| 1023 | y: lifetimeExtendedCleanupStack.end())) |
| 1024 | pushPendingCleanupToEHStack(entry: cleanup); |
| 1025 | lifetimeExtendedCleanupStack.truncate(N: oldLifetimeExtendedSize); |
| 1026 | } |
| 1027 | |