1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements pass that inlines CIR operations regions into the parent
10// function region.
11//
12//===----------------------------------------------------------------------===//
13
14#include "PassDetail.h"
15#include "mlir/Dialect/Func/IR/FuncOps.h"
16#include "mlir/IR/Block.h"
17#include "mlir/IR/Builders.h"
18#include "mlir/IR/PatternMatch.h"
19#include "mlir/Interfaces/SideEffectInterfaces.h"
20#include "mlir/Rewrite/PatternApplicator.h"
21#include "mlir/Support/LogicalResult.h"
22#include "mlir/Transforms/DialectConversion.h"
23#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
24#include "clang/CIR/Dialect/IR/CIRDialect.h"
25#include "clang/CIR/Dialect/Passes.h"
26#include "clang/CIR/Dialect/Transforms/CIRTransformUtils.h"
27#include "clang/CIR/MissingFeatures.h"
28#include "llvm/ADT/TypeSwitch.h"
29
30using namespace mlir;
31using namespace cir;
32
33namespace mlir {
34#define GEN_PASS_DEF_CIRFLATTENCFG
35#include "clang/CIR/Dialect/Passes.h.inc"
36} // namespace mlir
37
38namespace {
39
40/// Lowers operations with the terminator trait that have a single successor.
41void lowerTerminator(mlir::Operation *op, mlir::Block *dest,
42 mlir::PatternRewriter &rewriter,
43 mlir::DistinctAttr loopId = {},
44 cir::LoopHintAttr loopHints = {}) {
45 assert(op->hasTrait<mlir::OpTrait::IsTerminator>() && "not a terminator");
46 mlir::OpBuilder::InsertionGuard guard(rewriter);
47 rewriter.setInsertionPoint(op);
48 rewriter.replaceOpWithNewOp<cir::BrOp>(op, args: mlir::ValueRange{}, args&: loopId,
49 args&: loopHints, args&: dest);
50}
51
52/// Walks a region while skipping operations of type `Ops`. This ensures the
53/// callback is not applied to said operations and its children.
54template <typename... Ops>
55void walkRegionSkipping(
56 mlir::Region &region,
57 mlir::function_ref<mlir::WalkResult(mlir::Operation *)> callback) {
58 region.walk<mlir::WalkOrder::PreOrder>([&](mlir::Operation *op) {
59 if (isa<Ops...>(op))
60 return mlir::WalkResult::skip();
61 return callback(op);
62 });
63}
64
65/// Check whether a region contains any nested op with regions (i.e. structured
66/// CIR ops that must be flattened before their parent). The greedy pattern
67/// rewriter doesn't guarantee inside-out processing order — when a pattern
68/// fires and modifies IR, newly created ops go onto the worklist and can be
69/// visited in any order. So each flattening pattern must explicitly defer
70/// until its nested structured ops are flat.
71///
72/// CaseOps are excluded because they are structural children of SwitchOp and
73/// are handled by the SwitchOp flattening pattern. Ops from other dialects
74/// (e.g. OpenMP and OpenACC constructs) keep their regions after this pass, so
75/// they never block their parent; CIR ops nested inside them are still found.
76static bool hasNestedOpsToFlatten(mlir::Region &region) {
77 return region
78 .walk(callback: [](mlir::Operation *op) {
79 if (op->getNumRegions() > 0 && !isa<cir::CaseOp>(Val: op) &&
80 isa<cir::CIRDialect>(Val: op->getDialect()))
81 return mlir::WalkResult::interrupt();
82 return mlir::WalkResult::advance();
83 })
84 .wasInterrupted();
85}
86
87/// True if `op` is a non-returning terminator — currently `cir.unreachable`
88/// or `cir.trap`. Such terminators don't fall through and don't yield a
89/// value, so when flattening a region they can be left in place rather than
90/// being replaced with a branch to the continuation block. Add new ops here
91/// (e.g. a hypothetical `cir.abort`) so every flattening pattern picks them
92/// up at once.
93static bool isNonReturningTerminator(mlir::Operation *op) {
94 return mlir::isa_and_nonnull<cir::UnreachableOp, cir::TrapOp>(Val: op);
95}
96
97/// Rewrite the terminator of `region`'s exit block so that, after
98/// flattening, control falls through to `continueBlock`. The exit
99/// terminator is expected to be either:
100/// - `cir.yield`: replaced with `cir.br` to `continueBlock` (yielded
101/// args become the destination block's arguments).
102/// - non-returning (`cir.unreachable`, `cir.trap`): left in place — no
103/// branch is needed.
104///
105/// On success returns `success()`. If the terminator is anything else, an
106/// error is emitted and `failure()` is returned. NOTE: callers in this
107/// file have typically already mutated IR (splitBlock / createBlock) by
108/// the time this is invoked, so the MLIR pattern rewriter contract
109/// requires them to still return `success()` from the surrounding
110/// pattern; the `failure()` here just signals "stop trying to wire up
111/// this region".
112static mlir::LogicalResult
113rewriteRegionExitToContinue(mlir::PatternRewriter &rewriter,
114 mlir::Region &region, mlir::Block *continueBlock,
115 llvm::StringRef regionDescription) {
116 mlir::Operation *terminator = region.back().getTerminator();
117 rewriter.setInsertionPointToEnd(&region.back());
118 if (auto yieldOp = mlir::dyn_cast<cir::YieldOp>(Val: terminator)) {
119 rewriter.replaceOpWithNewOp<cir::BrOp>(
120 op: yieldOp, args: yieldOp.getArgs(), /*loop_id=*/args: nullptr, /*loop_hints=*/args: nullptr,
121 args&: continueBlock);
122 return mlir::success();
123 }
124 if (isNonReturningTerminator(op: terminator))
125 return mlir::success();
126 terminator->emitError(message: "unexpected terminator in ")
127 << regionDescription
128 << " region, expected yield, unreachable, or trap, got: "
129 << terminator->getName();
130 return mlir::failure();
131}
132
133struct CIRFlattenCFGPass : public impl::CIRFlattenCFGBase<CIRFlattenCFGPass> {
134
135 CIRFlattenCFGPass() = default;
136 void runOnOperation() override;
137};
138
139struct CIRIfFlattening : public mlir::OpRewritePattern<cir::IfOp> {
140 using OpRewritePattern<IfOp>::OpRewritePattern;
141
142 mlir::LogicalResult
143 matchAndRewrite(cir::IfOp ifOp,
144 mlir::PatternRewriter &rewriter) const override {
145 mlir::OpBuilder::InsertionGuard guard(rewriter);
146 mlir::Location loc = ifOp.getLoc();
147 bool emptyElse = ifOp.getElseRegion().empty();
148 mlir::Block *currentBlock = rewriter.getInsertionBlock();
149 mlir::Block *remainingOpsBlock =
150 rewriter.splitBlock(block: currentBlock, before: rewriter.getInsertionPoint());
151 mlir::Block *continueBlock;
152 if (ifOp->getResults().empty())
153 continueBlock = remainingOpsBlock;
154 else
155 llvm_unreachable("NYI");
156
157 // Inline the region
158 mlir::Block *thenBeforeBody = &ifOp.getThenRegion().front();
159 mlir::Block *thenAfterBody = &ifOp.getThenRegion().back();
160 rewriter.inlineRegionBefore(region&: ifOp.getThenRegion(), before: continueBlock);
161
162 rewriter.setInsertionPointToEnd(thenAfterBody);
163 if (auto thenYieldOp =
164 dyn_cast<cir::YieldOp>(Val: thenAfterBody->getTerminator())) {
165 rewriter.replaceOpWithNewOp<cir::BrOp>(
166 op: thenYieldOp, args: thenYieldOp.getArgs(),
167 /*loop_id=*/args: nullptr, /*loop_hints=*/args: nullptr, args&: continueBlock);
168 }
169
170 rewriter.setInsertionPointToEnd(continueBlock);
171
172 // Has else region: inline it.
173 mlir::Block *elseBeforeBody = nullptr;
174 mlir::Block *elseAfterBody = nullptr;
175 if (!emptyElse) {
176 elseBeforeBody = &ifOp.getElseRegion().front();
177 elseAfterBody = &ifOp.getElseRegion().back();
178 rewriter.inlineRegionBefore(region&: ifOp.getElseRegion(), before: continueBlock);
179 } else {
180 elseBeforeBody = elseAfterBody = continueBlock;
181 }
182
183 rewriter.setInsertionPointToEnd(currentBlock);
184 cir::BrCondOp::create(builder&: rewriter, location: loc, cond: ifOp.getCondition(), destTrue: thenBeforeBody,
185 destFalse: elseBeforeBody);
186
187 if (!emptyElse) {
188 rewriter.setInsertionPointToEnd(elseAfterBody);
189 if (auto elseYieldOP =
190 dyn_cast<cir::YieldOp>(Val: elseAfterBody->getTerminator())) {
191 rewriter.replaceOpWithNewOp<cir::BrOp>(
192 op: elseYieldOP, args: elseYieldOP.getArgs(), /*loop_id=*/args: nullptr,
193 /*loop_hints=*/args: nullptr, args&: continueBlock);
194 }
195 }
196
197 rewriter.replaceOp(op: ifOp, newValues: continueBlock->getArguments());
198 return mlir::success();
199 }
200};
201
202class CIRScopeOpFlattening : public mlir::OpRewritePattern<cir::ScopeOp> {
203public:
204 using OpRewritePattern<cir::ScopeOp>::OpRewritePattern;
205
206 mlir::LogicalResult
207 matchAndRewrite(cir::ScopeOp scopeOp,
208 mlir::PatternRewriter &rewriter) const override {
209 mlir::OpBuilder::InsertionGuard guard(rewriter);
210 mlir::Location loc = scopeOp.getLoc();
211
212 // Empty scope: just remove it.
213 // TODO: Remove this logic once CIR uses MLIR infrastructure to remove
214 // trivially dead operations. MLIR canonicalizer is too aggressive and we
215 // need to either (a) make sure all our ops model all side-effects and/or
216 // (b) have more options in the canonicalizer in MLIR to temper
217 // aggressiveness level.
218 if (scopeOp.isEmpty()) {
219 rewriter.eraseOp(op: scopeOp);
220 return mlir::success();
221 }
222
223 // Split the current block before the ScopeOp to create the inlining
224 // point.
225 mlir::Block *currentBlock = rewriter.getInsertionBlock();
226 mlir::Block *continueBlock =
227 rewriter.splitBlock(block: currentBlock, before: rewriter.getInsertionPoint());
228 if (scopeOp.getNumResults() > 0)
229 continueBlock->addArguments(types: scopeOp.getResultTypes(), locs: loc);
230
231 // Inline body region.
232 mlir::Block *beforeBody = &scopeOp.getScopeRegion().front();
233 mlir::Block *afterBody = &scopeOp.getScopeRegion().back();
234 rewriter.inlineRegionBefore(region&: scopeOp.getScopeRegion(), before: continueBlock);
235
236 // Save stack and then branch into the body of the region.
237 rewriter.setInsertionPointToEnd(currentBlock);
238 assert(!cir::MissingFeatures::stackSaveOp());
239 cir::BrOp::create(builder&: rewriter, location: loc, destOperands: mlir::ValueRange(), /*loop_id=*/nullptr,
240 /*loop_hints=*/nullptr, dest: beforeBody);
241
242 // Replace the scopeop return with a branch that jumps out of the body.
243 // Stack restore before leaving the body region.
244 rewriter.setInsertionPointToEnd(afterBody);
245 if (auto yieldOp = dyn_cast<cir::YieldOp>(Val: afterBody->getTerminator())) {
246 rewriter.replaceOpWithNewOp<cir::BrOp>(
247 op: yieldOp, args: yieldOp.getArgs(),
248 /*loop_id=*/args: nullptr, /*loop_hints=*/args: nullptr, args&: continueBlock);
249 }
250
251 // Replace the op with values return from the body region.
252 rewriter.replaceOp(op: scopeOp, newValues: continueBlock->getArguments());
253
254 return mlir::success();
255 }
256};
257
258class CIRSwitchOpFlattening : public mlir::OpRewritePattern<cir::SwitchOp> {
259public:
260 using OpRewritePattern<cir::SwitchOp>::OpRewritePattern;
261
262 inline void rewriteYieldOp(mlir::PatternRewriter &rewriter,
263 cir::YieldOp yieldOp,
264 mlir::Block *destination) const {
265 rewriter.setInsertionPoint(yieldOp);
266 rewriter.replaceOpWithNewOp<cir::BrOp>(op: yieldOp, args: yieldOp.getOperands(),
267 /*loop_id=*/args: nullptr,
268 /*loop_hints=*/args: nullptr, args&: destination);
269 }
270
271 // Return the new defaultDestination block.
272 Block *condBrToRangeDestination(cir::SwitchOp op,
273 mlir::PatternRewriter &rewriter,
274 mlir::Block *rangeDestination,
275 mlir::Block *defaultDestination,
276 const APInt &lowerBound,
277 const APInt &upperBound) const {
278 auto condType = mlir::cast<cir::IntType>(Val: op.getCondition().getType());
279 bool isSigned = condType.isSigned();
280 assert(
281 (isSigned ? lowerBound.sle(upperBound) : lowerBound.ule(upperBound)) &&
282 "Invalid range");
283 mlir::Block *resBlock = rewriter.createBlock(insertBefore: defaultDestination);
284
285 // Build the range check at the switch operand's width. The classic
286 // `sub x, lo; ule (x - lo), (hi - lo)` idiom is a cyclic-distance test that
287 // is correct for both signed and unsigned switches, so the comparison is
288 // always unsigned.
289 cir::IntType uIntType =
290 cir::IntType::get(context: op.getContext(), width: condType.getWidth(),
291 /*isSigned=*/is_signed: false);
292
293 cir::ConstantOp lowerBoundValue = cir::ConstantOp::create(
294 builder&: rewriter, location: op.getLoc(), value: cir::IntAttr::get(type: condType, value: lowerBound));
295 mlir::Value diffValue = cir::SubOp::create(
296 builder&: rewriter, location: op.getLoc(), lhs: op.getCondition(), rhs: lowerBoundValue);
297
298 // Use an unsigned comparison to check whether the condition is in range.
299 // cir.cmp takes its signedness from the operand type, so a signed
300 // difference needs a same-width integral cast to the unsigned type (a
301 // signedness reinterpretation) to make the `le` unsigned; an unsigned
302 // difference already has the right type.
303 if (isSigned)
304 diffValue = cir::CastOp::create(builder&: rewriter, location: op.getLoc(), result: uIntType,
305 kind: CastKind::integral, src: diffValue);
306
307 cir::ConstantOp rangeLength = cir::ConstantOp::create(
308 builder&: rewriter, location: op.getLoc(),
309 value: cir::IntAttr::get(type: uIntType, value: upperBound - lowerBound));
310
311 cir::CmpOp cmpResult = cir::CmpOp::create(
312 builder&: rewriter, location: op.getLoc(), kind: cir::CmpOpKind::le, lhs: diffValue, rhs: rangeLength);
313 cir::BrCondOp::create(builder&: rewriter, location: op.getLoc(), cond: cmpResult, destTrue: rangeDestination,
314 destFalse: defaultDestination);
315 return resBlock;
316 }
317
318 mlir::LogicalResult
319 matchAndRewrite(cir::SwitchOp op,
320 mlir::PatternRewriter &rewriter) const override {
321 // All nested structured CIR ops must be flattened before the switch.
322 // Break statements inside nested structured ops would create branches to
323 // blocks outside those ops' regions, which is invalid. Fail the match so
324 // the pattern rewriter will process them first.
325 for (mlir::Region &region : op->getRegions())
326 if (hasNestedOpsToFlatten(region))
327 return mlir::failure();
328
329 // Empty switch statement: just erase it.
330 if (op.getBody().hasOneBlock() &&
331 op.getBody().front().without_terminator().empty()) {
332 rewriter.eraseOp(op);
333 return mlir::success();
334 }
335
336 llvm::SmallVector<CaseOp> cases;
337 op.collectCases(cases);
338
339 // Create exit block from the next node of cir.switch op.
340 mlir::Block *exitBlock = rewriter.splitBlock(
341 block: rewriter.getBlock(), before: op->getNextNode()->getIterator());
342
343 // We lower cir.switch op in the following process:
344 // 1. Inline the region from the switch op after switch op.
345 // 2. Traverse each cir.case op:
346 // a. Record the entry block, block arguments and condition for every
347 // case. b. Inline the case region after the case op.
348 // 3. Replace the empty cir.switch.op with the new cir.switchflat op by the
349 // recorded block and conditions.
350
351 // First we have to handle the rewrite of all of the 'break' ops to make
352 // sure they now go to the right place, including the ones in the pre-case
353 // blcoks.
354 walkRegionSkipping<cir::LoopOpInterface, cir::SwitchOp>(
355 region&: op.getBody(), callback: [&](mlir::Operation *op) {
356 if (!isa<cir::BreakOp>(Val: op))
357 return mlir::WalkResult::advance();
358
359 lowerTerminator(op, dest: exitBlock, rewriter);
360 return mlir::WalkResult::skip();
361 });
362
363 // inline everything from switch body between the switch op and the exit
364 // block.
365 {
366 cir::YieldOp switchYield = nullptr;
367 // Clear switch operation.
368 for (mlir::Block &block :
369 llvm::make_early_inc_range(Range&: op.getBody().getBlocks()))
370 if (auto yieldOp = dyn_cast<cir::YieldOp>(Val: block.getTerminator()))
371 switchYield = yieldOp;
372
373 assert(!op.getBody().empty());
374 mlir::Block *originalBlock = op->getBlock();
375 mlir::Block *swopBlock =
376 rewriter.splitBlock(block: originalBlock, before: op->getIterator());
377 rewriter.inlineRegionBefore(region&: op.getBody(), before: exitBlock);
378
379 if (switchYield)
380 rewriteYieldOp(rewriter, yieldOp: switchYield, destination: exitBlock);
381
382 rewriter.setInsertionPointToEnd(originalBlock);
383 cir::BrOp::create(builder&: rewriter, location: op.getLoc(), dest: swopBlock);
384 }
385
386 // Allocate required data structures (disconsider default case in
387 // vectors).
388 llvm::SmallVector<mlir::APInt, 8> caseValues;
389 llvm::SmallVector<mlir::Block *, 8> caseDestinations;
390 llvm::SmallVector<mlir::ValueRange, 8> caseOperands;
391
392 llvm::SmallVector<std::pair<APInt, APInt>> rangeValues;
393 llvm::SmallVector<mlir::Block *> rangeDestinations;
394 llvm::SmallVector<mlir::ValueRange> rangeOperands;
395
396 // Initialize default case as optional.
397 mlir::Block *defaultDestination = exitBlock;
398 mlir::ValueRange defaultOperands = exitBlock->getArguments();
399
400 // Digest the case statements values and bodies.
401 for (cir::CaseOp caseOp : cases) {
402 mlir::Region &region = caseOp.getCaseRegion();
403
404 // Found default case: save destination and operands.
405 switch (caseOp.getKind()) {
406 case cir::CaseOpKind::Default:
407 defaultDestination = &region.front();
408 defaultOperands = defaultDestination->getArguments();
409 break;
410 case cir::CaseOpKind::Range:
411 assert(caseOp.getValue().size() == 2 &&
412 "Case range should have 2 case value");
413 rangeValues.push_back(
414 Elt: {cast<cir::IntAttr>(Val: caseOp.getValue()[0]).getValue(),
415 cast<cir::IntAttr>(Val: caseOp.getValue()[1]).getValue()});
416 rangeDestinations.push_back(Elt: &region.front());
417 rangeOperands.push_back(Elt: rangeDestinations.back()->getArguments());
418 break;
419 case cir::CaseOpKind::Anyof:
420 case cir::CaseOpKind::Equal:
421 // AnyOf cases kind can have multiple values, hence the loop below.
422 for (const mlir::Attribute &value : caseOp.getValue()) {
423 caseValues.push_back(Elt: cast<cir::IntAttr>(Val: value).getValue());
424 caseDestinations.push_back(Elt: &region.front());
425 caseOperands.push_back(Elt: caseDestinations.back()->getArguments());
426 }
427 break;
428 }
429
430 // Track fallthrough in cases.
431 for (mlir::Block &blk : region.getBlocks()) {
432 if (blk.getNumSuccessors())
433 continue;
434
435 if (auto yieldOp = dyn_cast<cir::YieldOp>(Val: blk.getTerminator())) {
436 mlir::Operation *nextOp = caseOp->getNextNode();
437 assert(nextOp && "caseOp is not expected to be the last op");
438 mlir::Block *oldBlock = nextOp->getBlock();
439 mlir::Block *newBlock =
440 rewriter.splitBlock(block: oldBlock, before: nextOp->getIterator());
441 rewriter.setInsertionPointToEnd(oldBlock);
442 cir::BrOp::create(builder&: rewriter, location: nextOp->getLoc(), destOperands: mlir::ValueRange(),
443 /*loop_id=*/nullptr, /*loop_hints=*/nullptr,
444 dest: newBlock);
445 rewriteYieldOp(rewriter, yieldOp, destination: newBlock);
446 }
447 }
448
449 mlir::Block *oldBlock = caseOp->getBlock();
450 mlir::Block *newBlock =
451 rewriter.splitBlock(block: oldBlock, before: caseOp->getIterator());
452
453 mlir::Block &entryBlock = caseOp.getCaseRegion().front();
454 rewriter.inlineRegionBefore(region&: caseOp.getCaseRegion(), before: newBlock);
455
456 // Create a branch to the entry of the inlined region.
457 rewriter.setInsertionPointToEnd(oldBlock);
458 cir::BrOp::create(builder&: rewriter, location: caseOp.getLoc(), dest: &entryBlock);
459 }
460
461 // Remove all cases since we've inlined the regions.
462 for (cir::CaseOp caseOp : cases) {
463 mlir::Block *caseBlock = caseOp->getBlock();
464 // Erase the block with no predecessors here to make the generated code
465 // simpler a little bit.
466 if (caseBlock->hasNoPredecessors())
467 rewriter.eraseBlock(block: caseBlock);
468 else
469 rewriter.eraseOp(op: caseOp);
470 }
471
472 bool isSigned =
473 mlir::cast<cir::IntType>(Val: op.getCondition().getType()).isSigned();
474 for (auto [rangeVal, operand, destination] :
475 llvm::zip(t&: rangeValues, u&: rangeOperands, args&: rangeDestinations)) {
476 APInt lowerBound = rangeVal.first;
477 APInt upperBound = rangeVal.second;
478
479 // An empty range (lo > hi in the switch's signedness) is unreachable.
480 if (isSigned ? lowerBound.sgt(RHS: upperBound) : lowerBound.ugt(RHS: upperBound))
481 continue;
482
483 // If range is small, add multiple switch instruction cases.
484 // This magical number is from the original CGStmt code.
485 constexpr uint64_t kSmallRangeThreshold = 64;
486 APInt rangeSize = upperBound - lowerBound;
487 if (rangeSize.ult(RHS: kSmallRangeThreshold)) {
488 // Expand into individual cases. rangeSize < kSmallRangeThreshold, so
489 // the inclusive case count (rangeSize + 1) fits in a uint64_t. Drive
490 // termination by the count rather than by comparing caseValue to
491 // upperBound: when upperBound is the type's maximum the final
492 // caseValue++ wraps past the top, which is harmless because the
493 // wrapped value is never used.
494 APInt caseValue = lowerBound;
495 for (uint64_t n = rangeSize.getZExtValue() + 1; n != 0; --n) {
496 caseValues.push_back(Elt: caseValue++);
497 caseOperands.push_back(Elt: operand);
498 caseDestinations.push_back(Elt: destination);
499 }
500 continue;
501 }
502
503 defaultDestination =
504 condBrToRangeDestination(op, rewriter, rangeDestination: destination,
505 defaultDestination, lowerBound, upperBound);
506 defaultOperands = operand;
507 }
508
509 // Set switch op to branch to the newly created blocks.
510 rewriter.setInsertionPoint(op);
511 rewriter.replaceOpWithNewOp<cir::SwitchFlatOp>(
512 op, args: op.getCondition(), args&: defaultDestination, args&: defaultOperands, args&: caseValues,
513 args&: caseDestinations, args&: caseOperands);
514
515 return mlir::success();
516 }
517};
518
519class CIRLoopOpInterfaceFlattening
520 : public mlir::OpInterfaceRewritePattern<cir::LoopOpInterface> {
521public:
522 using mlir::OpInterfaceRewritePattern<
523 cir::LoopOpInterface>::OpInterfaceRewritePattern;
524
525 inline void lowerConditionOp(cir::ConditionOp op, mlir::Block *body,
526 mlir::Block *exit,
527 mlir::PatternRewriter &rewriter,
528 mlir::DistinctAttr loopId = {},
529 cir::LoopHintAttr loopHints = {}) const {
530 mlir::OpBuilder::InsertionGuard guard(rewriter);
531 rewriter.setInsertionPoint(op);
532 rewriter.replaceOpWithNewOp<cir::BrCondOp>(
533 op, args: op.getCondition(), args: mlir::ValueRange{}, args: mlir::ValueRange{}, args&: loopId,
534 args&: loopHints, args&: body, args&: exit);
535 }
536
537 // Rewrite a loop that has a per-iteration cleanup region into a loop whose
538 // condition is always 'true' and whose body is a cir.cleanup.scope enclosing
539 // the original condition, body, and step regions.
540 //
541 // The condition test is sunk to the top of the scope body (a false
542 // result becomes a cir.break out of the loop) and, for a for loop, the step
543 // is appended to the end of the scope body. The loop's cleanup region becomes
544 // the cleanup region of a cir.cleanup.scope enclosing the new body.
545 //
546 // For example, a while loop:
547 //
548 // cir.while { <cond>; cir.condition(%c) }
549 // do { <body> }
550 // cleanup all { <cleanup> }
551 //
552 // becomes:
553 //
554 // cir.while { cir.condition(%true) } do {
555 // cir.cleanup.scope {
556 // <cond>
557 // cir.brcond %c ^body, ^cond_false
558 // ^cond_false:
559 // cir.break
560 // ^body:
561 // <body>
562 // } cleanup all { <cleanup> }
563 // cir.yield
564 // }
565 //
566 // A for loop is the same, except the step region is appended to the scope
567 // body (after the body) and both the body's normal end and any continue are
568 // redirected into the step so that the step runs before the cleanup. The init
569 // section of a for loop is already outside the loop op.
570 mlir::LogicalResult
571 rewriteLoopWithCleanup(cir::LoopOpInterface op,
572 mlir::PatternRewriter &rewriter) const {
573 mlir::Location loc = op.getLoc();
574 mlir::Region *stepRegion = op.maybeGetStep();
575
576 cir::CleanupKindAttr cleanupKind = op.maybeGetCleanupKind();
577 assert(cleanupKind && "loop cleanup region without a cleanup kind");
578
579 mlir::Region &condRegion = op.getCond();
580 mlir::Region &bodyRegion = op.getBody();
581 mlir::Region &cleanupRegion = *op.maybeGetCleanup();
582
583 // The cir.condition is the terminator of the condition region's last block
584 // (there can be more than one block if a nested scope was already
585 // flattened within the condition).
586 auto conditionOp =
587 cast<cir::ConditionOp>(Val: condRegion.back().getTerminator());
588 mlir::Value condVal = conditionOp.getCondition();
589
590 // Capture block references and exit terminators before moving blocks
591 // around. Block pointers stay valid across region inlining because blocks
592 // are reparented, not recreated.
593 mlir::Block *bodyFront = &bodyRegion.front();
594 mlir::Block *stepFront = stepRegion ? &stepRegion->front() : nullptr;
595
596 // For a for loop the body's normal end and any continue must run the step
597 // before the cleanup, so collect them to redirect into the step region. For
598 // a while loop they are genuine cleanup-scope exits and are left untouched.
599 llvm::SmallVector<cir::YieldOp> bodyYieldsToStep;
600 llvm::SmallVector<cir::ContinueOp> continuesToStep;
601 if (stepRegion) {
602 for (mlir::Block &blk : bodyRegion.getBlocks())
603 if (auto y = dyn_cast<cir::YieldOp>(Val: blk.getTerminator()))
604 bodyYieldsToStep.push_back(Elt: y);
605 op.walkBodySkippingNestedLoops(callback: [&](mlir::Operation *o) {
606 if (auto c = dyn_cast<cir::ContinueOp>(Val: o)) {
607 continuesToStep.push_back(Elt: c);
608 return mlir::WalkResult::skip();
609 }
610 return mlir::WalkResult::advance();
611 });
612 }
613
614 // Assemble the per-iteration blocks into the condition region, in
615 // execution order: [ cond blocks..., body blocks..., step blocks... ].
616 rewriter.inlineRegionBefore(region&: bodyRegion, parent&: condRegion, before: condRegion.end());
617 if (stepRegion)
618 rewriter.inlineRegionBefore(region&: *stepRegion, parent&: condRegion, before: condRegion.end());
619
620 // Replace cir.condition(%c) with a conditional branch into the body whose
621 // false edge breaks out of the loop.
622 mlir::Block *breakBlock =
623 rewriter.createBlock(parent: &condRegion, insertPt: condRegion.end());
624 rewriter.setInsertionPointToEnd(breakBlock);
625 cir::BreakOp::create(builder&: rewriter, location: conditionOp.getLoc());
626
627 rewriter.setInsertionPoint(conditionOp);
628 rewriter.replaceOpWithNewOp<cir::BrCondOp>(op: conditionOp, args&: condVal, args&: bodyFront,
629 args&: breakBlock);
630
631 // For a for loop, redirect the body's normal end and any continue to the
632 // step. The step's own yield remains the normal end of the iteration.
633 for (cir::YieldOp y : bodyYieldsToStep)
634 lowerTerminator(op: y, dest: stepFront, rewriter);
635 for (cir::ContinueOp c : continuesToStep)
636 lowerTerminator(op: c, dest: stepFront, rewriter);
637
638 // Build the trivial loop body: a single cir.cleanup.scope followed by a
639 // yield, in a fresh block (the body region's blocks were moved out above).
640 mlir::Block *newBodyBlock = rewriter.createBlock(parent: &bodyRegion);
641 rewriter.setInsertionPointToEnd(newBodyBlock);
642 auto emitYield = [](mlir::OpBuilder &b, mlir::Location l) {
643 cir::YieldOp::create(builder&: b, location: l);
644 };
645 auto scope = cir::CleanupScopeOp::create(
646 builder&: rewriter, location: loc, cleanupKind: cleanupKind.getValue(), bodyBuilder: emitYield, cleanupBuilder: emitYield);
647 cir::YieldOp::create(builder&: rewriter, location: loc);
648
649 // Move the assembled per-iteration blocks into the scope's body region and
650 // the loop's cleanup blocks into the scope's cleanup region, discarding the
651 // placeholder blocks the builder created.
652 mlir::Block *bodyPlaceholder = &scope.getBodyRegion().front();
653 rewriter.inlineRegionBefore(region&: condRegion, before: bodyPlaceholder);
654 rewriter.eraseBlock(block: bodyPlaceholder);
655
656 mlir::Block *cleanupPlaceholder = &scope.getCleanupRegion().front();
657 rewriter.inlineRegionBefore(region&: cleanupRegion, before: cleanupPlaceholder);
658 rewriter.eraseBlock(block: cleanupPlaceholder);
659
660 // Rebuild the loop condition region with an always-true condition.
661 mlir::Block *newCondBlock = rewriter.createBlock(parent: &condRegion);
662 rewriter.setInsertionPointToEnd(newCondBlock);
663 mlir::Value trueVal = cir::ConstantOp::create(
664 builder&: rewriter, location: loc, value: cir::BoolAttr::get(context: rewriter.getContext(), value: true));
665 cir::ConditionOp::create(builder&: rewriter, location: loc, condition: trueVal);
666
667 // For a for loop, rebuild the (now empty) step region with a trivial yield.
668 if (stepRegion) {
669 mlir::Block *newStepBlock = rewriter.createBlock(parent: stepRegion);
670 rewriter.setInsertionPointToEnd(newStepBlock);
671 cir::YieldOp::create(builder&: rewriter, location: loc);
672 }
673
674 // Drop the cleanup-kind attribute now that the loop's cleanup region is
675 // empty, so the trivial loop verifies and takes the no-cleanup flattening
676 // path when the greedy driver revisits it.
677 if (auto whileOp = mlir::dyn_cast<cir::WhileOp>(Val: op.getOperation()))
678 whileOp.removeCleanupKindAttr();
679 else if (auto forOp = mlir::dyn_cast<cir::ForOp>(Val: op.getOperation()))
680 forOp.removeCleanupKindAttr();
681
682 return mlir::success();
683 }
684
685 mlir::LogicalResult
686 matchAndRewrite(cir::LoopOpInterface op,
687 mlir::PatternRewriter &rewriter) const final {
688 // All nested structured CIR ops must be flattened before the loop.
689 // Break/continue statements inside nested structured ops would create
690 // branches to blocks outside those ops' regions, which is invalid. Fail
691 // the match so the pattern rewriter will process them first.
692 for (mlir::Region &region : op->getRegions())
693 if (hasNestedOpsToFlatten(region))
694 return mlir::failure();
695
696 // Loops with a per-iteration cleanup region need every iteration-exit edge
697 // routed through that cleanup, including the false-condition exit and any
698 // exceptions that might be thrown from the step region. Rather than trying
699 // to figure out all of the cleanup routing here, we sink the condition into
700 // the body region, hoist the step region (if any) and create a new
701 // cir.cleanup.scope enclosing the body region. A subsequent sweep of the
702 // pass will flatten the cir.cleanup.scope and the loop reusing the normal
703 // handlers.
704 if (op.maybeGetCleanup())
705 return rewriteLoopWithCleanup(op, rewriter);
706
707 // Setup CFG blocks.
708 mlir::Block *entry = rewriter.getInsertionBlock();
709 mlir::Block *exit =
710 rewriter.splitBlock(block: entry, before: rewriter.getInsertionPoint());
711 mlir::Block *cond = &op.getCond().front();
712 mlir::Block *body = &op.getBody().front();
713 mlir::Block *step =
714 (op.maybeGetStep() ? &op.maybeGetStep()->front() : nullptr);
715
716 // Setup loop entry branch.
717 mlir::Block *firstBlock = &op.getEntry().front();
718 rewriter.setInsertionPointToEnd(entry);
719 cir::BrOp::create(builder&: rewriter, location: op.getLoc(), dest: firstBlock);
720
721 // We attach the op's loop hints to the ops that break back to the
722 // 'beginning' of the operation. For a while/for it is the condition block,
723 // for a do-while it is the top of the body. All of those ops also get the
724 // loop's id, so that they can be told apart from other loops' hints.
725 cir::LoopHintAttr loopHints = op.getLoopHintsAttr();
726 mlir::DistinctAttr loopId = op.getLoopIdAttr();
727 auto idFor = [&](mlir::Block *target) {
728 return target == firstBlock ? loopId : mlir::DistinctAttr{};
729 };
730 auto hintsFor = [&](mlir::Block *target) {
731 return target == firstBlock ? loopHints : cir::LoopHintAttr{};
732 };
733
734 // Branch from condition region to body or exit. The ConditionOp may not
735 // be in the first block of the condition region if a cleanup scope was
736 // already flattened within it, introducing multiple blocks. The
737 // ConditionOp is always the terminator of the last block.
738 auto conditionOp =
739 cast<cir::ConditionOp>(Val: op.getCond().back().getTerminator());
740 lowerConditionOp(op: conditionOp, body, exit, rewriter, loopId: idFor(body),
741 loopHints: hintsFor(body));
742
743 // TODO(cir): Remove the walks below. It visits operations unnecessarily.
744 // However, to solve this we would likely need a custom DialectConversion
745 // driver to customize the order that operations are visited.
746
747 // Lower continue statements.
748 mlir::Block *dest = (step ? step : cond);
749 op.walkBodySkippingNestedLoops(callback: [&](mlir::Operation *op) {
750 if (!isa<cir::ContinueOp>(Val: op))
751 return mlir::WalkResult::advance();
752
753 lowerTerminator(op, dest, rewriter, loopId: idFor(dest), loopHints: hintsFor(dest));
754 return mlir::WalkResult::skip();
755 });
756
757 // Lower break statements.
758 walkRegionSkipping<cir::LoopOpInterface, cir::SwitchOp>(
759 region&: op.getBody(), callback: [&](mlir::Operation *op) {
760 if (!isa<cir::BreakOp>(Val: op))
761 return mlir::WalkResult::advance();
762
763 lowerTerminator(op, dest: exit, rewriter);
764 return mlir::WalkResult::skip();
765 });
766
767 // Lower optional body region yield.
768 for (mlir::Block &blk : op.getBody().getBlocks()) {
769 auto bodyYield = dyn_cast<cir::YieldOp>(Val: blk.getTerminator());
770 if (bodyYield)
771 lowerTerminator(op: bodyYield, dest, rewriter, loopId: idFor(dest), loopHints: hintsFor(dest));
772 }
773
774 // Lower mandatory step region yield. Like the condition region, the
775 // YieldOp may be in the last block rather than the first if a cleanup
776 // scope was already flattened within the step region.
777 if (step)
778 lowerTerminator(
779 op: cast<cir::YieldOp>(Val: op.maybeGetStep()->back().getTerminator()), dest: cond,
780 rewriter, loopId: idFor(cond), loopHints: hintsFor(cond));
781
782 // Move region contents out of the loop op.
783 rewriter.inlineRegionBefore(region&: op.getCond(), before: exit);
784 rewriter.inlineRegionBefore(region&: op.getBody(), before: exit);
785 if (step)
786 rewriter.inlineRegionBefore(region&: *op.maybeGetStep(), before: exit);
787
788 rewriter.eraseOp(op);
789 return mlir::success();
790 }
791};
792
793class CIRTernaryOpFlattening : public mlir::OpRewritePattern<cir::TernaryOp> {
794public:
795 using OpRewritePattern<cir::TernaryOp>::OpRewritePattern;
796
797 mlir::LogicalResult
798 matchAndRewrite(cir::TernaryOp op,
799 mlir::PatternRewriter &rewriter) const override {
800 Location loc = op->getLoc();
801 Block *condBlock = rewriter.getInsertionBlock();
802 Block::iterator opPosition = rewriter.getInsertionPoint();
803 Block *remainingOpsBlock = rewriter.splitBlock(block: condBlock, before: opPosition);
804 llvm::SmallVector<mlir::Location, 2> locs;
805 // Ternary result is optional, make sure to populate the location only
806 // when relevant.
807 if (op->getResultTypes().size())
808 locs.push_back(Elt: loc);
809 Block *continueBlock =
810 rewriter.createBlock(insertBefore: remainingOpsBlock, argTypes: op->getResultTypes(), locs);
811 cir::BrOp::create(builder&: rewriter, location: loc, dest: remainingOpsBlock);
812
813 Region &trueRegion = op.getTrueRegion();
814 Block *trueBlock = &trueRegion.front();
815 // Wire up the true region's exit (cir.yield -> br, cir.unreachable /
816 // cir.trap kept as-is). IR has already been modified by splitBlock /
817 // createBlock above, so per the MLIR pattern rewriter contract we must
818 // still return success() if the terminator turns out to be unexpected.
819 if (failed(Result: rewriteRegionExitToContinue(rewriter, region&: trueRegion, continueBlock,
820 regionDescription: "ternary true")))
821 return mlir::success();
822 rewriter.inlineRegionBefore(region&: trueRegion, before: continueBlock);
823
824 Block *falseBlock = continueBlock;
825 Region &falseRegion = op.getFalseRegion();
826
827 falseBlock = &falseRegion.front();
828 if (failed(Result: rewriteRegionExitToContinue(rewriter, region&: falseRegion, continueBlock,
829 regionDescription: "ternary false")))
830 return mlir::success();
831 rewriter.inlineRegionBefore(region&: falseRegion, before: continueBlock);
832
833 rewriter.setInsertionPointToEnd(condBlock);
834 cir::BrCondOp::create(builder&: rewriter, location: loc, cond: op.getCond(), destTrue: trueBlock, destFalse: falseBlock);
835
836 rewriter.replaceOp(op, newValues: continueBlock->getArguments());
837
838 // Ok, we're done!
839 return mlir::success();
840 }
841};
842
843// Get or create the cleanup destination slot for a function. This slot is
844// shared across all cleanup scopes in the function to track which exit path
845// to take after running cleanup code when there are multiple exits.
846static cir::AllocaOp getOrCreateCleanupDestSlot(cir::FuncOp funcOp,
847 mlir::PatternRewriter &rewriter,
848 mlir::Location loc) {
849 mlir::Block &entryBlock = funcOp.getBody().front();
850
851 // Look for an existing cleanup dest slot in the entry block.
852 auto it = llvm::find_if(Range&: entryBlock, P: [](auto &op) {
853 return mlir::isa<AllocaOp>(&op) &&
854 mlir::cast<AllocaOp>(&op).getCleanupDestSlot();
855 });
856 if (it != entryBlock.end())
857 return mlir::cast<cir::AllocaOp>(Val&: *it);
858
859 // Create a new cleanup dest slot at the start of the entry block.
860 mlir::OpBuilder::InsertionGuard guard(rewriter);
861 rewriter.setInsertionPointToStart(&entryBlock);
862 cir::IntType s32Type =
863 cir::IntType::get(context: rewriter.getContext(), width: 32, /*isSigned=*/is_signed: true);
864 cir::CIRDataLayout dataLayout(funcOp->getParentOfType<mlir::ModuleOp>());
865 cir::PointerType ptrToS32Type = cir::PointerType::get(
866 pointee: s32Type, addrSpace: dataLayout.getAllocaAddrSpace(ctx: rewriter.getContext()));
867 uint64_t alignment = dataLayout.getAlignment(ty: s32Type, useABIAlign: true).value();
868 auto allocaOp = cir::AllocaOp::create(
869 builder&: rewriter, location: loc, addr: ptrToS32Type, name: "__cleanup_dest_slot",
870 /*alignment=*/rewriter.getI64IntegerAttr(value: alignment));
871 allocaOp.setCleanupDestSlot(true);
872 return allocaOp;
873}
874
875/// Shared EH flattening utilities used by both CIRCleanupScopeOpFlattening
876/// and CIRTryOpFlattening.
877
878// Lifetime markers participate in existing EH paths, but do not require an
879// unwind edge on their own. Conservatively treat other cleanup code as real
880// cleanup, including regions with control flow.
881static bool isLifetimeMarkerOnly(mlir::Region &region) {
882 return llvm::hasSingleElement(C&: region) &&
883 llvm::any_of(
884 Range&: region.front(),
885 P: [](mlir::Operation &op) { return isa<cir::LifetimeEndOp>(Val: op); }) &&
886 llvm::all_of(Range&: region.front(), P: [](mlir::Operation &op) {
887 return isa<cir::LifetimeEndOp, cir::YieldOp>(Val: op);
888 });
889}
890
891// Look for an enclosing handler or non-marker EH cleanup that protects this
892// operation. Follow regions, rather than just parent ops: a try does not catch
893// exceptions from its own handlers, and a cleanup does not protect itself.
894static bool hasEnclosingEHRequirement(mlir::Operation *op) {
895 for (mlir::Region *region = op->getParentRegion(); region;
896 region = region->getParentRegion()) {
897 mlir::Operation *parent = region->getParentOp();
898 if (!parent || isa<cir::FuncOp>(Val: parent))
899 break;
900 if (auto tryOp = dyn_cast<cir::TryOp>(Val: parent)) {
901 auto handlers = tryOp.getHandlerTypesAttr();
902 if (region == &tryOp.getTryRegion() && handlers &&
903 llvm::any_of(Range&: handlers, P: [](mlir::Attribute handler) {
904 return !isa<cir::UnwindAttr>(Val: handler);
905 }))
906 return true;
907 } else if (auto cleanupOp = dyn_cast<cir::CleanupScopeOp>(Val: parent)) {
908 if (region == &cleanupOp.getBodyRegion() &&
909 cleanupOp.getCleanupKindAttr().isEH() &&
910 !isLifetimeMarkerOnly(region&: cleanupOp.getCleanupRegion()))
911 return true;
912 } else if (auto loopOp = dyn_cast<cir::LoopOpInterface>(Val: parent)) {
913 // The enclosing loop may not yet have been rewritten to a cleanup scope.
914 mlir::Region *cleanup = loopOp.maybeGetCleanup();
915 if (cleanup && region != cleanup && loopOp.maybeGetCleanupKind().isEH() &&
916 !isLifetimeMarkerOnly(region&: *cleanup))
917 return true;
918 }
919 }
920 return false;
921}
922
923// Collect all function calls in a region that may throw exceptions and need
924// to be replaced with try_call operations. Skips calls marked nothrow.
925// Nested cleanup scopes and try ops are always flattened before their
926// enclosing parents, so there are no nested regions to skip here.
927static void
928collectThrowingCalls(mlir::Region &region,
929 llvm::SmallVectorImpl<cir::CallOp> &callsToRewrite) {
930 region.walk(callback: [&](cir::CallOp callOp) {
931 // A musttail call is never an invoke. LLVM has no musttail invoke, and
932 // the tail call replaces the frame the cleanup would have run in, so
933 // there is nothing left to unwind through.
934 if (!callOp.getNothrow() && !callOp.getMusttail())
935 callsToRewrite.push_back(Elt: callOp);
936 });
937}
938
939// Collect all cir.throw operations in a region that need to be replaced
940// with cir.try_throw operations so they can unwind through an enclosing
941// cleanup or catch handler. Nested cleanup scopes and try ops are always
942// flattened before their enclosing parents, so there are no nested
943// regions to skip here.
944static void
945collectThrows(mlir::Region &region,
946 llvm::SmallVectorImpl<cir::ThrowOp> &throwsToRewrite) {
947 region.walk(
948 callback: [&](cir::ThrowOp throwOp) { throwsToRewrite.push_back(Elt: throwOp); });
949}
950
951// Collect all cir.resume operations in a region that come from
952// already-flattened try or cleanup scope operations. These resume ops need
953// to be chained through this scope's EH handler instead of unwinding
954// directly to the caller. Nested cleanup scopes and try ops are always
955// flattened before their enclosing parents, so there are no nested regions
956// to skip here.
957static void collectResumeOps(mlir::Region &region,
958 llvm::SmallVectorImpl<cir::ResumeOp> &resumeOps) {
959 region.walk(callback: [&](cir::ResumeOp resumeOp) { resumeOps.push_back(Elt: resumeOp); });
960}
961
962// Create a shared unwind destination block. The block contains a
963// cir.eh.initiate operation (optionally with the cleanup attribute) and a
964// branch to the given destination block, passing the eh_token.
965static mlir::Block *buildUnwindBlock(mlir::Block *dest, bool isCleanupOnly,
966 mlir::Location loc,
967 mlir::Block *insertBefore,
968 mlir::PatternRewriter &rewriter) {
969 mlir::Block *unwindBlock = rewriter.createBlock(insertBefore);
970 rewriter.setInsertionPointToEnd(unwindBlock);
971 auto ehInitiate =
972 cir::EhInitiateOp::create(builder&: rewriter, location: loc, /*cleanup=*/isCleanupOnly);
973 cir::BrOp::create(builder&: rewriter, location: loc, destOperands: mlir::ValueRange{ehInitiate.getEhToken()},
974 /*loop_id=*/nullptr, /*loop_hints=*/nullptr, dest);
975 return unwindBlock;
976}
977
978// Create a shared terminate unwind block for throwing calls in EH cleanup
979// regions. When an exception is thrown during cleanup (unwinding), the C++
980// standard requires that std::terminate() be called.
981static mlir::Block *buildTerminateUnwindBlock(mlir::Location loc,
982 mlir::Block *insertBefore,
983 mlir::PatternRewriter &rewriter) {
984 mlir::Block *terminateBlock = rewriter.createBlock(insertBefore);
985 rewriter.setInsertionPointToEnd(terminateBlock);
986 auto ehInitiate = cir::EhInitiateOp::create(builder&: rewriter, location: loc, /*cleanup=*/false);
987 cir::EhTerminateOp::create(builder&: rewriter, location: loc, eh_token: ehInitiate.getEhToken());
988 return terminateBlock;
989}
990
991class CIRCleanupScopeOpFlattening
992 : public mlir::OpRewritePattern<cir::CleanupScopeOp> {
993public:
994 using OpRewritePattern<cir::CleanupScopeOp>::OpRewritePattern;
995
996 struct CleanupExit {
997 // An operation that exits the cleanup scope (yield, break, continue,
998 // return, etc.)
999 mlir::Operation *exitOp;
1000
1001 // A unique identifier for this exit's destination (used for switch dispatch
1002 // when there are multiple exits).
1003 int destinationId;
1004
1005 CleanupExit(mlir::Operation *op, int id) : exitOp(op), destinationId(id) {}
1006 };
1007
1008 // Determine whether a goto operation transfers control to a label that
1009 // exists somewhere inside the given region (or any of its nested regions).
1010 // Label names are unique within a function, so finding a matching cir.label
1011 // inside the region implies that the goto definitely targets that label and
1012 // therefore stays within the region. If no match is found, the goto either
1013 // exits the region or its target is unknown; in either case the caller must
1014 // treat it as exiting the region.
1015 static bool gotoTargetsLabelInRegion(cir::GotoOp gotoOp,
1016 mlir::Region &region) {
1017 llvm::StringRef targetLabel = gotoOp.getLabel();
1018 return region
1019 .walk(callback: [&](cir::LabelOp labelOp) {
1020 if (labelOp.getLabel() == targetLabel)
1021 return mlir::WalkResult::interrupt();
1022 return mlir::WalkResult::advance();
1023 })
1024 .wasInterrupted();
1025 }
1026
1027 // Collect all operations that exit a cleanup scope body. Return, goto, break,
1028 // and continue can all require branches through the cleanup region. When a
1029 // loop is encountered, only return and goto are collected because break and
1030 // continue are handled by the loop and stay within the cleanup scope. When a
1031 // switch is encountered, return, goto and continue are collected because they
1032 // may all branch through the cleanup, but break is local to the switch. When
1033 // a nested cleanup scope is encountered, we recursively collect exits since
1034 // any return, goto, break, or continue from the nested cleanup will also
1035 // branch through the outer cleanup.
1036 //
1037 // A goto is only treated as an exit if its target label is not somewhere
1038 // inside the cleanup body region. Gotos whose target label is within the
1039 // cleanup body stay inside the cleanup scope and need no special handling
1040 // during flattening; they are simply inlined along with the rest of the
1041 // body region.
1042 //
1043 // A musttail return is likewise not an exit. It leaves the function
1044 // directly without running any cleanup, which is what the tail call means:
1045 // the frame those cleanups would run in no longer exists. CIRGen only emits
1046 // one when every cleanup it skips is redundant before a return.
1047 //
1048 // This function assigns unique destination IDs to each exit, which are
1049 // used when multi-exit cleanup scopes are flattened.
1050 void collectExits(mlir::Region &cleanupBodyRegion,
1051 llvm::SmallVectorImpl<CleanupExit> &exits,
1052 int &nextId) const {
1053 // Collect yield terminators from the body region. We do this separately
1054 // because yields in nested operations, including those in nested cleanup
1055 // scopes, won't branch through the outer cleanup region.
1056 for (mlir::Block &block : cleanupBodyRegion) {
1057 auto *terminator = block.getTerminator();
1058 if (isa<cir::YieldOp>(Val: terminator))
1059 exits.emplace_back(Args&: terminator, Args: nextId++);
1060 }
1061
1062 // Helper to decide whether an op is a goto that needs to be treated as an
1063 // exit from the cleanup scope being flattened. If op is a goto and targets
1064 // a label inside the cleanup body region, control stays within the cleanup
1065 // and we leave the goto in place.
1066 auto isGotoThatExitsCleanup = [&](mlir::Operation *op) {
1067 auto gotoOp = dyn_cast<cir::GotoOp>(Val: op);
1068 return gotoOp && !gotoTargetsLabelInRegion(gotoOp, region&: cleanupBodyRegion);
1069 };
1070
1071 // Helper to decide whether a return branches through the cleanup. The
1072 // return that completes a musttail call does not. LLVM requires the two
1073 // to stay adjacent, so it cannot be rewritten into a store and a branch
1074 // to the cleanup. CIRGen emits the pair adjacently in the same block, and
1075 // the predecessor is only ever a cir.call, never a cir.try_call, since a
1076 // try_call terminates its block and so cannot be followed by a return.
1077 auto isReturnThatExitsCleanup = [](mlir::Operation *op) {
1078 if (!isa<cir::ReturnOp>(Val: op))
1079 return false;
1080 auto callOp = dyn_cast_if_present<cir::CallOp>(Val: op->getPrevNode());
1081 return !callOp || !callOp.getMusttail();
1082 };
1083
1084 // Lambda to walk a loop and collect only returns and gotos.
1085 // Break and continue inside loops are handled by the loop itself.
1086 // Loops don't require special handling for nested switch or cleanup scopes
1087 // because break and continue never branch out of the loop.
1088 auto collectExitsInLoop = [&](mlir::Operation *loopOp) {
1089 loopOp->walk<mlir::WalkOrder::PreOrder>(callback: [&](mlir::Operation *nestedOp) {
1090 if (isa<cir::CoroCleanupOp>(Val: nestedOp) ||
1091 isReturnThatExitsCleanup(nestedOp) ||
1092 isGotoThatExitsCleanup(nestedOp)) {
1093 exits.emplace_back(Args&: nestedOp, Args: nextId++);
1094 }
1095 return mlir::WalkResult::advance();
1096 });
1097 };
1098
1099 // Forward declaration for mutual recursion.
1100 std::function<void(mlir::Region &, bool)> collectExitsInCleanup;
1101 std::function<void(mlir::Operation *)> collectExitsInSwitch;
1102
1103 // Lambda to collect exits from a switch. Collects return/goto/continue but
1104 // not break (handled by switch). For nested loops/cleanups, recurses.
1105 collectExitsInSwitch = [&](mlir::Operation *switchOp) {
1106 switchOp->walk<mlir::WalkOrder::PreOrder>(callback: [&](mlir::Operation *nestedOp) {
1107 if (isa<cir::CleanupScopeOp>(Val: nestedOp)) {
1108 // Walk the nested cleanup, but ignore break statements because they
1109 // will be handled by the switch we are currently walking.
1110 collectExitsInCleanup(
1111 cast<cir::CleanupScopeOp>(Val: nestedOp).getBodyRegion(),
1112 /*ignoreBreak=*/true);
1113 return mlir::WalkResult::skip();
1114 } else if (isa<cir::LoopOpInterface>(Val: nestedOp)) {
1115 collectExitsInLoop(nestedOp);
1116 return mlir::WalkResult::skip();
1117 } else if (isa<cir::ContinueOp, cir::CoReturnOp, cir::CoroCleanupOp>(
1118 Val: nestedOp) ||
1119 isReturnThatExitsCleanup(nestedOp) ||
1120 isGotoThatExitsCleanup(nestedOp)) {
1121 exits.emplace_back(Args&: nestedOp, Args: nextId++);
1122 }
1123 return mlir::WalkResult::advance();
1124 });
1125 };
1126
1127 // Lambda to collect exits from a cleanup scope body region. This collects
1128 // break (optionally), continue, return, and goto, handling nested loops,
1129 // switches, and cleanups appropriately.
1130 collectExitsInCleanup = [&](mlir::Region &region, bool ignoreBreak) {
1131 region.walk<mlir::WalkOrder::PreOrder>(callback: [&](mlir::Operation *op) {
1132 // We need special handling for break statements because if this cleanup
1133 // scope was nested within a switch op, break will be handled by the
1134 // switch operation and therefore won't exit the cleanup scope enclosing
1135 // the switch. We're only collecting exits from the cleanup that started
1136 // this walk. Exits from nested cleanups will be handled when we flatten
1137 // the nested cleanup.
1138 if (!ignoreBreak && isa<cir::BreakOp>(Val: op)) {
1139 exits.emplace_back(Args&: op, Args: nextId++);
1140 } else if (isa<cir::ContinueOp, cir::CoReturnOp, cir::CoroCleanupOp>(
1141 Val: op) ||
1142 isReturnThatExitsCleanup(op) || isGotoThatExitsCleanup(op)) {
1143 exits.emplace_back(Args&: op, Args: nextId++);
1144 } else if (isa<cir::CleanupScopeOp>(Val: op)) {
1145 // Recurse into nested cleanup's body region.
1146 collectExitsInCleanup(cast<cir::CleanupScopeOp>(Val: op).getBodyRegion(),
1147 /*ignoreBreak=*/ignoreBreak);
1148 return mlir::WalkResult::skip();
1149 } else if (isa<cir::LoopOpInterface>(Val: op)) {
1150 // This kicks off a separate walk rather than continuing to dig deeper
1151 // in the current walk because we need to handle break and continue
1152 // differently inside loops.
1153 collectExitsInLoop(op);
1154 return mlir::WalkResult::skip();
1155 } else if (isa<cir::SwitchOp>(Val: op)) {
1156 // This kicks off a separate walk rather than continuing to dig deeper
1157 // in the current walk because we need to handle break differently
1158 // inside switches.
1159 collectExitsInSwitch(op);
1160 return mlir::WalkResult::skip();
1161 }
1162 return mlir::WalkResult::advance();
1163 });
1164 };
1165
1166 // Collect exits from the body region.
1167 collectExitsInCleanup(cleanupBodyRegion, /*ignoreBreak=*/false);
1168 }
1169
1170 // Check if an operand's defining op should be moved to the destination block.
1171 // We only sink constants and simple loads. Anything else should be saved
1172 // to a temporary alloca and reloaded at the destination block.
1173 static bool shouldSinkReturnOperand(mlir::Value operand,
1174 cir::ReturnOp returnOp) {
1175 // Block arguments can't be moved
1176 mlir::Operation *defOp = operand.getDefiningOp();
1177 if (!defOp)
1178 return false;
1179
1180 // Only move constants and loads to the dispatch block. For anything else,
1181 // we'll store to a temporary and reload in the dispatch block.
1182 if (!mlir::isa<cir::ConstantOp, cir::LoadOp>(Val: defOp))
1183 return false;
1184
1185 // Check if the return is the only user
1186 if (!operand.hasOneUse())
1187 return false;
1188
1189 // Only move ops that are in the same block as the return.
1190 if (defOp->getBlock() != returnOp->getBlock())
1191 return false;
1192
1193 if (auto loadOp = mlir::dyn_cast<cir::LoadOp>(Val: defOp)) {
1194 // A volatile or atomic load must happen before the cleanup runs.
1195 if (loadOp.isVolatileOrAtomic())
1196 return false;
1197
1198 // Only attempt to move loads of allocas in the entry block.
1199 mlir::Value ptr = loadOp.getAddr();
1200 auto funcOp = returnOp->getParentOfType<cir::FuncOp>();
1201 assert(funcOp && "Return op has no function parent?");
1202 mlir::Block &funcEntryBlock = funcOp.getBody().front();
1203
1204 // Check if it's an alloca in the function entry block
1205 if (auto allocaOp =
1206 mlir::dyn_cast_if_present<cir::AllocaOp>(Val: ptr.getDefiningOp()))
1207 return allocaOp->getBlock() == &funcEntryBlock;
1208
1209 return false;
1210 }
1211
1212 // Make sure we only fall through to here with constants.
1213 assert(mlir::isa<cir::ConstantOp>(defOp) && "Expected constant op");
1214 return true;
1215 }
1216
1217 // For returns with operands in cleanup dispatch blocks, the operands may not
1218 // dominate the dispatch block. This function handles that by either sinking
1219 // the operand's defining op to the dispatch block (for constants and simple
1220 // loads) or by storing to a temporary alloca and reloading it.
1221 void
1222 getReturnOpOperands(cir::ReturnOp returnOp, mlir::Operation *exitOp,
1223 mlir::Location loc, mlir::PatternRewriter &rewriter,
1224 llvm::SmallVectorImpl<mlir::Value> &returnValues) const {
1225 mlir::Block *destBlock = rewriter.getInsertionBlock();
1226 auto funcOp = exitOp->getParentOfType<cir::FuncOp>();
1227 assert(funcOp && "Return op has no function parent?");
1228 mlir::Block &funcEntryBlock = funcOp.getBody().front();
1229
1230 for (mlir::Value operand : returnOp.getOperands()) {
1231 if (shouldSinkReturnOperand(operand, returnOp)) {
1232 // Sink the defining op to the dispatch block.
1233 mlir::Operation *defOp = operand.getDefiningOp();
1234 rewriter.moveOpBefore(op: defOp, block: destBlock, iterator: destBlock->end());
1235 returnValues.push_back(Elt: operand);
1236 } else {
1237 // Create an alloca in the function entry block.
1238 cir::AllocaOp alloca;
1239 {
1240 mlir::OpBuilder::InsertionGuard guard(rewriter);
1241 rewriter.setInsertionPointToStart(&funcEntryBlock);
1242 cir::CIRDataLayout dataLayout(
1243 funcOp->getParentOfType<mlir::ModuleOp>());
1244 uint64_t alignment =
1245 dataLayout.getAlignment(ty: operand.getType(), useABIAlign: true).value();
1246 cir::PointerType ptrType = cir::PointerType::get(pointee: operand.getType());
1247 alloca =
1248 cir::AllocaOp::create(builder&: rewriter, location: loc, addr: ptrType, name: "__ret_operand_tmp",
1249 alignment: rewriter.getI64IntegerAttr(value: alignment));
1250 }
1251
1252 // Store the operand value at the original return location.
1253 {
1254 mlir::OpBuilder::InsertionGuard guard(rewriter);
1255 rewriter.setInsertionPoint(exitOp);
1256 cir::StoreOp::create(builder&: rewriter, location: loc, value: operand, addr: alloca,
1257 /*isVolatile=*/is_volatile: false,
1258 /*isNontemporal=*/is_nontemporal: false,
1259 /*alignment=*/mlir::IntegerAttr(),
1260 sync_scope: cir::SyncScopeKindAttr(), mem_order: cir::MemOrderAttr());
1261 }
1262
1263 // Reload the value from the temporary alloca in the destination block.
1264 rewriter.setInsertionPointToEnd(destBlock);
1265 auto loaded =
1266 cir::LoadOp::create(builder&: rewriter, location: loc, addr: alloca, /*isDeref=*/false,
1267 /*isVolatile=*/is_volatile: false, /*isNontemporal=*/is_nontemporal: false,
1268 /*alignment=*/mlir::IntegerAttr(),
1269 sync_scope: cir::SyncScopeKindAttr(), mem_order: cir::MemOrderAttr(),
1270 /*invariant=*/false);
1271 returnValues.push_back(Elt: loaded);
1272 }
1273 }
1274 }
1275
1276 // Create the appropriate terminator for an exit operation in the dispatch
1277 // block. For return ops with operands, this handles the dominance issue by
1278 // either moving the operand's defining op to the dispatch block (if it's a
1279 // trivial use) or by storing to a temporary alloca and loading it.
1280 mlir::LogicalResult
1281 createExitTerminator(mlir::Operation *exitOp, mlir::Location loc,
1282 mlir::Block *continueBlock,
1283 mlir::PatternRewriter &rewriter) const {
1284 return llvm::TypeSwitch<mlir::Operation *, mlir::LogicalResult>(exitOp)
1285 .Case<cir::YieldOp>(caseFn: [&](auto) {
1286 // Yield becomes a branch to continue block.
1287 cir::BrOp::create(builder&: rewriter, location: loc, dest: continueBlock);
1288 return mlir::success();
1289 })
1290 .Case<cir::BreakOp>(caseFn: [&](auto) {
1291 // Break is preserved for later lowering by enclosing switch/loop.
1292 cir::BreakOp::create(builder&: rewriter, location: loc);
1293 return mlir::success();
1294 })
1295 .Case<cir::ContinueOp>(caseFn: [&](auto) {
1296 // Continue is preserved for later lowering by enclosing loop.
1297 cir::ContinueOp::create(builder&: rewriter, location: loc);
1298 return mlir::success();
1299 })
1300 .Case<cir::CoReturnOp>(caseFn: [&](auto) {
1301 // CoReturnOp does not carry a destination operand. The continuation
1302 // block determines whether execution proceeds to another cleanup or
1303 // to the coroutine's final suspend path.
1304 cir::CoReturnOp::create(builder&: rewriter, location: loc);
1305 return mlir::success();
1306 })
1307 .Case<cir::CoroCleanupOp>(caseFn: [&](auto) {
1308 // Like cir.co_return, it keeps propagating through the enclosing
1309 // cleanup scopes until CIRCoroutineFlattening rewrites it.
1310 cir::CoroCleanupOp::create(builder&: rewriter, location: loc);
1311 return mlir::success();
1312 })
1313 .Case<cir::ReturnOp>(caseFn: [&](auto returnOp) {
1314 // Return from the cleanup exit. Note, if this is a return inside a
1315 // nested cleanup scope, the flattening of the outer scope will handle
1316 // branching through the outer cleanup.
1317 if (returnOp.hasOperand()) {
1318 llvm::SmallVector<mlir::Value, 2> returnValues;
1319 getReturnOpOperands(returnOp, exitOp, loc, rewriter, returnValues);
1320 cir::ReturnOp::create(builder&: rewriter, location: loc, input: returnValues);
1321 } else {
1322 cir::ReturnOp::create(builder&: rewriter, location: loc);
1323 }
1324 return mlir::success();
1325 })
1326 .Case<cir::GotoOp>(caseFn: [&](auto gotoOp) {
1327 // Gotos that target a label within the cleanup body region are
1328 // filtered out by collectExits and never reach this code, so any
1329 // goto that does reach here transfers control out of the cleanup
1330 // scope. The goto is just moved to the exit block.
1331 cir::GotoOp::create(rewriter, loc, gotoOp.getLabel());
1332 return mlir::success();
1333 })
1334 .Default(defaultFn: [&](mlir::Operation *op) {
1335 cir::UnreachableOp::create(builder&: rewriter, location: loc);
1336 return op->emitError(
1337 message: "unexpected exit operation in cleanup scope body");
1338 });
1339 }
1340
1341#ifndef NDEBUG
1342 // Check that no block other than the last one in a region exits the region.
1343 static bool regionExitsOnlyFromLastBlock(mlir::Region &region) {
1344 for (mlir::Block &block : region) {
1345 if (&block == &region.back())
1346 continue;
1347 bool expectedTerminator =
1348 llvm::TypeSwitch<mlir::Operation *, bool>(block.getTerminator())
1349 // It is theoretically possible to have a cleanup block with
1350 // any of the following exits in non-final blocks, but we won't
1351 // currently generate any CIR that does that, and being able to
1352 // assume that it doesn't happen simplifies the implementation.
1353 // If we ever need to handle this case, the code will need to
1354 // be updated to handle it.
1355 .Case<cir::YieldOp, cir::ReturnOp, cir::ResumeFlatOp,
1356 cir::ContinueOp, cir::BreakOp, cir::GotoOp>(
1357 [](auto) { return false; })
1358 // We expect that call operations have not yet been rewritten
1359 // as try_call operations. A call can unwind out of the cleanup
1360 // scope, but we will be handling that during flattening. The
1361 // only case where a try_call could be present inside an
1362 // unflattened cleanup region is if the cleanup contained a
1363 // nested try-catch region, and that isn't expected as of the
1364 // time of this implementation. If it does, this could be
1365 // updated to tolerate it.
1366 .Case<cir::TryCallOp>([](auto) { return false; })
1367 // Likewise, we don't expect to find an EH dispatch operation
1368 // because we weren't expecting try-catch regions nested in the
1369 // cleanup region.
1370 .Case<cir::EhDispatchOp>([](auto) { return false; })
1371 // In theory, it would be possible to have a flattened switch
1372 // operation that does not exit the cleanup region. For now,
1373 // that's not happening.
1374 .Case<cir::SwitchFlatOp>([](auto) { return false; })
1375 // These aren't expected either, but if they occur, they don't
1376 // exit the region, so that's OK.
1377 .Case<cir::UnreachableOp, cir::TrapOp>([](auto) { return true; })
1378 // Indirect branches are not expected.
1379 .Case<cir::IndirectBrOp>([](auto) { return false; })
1380 // We do expect branches, but we don't expect them to leave
1381 // the region.
1382 .Case<cir::BrOp>([&](cir::BrOp brOp) {
1383 assert(brOp.getDest()->getParent() == &region &&
1384 "branch destination is not in the region");
1385 return true;
1386 })
1387 .Case<cir::BrCondOp>([&](cir::BrCondOp brCondOp) {
1388 assert(brCondOp.getDestTrue()->getParent() == &region &&
1389 "branch destination is not in the region");
1390 assert(brCondOp.getDestFalse()->getParent() == &region &&
1391 "branch destination is not in the region");
1392 return true;
1393 })
1394 // What else could there be?
1395 .Default([](mlir::Operation *) -> bool {
1396 llvm_unreachable("unexpected terminator in cleanup region");
1397 });
1398 if (!expectedTerminator)
1399 return false;
1400 }
1401 return true;
1402 }
1403#endif
1404
1405 // Build the EH cleanup block structure by cloning the cleanup region. The
1406 // cloned entry block gets an !cir.eh_token argument and a cir.begin_cleanup
1407 // inserted at the top. All cir.yield terminators that might exit the cleanup
1408 // region are replaced with cir.end_cleanup + cir.resume.
1409 //
1410 // For a single-block cleanup region, this produces:
1411 //
1412 // ^eh_cleanup(%eh_token : !cir.eh_token):
1413 // %ct = cir.begin_cleanup %eh_token : !cir.eh_token -> !cir.cleanup_token
1414 // <cloned cleanup operations>
1415 // cir.end_cleanup %ct : !cir.cleanup_token
1416 // cir.resume %eh_token : !cir.eh_token
1417 //
1418 // For a multi-block cleanup region (e.g. containing a flattened cir.if),
1419 // the same wrapping is applied around the cloned block structure: the entry
1420 // block gets begin_cleanup and all exit blocks (those terminated by yield)
1421 // get end_cleanup + resume.
1422 //
1423 // If this cleanup scope is nested within a TryOp, the resume will be updated
1424 // to branch to the catch dispatch block of the enclosing try operation when
1425 // the TryOp is flattened.
1426 mlir::Block *buildEHCleanupBlocks(cir::CleanupScopeOp cleanupOp,
1427 mlir::Location loc,
1428 mlir::Block *insertBefore,
1429 mlir::PatternRewriter &rewriter) const {
1430 assert(regionExitsOnlyFromLastBlock(cleanupOp.getCleanupRegion()) &&
1431 "cleanup region has exits in non-final blocks");
1432
1433 // Track the block before the insertion point so we can find the cloned
1434 // blocks after cloning.
1435 mlir::Block *blockBeforeClone = insertBefore->getPrevNode();
1436
1437 // Clone the entire cleanup region before insertBefore.
1438 rewriter.cloneRegionBefore(region&: cleanupOp.getCleanupRegion(), before: insertBefore);
1439
1440 // Find the first cloned block.
1441 mlir::Block *clonedEntry = blockBeforeClone
1442 ? blockBeforeClone->getNextNode()
1443 : &insertBefore->getParent()->front();
1444
1445 // Add the eh_token argument to the cloned entry block and insert
1446 // begin_cleanup at the top.
1447 auto ehTokenType = cir::EhTokenType::get(ctx: rewriter.getContext());
1448 mlir::Value ehToken = clonedEntry->addArgument(type: ehTokenType, loc);
1449
1450 rewriter.setInsertionPointToStart(clonedEntry);
1451 auto beginCleanup = cir::BeginCleanupOp::create(builder&: rewriter, location: loc, eh_token: ehToken);
1452
1453 // Replace the yield terminator in the last cloned block with
1454 // end_cleanup + resume.
1455 mlir::Block *lastClonedBlock = insertBefore->getPrevNode();
1456 auto yieldOp =
1457 mlir::dyn_cast<cir::YieldOp>(Val: lastClonedBlock->getTerminator());
1458 if (yieldOp) {
1459 rewriter.setInsertionPoint(yieldOp);
1460 cir::EndCleanupOp::create(builder&: rewriter, location: loc, cleanup_token: beginCleanup.getCleanupToken());
1461 rewriter.replaceOpWithNewOp<cir::ResumeOp>(op: yieldOp, args&: ehToken);
1462 } else {
1463 cleanupOp->emitError(message: "Not yet implemented: cleanup region terminated "
1464 "with non-yield operation");
1465 }
1466
1467 return clonedEntry;
1468 }
1469
1470 // Flatten a cleanup scope. The body region's exits branch to the cleanup
1471 // block, and the cleanup block branches to destination blocks whose contents
1472 // depend on the type of operation that exited the body region. Yield becomes
1473 // a branch to the block after the cleanup scope, break and continue are
1474 // preserved for later lowering by enclosing switch or loop, and return
1475 // is preserved as is.
1476 //
1477 // If there are multiple exits from the cleanup body, a destination slot and
1478 // switch dispatch are used to continue to the correct destination after the
1479 // cleanup is complete. A destination slot alloca is created at the function
1480 // entry block. Each exit operation is replaced by a store of its unique ID to
1481 // the destination slot and a branch to cleanup. An operation is appended to
1482 // the to branch to a dispatch block that loads the destination slot and uses
1483 // switch.flat to branch to the correct destination.
1484 //
1485 // If the cleanup scope requires EH cleanup, any call operations in the body
1486 // that may throw are replaced with cir.try_call operations that unwind to an
1487 // EH cleanup block. The cleanup block(s) will be terminated with a cir.resume
1488 // operation. If this cleanup scope is enclosed by a try operation, the
1489 // flattening of the try operation flattening will replace the cir.resume with
1490 // a branch to a catch dispatch block. Otherwise, the cir.resume operation
1491 // remains in place and will unwind to the caller.
1492 mlir::LogicalResult
1493 flattenCleanup(cir::CleanupScopeOp cleanupOp,
1494 llvm::SmallVectorImpl<CleanupExit> &exits,
1495 llvm::SmallVectorImpl<cir::CallOp> &callsToRewrite,
1496 llvm::SmallVectorImpl<cir::ThrowOp> &throwsToRewrite,
1497 llvm::SmallVectorImpl<cir::ResumeOp> &resumeOpsToChain,
1498 mlir::PatternRewriter &rewriter) const {
1499 mlir::Location loc = cleanupOp.getLoc();
1500 cir::CleanupKindAttr cleanupKind = cleanupOp.getCleanupKindAttr();
1501 bool hasNormalCleanup = cleanupKind.isNormal();
1502 bool hasEHCleanup = cleanupKind.isEH();
1503 bool isMultiExit = exits.size() > 1;
1504
1505 // If every exit from the body is a musttail return, nothing reaches the
1506 // normal cleanup, so it is dropped rather than emitted as dead code. The
1507 // EH path is still built. The only cleanup a musttail call may skip and
1508 // still unwind through is a lifetime marker, which is pushed as an "all"
1509 // cleanup, and the body can throw before reaching the tail call.
1510 bool emitNormalCleanup = hasNormalCleanup && !exits.empty();
1511
1512 // Get references to region blocks before inlining.
1513 mlir::Block *bodyEntry = &cleanupOp.getBodyRegion().front();
1514 mlir::Block *cleanupEntry = &cleanupOp.getCleanupRegion().front();
1515 mlir::Block *cleanupExit = &cleanupOp.getCleanupRegion().back();
1516 assert(regionExitsOnlyFromLastBlock(cleanupOp.getCleanupRegion()) &&
1517 "cleanup region has exits in non-final blocks");
1518 auto cleanupYield = dyn_cast<cir::YieldOp>(Val: cleanupExit->getTerminator());
1519 if (!cleanupYield) {
1520 return rewriter.notifyMatchFailure(arg&: cleanupOp,
1521 msg: "Not yet implemented: cleanup region "
1522 "terminated with non-yield operation");
1523 }
1524
1525 // For multiple exits from the body region, get or create a destination slot
1526 // at function entry. The slot is shared across all cleanup scopes in the
1527 // function. This is only needed if the cleanup scope requires normal
1528 // cleanup.
1529 cir::AllocaOp destSlot;
1530 if (isMultiExit && emitNormalCleanup) {
1531 auto funcOp = cleanupOp->getParentOfType<cir::FuncOp>();
1532 if (!funcOp)
1533 return cleanupOp->emitError(message: "cleanup scope not inside a function");
1534 destSlot = getOrCreateCleanupDestSlot(funcOp, rewriter, loc);
1535 }
1536
1537 // Split the current block to create the insertion point.
1538 mlir::Block *currentBlock = rewriter.getInsertionBlock();
1539 mlir::Block *continueBlock =
1540 rewriter.splitBlock(block: currentBlock, before: rewriter.getInsertionPoint());
1541
1542 // Build EH cleanup blocks if needed. This must be done before inlining
1543 // the cleanup region since buildEHCleanupBlocks clones from it. The unwind
1544 // block is inserted before the EH cleanup entry so that the final layout
1545 // is: body -> normal cleanup -> exit -> unwind -> EH cleanup -> continue.
1546 // EH cleanup blocks are needed when there are throwing calls or throws
1547 // that need to be rewritten, or when there are resume ops from
1548 // already-flattened inner cleanup scopes that need to chain through this
1549 // cleanup's EH handler.
1550 mlir::Block *unwindBlock = nullptr;
1551 mlir::Block *ehCleanupEntry = nullptr;
1552 if (hasEHCleanup && (!callsToRewrite.empty() || !throwsToRewrite.empty() ||
1553 !resumeOpsToChain.empty())) {
1554 ehCleanupEntry =
1555 buildEHCleanupBlocks(cleanupOp, loc, insertBefore: continueBlock, rewriter);
1556 // The unwind block is only needed when there are throwing calls or
1557 // throws that need a shared unwind destination. Resume ops from inner
1558 // cleanups branch directly to the EH cleanup entry.
1559 if (!callsToRewrite.empty() || !throwsToRewrite.empty())
1560 unwindBlock = buildUnwindBlock(dest: ehCleanupEntry, /*isCleanupOnly=*/true,
1561 loc, insertBefore: ehCleanupEntry, rewriter);
1562 }
1563
1564 // All normal flow blocks are inserted before this point — either before
1565 // the unwind block (if it exists), or before the EH cleanup entry (if EH
1566 // cleanup exists but no unwind block is needed), or before the continue
1567 // block.
1568 mlir::Block *normalInsertPt =
1569 unwindBlock ? unwindBlock
1570 : (ehCleanupEntry ? ehCleanupEntry : continueBlock);
1571
1572 // Inline the body region.
1573 rewriter.inlineRegionBefore(region&: cleanupOp.getBodyRegion(), before: normalInsertPt);
1574
1575 // Inline the cleanup region for the normal cleanup path.
1576 if (emitNormalCleanup)
1577 rewriter.inlineRegionBefore(region&: cleanupOp.getCleanupRegion(), before: normalInsertPt);
1578
1579 // Branch from current block to body entry.
1580 rewriter.setInsertionPointToEnd(currentBlock);
1581 cir::BrOp::create(builder&: rewriter, location: loc, dest: bodyEntry);
1582
1583 // Handle normal exits.
1584 mlir::LogicalResult result = mlir::success();
1585 if (emitNormalCleanup) {
1586 // Create the exit/dispatch block (after cleanup, before continue).
1587 mlir::Block *exitBlock = rewriter.createBlock(insertBefore: normalInsertPt);
1588
1589 // Rewrite the cleanup region's yield to branch to exit block.
1590 rewriter.setInsertionPoint(cleanupYield);
1591 rewriter.replaceOpWithNewOp<cir::BrOp>(op: cleanupYield, args&: exitBlock);
1592
1593 if (isMultiExit) {
1594 // Build the dispatch switch in the exit block.
1595 rewriter.setInsertionPointToEnd(exitBlock);
1596
1597 // Load the destination slot value.
1598 auto slotValue =
1599 cir::LoadOp::create(builder&: rewriter, location: loc, addr: destSlot, /*isDeref=*/false,
1600 /*isVolatile=*/is_volatile: false, /*isNontemporal=*/is_nontemporal: false,
1601 /*alignment=*/mlir::IntegerAttr(),
1602 sync_scope: cir::SyncScopeKindAttr(), mem_order: cir::MemOrderAttr(),
1603 /*invariant=*/false);
1604
1605 // Create destination blocks for each exit and collect switch case info.
1606 llvm::SmallVector<mlir::APInt, 8> caseValues;
1607 llvm::SmallVector<mlir::Block *, 8> caseDestinations;
1608 llvm::SmallVector<mlir::ValueRange, 8> caseOperands;
1609 cir::IntType s32Type =
1610 cir::IntType::get(context: rewriter.getContext(), width: 32, /*isSigned=*/is_signed: true);
1611
1612 for (const CleanupExit &exit : exits) {
1613 // Create a block for this destination.
1614 mlir::Block *destBlock = rewriter.createBlock(insertBefore: normalInsertPt);
1615 rewriter.setInsertionPointToEnd(destBlock);
1616 result =
1617 createExitTerminator(exitOp: exit.exitOp, loc, continueBlock, rewriter);
1618
1619 // Add to switch cases.
1620 caseValues.push_back(
1621 Elt: llvm::APInt(32, static_cast<uint64_t>(exit.destinationId), true));
1622 caseDestinations.push_back(Elt: destBlock);
1623 caseOperands.push_back(Elt: mlir::ValueRange());
1624
1625 // Replace the original exit op with: store dest ID, branch to
1626 // cleanup.
1627 rewriter.setInsertionPoint(exit.exitOp);
1628 auto destIdConst = cir::ConstantOp::create(
1629 builder&: rewriter, location: loc, value: cir::IntAttr::get(type: s32Type, value: exit.destinationId));
1630 cir::StoreOp::create(builder&: rewriter, location: loc, value: destIdConst, addr: destSlot,
1631 /*isVolatile=*/is_volatile: false,
1632 /*isNontemporal=*/is_nontemporal: false,
1633 /*alignment=*/mlir::IntegerAttr(),
1634 sync_scope: cir::SyncScopeKindAttr(), mem_order: cir::MemOrderAttr());
1635 rewriter.replaceOpWithNewOp<cir::BrOp>(op: exit.exitOp, args&: cleanupEntry);
1636
1637 // If the exit terminator creation failed, we're going to end up with
1638 // partially flattened code, but we'll also have reported an error so
1639 // that's OK. We need to finish out this function to keep the IR in a
1640 // valid state to help diagnose the error. This is a temporary
1641 // possibility during development. It shouldn't ever happen after the
1642 // implementation is complete.
1643 if (result.failed())
1644 break;
1645 }
1646
1647 // Create the default destination (unreachable).
1648 mlir::Block *defaultBlock = rewriter.createBlock(insertBefore: normalInsertPt);
1649 rewriter.setInsertionPointToEnd(defaultBlock);
1650 cir::UnreachableOp::create(builder&: rewriter, location: loc);
1651
1652 // Build the switch.flat operation in the exit block.
1653 rewriter.setInsertionPointToEnd(exitBlock);
1654 cir::SwitchFlatOp::create(builder&: rewriter, location: loc, condition: slotValue, defaultDestination: defaultBlock,
1655 defaultOperands: mlir::ValueRange(), caseValues,
1656 caseDestinations, caseOperands);
1657 } else {
1658 // Single exit: put the appropriate terminator directly in the exit
1659 // block.
1660 rewriter.setInsertionPointToEnd(exitBlock);
1661 mlir::Operation *exitOp = exits[0].exitOp;
1662 result = createExitTerminator(exitOp, loc, continueBlock, rewriter);
1663
1664 // Replace body exit with branch to cleanup entry.
1665 rewriter.setInsertionPoint(exitOp);
1666 rewriter.replaceOpWithNewOp<cir::BrOp>(op: exitOp, args&: cleanupEntry);
1667 }
1668 } else {
1669 // No normal cleanup to run: normal exits skip the cleanup entirely.
1670 // Replace yield exits with branches to the continue block.
1671 for (CleanupExit &exit : exits) {
1672 if (isa<cir::YieldOp>(Val: exit.exitOp)) {
1673 rewriter.setInsertionPoint(exit.exitOp);
1674 rewriter.replaceOpWithNewOp<cir::BrOp>(op: exit.exitOp, args&: continueBlock);
1675 }
1676 // Non-yield exits (break, continue, return) stay as-is since no normal
1677 // cleanup is needed.
1678 }
1679 }
1680
1681 // Replace non-nothrow calls and throws with try_call/try_throw
1682 // operations. All calls and throws within this cleanup scope share the
1683 // same unwind destination.
1684 if (hasEHCleanup) {
1685 for (cir::CallOp callOp : callsToRewrite)
1686 replaceCallWithTryCall(callOp, unwindDest: unwindBlock, loc, rewriter);
1687 for (cir::ThrowOp throwOp : throwsToRewrite)
1688 replaceThrowWithTryThrow(throwOp, unwindDest: unwindBlock, loc, rewriter);
1689 }
1690
1691 // Handle throwing calls and throws in EH cleanup blocks. When an
1692 // exception is thrown during cleanup code that runs on the exception
1693 // unwind path, the C++ standard requires that std::terminate() be
1694 // called. Replace such calls and throws with try_call/try_throw
1695 // operations that unwind to a terminate block containing
1696 // cir.eh.initiate + cir.eh.terminate.
1697 if (ehCleanupEntry) {
1698 llvm::SmallVector<cir::CallOp> ehCleanupThrowingCalls;
1699 llvm::SmallVector<cir::ThrowOp> ehCleanupThrows;
1700 for (mlir::Block *block = ehCleanupEntry; block != continueBlock;
1701 block = block->getNextNode()) {
1702 block->walk(callback: [&](mlir::Operation *op) {
1703 if (auto callOp = mlir::dyn_cast<cir::CallOp>(Val: op)) {
1704 if (!callOp.getNothrow())
1705 ehCleanupThrowingCalls.push_back(Elt: callOp);
1706 } else if (auto throwOp = mlir::dyn_cast<cir::ThrowOp>(Val: op)) {
1707 ehCleanupThrows.push_back(Elt: throwOp);
1708 }
1709 });
1710 }
1711 if (!ehCleanupThrowingCalls.empty() || !ehCleanupThrows.empty()) {
1712 mlir::Block *terminateBlock =
1713 buildTerminateUnwindBlock(loc, insertBefore: continueBlock, rewriter);
1714 for (cir::CallOp callOp : ehCleanupThrowingCalls)
1715 replaceCallWithTryCall(callOp, unwindDest: terminateBlock, loc, rewriter);
1716 for (cir::ThrowOp throwOp : ehCleanupThrows)
1717 replaceThrowWithTryThrow(throwOp, unwindDest: terminateBlock, loc, rewriter);
1718 }
1719 }
1720
1721 // Chain inner EH cleanup resume ops to this cleanup's EH handler.
1722 // Each cir.resume from an already-flattened inner cleanup is replaced
1723 // with a branch to the outer EH cleanup entry, passing the eh_token
1724 // from the inner's begin_cleanup so that the same in-flight exception
1725 // flows through the outer cleanup before unwinding to the caller.
1726 if (ehCleanupEntry) {
1727 for (cir::ResumeOp resumeOp : resumeOpsToChain) {
1728 mlir::Value ehToken = resumeOp.getEhToken();
1729 rewriter.setInsertionPoint(resumeOp);
1730 rewriter.replaceOpWithNewOp<cir::BrOp>(
1731 op: resumeOp, args: mlir::ValueRange{ehToken}, /*loop_id=*/args: nullptr,
1732 /*loop_hints=*/args: nullptr, args&: ehCleanupEntry);
1733 }
1734 }
1735
1736 // Erase the original cleanup scope op.
1737 rewriter.eraseOp(op: cleanupOp);
1738
1739 // Always return success because the IR has been modified (blocks split,
1740 // regions inlined, ops erased, etc.). The MLIR pattern rewriter contract
1741 // requires that if a pattern modifies IR, it must return success().
1742 return mlir::success();
1743 }
1744
1745 mlir::LogicalResult
1746 matchAndRewrite(cir::CleanupScopeOp cleanupOp,
1747 mlir::PatternRewriter &rewriter) const override {
1748 mlir::OpBuilder::InsertionGuard guard(rewriter);
1749
1750 // All nested structured CIR ops must be flattened before the cleanup scope.
1751 // Operations like loops, switches, scopes, and ifs may contain exits
1752 // (return, break, continue) that the cleanup scope will replace with
1753 // branches to the cleanup entry. If those exits are inside a structured
1754 // op's region, the branch would reference a block outside that region,
1755 // which is invalid. Fail the match so they are processed first.
1756 //
1757 // Before checking, erase any trivially dead nested cleanup scopes. These
1758 // arise from deactivated cleanups (e.g. partial-construction guards for
1759 // lambda captures). The greedy rewriter may have already DCE'd them, but
1760 // when a trivially dead nested op is erased first, the parent isn't always
1761 // re-added to the worklist, so we handle it here.
1762 llvm::SmallVector<cir::CleanupScopeOp> deadNestedOps;
1763 cleanupOp.getBodyRegion().walk(callback: [&](cir::CleanupScopeOp nested) {
1764 if (mlir::isOpTriviallyDead(op: nested))
1765 deadNestedOps.push_back(Elt: nested);
1766 });
1767 for (auto op : deadNestedOps)
1768 rewriter.eraseOp(op);
1769
1770 if (hasNestedOpsToFlatten(region&: cleanupOp.getBodyRegion()))
1771 return mlir::failure();
1772
1773 bool hasEHCleanup = cleanupOp.getCleanupKindAttr().isEH();
1774
1775 // Collect all exits from the body region.
1776 llvm::SmallVector<CleanupExit> exits;
1777 int nextId = 0;
1778 collectExits(cleanupBodyRegion&: cleanupOp.getBodyRegion(), exits, nextId);
1779
1780#ifndef NDEBUG
1781 // A body with no exits at all only happens when every path out of it is a
1782 // musttail return, which leaves the function without running the cleanup.
1783 bool hasMustTailCall = false;
1784 cleanupOp.getBodyRegion().walk(
1785 [&](cir::CallOp callOp) { hasMustTailCall |= callOp.getMusttail(); });
1786 assert((!exits.empty() || hasMustTailCall) &&
1787 "cleanup scope body has no exit");
1788#endif
1789
1790 // Collect non-nothrow calls and throws that need to be converted to
1791 // try_call/try_throw. A marker-only cleanup must not introduce an unwind
1792 // edge unless an enclosing handler or real EH cleanup requires one.
1793 llvm::SmallVector<cir::CallOp> callsToRewrite;
1794 llvm::SmallVector<cir::ThrowOp> throwsToRewrite;
1795 if (hasEHCleanup && (!isLifetimeMarkerOnly(region&: cleanupOp.getCleanupRegion()) ||
1796 hasEnclosingEHRequirement(op: cleanupOp))) {
1797 collectThrowingCalls(region&: cleanupOp.getBodyRegion(), callsToRewrite);
1798 collectThrows(region&: cleanupOp.getBodyRegion(), throwsToRewrite);
1799 }
1800
1801 // Collect resume ops from already-flattened inner cleanup scopes that
1802 // need to chain through this cleanup's EH handler, including lifetime
1803 // markers even when they did not introduce any unwind edges themselves.
1804 llvm::SmallVector<cir::ResumeOp> resumeOpsToChain;
1805 if (hasEHCleanup)
1806 collectResumeOps(region&: cleanupOp.getBodyRegion(), resumeOps&: resumeOpsToChain);
1807
1808 return flattenCleanup(cleanupOp, exits, callsToRewrite, throwsToRewrite,
1809 resumeOpsToChain, rewriter);
1810 }
1811};
1812
1813// Trace an !cir.eh_token value back through block arguments to find the
1814// cir.eh.initiate operation that defines it. Returns {} if the defining op
1815// cannot be found (e.g. multiple predecessors).
1816static cir::EhInitiateOp traceToEhInitiate(mlir::Value ehToken) {
1817 while (ehToken) {
1818 if (auto initiate = ehToken.getDefiningOp<cir::EhInitiateOp>())
1819 return initiate;
1820 auto blockArg = mlir::dyn_cast<mlir::BlockArgument>(Val&: ehToken);
1821 if (!blockArg)
1822 return {};
1823 mlir::Block *pred = blockArg.getOwner()->getSinglePredecessor();
1824 if (!pred)
1825 return {};
1826 auto brOp = mlir::dyn_cast<cir::BrOp>(Val: pred->getTerminator());
1827 if (!brOp)
1828 return {};
1829 ehToken = brOp.getDestOperands()[blockArg.getArgNumber()];
1830 }
1831 return {};
1832}
1833
1834class CIRTryOpFlattening : public mlir::OpRewritePattern<cir::TryOp> {
1835public:
1836 using OpRewritePattern<cir::TryOp>::OpRewritePattern;
1837
1838 // Build the catch dispatch block with a cir.eh.dispatch operation.
1839 // The dispatch block receives an !cir.eh_token argument and dispatches
1840 // to the appropriate catch handler blocks based on exception types.
1841 mlir::Block *buildCatchDispatchBlock(
1842 cir::TryOp tryOp, mlir::ArrayAttr handlerTypes,
1843 llvm::SmallVectorImpl<mlir::Block *> &catchHandlerBlocks,
1844 mlir::Location loc, mlir::Block *insertBefore,
1845 mlir::PatternRewriter &rewriter) const {
1846 mlir::Block *dispatchBlock = rewriter.createBlock(insertBefore);
1847 auto ehTokenType = cir::EhTokenType::get(ctx: rewriter.getContext());
1848 mlir::Value ehToken = dispatchBlock->addArgument(type: ehTokenType, loc);
1849
1850 rewriter.setInsertionPointToEnd(dispatchBlock);
1851
1852 // Build the catch types and destinations for the dispatch.
1853 llvm::SmallVector<mlir::Attribute> catchTypeAttrs;
1854 llvm::SmallVector<mlir::Block *> catchDests;
1855 mlir::Block *defaultDest = nullptr;
1856 bool defaultIsCatchAll = false;
1857
1858 // A filter try operation has exactly two handlers. The unexpected region
1859 // is the destination of the filter clause (taken when the exception is
1860 // *not* permitted). The filter region is the unwind destination (taken
1861 // when it is).
1862 cir::EhFilterAttr filterAttr;
1863 mlir::Block *filterUnwindDest = nullptr;
1864 mlir::Block *filterClauseDest = nullptr;
1865
1866 for (auto zipped : llvm::zip(t&: handlerTypes, u&: catchHandlerBlocks)) {
1867 mlir::Attribute typeAttr = std::get<0>(t&: zipped);
1868 mlir::Block *handlerBlock = std::get<1>(t&: zipped);
1869 llvm::TypeSwitch<mlir::Attribute>(typeAttr)
1870 .Case<cir::CatchAllAttr>(caseFn: [&](auto) {
1871 assert(!defaultDest && "multiple catch_all or unwind handlers");
1872 defaultDest = handlerBlock;
1873 defaultIsCatchAll = true;
1874 })
1875 .Case<cir::UnwindAttr>(caseFn: [&](auto) {
1876 assert(!defaultDest && "multiple catch_all or unwind handlers");
1877 defaultDest = handlerBlock;
1878 defaultIsCatchAll = false;
1879 })
1880 .Case<cir::EhFilterAttr>(caseFn: [&](cir::EhFilterAttr ehFilter) {
1881 assert(!filterAttr && "multiple filter handlers");
1882 filterAttr = ehFilter;
1883 filterUnwindDest = handlerBlock;
1884 })
1885 .Case<cir::EhUnexpectedAttr>(caseFn: [&](auto) {
1886 assert(!filterClauseDest && "multiple unexpected handlers");
1887 filterClauseDest = handlerBlock;
1888 })
1889 .Default(defaultFn: [&](mlir::Attribute attr) {
1890 // Typed catch handler (GlobalViewAttr with type info).
1891 catchTypeAttrs.push_back(Elt: attr);
1892 catchDests.push_back(Elt: handlerBlock);
1893 });
1894 }
1895
1896 if (filterAttr) {
1897 assert(filterUnwindDest && filterClauseDest &&
1898 "filter handler requires an unexpected handler");
1899 assert(!defaultDest &&
1900 "filter handler cannot share a dispatch with catch_all or unwind");
1901 catchTypeAttrs.push_back(Elt: filterAttr);
1902 catchDests.push_back(Elt: filterClauseDest);
1903 defaultDest = filterUnwindDest;
1904 defaultIsCatchAll = false;
1905 }
1906
1907 assert(defaultDest && "dispatch must have a catch_all or unwind handler");
1908
1909 mlir::ArrayAttr catchTypesArrayAttr;
1910 if (!catchTypeAttrs.empty())
1911 catchTypesArrayAttr = rewriter.getArrayAttr(value: catchTypeAttrs);
1912
1913 cir::EhDispatchOp::create(builder&: rewriter, location: loc, eh_token: ehToken, catch_types: catchTypesArrayAttr,
1914 default_is_catch_all: defaultIsCatchAll, default_destination: defaultDest, catch_destinations: catchDests);
1915
1916 return dispatchBlock;
1917 }
1918
1919 // Flatten a single catch handler region. Each handler region has an
1920 // !cir.eh_token argument and starts with cir.begin_catch, followed by
1921 // a cir.cleanup.scope containing the handler body (with cir.end_catch in
1922 // its cleanup region), and ending with cir.yield.
1923 //
1924 // After flattening, the handler region becomes a block that receives the
1925 // eh_token, calls begin_catch, runs the handler body inline, calls
1926 // end_catch, and branches to the continue block.
1927 //
1928 // The cleanup scope inside the catch handler is expected to have been
1929 // flattened before we get here, so what we see in the handler region is
1930 // already flat code with begin_catch at the top and end_catch in any place
1931 // that we would exit the catch handler. We just need to inline the region
1932 // and fix up terminators.
1933 mlir::Block *flattenCatchHandler(mlir::Region &handlerRegion,
1934 mlir::Block *continueBlock,
1935 mlir::Location loc,
1936 mlir::Block *insertBefore,
1937 mlir::PatternRewriter &rewriter) const {
1938 // The handler region entry block has the !cir.eh_token argument.
1939 mlir::Block *handlerEntry = &handlerRegion.front();
1940
1941 // Inline the handler region before insertBefore.
1942 rewriter.inlineRegionBefore(region&: handlerRegion, before: insertBefore);
1943
1944 // Replace yield terminators in the handler with branches to continue.
1945 for (mlir::Block &block : llvm::make_range(x: handlerEntry->getIterator(),
1946 y: insertBefore->getIterator())) {
1947 if (auto yieldOp = dyn_cast<cir::YieldOp>(Val: block.getTerminator())) {
1948 // Verify that end_catch is the last non-branch operation before
1949 // this yield. After cleanup scope flattening, end_catch may be
1950 // in a predecessor block rather than immediately before the yield.
1951 // Walk back through predecessors (including multi-predecessor
1952 // blocks), verifying that each intermediate block contains only a
1953 // branch terminator, until we find end_catch as the last
1954 // non-terminator in some block.
1955 // Verify that end_catch is reachable on some predecessor path
1956 // before this yield. After cleanup scope flattening, end_catch
1957 // may be separated from yield by conditional branches (e.g.,
1958 // from flattened cir.if inside the catch body).
1959 assert(([&]() {
1960 if (mlir::Operation *prev = yieldOp->getPrevNode())
1961 return isa<cir::EndCatchOp>(prev);
1962 llvm::SmallPtrSet<mlir::Block *, 8> visited;
1963 llvm::SmallVector<mlir::Block *, 4> worklist;
1964 for (mlir::Block *pred : block.getPredecessors())
1965 worklist.push_back(pred);
1966 while (!worklist.empty()) {
1967 mlir::Block *b = worklist.pop_back_val();
1968 if (!visited.insert(b).second)
1969 continue;
1970 mlir::Operation *term = b->getTerminator();
1971 if (mlir::Operation *prev = term->getPrevNode()) {
1972 if (isa<cir::EndCatchOp>(prev))
1973 return true;
1974 }
1975 for (mlir::Block *pred : b->getPredecessors())
1976 worklist.push_back(pred);
1977 }
1978 return false;
1979 }()) &&
1980 "expected end_catch reachable before yield "
1981 "in catch handler");
1982 rewriter.setInsertionPoint(yieldOp);
1983 rewriter.replaceOpWithNewOp<cir::BrOp>(op: yieldOp, args&: continueBlock);
1984 }
1985 }
1986
1987 return handlerEntry;
1988 }
1989
1990 // Flatten an unwind, filter, or unexpected handler region. These regions
1991 // do not catch the exception. They resume, are unreachable, or call
1992 // cir.eh.unexpected. We inline them and leave the terminator in place. If
1993 // this try op is nested inside an EH cleanup or another try op, the
1994 // enclosing op will rewrite a resume as a branch to its cleanup or dispatch
1995 // block when it is flattened. Otherwise, a resume will unwind to the caller.
1996 mlir::Block *flattenUnwindHandler(mlir::Region &unwindRegion,
1997 mlir::Location loc,
1998 mlir::Block *insertBefore,
1999 mlir::PatternRewriter &rewriter) const {
2000 mlir::Block *unwindEntry = &unwindRegion.front();
2001 rewriter.inlineRegionBefore(region&: unwindRegion, before: insertBefore);
2002 return unwindEntry;
2003 }
2004
2005 mlir::LogicalResult
2006 matchAndRewrite(cir::TryOp tryOp,
2007 mlir::PatternRewriter &rewriter) const override {
2008 // All nested structured CIR ops must be flattened before the try op.
2009 // Cleanup scopes and nested try ops need to be flat so EH cleanup is
2010 // properly handled. Other structured ops (scopes, ifs, loops, switches,
2011 // ternaries) must be flat because replaceCallWithTryCall creates try_call
2012 // ops whose unwind destination is outside the structured op's region,
2013 // which would be an invalid cross-region reference.
2014 for (mlir::Region &region : tryOp->getRegions())
2015 if (hasNestedOpsToFlatten(region))
2016 return mlir::failure();
2017
2018 mlir::OpBuilder::InsertionGuard guard(rewriter);
2019 mlir::Location loc = tryOp.getLoc();
2020
2021 mlir::ArrayAttr handlerTypes = tryOp.getHandlerTypesAttr();
2022 mlir::MutableArrayRef<mlir::Region> handlerRegions =
2023 tryOp.getHandlerRegions();
2024
2025 // Collect throwing calls and throws in the try body.
2026 llvm::SmallVector<cir::CallOp> callsToRewrite;
2027 collectThrowingCalls(region&: tryOp.getTryRegion(), callsToRewrite);
2028 llvm::SmallVector<cir::ThrowOp> throwsToRewrite;
2029 collectThrows(region&: tryOp.getTryRegion(), throwsToRewrite);
2030
2031 // Collect resume ops from already-flattened cleanup scopes in the try body.
2032 llvm::SmallVector<cir::ResumeOp> resumeOpsToChain;
2033 collectResumeOps(region&: tryOp.getTryRegion(), resumeOps&: resumeOpsToChain);
2034
2035 // Split the current block and inline the try body.
2036 mlir::Block *currentBlock = rewriter.getInsertionBlock();
2037 mlir::Block *continueBlock =
2038 rewriter.splitBlock(block: currentBlock, before: rewriter.getInsertionPoint());
2039
2040 // Get references to try body blocks before inlining.
2041 mlir::Block *bodyEntry = &tryOp.getTryRegion().front();
2042 mlir::Block *bodyExit = &tryOp.getTryRegion().back();
2043
2044 // Inline the try body region before the continue block.
2045 rewriter.inlineRegionBefore(region&: tryOp.getTryRegion(), before: continueBlock);
2046
2047 // Branch from the current block to the body entry.
2048 rewriter.setInsertionPointToEnd(currentBlock);
2049 cir::BrOp::create(builder&: rewriter, location: loc, dest: bodyEntry);
2050
2051 // Replace the try body's yield terminator with a branch to continue.
2052 if (auto bodyYield = dyn_cast<cir::YieldOp>(Val: bodyExit->getTerminator())) {
2053 rewriter.setInsertionPoint(bodyYield);
2054 rewriter.replaceOpWithNewOp<cir::BrOp>(op: bodyYield, args&: continueBlock);
2055 }
2056
2057 // If there are no handlers, we're done.
2058 if (!handlerTypes || handlerTypes.empty()) {
2059 rewriter.eraseOp(op: tryOp);
2060 return mlir::success();
2061 }
2062
2063 // If there are no throwing calls, no throws, and no resume ops from
2064 // inner cleanup scopes, exceptions cannot reach the catch handlers.
2065 // Drop all uses from the (unreachable) handler regions before erasing
2066 // the try op, since handler ops may reference values that were inlined
2067 // from the try body into the parent block.
2068 if (callsToRewrite.empty() && throwsToRewrite.empty() &&
2069 resumeOpsToChain.empty()) {
2070 for (mlir::Region &handlerRegion : handlerRegions)
2071 for (mlir::Block &block : handlerRegion)
2072 block.dropAllDefinedValueUses();
2073 rewriter.eraseOp(op: tryOp);
2074 return mlir::success();
2075 }
2076
2077 // Build the catch handler blocks.
2078
2079 // First, flatten all handler regions and collect the entry blocks.
2080 llvm::SmallVector<mlir::Block *> catchHandlerBlocks;
2081
2082 for (const auto &[idx, typeAttr] : llvm::enumerate(First&: handlerTypes)) {
2083 mlir::Region &handlerRegion = handlerRegions[idx];
2084
2085 if (mlir::isa<cir::UnwindAttr, cir::EhFilterAttr, cir::EhUnexpectedAttr>(
2086 Val: typeAttr)) {
2087 mlir::Block *unwindEntry =
2088 flattenUnwindHandler(unwindRegion&: handlerRegion, loc, insertBefore: continueBlock, rewriter);
2089 catchHandlerBlocks.push_back(Elt: unwindEntry);
2090 } else {
2091 mlir::Block *handlerEntry = flattenCatchHandler(
2092 handlerRegion, continueBlock, loc, insertBefore: continueBlock, rewriter);
2093 catchHandlerBlocks.push_back(Elt: handlerEntry);
2094 }
2095 }
2096
2097 // Build the catch dispatch block.
2098 mlir::Block *dispatchBlock =
2099 buildCatchDispatchBlock(tryOp, handlerTypes, catchHandlerBlocks, loc,
2100 insertBefore: catchHandlerBlocks.front(), rewriter);
2101
2102 // Check whether the try has a catch-all handler. When catch-all is
2103 // present, the personality function will always stop unwinding at this
2104 // frame (because catch-all matches every exception type). The LLVM
2105 // landingpad therefore needs "catch ptr null" rather than "cleanup".
2106 // The downstream pipeline (EHABILowering + LowerToLLVM) emits
2107 // "catch ptr null" when the EhInitiateOp has neither cleanup nor typed
2108 // catch types, so we clear the cleanup flag on every EhInitiateOp that
2109 // feeds into a dispatch with a catch-all handler.
2110 bool hasCatchAll =
2111 handlerTypes && llvm::any_of(Range&: handlerTypes, P: [](mlir::Attribute attr) {
2112 return mlir::isa<cir::CatchAllAttr>(Val: attr);
2113 });
2114
2115 // Build a block to be the unwind desination for throwing calls/throws
2116 // and replace the calls/throws with try_call/try_throw ops. Note that
2117 // the unwind block created here is something different than the unwind
2118 // handler that we may have created above. The unwind handler continues
2119 // unwinding after uncaught exceptions. This is the block that will
2120 // eventually become the landing pad for invoke instructions.
2121 bool isCleanupOnly = tryOp.getCleanup() && !hasCatchAll;
2122 if (!callsToRewrite.empty() || !throwsToRewrite.empty()) {
2123 // Create a shared unwind block for all throwing calls/throws.
2124 mlir::Block *unwindBlock = buildUnwindBlock(dest: dispatchBlock, isCleanupOnly,
2125 loc, insertBefore: dispatchBlock, rewriter);
2126
2127 for (cir::CallOp callOp : callsToRewrite)
2128 replaceCallWithTryCall(callOp, unwindDest: unwindBlock, loc, rewriter);
2129 for (cir::ThrowOp throwOp : throwsToRewrite)
2130 replaceThrowWithTryThrow(throwOp, unwindDest: unwindBlock, loc, rewriter);
2131 }
2132
2133 // Chain resume ops from inner cleanup scopes.
2134 // Resume ops from already-flattened cleanup scopes within the try body
2135 // should branch to the catch dispatch block instead of unwinding directly.
2136 for (cir::ResumeOp resumeOp : resumeOpsToChain) {
2137 // When there is a catch-all handler, clear the cleanup flag on the
2138 // cir.eh.initiate that produced this token. With catch-all, the LLVM
2139 // landingpad needs "catch ptr null" instead of "cleanup".
2140 if (hasCatchAll) {
2141 if (auto ehInitiate = traceToEhInitiate(ehToken: resumeOp.getEhToken())) {
2142 rewriter.modifyOpInPlace(root: ehInitiate,
2143 callable: [&] { ehInitiate.removeCleanupAttr(); });
2144 }
2145 }
2146
2147 mlir::Value ehToken = resumeOp.getEhToken();
2148 rewriter.setInsertionPoint(resumeOp);
2149 rewriter.replaceOpWithNewOp<cir::BrOp>(
2150 op: resumeOp, args: mlir::ValueRange{ehToken}, /*loop_id=*/args: nullptr,
2151 /*loop_hints=*/args: nullptr, args&: dispatchBlock);
2152 }
2153
2154 // Finally, erase the original try op ----
2155 rewriter.eraseOp(op: tryOp);
2156
2157 return mlir::success();
2158 }
2159};
2160
2161static void
2162collectCoReturnsOps(mlir::Region &region,
2163 llvm::SmallVectorImpl<cir::CoReturnOp> &coReturnOps) {
2164 region.walk(
2165 callback: [&](cir::CoReturnOp coReturnOp) { coReturnOps.push_back(Elt: coReturnOp); });
2166}
2167
2168// Flatten `cir.coroutine`. This runs after every nested op (awaits, cir.scope,
2169// cleanup scopes, ...) has been flattened. The regions are inlined in order,
2170// the same layout classic codegen produces:
2171//
2172// initialSuspend ---> body ---> finalSuspend ---> destroy ---> exit
2173// | ^ ^ ^
2174// +-------------+ | |
2175// cir.co_return | |
2176// await destroy case (cir.coro.cleanup) -------------+ |
2177// await suspended case (cir.coro.exit) -------------------------+
2178//
2179// Completing the final suspend and an await's destroy case both free the frame
2180// (the destroy region, cir.coro.intrinsic.free) and then continue to the exit
2181// region. A suspended coroutine (cir.coro.exit) must keep its frame, so it
2182// branches straight to the exit region.
2183class CIRCoroutineFlattening : public mlir::OpRewritePattern<cir::CoroutineOp> {
2184 // cir.coro.exit and cir.coro.cleanup are placeholders produced by flattening
2185 // a cir.await, which can only appear in the initialSuspend, body and
2186 // finalSuspend regions.
2187 template <class PlaceholderOp>
2188 void
2189 collectAwaitPlaceholders(cir::CoroutineOp coroOp,
2190 llvm::SmallVectorImpl<PlaceholderOp> &ops) const {
2191 auto collect = [&](PlaceholderOp op) { ops.push_back(op); };
2192 coroOp.getInitialSuspend().walk(collect);
2193 coroOp.getBody().walk(collect);
2194 coroOp.getFinalSuspend().walk(collect);
2195 }
2196
2197 // Replace the cir.yield that ends `region` with a branch to `dest`.
2198 void replaceYieldWithBr(mlir::PatternRewriter &rewriter, mlir::Region &region,
2199 mlir::Block *dest) const {
2200 auto yieldOp = cast<cir::YieldOp>(Val: region.back().getTerminator());
2201 rewriter.setInsertionPoint(yieldOp);
2202 rewriter.replaceOpWithNewOp<cir::BrOp>(op: yieldOp, args&: dest, args: yieldOp.getArgs());
2203 }
2204
2205 // initialSuspend ends in a cir.yield once the init await is done. Replace it
2206 // with a branch to the body.
2207 void inlineInitialSuspend(mlir::PatternRewriter &rewriter,
2208 cir::CoroutineOp coroOp, mlir::Block *insertBefore,
2209 mlir::Block *beforeBody) const {
2210 mlir::Region &initialRegion = coroOp.getInitialSuspend();
2211 replaceYieldWithBr(rewriter, region&: initialRegion, dest: beforeBody);
2212 rewriter.inlineRegionBefore(region&: initialRegion, before: insertBefore);
2213 }
2214
2215 // The body either falls off the end (cir.yield) or ends in a
2216 // cir.co_return. Both, like every other cir.co_return in the body, branch
2217 // to the final suspend.
2218 void inlineBody(mlir::PatternRewriter &rewriter, cir::CoroutineOp coroOp,
2219 mlir::Block *insertBefore, mlir::Block *beforeFinal) const {
2220 mlir::Region &bodyRegion = coroOp.getBody();
2221
2222 llvm::SmallVector<cir::CoReturnOp> coReturnsOps;
2223 collectCoReturnsOps(region&: bodyRegion, coReturnOps&: coReturnsOps);
2224
2225 mlir::Operation *bodyTerminator = bodyRegion.back().getTerminator();
2226 if (auto bodyYield = dyn_cast<cir::YieldOp>(Val: bodyTerminator)) {
2227 rewriter.setInsertionPoint(bodyYield);
2228 rewriter.replaceOpWithNewOp<cir::BrOp>(op: bodyYield, args&: beforeFinal,
2229 args: mlir::ValueRange{});
2230 }
2231
2232 for (auto &coReturn : coReturnsOps) {
2233 rewriter.setInsertionPoint(coReturn);
2234 rewriter.replaceOpWithNewOp<cir::BrOp>(op: coReturn, args&: beforeFinal,
2235 args: mlir::ValueRange{});
2236 }
2237
2238 rewriter.inlineRegionBefore(region&: bodyRegion, before: insertBefore);
2239 }
2240
2241 // finalSuspend ends in a cir.yield once the final await completes. The
2242 // coroutine is done, so branch to the destroy region to free the frame.
2243 void inlineFinalSuspend(mlir::PatternRewriter &rewriter,
2244 cir::CoroutineOp coroOp, mlir::Block *insertBefore,
2245 mlir::Block *beforeDestroy) const {
2246 mlir::Region &finalRegion = coroOp.getFinalSuspend();
2247 replaceYieldWithBr(rewriter, region&: finalRegion, dest: beforeDestroy);
2248 rewriter.inlineRegionBefore(region&: finalRegion, before: insertBefore);
2249 }
2250
2251 // The destroy region frees the frame (cir.coro.intrinsic.free) and always
2252 // continues to the exit region.
2253 void inlineDestroy(mlir::PatternRewriter &rewriter, cir::CoroutineOp coroOp,
2254 mlir::Block *insertBefore, mlir::Block *beforeExit) const {
2255 mlir::Region &destroyRegion = coroOp.getDestroy();
2256 replaceYieldWithBr(rewriter, region&: destroyRegion, dest: beforeExit);
2257 rewriter.inlineRegionBefore(region&: destroyRegion, before: insertBefore);
2258 }
2259
2260 // The exit region ends in a cir.return, so nothing falls through it.
2261 void inlineExit(mlir::PatternRewriter &rewriter, cir::CoroutineOp coroOp,
2262 mlir::Block *insertBefore) const {
2263 rewriter.inlineRegionBefore(region&: coroOp.getExit(), before: insertBefore);
2264 }
2265
2266 // A suspended coroutine keeps its frame, so each cir.coro.exit branches
2267 // straight to the exit region without running the destroy region.
2268 void lowerCoroExits(mlir::PatternRewriter &rewriter,
2269 llvm::ArrayRef<cir::CoroExitOp> coroExitsOps,
2270 mlir::Block *beforeExit) const {
2271 for (cir::CoroExitOp coroExit : coroExitsOps) {
2272 // TODO(cir): once the GRO region is supported, this may need to branch
2273 // there instead of to the exit region.
2274 assert(!cir::MissingFeatures::coroGetRetObjManager());
2275 rewriter.setInsertionPoint(coroExit);
2276 rewriter.replaceOpWithNewOp<cir::BrOp>(op: coroExit, args&: beforeExit,
2277 args: mlir::ValueRange{});
2278 }
2279 }
2280
2281 // An await's destroy case: by now each cir.coro.cleanup has already gone
2282 // through the enclosing cleanup scopes (running their cleanups) and sits
2283 // directly in the coroutine. Replace it with a branch to the destroy region,
2284 // which frees the frame and then reaches the exit region.
2285 void lowerCoroCleanups(mlir::PatternRewriter &rewriter,
2286 llvm::ArrayRef<cir::CoroCleanupOp> coroCleanupOps,
2287 mlir::Block *beforeDestroy) const {
2288 for (cir::CoroCleanupOp coroCleanup : coroCleanupOps) {
2289 rewriter.setInsertionPoint(coroCleanup);
2290 rewriter.replaceOpWithNewOp<cir::BrOp>(op: coroCleanup, args&: beforeDestroy,
2291 args: mlir::ValueRange{});
2292 }
2293 }
2294
2295public:
2296 using OpRewritePattern<cir::CoroutineOp>::OpRewritePattern;
2297
2298 mlir::LogicalResult
2299 matchAndRewrite(cir::CoroutineOp coroOp,
2300 mlir::PatternRewriter &rewriter) const override {
2301
2302 assert(!cir::MissingFeatures::coroUnwindRegion());
2303 for (mlir::Region &region : coroOp->getRegions())
2304 if (hasNestedOpsToFlatten(region))
2305 return mlir::failure();
2306
2307 // Collect the await placeholders before the regions are moved. Each
2308 // cir.coro.exit becomes a branch to the exit region, and each
2309 // cir.coro.cleanup a branch to the destroy region.
2310 llvm::SmallVector<cir::CoroExitOp> coroExitsOps;
2311 collectAwaitPlaceholders(coroOp, ops&: coroExitsOps);
2312 llvm::SmallVector<cir::CoroCleanupOp> coroCleanupOps;
2313 collectAwaitPlaceholders(coroOp, ops&: coroCleanupOps);
2314
2315 // Take the entry blocks before the regions are moved.
2316 mlir::Block *beforeInitial = &coroOp.getInitialSuspend().front();
2317 mlir::Block *beforeBody = &coroOp.getBody().front();
2318 mlir::Block *beforeFinal = &coroOp.getFinalSuspend().front();
2319 mlir::Block *beforeDestroy = &coroOp.getDestroy().front();
2320 mlir::Block *beforeExit = &coroOp.getExit().front();
2321
2322 mlir::Block *coroBlock = rewriter.getInsertionBlock();
2323 // Split the current block before the CoroutineOp to create the inlining
2324 // point.
2325 mlir::Block *remainingOpsBlock =
2326 rewriter.splitBlock(block: coroBlock, before: rewriter.getInsertionPoint());
2327
2328 lowerCoroExits(rewriter, coroExitsOps, beforeExit);
2329 lowerCoroCleanups(rewriter, coroCleanupOps, beforeDestroy);
2330
2331 inlineInitialSuspend(rewriter, coroOp, insertBefore: remainingOpsBlock, beforeBody);
2332 inlineBody(rewriter, coroOp, insertBefore: remainingOpsBlock, beforeFinal);
2333 inlineFinalSuspend(rewriter, coroOp, insertBefore: remainingOpsBlock, beforeDestroy);
2334 inlineDestroy(rewriter, coroOp, insertBefore: remainingOpsBlock, beforeExit);
2335 inlineExit(rewriter, coroOp, insertBefore: remainingOpsBlock);
2336
2337 rewriter.setInsertionPointToEnd(coroBlock);
2338 cir::BrOp::create(builder&: rewriter, location: coroOp.getLoc(), dest: beforeInitial,
2339 destOperands: mlir::ValueRange{});
2340
2341 auto func = coroOp->getParentOfType<cir::FuncOp>();
2342 // The coroutine regions have been flattened, so the original coroutine
2343 // attribute no longer verifies. Preserve the information that this function
2344 // originated from a coroutine for LLVM lowering.
2345 func.setCoroutineKind(cir::CoroutineKind::FlattenedCoroutine);
2346
2347 rewriter.eraseOp(op: coroOp);
2348 return mlir::success();
2349 }
2350};
2351
2352class CIRAwaitOpFlattening : public mlir::OpRewritePattern<cir::AwaitOp> {
2353 // The ready region ends in a cir.condition that holds the result of
2354 // `await_ready()`. Replace it with a cir.brcond: if the awaiter is already
2355 // ready, skip the suspension and go straight to the resume region
2356 // otherwise go to the suspend region.
2357 void flattenReadyRegion(mlir::PatternRewriter &rewriter, cir::AwaitOp awaitOp,
2358 mlir::Block *resumeEntry, mlir::Block *suspendEntry,
2359 mlir::Block *remainingOpsBlock) const {
2360 mlir::Region &readyRegion = awaitOp.getReady();
2361 auto conditionOp =
2362 cast<cir::ConditionOp>(Val: readyRegion.back().getTerminator());
2363 rewriter.setInsertionPoint(conditionOp);
2364 rewriter.replaceOpWithNewOp<cir::BrCondOp>(
2365 op: conditionOp, args: conditionOp.getCondition(), args&: resumeEntry, args&: suspendEntry);
2366 rewriter.inlineRegionBefore(region&: readyRegion, before: remainingOpsBlock);
2367 }
2368
2369 // The suspend region runs `await_suspend()` and ends in a
2370 // cir.coro.suspend_point. Emit cir.coro.intrinsic.save and
2371 // cir.coro.intrinsic.suspend around it, and replace the suspend point with a
2372 // cir.switch.flat on the suspend result, which decides whether the coroutine
2373 // resumes, is destroyed or stays suspended.
2374 void flattenSuspendRegion(mlir::PatternRewriter &rewriter,
2375 cir::AwaitOp awaitOp, mlir::Block *resumeEntry,
2376 mlir::Block *remainingOpsBlock) const {
2377 mlir::Region &suspendRegion = awaitOp.getSuspend();
2378 auto suspendPoint =
2379 cast<cir::CoroSuspendPoint>(Val: suspendRegion.back().getTerminator());
2380 cir::CoroSuspendOp coroSuspendIntri =
2381 emitCoroSuspend(rewriter, awaitOp, suspendPoint);
2382
2383 // Inline the suspend region before creating the destroy and suspended
2384 // blocks, so they are placed right after it, in front of the continuation.
2385 rewriter.inlineRegionBefore(region&: suspendRegion, before: remainingOpsBlock);
2386
2387 mlir::Location loc = awaitOp.getLoc();
2388 mlir::Block *cleanupBlock =
2389 createDestroyBlock(rewriter, loc, insertBefore: remainingOpsBlock);
2390 mlir::Block *retBlock =
2391 createSuspendedBlock(rewriter, loc, insertBefore: remainingOpsBlock);
2392
2393 // 0 resumes, 1 destroys and the default (-1) stays suspended.
2394 llvm::SmallVector<mlir::APInt> caseValues{mlir::APInt(8, 0),
2395 mlir::APInt(8, 1)};
2396 llvm::SmallVector<mlir::Block *> caseDestinations{resumeEntry,
2397 cleanupBlock};
2398 llvm::SmallVector<mlir::ValueRange> caseOperands{caseValues.size()};
2399
2400 // Dispatch to the appropriate coroutine path based on the result of
2401 // `cir.coro.intrinsic.suspend`: resume, destroy, or suspend.
2402 rewriter.setInsertionPoint(suspendPoint);
2403 rewriter.replaceOpWithNewOp<cir::SwitchFlatOp>(
2404 op: suspendPoint, args: coroSuspendIntri.getResult(), args&: retBlock,
2405 args: mlir::ValueRange{}, args&: caseValues, args&: caseDestinations, args&: caseOperands);
2406 }
2407
2408 // Emit cir.coro.intrinsic.save at the start of the suspend region and
2409 // cir.coro.intrinsic.suspend right before its cir.coro.suspend_point.
2410 cir::CoroSuspendOp emitCoroSuspend(mlir::PatternRewriter &rewriter,
2411 cir::AwaitOp awaitOp,
2412 cir::CoroSuspendPoint suspendPoint) const {
2413 mlir::Location loc = awaitOp.getLoc();
2414
2415 // cir.coro.intrinsic.suspend takes the token produced by
2416 // cir.coro.intrinsic.save as its first argument, so emit the save first.
2417 rewriter.setInsertionPointToStart(&awaitOp.getSuspend().front());
2418
2419 // The save goes first in the suspend region, before await_suspend()
2420 // runs, so the coroutine is already considered suspended (and can be
2421 // resumed) from that point.
2422 auto voidPtrTy = cir::PointerType::get(pointee: cir::VoidType::get(ctx: getContext()));
2423 auto nullPtr = cir::ConstantOp::create(
2424 builder&: rewriter, location: loc,
2425 value: cir::ConstPtrAttr::get(type: voidPtrTy, value: rewriter.getI64IntegerAttr(value: 0)));
2426 auto coroSave = cir::CoroSaveOp::create(builder&: rewriter, location: loc, coroaddr: nullPtr);
2427 rewriter.setInsertionPoint(suspendPoint);
2428 if (suspendPoint.getCond() != nullptr) {
2429 auto *oldinsertpoint = rewriter.getInsertionBlock();
2430 mlir::Block *suspendBlock = rewriter.splitBlock(
2431 block: rewriter.getInsertionBlock(), before: rewriter.getInsertionPoint());
2432 mlir::OpBuilder::InsertionGuard guard(rewriter);
2433 rewriter.setInsertionPointToEnd(oldinsertpoint);
2434 cir::BrCondOp::create(builder&: rewriter, location: loc, cond: suspendPoint.getCond(), destTrue: suspendBlock,
2435 destFalse: &awaitOp.getResume().front(), destOperandsTrue: mlir::ValueRange{},
2436 destOperandsFalse: mlir::ValueRange{});
2437 }
2438 rewriter.setInsertionPoint(suspendPoint);
2439
2440 // The second argument to `cir.coro.intrinsic.suspend` indicates whether
2441 // this is the final suspend point. Coroutines suspended at a final suspend
2442 // point are considered done (`cir.coro.intrinsic.done` returns true) and
2443 // may only be destroyed; resuming them is undefined behavior.
2444 bool isFinalSuspend = awaitOp.getKind() == cir::AwaitKind::Final;
2445 auto isFinalCoroSuspend = cir::ConstantOp::create(
2446 builder&: rewriter, location: loc, value: cir::BoolAttr::get(context: getContext(), value: isFinalSuspend));
2447
2448 // cir.coro.intrinsic.suspend returns:
2449 // -1 : coroutine suspended
2450 // 0 : coroutine resumed
2451 // 1 : coroutine destroyed
2452 return cir::CoroSuspendOp::create(
2453 builder&: rewriter, location: loc,
2454 operands: mlir::ValueRange{coroSave.getResult(), isFinalCoroSuspend});
2455 }
2456
2457 // Destroy case. Classic codegen uses EmitBranchThroughCleanup() so every
2458 // enclosing cleanup runs before the frame is freed. Here the destroy block
2459 // ends in a cir.coro.cleanup which, like cir.co_return, leaves every
2460 // enclosing cir.cleanup.scope (and loop) through its cleanup. Once it reaches
2461 // the coroutine, CIRCoroutineFlattening turns it into a branch to the
2462 // destroy region, which frees the frame (cir.coro.intrinsic.free) and then
2463 // reaches the exit region.
2464 mlir::Block *createDestroyBlock(mlir::PatternRewriter &rewriter,
2465 mlir::Location loc,
2466 mlir::Block *insertBefore) const {
2467 mlir::OpBuilder::InsertionGuard guard(rewriter);
2468 mlir::Block *cleanupBlock = rewriter.createBlock(insertBefore);
2469 cir::CoroCleanupOp::create(builder&: rewriter, location: loc);
2470 return cleanupBlock;
2471 }
2472
2473 // Suspended case: cir.coro.exit, which CIRCoroutineFlattening turns into
2474 // a branch to the exit region.
2475 mlir::Block *createSuspendedBlock(mlir::PatternRewriter &rewriter,
2476 mlir::Location loc,
2477 mlir::Block *insertBefore) const {
2478 mlir::OpBuilder::InsertionGuard guard(rewriter);
2479 assert(!cir::MissingFeatures::coroGetRetObjManager());
2480 mlir::Block *retBlock = rewriter.createBlock(insertBefore);
2481 cir::CoroExitOp::create(builder&: rewriter, location: loc);
2482 return retBlock;
2483 }
2484
2485 void flattenResumeRegion(mlir::PatternRewriter &rewriter,
2486 cir::AwaitOp awaitOp,
2487 mlir::Block *remainingOpsBlock) const {
2488 mlir::Region &resumeRegion = awaitOp.getResume();
2489 auto resumeYield = cast<cir::YieldOp>(Val: resumeRegion.back().getTerminator());
2490 rewriter.setInsertionPoint(resumeYield);
2491 // Once the coroutine resumes, continue with the operations that
2492 // originally followed the await.
2493 rewriter.replaceOpWithNewOp<cir::BrOp>(op: resumeYield, args&: remainingOpsBlock);
2494 rewriter.inlineRegionBefore(region&: resumeRegion, before: remainingOpsBlock);
2495 }
2496
2497public:
2498 using OpRewritePattern<cir::AwaitOp>::OpRewritePattern;
2499
2500 // Flatten `cir.await` into the following control flow:
2501 //
2502 // awaitBlock
2503 // |
2504 // v
2505 // ready await_ready()
2506 // true / \ false
2507 // | v
2508 // | suspend await_suspend(), with
2509 // | / | \ cir.coro.intrinsic.save/suspend
2510 // | 0/ 1| \-1 (default)
2511 // v v v v
2512 // resume destroy suspended
2513 // | cir.coro cir.coro.exit
2514 // | .cleanup
2515 // v
2516 // continuation ops after the await
2517 //
2518 // `destroy` leaves through the enclosing cleanups to the coroutine's destroy
2519 // region (see createDestroyBlock), and `suspended` goes straight to its exit
2520 // region (see CIRCoroutineFlattening).
2521 mlir::LogicalResult
2522 matchAndRewrite(cir::AwaitOp awaitOp,
2523 mlir::PatternRewriter &rewriter) const override {
2524 // Nested ops (cir.if, cir.scope, cir.try, cleanup scopes, ...) must be
2525 // flattened first. Fail here so the rewriter processes them before
2526 // flattening this op.
2527 for (mlir::Region &region : awaitOp->getRegions())
2528 if (hasNestedOpsToFlatten(region))
2529 return mlir::failure();
2530
2531 mlir::Block *awaitBlock = rewriter.getInsertionBlock();
2532 // Split the current block before the AwaitOp to create the inlining
2533 // point.
2534 mlir::Block *remainingOpsBlock =
2535 rewriter.splitBlock(block: awaitBlock, before: rewriter.getInsertionPoint());
2536
2537 // Take the entry blocks before the regions are inlined.
2538 mlir::Block *readyEntry = &awaitOp.getReady().front();
2539 mlir::Block *suspendEntry = &awaitOp.getSuspend().front();
2540 mlir::Block *resumeEntry = &awaitOp.getResume().front();
2541
2542 flattenReadyRegion(rewriter, awaitOp, resumeEntry, suspendEntry,
2543 remainingOpsBlock);
2544
2545 // After inlining the ready region, branch from the original await block
2546 // to the beginning of the inlined ready region.
2547 rewriter.setInsertionPointToEnd(awaitBlock);
2548 cir::BrOp::create(builder&: rewriter, location: awaitOp.getLoc(), dest: readyEntry);
2549
2550 flattenSuspendRegion(rewriter, awaitOp, resumeEntry, remainingOpsBlock);
2551 flattenResumeRegion(rewriter, awaitOp, remainingOpsBlock);
2552
2553 rewriter.eraseOp(op: awaitOp);
2554 return mlir::success();
2555 }
2556};
2557
2558void populateFlattenCFGPatterns(RewritePatternSet &patterns) {
2559 patterns
2560 .add<CIRIfFlattening, CIRLoopOpInterfaceFlattening, CIRScopeOpFlattening,
2561 CIRSwitchOpFlattening, CIRTernaryOpFlattening,
2562 CIRCleanupScopeOpFlattening, CIRTryOpFlattening,
2563 CIRAwaitOpFlattening, CIRCoroutineFlattening>(arg: patterns.getContext());
2564}
2565
2566namespace {
2567// An implementation of RewriterBase::Listener that determines whether the IR
2568// has been modified since the last time it was 'reset'. At the moment, this is
2569// the only use for something like this, but we might wish to move this
2570// somewhere if someone else needs similar functionality in the future.
2571class MLIRChangedListener final : public mlir::RewriterBase::Listener {
2572 bool hasChanged = false;
2573
2574public:
2575 void reset() { hasChanged = false; }
2576
2577 bool changed() const { return hasChanged; }
2578
2579 void notifyBlockErased(Block *) override { hasChanged = true; }
2580 void notifyOperationModified(Operation *) override { hasChanged = true; }
2581 void notifyOperationReplaced(Operation *, Operation *) override {
2582 hasChanged = true;
2583 }
2584 void notifyOperationReplaced(Operation *, ValueRange) override {
2585 hasChanged = true;
2586 }
2587 void notifyOperationErased(Operation *) override { hasChanged = true; }
2588
2589 // notifyPatternBegin, notifyPatternEnd, notifyMatchFailure all skipped, since
2590 // they don't modify.
2591 void notifyOperationInserted(Operation *,
2592 mlir::IRRewriter::InsertPoint) override {
2593 hasChanged = true;
2594 }
2595 void notifyBlockInserted(Block *, Region *, Region::iterator) override {
2596 hasChanged = true;
2597 }
2598};
2599} // namespace
2600
2601void CIRFlattenCFGPass::runOnOperation() {
2602 RewritePatternSet patternList(&getContext());
2603 populateFlattenCFGPatterns(patterns&: patternList);
2604 FrozenRewritePatternSet patterns(std::move(patternList));
2605
2606 PatternApplicator applicator(patterns);
2607 // We need _A_ cost model, and everything here is the same cost-model, so this
2608 // is effectively a no-op, but necessary to use the PatternApplicator.
2609 applicator.applyDefaultCostModel();
2610
2611 mlir::PatternRewriter rewriter(&getContext());
2612 MLIRChangedListener changedListener;
2613 rewriter.setListener(&changedListener);
2614
2615 do {
2616 changedListener.reset();
2617
2618 // Collect flatten candidates post-order so an inner op is handled before
2619 // its parent; op pointers stay valid across the block splits / region
2620 // inlines the patterns perform (a pattern only erases the matched op and
2621 // its descendants, which are visited first), so the list can be iterated
2622 // directly.
2623 llvm::SmallVector<Operation *, 16> ops;
2624 getOperation()->walk<mlir::WalkOrder::PostOrder>(callback: [&](Operation *op) {
2625 if (isa<IfOp, ScopeOp, SwitchOp, LoopOpInterface, TernaryOp,
2626 CleanupScopeOp, TryOp, AwaitOp, CoroutineOp>(Val: op))
2627 ops.push_back(Elt: op);
2628 });
2629
2630 for (mlir::Operation *op : ops) {
2631 rewriter.setInsertionPoint(op);
2632 (void)applicator.matchAndRewrite(op, rewriter);
2633 }
2634 } while (changedListener.changed());
2635}
2636
2637} // namespace
2638
2639namespace mlir {
2640
2641std::unique_ptr<Pass> createCIRFlattenCFGPass() {
2642 return std::make_unique<CIRFlattenCFGPass>();
2643}
2644
2645} // namespace mlir
2646