1//===----- CGCoroutine.cpp - Emit CIR Code for C++ coroutines -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code dealing with C++ code generation of coroutines.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenFunction.h"
14#include "mlir/Support/LLVM.h"
15#include "clang/AST/StmtCXX.h"
16#include "clang/AST/StmtVisitor.h"
17#include "clang/Basic/TargetInfo.h"
18#include "clang/CIR/Dialect/IR/CIRDialect.h"
19#include "clang/CIR/MissingFeatures.h"
20
21using namespace clang;
22using namespace clang::CIRGen;
23
24struct clang::CIRGen::CGCoroData {
25 // What is the current await expression kind and how many
26 // await/yield expressions were encountered so far.
27 // These are used to generate pretty labels for await expressions in LLVM IR.
28 cir::AwaitKind currentAwaitKind = cir::AwaitKind::Init;
29 // Stores the __builtin_coro_id emitted in the function so that we can supply
30 // it as the first argument to other builtins.
31 cir::CoroIdOp coroId = nullptr;
32
33 // Stores the result of __builtin_coro_begin call.
34 mlir::Value coroBegin = nullptr;
35
36 // How many co_return statements are in the coroutine. Used to decide whether
37 // we need to add co_return; equivalent at the end of the user authored body.
38 unsigned coreturnCount = 0;
39
40 // The promise type's 'unhandled_exception' handler, if it defines one.
41 Stmt *exceptionHandler = nullptr;
42
43 // Stores the last emitted coro.free for the deallocate expressions, we use it
44 // to wrap dealloc code with if(auto mem = coro.free) dealloc(mem).
45 cir::CoroFreeOp lastCoroFree = nullptr;
46
47 // A temporary bool alloca that stores whether 'await_resume' threw an
48 // exception. If it did, 'true' is stored in this variable, and the coroutine
49 // body must be skipped. If the promise type does not define an exception
50 // handler, this is null.
51 Address resumeEHVar = Address::invalid();
52
53 // If coro.id came from the builtin, remember the expression to give better
54 // diagnostic. If CoroIdExpr is nullptr, the coro.id was created by
55 // EmitCoroutineBody.
56 CallExpr const *coroIdExpr = nullptr;
57};
58
59// Defining these here allows to keep CGCoroData private to this file.
60CIRGenFunction::CGCoroInfo::CGCoroInfo() {}
61CIRGenFunction::CGCoroInfo::~CGCoroInfo() {}
62
63namespace {
64// FIXME: both GetParamRef and ParamReferenceReplacerRAII are good template
65// candidates to be shared among LLVM / CIR codegen.
66
67// Hunts for the parameter reference in the parameter copy/move declaration.
68struct GetParamRef : public StmtVisitor<GetParamRef> {
69public:
70 DeclRefExpr *expr = nullptr;
71 GetParamRef() {}
72 void VisitDeclRefExpr(DeclRefExpr *e) {
73 assert(expr == nullptr && "multilple declref in param move");
74 expr = e;
75 }
76 void VisitStmt(Stmt *s) {
77 for (Stmt *c : s->children()) {
78 if (c)
79 Visit(S: c);
80 }
81 }
82};
83
84// This class replaces references to parameters to their copies by changing
85// the addresses in CGF.LocalDeclMap and restoring back the original values in
86// its destructor.
87struct ParamReferenceReplacerRAII {
88 CIRGenFunction::DeclMapTy savedLocals;
89 CIRGenFunction::DeclMapTy &localDeclMap;
90
91 ParamReferenceReplacerRAII(CIRGenFunction::DeclMapTy &localDeclMap)
92 : localDeclMap(localDeclMap) {}
93
94 void addCopy(const DeclStmt *pm) {
95 // Figure out what param it refers to.
96
97 assert(pm->isSingleDecl());
98 const VarDecl *vd = static_cast<const VarDecl *>(pm->getSingleDecl());
99 const Expr *initExpr = vd->getInit();
100 GetParamRef visitor;
101 visitor.Visit(S: const_cast<Expr *>(initExpr));
102 assert(visitor.expr);
103 DeclRefExpr *dreOrig = visitor.expr;
104 auto *pd = dreOrig->getDecl();
105
106 auto it = localDeclMap.find(Val: pd);
107 assert(it != localDeclMap.end() && "parameter is not found");
108 savedLocals.insert(KV: {pd, it->second});
109
110 auto copyIt = localDeclMap.find(Val: vd);
111 assert(copyIt != localDeclMap.end() && "parameter copy is not found");
112 it->second = copyIt->getSecond();
113 }
114
115 ~ParamReferenceReplacerRAII() {
116 for (auto &&savedLocal : savedLocals) {
117 localDeclMap.insert(KV: {savedLocal.first, savedLocal.second});
118 }
119 }
120};
121} // namespace
122
123RValue CIRGenFunction::emitCoroutineFrame() {
124 if (curCoro.data && curCoro.data->coroBegin) {
125 return RValue::get(v: curCoro.data->coroBegin);
126 }
127 cgm.errorNYI(feature: "NYI");
128 return RValue();
129}
130
131static void createCoroData(CIRGenFunction &cgf,
132 CIRGenFunction::CGCoroInfo &curCoro,
133 cir::CoroIdOp coroId,
134 CallExpr const *coroIdExpr = nullptr) {
135
136 if (curCoro.data) {
137 if (curCoro.data->coroIdExpr)
138 cgf.cgm.error(loc: coroIdExpr->getBeginLoc(),
139 error: "only one __builtin_coro_id can be used in a function");
140 else if (coroIdExpr)
141 cgf.cgm.error(loc: coroIdExpr->getBeginLoc(),
142 error: "__builtin_coro_id shall not be used in a C++ coroutine");
143 else
144 llvm_unreachable("EmitCoroutineBodyStatement called twice?");
145
146 return;
147 }
148
149 curCoro.data = std::make_unique<CGCoroData>();
150 curCoro.data->coroId = coroId;
151 curCoro.data->coroIdExpr = coroIdExpr;
152}
153
154static mlir::LogicalResult
155emitBodyAndFallthrough(CIRGenFunction &cgf, const CoroutineBodyStmt &s,
156 Stmt *body,
157 const CIRGenFunction::LexicalScope *currLexScope) {
158 {
159 // Destroy the body's local variables, and the temporaries they extend, when
160 // the body ends, before the fall-through handler, the implicit
161 // `co_return;`, calls return_void(). The body stays in the current lexical
162 // scope so that a co_return at its top level is recorded in currLexScope.
163 CIRGenFunction::RunCleanupsScope bodyScope(cgf);
164 if (cgf.emitStmt(s: body, /*useCurrentScope=*/true).failed())
165 return mlir::failure();
166 }
167
168 // Note that classic codegen checks CanFallthrough by looking into the
169 // availability of the insert block which is kinda brittle and unintuitive,
170 // seems to be related with how landing pads are handled.
171 //
172 // CIRGen handles this by checking pre-existing co_returns in the current
173 // scope instead.
174
175 // From LLVM IR Gen: const bool CanFallthrough = Builder.GetInsertBlock();
176 const bool canFallthrough = !currLexScope->hasCoreturn();
177 if (canFallthrough)
178 if (Stmt *onFallthrough = s.getFallthroughHandler())
179 if (cgf.emitStmt(s: onFallthrough, /*useCurrentScope=*/true).failed())
180 return mlir::failure();
181
182 return mlir::success();
183}
184
185cir::CoroIdOp CIRGenFunction::emitCoroIDBuiltinCall(const CallExpr *e) {
186 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
187
188 llvm::SmallVector<mlir::Value, 4> args;
189 for (const Expr *arg : e->arguments())
190 args.push_back(Elt: emitScalarExpr(e: arg));
191
192 auto coroId = cir::CoroIdOp::create(builder&: cgm.getBuilder(), location: loc, operands: args);
193 createCoroData(cgf&: *this, curCoro, coroId, coroIdExpr: e);
194 return coroId;
195}
196
197cir::CoroAllocOp CIRGenFunction::emitCoroAllocBuiltinCall(const CallExpr *e) {
198 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
199 if (!curCoro.data || !curCoro.data->coroId) {
200 cgm.error(loc: e->getBeginLoc(), error: "this builtin expect that __builtin_coro_id has"
201 " been used earlier in this function");
202 return {};
203 }
204
205 return cir::CoroAllocOp::create(
206 builder&: cgm.getBuilder(), location: loc,
207 operands: mlir::ValueRange{curCoro.data->coroId.getResult()});
208}
209
210cir::CoroBeginOp CIRGenFunction::emitCoroBeginBuiltinCall(const CallExpr *e) {
211
212 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
213 if (!curCoro.data || !curCoro.data->coroId) {
214 cgm.error(loc: e->getBeginLoc(), error: "this builtin expect that __builtin_coro_id has"
215 " been used earlier in this function");
216 return {};
217 }
218 llvm::SmallVector<mlir::Value, 2> args;
219 args.push_back(Elt: curCoro.data->coroId.getResult());
220 for (const Expr *arg : e->arguments())
221 args.push_back(Elt: emitScalarExpr(e: arg));
222
223 auto coroBegin = cir::CoroBeginOp::create(builder&: cgm.getBuilder(), location: loc, operands: args);
224 curCoro.data->coroBegin = coroBegin;
225 return coroBegin;
226}
227
228cir::CoroEndOp CIRGenFunction::emitCoroEndBuiltinCall(const CallExpr *e) {
229
230 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
231 CIRGenBuilderTy &builder = cgm.getBuilder();
232 llvm::SmallVector<mlir::Value, 3> args;
233 for (const Expr *arg : e->arguments())
234 args.push_back(Elt: emitScalarExpr(e: arg));
235 args.push_back(Elt: cir::TokenNoneOp::create(builder, location: loc));
236 return cir::CoroEndOp::create(builder, location: loc, resultTypes: {cgm.voidTy}, operands: args);
237}
238
239cir::CoroFreeOp CIRGenFunction::emitCoroFreeBuiltin(const CallExpr *e) {
240 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
241
242 if (!curCoro.data || !curCoro.data->coroId) {
243 cgm.error(loc: e->getBeginLoc(), error: "this builtin expect that __builtin_coro_id has"
244 " been used earlier in this function");
245 return {};
246 }
247
248 auto coroFree =
249 cir::CoroFreeOp::create(builder&: cgm.getBuilder(), location: loc,
250 operands: mlir::ValueRange{curCoro.data->coroId.getResult(),
251 curCoro.data->coroBegin});
252
253 curCoro.data->lastCoroFree = coroFree;
254 return coroFree;
255}
256
257cir::CoroSizeOp CIRGenFunction::emitCoroSizeBuiltinCall(const CallExpr *e) {
258 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
259 return cir::CoroSizeOp::create(builder&: cgm.getBuilder(), location: loc,
260 result: convertType(t: e->getType()));
261}
262
263cir::CoroAlignOp CIRGenFunction::emitCoroAlignBuiltinCall(const CallExpr *e) {
264 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
265 return cir::CoroAlignOp::create(builder&: cgm.getBuilder(), location: loc,
266 result: convertType(t: e->getType()));
267}
268
269cir::CoroPromiseOp
270CIRGenFunction::emitCoroPromiseBuiltinCall(const CallExpr *e) {
271 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
272
273 llvm::SmallVector<mlir::Value, 3> args;
274 for (const Expr *arg : e->arguments())
275 args.push_back(Elt: emitScalarExpr(e: arg));
276
277 auto coroPromise = cir::CoroPromiseOp::create(builder&: cgm.getBuilder(), location: loc, operands: args);
278 return coroPromise;
279}
280
281cir::CoroDoneOp CIRGenFunction::emitCoroDoneBuiltinCall(const CallExpr *e) {
282 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
283 return cir::CoroDoneOp::create(builder&: cgm.getBuilder(), location: loc,
284 handle: emitScalarExpr(e: e->getArg(Arg: 0)));
285}
286
287cir::CoroResumeOp CIRGenFunction::emitCoroResumeBuiltinCall(const CallExpr *e) {
288 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
289 return cir::CoroResumeOp::create(builder&: cgm.getBuilder(), location: loc,
290 handle: emitScalarExpr(e: e->getArg(Arg: 0)));
291}
292
293cir::CoroDestroyOp
294CIRGenFunction::emitCoroDestroyBuiltinCall(const CallExpr *e) {
295 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
296 return cir::CoroDestroyOp::create(builder&: cgm.getBuilder(), location: loc,
297 handle: emitScalarExpr(e: e->getArg(Arg: 0)));
298}
299
300cir::CoroNoopOp CIRGenFunction::emitCoroNoopBuiltinCall(const CallExpr *e) {
301 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
302 return cir::CoroNoopOp::create(builder&: cgm.getBuilder(), location: loc);
303}
304
305cir::CoroSuspendOp
306CIRGenFunction::emitCoroSuspendBuiltinCall(const CallExpr *e) {
307 mlir::Location loc = getLoc(srcLoc: e->getBeginLoc());
308 llvm::SmallVector<mlir::Value, 2> args;
309 args.push_back(Elt: cir::TokenNoneOp::create(builder, location: loc));
310 args.push_back(Elt: emitScalarExpr(e: e->getArg(Arg: 0)));
311
312 return cir::CoroSuspendOp::create(builder&: cgm.getBuilder(), location: loc, operands: args);
313}
314
315static mlir::LogicalResult
316coroutineBodyExceptionHelper(CIRGenFunction &cgf, const CoroutineBodyStmt &s) {
317
318 CXXCatchStmt catchStmt(s.getBeginLoc(), /*exDecl=*/nullptr,
319 cgf.curCoro.data->exceptionHandler);
320 auto *tryStmt = CXXTryStmt::Create(C: cgf.getContext(), tryLoc: s.getBeginLoc(),
321 tryBlock: s.getBody(), handlers: &catchStmt);
322 struct handlerEmitter final : CIRGenFunction::cxxTryBodyEmitter {
323 const CoroutineBodyStmt &s;
324
325 handlerEmitter(const CoroutineBodyStmt &s) : s(s) /*, scope(scope)*/ {}
326 mlir::LogicalResult operator()(CIRGenFunction &cgf) override {
327 return emitBodyAndFallthrough(cgf, s, body: s.getBody(), currLexScope: cgf.curLexScope);
328 }
329 ~handlerEmitter() override = default;
330 } emitter{s};
331
332 mlir::LogicalResult res = cgf.emitCXXTryStmt(s: *tryStmt, bodyCallback&: emitter);
333
334 return res;
335}
336
337mlir::LogicalResult
338CIRGenFunction::emitCoroutineBody(const CoroutineBodyStmt &s) {
339 mlir::Location openCurlyLoc = getLoc(srcLoc: s.getBeginLoc());
340 cir::ConstantOp nullPtrCst = builder.getNullPtr(ty: voidPtrTy, loc: openCurlyLoc);
341
342 auto fn = mlir::cast<cir::FuncOp>(Val: curFn);
343 fn.setCoroutineKind(cir::CoroutineKind::Coroutine);
344 const TargetInfo &ti = cgm.getASTContext().getTargetInfo();
345 unsigned newAlign = ti.getNewAlign() / ti.getCharWidth();
346
347 cir::CoroIdOp coroId = cir::CoroIdOp::create(
348 builder&: cgm.getBuilder(), location: openCurlyLoc,
349 operands: mlir::ValueRange{builder.getUInt32(c: newAlign, loc: openCurlyLoc), nullPtrCst,
350 nullPtrCst, nullPtrCst});
351 createCoroData(cgf&: *this, curCoro, coroId);
352
353 // Backend is allowed to elide memory allocations, to help it, emit
354 // auto mem = coro.alloc() ? 0 : ... allocation code ...;
355 cir::CoroAllocOp coroAlloc = cir::CoroAllocOp::create(
356 builder&: cgm.getBuilder(), location: openCurlyLoc,
357 operands: mlir::ValueRange{curCoro.data->coroId.getResult()});
358
359 // Initialize address of coroutine frame to null
360 CanQualType astVoidPtrTy = cgm.getASTContext().VoidPtrTy;
361 mlir::Type allocaTy = convertTypeForMem(t: astVoidPtrTy);
362 Address coroFrame =
363 createTempAlloca(ty: allocaTy, align: getContext().getTypeAlignInChars(T: astVoidPtrTy),
364 loc: openCurlyLoc, name: "__coro_frame_addr",
365 /*ArraySize=*/arraySize: nullptr);
366
367 mlir::Value storeAddr = coroFrame.getPointer();
368 builder.CIRBaseBuilderTy::createStore(loc: openCurlyLoc, val: nullPtrCst, dst: storeAddr);
369 mlir::LogicalResult res = mlir::success();
370 cir::IfOp::create(
371 builder, location: openCurlyLoc, cond: coroAlloc.getResult(),
372 /*withElseRegion=*/false,
373 /*thenBuilder=*/[&](mlir::OpBuilder &b, mlir::Location loc) {
374 mlir::Value allocatedPtr = emitScalarExpr(e: s.getAllocate());
375 builder.CIRBaseBuilderTy::createStore(loc, val: allocatedPtr, dst: storeAddr);
376 // Handle allocation failure if 'ReturnStmtOnAllocFailure' was provided.
377 if (Stmt *retOnAllocFailure = s.getReturnStmtOnAllocFailure()) {
378 mlir::Value isPtrNull = builder.createPtrIsNull(ptr: allocatedPtr);
379 assert(!cir::MissingFeatures::emitCondLikelihoodViaExpectIntrinsic());
380 cir::IfOp::create(builder, location: loc, cond: isPtrNull, /*withElseRegion=*/false,
381 thenBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) {
382 res = emitStmt(s: retOnAllocFailure,
383 /*useCurrentScope=*/true);
384 cir::UnreachableOp::create(builder, location: loc);
385 });
386 }
387 cir::YieldOp::create(builder, location: loc);
388 });
389
390 if (res.failed())
391 return res;
392
393 curCoro.data->coroBegin = cir::CoroBeginOp::create(
394 builder&: cgm.getBuilder(), location: openCurlyLoc,
395 operands: mlir::ValueRange{
396 curCoro.data->coroId.getResult(),
397 cir::LoadOp::create(builder, location: openCurlyLoc, result: allocaTy, addr: storeAddr)});
398
399 {
400 assert(!cir::MissingFeatures::generateDebugInfo());
401 ParamReferenceReplacerRAII paramReplacer(localDeclMap);
402
403 // Create mapping between parameters and copy-params for coroutine
404 // function.
405 llvm::ArrayRef<const Stmt *> paramMoves = s.getParamMoves();
406 assert((paramMoves.size() == 0 || (paramMoves.size() == fnArgs.size())) &&
407 "ParamMoves and FnArgs should be the same size for coroutine "
408 "function");
409 // For zipping the arg map into debug info.
410 assert(!cir::MissingFeatures::generateDebugInfo());
411
412 // Capture frame cleanups for destroy: promise first, then parameter copies,
413 // before freeing the frame ([dcl.fct.def.coroutine]p11, p13). Local cleanup
414 // scopes would end their lifetimes too early.
415
416 assert(!cir::MissingFeatures::coroUnwindRegion());
417 CapturedCleanups frameCleanups(*this, /*active=*/true);
418 // Whether the destructor of the promise or of a parameter copy may throw.
419 bool frameDestructorMayThrow = false;
420 auto emitFrameObject = [&](const VarDecl &var) {
421 emitAutoVarDeclWithCapturedCleanups(d: var, cleanups&: frameCleanups);
422 if (destructorMayThrow(type: var.getType()))
423 frameDestructorMayThrow = true;
424 };
425
426 // Create parameter copies. We do it before creating a promise, since an
427 // evolution of coroutine TS may allow promise constructor to observe
428 // parameter copies.
429 assert(!cir::MissingFeatures::coroOutsideFrameMD());
430 for (auto *pm : paramMoves) {
431 const auto *paramMove = cast<DeclStmt>(Val: pm);
432 emitFrameObject(cast<VarDecl>(Val: *paramMove->getSingleDecl()));
433 paramReplacer.addCopy(pm: paramMove);
434 }
435
436 // Builds `initial_suspend`.
437 auto initialSuspendBuilder = [&]() -> mlir::LogicalResult {
438 emitFrameObject(*s.getPromiseDecl());
439 // Update coro.id to refer to the promise. We could not do it earlier
440 // because the promise local variable was not emitted yet.
441 Address promiseAddr = getAddrOfLocalVar(vd: s.getPromiseDecl());
442 {
443 mlir::OpBuilder::InsertionGuard guard(builder);
444 builder.setInsertionPoint(coroId);
445 mlir::Value promisePtr = builder.createBitcast(
446 src: promiseAddr.emitRawPointer(), newTy: builder.getVoidPtrTy());
447 coroId.getPromiseMutable().assign(value: promisePtr);
448 }
449 // returnValue should be valid as long as the coroutine's return type
450 // is not void. The assertion could help us to reduce the check later.
451 assert(returnValue.isValid() == (bool)s.getReturnStmt());
452 // Now we have the promise, initialize the GRO.
453 // We need to emit `get_return_object` first. According to:
454 // [dcl.fct.def.coroutine]p7
455 // The call to get_return_­object is sequenced before the call to
456 // initial_suspend and is invoked at most once.
457 //
458 // So we couldn't emit return value when we emit return statment,
459 // otherwise the call to get_return_object wouldn't be in front
460 // of initial_suspend.
461 if (returnValue.isValid()) {
462 // Sema may omit ExprWithCleanups when the return types match. Clean
463 // up the GRO full-expression with its temporaries here explicitly.
464 FullExprCleanupScope groScope(*this, s.getReturnValue());
465 emitAnyExprToMem(e: s.getReturnValue(), location: returnValue,
466 quals: s.getReturnValue()->getType().getQualifiers(),
467 /*isInit*/ isInitializer: true);
468 }
469
470 curCoro.data->currentAwaitKind = cir::AwaitKind::Init;
471 curCoro.data->exceptionHandler = s.getExceptionHandler();
472
473 if (emitStmt(s: s.getInitSuspendStmt(), /*useCurrentScope=*/true).failed())
474 return mlir::failure();
475
476 cir::YieldOp::create(builder, location: openCurlyLoc);
477 return mlir::success();
478 };
479
480 // Builds `body`: the user-written coroutine code, including its
481 // implicit `try { ... } catch (...) { unhandled_exception(); }`.
482 auto bodyBuilder = [&]() -> mlir::LogicalResult {
483 RunCleanupsScope bodyRegionScope(*this);
484 curCoro.data->currentAwaitKind = cir::AwaitKind::User;
485
486 if (curCoro.data->exceptionHandler) {
487 // This bit of code is supposed to do:
488 //
489 // if (await-resume-didnt-throw-exception) {
490 // try {
491 // coroutine-body
492 // } catch (...) {
493 // unhandled_exception();
494 // }
495 // }
496 //
497 // IF resume couldn't have thrown an exception(await_resume is
498 // noexcept), we skip the 'if'.
499 //
500 // Note that we've reversed the condition of the 'if' from classic
501 // codegen so that we don't need an 'else' block.
502 if (curCoro.data->resumeEHVar.isValid()) {
503 mlir::Value shouldSkip = builder.createFlagLoad(
504 loc: openCurlyLoc, addr: curCoro.data->resumeEHVar.getPointer());
505 mlir::LogicalResult res = mlir::success();
506 cir::IfOp::create(builder, location: openCurlyLoc, cond: shouldSkip,
507 /*withElseRegion=*/false,
508 thenBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) {
509 res = coroutineBodyExceptionHelper(cgf&: *this, s);
510 builder.createYield(loc: openCurlyLoc);
511 });
512
513 if (res.failed())
514 return mlir::failure();
515
516 } else if (coroutineBodyExceptionHelper(cgf&: *this, s).failed()) {
517 return mlir::failure();
518 }
519 } else if (emitBodyAndFallthrough(cgf&: *this, s, body: s.getBody(), currLexScope: curLexScope)
520 .failed()) {
521 return mlir::failure();
522 }
523
524 return mlir::success();
525 };
526
527 // Builds `final_suspend`: only emitted at all if the body can actually
528 // reach it (an explicit co_return, or falling off the end).
529 auto finalSuspendBuilder = [&]() -> mlir::LogicalResult {
530 // Note that LLVM checks CanFallthrough by looking into the availability
531 // of the insert block which is kinda brittle and unintuitive, seems to be
532 // related with how landing pads are handled.
533 //
534 // CIRGen handles this by checking pre-existing co_returns in the current
535 // scope instead.
536 //
537 // From LLVM IR Gen: const bool CanFallthrough = Builder.GetInsertBlock();
538 const bool canFallthrough = curLexScope->hasCoreturn();
539 const bool hasCoreturns = curCoro.data->coreturnCount > 0;
540 if (canFallthrough || hasCoreturns) {
541 curCoro.data->currentAwaitKind = cir::AwaitKind::Final;
542 if (emitStmt(s: s.getFinalSuspendStmt(), /*useCurrentScope=*/true)
543 .failed())
544 return mlir::failure();
545 }
546 cir::YieldOp::create(builder, location: openCurlyLoc);
547 return mlir::success();
548 };
549
550 // Emit "if (coro.free(CoroId, CoroBegin)) Deallocate;".
551 auto emitDeallocate = [&]() -> mlir::LogicalResult {
552 Stmt *deallocate = s.getDeallocate();
553 if (emitStmt(s: deallocate, /*useCurrentScope=*/true).failed()) {
554 cgm.error(loc: deallocate->getBeginLoc(),
555 error: "failed to emit coroutine deallocation expression");
556 return mlir::failure();
557 }
558
559 cir::CoroFreeOp coroFree = curCoro.data->lastCoroFree;
560
561 if (!coroFree) {
562 cgm.error(loc: deallocate->getBeginLoc(),
563 error: "Deallocation expression does not refer to coro.free");
564 return mlir::failure();
565 }
566 {
567 mlir::OpBuilder::InsertionGuard guard(builder);
568 builder.setInsertionPointAfter(coroFree);
569 mlir::Value isPtrNotNull =
570 builder.createPtrIsNotNull(ptr: coroFree.getResult());
571
572 llvm::SmallVector<mlir::Operation *> opsToMove;
573 mlir::Block *block = builder.getInsertionBlock();
574 mlir::Block::iterator it(isPtrNotNull.getDefiningOp());
575
576 for (++it; it != block->end(); ++it)
577 opsToMove.push_back(Elt: &*it);
578
579 auto ifOp = cir::IfOp::create(
580 builder, location: getLoc(srcLoc: deallocate->getSourceRange()), cond: isPtrNotNull,
581 /*withElseRegion*/ false,
582 thenBuilder: [&](mlir::OpBuilder &builder, mlir::Location loc) {
583 cir::YieldOp::create(builder, location: loc);
584 });
585
586 mlir::Operation *yieldOp = ifOp.getThenRegion().back().getTerminator();
587 for (auto *op : opsToMove)
588 op->moveBefore(existingOp: yieldOp);
589 }
590 return mlir::success();
591 };
592
593 // Destroy the promise and the parameter copies, then free the frame. If one
594 // of the destructors may throw an exception that can leave the coroutine,
595 // emit them like classic CodeGen's cleanups: the deallocation is the
596 // cleanup of an outer cir.cleanup.scope, and each destructor has its own
597 // nested one, the first constructed object outermost. An exception from one
598 // of the destructors then still destroys the objects constructed before it
599 // and frees the frame.
600 auto destroyBuilder = [&]() -> mlir::LogicalResult {
601 if (!getLangOpts().Exceptions || inEHSpecTerminateScope() ||
602 !frameDestructorMayThrow) {
603 frameCleanups.emitIntoCleanupRegion(loc: openCurlyLoc);
604 if (emitDeallocate().failed())
605 return mlir::failure();
606 cir::YieldOp::create(builder, location: openCurlyLoc);
607 return mlir::success();
608 }
609
610 mlir::LogicalResult deallocRes = mlir::success();
611 auto deallocScope = cir::CleanupScopeOp::create(
612 builder, location: openCurlyLoc, cleanupKind: cir::CleanupKind::All,
613 /*bodyBuilder=*/[](mlir::OpBuilder &, mlir::Location) {},
614 /*cleanupBuilder=*/
615 [&](mlir::OpBuilder &b, mlir::Location loc) {
616 deallocRes = emitDeallocate();
617 cir::YieldOp::create(builder&: b, location: loc);
618 });
619 builder.setInsertionPointToEnd(&deallocScope.getBodyRegion().back());
620 frameCleanups.emitAsNestedScopes(loc: openCurlyLoc);
621 cir::YieldOp::create(builder, location: openCurlyLoc);
622 builder.setInsertionPointAfter(deallocScope);
623 cir::YieldOp::create(builder, location: openCurlyLoc);
624 return deallocRes;
625 };
626
627 // Builds `exit`: coro.end(/*unwind*/ false) followed by the actual return
628 // to the caller.
629
630 assert(!cir::MissingFeatures::coroGetRetObjManager());
631 auto exitBuilder = [&]() {
632 cir::ConstantOp nullHandler =
633 builder.getNullPtr(ty: builder.getVoidPtrTy(), loc: openCurlyLoc);
634 cir::ConstantOp noUnwind = builder.getBool(state: false, loc: openCurlyLoc);
635 auto tkNone = cir::TokenNoneOp::create(builder, location: openCurlyLoc);
636 cir::CoroEndOp::create(builder, location: openCurlyLoc, handle: nullHandler, unwind: noUnwind,
637 resultToken: tkNone);
638
639 if (auto *ret = cast_or_null<ReturnStmt>(Val: s.getReturnStmt())) {
640 // Since we already emitted the return value above, so we shouldn't
641 // emit it again here.
642 Expr *previousRetValue = ret->getRetValue();
643 ret->setRetValue(nullptr);
644 if (emitStmt(s: ret, /*useCurrentScope=*/true).failed())
645 return mlir::failure();
646 mlir::Block *block = builder.getInsertionBlock();
647 // emitReturnStmt() always creates a new insertion block after emitting
648 // the return. That block is unreachable in this case, so erase it.
649 block->erase();
650 // Set the return value back. The code generator, as the AST
651 // **Consumer**, shouldn't change the AST.
652 ret->setRetValue(previousRetValue);
653 } else {
654 cir::ReturnOp::create(builder, location: openCurlyLoc);
655 }
656
657 return mlir::success();
658 };
659
660 cir::CoroutineOp coro = cir::CoroutineOp::create(
661 builder, location: openCurlyLoc,
662 /*initialSuspendBuilder=*/
663 [&](mlir::OpBuilder &b, mlir::Location loc) {
664 if (initialSuspendBuilder().failed())
665 res = mlir::failure();
666 },
667 /*bodyBuilder=*/
668 [&](mlir::OpBuilder &b, mlir::Location loc) {
669 if (bodyBuilder().failed())
670 res = mlir::failure();
671 },
672 /*finalSuspendBuilder=*/
673 [&](mlir::OpBuilder &b, mlir::Location loc) {
674 if (finalSuspendBuilder().failed())
675 res = mlir::failure();
676 },
677 /*destroyBuilder=*/
678 [&](mlir::OpBuilder &b, mlir::Location loc) {
679 if (destroyBuilder().failed())
680 res = mlir::failure();
681 },
682 /*exitBuilder=*/
683 [&](mlir::OpBuilder &b, mlir::Location loc) {
684 if (exitBuilder().failed())
685 res = mlir::failure();
686 });
687
688 if (coro && !coro.getBody().empty()) {
689 mlir::Block &lastBlock = coro.getBody().back();
690 mlir::Block *prevBlock = lastBlock.getPrevNode();
691 if (lastBlock.empty() && lastBlock.hasNoPredecessors() && prevBlock &&
692 !prevBlock->empty() && isa<cir::CoReturnOp>(Val: prevBlock->back())) {
693 lastBlock.erase();
694 } else if (!lastBlock.mightHaveTerminator()) {
695 mlir::OpBuilder::InsertionGuard guard(builder);
696 builder.setInsertionPointToEnd(&lastBlock);
697 cir::YieldOp::create(builder, location: openCurlyLoc);
698 }
699 }
700 }
701
702 return res;
703}
704
705static bool functionCanThrow(const FunctionDecl *d) {
706 const auto *proto = d->getType()->getAs<FunctionProtoType>();
707 if (!proto)
708 return true;
709 return !isNoexceptExceptionSpec(ESpecType: proto->getExceptionSpecType()) ||
710 proto->canThrow() != CT_Cannot;
711}
712
713static bool stmtCanThrow(const Stmt *s) {
714 if (const auto *ce = dyn_cast<CallExpr>(Val: s)) {
715 const FunctionDecl *callee = ce->getDirectCallee();
716 if (!callee || functionCanThrow(d: callee))
717 return true;
718 }
719
720 // The destructor call of a bound temporary is not among the children.
721 if (const auto *te = dyn_cast<CXXBindTemporaryExpr>(Val: s))
722 if (functionCanThrow(d: te->getTemporary()->getDestructor()))
723 return true;
724
725 return llvm::any_of(Range: s->children(), P: [](const Stmt *child) {
726 return child && stmtCanThrow(s: child);
727 });
728}
729
730// Given a suspend expression which roughly looks like:
731//
732// auto && x = CommonExpr();
733// if (!x.await_ready()) {
734// x.await_suspend(...); (*)
735// }
736// x.await_resume();
737//
738// where the result of the entire expression is the result of x.await_resume()
739//
740// (*) If x.await_suspend return type is bool, it allows to veto a suspend:
741// if (x.await_suspend(...))
742// llvm_coro_suspend();
743//
744// This is more higher level than LLVM codegen, for that one see llvm's
745// docs/Coroutines.md for more details.
746namespace {
747struct LValueOrRValue {
748 LValue lv;
749 RValue rv;
750};
751} // namespace
752
753static bool isResumeExprInTryCatch(const CGCoroData &coro, cir::AwaitKind kind,
754 const CoroutineSuspendExpr &s) {
755 return coro.exceptionHandler && kind == cir::AwaitKind::Init &&
756 stmtCanThrow(s: s.getResumeExpr());
757}
758
759static LValueOrRValue
760emitSuspendExpression(CIRGenFunction &cgf, CGCoroData &coro,
761 CoroutineSuspendExpr const &s, cir::AwaitKind kind,
762 AggValueSlot aggSlot, bool ignoreResult,
763 mlir::Block *scopeParentBlock,
764 mlir::Value &tmpResumeRValAddr, bool forLValue) {
765 [[maybe_unused]] mlir::LogicalResult awaitBuild = mlir::success();
766 LValueOrRValue awaitRes;
767
768 CIRGenFunction::OpaqueValueMapping binder =
769 CIRGenFunction::OpaqueValueMapping(cgf, s.getOpaqueValue());
770 CIRGenBuilderTy &builder = cgf.getBuilder();
771
772 // Exception handling requires additional IR. We avoid generating it when
773 // neither the await_resume() call nor the destruction of its result can
774 // throw.
775 const bool resumeInTry = isResumeExprInTryCatch(coro, kind, s);
776
777 // Resume-expression temporaries live until the enclosing full-expression
778 // ends. Defer their cleanups and guard them with construction flags, since
779 // destroying the coroutine while suspended skips their construction.
780 // The initial-suspend try/catch cleans up its temporaries in the try block.
781 std::optional<CIRGenFunction::ConditionalEvaluation> resumeEval;
782 const bool resumeDefersCleanups =
783 !resumeInTry && CIRGenFunction::resumeExprDefersCleanups(s);
784 if (resumeDefersCleanups && cgf.currentFullExprCleanupScope)
785 resumeEval.emplace(args&: cgf, args: cgf.getLoc(srcLoc: s.getSourceRange()));
786 else if (resumeDefersCleanups)
787 cgf.cgm.errorNYI(s.getSourceRange(),
788 "resume expression cleanups without a full-expression "
789 "cleanup scope");
790 [[maybe_unused]] EHScopeStack::stable_iterator ehDepth =
791 cgf.ehStack.stable_begin();
792
793 // If the await_resume() result needs a destructor, take over its
794 // destruction, unless the destination owns it. The try/catch path destroys
795 // the result itself.
796 const CXXBindTemporaryExpr *resultBind = nullptr;
797 if (!resumeInTry && !aggSlot.isExternallyDestructed())
798 resultBind = dyn_cast<CXXBindTemporaryExpr>(Val: s.getResumeExpr());
799 if (resultBind) {
800 // Emit the result into a slot created outside of the cir.await, so that it
801 // is still available after it.
802 if (aggSlot.isIgnored())
803 aggSlot = cgf.createAggTemp(ty: resultBind->getType(),
804 loc: cgf.getLoc(srcLoc: resultBind->getSourceRange()),
805 name: "agg.tmp.ensured");
806 // Don't push the destructor from within the resume region, where its
807 // cir.cleanup.scope would capture the region's terminator.
808 aggSlot.setExternallyDestructed();
809 }
810
811 [[maybe_unused]] cir::AwaitOp awaitOp = cir::AwaitOp::create(
812 builder, location: cgf.getLoc(srcLoc: s.getSourceRange()), kind,
813 /*readyBuilder=*/
814 [&](mlir::OpBuilder &b, mlir::Location loc) {
815 Expr *condExpr = s.getReadyExpr()->IgnoreParens();
816 builder.createCondition(condition: cgf.evaluateExprAsBool(e: condExpr));
817 },
818 /*suspendBuilder=*/
819 [&](mlir::OpBuilder &b, mlir::Location loc) {
820 // Note that differently from LLVM codegen we do not emit coro.save
821 // and coro.suspend here, that should be done as part of lowering this
822 // to LLVM dialect (or some other MLIR dialect)
823
824 switch (s.getSuspendReturnType()) {
825 case CoroutineSuspendExpr::SuspendReturnType::SuspendBool: {
826 mlir::Value suspendRet = cgf.evaluateExprAsBool(e: s.getSuspendExpr());
827 // Veto suspension if requested by bool returning await_suspend.
828 cir::CoroSuspendPoint::create(builder, location: loc, cond: suspendRet);
829 break;
830 }
831 case CoroutineSuspendExpr::SuspendReturnType::SuspendVoid:
832 cgf.emitScalarExpr(e: s.getSuspendExpr());
833 // Signals the parent that execution flows to next region.
834 cir::CoroSuspendPoint::create(builder, location: loc);
835 break;
836 case CoroutineSuspendExpr::SuspendReturnType::SuspendHandle:
837 cgf.cgm.errorNYI(s.getSourceRange(),
838 "await_suspend returning handle");
839 cir::CoroSuspendPoint::create(builder, location: loc);
840 break;
841 }
842 },
843 /*resumeBuilder=*/
844 [&](mlir::OpBuilder &b, mlir::Location loc) {
845 if (resumeEval)
846 resumeEval->beginEvaluation();
847 if (resumeInTry) {
848 // we are basically just emitting:
849 // resumeEh = false;
850 // try {
851 // resumeExpr();
852 // resumeEh = true;
853 // } catch(...) {
854 // exceptionHandler();
855 // }
856 // Note the values of resumeEh are reversed from classic codegen,
857 // simply so we can use an 'IfOp' without a 'else' later.
858 ASTContext &ctx = cgf.getContext();
859 SourceLocation resumeLoc = s.getResumeExpr()->getExprLoc();
860 mlir::Location mlirLoc = cgf.getLoc(srcLoc: resumeLoc);
861 coro.resumeEHVar = cgf.createTempAlloca(
862 ty: builder.getBoolTy(), align: ctx.getTypeAlignInChars(T: ctx.BoolTy), loc: mlirLoc,
863 name: "resume.eh");
864 builder.createFlagStore(loc: mlirLoc, val: false,
865 dst: coro.resumeEHVar.getPointer());
866
867 CXXCatchStmt catchStmt(resumeLoc,
868 /*exDecl=*/nullptr, coro.exceptionHandler);
869 auto *tryBody =
870 CompoundStmt::Create(C: ctx, Stmts: s.getResumeExpr(), FPFeatures: FPOptionsOverride(),
871 LB: resumeLoc, RB: resumeLoc);
872 CXXTryStmt *tryStmt =
873 CXXTryStmt::Create(C: ctx, tryLoc: resumeLoc, tryBlock: tryBody, handlers: &catchStmt);
874
875 struct resumeEmitter final : CIRGenFunction::cxxTryBodyEmitter {
876 const CXXTryStmt &tryStmt;
877 const Expr *resumeExpr;
878 mlir::Location loc;
879 mlir::Value resumeEHVar;
880 resumeEmitter(const CXXTryStmt &tryStmt, const Expr *resumeExpr,
881 mlir::Location loc, Address resumeEHVar)
882 : tryStmt(tryStmt), resumeExpr(resumeExpr), loc(loc),
883 resumeEHVar(resumeEHVar.getPointer()) {}
884
885 mlir::LogicalResult operator()(CIRGenFunction &cgf) override {
886 // Destroy resume temporaries in the try block before marking
887 // resumption successful, so destructor exceptions are caught too.
888 CIRGenFunction::FullExprCleanupScope fullExpr(cgf, resumeExpr);
889 mlir::LogicalResult res =
890 cgf.emitStmt(s: tryStmt.getTryBlock(), /*useCurrentScope=*/true);
891 fullExpr.exit();
892 cgf.getBuilder().createFlagStore(loc, val: true, dst: resumeEHVar);
893 return res;
894 }
895
896 ~resumeEmitter() override = default;
897 } emitter{*tryStmt, s.getResumeExpr(), mlirLoc, coro.resumeEHVar};
898
899 awaitBuild = cgf.emitCXXTryStmt(s: *tryStmt, bodyCallback&: emitter);
900 // We are not supposed to obtain the value from init suspend
901 // await_resume().
902 awaitRes.rv = RValue::getIgnored();
903 } else if (forLValue) {
904 awaitRes.lv = cgf.emitLValue(e: s.getResumeExpr());
905 assert(awaitRes.lv.isSimple() &&
906 "expected simple lvalue for coroutine resume");
907 mlir::Value ptr = awaitRes.lv.getPointer();
908 llvm::StringRef tmpName = kind == cir::AwaitKind::Yield
909 ? "__coyield_resume_lval"
910 : "__coawait_resume_lval";
911 tmpResumeRValAddr = cgf.emitAlloca(
912 name: tmpName, ty: ptr.getType(), loc, alignment: cgf.getPointerAlign(),
913 ip: builder.getBestAllocaInsertPoint(block: scopeParentBlock));
914 builder.CIRBaseBuilderTy::createStore(loc, val: ptr, dst: tmpResumeRValAddr);
915 } else {
916 awaitRes.rv =
917 cgf.emitAnyExpr(e: s.getResumeExpr(), aggSlot, ignoreResult);
918 if (!awaitRes.rv.isIgnored()) {
919 RValue rv = awaitRes.rv;
920 if (rv.isScalar() || rv.isComplex()) {
921 mlir::Value value =
922 rv.isScalar() ? rv.getValue() : rv.getComplexValue();
923 tmpResumeRValAddr = cgf.emitAlloca(
924 name: "__coawait_resume_rval", ty: value.getType(), loc,
925 alignment: CharUnits::One(),
926 ip: builder.getBestAllocaInsertPoint(block: scopeParentBlock));
927 // Store the rvalue so we can reload it before the promise call.
928 builder.CIRBaseBuilderTy::createStore(loc, val: value,
929 dst: tmpResumeRValAddr);
930 } else {
931 assert(rv.isAggregate() && "unexpected rvalue kind");
932 }
933 }
934 }
935
936 if (resumeEval)
937 resumeEval->endEvaluation();
938
939 // Returns control back to parent.
940 cir::YieldOp::create(builder, location: loc);
941 });
942
943 assert((cgf.ehStack.stable_begin() == ehDepth ||
944 (resumeDefersCleanups && !resumeEval)) &&
945 "a resume expression cleanup was pushed in the resume region");
946
947 // Push the destructor right after the cir.await, where the result starts to
948 // exist, and not around it: destroying the coroutine at the suspend point
949 // would destroy the not yet constructed result. The cleanup runs at the end
950 // of the full-expression.
951 if (resultBind)
952 cgf.emitCXXTemporary(temporary: resultBind->getTemporary(), tempType: resultBind->getType(),
953 ptr: aggSlot.getAddress());
954
955 assert(awaitBuild.succeeded() && "Should know how to codegen");
956 return awaitRes;
957}
958
959static RValue emitSuspendExpr(CIRGenFunction &cgf,
960 const CoroutineSuspendExpr &e,
961 cir::AwaitKind kind, AggValueSlot aggSlot,
962 bool ignoreResult) {
963 RValue rval;
964 mlir::Location scopeLoc = cgf.getLoc(srcLoc: e.getSourceRange());
965
966 // Since we model suspend / resume as an inner region, we must store
967 // resume scalar results in a tmp alloca, and load it after we build the
968 // suspend expression. An alternative way to do this would be to make
969 // every region return a value when promise.return_value() is used, but
970 // it's a bit awkward given that resume is the only region that actually
971 // returns a value.
972 mlir::Block *currEntryBlock = cgf.curLexScope->getEntryBlock();
973 [[maybe_unused]] mlir::Value tmpResumeRValAddr;
974
975 // No need to explicitly wrap this into a scope since the AST already uses a
976 // ExprWithCleanups, which will wrap this into a cir.scope anyways.
977 rval = emitSuspendExpression(cgf, coro&: *cgf.curCoro.data, s: e, kind, aggSlot,
978 ignoreResult, scopeParentBlock: currEntryBlock, tmpResumeRValAddr,
979 /*forLValue*/ false)
980 .rv;
981
982 if (ignoreResult || rval.isIgnored())
983 return rval;
984
985 if (rval.isScalar()) {
986 rval = RValue::get(v: cir::LoadOp::create(builder&: cgf.getBuilder(), location: scopeLoc,
987 result: rval.getValue().getType(),
988 addr: tmpResumeRValAddr));
989 } else if (rval.isAggregate()) {
990 return rval;
991 } else { // complex
992 rval = RValue::getComplex(v: cir::LoadOp::create(
993 builder&: cgf.getBuilder(), location: scopeLoc, result: rval.getComplexValue().getType(),
994 addr: tmpResumeRValAddr));
995 }
996 return rval;
997}
998
999RValue CIRGenFunction::emitCoawaitExpr(const CoawaitExpr &e,
1000 AggValueSlot aggSlot,
1001 bool ignoreResult) {
1002 return emitSuspendExpr(cgf&: *this, e, kind: curCoro.data->currentAwaitKind, aggSlot,
1003 ignoreResult);
1004}
1005
1006RValue CIRGenFunction::emitCoyieldExpr(const CoyieldExpr &e,
1007 AggValueSlot aggSlot,
1008 bool ignoreResult) {
1009 return emitSuspendExpr(cgf&: *this, e, kind: cir::AwaitKind::Yield, aggSlot,
1010 ignoreResult);
1011}
1012
1013static QualType
1014getCoroutineSuspendExprReturnType(const ASTContext &ctx,
1015 const CoroutineSuspendExpr *e) {
1016 const auto *re = e->getResumeExpr();
1017 assert(isa<CallExpr>(re) && "unexpected suspend expression type");
1018 return cast<CallExpr>(Val: re)->getCallReturnType(Ctx: ctx);
1019}
1020
1021static LValue emitSuspendLValue(CIRGenFunction &cgf,
1022 const CoroutineSuspendExpr *e,
1023 cir::AwaitKind kind) {
1024 QualType retTy = getCoroutineSuspendExprReturnType(ctx: cgf.getContext(), e);
1025 assert(retTy->isReferenceType() &&
1026 "Can't have a scalar return unless the return type is a "
1027 "reference type!");
1028 mlir::Location scopeLoc = cgf.getLoc(srcLoc: e->getSourceRange());
1029 mlir::Block *currEntryBlock = cgf.curLexScope->getEntryBlock();
1030 mlir::Value tmpResumeLValAddr;
1031 LValue lv = emitSuspendExpression(cgf, coro&: *cgf.curCoro.data, s: *e, kind,
1032 aggSlot: AggValueSlot::ignored(),
1033 /*ignoreResult=*/false, scopeParentBlock: currEntryBlock,
1034 tmpResumeRValAddr&: tmpResumeLValAddr, /*forLValue=*/true)
1035 .lv;
1036 if (!tmpResumeLValAddr) {
1037 assert(isResumeExprInTryCatch(*cgf.curCoro.data, kind, *e) &&
1038 "expected resume in try/catch");
1039 QualType pointeeTy = retTy->getPointeeType();
1040 mlir::Type elemTy = cgf.convertType(t: pointeeTy);
1041 mlir::Value nullPtr = cgf.getBuilder().getNullPtr(
1042 ty: cgf.getBuilder().getPointerTo(ty: elemTy), loc: scopeLoc);
1043 return cgf.makeNaturalAlignAddrLValue(val: nullPtr, ty: pointeeTy);
1044 }
1045
1046 Address addr = lv.getAddress();
1047 mlir::Value loadedPtr =
1048 cir::LoadOp::create(builder&: cgf.getBuilder(), location: scopeLoc,
1049 result: addr.getPointer().getType(), addr: tmpResumeLValAddr);
1050 lv.setAddress(addr.withPointer(newPtr: loadedPtr));
1051 return lv;
1052}
1053
1054LValue CIRGenFunction::emitCoawaitLValue(const CoawaitExpr *e) {
1055 return emitSuspendLValue(cgf&: *this, e, kind: curCoro.data->currentAwaitKind);
1056}
1057
1058LValue CIRGenFunction::emitCoyieldLValue(const CoyieldExpr *e) {
1059 return emitSuspendLValue(cgf&: *this, e, kind: cir::AwaitKind::Yield);
1060}
1061
1062mlir::LogicalResult CIRGenFunction::emitCoreturnStmt(CoreturnStmt const &s) {
1063 ++curCoro.data->coreturnCount;
1064
1065 const Expr *rv = s.getOperand();
1066 if (rv && rv->getType()->isVoidType() && !isa<InitListExpr>(Val: rv)) {
1067 // Make sure to evaluate the non initlist expression of a co_return
1068 // with a void expression for side effects.
1069 RunCleanupsScope cleanupScope(*this);
1070 emitIgnoredExpr(e: rv);
1071 }
1072
1073 if (emitStmt(s: s.getPromiseCall(), /*useCurrentScope=*/true).failed())
1074 return mlir::failure();
1075 mlir::Location loc = getLoc(srcLoc: s.getSourceRange());
1076 cir::CoReturnOp::create(builder, location: loc);
1077 curLexScope->markCoreturnEmitted();
1078
1079 // Statements after a co_return may stil be reachable through a label.
1080 // Continue codegen.
1081 builder.createBlock(parent: builder.getBlock()->getParent());
1082 return mlir::success();
1083}
1084