| 1 | //===----- CGCoroutine.cpp - Emit CIR Code for C++ coroutines -------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This contains code dealing with C++ code generation of coroutines. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "CIRGenFunction.h" |
| 14 | #include "mlir/Support/LLVM.h" |
| 15 | #include "clang/AST/StmtCXX.h" |
| 16 | #include "clang/AST/StmtVisitor.h" |
| 17 | #include "clang/Basic/TargetInfo.h" |
| 18 | #include "clang/CIR/Dialect/IR/CIRDialect.h" |
| 19 | #include "clang/CIR/MissingFeatures.h" |
| 20 | |
| 21 | using namespace clang; |
| 22 | using namespace clang::CIRGen; |
| 23 | |
| 24 | struct clang::CIRGen::CGCoroData { |
| 25 | // What is the current await expression kind and how many |
| 26 | // await/yield expressions were encountered so far. |
| 27 | // These are used to generate pretty labels for await expressions in LLVM IR. |
| 28 | cir::AwaitKind currentAwaitKind = cir::AwaitKind::Init; |
| 29 | // Stores the __builtin_coro_id emitted in the function so that we can supply |
| 30 | // it as the first argument to other builtins. |
| 31 | cir::CoroIdOp coroId = nullptr; |
| 32 | |
| 33 | // Stores the result of __builtin_coro_begin call. |
| 34 | mlir::Value coroBegin = nullptr; |
| 35 | |
| 36 | // How many co_return statements are in the coroutine. Used to decide whether |
| 37 | // we need to add co_return; equivalent at the end of the user authored body. |
| 38 | unsigned coreturnCount = 0; |
| 39 | |
| 40 | // The promise type's 'unhandled_exception' handler, if it defines one. |
| 41 | Stmt *exceptionHandler = nullptr; |
| 42 | |
| 43 | // Stores the last emitted coro.free for the deallocate expressions, we use it |
| 44 | // to wrap dealloc code with if(auto mem = coro.free) dealloc(mem). |
| 45 | cir::CoroFreeOp lastCoroFree = nullptr; |
| 46 | |
| 47 | // A temporary bool alloca that stores whether 'await_resume' threw an |
| 48 | // exception. If it did, 'true' is stored in this variable, and the coroutine |
| 49 | // body must be skipped. If the promise type does not define an exception |
| 50 | // handler, this is null. |
| 51 | Address resumeEHVar = Address::invalid(); |
| 52 | |
| 53 | // If coro.id came from the builtin, remember the expression to give better |
| 54 | // diagnostic. If CoroIdExpr is nullptr, the coro.id was created by |
| 55 | // EmitCoroutineBody. |
| 56 | CallExpr const *coroIdExpr = nullptr; |
| 57 | }; |
| 58 | |
| 59 | // Defining these here allows to keep CGCoroData private to this file. |
| 60 | CIRGenFunction::CGCoroInfo::CGCoroInfo() {} |
| 61 | CIRGenFunction::CGCoroInfo::~CGCoroInfo() {} |
| 62 | |
| 63 | namespace { |
| 64 | // FIXME: both GetParamRef and ParamReferenceReplacerRAII are good template |
| 65 | // candidates to be shared among LLVM / CIR codegen. |
| 66 | |
| 67 | // Hunts for the parameter reference in the parameter copy/move declaration. |
| 68 | struct GetParamRef : public StmtVisitor<GetParamRef> { |
| 69 | public: |
| 70 | DeclRefExpr *expr = nullptr; |
| 71 | GetParamRef() {} |
| 72 | void VisitDeclRefExpr(DeclRefExpr *e) { |
| 73 | assert(expr == nullptr && "multilple declref in param move" ); |
| 74 | expr = e; |
| 75 | } |
| 76 | void VisitStmt(Stmt *s) { |
| 77 | for (Stmt *c : s->children()) { |
| 78 | if (c) |
| 79 | Visit(S: c); |
| 80 | } |
| 81 | } |
| 82 | }; |
| 83 | |
| 84 | // This class replaces references to parameters to their copies by changing |
| 85 | // the addresses in CGF.LocalDeclMap and restoring back the original values in |
| 86 | // its destructor. |
| 87 | struct ParamReferenceReplacerRAII { |
| 88 | CIRGenFunction::DeclMapTy savedLocals; |
| 89 | CIRGenFunction::DeclMapTy &localDeclMap; |
| 90 | |
| 91 | ParamReferenceReplacerRAII(CIRGenFunction::DeclMapTy &localDeclMap) |
| 92 | : localDeclMap(localDeclMap) {} |
| 93 | |
| 94 | void addCopy(const DeclStmt *pm) { |
| 95 | // Figure out what param it refers to. |
| 96 | |
| 97 | assert(pm->isSingleDecl()); |
| 98 | const VarDecl *vd = static_cast<const VarDecl *>(pm->getSingleDecl()); |
| 99 | const Expr *initExpr = vd->getInit(); |
| 100 | GetParamRef visitor; |
| 101 | visitor.Visit(S: const_cast<Expr *>(initExpr)); |
| 102 | assert(visitor.expr); |
| 103 | DeclRefExpr *dreOrig = visitor.expr; |
| 104 | auto *pd = dreOrig->getDecl(); |
| 105 | |
| 106 | auto it = localDeclMap.find(Val: pd); |
| 107 | assert(it != localDeclMap.end() && "parameter is not found" ); |
| 108 | savedLocals.insert(KV: {pd, it->second}); |
| 109 | |
| 110 | auto copyIt = localDeclMap.find(Val: vd); |
| 111 | assert(copyIt != localDeclMap.end() && "parameter copy is not found" ); |
| 112 | it->second = copyIt->getSecond(); |
| 113 | } |
| 114 | |
| 115 | ~ParamReferenceReplacerRAII() { |
| 116 | for (auto &&savedLocal : savedLocals) { |
| 117 | localDeclMap.insert(KV: {savedLocal.first, savedLocal.second}); |
| 118 | } |
| 119 | } |
| 120 | }; |
| 121 | } // namespace |
| 122 | |
| 123 | RValue CIRGenFunction::emitCoroutineFrame() { |
| 124 | if (curCoro.data && curCoro.data->coroBegin) { |
| 125 | return RValue::get(v: curCoro.data->coroBegin); |
| 126 | } |
| 127 | cgm.errorNYI(feature: "NYI" ); |
| 128 | return RValue(); |
| 129 | } |
| 130 | |
| 131 | static void createCoroData(CIRGenFunction &cgf, |
| 132 | CIRGenFunction::CGCoroInfo &curCoro, |
| 133 | cir::CoroIdOp coroId, |
| 134 | CallExpr const *coroIdExpr = nullptr) { |
| 135 | |
| 136 | if (curCoro.data) { |
| 137 | if (curCoro.data->coroIdExpr) |
| 138 | cgf.cgm.error(loc: coroIdExpr->getBeginLoc(), |
| 139 | error: "only one __builtin_coro_id can be used in a function" ); |
| 140 | else if (coroIdExpr) |
| 141 | cgf.cgm.error(loc: coroIdExpr->getBeginLoc(), |
| 142 | error: "__builtin_coro_id shall not be used in a C++ coroutine" ); |
| 143 | else |
| 144 | llvm_unreachable("EmitCoroutineBodyStatement called twice?" ); |
| 145 | |
| 146 | return; |
| 147 | } |
| 148 | |
| 149 | curCoro.data = std::make_unique<CGCoroData>(); |
| 150 | curCoro.data->coroId = coroId; |
| 151 | curCoro.data->coroIdExpr = coroIdExpr; |
| 152 | } |
| 153 | |
| 154 | static mlir::LogicalResult |
| 155 | emitBodyAndFallthrough(CIRGenFunction &cgf, const CoroutineBodyStmt &s, |
| 156 | Stmt *body, |
| 157 | const CIRGenFunction::LexicalScope *currLexScope) { |
| 158 | { |
| 159 | // Destroy the body's local variables, and the temporaries they extend, when |
| 160 | // the body ends, before the fall-through handler, the implicit |
| 161 | // `co_return;`, calls return_void(). The body stays in the current lexical |
| 162 | // scope so that a co_return at its top level is recorded in currLexScope. |
| 163 | CIRGenFunction::RunCleanupsScope bodyScope(cgf); |
| 164 | if (cgf.emitStmt(s: body, /*useCurrentScope=*/true).failed()) |
| 165 | return mlir::failure(); |
| 166 | } |
| 167 | |
| 168 | // Note that classic codegen checks CanFallthrough by looking into the |
| 169 | // availability of the insert block which is kinda brittle and unintuitive, |
| 170 | // seems to be related with how landing pads are handled. |
| 171 | // |
| 172 | // CIRGen handles this by checking pre-existing co_returns in the current |
| 173 | // scope instead. |
| 174 | |
| 175 | // From LLVM IR Gen: const bool CanFallthrough = Builder.GetInsertBlock(); |
| 176 | const bool canFallthrough = !currLexScope->hasCoreturn(); |
| 177 | if (canFallthrough) |
| 178 | if (Stmt *onFallthrough = s.getFallthroughHandler()) |
| 179 | if (cgf.emitStmt(s: onFallthrough, /*useCurrentScope=*/true).failed()) |
| 180 | return mlir::failure(); |
| 181 | |
| 182 | return mlir::success(); |
| 183 | } |
| 184 | |
| 185 | cir::CoroIdOp CIRGenFunction::emitCoroIDBuiltinCall(const CallExpr *e) { |
| 186 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 187 | |
| 188 | llvm::SmallVector<mlir::Value, 4> args; |
| 189 | for (const Expr *arg : e->arguments()) |
| 190 | args.push_back(Elt: emitScalarExpr(e: arg)); |
| 191 | |
| 192 | auto coroId = cir::CoroIdOp::create(builder&: cgm.getBuilder(), location: loc, operands: args); |
| 193 | createCoroData(cgf&: *this, curCoro, coroId, coroIdExpr: e); |
| 194 | return coroId; |
| 195 | } |
| 196 | |
| 197 | cir::CoroAllocOp CIRGenFunction::emitCoroAllocBuiltinCall(const CallExpr *e) { |
| 198 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 199 | if (!curCoro.data || !curCoro.data->coroId) { |
| 200 | cgm.error(loc: e->getBeginLoc(), error: "this builtin expect that __builtin_coro_id has" |
| 201 | " been used earlier in this function" ); |
| 202 | return {}; |
| 203 | } |
| 204 | |
| 205 | return cir::CoroAllocOp::create( |
| 206 | builder&: cgm.getBuilder(), location: loc, |
| 207 | operands: mlir::ValueRange{curCoro.data->coroId.getResult()}); |
| 208 | } |
| 209 | |
| 210 | cir::CoroBeginOp CIRGenFunction::emitCoroBeginBuiltinCall(const CallExpr *e) { |
| 211 | |
| 212 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 213 | if (!curCoro.data || !curCoro.data->coroId) { |
| 214 | cgm.error(loc: e->getBeginLoc(), error: "this builtin expect that __builtin_coro_id has" |
| 215 | " been used earlier in this function" ); |
| 216 | return {}; |
| 217 | } |
| 218 | llvm::SmallVector<mlir::Value, 2> args; |
| 219 | args.push_back(Elt: curCoro.data->coroId.getResult()); |
| 220 | for (const Expr *arg : e->arguments()) |
| 221 | args.push_back(Elt: emitScalarExpr(e: arg)); |
| 222 | |
| 223 | auto coroBegin = cir::CoroBeginOp::create(builder&: cgm.getBuilder(), location: loc, operands: args); |
| 224 | curCoro.data->coroBegin = coroBegin; |
| 225 | return coroBegin; |
| 226 | } |
| 227 | |
| 228 | cir::CoroEndOp CIRGenFunction::emitCoroEndBuiltinCall(const CallExpr *e) { |
| 229 | |
| 230 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 231 | CIRGenBuilderTy &builder = cgm.getBuilder(); |
| 232 | llvm::SmallVector<mlir::Value, 3> args; |
| 233 | for (const Expr *arg : e->arguments()) |
| 234 | args.push_back(Elt: emitScalarExpr(e: arg)); |
| 235 | args.push_back(Elt: cir::TokenNoneOp::create(builder, location: loc)); |
| 236 | return cir::CoroEndOp::create(builder, location: loc, resultTypes: {cgm.voidTy}, operands: args); |
| 237 | } |
| 238 | |
| 239 | cir::CoroFreeOp CIRGenFunction::emitCoroFreeBuiltin(const CallExpr *e) { |
| 240 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 241 | |
| 242 | if (!curCoro.data || !curCoro.data->coroId) { |
| 243 | cgm.error(loc: e->getBeginLoc(), error: "this builtin expect that __builtin_coro_id has" |
| 244 | " been used earlier in this function" ); |
| 245 | return {}; |
| 246 | } |
| 247 | |
| 248 | auto coroFree = |
| 249 | cir::CoroFreeOp::create(builder&: cgm.getBuilder(), location: loc, |
| 250 | operands: mlir::ValueRange{curCoro.data->coroId.getResult(), |
| 251 | curCoro.data->coroBegin}); |
| 252 | |
| 253 | curCoro.data->lastCoroFree = coroFree; |
| 254 | return coroFree; |
| 255 | } |
| 256 | |
| 257 | cir::CoroSizeOp CIRGenFunction::emitCoroSizeBuiltinCall(const CallExpr *e) { |
| 258 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 259 | return cir::CoroSizeOp::create(builder&: cgm.getBuilder(), location: loc, |
| 260 | result: convertType(t: e->getType())); |
| 261 | } |
| 262 | |
| 263 | cir::CoroAlignOp CIRGenFunction::emitCoroAlignBuiltinCall(const CallExpr *e) { |
| 264 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 265 | return cir::CoroAlignOp::create(builder&: cgm.getBuilder(), location: loc, |
| 266 | result: convertType(t: e->getType())); |
| 267 | } |
| 268 | |
| 269 | cir::CoroPromiseOp |
| 270 | CIRGenFunction::emitCoroPromiseBuiltinCall(const CallExpr *e) { |
| 271 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 272 | |
| 273 | llvm::SmallVector<mlir::Value, 3> args; |
| 274 | for (const Expr *arg : e->arguments()) |
| 275 | args.push_back(Elt: emitScalarExpr(e: arg)); |
| 276 | |
| 277 | auto coroPromise = cir::CoroPromiseOp::create(builder&: cgm.getBuilder(), location: loc, operands: args); |
| 278 | return coroPromise; |
| 279 | } |
| 280 | |
| 281 | cir::CoroDoneOp CIRGenFunction::emitCoroDoneBuiltinCall(const CallExpr *e) { |
| 282 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 283 | return cir::CoroDoneOp::create(builder&: cgm.getBuilder(), location: loc, |
| 284 | handle: emitScalarExpr(e: e->getArg(Arg: 0))); |
| 285 | } |
| 286 | |
| 287 | cir::CoroResumeOp CIRGenFunction::emitCoroResumeBuiltinCall(const CallExpr *e) { |
| 288 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 289 | return cir::CoroResumeOp::create(builder&: cgm.getBuilder(), location: loc, |
| 290 | handle: emitScalarExpr(e: e->getArg(Arg: 0))); |
| 291 | } |
| 292 | |
| 293 | cir::CoroDestroyOp |
| 294 | CIRGenFunction::emitCoroDestroyBuiltinCall(const CallExpr *e) { |
| 295 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 296 | return cir::CoroDestroyOp::create(builder&: cgm.getBuilder(), location: loc, |
| 297 | handle: emitScalarExpr(e: e->getArg(Arg: 0))); |
| 298 | } |
| 299 | |
| 300 | cir::CoroNoopOp CIRGenFunction::emitCoroNoopBuiltinCall(const CallExpr *e) { |
| 301 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 302 | return cir::CoroNoopOp::create(builder&: cgm.getBuilder(), location: loc); |
| 303 | } |
| 304 | |
| 305 | cir::CoroSuspendOp |
| 306 | CIRGenFunction::emitCoroSuspendBuiltinCall(const CallExpr *e) { |
| 307 | mlir::Location loc = getLoc(srcLoc: e->getBeginLoc()); |
| 308 | llvm::SmallVector<mlir::Value, 2> args; |
| 309 | args.push_back(Elt: cir::TokenNoneOp::create(builder, location: loc)); |
| 310 | args.push_back(Elt: emitScalarExpr(e: e->getArg(Arg: 0))); |
| 311 | |
| 312 | return cir::CoroSuspendOp::create(builder&: cgm.getBuilder(), location: loc, operands: args); |
| 313 | } |
| 314 | |
| 315 | static mlir::LogicalResult |
| 316 | coroutineBodyExceptionHelper(CIRGenFunction &cgf, const CoroutineBodyStmt &s) { |
| 317 | |
| 318 | CXXCatchStmt catchStmt(s.getBeginLoc(), /*exDecl=*/nullptr, |
| 319 | cgf.curCoro.data->exceptionHandler); |
| 320 | auto *tryStmt = CXXTryStmt::Create(C: cgf.getContext(), tryLoc: s.getBeginLoc(), |
| 321 | tryBlock: s.getBody(), handlers: &catchStmt); |
| 322 | struct handlerEmitter final : CIRGenFunction::cxxTryBodyEmitter { |
| 323 | const CoroutineBodyStmt &s; |
| 324 | |
| 325 | handlerEmitter(const CoroutineBodyStmt &s) : s(s) /*, scope(scope)*/ {} |
| 326 | mlir::LogicalResult operator()(CIRGenFunction &cgf) override { |
| 327 | return emitBodyAndFallthrough(cgf, s, body: s.getBody(), currLexScope: cgf.curLexScope); |
| 328 | } |
| 329 | ~handlerEmitter() override = default; |
| 330 | } emitter{s}; |
| 331 | |
| 332 | mlir::LogicalResult res = cgf.emitCXXTryStmt(s: *tryStmt, bodyCallback&: emitter); |
| 333 | |
| 334 | return res; |
| 335 | } |
| 336 | |
| 337 | mlir::LogicalResult |
| 338 | CIRGenFunction::emitCoroutineBody(const CoroutineBodyStmt &s) { |
| 339 | mlir::Location openCurlyLoc = getLoc(srcLoc: s.getBeginLoc()); |
| 340 | cir::ConstantOp nullPtrCst = builder.getNullPtr(ty: voidPtrTy, loc: openCurlyLoc); |
| 341 | |
| 342 | auto fn = mlir::cast<cir::FuncOp>(Val: curFn); |
| 343 | fn.setCoroutineKind(cir::CoroutineKind::Coroutine); |
| 344 | const TargetInfo &ti = cgm.getASTContext().getTargetInfo(); |
| 345 | unsigned newAlign = ti.getNewAlign() / ti.getCharWidth(); |
| 346 | |
| 347 | cir::CoroIdOp coroId = cir::CoroIdOp::create( |
| 348 | builder&: cgm.getBuilder(), location: openCurlyLoc, |
| 349 | operands: mlir::ValueRange{builder.getUInt32(c: newAlign, loc: openCurlyLoc), nullPtrCst, |
| 350 | nullPtrCst, nullPtrCst}); |
| 351 | createCoroData(cgf&: *this, curCoro, coroId); |
| 352 | |
| 353 | // Backend is allowed to elide memory allocations, to help it, emit |
| 354 | // auto mem = coro.alloc() ? 0 : ... allocation code ...; |
| 355 | cir::CoroAllocOp coroAlloc = cir::CoroAllocOp::create( |
| 356 | builder&: cgm.getBuilder(), location: openCurlyLoc, |
| 357 | operands: mlir::ValueRange{curCoro.data->coroId.getResult()}); |
| 358 | |
| 359 | // Initialize address of coroutine frame to null |
| 360 | CanQualType astVoidPtrTy = cgm.getASTContext().VoidPtrTy; |
| 361 | mlir::Type allocaTy = convertTypeForMem(t: astVoidPtrTy); |
| 362 | Address coroFrame = |
| 363 | createTempAlloca(ty: allocaTy, align: getContext().getTypeAlignInChars(T: astVoidPtrTy), |
| 364 | loc: openCurlyLoc, name: "__coro_frame_addr" , |
| 365 | /*ArraySize=*/arraySize: nullptr); |
| 366 | |
| 367 | mlir::Value storeAddr = coroFrame.getPointer(); |
| 368 | builder.CIRBaseBuilderTy::createStore(loc: openCurlyLoc, val: nullPtrCst, dst: storeAddr); |
| 369 | mlir::LogicalResult res = mlir::success(); |
| 370 | cir::IfOp::create( |
| 371 | builder, location: openCurlyLoc, cond: coroAlloc.getResult(), |
| 372 | /*withElseRegion=*/false, |
| 373 | /*thenBuilder=*/[&](mlir::OpBuilder &b, mlir::Location loc) { |
| 374 | mlir::Value allocatedPtr = emitScalarExpr(e: s.getAllocate()); |
| 375 | builder.CIRBaseBuilderTy::createStore(loc, val: allocatedPtr, dst: storeAddr); |
| 376 | // Handle allocation failure if 'ReturnStmtOnAllocFailure' was provided. |
| 377 | if (Stmt *retOnAllocFailure = s.getReturnStmtOnAllocFailure()) { |
| 378 | mlir::Value isPtrNull = builder.createPtrIsNull(ptr: allocatedPtr); |
| 379 | assert(!cir::MissingFeatures::emitCondLikelihoodViaExpectIntrinsic()); |
| 380 | cir::IfOp::create(builder, location: loc, cond: isPtrNull, /*withElseRegion=*/false, |
| 381 | thenBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 382 | res = emitStmt(s: retOnAllocFailure, |
| 383 | /*useCurrentScope=*/true); |
| 384 | cir::UnreachableOp::create(builder, location: loc); |
| 385 | }); |
| 386 | } |
| 387 | cir::YieldOp::create(builder, location: loc); |
| 388 | }); |
| 389 | |
| 390 | if (res.failed()) |
| 391 | return res; |
| 392 | |
| 393 | curCoro.data->coroBegin = cir::CoroBeginOp::create( |
| 394 | builder&: cgm.getBuilder(), location: openCurlyLoc, |
| 395 | operands: mlir::ValueRange{ |
| 396 | curCoro.data->coroId.getResult(), |
| 397 | cir::LoadOp::create(builder, location: openCurlyLoc, result: allocaTy, addr: storeAddr)}); |
| 398 | |
| 399 | { |
| 400 | assert(!cir::MissingFeatures::generateDebugInfo()); |
| 401 | ParamReferenceReplacerRAII paramReplacer(localDeclMap); |
| 402 | |
| 403 | // Create mapping between parameters and copy-params for coroutine |
| 404 | // function. |
| 405 | llvm::ArrayRef<const Stmt *> paramMoves = s.getParamMoves(); |
| 406 | assert((paramMoves.size() == 0 || (paramMoves.size() == fnArgs.size())) && |
| 407 | "ParamMoves and FnArgs should be the same size for coroutine " |
| 408 | "function" ); |
| 409 | // For zipping the arg map into debug info. |
| 410 | assert(!cir::MissingFeatures::generateDebugInfo()); |
| 411 | |
| 412 | // Capture frame cleanups for destroy: promise first, then parameter copies, |
| 413 | // before freeing the frame ([dcl.fct.def.coroutine]p11, p13). Local cleanup |
| 414 | // scopes would end their lifetimes too early. |
| 415 | |
| 416 | assert(!cir::MissingFeatures::coroUnwindRegion()); |
| 417 | CapturedCleanups frameCleanups(*this, /*active=*/true); |
| 418 | // Whether the destructor of the promise or of a parameter copy may throw. |
| 419 | bool frameDestructorMayThrow = false; |
| 420 | auto emitFrameObject = [&](const VarDecl &var) { |
| 421 | emitAutoVarDeclWithCapturedCleanups(d: var, cleanups&: frameCleanups); |
| 422 | if (destructorMayThrow(type: var.getType())) |
| 423 | frameDestructorMayThrow = true; |
| 424 | }; |
| 425 | |
| 426 | // Create parameter copies. We do it before creating a promise, since an |
| 427 | // evolution of coroutine TS may allow promise constructor to observe |
| 428 | // parameter copies. |
| 429 | assert(!cir::MissingFeatures::coroOutsideFrameMD()); |
| 430 | for (auto *pm : paramMoves) { |
| 431 | const auto *paramMove = cast<DeclStmt>(Val: pm); |
| 432 | emitFrameObject(cast<VarDecl>(Val: *paramMove->getSingleDecl())); |
| 433 | paramReplacer.addCopy(pm: paramMove); |
| 434 | } |
| 435 | |
| 436 | // Builds `initial_suspend`. |
| 437 | auto initialSuspendBuilder = [&]() -> mlir::LogicalResult { |
| 438 | emitFrameObject(*s.getPromiseDecl()); |
| 439 | // Update coro.id to refer to the promise. We could not do it earlier |
| 440 | // because the promise local variable was not emitted yet. |
| 441 | Address promiseAddr = getAddrOfLocalVar(vd: s.getPromiseDecl()); |
| 442 | { |
| 443 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 444 | builder.setInsertionPoint(coroId); |
| 445 | mlir::Value promisePtr = builder.createBitcast( |
| 446 | src: promiseAddr.emitRawPointer(), newTy: builder.getVoidPtrTy()); |
| 447 | coroId.getPromiseMutable().assign(value: promisePtr); |
| 448 | } |
| 449 | // returnValue should be valid as long as the coroutine's return type |
| 450 | // is not void. The assertion could help us to reduce the check later. |
| 451 | assert(returnValue.isValid() == (bool)s.getReturnStmt()); |
| 452 | // Now we have the promise, initialize the GRO. |
| 453 | // We need to emit `get_return_object` first. According to: |
| 454 | // [dcl.fct.def.coroutine]p7 |
| 455 | // The call to get_return_Âobject is sequenced before the call to |
| 456 | // initial_suspend and is invoked at most once. |
| 457 | // |
| 458 | // So we couldn't emit return value when we emit return statment, |
| 459 | // otherwise the call to get_return_object wouldn't be in front |
| 460 | // of initial_suspend. |
| 461 | if (returnValue.isValid()) { |
| 462 | // Sema may omit ExprWithCleanups when the return types match. Clean |
| 463 | // up the GRO full-expression with its temporaries here explicitly. |
| 464 | FullExprCleanupScope groScope(*this, s.getReturnValue()); |
| 465 | emitAnyExprToMem(e: s.getReturnValue(), location: returnValue, |
| 466 | quals: s.getReturnValue()->getType().getQualifiers(), |
| 467 | /*isInit*/ isInitializer: true); |
| 468 | } |
| 469 | |
| 470 | curCoro.data->currentAwaitKind = cir::AwaitKind::Init; |
| 471 | curCoro.data->exceptionHandler = s.getExceptionHandler(); |
| 472 | |
| 473 | if (emitStmt(s: s.getInitSuspendStmt(), /*useCurrentScope=*/true).failed()) |
| 474 | return mlir::failure(); |
| 475 | |
| 476 | cir::YieldOp::create(builder, location: openCurlyLoc); |
| 477 | return mlir::success(); |
| 478 | }; |
| 479 | |
| 480 | // Builds `body`: the user-written coroutine code, including its |
| 481 | // implicit `try { ... } catch (...) { unhandled_exception(); }`. |
| 482 | auto bodyBuilder = [&]() -> mlir::LogicalResult { |
| 483 | RunCleanupsScope bodyRegionScope(*this); |
| 484 | curCoro.data->currentAwaitKind = cir::AwaitKind::User; |
| 485 | |
| 486 | if (curCoro.data->exceptionHandler) { |
| 487 | // This bit of code is supposed to do: |
| 488 | // |
| 489 | // if (await-resume-didnt-throw-exception) { |
| 490 | // try { |
| 491 | // coroutine-body |
| 492 | // } catch (...) { |
| 493 | // unhandled_exception(); |
| 494 | // } |
| 495 | // } |
| 496 | // |
| 497 | // IF resume couldn't have thrown an exception(await_resume is |
| 498 | // noexcept), we skip the 'if'. |
| 499 | // |
| 500 | // Note that we've reversed the condition of the 'if' from classic |
| 501 | // codegen so that we don't need an 'else' block. |
| 502 | if (curCoro.data->resumeEHVar.isValid()) { |
| 503 | mlir::Value shouldSkip = builder.createFlagLoad( |
| 504 | loc: openCurlyLoc, addr: curCoro.data->resumeEHVar.getPointer()); |
| 505 | mlir::LogicalResult res = mlir::success(); |
| 506 | cir::IfOp::create(builder, location: openCurlyLoc, cond: shouldSkip, |
| 507 | /*withElseRegion=*/false, |
| 508 | thenBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 509 | res = coroutineBodyExceptionHelper(cgf&: *this, s); |
| 510 | builder.createYield(loc: openCurlyLoc); |
| 511 | }); |
| 512 | |
| 513 | if (res.failed()) |
| 514 | return mlir::failure(); |
| 515 | |
| 516 | } else if (coroutineBodyExceptionHelper(cgf&: *this, s).failed()) { |
| 517 | return mlir::failure(); |
| 518 | } |
| 519 | } else if (emitBodyAndFallthrough(cgf&: *this, s, body: s.getBody(), currLexScope: curLexScope) |
| 520 | .failed()) { |
| 521 | return mlir::failure(); |
| 522 | } |
| 523 | |
| 524 | return mlir::success(); |
| 525 | }; |
| 526 | |
| 527 | // Builds `final_suspend`: only emitted at all if the body can actually |
| 528 | // reach it (an explicit co_return, or falling off the end). |
| 529 | auto finalSuspendBuilder = [&]() -> mlir::LogicalResult { |
| 530 | // Note that LLVM checks CanFallthrough by looking into the availability |
| 531 | // of the insert block which is kinda brittle and unintuitive, seems to be |
| 532 | // related with how landing pads are handled. |
| 533 | // |
| 534 | // CIRGen handles this by checking pre-existing co_returns in the current |
| 535 | // scope instead. |
| 536 | // |
| 537 | // From LLVM IR Gen: const bool CanFallthrough = Builder.GetInsertBlock(); |
| 538 | const bool canFallthrough = curLexScope->hasCoreturn(); |
| 539 | const bool hasCoreturns = curCoro.data->coreturnCount > 0; |
| 540 | if (canFallthrough || hasCoreturns) { |
| 541 | curCoro.data->currentAwaitKind = cir::AwaitKind::Final; |
| 542 | if (emitStmt(s: s.getFinalSuspendStmt(), /*useCurrentScope=*/true) |
| 543 | .failed()) |
| 544 | return mlir::failure(); |
| 545 | } |
| 546 | cir::YieldOp::create(builder, location: openCurlyLoc); |
| 547 | return mlir::success(); |
| 548 | }; |
| 549 | |
| 550 | // Emit "if (coro.free(CoroId, CoroBegin)) Deallocate;". |
| 551 | auto emitDeallocate = [&]() -> mlir::LogicalResult { |
| 552 | Stmt *deallocate = s.getDeallocate(); |
| 553 | if (emitStmt(s: deallocate, /*useCurrentScope=*/true).failed()) { |
| 554 | cgm.error(loc: deallocate->getBeginLoc(), |
| 555 | error: "failed to emit coroutine deallocation expression" ); |
| 556 | return mlir::failure(); |
| 557 | } |
| 558 | |
| 559 | cir::CoroFreeOp coroFree = curCoro.data->lastCoroFree; |
| 560 | |
| 561 | if (!coroFree) { |
| 562 | cgm.error(loc: deallocate->getBeginLoc(), |
| 563 | error: "Deallocation expression does not refer to coro.free" ); |
| 564 | return mlir::failure(); |
| 565 | } |
| 566 | { |
| 567 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 568 | builder.setInsertionPointAfter(coroFree); |
| 569 | mlir::Value isPtrNotNull = |
| 570 | builder.createPtrIsNotNull(ptr: coroFree.getResult()); |
| 571 | |
| 572 | llvm::SmallVector<mlir::Operation *> opsToMove; |
| 573 | mlir::Block *block = builder.getInsertionBlock(); |
| 574 | mlir::Block::iterator it(isPtrNotNull.getDefiningOp()); |
| 575 | |
| 576 | for (++it; it != block->end(); ++it) |
| 577 | opsToMove.push_back(Elt: &*it); |
| 578 | |
| 579 | auto ifOp = cir::IfOp::create( |
| 580 | builder, location: getLoc(srcLoc: deallocate->getSourceRange()), cond: isPtrNotNull, |
| 581 | /*withElseRegion*/ false, |
| 582 | thenBuilder: [&](mlir::OpBuilder &builder, mlir::Location loc) { |
| 583 | cir::YieldOp::create(builder, location: loc); |
| 584 | }); |
| 585 | |
| 586 | mlir::Operation *yieldOp = ifOp.getThenRegion().back().getTerminator(); |
| 587 | for (auto *op : opsToMove) |
| 588 | op->moveBefore(existingOp: yieldOp); |
| 589 | } |
| 590 | return mlir::success(); |
| 591 | }; |
| 592 | |
| 593 | // Destroy the promise and the parameter copies, then free the frame. If one |
| 594 | // of the destructors may throw an exception that can leave the coroutine, |
| 595 | // emit them like classic CodeGen's cleanups: the deallocation is the |
| 596 | // cleanup of an outer cir.cleanup.scope, and each destructor has its own |
| 597 | // nested one, the first constructed object outermost. An exception from one |
| 598 | // of the destructors then still destroys the objects constructed before it |
| 599 | // and frees the frame. |
| 600 | auto destroyBuilder = [&]() -> mlir::LogicalResult { |
| 601 | if (!getLangOpts().Exceptions || inEHSpecTerminateScope() || |
| 602 | !frameDestructorMayThrow) { |
| 603 | frameCleanups.emitIntoCleanupRegion(loc: openCurlyLoc); |
| 604 | if (emitDeallocate().failed()) |
| 605 | return mlir::failure(); |
| 606 | cir::YieldOp::create(builder, location: openCurlyLoc); |
| 607 | return mlir::success(); |
| 608 | } |
| 609 | |
| 610 | mlir::LogicalResult deallocRes = mlir::success(); |
| 611 | auto deallocScope = cir::CleanupScopeOp::create( |
| 612 | builder, location: openCurlyLoc, cleanupKind: cir::CleanupKind::All, |
| 613 | /*bodyBuilder=*/[](mlir::OpBuilder &, mlir::Location) {}, |
| 614 | /*cleanupBuilder=*/ |
| 615 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 616 | deallocRes = emitDeallocate(); |
| 617 | cir::YieldOp::create(builder&: b, location: loc); |
| 618 | }); |
| 619 | builder.setInsertionPointToEnd(&deallocScope.getBodyRegion().back()); |
| 620 | frameCleanups.emitAsNestedScopes(loc: openCurlyLoc); |
| 621 | cir::YieldOp::create(builder, location: openCurlyLoc); |
| 622 | builder.setInsertionPointAfter(deallocScope); |
| 623 | cir::YieldOp::create(builder, location: openCurlyLoc); |
| 624 | return deallocRes; |
| 625 | }; |
| 626 | |
| 627 | // Builds `exit`: coro.end(/*unwind*/ false) followed by the actual return |
| 628 | // to the caller. |
| 629 | |
| 630 | assert(!cir::MissingFeatures::coroGetRetObjManager()); |
| 631 | auto exitBuilder = [&]() { |
| 632 | cir::ConstantOp nullHandler = |
| 633 | builder.getNullPtr(ty: builder.getVoidPtrTy(), loc: openCurlyLoc); |
| 634 | cir::ConstantOp noUnwind = builder.getBool(state: false, loc: openCurlyLoc); |
| 635 | auto tkNone = cir::TokenNoneOp::create(builder, location: openCurlyLoc); |
| 636 | cir::CoroEndOp::create(builder, location: openCurlyLoc, handle: nullHandler, unwind: noUnwind, |
| 637 | resultToken: tkNone); |
| 638 | |
| 639 | if (auto *ret = cast_or_null<ReturnStmt>(Val: s.getReturnStmt())) { |
| 640 | // Since we already emitted the return value above, so we shouldn't |
| 641 | // emit it again here. |
| 642 | Expr *previousRetValue = ret->getRetValue(); |
| 643 | ret->setRetValue(nullptr); |
| 644 | if (emitStmt(s: ret, /*useCurrentScope=*/true).failed()) |
| 645 | return mlir::failure(); |
| 646 | mlir::Block *block = builder.getInsertionBlock(); |
| 647 | // emitReturnStmt() always creates a new insertion block after emitting |
| 648 | // the return. That block is unreachable in this case, so erase it. |
| 649 | block->erase(); |
| 650 | // Set the return value back. The code generator, as the AST |
| 651 | // **Consumer**, shouldn't change the AST. |
| 652 | ret->setRetValue(previousRetValue); |
| 653 | } else { |
| 654 | cir::ReturnOp::create(builder, location: openCurlyLoc); |
| 655 | } |
| 656 | |
| 657 | return mlir::success(); |
| 658 | }; |
| 659 | |
| 660 | cir::CoroutineOp coro = cir::CoroutineOp::create( |
| 661 | builder, location: openCurlyLoc, |
| 662 | /*initialSuspendBuilder=*/ |
| 663 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 664 | if (initialSuspendBuilder().failed()) |
| 665 | res = mlir::failure(); |
| 666 | }, |
| 667 | /*bodyBuilder=*/ |
| 668 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 669 | if (bodyBuilder().failed()) |
| 670 | res = mlir::failure(); |
| 671 | }, |
| 672 | /*finalSuspendBuilder=*/ |
| 673 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 674 | if (finalSuspendBuilder().failed()) |
| 675 | res = mlir::failure(); |
| 676 | }, |
| 677 | /*destroyBuilder=*/ |
| 678 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 679 | if (destroyBuilder().failed()) |
| 680 | res = mlir::failure(); |
| 681 | }, |
| 682 | /*exitBuilder=*/ |
| 683 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 684 | if (exitBuilder().failed()) |
| 685 | res = mlir::failure(); |
| 686 | }); |
| 687 | |
| 688 | if (coro && !coro.getBody().empty()) { |
| 689 | mlir::Block &lastBlock = coro.getBody().back(); |
| 690 | mlir::Block *prevBlock = lastBlock.getPrevNode(); |
| 691 | if (lastBlock.empty() && lastBlock.hasNoPredecessors() && prevBlock && |
| 692 | !prevBlock->empty() && isa<cir::CoReturnOp>(Val: prevBlock->back())) { |
| 693 | lastBlock.erase(); |
| 694 | } else if (!lastBlock.mightHaveTerminator()) { |
| 695 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 696 | builder.setInsertionPointToEnd(&lastBlock); |
| 697 | cir::YieldOp::create(builder, location: openCurlyLoc); |
| 698 | } |
| 699 | } |
| 700 | } |
| 701 | |
| 702 | return res; |
| 703 | } |
| 704 | |
| 705 | static bool functionCanThrow(const FunctionDecl *d) { |
| 706 | const auto *proto = d->getType()->getAs<FunctionProtoType>(); |
| 707 | if (!proto) |
| 708 | return true; |
| 709 | return !isNoexceptExceptionSpec(ESpecType: proto->getExceptionSpecType()) || |
| 710 | proto->canThrow() != CT_Cannot; |
| 711 | } |
| 712 | |
| 713 | static bool stmtCanThrow(const Stmt *s) { |
| 714 | if (const auto *ce = dyn_cast<CallExpr>(Val: s)) { |
| 715 | const FunctionDecl *callee = ce->getDirectCallee(); |
| 716 | if (!callee || functionCanThrow(d: callee)) |
| 717 | return true; |
| 718 | } |
| 719 | |
| 720 | // The destructor call of a bound temporary is not among the children. |
| 721 | if (const auto *te = dyn_cast<CXXBindTemporaryExpr>(Val: s)) |
| 722 | if (functionCanThrow(d: te->getTemporary()->getDestructor())) |
| 723 | return true; |
| 724 | |
| 725 | return llvm::any_of(Range: s->children(), P: [](const Stmt *child) { |
| 726 | return child && stmtCanThrow(s: child); |
| 727 | }); |
| 728 | } |
| 729 | |
| 730 | // Given a suspend expression which roughly looks like: |
| 731 | // |
| 732 | // auto && x = CommonExpr(); |
| 733 | // if (!x.await_ready()) { |
| 734 | // x.await_suspend(...); (*) |
| 735 | // } |
| 736 | // x.await_resume(); |
| 737 | // |
| 738 | // where the result of the entire expression is the result of x.await_resume() |
| 739 | // |
| 740 | // (*) If x.await_suspend return type is bool, it allows to veto a suspend: |
| 741 | // if (x.await_suspend(...)) |
| 742 | // llvm_coro_suspend(); |
| 743 | // |
| 744 | // This is more higher level than LLVM codegen, for that one see llvm's |
| 745 | // docs/Coroutines.md for more details. |
| 746 | namespace { |
| 747 | struct LValueOrRValue { |
| 748 | LValue lv; |
| 749 | RValue rv; |
| 750 | }; |
| 751 | } // namespace |
| 752 | |
| 753 | static bool isResumeExprInTryCatch(const CGCoroData &coro, cir::AwaitKind kind, |
| 754 | const CoroutineSuspendExpr &s) { |
| 755 | return coro.exceptionHandler && kind == cir::AwaitKind::Init && |
| 756 | stmtCanThrow(s: s.getResumeExpr()); |
| 757 | } |
| 758 | |
| 759 | static LValueOrRValue |
| 760 | emitSuspendExpression(CIRGenFunction &cgf, CGCoroData &coro, |
| 761 | CoroutineSuspendExpr const &s, cir::AwaitKind kind, |
| 762 | AggValueSlot aggSlot, bool ignoreResult, |
| 763 | mlir::Block *scopeParentBlock, |
| 764 | mlir::Value &tmpResumeRValAddr, bool forLValue) { |
| 765 | [[maybe_unused]] mlir::LogicalResult awaitBuild = mlir::success(); |
| 766 | LValueOrRValue awaitRes; |
| 767 | |
| 768 | CIRGenFunction::OpaqueValueMapping binder = |
| 769 | CIRGenFunction::OpaqueValueMapping(cgf, s.getOpaqueValue()); |
| 770 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 771 | |
| 772 | // Exception handling requires additional IR. We avoid generating it when |
| 773 | // neither the await_resume() call nor the destruction of its result can |
| 774 | // throw. |
| 775 | const bool resumeInTry = isResumeExprInTryCatch(coro, kind, s); |
| 776 | |
| 777 | // Resume-expression temporaries live until the enclosing full-expression |
| 778 | // ends. Defer their cleanups and guard them with construction flags, since |
| 779 | // destroying the coroutine while suspended skips their construction. |
| 780 | // The initial-suspend try/catch cleans up its temporaries in the try block. |
| 781 | std::optional<CIRGenFunction::ConditionalEvaluation> resumeEval; |
| 782 | const bool resumeDefersCleanups = |
| 783 | !resumeInTry && CIRGenFunction::resumeExprDefersCleanups(s); |
| 784 | if (resumeDefersCleanups && cgf.currentFullExprCleanupScope) |
| 785 | resumeEval.emplace(args&: cgf, args: cgf.getLoc(srcLoc: s.getSourceRange())); |
| 786 | else if (resumeDefersCleanups) |
| 787 | cgf.cgm.errorNYI(s.getSourceRange(), |
| 788 | "resume expression cleanups without a full-expression " |
| 789 | "cleanup scope" ); |
| 790 | [[maybe_unused]] EHScopeStack::stable_iterator ehDepth = |
| 791 | cgf.ehStack.stable_begin(); |
| 792 | |
| 793 | // If the await_resume() result needs a destructor, take over its |
| 794 | // destruction, unless the destination owns it. The try/catch path destroys |
| 795 | // the result itself. |
| 796 | const CXXBindTemporaryExpr *resultBind = nullptr; |
| 797 | if (!resumeInTry && !aggSlot.isExternallyDestructed()) |
| 798 | resultBind = dyn_cast<CXXBindTemporaryExpr>(Val: s.getResumeExpr()); |
| 799 | if (resultBind) { |
| 800 | // Emit the result into a slot created outside of the cir.await, so that it |
| 801 | // is still available after it. |
| 802 | if (aggSlot.isIgnored()) |
| 803 | aggSlot = cgf.createAggTemp(ty: resultBind->getType(), |
| 804 | loc: cgf.getLoc(srcLoc: resultBind->getSourceRange()), |
| 805 | name: "agg.tmp.ensured" ); |
| 806 | // Don't push the destructor from within the resume region, where its |
| 807 | // cir.cleanup.scope would capture the region's terminator. |
| 808 | aggSlot.setExternallyDestructed(); |
| 809 | } |
| 810 | |
| 811 | [[maybe_unused]] cir::AwaitOp awaitOp = cir::AwaitOp::create( |
| 812 | builder, location: cgf.getLoc(srcLoc: s.getSourceRange()), kind, |
| 813 | /*readyBuilder=*/ |
| 814 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 815 | Expr *condExpr = s.getReadyExpr()->IgnoreParens(); |
| 816 | builder.createCondition(condition: cgf.evaluateExprAsBool(e: condExpr)); |
| 817 | }, |
| 818 | /*suspendBuilder=*/ |
| 819 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 820 | // Note that differently from LLVM codegen we do not emit coro.save |
| 821 | // and coro.suspend here, that should be done as part of lowering this |
| 822 | // to LLVM dialect (or some other MLIR dialect) |
| 823 | |
| 824 | switch (s.getSuspendReturnType()) { |
| 825 | case CoroutineSuspendExpr::SuspendReturnType::SuspendBool: { |
| 826 | mlir::Value suspendRet = cgf.evaluateExprAsBool(e: s.getSuspendExpr()); |
| 827 | // Veto suspension if requested by bool returning await_suspend. |
| 828 | cir::CoroSuspendPoint::create(builder, location: loc, cond: suspendRet); |
| 829 | break; |
| 830 | } |
| 831 | case CoroutineSuspendExpr::SuspendReturnType::SuspendVoid: |
| 832 | cgf.emitScalarExpr(e: s.getSuspendExpr()); |
| 833 | // Signals the parent that execution flows to next region. |
| 834 | cir::CoroSuspendPoint::create(builder, location: loc); |
| 835 | break; |
| 836 | case CoroutineSuspendExpr::SuspendReturnType::SuspendHandle: |
| 837 | cgf.cgm.errorNYI(s.getSourceRange(), |
| 838 | "await_suspend returning handle" ); |
| 839 | cir::CoroSuspendPoint::create(builder, location: loc); |
| 840 | break; |
| 841 | } |
| 842 | }, |
| 843 | /*resumeBuilder=*/ |
| 844 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 845 | if (resumeEval) |
| 846 | resumeEval->beginEvaluation(); |
| 847 | if (resumeInTry) { |
| 848 | // we are basically just emitting: |
| 849 | // resumeEh = false; |
| 850 | // try { |
| 851 | // resumeExpr(); |
| 852 | // resumeEh = true; |
| 853 | // } catch(...) { |
| 854 | // exceptionHandler(); |
| 855 | // } |
| 856 | // Note the values of resumeEh are reversed from classic codegen, |
| 857 | // simply so we can use an 'IfOp' without a 'else' later. |
| 858 | ASTContext &ctx = cgf.getContext(); |
| 859 | SourceLocation resumeLoc = s.getResumeExpr()->getExprLoc(); |
| 860 | mlir::Location mlirLoc = cgf.getLoc(srcLoc: resumeLoc); |
| 861 | coro.resumeEHVar = cgf.createTempAlloca( |
| 862 | ty: builder.getBoolTy(), align: ctx.getTypeAlignInChars(T: ctx.BoolTy), loc: mlirLoc, |
| 863 | name: "resume.eh" ); |
| 864 | builder.createFlagStore(loc: mlirLoc, val: false, |
| 865 | dst: coro.resumeEHVar.getPointer()); |
| 866 | |
| 867 | CXXCatchStmt catchStmt(resumeLoc, |
| 868 | /*exDecl=*/nullptr, coro.exceptionHandler); |
| 869 | auto *tryBody = |
| 870 | CompoundStmt::Create(C: ctx, Stmts: s.getResumeExpr(), FPFeatures: FPOptionsOverride(), |
| 871 | LB: resumeLoc, RB: resumeLoc); |
| 872 | CXXTryStmt *tryStmt = |
| 873 | CXXTryStmt::Create(C: ctx, tryLoc: resumeLoc, tryBlock: tryBody, handlers: &catchStmt); |
| 874 | |
| 875 | struct resumeEmitter final : CIRGenFunction::cxxTryBodyEmitter { |
| 876 | const CXXTryStmt &tryStmt; |
| 877 | const Expr *resumeExpr; |
| 878 | mlir::Location loc; |
| 879 | mlir::Value resumeEHVar; |
| 880 | resumeEmitter(const CXXTryStmt &tryStmt, const Expr *resumeExpr, |
| 881 | mlir::Location loc, Address resumeEHVar) |
| 882 | : tryStmt(tryStmt), resumeExpr(resumeExpr), loc(loc), |
| 883 | resumeEHVar(resumeEHVar.getPointer()) {} |
| 884 | |
| 885 | mlir::LogicalResult operator()(CIRGenFunction &cgf) override { |
| 886 | // Destroy resume temporaries in the try block before marking |
| 887 | // resumption successful, so destructor exceptions are caught too. |
| 888 | CIRGenFunction::FullExprCleanupScope fullExpr(cgf, resumeExpr); |
| 889 | mlir::LogicalResult res = |
| 890 | cgf.emitStmt(s: tryStmt.getTryBlock(), /*useCurrentScope=*/true); |
| 891 | fullExpr.exit(); |
| 892 | cgf.getBuilder().createFlagStore(loc, val: true, dst: resumeEHVar); |
| 893 | return res; |
| 894 | } |
| 895 | |
| 896 | ~resumeEmitter() override = default; |
| 897 | } emitter{*tryStmt, s.getResumeExpr(), mlirLoc, coro.resumeEHVar}; |
| 898 | |
| 899 | awaitBuild = cgf.emitCXXTryStmt(s: *tryStmt, bodyCallback&: emitter); |
| 900 | // We are not supposed to obtain the value from init suspend |
| 901 | // await_resume(). |
| 902 | awaitRes.rv = RValue::getIgnored(); |
| 903 | } else if (forLValue) { |
| 904 | awaitRes.lv = cgf.emitLValue(e: s.getResumeExpr()); |
| 905 | assert(awaitRes.lv.isSimple() && |
| 906 | "expected simple lvalue for coroutine resume" ); |
| 907 | mlir::Value ptr = awaitRes.lv.getPointer(); |
| 908 | llvm::StringRef tmpName = kind == cir::AwaitKind::Yield |
| 909 | ? "__coyield_resume_lval" |
| 910 | : "__coawait_resume_lval" ; |
| 911 | tmpResumeRValAddr = cgf.emitAlloca( |
| 912 | name: tmpName, ty: ptr.getType(), loc, alignment: cgf.getPointerAlign(), |
| 913 | ip: builder.getBestAllocaInsertPoint(block: scopeParentBlock)); |
| 914 | builder.CIRBaseBuilderTy::createStore(loc, val: ptr, dst: tmpResumeRValAddr); |
| 915 | } else { |
| 916 | awaitRes.rv = |
| 917 | cgf.emitAnyExpr(e: s.getResumeExpr(), aggSlot, ignoreResult); |
| 918 | if (!awaitRes.rv.isIgnored()) { |
| 919 | RValue rv = awaitRes.rv; |
| 920 | if (rv.isScalar() || rv.isComplex()) { |
| 921 | mlir::Value value = |
| 922 | rv.isScalar() ? rv.getValue() : rv.getComplexValue(); |
| 923 | tmpResumeRValAddr = cgf.emitAlloca( |
| 924 | name: "__coawait_resume_rval" , ty: value.getType(), loc, |
| 925 | alignment: CharUnits::One(), |
| 926 | ip: builder.getBestAllocaInsertPoint(block: scopeParentBlock)); |
| 927 | // Store the rvalue so we can reload it before the promise call. |
| 928 | builder.CIRBaseBuilderTy::createStore(loc, val: value, |
| 929 | dst: tmpResumeRValAddr); |
| 930 | } else { |
| 931 | assert(rv.isAggregate() && "unexpected rvalue kind" ); |
| 932 | } |
| 933 | } |
| 934 | } |
| 935 | |
| 936 | if (resumeEval) |
| 937 | resumeEval->endEvaluation(); |
| 938 | |
| 939 | // Returns control back to parent. |
| 940 | cir::YieldOp::create(builder, location: loc); |
| 941 | }); |
| 942 | |
| 943 | assert((cgf.ehStack.stable_begin() == ehDepth || |
| 944 | (resumeDefersCleanups && !resumeEval)) && |
| 945 | "a resume expression cleanup was pushed in the resume region" ); |
| 946 | |
| 947 | // Push the destructor right after the cir.await, where the result starts to |
| 948 | // exist, and not around it: destroying the coroutine at the suspend point |
| 949 | // would destroy the not yet constructed result. The cleanup runs at the end |
| 950 | // of the full-expression. |
| 951 | if (resultBind) |
| 952 | cgf.emitCXXTemporary(temporary: resultBind->getTemporary(), tempType: resultBind->getType(), |
| 953 | ptr: aggSlot.getAddress()); |
| 954 | |
| 955 | assert(awaitBuild.succeeded() && "Should know how to codegen" ); |
| 956 | return awaitRes; |
| 957 | } |
| 958 | |
| 959 | static RValue emitSuspendExpr(CIRGenFunction &cgf, |
| 960 | const CoroutineSuspendExpr &e, |
| 961 | cir::AwaitKind kind, AggValueSlot aggSlot, |
| 962 | bool ignoreResult) { |
| 963 | RValue rval; |
| 964 | mlir::Location scopeLoc = cgf.getLoc(srcLoc: e.getSourceRange()); |
| 965 | |
| 966 | // Since we model suspend / resume as an inner region, we must store |
| 967 | // resume scalar results in a tmp alloca, and load it after we build the |
| 968 | // suspend expression. An alternative way to do this would be to make |
| 969 | // every region return a value when promise.return_value() is used, but |
| 970 | // it's a bit awkward given that resume is the only region that actually |
| 971 | // returns a value. |
| 972 | mlir::Block *currEntryBlock = cgf.curLexScope->getEntryBlock(); |
| 973 | [[maybe_unused]] mlir::Value tmpResumeRValAddr; |
| 974 | |
| 975 | // No need to explicitly wrap this into a scope since the AST already uses a |
| 976 | // ExprWithCleanups, which will wrap this into a cir.scope anyways. |
| 977 | rval = emitSuspendExpression(cgf, coro&: *cgf.curCoro.data, s: e, kind, aggSlot, |
| 978 | ignoreResult, scopeParentBlock: currEntryBlock, tmpResumeRValAddr, |
| 979 | /*forLValue*/ false) |
| 980 | .rv; |
| 981 | |
| 982 | if (ignoreResult || rval.isIgnored()) |
| 983 | return rval; |
| 984 | |
| 985 | if (rval.isScalar()) { |
| 986 | rval = RValue::get(v: cir::LoadOp::create(builder&: cgf.getBuilder(), location: scopeLoc, |
| 987 | result: rval.getValue().getType(), |
| 988 | addr: tmpResumeRValAddr)); |
| 989 | } else if (rval.isAggregate()) { |
| 990 | return rval; |
| 991 | } else { // complex |
| 992 | rval = RValue::getComplex(v: cir::LoadOp::create( |
| 993 | builder&: cgf.getBuilder(), location: scopeLoc, result: rval.getComplexValue().getType(), |
| 994 | addr: tmpResumeRValAddr)); |
| 995 | } |
| 996 | return rval; |
| 997 | } |
| 998 | |
| 999 | RValue CIRGenFunction::emitCoawaitExpr(const CoawaitExpr &e, |
| 1000 | AggValueSlot aggSlot, |
| 1001 | bool ignoreResult) { |
| 1002 | return emitSuspendExpr(cgf&: *this, e, kind: curCoro.data->currentAwaitKind, aggSlot, |
| 1003 | ignoreResult); |
| 1004 | } |
| 1005 | |
| 1006 | RValue CIRGenFunction::emitCoyieldExpr(const CoyieldExpr &e, |
| 1007 | AggValueSlot aggSlot, |
| 1008 | bool ignoreResult) { |
| 1009 | return emitSuspendExpr(cgf&: *this, e, kind: cir::AwaitKind::Yield, aggSlot, |
| 1010 | ignoreResult); |
| 1011 | } |
| 1012 | |
| 1013 | static QualType |
| 1014 | getCoroutineSuspendExprReturnType(const ASTContext &ctx, |
| 1015 | const CoroutineSuspendExpr *e) { |
| 1016 | const auto *re = e->getResumeExpr(); |
| 1017 | assert(isa<CallExpr>(re) && "unexpected suspend expression type" ); |
| 1018 | return cast<CallExpr>(Val: re)->getCallReturnType(Ctx: ctx); |
| 1019 | } |
| 1020 | |
| 1021 | static LValue emitSuspendLValue(CIRGenFunction &cgf, |
| 1022 | const CoroutineSuspendExpr *e, |
| 1023 | cir::AwaitKind kind) { |
| 1024 | QualType retTy = getCoroutineSuspendExprReturnType(ctx: cgf.getContext(), e); |
| 1025 | assert(retTy->isReferenceType() && |
| 1026 | "Can't have a scalar return unless the return type is a " |
| 1027 | "reference type!" ); |
| 1028 | mlir::Location scopeLoc = cgf.getLoc(srcLoc: e->getSourceRange()); |
| 1029 | mlir::Block *currEntryBlock = cgf.curLexScope->getEntryBlock(); |
| 1030 | mlir::Value tmpResumeLValAddr; |
| 1031 | LValue lv = emitSuspendExpression(cgf, coro&: *cgf.curCoro.data, s: *e, kind, |
| 1032 | aggSlot: AggValueSlot::ignored(), |
| 1033 | /*ignoreResult=*/false, scopeParentBlock: currEntryBlock, |
| 1034 | tmpResumeRValAddr&: tmpResumeLValAddr, /*forLValue=*/true) |
| 1035 | .lv; |
| 1036 | if (!tmpResumeLValAddr) { |
| 1037 | assert(isResumeExprInTryCatch(*cgf.curCoro.data, kind, *e) && |
| 1038 | "expected resume in try/catch" ); |
| 1039 | QualType pointeeTy = retTy->getPointeeType(); |
| 1040 | mlir::Type elemTy = cgf.convertType(t: pointeeTy); |
| 1041 | mlir::Value nullPtr = cgf.getBuilder().getNullPtr( |
| 1042 | ty: cgf.getBuilder().getPointerTo(ty: elemTy), loc: scopeLoc); |
| 1043 | return cgf.makeNaturalAlignAddrLValue(val: nullPtr, ty: pointeeTy); |
| 1044 | } |
| 1045 | |
| 1046 | Address addr = lv.getAddress(); |
| 1047 | mlir::Value loadedPtr = |
| 1048 | cir::LoadOp::create(builder&: cgf.getBuilder(), location: scopeLoc, |
| 1049 | result: addr.getPointer().getType(), addr: tmpResumeLValAddr); |
| 1050 | lv.setAddress(addr.withPointer(newPtr: loadedPtr)); |
| 1051 | return lv; |
| 1052 | } |
| 1053 | |
| 1054 | LValue CIRGenFunction::emitCoawaitLValue(const CoawaitExpr *e) { |
| 1055 | return emitSuspendLValue(cgf&: *this, e, kind: curCoro.data->currentAwaitKind); |
| 1056 | } |
| 1057 | |
| 1058 | LValue CIRGenFunction::emitCoyieldLValue(const CoyieldExpr *e) { |
| 1059 | return emitSuspendLValue(cgf&: *this, e, kind: cir::AwaitKind::Yield); |
| 1060 | } |
| 1061 | |
| 1062 | mlir::LogicalResult CIRGenFunction::emitCoreturnStmt(CoreturnStmt const &s) { |
| 1063 | ++curCoro.data->coreturnCount; |
| 1064 | |
| 1065 | const Expr *rv = s.getOperand(); |
| 1066 | if (rv && rv->getType()->isVoidType() && !isa<InitListExpr>(Val: rv)) { |
| 1067 | // Make sure to evaluate the non initlist expression of a co_return |
| 1068 | // with a void expression for side effects. |
| 1069 | RunCleanupsScope cleanupScope(*this); |
| 1070 | emitIgnoredExpr(e: rv); |
| 1071 | } |
| 1072 | |
| 1073 | if (emitStmt(s: s.getPromiseCall(), /*useCurrentScope=*/true).failed()) |
| 1074 | return mlir::failure(); |
| 1075 | mlir::Location loc = getLoc(srcLoc: s.getSourceRange()); |
| 1076 | cir::CoReturnOp::create(builder, location: loc); |
| 1077 | curLexScope->markCoreturnEmitted(); |
| 1078 | |
| 1079 | // Statements after a co_return may stil be reachable through a label. |
| 1080 | // Continue codegen. |
| 1081 | builder.createBlock(parent: builder.getBlock()->getParent()); |
| 1082 | return mlir::success(); |
| 1083 | } |
| 1084 | |