1//===- AsyncToAsyncRuntime.cpp - Lower from Async to Async Runtime --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements lowering from high level async operations to async.coro
10// and async.runtime operations.
11//
12//===----------------------------------------------------------------------===//
13
14#include <utility>
15
16#include "mlir/Dialect/Async/Passes.h"
17
18#include "PassDetail.h"
19#include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
20#include "mlir/Dialect/Arith/IR/Arith.h"
21#include "mlir/Dialect/Async/IR/Async.h"
22#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
23#include "mlir/Dialect/Func/IR/FuncOps.h"
24#include "mlir/Dialect/SCF/IR/SCFDialect.h"
25#include "mlir/IR/IRMapping.h"
26#include "mlir/IR/PatternMatch.h"
27#include "mlir/Transforms/DialectConversion.h"
28#include "mlir/Transforms/RegionUtils.h"
29#include "llvm/Support/Debug.h"
30#include <optional>
31
32namespace mlir {
33#define GEN_PASS_DEF_ASYNCTOASYNCRUNTIMEPASS
34#define GEN_PASS_DEF_ASYNCFUNCTOASYNCRUNTIMEPASS
35#include "mlir/Dialect/Async/Passes.h.inc"
36} // namespace mlir
37
38using namespace mlir;
39using namespace mlir::async;
40
41#define DEBUG_TYPE "async-to-async-runtime"
42// Prefix for functions outlined from `async.execute` op regions.
43static constexpr const char kAsyncFnPrefix[] = "async_execute_fn";
44
45namespace {
46
47class AsyncToAsyncRuntimePass
48 : public impl::AsyncToAsyncRuntimePassBase<AsyncToAsyncRuntimePass> {
49public:
50 AsyncToAsyncRuntimePass() = default;
51 void runOnOperation() override;
52};
53
54} // namespace
55
56namespace {
57
58class AsyncFuncToAsyncRuntimePass
59 : public impl::AsyncFuncToAsyncRuntimePassBase<
60 AsyncFuncToAsyncRuntimePass> {
61public:
62 AsyncFuncToAsyncRuntimePass() = default;
63 void runOnOperation() override;
64};
65
66} // namespace
67
68/// Function targeted for coroutine transformation has two additional blocks at
69/// the end: coroutine cleanup and coroutine suspension.
70///
71/// async.await op lowering additionaly creates a resume block for each
72/// operation to enable non-blocking waiting via coroutine suspension.
73namespace {
74struct CoroMachinery {
75 func::FuncOp func;
76
77 // Async function returns an optional token, followed by some async values
78 //
79 // async.func @foo() -> !async.value<T> {
80 // %cst = arith.constant 42.0 : T
81 // return %cst: T
82 // }
83 // Async execute region returns a completion token, and an async value for
84 // each yielded value.
85 //
86 // %token, %result = async.execute -> !async.value<T> {
87 // %0 = arith.constant ... : T
88 // async.yield %0 : T
89 // }
90 std::optional<Value> asyncToken; // returned completion token
91 llvm::SmallVector<Value, 4> returnValues; // returned async values
92
93 Value coroId; // coroutine id (!async.coro.id value)
94 Value coroHandle; // coroutine handle (!async.coro.getHandle value)
95 Block *entry; // coroutine entry block
96 std::optional<Block *> setError; // set returned values to error state
97 Block *cleanup; // coroutine cleanup block
98
99 // Coroutine cleanup block for destroy after the coroutine is resumed,
100 // e.g. async.coro.suspend state, [suspend], [resume], [destroy]
101 //
102 // This cleanup block is a duplicate of the cleanup block followed by the
103 // resume block. The purpose of having a duplicate cleanup block for destroy
104 // is to make the CFG clear so that the control flow analysis won't confuse.
105 //
106 // The overall structure of the lowered CFG can be the following,
107 //
108 // Entry (calling async.coro.suspend)
109 // | \
110 // Resume Destroy (duplicate of Cleanup)
111 // | |
112 // Cleanup |
113 // | /
114 // End (ends the corontine)
115 //
116 // If there is resume-specific cleanup logic, it can go into the Cleanup
117 // block but not the destroy block. Otherwise, it can fail block dominance
118 // check.
119 //
120 // This block is created lazily by `setupCleanupForDestroyBlock` only when a
121 // suspension point needs a destroy successor, so that functions without any
122 // coroutine suspends (e.g. an `async.func` body with no `await`) don't end
123 // up with dead code.
124 std::optional<Block *> cleanupForDestroy;
125 Block *suspend; // coroutine suspension block
126};
127} // namespace
128
129using FuncCoroMapPtr =
130 std::shared_ptr<llvm::DenseMap<func::FuncOp, CoroMachinery>>;
131
132/// Utility to partially update the regular function CFG to the coroutine CFG
133/// compatible with LLVM coroutines switched-resume lowering using
134/// `async.runtime.*` and `async.coro.*` operations. Adds a new entry block
135/// that branches into preexisting entry block. Also inserts trailing blocks.
136///
137/// The result types of the passed `func` start with an optional `async.token`
138/// and be continued with some number of `async.value`s.
139///
140/// See LLVM coroutines documentation: https://llvm.org/docs/Coroutines.html
141///
142/// - `entry` block sets up the coroutine.
143/// - `set_error` block sets completion token and async values state to error.
144/// - `cleanup` block cleans up the coroutine state.
145/// - `suspend block after the @llvm.coro.end() defines what value will be
146/// returned to the initial caller of a coroutine. Everything before the
147/// @llvm.coro.end() will be executed at every suspension point.
148///
149/// Coroutine structure (only the important bits):
150///
151/// func @some_fn(<function-arguments>) -> (!async.token, !async.value<T>)
152/// {
153/// ^entry(<function-arguments>):
154/// %token = <async token> : !async.token // create async runtime token
155/// %value = <async value> : !async.value<T> // create async value
156/// %id = async.coro.getId // create a coroutine id
157/// %hdl = async.coro.begin %id // create a coroutine handle
158/// cf.br ^preexisting_entry_block
159///
160/// /* preexisting blocks modified to branch to the cleanup block */
161///
162/// ^set_error: // this block created lazily only if needed (see code below)
163/// async.runtime.set_error %token : !async.token
164/// async.runtime.set_error %value : !async.value<T>
165/// cf.br ^cleanup
166///
167/// ^cleanup:
168/// async.coro.free %hdl // delete the coroutine state
169/// cf.br ^suspend
170///
171/// ^suspend:
172/// async.coro.end %hdl // marks the end of a coroutine
173/// return %token, %value : !async.token, !async.value<T>
174/// }
175///
176static CoroMachinery setupCoroMachinery(func::FuncOp func) {
177 assert(!func.getBlocks().empty() && "Function must have an entry block");
178
179 MLIRContext *ctx = func.getContext();
180 Block *entryBlock = &func.getBlocks().front();
181 Block *originalEntryBlock =
182 entryBlock->splitBlock(splitBefore: entryBlock->getOperations().begin());
183 auto builder = ImplicitLocOpBuilder::atBlockBegin(loc: func->getLoc(), block: entryBlock);
184
185 // ------------------------------------------------------------------------ //
186 // Allocate async token/values that we will return from a ramp function.
187 // ------------------------------------------------------------------------ //
188
189 // We treat TokenType as state update marker to represent side-effects of
190 // async computations
191 bool isStateful = isa<async::TokenType>(Val: func.getResultTypes().front());
192
193 std::optional<Value> retToken;
194 if (isStateful)
195 retToken.emplace(
196 args: RuntimeCreateOp::create(builder, result: async::TokenType::get(ctx)));
197
198 llvm::SmallVector<Value, 4> retValues;
199 ArrayRef<Type> resValueTypes =
200 isStateful ? func.getResultTypes().drop_front() : func.getResultTypes();
201 for (auto resType : resValueTypes)
202 retValues.emplace_back(
203 Args: RuntimeCreateOp::create(builder, result: resType).getResult());
204
205 // ------------------------------------------------------------------------ //
206 // Initialize coroutine: get coroutine id and coroutine handle.
207 // ------------------------------------------------------------------------ //
208 auto coroIdOp = CoroIdOp::create(builder, id: CoroIdType::get(ctx));
209 auto coroHdlOp =
210 CoroBeginOp::create(builder, handle: CoroHandleType::get(ctx), id: coroIdOp.getId());
211 cf::BranchOp::create(builder, dest: originalEntryBlock);
212
213 Block *cleanupBlock = func.addBlock();
214 Block *suspendBlock = func.addBlock();
215
216 // ------------------------------------------------------------------------ //
217 // Coroutine cleanup block: deallocate coroutine frame, free the memory.
218 // ------------------------------------------------------------------------ //
219 // The matching "destroy" cleanup block is materialized lazily by
220 // `setupCleanupForDestroyBlock` only when a suspend point needs it.
221 builder.setInsertionPointToStart(cleanupBlock);
222 CoroFreeOp::create(builder, id: coroIdOp.getId(), handle: coroHdlOp.getHandle());
223 cf::BranchOp::create(builder, dest: suspendBlock);
224
225 // ------------------------------------------------------------------------ //
226 // Coroutine suspend block: mark the end of a coroutine and return allocated
227 // async token.
228 // ------------------------------------------------------------------------ //
229 builder.setInsertionPointToStart(suspendBlock);
230
231 // Mark the end of a coroutine: async.coro.end
232 CoroEndOp::create(builder, handle: coroHdlOp.getHandle());
233
234 // Return created optional `async.token` and `async.values` from the suspend
235 // block. This will be the return value of a coroutine ramp function.
236 SmallVector<Value, 4> ret;
237 if (retToken)
238 ret.push_back(Elt: *retToken);
239 llvm::append_range(C&: ret, R&: retValues);
240 func::ReturnOp::create(builder, operands: ret);
241
242 // `async.await` op lowering will create resume blocks for async
243 // continuations, and will conditionally branch to cleanup or suspend blocks.
244
245 // The switch-resumed API based coroutine should be marked with
246 // presplitcoroutine attribute to mark the function as a coroutine.
247 func->setDiscardableAttr(
248 name: "llvm.passthrough",
249 value: builder.getArrayAttr(value: StringAttr::get(context: ctx, bytes: "presplitcoroutine")));
250
251 CoroMachinery machinery;
252 machinery.func = func;
253 machinery.asyncToken = retToken;
254 machinery.returnValues = retValues;
255 machinery.coroId = coroIdOp.getId();
256 machinery.coroHandle = coroHdlOp.getHandle();
257 machinery.entry = entryBlock;
258 machinery.setError = std::nullopt; // created lazily only if needed
259 machinery.cleanup = cleanupBlock;
260 machinery.cleanupForDestroy = std::nullopt; // created lazily only if needed
261 machinery.suspend = suspendBlock;
262 return machinery;
263}
264
265// Lazily creates `set_error` block only if it is required for lowering to the
266// runtime operations (see for example lowering of assert operation).
267static Block *setupSetErrorBlock(CoroMachinery &coro) {
268 if (coro.setError)
269 return *coro.setError;
270
271 coro.setError = coro.func.addBlock();
272 (*coro.setError)->moveBefore(block: coro.cleanup);
273
274 auto builder =
275 ImplicitLocOpBuilder::atBlockBegin(loc: coro.func->getLoc(), block: *coro.setError);
276
277 // Coroutine set_error block: set error on token and all returned values.
278 if (coro.asyncToken)
279 RuntimeSetErrorOp::create(builder, operand: *coro.asyncToken);
280
281 for (Value retValue : coro.returnValues)
282 RuntimeSetErrorOp::create(builder, operand: retValue);
283
284 // Branch into the cleanup block.
285 cf::BranchOp::create(builder, dest: coro.cleanup);
286
287 return *coro.setError;
288}
289
290// Lazily creates the `cleanupForDestroy` block only if a suspension point
291// actually needs a destroy successor. This avoids leaving an unreachable
292// cleanup block behind in coroutines that never suspend.
293static Block *setupCleanupForDestroyBlock(ImplicitLocOpBuilder &builder,
294 CoroMachinery &coro) {
295 if (coro.cleanupForDestroy)
296 return *coro.cleanupForDestroy;
297 OpBuilder::InsertionGuard guard(builder);
298 coro.cleanupForDestroy = builder.createBlock(insertBefore: coro.suspend);
299 CoroFreeOp::create(builder, id: coro.coroId, handle: coro.coroHandle);
300 cf::BranchOp::create(builder, dest: coro.suspend);
301 return *coro.cleanupForDestroy;
302}
303
304//===----------------------------------------------------------------------===//
305// async.execute op outlining to the coroutine functions.
306//===----------------------------------------------------------------------===//
307
308/// Outline the body region attached to the `async.execute` op into a standalone
309/// function.
310///
311/// Note that this is not reversible transformation.
312static std::pair<func::FuncOp, CoroMachinery>
313outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
314 ModuleOp module = execute->getParentOfType<ModuleOp>();
315
316 MLIRContext *ctx = module.getContext();
317 Location loc = execute.getLoc();
318
319 // Make sure that all constants will be inside the outlined async function to
320 // reduce the number of function arguments.
321 cloneConstantsIntoTheRegion(region&: execute.getBodyRegion());
322
323 // Collect all outlined function inputs.
324 SetVector<mlir::Value> functionInputs(llvm::from_range,
325 execute.getDependencies());
326 functionInputs.insert_range(R: execute.getBodyOperands());
327 getUsedValuesDefinedAbove(regions: execute.getBodyRegion(), values&: functionInputs);
328
329 // Collect types for the outlined function inputs and outputs.
330 auto typesRange = llvm::map_range(
331 C&: functionInputs, F: [](Value value) { return value.getType(); });
332 SmallVector<Type, 4> inputTypes(typesRange.begin(), typesRange.end());
333 auto outputTypes = execute.getResultTypes();
334
335 auto funcType = FunctionType::get(context: ctx, inputs: inputTypes, results: outputTypes);
336 auto funcAttrs = ArrayRef<NamedAttribute>();
337
338 // TODO: Derive outlined function name from the parent FuncOp (support
339 // multiple nested async.execute operations).
340 func::FuncOp func =
341 func::FuncOp::create(location: loc, name: kAsyncFnPrefix, type: funcType, attrs: funcAttrs);
342 symbolTable.insert(symbol: func);
343
344 SymbolTable::setSymbolVisibility(symbol: func, vis: SymbolTable::Visibility::Private);
345 auto builder = ImplicitLocOpBuilder::atBlockBegin(loc, block: func.addEntryBlock());
346
347 // Prepare for coroutine conversion by creating the body of the function.
348 {
349 size_t numDependencies = execute.getDependencies().size();
350 size_t numOperands = execute.getBodyOperands().size();
351
352 // Await on all dependencies before starting to execute the body region.
353 for (size_t i = 0; i < numDependencies; ++i)
354 AwaitOp::create(builder, operand: func.getArgument(idx: i));
355
356 // Await on all async value operands and unwrap the payload.
357 SmallVector<Value, 4> unwrappedOperands(numOperands);
358 for (size_t i = 0; i < numOperands; ++i) {
359 Value operand = func.getArgument(idx: numDependencies + i);
360 unwrappedOperands[i] = AwaitOp::create(builder, location: loc, operand).getResult();
361 }
362
363 // Map from function inputs defined above the execute op to the function
364 // arguments.
365 IRMapping valueMapping;
366 valueMapping.map(from&: functionInputs, to: func.getArguments());
367 valueMapping.map(from: execute.getBodyRegion().getArguments(), to&: unwrappedOperands);
368
369 // Clone all operations from the execute operation body into the outlined
370 // function body.
371 for (Operation &op : execute.getBodyRegion().getOps())
372 builder.clone(op, mapper&: valueMapping);
373 }
374
375 // Adding entry/cleanup/suspend blocks.
376 CoroMachinery coro = setupCoroMachinery(func);
377
378 // Suspend async function at the end of an entry block, and resume it using
379 // Async resume operation (execution will be resumed in a thread managed by
380 // the async runtime).
381 {
382 cf::BranchOp branch = cast<cf::BranchOp>(Val: coro.entry->getTerminator());
383 builder.setInsertionPointToEnd(coro.entry);
384
385 // Save the coroutine state: async.coro.save
386 auto coroSaveOp =
387 CoroSaveOp::create(builder, state: CoroStateType::get(ctx), handle: coro.coroHandle);
388
389 // Pass coroutine to the runtime to be resumed on a runtime managed
390 // thread.
391 RuntimeResumeOp::create(builder, handle: coro.coroHandle);
392
393 // Add async.coro.suspend as a suspended block terminator.
394 Block *destroy = setupCleanupForDestroyBlock(builder, coro);
395 CoroSuspendOp::create(builder, state: coroSaveOp.getState(), suspendDest: coro.suspend,
396 resumeDest: branch.getDest(), cleanupDest: destroy);
397
398 branch.erase();
399 }
400
401 // Replace the original `async.execute` with a call to outlined function.
402 {
403 ImplicitLocOpBuilder callBuilder(loc, execute);
404 auto callOutlinedFunc = func::CallOp::create(builder&: callBuilder, callee: func.getName(),
405 results: execute.getResultTypes(),
406 operands: functionInputs.getArrayRef());
407 execute.replaceAllUsesWith(values: callOutlinedFunc.getResults());
408 execute.erase();
409 }
410
411 return {func, coro};
412}
413
414//===----------------------------------------------------------------------===//
415// Convert async.create_group operation to async.runtime.create_group
416//===----------------------------------------------------------------------===//
417
418namespace {
419class CreateGroupOpLowering : public OpConversionPattern<CreateGroupOp> {
420public:
421 using Base::Base;
422
423 LogicalResult
424 matchAndRewrite(CreateGroupOp op, OpAdaptor adaptor,
425 ConversionPatternRewriter &rewriter) const override {
426 rewriter.replaceOpWithNewOp<RuntimeCreateGroupOp>(
427 op, args: GroupType::get(ctx: op->getContext()), args: adaptor.getOperands());
428 return success();
429 }
430};
431} // namespace
432
433//===----------------------------------------------------------------------===//
434// Convert async.add_to_group operation to async.runtime.add_to_group.
435//===----------------------------------------------------------------------===//
436
437namespace {
438class AddToGroupOpLowering : public OpConversionPattern<AddToGroupOp> {
439public:
440 using Base::Base;
441
442 LogicalResult
443 matchAndRewrite(AddToGroupOp op, OpAdaptor adaptor,
444 ConversionPatternRewriter &rewriter) const override {
445 rewriter.replaceOpWithNewOp<RuntimeAddToGroupOp>(
446 op, args: rewriter.getIndexType(), args: adaptor.getOperands());
447 return success();
448 }
449};
450} // namespace
451
452//===----------------------------------------------------------------------===//
453// Convert async.func, async.return and async.call operations to non-blocking
454// operations based on llvm coroutine
455//===----------------------------------------------------------------------===//
456
457namespace {
458
459//===----------------------------------------------------------------------===//
460// Convert async.func operation to func.func
461//===----------------------------------------------------------------------===//
462
463class AsyncFuncOpLowering : public OpConversionPattern<async::FuncOp> {
464public:
465 AsyncFuncOpLowering(MLIRContext *ctx, FuncCoroMapPtr coros)
466 : OpConversionPattern<async::FuncOp>(ctx), coros(std::move(coros)) {}
467
468 LogicalResult
469 matchAndRewrite(async::FuncOp op, OpAdaptor adaptor,
470 ConversionPatternRewriter &rewriter) const override {
471 Location loc = op->getLoc();
472
473 auto newFuncOp =
474 func::FuncOp::create(builder&: rewriter, location: loc, name: op.getName(), type: op.getFunctionType());
475
476 SymbolTable::setSymbolVisibility(symbol: newFuncOp,
477 vis: SymbolTable::getSymbolVisibility(symbol: op));
478 // Copy over the discardable attributes.
479 for (const auto &namedAttr : op->getDiscardableAttrDictionary().getValue())
480 newFuncOp->setDiscardableAttr(name: namedAttr.getName(), value: namedAttr.getValue());
481
482 rewriter.inlineRegionBefore(region&: op.getBody(), parent&: newFuncOp.getBody(),
483 before: newFuncOp.end());
484
485 CoroMachinery coro = setupCoroMachinery(newFuncOp);
486 (*coros)[newFuncOp] = coro;
487 // no initial suspend, we should hot-start
488
489 rewriter.eraseOp(op);
490 return success();
491 }
492
493private:
494 FuncCoroMapPtr coros;
495};
496
497//===----------------------------------------------------------------------===//
498// Convert async.call operation to func.call
499//===----------------------------------------------------------------------===//
500
501class AsyncCallOpLowering : public OpConversionPattern<async::CallOp> {
502public:
503 AsyncCallOpLowering(MLIRContext *ctx)
504 : OpConversionPattern<async::CallOp>(ctx) {}
505
506 LogicalResult
507 matchAndRewrite(async::CallOp op, OpAdaptor adaptor,
508 ConversionPatternRewriter &rewriter) const override {
509 rewriter.replaceOpWithNewOp<func::CallOp>(
510 op, args: op.getCallee(), args: op.getResultTypes(), args: op.getOperands());
511 return success();
512 }
513};
514
515//===----------------------------------------------------------------------===//
516// Convert async.return operation to async.runtime operations.
517//===----------------------------------------------------------------------===//
518
519class AsyncReturnOpLowering : public OpConversionPattern<async::ReturnOp> {
520public:
521 AsyncReturnOpLowering(MLIRContext *ctx, FuncCoroMapPtr coros)
522 : OpConversionPattern<async::ReturnOp>(ctx), coros(std::move(coros)) {}
523
524 LogicalResult
525 matchAndRewrite(async::ReturnOp op, OpAdaptor adaptor,
526 ConversionPatternRewriter &rewriter) const override {
527 auto func = op->template getParentOfType<func::FuncOp>();
528 auto funcCoro = coros->find(Val: func);
529 if (funcCoro == coros->end())
530 return rewriter.notifyMatchFailure(
531 arg&: op, msg: "operation is not inside the async coroutine function");
532
533 Location loc = op->getLoc();
534 const CoroMachinery &coro = funcCoro->getSecond();
535 rewriter.setInsertionPointAfter(op);
536
537 // Store return values into the async values storage and switch async
538 // values state to available.
539 for (auto tuple : llvm::zip(t: adaptor.getOperands(), u: coro.returnValues)) {
540 Value returnValue = std::get<0>(t&: tuple);
541 Value asyncValue = std::get<1>(t&: tuple);
542 RuntimeStoreOp::create(builder&: rewriter, location: loc, value: returnValue, storage: asyncValue);
543 RuntimeSetAvailableOp::create(builder&: rewriter, location: loc, operand: asyncValue);
544 }
545
546 if (coro.asyncToken)
547 // Switch the coroutine completion token to available state.
548 RuntimeSetAvailableOp::create(builder&: rewriter, location: loc, operand: *coro.asyncToken);
549
550 rewriter.eraseOp(op);
551 cf::BranchOp::create(builder&: rewriter, location: loc, dest: coro.cleanup);
552 return success();
553 }
554
555private:
556 FuncCoroMapPtr coros;
557};
558} // namespace
559
560//===----------------------------------------------------------------------===//
561// Convert async.await and async.await_all operations to the async.runtime.await
562// or async.runtime.await_and_resume operations.
563//===----------------------------------------------------------------------===//
564
565namespace {
566template <typename AwaitType, typename AwaitableType>
567class AwaitOpLoweringBase : public OpConversionPattern<AwaitType> {
568 using AwaitAdaptor = typename AwaitType::Adaptor;
569
570public:
571 AwaitOpLoweringBase(MLIRContext *ctx, FuncCoroMapPtr coros,
572 bool shouldLowerBlockingWait)
573 : OpConversionPattern<AwaitType>(ctx), coros(std::move(coros)),
574 shouldLowerBlockingWait(shouldLowerBlockingWait) {}
575
576 LogicalResult
577 matchAndRewrite(AwaitType op, typename AwaitType::Adaptor adaptor,
578 ConversionPatternRewriter &rewriter) const override {
579 // We can only await on one the `AwaitableType` (for `await` it can be
580 // a `token` or a `value`, for `await_all` it must be a `group`).
581 if (!isa<AwaitableType>(op.getOperand().getType()))
582 return rewriter.notifyMatchFailure(op, "unsupported awaitable type");
583
584 // Check if await operation is inside the coroutine function.
585 auto func = op->template getParentOfType<func::FuncOp>();
586 auto funcCoro = coros->find(func);
587 const bool isInCoroutine = funcCoro != coros->end();
588
589 Location loc = op->getLoc();
590 Value operand = adaptor.getOperand();
591
592 Type i1 = rewriter.getI1Type();
593
594 // Delay lowering to block wait in case await op is inside async.execute
595 if (!isInCoroutine && !shouldLowerBlockingWait)
596 return failure();
597
598 // Inside regular functions we use the blocking wait operation to wait for
599 // the async object (token, value or group) to become available.
600 if (!isInCoroutine) {
601 ImplicitLocOpBuilder builder(loc, rewriter);
602 RuntimeAwaitOp::create(builder, location: loc, operand);
603
604 // Assert that the awaited operands is not in the error state.
605 Value isError = RuntimeIsErrorOp::create(builder, is_error: i1, operand);
606 Value notError = arith::XOrIOp::create(
607 builder, lhs: isError,
608 rhs: arith::ConstantOp::create(builder, location: loc, result: i1,
609 value: builder.getIntegerAttr(type: i1, value: 1)));
610
611 cf::AssertOp::create(builder, arg: notError,
612 msg: "Awaited async operand is in error state");
613 }
614
615 // Inside the coroutine we convert await operation into coroutine suspension
616 // point, and resume execution asynchronously.
617 if (isInCoroutine) {
618 CoroMachinery &coro = funcCoro->getSecond();
619 Block *suspended = op->getBlock();
620
621 ImplicitLocOpBuilder builder(loc, rewriter);
622 MLIRContext *ctx = op->getContext();
623
624 // Save the coroutine state and resume on a runtime managed thread when
625 // the operand becomes available.
626 auto coroSaveOp =
627 CoroSaveOp::create(builder, state: CoroStateType::get(ctx), handle: coro.coroHandle);
628 RuntimeAwaitAndResumeOp::create(builder, operand, handle: coro.coroHandle);
629
630 // Split the entry block before the await operation.
631 Block *resume = rewriter.splitBlock(block: suspended, before: Block::iterator(op));
632
633 // Add async.coro.suspend as a suspended block terminator.
634 Block *destroy = setupCleanupForDestroyBlock(builder, coro);
635 builder.setInsertionPointToEnd(suspended);
636 CoroSuspendOp::create(builder, state: coroSaveOp.getState(), suspendDest: coro.suspend,
637 resumeDest: resume, cleanupDest: destroy);
638
639 // Split the resume block into error checking and continuation.
640 Block *continuation = rewriter.splitBlock(block: resume, before: Block::iterator(op));
641
642 // Check if the awaited value is in the error state.
643 builder.setInsertionPointToStart(resume);
644 auto isError = RuntimeIsErrorOp::create(builder, location: loc, is_error: i1, operand);
645 cf::CondBranchOp::create(builder, condition: isError,
646 /*trueDest=*/setupSetErrorBlock(coro),
647 /*trueArgs=*/trueOperands: ArrayRef<Value>(),
648 /*falseDest=*/continuation,
649 /*falseArgs=*/falseOperands: ArrayRef<Value>());
650
651 // Make sure that replacement value will be constructed in the
652 // continuation block.
653 rewriter.setInsertionPointToStart(continuation);
654 }
655
656 // Erase or replace the await operation with the new value.
657 if (Value replaceWith = getReplacementValue(op, operand, rewriter))
658 rewriter.replaceOp(op, replaceWith);
659 else
660 rewriter.eraseOp(op);
661
662 return success();
663 }
664
665 virtual Value getReplacementValue(AwaitType op, Value operand,
666 ConversionPatternRewriter &rewriter) const {
667 return Value();
668 }
669
670private:
671 FuncCoroMapPtr coros;
672 bool shouldLowerBlockingWait;
673};
674
675/// Lowering for `async.await` with a token operand.
676class AwaitTokenOpLowering
677 : public AwaitOpLoweringBase<AwaitOp, async::TokenType> {
678 using Base = AwaitOpLoweringBase<AwaitOp, async::TokenType>;
679
680public:
681 using Base::Base;
682};
683
684/// Lowering for `async.await` with a value operand.
685class AwaitValueOpLowering : public AwaitOpLoweringBase<AwaitOp, ValueType> {
686 using Base = AwaitOpLoweringBase<AwaitOp, ValueType>;
687
688public:
689 using Base::Base;
690
691 Value
692 getReplacementValue(AwaitOp op, Value operand,
693 ConversionPatternRewriter &rewriter) const override {
694 // Load from the async value storage.
695 auto valueType = cast<ValueType>(Val: operand.getType()).getValueType();
696 return RuntimeLoadOp::create(builder&: rewriter, location: op->getLoc(), result: valueType, storage: operand);
697 }
698};
699
700/// Lowering for `async.await_all` operation.
701class AwaitAllOpLowering : public AwaitOpLoweringBase<AwaitAllOp, GroupType> {
702 using Base = AwaitOpLoweringBase<AwaitAllOp, GroupType>;
703
704public:
705 using Base::Base;
706};
707
708} // namespace
709
710//===----------------------------------------------------------------------===//
711// Convert async.yield operation to async.runtime operations.
712//===----------------------------------------------------------------------===//
713
714class YieldOpLowering : public OpConversionPattern<async::YieldOp> {
715public:
716 YieldOpLowering(MLIRContext *ctx, FuncCoroMapPtr coros)
717 : OpConversionPattern<async::YieldOp>(ctx), coros(std::move(coros)) {}
718
719 LogicalResult
720 matchAndRewrite(async::YieldOp op, OpAdaptor adaptor,
721 ConversionPatternRewriter &rewriter) const override {
722 // Check if yield operation is inside the async coroutine function.
723 auto func = op->template getParentOfType<func::FuncOp>();
724 auto funcCoro = coros->find(Val: func);
725 if (funcCoro == coros->end())
726 return rewriter.notifyMatchFailure(
727 arg&: op, msg: "operation is not inside the async coroutine function");
728
729 Location loc = op->getLoc();
730 const CoroMachinery &coro = funcCoro->getSecond();
731
732 // Store yielded values into the async values storage and switch async
733 // values state to available.
734 for (auto tuple : llvm::zip(t: adaptor.getOperands(), u: coro.returnValues)) {
735 Value yieldValue = std::get<0>(t&: tuple);
736 Value asyncValue = std::get<1>(t&: tuple);
737 RuntimeStoreOp::create(builder&: rewriter, location: loc, value: yieldValue, storage: asyncValue);
738 RuntimeSetAvailableOp::create(builder&: rewriter, location: loc, operand: asyncValue);
739 }
740
741 if (coro.asyncToken)
742 // Switch the coroutine completion token to available state.
743 RuntimeSetAvailableOp::create(builder&: rewriter, location: loc, operand: *coro.asyncToken);
744
745 cf::BranchOp::create(builder&: rewriter, location: loc, dest: coro.cleanup);
746 rewriter.eraseOp(op);
747
748 return success();
749 }
750
751private:
752 FuncCoroMapPtr coros;
753};
754
755//===----------------------------------------------------------------------===//
756// Convert cf.assert operation to cf.cond_br into `set_error` block.
757//===----------------------------------------------------------------------===//
758
759class AssertOpLowering : public OpConversionPattern<cf::AssertOp> {
760public:
761 AssertOpLowering(MLIRContext *ctx, FuncCoroMapPtr coros)
762 : OpConversionPattern<cf::AssertOp>(ctx), coros(std::move(coros)) {}
763
764 LogicalResult
765 matchAndRewrite(cf::AssertOp op, OpAdaptor adaptor,
766 ConversionPatternRewriter &rewriter) const override {
767 // Check if assert operation is inside the async coroutine function.
768 auto func = op->template getParentOfType<func::FuncOp>();
769 auto funcCoro = coros->find(Val: func);
770 if (funcCoro == coros->end())
771 return rewriter.notifyMatchFailure(
772 arg&: op, msg: "operation is not inside the async coroutine function");
773
774 Location loc = op->getLoc();
775 CoroMachinery &coro = funcCoro->getSecond();
776
777 Block *cont = rewriter.splitBlock(block: op->getBlock(), before: Block::iterator(op));
778 rewriter.setInsertionPointToEnd(cont->getPrevNode());
779 cf::CondBranchOp::create(builder&: rewriter, location: loc, condition: adaptor.getArg(),
780 /*trueDest=*/cont,
781 /*trueArgs=*/trueOperands: ArrayRef<Value>(),
782 /*falseDest=*/setupSetErrorBlock(coro),
783 /*falseArgs=*/falseOperands: ArrayRef<Value>());
784 rewriter.eraseOp(op);
785
786 return success();
787 }
788
789private:
790 FuncCoroMapPtr coros;
791};
792
793//===----------------------------------------------------------------------===//
794void AsyncToAsyncRuntimePass::runOnOperation() {
795 ModuleOp module = getOperation();
796 SymbolTable symbolTable(module);
797
798 // Functions with coroutine CFG setups, which are results of outlining
799 // `async.execute` body regions
800 FuncCoroMapPtr coros =
801 std::make_shared<llvm::DenseMap<func::FuncOp, CoroMachinery>>();
802
803 module.walk(callback: [&](ExecuteOp execute) {
804 coros->insert(KV: outlineExecuteOp(symbolTable, execute));
805 });
806
807 LLVM_DEBUG({
808 llvm::dbgs() << "Outlined " << coros->size()
809 << " functions built from async.execute operations\n";
810 });
811
812 // Returns true if operation is inside the coroutine.
813 auto isInCoroutine = [&](Operation *op) -> bool {
814 auto parentFunc = op->getParentOfType<func::FuncOp>();
815 return coros->contains(Val: parentFunc);
816 };
817
818 // Lower async operations to async.runtime operations.
819 MLIRContext *ctx = module->getContext();
820 RewritePatternSet asyncPatterns(ctx);
821
822 // Conversion to async runtime augments original CFG with the coroutine CFG,
823 // and we have to make sure that structured control flow operations with async
824 // operations in nested regions will be converted to branch-based control flow
825 // before we add the coroutine basic blocks.
826 populateSCFToControlFlowConversionPatterns(patterns&: asyncPatterns);
827
828 // Async lowering does not use type converter because it must preserve all
829 // types for async.runtime operations.
830 asyncPatterns.add<CreateGroupOpLowering, AddToGroupOpLowering>(arg&: ctx);
831
832 asyncPatterns
833 .add<AwaitTokenOpLowering, AwaitValueOpLowering, AwaitAllOpLowering>(
834 arg&: ctx, args&: coros, /*should_lower_blocking_wait=*/args: true);
835
836 // Lower assertions to conditional branches into error blocks.
837 asyncPatterns.add<YieldOpLowering, AssertOpLowering>(arg&: ctx, args&: coros);
838
839 // All high level async operations must be lowered to the runtime operations.
840 ConversionTarget runtimeTarget(*ctx);
841 runtimeTarget.addLegalDialect<AsyncDialect, func::FuncDialect>();
842 runtimeTarget.addIllegalOp<CreateGroupOp, AddToGroupOp>();
843 runtimeTarget.addIllegalOp<ExecuteOp, AwaitOp, AwaitAllOp, async::YieldOp>();
844
845 // Decide if structured control flow has to be lowered to branch-based CFG.
846 runtimeTarget.addDynamicallyLegalDialect<scf::SCFDialect>(callback: [&](Operation *op) {
847 auto walkResult = op->walk(callback: [&](Operation *nested) {
848 bool isAsync = isa<async::AsyncDialect>(Val: nested->getDialect());
849 return isAsync && isInCoroutine(nested) ? WalkResult::interrupt()
850 : WalkResult::advance();
851 });
852 return !walkResult.wasInterrupted();
853 });
854 runtimeTarget.addLegalOp<cf::AssertOp, arith::XOrIOp, arith::ConstantOp,
855 func::ConstantOp, cf::BranchOp, cf::CondBranchOp>();
856
857 // Assertions must be converted to runtime errors inside async functions.
858 runtimeTarget.addDynamicallyLegalOp<cf::AssertOp>(
859 callback: [&](cf::AssertOp op) -> bool {
860 auto func = op->getParentOfType<func::FuncOp>();
861 return !coros->contains(Val: func);
862 });
863
864 if (failed(Result: applyPartialConversion(op: module, target: runtimeTarget,
865 patterns: std::move(asyncPatterns)))) {
866 signalPassFailure();
867 return;
868 }
869}
870
871//===----------------------------------------------------------------------===//
872void mlir::populateAsyncFuncToAsyncRuntimeConversionPatterns(
873 RewritePatternSet &patterns, ConversionTarget &target) {
874 // Functions with coroutine CFG setups, which are results of converting
875 // async.func.
876 FuncCoroMapPtr coros =
877 std::make_shared<llvm::DenseMap<func::FuncOp, CoroMachinery>>();
878 MLIRContext *ctx = patterns.getContext();
879 // Lower async.func to func.func with coroutine cfg.
880 patterns.add<AsyncCallOpLowering>(arg&: ctx);
881 patterns.add<AsyncFuncOpLowering, AsyncReturnOpLowering>(arg&: ctx, args&: coros);
882
883 patterns.add<AwaitTokenOpLowering, AwaitValueOpLowering, AwaitAllOpLowering>(
884 arg&: ctx, args&: coros, /*should_lower_blocking_wait=*/args: false);
885 patterns.add<YieldOpLowering, AssertOpLowering>(arg&: ctx, args&: coros);
886
887 target.addDynamicallyLegalOp<AwaitOp, AwaitAllOp, YieldOp, cf::AssertOp>(
888 callback: [coros](Operation *op) {
889 auto exec = op->getParentOfType<ExecuteOp>();
890 auto func = op->getParentOfType<func::FuncOp>();
891 return exec || !coros->contains(Val: func);
892 });
893}
894
895void AsyncFuncToAsyncRuntimePass::runOnOperation() {
896 ModuleOp module = getOperation();
897
898 // Lower async operations to async.runtime operations.
899 MLIRContext *ctx = module->getContext();
900 RewritePatternSet asyncPatterns(ctx);
901 ConversionTarget runtimeTarget(*ctx);
902
903 // Lower async.func to func.func with coroutine cfg.
904 populateAsyncFuncToAsyncRuntimeConversionPatterns(patterns&: asyncPatterns,
905 target&: runtimeTarget);
906
907 runtimeTarget.addLegalDialect<AsyncDialect, func::FuncDialect>();
908 runtimeTarget.addIllegalOp<async::FuncOp, async::CallOp, async::ReturnOp>();
909
910 runtimeTarget.addLegalOp<arith::XOrIOp, arith::ConstantOp, func::ConstantOp,
911 cf::BranchOp, cf::CondBranchOp>();
912
913 if (failed(Result: applyPartialConversion(op: module, target: runtimeTarget,
914 patterns: std::move(asyncPatterns)))) {
915 signalPassFailure();
916 return;
917 }
918}
919