1//===- AsyncToLLVM.cpp - Convert Async to LLVM dialect --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h"
10
11#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
12#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
13#include "mlir/Conversion/LLVMCommon/Pattern.h"
14#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/Async/IR/Async.h"
17#include "mlir/Dialect/Func/IR/FuncOps.h"
18#include "mlir/Dialect/Func/Transforms/FuncConversions.h"
19#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
20#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
22#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
23#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
24#include "mlir/Dialect/LLVMIR/LLVMCoroIntrinsicOps.h"
25#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
26#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
27#include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h"
28#include "mlir/IR/TypeUtilities.h"
29#include "mlir/Pass/Pass.h"
30#include "mlir/Transforms/DialectConversion.h"
31#include "llvm/ADT/TypeSwitch.h"
32
33namespace mlir {
34#define GEN_PASS_DEF_CONVERTASYNCTOLLVMPASS
35#include "mlir/Conversion/Passes.h.inc"
36} // namespace mlir
37
38#define DEBUG_TYPE "convert-async-to-llvm"
39
40using namespace mlir;
41using namespace mlir::async;
42
43//===----------------------------------------------------------------------===//
44// Async Runtime C API declaration.
45//===----------------------------------------------------------------------===//
46
47static constexpr const char *kAddRef = "mlirAsyncRuntimeAddRef";
48static constexpr const char *kDropRef = "mlirAsyncRuntimeDropRef";
49static constexpr const char *kCreateToken = "mlirAsyncRuntimeCreateToken";
50static constexpr const char *kCreateValue = "mlirAsyncRuntimeCreateValue";
51static constexpr const char *kCreateGroup = "mlirAsyncRuntimeCreateGroup";
52static constexpr const char *kEmplaceToken = "mlirAsyncRuntimeEmplaceToken";
53static constexpr const char *kEmplaceValue = "mlirAsyncRuntimeEmplaceValue";
54static constexpr const char *kSetTokenError = "mlirAsyncRuntimeSetTokenError";
55static constexpr const char *kSetValueError = "mlirAsyncRuntimeSetValueError";
56static constexpr const char *kIsTokenError = "mlirAsyncRuntimeIsTokenError";
57static constexpr const char *kIsValueError = "mlirAsyncRuntimeIsValueError";
58static constexpr const char *kIsGroupError = "mlirAsyncRuntimeIsGroupError";
59static constexpr const char *kAwaitToken = "mlirAsyncRuntimeAwaitToken";
60static constexpr const char *kAwaitValue = "mlirAsyncRuntimeAwaitValue";
61static constexpr const char *kAwaitGroup = "mlirAsyncRuntimeAwaitAllInGroup";
62static constexpr const char *kExecute = "mlirAsyncRuntimeExecute";
63static constexpr const char *kGetValueStorage =
64 "mlirAsyncRuntimeGetValueStorage";
65static constexpr const char *kAddTokenToGroup =
66 "mlirAsyncRuntimeAddTokenToGroup";
67static constexpr const char *kAwaitTokenAndExecute =
68 "mlirAsyncRuntimeAwaitTokenAndExecute";
69static constexpr const char *kAwaitValueAndExecute =
70 "mlirAsyncRuntimeAwaitValueAndExecute";
71static constexpr const char *kAwaitAllAndExecute =
72 "mlirAsyncRuntimeAwaitAllInGroupAndExecute";
73static constexpr const char *kGetNumWorkerThreads =
74 "mlirAsyncRuntimGetNumWorkerThreads";
75
76namespace {
77/// Async Runtime API function types.
78///
79/// Because we can't create API function signature for type parametrized
80/// async.getValue type, we use opaque pointers (!llvm.ptr) instead. After
81/// lowering all async data types become opaque pointers at runtime.
82struct AsyncAPI {
83 // All async types are lowered to opaque LLVM pointers at runtime.
84 static LLVM::LLVMPointerType opaquePointerType(MLIRContext *ctx) {
85 return LLVM::LLVMPointerType::get(context: ctx);
86 }
87
88 static mlir::TokenType tokenType(MLIRContext *ctx) {
89 return mlir::TokenType::get(ctx);
90 }
91
92 static FunctionType addOrDropRefFunctionType(MLIRContext *ctx) {
93 auto ref = opaquePointerType(ctx);
94 auto count = IntegerType::get(context: ctx, width: 64);
95 return FunctionType::get(context: ctx, inputs: {ref, count}, results: {});
96 }
97
98 static FunctionType createTokenFunctionType(MLIRContext *ctx) {
99 return FunctionType::get(context: ctx, inputs: {}, results: {async::TokenType::get(ctx)});
100 }
101
102 static FunctionType createValueFunctionType(MLIRContext *ctx) {
103 auto i64 = IntegerType::get(context: ctx, width: 64);
104 auto value = opaquePointerType(ctx);
105 return FunctionType::get(context: ctx, inputs: {i64}, results: {value});
106 }
107
108 static FunctionType createGroupFunctionType(MLIRContext *ctx) {
109 auto i64 = IntegerType::get(context: ctx, width: 64);
110 return FunctionType::get(context: ctx, inputs: {i64}, results: {GroupType::get(ctx)});
111 }
112
113 static FunctionType getValueStorageFunctionType(MLIRContext *ctx) {
114 auto ptrType = opaquePointerType(ctx);
115 return FunctionType::get(context: ctx, inputs: {ptrType}, results: {ptrType});
116 }
117
118 static FunctionType emplaceTokenFunctionType(MLIRContext *ctx) {
119 return FunctionType::get(context: ctx, inputs: {async::TokenType::get(ctx)}, results: {});
120 }
121
122 static FunctionType emplaceValueFunctionType(MLIRContext *ctx) {
123 auto value = opaquePointerType(ctx);
124 return FunctionType::get(context: ctx, inputs: {value}, results: {});
125 }
126
127 static FunctionType setTokenErrorFunctionType(MLIRContext *ctx) {
128 return FunctionType::get(context: ctx, inputs: {async::TokenType::get(ctx)}, results: {});
129 }
130
131 static FunctionType setValueErrorFunctionType(MLIRContext *ctx) {
132 auto value = opaquePointerType(ctx);
133 return FunctionType::get(context: ctx, inputs: {value}, results: {});
134 }
135
136 static FunctionType isTokenErrorFunctionType(MLIRContext *ctx) {
137 auto i1 = IntegerType::get(context: ctx, width: 1);
138 return FunctionType::get(context: ctx, inputs: {async::TokenType::get(ctx)}, results: {i1});
139 }
140
141 static FunctionType isValueErrorFunctionType(MLIRContext *ctx) {
142 auto value = opaquePointerType(ctx);
143 auto i1 = IntegerType::get(context: ctx, width: 1);
144 return FunctionType::get(context: ctx, inputs: {value}, results: {i1});
145 }
146
147 static FunctionType isGroupErrorFunctionType(MLIRContext *ctx) {
148 auto i1 = IntegerType::get(context: ctx, width: 1);
149 return FunctionType::get(context: ctx, inputs: {GroupType::get(ctx)}, results: {i1});
150 }
151
152 static FunctionType awaitTokenFunctionType(MLIRContext *ctx) {
153 return FunctionType::get(context: ctx, inputs: {async::TokenType::get(ctx)}, results: {});
154 }
155
156 static FunctionType awaitValueFunctionType(MLIRContext *ctx) {
157 auto value = opaquePointerType(ctx);
158 return FunctionType::get(context: ctx, inputs: {value}, results: {});
159 }
160
161 static FunctionType awaitGroupFunctionType(MLIRContext *ctx) {
162 return FunctionType::get(context: ctx, inputs: {GroupType::get(ctx)}, results: {});
163 }
164
165 static FunctionType executeFunctionType(MLIRContext *ctx) {
166 auto ptrType = opaquePointerType(ctx);
167 return FunctionType::get(context: ctx, inputs: {ptrType, ptrType}, results: {});
168 }
169
170 static FunctionType addTokenToGroupFunctionType(MLIRContext *ctx) {
171 auto i64 = IntegerType::get(context: ctx, width: 64);
172 return FunctionType::get(
173 context: ctx, inputs: {async::TokenType::get(ctx), GroupType::get(ctx)}, results: {i64});
174 }
175
176 static FunctionType awaitTokenAndExecuteFunctionType(MLIRContext *ctx) {
177 auto ptrType = opaquePointerType(ctx);
178 return FunctionType::get(
179 context: ctx, inputs: {async::TokenType::get(ctx), ptrType, ptrType}, results: {});
180 }
181
182 static FunctionType awaitValueAndExecuteFunctionType(MLIRContext *ctx) {
183 auto ptrType = opaquePointerType(ctx);
184 return FunctionType::get(context: ctx, inputs: {ptrType, ptrType, ptrType}, results: {});
185 }
186
187 static FunctionType awaitAllAndExecuteFunctionType(MLIRContext *ctx) {
188 auto ptrType = opaquePointerType(ctx);
189 return FunctionType::get(context: ctx, inputs: {GroupType::get(ctx), ptrType, ptrType}, results: {});
190 }
191
192 static FunctionType getNumWorkerThreads(MLIRContext *ctx) {
193 return FunctionType::get(context: ctx, inputs: {}, results: {IndexType::get(context: ctx)});
194 }
195
196 // Auxiliary coroutine resume intrinsic wrapper.
197 static Type resumeFunctionType(MLIRContext *ctx) {
198 auto voidTy = LLVM::LLVMVoidType::get(ctx);
199 auto ptrType = opaquePointerType(ctx);
200 return LLVM::LLVMFunctionType::get(result: voidTy, arguments: {ptrType}, isVarArg: false);
201 }
202};
203} // namespace
204
205/// Adds Async Runtime C API declarations to the module.
206static void addAsyncRuntimeApiDeclarations(ModuleOp module) {
207 auto builder =
208 ImplicitLocOpBuilder::atBlockEnd(loc: module.getLoc(), block: module.getBody());
209
210 auto addFuncDecl = [&](StringRef name, FunctionType type) {
211 if (module.lookupSymbol(name))
212 return;
213 func::FuncOp::create(builder, name, type).setPrivate();
214 };
215
216 MLIRContext *ctx = module.getContext();
217 addFuncDecl(kAddRef, AsyncAPI::addOrDropRefFunctionType(ctx));
218 addFuncDecl(kDropRef, AsyncAPI::addOrDropRefFunctionType(ctx));
219 addFuncDecl(kCreateToken, AsyncAPI::createTokenFunctionType(ctx));
220 addFuncDecl(kCreateValue, AsyncAPI::createValueFunctionType(ctx));
221 addFuncDecl(kCreateGroup, AsyncAPI::createGroupFunctionType(ctx));
222 addFuncDecl(kEmplaceToken, AsyncAPI::emplaceTokenFunctionType(ctx));
223 addFuncDecl(kEmplaceValue, AsyncAPI::emplaceValueFunctionType(ctx));
224 addFuncDecl(kSetTokenError, AsyncAPI::setTokenErrorFunctionType(ctx));
225 addFuncDecl(kSetValueError, AsyncAPI::setValueErrorFunctionType(ctx));
226 addFuncDecl(kIsTokenError, AsyncAPI::isTokenErrorFunctionType(ctx));
227 addFuncDecl(kIsValueError, AsyncAPI::isValueErrorFunctionType(ctx));
228 addFuncDecl(kIsGroupError, AsyncAPI::isGroupErrorFunctionType(ctx));
229 addFuncDecl(kAwaitToken, AsyncAPI::awaitTokenFunctionType(ctx));
230 addFuncDecl(kAwaitValue, AsyncAPI::awaitValueFunctionType(ctx));
231 addFuncDecl(kAwaitGroup, AsyncAPI::awaitGroupFunctionType(ctx));
232 addFuncDecl(kExecute, AsyncAPI::executeFunctionType(ctx));
233 addFuncDecl(kGetValueStorage, AsyncAPI::getValueStorageFunctionType(ctx));
234 addFuncDecl(kAddTokenToGroup, AsyncAPI::addTokenToGroupFunctionType(ctx));
235 addFuncDecl(kAwaitTokenAndExecute,
236 AsyncAPI::awaitTokenAndExecuteFunctionType(ctx));
237 addFuncDecl(kAwaitValueAndExecute,
238 AsyncAPI::awaitValueAndExecuteFunctionType(ctx));
239 addFuncDecl(kAwaitAllAndExecute,
240 AsyncAPI::awaitAllAndExecuteFunctionType(ctx));
241 addFuncDecl(kGetNumWorkerThreads, AsyncAPI::getNumWorkerThreads(ctx));
242}
243
244//===----------------------------------------------------------------------===//
245// Coroutine resume function wrapper.
246//===----------------------------------------------------------------------===//
247
248static constexpr const char *kResume = "__resume";
249
250/// A function that takes a coroutine handle and calls a `llvm.coro.resume`
251/// intrinsics. We need this function to be able to pass it to the async
252/// runtime execute API.
253static void addResumeFunction(ModuleOp module) {
254 if (module.lookupSymbol(name: kResume))
255 return;
256
257 MLIRContext *ctx = module.getContext();
258 auto loc = module.getLoc();
259 auto moduleBuilder = ImplicitLocOpBuilder::atBlockEnd(loc, block: module.getBody());
260
261 auto voidTy = LLVM::LLVMVoidType::get(ctx);
262 Type ptrType = AsyncAPI::opaquePointerType(ctx);
263
264 auto resumeOp = LLVM::LLVMFuncOp::create(
265 builder&: moduleBuilder, name: kResume, type: LLVM::LLVMFunctionType::get(result: voidTy, arguments: {ptrType}));
266 resumeOp.setPrivate();
267
268 auto *block = resumeOp.addEntryBlock(builder&: moduleBuilder);
269 auto blockBuilder = ImplicitLocOpBuilder::atBlockEnd(loc, block);
270
271 LLVM::CoroResumeOp::create(builder&: blockBuilder, handle: resumeOp.getArgument(idx: 0));
272 LLVM::ReturnOp::create(builder&: blockBuilder, args: ValueRange());
273}
274
275//===----------------------------------------------------------------------===//
276// Convert Async dialect types to LLVM types.
277//===----------------------------------------------------------------------===//
278
279namespace {
280/// AsyncRuntimeTypeConverter only converts types from the Async dialect to
281/// their runtime type (opaque pointers) and does not convert any other types.
282class AsyncRuntimeTypeConverter : public TypeConverter {
283public:
284 AsyncRuntimeTypeConverter(const LowerToLLVMOptions &options) {
285 addConversion(callback: [](Type type) { return type; });
286 addConversion(callback: [](Type type) { return convertAsyncTypes(type); });
287
288 // Use UnrealizedConversionCast as the bridge so that we don't need to pull
289 // in patterns for other dialects.
290 auto addUnrealizedCast = [](OpBuilder &builder, Type type,
291 ValueRange inputs, Location loc) -> Value {
292 auto cast =
293 UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs);
294 return cast.getResult(i: 0);
295 };
296
297 addSourceMaterialization(callback&: addUnrealizedCast);
298 addTargetMaterialization(callback&: addUnrealizedCast);
299 }
300
301 static std::optional<Type> convertAsyncTypes(Type type) {
302 if (isa<async::TokenType, GroupType, ValueType>(Val: type))
303 return AsyncAPI::opaquePointerType(ctx: type.getContext());
304
305 if (isa<CoroIdType, CoroStateType>(Val: type))
306 return AsyncAPI::tokenType(ctx: type.getContext());
307 if (isa<CoroHandleType>(Val: type))
308 return AsyncAPI::opaquePointerType(ctx: type.getContext());
309
310 return std::nullopt;
311 }
312};
313
314/// Base class for conversion patterns requiring AsyncRuntimeTypeConverter
315/// as type converter. Allows access to it via the 'getTypeConverter'
316/// convenience method.
317template <typename SourceOp>
318class AsyncOpConversionPattern : public OpConversionPattern<SourceOp> {
319
320 using Base = OpConversionPattern<SourceOp>;
321
322public:
323 AsyncOpConversionPattern(const AsyncRuntimeTypeConverter &typeConverter,
324 MLIRContext *context)
325 : Base(typeConverter, context) {}
326
327 /// Returns the 'AsyncRuntimeTypeConverter' of the pattern.
328 const AsyncRuntimeTypeConverter *getTypeConverter() const {
329 return static_cast<const AsyncRuntimeTypeConverter *>(
330 Base::getTypeConverter());
331 }
332};
333
334} // namespace
335
336//===----------------------------------------------------------------------===//
337// Convert async.coro.id to @llvm.coro.id intrinsic.
338//===----------------------------------------------------------------------===//
339
340namespace {
341class CoroIdOpConversion : public AsyncOpConversionPattern<CoroIdOp> {
342public:
343 using AsyncOpConversionPattern::AsyncOpConversionPattern;
344
345 LogicalResult
346 matchAndRewrite(CoroIdOp op, OpAdaptor adaptor,
347 ConversionPatternRewriter &rewriter) const override {
348 auto token = AsyncAPI::tokenType(ctx: op->getContext());
349 auto ptrType = AsyncAPI::opaquePointerType(ctx: op->getContext());
350 auto loc = op->getLoc();
351
352 // Constants for initializing coroutine frame.
353 auto constZero =
354 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: 0);
355 auto nullPtr = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrType);
356
357 // Get coroutine id: @llvm.coro.id.
358 rewriter.replaceOpWithNewOp<LLVM::CoroIdOp>(
359 op, args&: token, args: ValueRange({constZero, nullPtr, nullPtr, nullPtr}));
360
361 return success();
362 }
363};
364} // namespace
365
366//===----------------------------------------------------------------------===//
367// Convert async.coro.begin to @llvm.coro.begin intrinsic.
368//===----------------------------------------------------------------------===//
369
370namespace {
371class CoroBeginOpConversion : public AsyncOpConversionPattern<CoroBeginOp> {
372public:
373 using AsyncOpConversionPattern::AsyncOpConversionPattern;
374
375 LogicalResult
376 matchAndRewrite(CoroBeginOp op, OpAdaptor adaptor,
377 ConversionPatternRewriter &rewriter) const override {
378 auto ptrType = AsyncAPI::opaquePointerType(ctx: op->getContext());
379 auto loc = op->getLoc();
380
381 // Get coroutine frame size: @llvm.coro.size.i64.
382 Value coroSize =
383 LLVM::CoroSizeOp::create(builder&: rewriter, location: loc, res: rewriter.getI64Type());
384 // Get coroutine frame alignment: @llvm.coro.align.i64.
385 Value coroAlign =
386 LLVM::CoroAlignOp::create(builder&: rewriter, location: loc, res: rewriter.getI64Type());
387
388 // Round up the size to be multiple of the alignment. Since aligned_alloc
389 // requires the size parameter be an integral multiple of the alignment
390 // parameter.
391 auto makeConstant = [&](uint64_t c) {
392 return LLVM::ConstantOp::create(builder&: rewriter, location: op->getLoc(),
393 type: rewriter.getI64Type(), value: c);
394 };
395 coroSize = LLVM::AddOp::create(builder&: rewriter, location: op->getLoc(), lhs: coroSize, rhs: coroAlign);
396 coroSize =
397 LLVM::SubOp::create(builder&: rewriter, location: op->getLoc(), lhs: coroSize, rhs: makeConstant(1));
398 Value negCoroAlign =
399 LLVM::SubOp::create(builder&: rewriter, location: op->getLoc(), lhs: makeConstant(0), rhs: coroAlign);
400 coroSize =
401 LLVM::AndOp::create(builder&: rewriter, location: op->getLoc(), lhs: coroSize, rhs: negCoroAlign);
402
403 // Allocate memory for the coroutine frame.
404 auto allocFuncOp = LLVM::lookupOrCreateAlignedAllocFn(
405 b&: rewriter, moduleOp: op->getParentOfType<ModuleOp>(), indexType: rewriter.getI64Type());
406 if (failed(Result: allocFuncOp))
407 return failure();
408 auto coroAlloc = LLVM::CallOp::create(builder&: rewriter, location: loc, func: allocFuncOp.value(),
409 args: ValueRange{coroAlign, coroSize});
410
411 // Begin a coroutine: @llvm.coro.begin.
412 auto coroId = CoroBeginOpAdaptor(adaptor.getOperands()).getId();
413 rewriter.replaceOpWithNewOp<LLVM::CoroBeginOp>(
414 op, args&: ptrType, args: ValueRange({coroId, coroAlloc.getResult()}));
415
416 return success();
417 }
418};
419} // namespace
420
421//===----------------------------------------------------------------------===//
422// Convert async.coro.free to @llvm.coro.free intrinsic.
423//===----------------------------------------------------------------------===//
424
425namespace {
426class CoroFreeOpConversion : public AsyncOpConversionPattern<CoroFreeOp> {
427public:
428 using AsyncOpConversionPattern::AsyncOpConversionPattern;
429
430 LogicalResult
431 matchAndRewrite(CoroFreeOp op, OpAdaptor adaptor,
432 ConversionPatternRewriter &rewriter) const override {
433 auto ptrType = AsyncAPI::opaquePointerType(ctx: op->getContext());
434 auto loc = op->getLoc();
435
436 // Get a pointer to the coroutine frame memory: @llvm.coro.free.
437 auto coroMem =
438 LLVM::CoroFreeOp::create(builder&: rewriter, location: loc, resultTypes: ptrType, operands: adaptor.getOperands());
439
440 // Free the memory.
441 auto freeFuncOp =
442 LLVM::lookupOrCreateFreeFn(b&: rewriter, moduleOp: op->getParentOfType<ModuleOp>());
443 if (failed(Result: freeFuncOp))
444 return failure();
445 rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, args&: freeFuncOp.value(),
446 args: ValueRange(coroMem.getResult()));
447
448 return success();
449 }
450};
451} // namespace
452
453//===----------------------------------------------------------------------===//
454// Convert async.coro.end to @llvm.coro.end intrinsic.
455//===----------------------------------------------------------------------===//
456
457namespace {
458class CoroEndOpConversion : public OpConversionPattern<CoroEndOp> {
459public:
460 using Base::Base;
461
462 LogicalResult
463 matchAndRewrite(CoroEndOp op, OpAdaptor adaptor,
464 ConversionPatternRewriter &rewriter) const override {
465 // We are not in the block that is part of the unwind sequence.
466 auto constFalse =
467 LLVM::ConstantOp::create(builder&: rewriter, location: op->getLoc(), res: rewriter.getI1Type(),
468 value: rewriter.getBoolAttr(value: false));
469 auto noneToken = LLVM::NoneTokenOp::create(builder&: rewriter, location: op->getLoc());
470
471 // Mark the end of a coroutine: @llvm.coro.end.
472 auto coroHdl = adaptor.getHandle();
473 LLVM::CoroEndOp::create(builder&: rewriter, location: op->getLoc(), resultTypes: rewriter.getI1Type(),
474 operands: ValueRange({coroHdl, constFalse, noneToken}));
475 rewriter.eraseOp(op);
476
477 return success();
478 }
479};
480} // namespace
481
482//===----------------------------------------------------------------------===//
483// Convert async.coro.save to @llvm.coro.save intrinsic.
484//===----------------------------------------------------------------------===//
485
486namespace {
487class CoroSaveOpConversion : public OpConversionPattern<CoroSaveOp> {
488public:
489 using Base::Base;
490
491 LogicalResult
492 matchAndRewrite(CoroSaveOp op, OpAdaptor adaptor,
493 ConversionPatternRewriter &rewriter) const override {
494 // Save the coroutine state: @llvm.coro.save
495 rewriter.replaceOpWithNewOp<LLVM::CoroSaveOp>(
496 op, args: AsyncAPI::tokenType(ctx: op->getContext()), args: adaptor.getOperands());
497
498 return success();
499 }
500};
501} // namespace
502
503//===----------------------------------------------------------------------===//
504// Convert async.coro.suspend to @llvm.coro.suspend intrinsic.
505//===----------------------------------------------------------------------===//
506
507namespace {
508
509/// Convert async.coro.suspend to the @llvm.coro.suspend intrinsic call, and
510/// branch to the appropriate block based on the return code.
511///
512/// Before:
513///
514/// ^suspended:
515/// "opBefore"(...)
516/// async.coro.suspend %state, ^suspend, ^resume, ^cleanup
517/// ^resume:
518/// "op"(...)
519/// ^cleanup: ...
520/// ^suspend: ...
521///
522/// After:
523///
524/// ^suspended:
525/// "opBefore"(...)
526/// %suspend = llmv.intr.coro.suspend ...
527/// switch %suspend [-1: ^suspend, 0: ^resume, 1: ^cleanup]
528/// ^resume:
529/// "op"(...)
530/// ^cleanup: ...
531/// ^suspend: ...
532///
533class CoroSuspendOpConversion : public OpConversionPattern<CoroSuspendOp> {
534public:
535 using Base::Base;
536
537 LogicalResult
538 matchAndRewrite(CoroSuspendOp op, OpAdaptor adaptor,
539 ConversionPatternRewriter &rewriter) const override {
540 auto i8 = rewriter.getIntegerType(width: 8);
541 auto i32 = rewriter.getI32Type();
542 auto loc = op->getLoc();
543
544 // This is not a final suspension point.
545 auto constFalse = LLVM::ConstantOp::create(
546 builder&: rewriter, location: loc, res: rewriter.getI1Type(), value: rewriter.getBoolAttr(value: false));
547
548 // Suspend a coroutine: @llvm.coro.suspend
549 auto coroState = adaptor.getState();
550 auto coroSuspend = LLVM::CoroSuspendOp::create(
551 builder&: rewriter, location: loc, resultTypes: i8, operands: ValueRange({coroState, constFalse}));
552
553 // Cast return code to i32.
554
555 // After a suspension point decide if we should branch into resume, cleanup
556 // or suspend block of the coroutine (see @llvm.coro.suspend return code
557 // documentation).
558 llvm::SmallVector<int32_t, 2> caseValues = {0, 1};
559 llvm::SmallVector<Block *, 2> caseDest = {op.getResumeDest(),
560 op.getCleanupDest()};
561 rewriter.replaceOpWithNewOp<LLVM::SwitchOp>(
562 op, args: LLVM::SExtOp::create(builder&: rewriter, location: loc, res: i32, arg: coroSuspend.getResult()),
563 /*defaultDestination=*/args: op.getSuspendDest(),
564 /*defaultOperands=*/args: ValueRange(),
565 /*caseValues=*/args&: caseValues,
566 /*caseDestinations=*/args&: caseDest,
567 /*caseOperands=*/args: ArrayRef<ValueRange>({ValueRange(), ValueRange()}),
568 /*branchWeights=*/args: ArrayRef<int32_t>());
569
570 return success();
571 }
572};
573} // namespace
574
575//===----------------------------------------------------------------------===//
576// Convert async.runtime.create to the corresponding runtime API call.
577//
578// To allocate storage for the async values we use getelementptr trick:
579// http://nondot.org/sabre/LLVMNotes/SizeOf-OffsetOf-VariableSizedStructs.txt
580//===----------------------------------------------------------------------===//
581
582namespace {
583class RuntimeCreateOpLowering : public ConvertOpToLLVMPattern<RuntimeCreateOp> {
584public:
585 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
586
587 LogicalResult
588 matchAndRewrite(RuntimeCreateOp op, OpAdaptor adaptor,
589 ConversionPatternRewriter &rewriter) const override {
590 const TypeConverter *converter = getTypeConverter();
591 Type resultType = op->getResultTypes()[0];
592
593 // Tokens creation maps to a simple function call.
594 if (isa<async::TokenType>(Val: resultType)) {
595 rewriter.replaceOpWithNewOp<func::CallOp>(
596 op, args: kCreateToken, args: converter->convertType(t: resultType));
597 return success();
598 }
599
600 // To create a value we need to compute the storage requirement.
601 if (auto value = dyn_cast<ValueType>(Val&: resultType)) {
602 // Returns the size requirements for the async value storage.
603 auto sizeOf = [&](ValueType valueType) -> Value {
604 auto loc = op->getLoc();
605 auto i64 = rewriter.getI64Type();
606
607 auto storedType = converter->convertType(t: valueType.getValueType());
608 auto storagePtrType =
609 AsyncAPI::opaquePointerType(ctx: rewriter.getContext());
610
611 // %Size = getelementptr %T* null, int 1
612 // %SizeI = ptrtoint %T* %Size to i64
613 auto nullPtr = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: storagePtrType);
614 auto gep =
615 LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: storagePtrType, elementType: storedType,
616 basePtr: nullPtr, indices: ArrayRef<LLVM::GEPArg>{1});
617 return LLVM::PtrToIntOp::create(builder&: rewriter, location: loc, res: i64, arg: gep);
618 };
619
620 rewriter.replaceOpWithNewOp<func::CallOp>(op, args: kCreateValue, args&: resultType,
621 args: sizeOf(value));
622
623 return success();
624 }
625
626 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported async type");
627 }
628};
629} // namespace
630
631//===----------------------------------------------------------------------===//
632// Convert async.runtime.create_group to the corresponding runtime API call.
633//===----------------------------------------------------------------------===//
634
635namespace {
636class RuntimeCreateGroupOpLowering
637 : public ConvertOpToLLVMPattern<RuntimeCreateGroupOp> {
638public:
639 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
640
641 LogicalResult
642 matchAndRewrite(RuntimeCreateGroupOp op, OpAdaptor adaptor,
643 ConversionPatternRewriter &rewriter) const override {
644 const TypeConverter *converter = getTypeConverter();
645 Type resultType = op.getResult().getType();
646
647 rewriter.replaceOpWithNewOp<func::CallOp>(
648 op, args: kCreateGroup, args: converter->convertType(t: resultType),
649 args: adaptor.getOperands());
650 return success();
651 }
652};
653} // namespace
654
655//===----------------------------------------------------------------------===//
656// Convert async.runtime.set_available to the corresponding runtime API call.
657//===----------------------------------------------------------------------===//
658
659namespace {
660class RuntimeSetAvailableOpLowering
661 : public OpConversionPattern<RuntimeSetAvailableOp> {
662public:
663 using Base::Base;
664
665 LogicalResult
666 matchAndRewrite(RuntimeSetAvailableOp op, OpAdaptor adaptor,
667 ConversionPatternRewriter &rewriter) const override {
668 StringRef apiFuncName =
669 TypeSwitch<Type, StringRef>(op.getOperand().getType())
670 .Case<async::TokenType>(caseFn: [](Type) { return kEmplaceToken; })
671 .Case<ValueType>(caseFn: [](Type) { return kEmplaceValue; });
672
673 rewriter.replaceOpWithNewOp<func::CallOp>(op, args&: apiFuncName, args: TypeRange(),
674 args: adaptor.getOperands());
675
676 return success();
677 }
678};
679} // namespace
680
681//===----------------------------------------------------------------------===//
682// Convert async.runtime.set_error to the corresponding runtime API call.
683//===----------------------------------------------------------------------===//
684
685namespace {
686class RuntimeSetErrorOpLowering
687 : public OpConversionPattern<RuntimeSetErrorOp> {
688public:
689 using Base::Base;
690
691 LogicalResult
692 matchAndRewrite(RuntimeSetErrorOp op, OpAdaptor adaptor,
693 ConversionPatternRewriter &rewriter) const override {
694 StringRef apiFuncName =
695 TypeSwitch<Type, StringRef>(op.getOperand().getType())
696 .Case<async::TokenType>(caseFn: [](Type) { return kSetTokenError; })
697 .Case<ValueType>(caseFn: [](Type) { return kSetValueError; });
698
699 rewriter.replaceOpWithNewOp<func::CallOp>(op, args&: apiFuncName, args: TypeRange(),
700 args: adaptor.getOperands());
701
702 return success();
703 }
704};
705} // namespace
706
707//===----------------------------------------------------------------------===//
708// Convert async.runtime.is_error to the corresponding runtime API call.
709//===----------------------------------------------------------------------===//
710
711namespace {
712class RuntimeIsErrorOpLowering : public OpConversionPattern<RuntimeIsErrorOp> {
713public:
714 using Base::Base;
715
716 LogicalResult
717 matchAndRewrite(RuntimeIsErrorOp op, OpAdaptor adaptor,
718 ConversionPatternRewriter &rewriter) const override {
719 StringRef apiFuncName =
720 TypeSwitch<Type, StringRef>(op.getOperand().getType())
721 .Case<async::TokenType>(caseFn: [](Type) { return kIsTokenError; })
722 .Case<GroupType>(caseFn: [](Type) { return kIsGroupError; })
723 .Case<ValueType>(caseFn: [](Type) { return kIsValueError; });
724
725 rewriter.replaceOpWithNewOp<func::CallOp>(
726 op, args&: apiFuncName, args: rewriter.getI1Type(), args: adaptor.getOperands());
727 return success();
728 }
729};
730} // namespace
731
732//===----------------------------------------------------------------------===//
733// Convert async.runtime.await to the corresponding runtime API call.
734//===----------------------------------------------------------------------===//
735
736namespace {
737class RuntimeAwaitOpLowering : public OpConversionPattern<RuntimeAwaitOp> {
738public:
739 using Base::Base;
740
741 LogicalResult
742 matchAndRewrite(RuntimeAwaitOp op, OpAdaptor adaptor,
743 ConversionPatternRewriter &rewriter) const override {
744 StringRef apiFuncName =
745 TypeSwitch<Type, StringRef>(op.getOperand().getType())
746 .Case<async::TokenType>(caseFn: [](Type) { return kAwaitToken; })
747 .Case<ValueType>(caseFn: [](Type) { return kAwaitValue; })
748 .Case<GroupType>(caseFn: [](Type) { return kAwaitGroup; });
749
750 func::CallOp::create(builder&: rewriter, location: op->getLoc(), callee: apiFuncName, results: TypeRange(),
751 operands: adaptor.getOperands());
752 rewriter.eraseOp(op);
753
754 return success();
755 }
756};
757} // namespace
758
759//===----------------------------------------------------------------------===//
760// Convert async.runtime.await_and_resume to the corresponding runtime API call.
761//===----------------------------------------------------------------------===//
762
763namespace {
764class RuntimeAwaitAndResumeOpLowering
765 : public AsyncOpConversionPattern<RuntimeAwaitAndResumeOp> {
766public:
767 using AsyncOpConversionPattern::AsyncOpConversionPattern;
768
769 LogicalResult
770 matchAndRewrite(RuntimeAwaitAndResumeOp op, OpAdaptor adaptor,
771 ConversionPatternRewriter &rewriter) const override {
772 StringRef apiFuncName =
773 TypeSwitch<Type, StringRef>(op.getOperand().getType())
774 .Case<async::TokenType>(caseFn: [](Type) { return kAwaitTokenAndExecute; })
775 .Case<ValueType>(caseFn: [](Type) { return kAwaitValueAndExecute; })
776 .Case<GroupType>(caseFn: [](Type) { return kAwaitAllAndExecute; });
777
778 Value operand = adaptor.getOperand();
779 Value handle = adaptor.getHandle();
780
781 // A pointer to coroutine resume intrinsic wrapper.
782 addResumeFunction(module: op->getParentOfType<ModuleOp>());
783 auto resumePtr = LLVM::AddressOfOp::create(
784 builder&: rewriter, location: op->getLoc(),
785 res: AsyncAPI::opaquePointerType(ctx: rewriter.getContext()), global_name: kResume);
786
787 func::CallOp::create(builder&: rewriter, location: op->getLoc(), callee: apiFuncName, results: TypeRange(),
788 operands: ValueRange({operand, handle, resumePtr.getRes()}));
789 rewriter.eraseOp(op);
790
791 return success();
792 }
793};
794} // namespace
795
796//===----------------------------------------------------------------------===//
797// Convert async.runtime.resume to the corresponding runtime API call.
798//===----------------------------------------------------------------------===//
799
800namespace {
801class RuntimeResumeOpLowering
802 : public AsyncOpConversionPattern<RuntimeResumeOp> {
803public:
804 using AsyncOpConversionPattern::AsyncOpConversionPattern;
805
806 LogicalResult
807 matchAndRewrite(RuntimeResumeOp op, OpAdaptor adaptor,
808 ConversionPatternRewriter &rewriter) const override {
809 // A pointer to coroutine resume intrinsic wrapper.
810 addResumeFunction(module: op->getParentOfType<ModuleOp>());
811 auto resumePtr = LLVM::AddressOfOp::create(
812 builder&: rewriter, location: op->getLoc(),
813 res: AsyncAPI::opaquePointerType(ctx: rewriter.getContext()), global_name: kResume);
814
815 // Call async runtime API to execute a coroutine in the managed thread.
816 auto coroHdl = adaptor.getHandle();
817 rewriter.replaceOpWithNewOp<func::CallOp>(
818 op, args: TypeRange(), args: kExecute, args: ValueRange({coroHdl, resumePtr.getRes()}));
819
820 return success();
821 }
822};
823} // namespace
824
825//===----------------------------------------------------------------------===//
826// Convert async.runtime.store to the corresponding runtime API call.
827//===----------------------------------------------------------------------===//
828
829namespace {
830class RuntimeStoreOpLowering : public ConvertOpToLLVMPattern<RuntimeStoreOp> {
831public:
832 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
833
834 LogicalResult
835 matchAndRewrite(RuntimeStoreOp op, OpAdaptor adaptor,
836 ConversionPatternRewriter &rewriter) const override {
837 Location loc = op->getLoc();
838
839 // Get a pointer to the async value storage from the runtime.
840 auto ptrType = AsyncAPI::opaquePointerType(ctx: rewriter.getContext());
841 auto storage = adaptor.getStorage();
842 auto storagePtr = func::CallOp::create(builder&: rewriter, location: loc, callee: kGetValueStorage,
843 results: TypeRange(ptrType), operands: storage);
844
845 // Cast from i8* to the LLVM pointer type.
846 auto valueType = op.getValue().getType();
847 auto llvmValueType = getTypeConverter()->convertType(t: valueType);
848 if (!llvmValueType)
849 return rewriter.notifyMatchFailure(
850 arg&: op, msg: "failed to convert stored value type to LLVM type");
851
852 Value castedStoragePtr = storagePtr.getResult(i: 0);
853 // Store the yielded value into the async value storage.
854 auto value = adaptor.getValue();
855 LLVM::StoreOp::create(builder&: rewriter, location: loc, value, addr: castedStoragePtr);
856
857 // Erase the original runtime store operation.
858 rewriter.eraseOp(op);
859
860 return success();
861 }
862};
863} // namespace
864
865//===----------------------------------------------------------------------===//
866// Convert async.runtime.load to the corresponding runtime API call.
867//===----------------------------------------------------------------------===//
868
869namespace {
870class RuntimeLoadOpLowering : public ConvertOpToLLVMPattern<RuntimeLoadOp> {
871public:
872 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
873
874 LogicalResult
875 matchAndRewrite(RuntimeLoadOp op, OpAdaptor adaptor,
876 ConversionPatternRewriter &rewriter) const override {
877 Location loc = op->getLoc();
878
879 // Get a pointer to the async value storage from the runtime.
880 auto ptrType = AsyncAPI::opaquePointerType(ctx: rewriter.getContext());
881 auto storage = adaptor.getStorage();
882 auto storagePtr = func::CallOp::create(builder&: rewriter, location: loc, callee: kGetValueStorage,
883 results: TypeRange(ptrType), operands: storage);
884
885 // Cast from i8* to the LLVM pointer type.
886 auto valueType = op.getResult().getType();
887 auto llvmValueType = getTypeConverter()->convertType(t: valueType);
888 if (!llvmValueType)
889 return rewriter.notifyMatchFailure(
890 arg&: op, msg: "failed to convert loaded value type to LLVM type");
891
892 Value castedStoragePtr = storagePtr.getResult(i: 0);
893
894 // Load from the casted pointer.
895 rewriter.replaceOpWithNewOp<LLVM::LoadOp>(op, args&: llvmValueType,
896 args&: castedStoragePtr);
897
898 return success();
899 }
900};
901} // namespace
902
903//===----------------------------------------------------------------------===//
904// Convert async.runtime.add_to_group to the corresponding runtime API call.
905//===----------------------------------------------------------------------===//
906
907namespace {
908class RuntimeAddToGroupOpLowering
909 : public OpConversionPattern<RuntimeAddToGroupOp> {
910public:
911 using Base::Base;
912
913 LogicalResult
914 matchAndRewrite(RuntimeAddToGroupOp op, OpAdaptor adaptor,
915 ConversionPatternRewriter &rewriter) const override {
916 // Currently we can only add tokens to the group.
917 if (!isa<async::TokenType>(Val: op.getOperand().getType()))
918 return rewriter.notifyMatchFailure(arg&: op, msg: "only token type is supported");
919
920 // Replace with a runtime API function call.
921 rewriter.replaceOpWithNewOp<func::CallOp>(
922 op, args: kAddTokenToGroup, args: rewriter.getI64Type(), args: adaptor.getOperands());
923
924 return success();
925 }
926};
927} // namespace
928
929//===----------------------------------------------------------------------===//
930// Convert async.runtime.num_worker_threads to the corresponding runtime API
931// call.
932//===----------------------------------------------------------------------===//
933
934namespace {
935class RuntimeNumWorkerThreadsOpLowering
936 : public OpConversionPattern<RuntimeNumWorkerThreadsOp> {
937public:
938 using Base::Base;
939
940 LogicalResult
941 matchAndRewrite(RuntimeNumWorkerThreadsOp op, OpAdaptor adaptor,
942 ConversionPatternRewriter &rewriter) const override {
943
944 // Replace with a runtime API function call.
945 rewriter.replaceOpWithNewOp<func::CallOp>(op, args: kGetNumWorkerThreads,
946 args: rewriter.getIndexType());
947
948 return success();
949 }
950};
951} // namespace
952
953//===----------------------------------------------------------------------===//
954// Async reference counting ops lowering (`async.runtime.add_ref` and
955// `async.runtime.drop_ref` to the corresponding API calls).
956//===----------------------------------------------------------------------===//
957
958namespace {
959template <typename RefCountingOp>
960class RefCountingOpLowering : public OpConversionPattern<RefCountingOp> {
961public:
962 explicit RefCountingOpLowering(const TypeConverter &converter,
963 MLIRContext *ctx, StringRef apiFunctionName)
964 : OpConversionPattern<RefCountingOp>(converter, ctx),
965 apiFunctionName(apiFunctionName) {}
966
967 LogicalResult
968 matchAndRewrite(RefCountingOp op, typename RefCountingOp::Adaptor adaptor,
969 ConversionPatternRewriter &rewriter) const override {
970 auto count =
971 arith::ConstantOp::create(rewriter, op->getLoc(), rewriter.getI64Type(),
972 rewriter.getI64IntegerAttr(value: op.getCount()));
973
974 auto operand = adaptor.getOperand();
975 rewriter.replaceOpWithNewOp<func::CallOp>(op, TypeRange(), apiFunctionName,
976 ValueRange({operand, count}));
977
978 return success();
979 }
980
981private:
982 StringRef apiFunctionName;
983};
984
985class RuntimeAddRefOpLowering : public RefCountingOpLowering<RuntimeAddRefOp> {
986public:
987 explicit RuntimeAddRefOpLowering(const TypeConverter &converter,
988 MLIRContext *ctx)
989 : RefCountingOpLowering(converter, ctx, kAddRef) {}
990};
991
992class RuntimeDropRefOpLowering
993 : public RefCountingOpLowering<RuntimeDropRefOp> {
994public:
995 explicit RuntimeDropRefOpLowering(const TypeConverter &converter,
996 MLIRContext *ctx)
997 : RefCountingOpLowering(converter, ctx, kDropRef) {}
998};
999} // namespace
1000
1001//===----------------------------------------------------------------------===//
1002// Convert return operations that return async values from async regions.
1003//===----------------------------------------------------------------------===//
1004
1005namespace {
1006class ReturnOpOpConversion : public OpConversionPattern<func::ReturnOp> {
1007public:
1008 using Base::Base;
1009
1010 LogicalResult
1011 matchAndRewrite(func::ReturnOp op, OpAdaptor adaptor,
1012 ConversionPatternRewriter &rewriter) const override {
1013 rewriter.replaceOpWithNewOp<func::ReturnOp>(op, args: adaptor.getOperands());
1014 return success();
1015 }
1016};
1017} // namespace
1018
1019//===----------------------------------------------------------------------===//
1020
1021namespace {
1022struct ConvertAsyncToLLVMPass
1023 : public impl::ConvertAsyncToLLVMPassBase<ConvertAsyncToLLVMPass> {
1024 using Base::Base;
1025
1026 void runOnOperation() override;
1027};
1028} // namespace
1029
1030void ConvertAsyncToLLVMPass::runOnOperation() {
1031 ModuleOp module = getOperation();
1032 MLIRContext *ctx = module->getContext();
1033
1034 LowerToLLVMOptions options(ctx);
1035
1036 // Add declarations for most functions required by the coroutines lowering.
1037 // We delay adding the resume function until it's needed because it currently
1038 // fails to compile unless '-O0' is specified.
1039 addAsyncRuntimeApiDeclarations(module);
1040
1041 // Lower async.runtime and async.coro operations to Async Runtime API and
1042 // LLVM coroutine intrinsics.
1043
1044 // Convert async dialect types and operations to LLVM dialect.
1045 AsyncRuntimeTypeConverter converter(options);
1046 RewritePatternSet patterns(ctx);
1047
1048 // We use conversion to LLVM type to lower async.runtime load and store
1049 // operations.
1050 LLVMTypeConverter llvmConverter(ctx, options);
1051 llvmConverter.addConversion(callback: [&](Type type) {
1052 return AsyncRuntimeTypeConverter::convertAsyncTypes(type);
1053 });
1054
1055 // Convert async types in function signatures and function calls.
1056 populateFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(patterns,
1057 converter);
1058 populateCallOpTypeConversionPattern(patterns, converter);
1059
1060 // Convert return operations inside async.execute regions.
1061 patterns.add<ReturnOpOpConversion>(arg&: converter, args&: ctx);
1062
1063 // Lower async.runtime operations to the async runtime API calls.
1064 patterns.add<RuntimeSetAvailableOpLowering, RuntimeSetErrorOpLowering,
1065 RuntimeIsErrorOpLowering, RuntimeAwaitOpLowering,
1066 RuntimeAwaitAndResumeOpLowering, RuntimeResumeOpLowering,
1067 RuntimeAddToGroupOpLowering, RuntimeNumWorkerThreadsOpLowering,
1068 RuntimeAddRefOpLowering, RuntimeDropRefOpLowering>(arg&: converter,
1069 args&: ctx);
1070
1071 // Lower async.runtime operations that rely on LLVM type converter to convert
1072 // from async value payload type to the LLVM type.
1073 patterns.add<RuntimeCreateOpLowering, RuntimeCreateGroupOpLowering,
1074 RuntimeStoreOpLowering, RuntimeLoadOpLowering>(arg&: llvmConverter);
1075
1076 // Lower async coroutine operations to LLVM coroutine intrinsics.
1077 patterns
1078 .add<CoroIdOpConversion, CoroBeginOpConversion, CoroFreeOpConversion,
1079 CoroEndOpConversion, CoroSaveOpConversion, CoroSuspendOpConversion>(
1080 arg&: converter, args&: ctx);
1081
1082 ConversionTarget target(*ctx);
1083 target.addLegalOp<arith::ConstantOp, func::ConstantOp,
1084 UnrealizedConversionCastOp>();
1085 target.addLegalDialect<LLVM::LLVMDialect>();
1086
1087 // All operations from Async dialect must be lowered to the runtime API and
1088 // LLVM intrinsics calls.
1089 target.addIllegalDialect<AsyncDialect>();
1090
1091 // Add dynamic legality constraints to apply conversions defined above.
1092 target.addDynamicallyLegalOp<func::FuncOp>(callback: [&](func::FuncOp op) {
1093 return converter.isSignatureLegal(ty: op.getFunctionType());
1094 });
1095 target.addDynamicallyLegalOp<func::ReturnOp>(callback: [&](func::ReturnOp op) {
1096 return converter.isLegal(range: op.getOperandTypes());
1097 });
1098 target.addDynamicallyLegalOp<func::CallOp>(callback: [&](func::CallOp op) {
1099 return converter.isSignatureLegal(ty: op.getCalleeType());
1100 });
1101
1102 if (failed(Result: applyPartialConversion(op: module, target, patterns: std::move(patterns))))
1103 signalPassFailure();
1104}
1105
1106//===----------------------------------------------------------------------===//
1107// Patterns for structural type conversions for the Async dialect operations.
1108//===----------------------------------------------------------------------===//
1109
1110namespace {
1111class ConvertExecuteOpTypes : public OpConversionPattern<ExecuteOp> {
1112public:
1113 using Base::Base;
1114 LogicalResult
1115 matchAndRewrite(ExecuteOp op, OpAdaptor adaptor,
1116 ConversionPatternRewriter &rewriter) const override {
1117 ExecuteOp newOp =
1118 cast<ExecuteOp>(Val: rewriter.cloneWithoutRegions(op&: *op.getOperation()));
1119 rewriter.inlineRegionBefore(region&: op.getRegion(), parent&: newOp.getRegion(),
1120 before: newOp.getRegion().end());
1121
1122 // Set operands and update block argument and result types.
1123 newOp->setOperands(adaptor.getOperands());
1124 if (failed(Result: rewriter.convertRegionTypes(region: &newOp.getRegion(), converter: *typeConverter)))
1125 return failure();
1126 for (auto result : newOp.getResults())
1127 result.setType(typeConverter->convertType(t: result.getType()));
1128
1129 rewriter.replaceOp(op, newValues: newOp.getResults());
1130 return success();
1131 }
1132};
1133
1134// Dummy pattern to trigger the appropriate type conversion / materialization.
1135class ConvertAwaitOpTypes : public OpConversionPattern<AwaitOp> {
1136public:
1137 using Base::Base;
1138 LogicalResult
1139 matchAndRewrite(AwaitOp op, OpAdaptor adaptor,
1140 ConversionPatternRewriter &rewriter) const override {
1141 rewriter.replaceOpWithNewOp<AwaitOp>(op, args: adaptor.getOperands().front());
1142 return success();
1143 }
1144};
1145
1146// Dummy pattern to trigger the appropriate type conversion / materialization.
1147class ConvertYieldOpTypes : public OpConversionPattern<async::YieldOp> {
1148public:
1149 using Base::Base;
1150 LogicalResult
1151 matchAndRewrite(async::YieldOp op, OpAdaptor adaptor,
1152 ConversionPatternRewriter &rewriter) const override {
1153 rewriter.replaceOpWithNewOp<async::YieldOp>(op, args: adaptor.getOperands());
1154 return success();
1155 }
1156};
1157} // namespace
1158
1159void mlir::populateAsyncStructuralTypeConversionsAndLegality(
1160 TypeConverter &typeConverter, RewritePatternSet &patterns,
1161 ConversionTarget &target) {
1162 typeConverter.addConversion(callback: [&](async::TokenType type) { return type; });
1163 typeConverter.addConversion(callback: [&](ValueType type) {
1164 Type converted = typeConverter.convertType(t: type.getValueType());
1165 return converted ? ValueType::get(valueType: converted) : converted;
1166 });
1167
1168 patterns.add<ConvertExecuteOpTypes, ConvertAwaitOpTypes, ConvertYieldOpTypes>(
1169 arg&: typeConverter, args: patterns.getContext());
1170
1171 target.addDynamicallyLegalOp<AwaitOp, ExecuteOp, async::YieldOp>(
1172 callback: [&](Operation *op) { return typeConverter.isLegal(op); });
1173}
1174