| 1 | //===- Async.cpp - MLIR Async Operations ----------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Async/IR/Async.h" |
| 10 | |
| 11 | #include "mlir/Interfaces/FunctionImplementation.h" |
| 12 | |
| 13 | using namespace mlir; |
| 14 | using namespace mlir::async; |
| 15 | |
| 16 | #include "mlir/Dialect/Async/IR/AsyncOpsDialect.cpp.inc" |
| 17 | |
| 18 | void AsyncDialect::initialize() { |
| 19 | addOperations< |
| 20 | #define GET_OP_LIST |
| 21 | #include "mlir/Dialect/Async/IR/AsyncOps.cpp.inc" |
| 22 | >(); |
| 23 | addTypes< |
| 24 | #define GET_TYPEDEF_LIST |
| 25 | #include "mlir/Dialect/Async/IR/AsyncOpsTypes.cpp.inc" |
| 26 | >(); |
| 27 | } |
| 28 | |
| 29 | //===----------------------------------------------------------------------===// |
| 30 | /// ExecuteOp |
| 31 | //===----------------------------------------------------------------------===// |
| 32 | |
| 33 | constexpr char kOperandSegmentSizesAttr[] = "operandSegmentSizes" ; |
| 34 | |
| 35 | OperandRange ExecuteOp::getEntrySuccessorOperands(RegionSuccessor successor) { |
| 36 | assert(successor.getSuccessor() == &getBodyRegion() && |
| 37 | "invalid region index" ); |
| 38 | return getBodyOperands(); |
| 39 | } |
| 40 | |
| 41 | bool ExecuteOp::areTypesCompatible(Type lhs, Type rhs) { |
| 42 | const auto getValueOrTokenType = [](Type type) { |
| 43 | if (auto value = llvm::dyn_cast<ValueType>(Val&: type)) |
| 44 | return value.getValueType(); |
| 45 | return type; |
| 46 | }; |
| 47 | return getValueOrTokenType(lhs) == getValueOrTokenType(rhs); |
| 48 | } |
| 49 | |
| 50 | void ExecuteOp::getSuccessorRegions(RegionBranchPoint point, |
| 51 | SmallVectorImpl<RegionSuccessor> ®ions) { |
| 52 | // The `body` region branch back to the parent operation. |
| 53 | if (!point.isParent() && |
| 54 | point.getTerminatorPredecessorOrNull()->getParentRegion() == |
| 55 | &getBodyRegion()) { |
| 56 | regions.push_back(Elt: RegionSuccessor(getOperation())); |
| 57 | return; |
| 58 | } |
| 59 | |
| 60 | // Otherwise the successor is the body region. |
| 61 | regions.push_back(Elt: RegionSuccessor(&getBodyRegion())); |
| 62 | } |
| 63 | |
| 64 | ValueRange ExecuteOp::getSuccessorInputs(RegionSuccessor successor) { |
| 65 | return successor.isOperation() ? ValueRange(getBodyResults()) |
| 66 | : ValueRange(getBodyRegion().getArguments()); |
| 67 | } |
| 68 | |
| 69 | void ExecuteOp::build(OpBuilder &builder, OperationState &result, |
| 70 | TypeRange resultTypes, ValueRange dependencies, |
| 71 | ValueRange operands, BodyBuilderFn bodyBuilder) { |
| 72 | OpBuilder::InsertionGuard guard(builder); |
| 73 | result.addOperands(newOperands: dependencies); |
| 74 | result.addOperands(newOperands: operands); |
| 75 | |
| 76 | // Add derived `operandSegmentSizes` attribute based on parsed operands. |
| 77 | int32_t numDependencies = dependencies.size(); |
| 78 | int32_t numOperands = operands.size(); |
| 79 | auto operandSegmentSizes = |
| 80 | builder.getDenseI32ArrayAttr(values: {numDependencies, numOperands}); |
| 81 | result.addAttribute(name: kOperandSegmentSizesAttr, attr: operandSegmentSizes); |
| 82 | |
| 83 | // First result is always a token, and then `resultTypes` wrapped into |
| 84 | // `async.value`. |
| 85 | result.addTypes(newTypes: {async::TokenType::get(ctx: result.getContext())}); |
| 86 | for (Type type : resultTypes) |
| 87 | result.addTypes(newTypes: ValueType::get(valueType: type)); |
| 88 | |
| 89 | // Add a body region with block arguments as unwrapped async value operands. |
| 90 | Region *bodyRegion = result.addRegion(); |
| 91 | Block *bodyBlock = builder.createBlock(parent: bodyRegion); |
| 92 | for (Value operand : operands) { |
| 93 | auto valueType = llvm::dyn_cast<ValueType>(Val: operand.getType()); |
| 94 | bodyBlock->addArgument(type: valueType ? valueType.getValueType() |
| 95 | : operand.getType(), |
| 96 | loc: operand.getLoc()); |
| 97 | } |
| 98 | |
| 99 | // Create the default terminator if the builder is not provided and if the |
| 100 | // expected result is empty. Otherwise, leave this to the caller |
| 101 | // because we don't know which values to return from the execute op. |
| 102 | if (resultTypes.empty() && !bodyBuilder) { |
| 103 | async::YieldOp::create(builder, location: result.location, operands: ValueRange()); |
| 104 | } else if (bodyBuilder) { |
| 105 | bodyBuilder(builder, result.location, bodyBlock->getArguments()); |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | void ExecuteOp::print(OpAsmPrinter &p) { |
| 110 | // [%tokens,...] |
| 111 | if (!getDependencies().empty()) |
| 112 | p << " [" << getDependencies() << "]" ; |
| 113 | |
| 114 | // (%value as %unwrapped: !async.value<!arg.type>, ...) |
| 115 | if (!getBodyOperands().empty()) { |
| 116 | p << " (" ; |
| 117 | Block *entry = getBodyRegion().empty() ? nullptr : &getBodyRegion().front(); |
| 118 | llvm::interleaveComma( |
| 119 | c: getBodyOperands(), os&: p, each_fn: [&, n = 0](Value operand) mutable { |
| 120 | Value argument = entry ? entry->getArgument(i: n++) : Value(); |
| 121 | p << operand << " as " << argument << ": " << operand.getType(); |
| 122 | }); |
| 123 | p << ")" ; |
| 124 | } |
| 125 | |
| 126 | // -> (!async.value<!return.type>, ...) |
| 127 | p.printOptionalArrowTypeList(types: llvm::drop_begin(RangeOrContainer: getResultTypes())); |
| 128 | p.printOptionalAttrDictWithKeyword( |
| 129 | attrs: (*this)->getDiscardableAttrDictionary().getValue(), |
| 130 | elidedAttrs: {kOperandSegmentSizesAttr}); |
| 131 | p << ' '; |
| 132 | p.printRegion(blocks&: getBodyRegion(), /*printEntryBlockArgs=*/false); |
| 133 | } |
| 134 | |
| 135 | ParseResult ExecuteOp::parse(OpAsmParser &parser, OperationState &result) { |
| 136 | MLIRContext *ctx = result.getContext(); |
| 137 | |
| 138 | // Sizes of parsed variadic operands, will be updated below after parsing. |
| 139 | int32_t numDependencies = 0; |
| 140 | |
| 141 | auto tokenTy = async::TokenType::get(ctx); |
| 142 | |
| 143 | // Parse dependency tokens. |
| 144 | if (succeeded(Result: parser.parseOptionalLSquare())) { |
| 145 | SmallVector<OpAsmParser::UnresolvedOperand, 4> tokenArgs; |
| 146 | if (parser.parseOperandList(result&: tokenArgs) || |
| 147 | parser.resolveOperands(operands&: tokenArgs, type: tokenTy, result&: result.operands) || |
| 148 | parser.parseRSquare()) |
| 149 | return failure(); |
| 150 | |
| 151 | numDependencies = tokenArgs.size(); |
| 152 | } |
| 153 | |
| 154 | // Parse async value operands (%value as %unwrapped : !async.value<!type>). |
| 155 | SmallVector<OpAsmParser::UnresolvedOperand, 4> valueArgs; |
| 156 | SmallVector<OpAsmParser::Argument, 4> unwrappedArgs; |
| 157 | SmallVector<Type, 4> valueTypes; |
| 158 | |
| 159 | // Parse a single instance of `%value as %unwrapped : !async.value<!type>`. |
| 160 | auto parseAsyncValueArg = [&]() -> ParseResult { |
| 161 | if (parser.parseOperand(result&: valueArgs.emplace_back()) || |
| 162 | parser.parseKeyword(keyword: "as" ) || |
| 163 | parser.parseArgument(result&: unwrappedArgs.emplace_back()) || |
| 164 | parser.parseColonType(result&: valueTypes.emplace_back())) |
| 165 | return failure(); |
| 166 | |
| 167 | auto valueTy = llvm::dyn_cast<ValueType>(Val&: valueTypes.back()); |
| 168 | unwrappedArgs.back().type = valueTy ? valueTy.getValueType() : Type(); |
| 169 | return success(); |
| 170 | }; |
| 171 | |
| 172 | auto argsLoc = parser.getCurrentLocation(); |
| 173 | if (parser.parseCommaSeparatedList(delimiter: OpAsmParser::Delimiter::OptionalParen, |
| 174 | parseElementFn: parseAsyncValueArg) || |
| 175 | parser.resolveOperands(operands&: valueArgs, types&: valueTypes, loc: argsLoc, result&: result.operands)) |
| 176 | return failure(); |
| 177 | |
| 178 | int32_t numOperands = valueArgs.size(); |
| 179 | |
| 180 | // Add derived `operandSegmentSizes` attribute based on parsed operands. |
| 181 | auto operandSegmentSizes = |
| 182 | parser.getBuilder().getDenseI32ArrayAttr(values: {numDependencies, numOperands}); |
| 183 | result.addAttribute(name: kOperandSegmentSizesAttr, attr: operandSegmentSizes); |
| 184 | |
| 185 | // Parse the types of results returned from the async execute op. |
| 186 | SmallVector<Type, 4> resultTypes; |
| 187 | NamedAttrList attrs; |
| 188 | if (parser.parseOptionalArrowTypeList(result&: resultTypes) || |
| 189 | // Async execute first result is always a completion token. |
| 190 | parser.addTypeToList(type: tokenTy, result&: result.types) || |
| 191 | parser.addTypesToList(types: resultTypes, result&: result.types) || |
| 192 | // Parse operation attributes. |
| 193 | parser.parseOptionalAttrDictWithKeyword(result&: attrs)) |
| 194 | return failure(); |
| 195 | |
| 196 | result.addAttributes(newAttributes: attrs); |
| 197 | |
| 198 | // Parse asynchronous region. |
| 199 | Region *body = result.addRegion(); |
| 200 | return parser.parseRegion(region&: *body, /*arguments=*/unwrappedArgs); |
| 201 | } |
| 202 | |
| 203 | LogicalResult ExecuteOp::verifyRegions() { |
| 204 | // Unwrap async.execute value operands types. |
| 205 | auto unwrappedTypes = llvm::map_range(C: getBodyOperands(), F: [](Value operand) { |
| 206 | return llvm::cast<ValueType>(Val: operand.getType()).getValueType(); |
| 207 | }); |
| 208 | |
| 209 | // Verify that unwrapped argument types matches the body region arguments. |
| 210 | if (getBodyRegion().getArgumentTypes() != unwrappedTypes) |
| 211 | return emitOpError(message: "async body region argument types do not match the " |
| 212 | "execute operation arguments types" ); |
| 213 | |
| 214 | return success(); |
| 215 | } |
| 216 | |
| 217 | //===----------------------------------------------------------------------===// |
| 218 | /// CreateGroupOp |
| 219 | //===----------------------------------------------------------------------===// |
| 220 | |
| 221 | LogicalResult CreateGroupOp::canonicalize(CreateGroupOp op, |
| 222 | PatternRewriter &rewriter) { |
| 223 | // Find all `await_all` users of the group. |
| 224 | llvm::SmallVector<AwaitAllOp> awaitAllUsers; |
| 225 | |
| 226 | auto isAwaitAll = [&](Operation *op) -> bool { |
| 227 | if (AwaitAllOp awaitAll = dyn_cast<AwaitAllOp>(Val: op)) { |
| 228 | awaitAllUsers.push_back(Elt: awaitAll); |
| 229 | return true; |
| 230 | } |
| 231 | return false; |
| 232 | }; |
| 233 | |
| 234 | // Check if all users of the group are `await_all` operations. |
| 235 | if (!llvm::all_of(Range: op->getUsers(), P: isAwaitAll)) |
| 236 | return failure(); |
| 237 | |
| 238 | // If group is only awaited without adding anything to it, we can safely erase |
| 239 | // the create operation and all users. |
| 240 | for (AwaitAllOp awaitAll : awaitAllUsers) |
| 241 | rewriter.eraseOp(op: awaitAll); |
| 242 | rewriter.eraseOp(op); |
| 243 | |
| 244 | return success(); |
| 245 | } |
| 246 | |
| 247 | //===----------------------------------------------------------------------===// |
| 248 | /// AwaitOp |
| 249 | //===----------------------------------------------------------------------===// |
| 250 | |
| 251 | void AwaitOp::build(OpBuilder &builder, OperationState &result, Value operand, |
| 252 | ArrayRef<NamedAttribute> attrs) { |
| 253 | result.addOperands(newOperands: {operand}); |
| 254 | result.attributes.append(inStart: attrs.begin(), inEnd: attrs.end()); |
| 255 | |
| 256 | // Add unwrapped async.value type to the returned values types. |
| 257 | if (auto valueType = llvm::dyn_cast<ValueType>(Val: operand.getType())) |
| 258 | result.addTypes(newTypes: valueType.getValueType()); |
| 259 | } |
| 260 | |
| 261 | static ParseResult parseAwaitResultType(OpAsmParser &parser, Type &operandType, |
| 262 | Type &resultType) { |
| 263 | if (parser.parseType(result&: operandType)) |
| 264 | return failure(); |
| 265 | |
| 266 | // Add unwrapped async.value type to the returned values types. |
| 267 | if (auto valueType = llvm::dyn_cast<ValueType>(Val&: operandType)) |
| 268 | resultType = valueType.getValueType(); |
| 269 | |
| 270 | return success(); |
| 271 | } |
| 272 | |
| 273 | static void printAwaitResultType(OpAsmPrinter &p, Operation *op, |
| 274 | Type operandType, Type resultType) { |
| 275 | p << operandType; |
| 276 | } |
| 277 | |
| 278 | LogicalResult AwaitOp::verify() { |
| 279 | Type argType = getOperand().getType(); |
| 280 | |
| 281 | // Awaiting on a token does not have any results. |
| 282 | if (llvm::isa<async::TokenType>(Val: argType) && !getResultTypes().empty()) |
| 283 | return emitOpError(message: "awaiting on a token must have empty result" ); |
| 284 | |
| 285 | // Awaiting on a value unwraps the async value type. |
| 286 | if (auto value = llvm::dyn_cast<ValueType>(Val&: argType)) { |
| 287 | if (*getResultType() != value.getValueType()) |
| 288 | return emitOpError() << "result type " << *getResultType() |
| 289 | << " does not match async value type " |
| 290 | << value.getValueType(); |
| 291 | } |
| 292 | |
| 293 | return success(); |
| 294 | } |
| 295 | |
| 296 | //===----------------------------------------------------------------------===// |
| 297 | // FuncOp |
| 298 | //===----------------------------------------------------------------------===// |
| 299 | |
| 300 | void FuncOp::build(OpBuilder &builder, OperationState &state, StringRef name, |
| 301 | FunctionType type, ArrayRef<NamedAttribute> attrs, |
| 302 | ArrayRef<DictionaryAttr> argAttrs) { |
| 303 | state.getOrAddProperties<Properties>().sym_name = builder.getStringAttr(bytes: name); |
| 304 | state.addAttribute(name: getFunctionTypeAttrName(name: state.name), attr: TypeAttr::get(type)); |
| 305 | |
| 306 | state.attributes.append(inStart: attrs.begin(), inEnd: attrs.end()); |
| 307 | state.addRegion(); |
| 308 | |
| 309 | if (argAttrs.empty()) |
| 310 | return; |
| 311 | assert(type.getNumInputs() == argAttrs.size()); |
| 312 | call_interface_impl::addArgAndResultAttrs( |
| 313 | builder, result&: state, argAttrs, /*resultAttrs=*/{}, |
| 314 | argAttrsName: getArgAttrsAttrName(name: state.name), resAttrsName: getResAttrsAttrName(name: state.name)); |
| 315 | } |
| 316 | |
| 317 | ParseResult FuncOp::parse(OpAsmParser &parser, OperationState &result) { |
| 318 | auto buildFuncType = |
| 319 | [](Builder &builder, ArrayRef<Type> argTypes, ArrayRef<Type> results, |
| 320 | function_interface_impl::VariadicFlag, |
| 321 | std::string &) { return builder.getFunctionType(inputs: argTypes, results); }; |
| 322 | |
| 323 | return function_interface_impl::parseFunctionOp( |
| 324 | parser, result, /*allowVariadic=*/false, |
| 325 | typeAttrName: getFunctionTypeAttrName(name: result.name), funcTypeBuilder: buildFuncType, |
| 326 | argAttrsName: getArgAttrsAttrName(name: result.name), resAttrsName: getResAttrsAttrName(name: result.name)); |
| 327 | } |
| 328 | |
| 329 | void FuncOp::print(OpAsmPrinter &p) { |
| 330 | function_interface_impl::printFunctionOp( |
| 331 | p, op: *this, /*isVariadic=*/false, typeAttrName: getFunctionTypeAttrName(), |
| 332 | argAttrsName: getArgAttrsAttrName(), resAttrsName: getResAttrsAttrName()); |
| 333 | } |
| 334 | |
| 335 | /// Check that the result type of async.func is not void and must be |
| 336 | /// some async token or async values. |
| 337 | LogicalResult FuncOp::verify() { |
| 338 | auto resultTypes = getResultTypes(); |
| 339 | if (resultTypes.empty()) |
| 340 | return emitOpError() |
| 341 | << "result is expected to be at least of size 1, but got " |
| 342 | << resultTypes.size(); |
| 343 | |
| 344 | for (unsigned i = 0, e = resultTypes.size(); i != e; ++i) { |
| 345 | auto type = resultTypes[i]; |
| 346 | if (!llvm::isa<async::TokenType>(Val: type) && !llvm::isa<ValueType>(Val: type)) |
| 347 | return emitOpError() << "result type must be async value type or async " |
| 348 | "token type, but got " |
| 349 | << type; |
| 350 | // We only allow AsyncToken appear as the first return value |
| 351 | if (llvm::isa<async::TokenType>(Val: type) && i != 0) { |
| 352 | return emitOpError() |
| 353 | << " results' (optional) async token type is expected " |
| 354 | "to appear as the 1st return value, but got " |
| 355 | << i + 1; |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | return success(); |
| 360 | } |
| 361 | |
| 362 | //===----------------------------------------------------------------------===// |
| 363 | /// CallOp |
| 364 | //===----------------------------------------------------------------------===// |
| 365 | |
| 366 | LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 367 | // Check that the callee attribute was specified. |
| 368 | auto fnAttr = getCalleeAttr(); |
| 369 | if (!fnAttr) |
| 370 | return emitOpError(message: "requires a 'callee' symbol reference attribute" ); |
| 371 | FuncOp fn = symbolTable.lookupNearestSymbolFrom<FuncOp>(from: *this, symbol: fnAttr); |
| 372 | if (!fn) |
| 373 | return emitOpError() << "'" << fnAttr.getValue() |
| 374 | << "' does not reference a valid async function" ; |
| 375 | |
| 376 | // Verify that the operand and result types match the callee. |
| 377 | return call_interface_impl::verifyCallOpInterface(call: *this, callable: fn); |
| 378 | } |
| 379 | |
| 380 | FunctionType CallOp::getCalleeType() { |
| 381 | return FunctionType::get(context: getContext(), inputs: getOperandTypes(), results: getResultTypes()); |
| 382 | } |
| 383 | |
| 384 | //===----------------------------------------------------------------------===// |
| 385 | /// ReturnOp |
| 386 | //===----------------------------------------------------------------------===// |
| 387 | |
| 388 | LogicalResult ReturnOp::verify() { |
| 389 | auto funcOp = (*this)->getParentOfType<FuncOp>(); |
| 390 | ArrayRef<Type> resultTypes = funcOp.isStateful() |
| 391 | ? funcOp.getResultTypes().drop_front() |
| 392 | : funcOp.getResultTypes(); |
| 393 | // Get the underlying value types from async types returned from the |
| 394 | // parent `async.func` operation. |
| 395 | auto types = llvm::map_range(C&: resultTypes, F: [](const Type &result) { |
| 396 | return llvm::cast<ValueType>(Val: result).getValueType(); |
| 397 | }); |
| 398 | |
| 399 | if (getOperandTypes() != types) |
| 400 | return emitOpError(message: "operand types do not match the types returned from " |
| 401 | "the parent FuncOp" ); |
| 402 | |
| 403 | return success(); |
| 404 | } |
| 405 | |
| 406 | //===----------------------------------------------------------------------===// |
| 407 | // TableGen'd op method definitions |
| 408 | //===----------------------------------------------------------------------===// |
| 409 | |
| 410 | #define GET_OP_CLASSES |
| 411 | #include "mlir/Dialect/Async/IR/AsyncOps.cpp.inc" |
| 412 | |
| 413 | //===----------------------------------------------------------------------===// |
| 414 | // TableGen'd type method definitions |
| 415 | //===----------------------------------------------------------------------===// |
| 416 | |
| 417 | #define GET_TYPEDEF_CLASSES |
| 418 | #include "mlir/Dialect/Async/IR/AsyncOpsTypes.cpp.inc" |
| 419 | |
| 420 | void ValueType::print(AsmPrinter &printer) const { |
| 421 | printer << "<" ; |
| 422 | printer.printType(type: getValueType()); |
| 423 | printer << '>'; |
| 424 | } |
| 425 | |
| 426 | Type ValueType::parse(mlir::AsmParser &parser) { |
| 427 | Type ty; |
| 428 | if (parser.parseLess() || parser.parseType(result&: ty) || parser.parseGreater()) { |
| 429 | parser.emitError(loc: parser.getNameLoc(), message: "failed to parse async value type" ); |
| 430 | return Type(); |
| 431 | } |
| 432 | return ValueType::get(valueType: ty); |
| 433 | } |
| 434 | |