| 1 | //===- TestOpDefs.cpp - MLIR Test Dialect Operation Hooks -----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "TestDialect.h" |
| 10 | #include "TestOps.h" |
| 11 | #include "mlir/Dialect/Bufferization/IR/Bufferization.h" |
| 12 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 13 | #include "mlir/IR/Verifier.h" |
| 14 | #include "mlir/Interfaces/FunctionImplementation.h" |
| 15 | #include "mlir/Interfaces/MemorySlotInterfaces.h" |
| 16 | #include "llvm/ADT/SmallVectorExtras.h" |
| 17 | #include "llvm/ADT/TypeSwitch.h" |
| 18 | |
| 19 | using namespace mlir; |
| 20 | using namespace test; |
| 21 | |
| 22 | //===----------------------------------------------------------------------===// |
| 23 | // OverridenSymbolVisibilityOp |
| 24 | //===----------------------------------------------------------------------===// |
| 25 | |
| 26 | SymbolTable::Visibility OverriddenSymbolVisibilityOp::getVisibility() { |
| 27 | return SymbolTable::Visibility::Private; |
| 28 | } |
| 29 | |
| 30 | StringAttr OverriddenSymbolVisibilityOp::getNameAttr() { |
| 31 | return getSymNameAttr(); |
| 32 | } |
| 33 | |
| 34 | static StringLiteral getVisibilityString(SymbolTable::Visibility visibility) { |
| 35 | switch (visibility) { |
| 36 | case SymbolTable::Visibility::Private: |
| 37 | return "private" ; |
| 38 | case SymbolTable::Visibility::Nested: |
| 39 | return "nested" ; |
| 40 | case SymbolTable::Visibility::Public: |
| 41 | return "public" ; |
| 42 | } |
| 43 | llvm_unreachable("Unknown SymbolTable::Visibility" ); |
| 44 | } |
| 45 | |
| 46 | void OverriddenSymbolVisibilityOp::setVisibility( |
| 47 | SymbolTable::Visibility visibility) { |
| 48 | |
| 49 | emitOpError("cannot change visibility of symbol to " ) |
| 50 | << getVisibilityString(visibility); |
| 51 | } |
| 52 | |
| 53 | //===----------------------------------------------------------------------===// |
| 54 | // TestBranchOp |
| 55 | //===----------------------------------------------------------------------===// |
| 56 | |
| 57 | SuccessorOperands TestBranchOp::getSuccessorOperands(unsigned index) { |
| 58 | assert(index == 0 && "invalid successor index" ); |
| 59 | return SuccessorOperands(getTargetOperandsMutable()); |
| 60 | } |
| 61 | |
| 62 | LogicalResult TestBranchOp::fold(FoldAdaptor adaptor, |
| 63 | SmallVectorImpl<OpFoldResult> &results) { |
| 64 | if (!getFoldToLast()) |
| 65 | return failure(); |
| 66 | (*this)->setSuccessor(&(*this)->getParentRegion()->back(), 0); |
| 67 | removeFoldToLastAttr(); |
| 68 | return success(); |
| 69 | } |
| 70 | |
| 71 | //===----------------------------------------------------------------------===// |
| 72 | // TestProducingBranchOp |
| 73 | //===----------------------------------------------------------------------===// |
| 74 | |
| 75 | SuccessorOperands TestProducingBranchOp::getSuccessorOperands(unsigned index) { |
| 76 | assert(index <= 1 && "invalid successor index" ); |
| 77 | if (index == 1) |
| 78 | return SuccessorOperands(getFirstOperandsMutable()); |
| 79 | return SuccessorOperands(getSecondOperandsMutable()); |
| 80 | } |
| 81 | |
| 82 | //===----------------------------------------------------------------------===// |
| 83 | // TestInternalBranchOp |
| 84 | //===----------------------------------------------------------------------===// |
| 85 | |
| 86 | SuccessorOperands TestInternalBranchOp::getSuccessorOperands(unsigned index) { |
| 87 | assert(index <= 1 && "invalid successor index" ); |
| 88 | if (index == 0) |
| 89 | return SuccessorOperands(0, getSuccessOperandsMutable()); |
| 90 | return SuccessorOperands(1, getErrorOperandsMutable()); |
| 91 | } |
| 92 | |
| 93 | //===----------------------------------------------------------------------===// |
| 94 | // TestCallOp |
| 95 | //===----------------------------------------------------------------------===// |
| 96 | |
| 97 | LogicalResult TestCallOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 98 | // Check that the callee attribute was specified. |
| 99 | auto fnAttr = getCalleeAttr(); |
| 100 | if (!fnAttr) |
| 101 | return emitOpError("requires a 'callee' symbol reference attribute" ); |
| 102 | if (!symbolTable.lookupNearestSymbolFrom<FunctionOpInterface>(*this, fnAttr)) |
| 103 | return emitOpError() << "'" << fnAttr.getValue() |
| 104 | << "' does not reference a valid function" ; |
| 105 | return success(); |
| 106 | } |
| 107 | |
| 108 | //===----------------------------------------------------------------------===// |
| 109 | // FoldToCallOp |
| 110 | //===----------------------------------------------------------------------===// |
| 111 | |
| 112 | namespace { |
| 113 | struct FoldToCallOpPattern : public OpRewritePattern<FoldToCallOp> { |
| 114 | using OpRewritePattern<FoldToCallOp>::OpRewritePattern; |
| 115 | |
| 116 | LogicalResult matchAndRewrite(FoldToCallOp op, |
| 117 | PatternRewriter &rewriter) const override { |
| 118 | rewriter.replaceOpWithNewOp<func::CallOp>(op, TypeRange(), |
| 119 | op.getCalleeAttr(), ValueRange()); |
| 120 | return success(); |
| 121 | } |
| 122 | }; |
| 123 | } // namespace |
| 124 | |
| 125 | void FoldToCallOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 126 | MLIRContext *context) { |
| 127 | results.add<FoldToCallOpPattern>(context); |
| 128 | } |
| 129 | |
| 130 | //===----------------------------------------------------------------------===// |
| 131 | // IsolatedRegionOp - test parsing passthrough operands |
| 132 | //===----------------------------------------------------------------------===// |
| 133 | |
| 134 | ParseResult IsolatedRegionOp::parse(OpAsmParser &parser, |
| 135 | OperationState &result) { |
| 136 | // Parse the input operand. |
| 137 | OpAsmParser::Argument argInfo; |
| 138 | argInfo.type = parser.getBuilder().getIndexType(); |
| 139 | if (parser.parseOperand(argInfo.ssaName) || |
| 140 | parser.resolveOperand(argInfo.ssaName, argInfo.type, result.operands)) |
| 141 | return failure(); |
| 142 | |
| 143 | // Parse the body region, and reuse the operand info as the argument info. |
| 144 | Region *body = result.addRegion(); |
| 145 | return parser.parseRegion(*body, argInfo, /*enableNameShadowing=*/true); |
| 146 | } |
| 147 | |
| 148 | void IsolatedRegionOp::print(OpAsmPrinter &p) { |
| 149 | p << ' '; |
| 150 | p.printOperand(getOperand()); |
| 151 | p.shadowRegionArgs(getRegion(), getOperand()); |
| 152 | p << ' '; |
| 153 | p.printRegion(getRegion(), /*printEntryBlockArgs=*/false); |
| 154 | } |
| 155 | |
| 156 | //===----------------------------------------------------------------------===// |
| 157 | // SSACFGRegionOp |
| 158 | //===----------------------------------------------------------------------===// |
| 159 | |
| 160 | RegionKind SSACFGRegionOp::getRegionKind(unsigned index) { |
| 161 | return RegionKind::SSACFG; |
| 162 | } |
| 163 | |
| 164 | //===----------------------------------------------------------------------===// |
| 165 | // GraphRegionOp |
| 166 | //===----------------------------------------------------------------------===// |
| 167 | |
| 168 | RegionKind GraphRegionOp::getRegionKind(unsigned index) { |
| 169 | return RegionKind::Graph; |
| 170 | } |
| 171 | |
| 172 | //===----------------------------------------------------------------------===// |
| 173 | // IsolatedGraphRegionOp |
| 174 | //===----------------------------------------------------------------------===// |
| 175 | |
| 176 | RegionKind IsolatedGraphRegionOp::getRegionKind(unsigned index) { |
| 177 | return RegionKind::Graph; |
| 178 | } |
| 179 | |
| 180 | //===----------------------------------------------------------------------===// |
| 181 | // AffineScopeOp |
| 182 | //===----------------------------------------------------------------------===// |
| 183 | |
| 184 | ParseResult AffineScopeOp::parse(OpAsmParser &parser, OperationState &result) { |
| 185 | // Parse the body region, and reuse the operand info as the argument info. |
| 186 | Region *body = result.addRegion(); |
| 187 | return parser.parseRegion(*body, /*arguments=*/{}, /*argTypes=*/{}); |
| 188 | } |
| 189 | |
| 190 | void AffineScopeOp::print(OpAsmPrinter &p) { |
| 191 | p << " " ; |
| 192 | p.printRegion(getRegion(), /*printEntryBlockArgs=*/false); |
| 193 | } |
| 194 | |
| 195 | //===----------------------------------------------------------------------===// |
| 196 | // TestRemoveOpWithInnerOps |
| 197 | //===----------------------------------------------------------------------===// |
| 198 | |
| 199 | namespace { |
| 200 | struct TestRemoveOpWithInnerOps |
| 201 | : public OpRewritePattern<TestOpWithRegionPattern> { |
| 202 | using OpRewritePattern<TestOpWithRegionPattern>::OpRewritePattern; |
| 203 | |
| 204 | void initialize() { setDebugName("TestRemoveOpWithInnerOps" ); } |
| 205 | |
| 206 | LogicalResult matchAndRewrite(TestOpWithRegionPattern op, |
| 207 | PatternRewriter &rewriter) const override { |
| 208 | rewriter.eraseOp(op: op); |
| 209 | return success(); |
| 210 | } |
| 211 | }; |
| 212 | } // namespace |
| 213 | |
| 214 | //===----------------------------------------------------------------------===// |
| 215 | // TestOpWithRegionPattern |
| 216 | //===----------------------------------------------------------------------===// |
| 217 | |
| 218 | void TestOpWithRegionPattern::getCanonicalizationPatterns( |
| 219 | RewritePatternSet &results, MLIRContext *context) { |
| 220 | results.add<TestRemoveOpWithInnerOps>(context); |
| 221 | } |
| 222 | |
| 223 | //===----------------------------------------------------------------------===// |
| 224 | // TestOpWithRegionFold |
| 225 | //===----------------------------------------------------------------------===// |
| 226 | |
| 227 | OpFoldResult TestOpWithRegionFold::fold(FoldAdaptor adaptor) { |
| 228 | return getOperand(); |
| 229 | } |
| 230 | |
| 231 | //===----------------------------------------------------------------------===// |
| 232 | // TestOpConstant |
| 233 | //===----------------------------------------------------------------------===// |
| 234 | |
| 235 | OpFoldResult TestOpConstant::fold(FoldAdaptor adaptor) { return getValue(); } |
| 236 | |
| 237 | //===----------------------------------------------------------------------===// |
| 238 | // TestOpWithVariadicResultsAndFolder |
| 239 | //===----------------------------------------------------------------------===// |
| 240 | |
| 241 | LogicalResult TestOpWithVariadicResultsAndFolder::fold( |
| 242 | FoldAdaptor adaptor, SmallVectorImpl<OpFoldResult> &results) { |
| 243 | for (Value input : this->getOperands()) { |
| 244 | results.push_back(input); |
| 245 | } |
| 246 | return success(); |
| 247 | } |
| 248 | |
| 249 | //===----------------------------------------------------------------------===// |
| 250 | // TestOpInPlaceFold |
| 251 | //===----------------------------------------------------------------------===// |
| 252 | |
| 253 | OpFoldResult TestOpInPlaceFold::fold(FoldAdaptor adaptor) { |
| 254 | // Exercise the fact that an operation created with createOrFold should be |
| 255 | // allowed to access its parent block. |
| 256 | assert(getOperation()->getBlock() && |
| 257 | "expected that operation is not unlinked" ); |
| 258 | |
| 259 | if (adaptor.getOp() && !getProperties().attr) { |
| 260 | // The folder adds "attr" if not present. |
| 261 | getProperties().attr = dyn_cast_or_null<IntegerAttr>(adaptor.getOp()); |
| 262 | return getResult(); |
| 263 | } |
| 264 | return {}; |
| 265 | } |
| 266 | |
| 267 | //===----------------------------------------------------------------------===// |
| 268 | // OpWithInferTypeInterfaceOp |
| 269 | //===----------------------------------------------------------------------===// |
| 270 | |
| 271 | LogicalResult OpWithInferTypeInterfaceOp::inferReturnTypes( |
| 272 | MLIRContext *, std::optional<Location> location, ValueRange operands, |
| 273 | DictionaryAttr attributes, PropertyRef properties, RegionRange regions, |
| 274 | SmallVectorImpl<Type> &inferredReturnTypes) { |
| 275 | if (operands[0].getType() != operands[1].getType()) { |
| 276 | return emitOptionalError(location, "operand type mismatch " , |
| 277 | operands[0].getType(), " vs " , |
| 278 | operands[1].getType()); |
| 279 | } |
| 280 | inferredReturnTypes.assign({operands[0].getType()}); |
| 281 | return success(); |
| 282 | } |
| 283 | |
| 284 | //===----------------------------------------------------------------------===// |
| 285 | // OpWithShapedTypeInferTypeInterfaceOp |
| 286 | //===----------------------------------------------------------------------===// |
| 287 | |
| 288 | LogicalResult OpWithShapedTypeInferTypeInterfaceOp::inferReturnTypeComponents( |
| 289 | MLIRContext *context, std::optional<Location> location, |
| 290 | ValueShapeRange operands, DictionaryAttr attributes, PropertyRef properties, |
| 291 | RegionRange regions, |
| 292 | SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) { |
| 293 | // Create return type consisting of the last element of the first operand. |
| 294 | auto operandType = operands.front().getType(); |
| 295 | auto sval = dyn_cast<ShapedType>(operandType); |
| 296 | if (!sval) |
| 297 | return emitOptionalError(location, "only shaped type operands allowed" ); |
| 298 | int64_t dim = sval.hasRank() ? sval.getShape().front() : ShapedType::kDynamic; |
| 299 | auto type = IntegerType::get(context, 17); |
| 300 | |
| 301 | Attribute encoding; |
| 302 | if (auto rankedTy = dyn_cast<RankedTensorType>(sval)) |
| 303 | encoding = rankedTy.getEncoding(); |
| 304 | inferredReturnShapes.push_back(ShapedTypeComponents({dim}, type, encoding)); |
| 305 | return success(); |
| 306 | } |
| 307 | |
| 308 | LogicalResult OpWithShapedTypeInferTypeInterfaceOp::reifyReturnTypeShapes( |
| 309 | OpBuilder &builder, ValueRange operands, |
| 310 | llvm::SmallVectorImpl<Value> &shapes) { |
| 311 | shapes = SmallVector<Value, 1>{ |
| 312 | builder.createOrFold<tensor::DimOp>(getLoc(), operands.front(), 0)}; |
| 313 | return success(); |
| 314 | } |
| 315 | |
| 316 | //===----------------------------------------------------------------------===// |
| 317 | // OpWithResultShapeInterfaceOp |
| 318 | //===----------------------------------------------------------------------===// |
| 319 | |
| 320 | LogicalResult OpWithResultShapeInterfaceOp::reifyReturnTypeShapes( |
| 321 | OpBuilder &builder, ValueRange operands, |
| 322 | llvm::SmallVectorImpl<Value> &shapes) { |
| 323 | Location loc = getLoc(); |
| 324 | shapes.reserve(operands.size()); |
| 325 | for (Value operand : llvm::reverse(operands)) { |
| 326 | auto rank = cast<RankedTensorType>(operand.getType()).getRank(); |
| 327 | auto currShape = llvm::map_to_vector<4>( |
| 328 | llvm::seq<int64_t>(0, rank), [&](int64_t dim) -> Value { |
| 329 | return builder.createOrFold<tensor::DimOp>(loc, operand, dim); |
| 330 | }); |
| 331 | shapes.push_back(tensor::FromElementsOp::create( |
| 332 | builder, getLoc(), |
| 333 | RankedTensorType::get({rank}, builder.getIndexType()), currShape)); |
| 334 | } |
| 335 | return success(); |
| 336 | } |
| 337 | |
| 338 | //===----------------------------------------------------------------------===// |
| 339 | // ReifyShapedTypeUsingReifyResultShapesOp |
| 340 | //===----------------------------------------------------------------------===// |
| 341 | |
| 342 | LogicalResult ReifyShapedTypeUsingReifyResultShapesOp::reifyResultShapes( |
| 343 | OpBuilder &builder, ReifiedRankedShapedTypeDims &shapes) { |
| 344 | Location loc = getLoc(); |
| 345 | shapes.reserve(getNumOperands()); |
| 346 | for (Value operand : llvm::reverse(getOperands())) { |
| 347 | auto tensorType = cast<RankedTensorType>(operand.getType()); |
| 348 | auto currShape = llvm::map_to_vector<4>( |
| 349 | llvm::seq<int64_t>(0, tensorType.getRank()), |
| 350 | [&](int64_t dim) -> OpFoldResult { |
| 351 | return tensorType.isDynamicDim(dim) |
| 352 | ? static_cast<OpFoldResult>( |
| 353 | builder.createOrFold<tensor::DimOp>(loc, operand, |
| 354 | dim)) |
| 355 | : static_cast<OpFoldResult>( |
| 356 | builder.getIndexAttr(tensorType.getDimSize(dim))); |
| 357 | }); |
| 358 | shapes.emplace_back(std::move(currShape)); |
| 359 | } |
| 360 | return success(); |
| 361 | } |
| 362 | |
| 363 | //===----------------------------------------------------------------------===// |
| 364 | // ReifyShapedTypeUsingReifyShapeOfResultOp |
| 365 | //===----------------------------------------------------------------------===// |
| 366 | |
| 367 | LogicalResult ReifyShapedTypeUsingReifyShapeOfResultOp::reifyResultShapes( |
| 368 | OpBuilder &builder, ReifiedRankedShapedTypeDims &shapes) { |
| 369 | return failure(); |
| 370 | } |
| 371 | |
| 372 | FailureOr<SmallVector<OpFoldResult>> |
| 373 | ReifyShapedTypeUsingReifyShapeOfResultOp::reifyShapeOfResult(OpBuilder &builder, |
| 374 | int resultIndex) { |
| 375 | Location loc = getLoc(); |
| 376 | Value sourceOperand = getOperand(getNumOperands() - 1 - resultIndex); |
| 377 | SmallVector<OpFoldResult> shape = |
| 378 | tensor::getMixedSizes(builder, loc, sourceOperand); |
| 379 | return shape; |
| 380 | } |
| 381 | |
| 382 | //===----------------------------------------------------------------------===// |
| 383 | // ReifyShapedTypeUsingReifyDimOfResultOp |
| 384 | //===----------------------------------------------------------------------===// |
| 385 | |
| 386 | LogicalResult ReifyShapedTypeUsingReifyDimOfResultOp::reifyResultShapes( |
| 387 | OpBuilder &builder, ReifiedRankedShapedTypeDims &shapes) { |
| 388 | return failure(); |
| 389 | } |
| 390 | |
| 391 | FailureOr<SmallVector<OpFoldResult>> |
| 392 | ReifyShapedTypeUsingReifyDimOfResultOp::reifyShapeOfResult(OpBuilder &builder, |
| 393 | int resultIndex) { |
| 394 | return failure(); |
| 395 | } |
| 396 | |
| 397 | FailureOr<OpFoldResult> |
| 398 | ReifyShapedTypeUsingReifyDimOfResultOp::reifyDimOfResult(OpBuilder &builder, |
| 399 | int resultIndex, |
| 400 | int dim) { |
| 401 | Location loc = getLoc(); |
| 402 | Value sourceOperand = getOperand(getNumOperands() - 1 - resultIndex); |
| 403 | OpFoldResult shape = tensor::getMixedSize(builder, loc, sourceOperand, dim); |
| 404 | return shape; |
| 405 | } |
| 406 | |
| 407 | //===----------------------------------------------------------------------===// |
| 408 | // UnreifableResultShapesOp |
| 409 | //===----------------------------------------------------------------------===// |
| 410 | |
| 411 | LogicalResult UnreifiableResultShapesOp::reifyResultShapes( |
| 412 | OpBuilder &builder, ReifiedRankedShapedTypeDims &shapes) { |
| 413 | Location loc = getLoc(); |
| 414 | shapes.resize(1); |
| 415 | shapes[0] = {tensor::getMixedSize(builder, loc, getOperand(), 0), |
| 416 | OpFoldResult()}; |
| 417 | return success(); |
| 418 | } |
| 419 | |
| 420 | //===----------------------------------------------------------------------===// |
| 421 | // UnreifableResultShapeOp |
| 422 | //===----------------------------------------------------------------------===// |
| 423 | |
| 424 | LogicalResult UnreifiableResultShapeOp::reifyResultShapes( |
| 425 | OpBuilder &builder, ReifiedRankedShapedTypeDims &shapes) { |
| 426 | return failure(); |
| 427 | } |
| 428 | |
| 429 | FailureOr<SmallVector<OpFoldResult>> |
| 430 | UnreifiableResultShapeOp::reifyShapeOfResult(OpBuilder &builder, |
| 431 | int resultIndex) { |
| 432 | SmallVector<OpFoldResult> shape = { |
| 433 | tensor::getMixedSize(builder, getLoc(), getOperand(), 0), OpFoldResult()}; |
| 434 | return shape; |
| 435 | } |
| 436 | |
| 437 | //===----------------------------------------------------------------------===// |
| 438 | // UnreifableResultShapeOp |
| 439 | //===----------------------------------------------------------------------===// |
| 440 | |
| 441 | LogicalResult UnreifiableDimOfResultShapeOp::reifyResultShapes( |
| 442 | OpBuilder &builder, ReifiedRankedShapedTypeDims &shapes) { |
| 443 | return failure(); |
| 444 | } |
| 445 | |
| 446 | FailureOr<SmallVector<OpFoldResult>> |
| 447 | UnreifiableDimOfResultShapeOp::reifyShapeOfResult(OpBuilder &builder, |
| 448 | int resultIndex) { |
| 449 | return failure(); |
| 450 | } |
| 451 | |
| 452 | FailureOr<OpFoldResult> |
| 453 | UnreifiableDimOfResultShapeOp::reifyDimOfResult(OpBuilder &builder, |
| 454 | int resultIndex, int dim) { |
| 455 | if (dim == 0) |
| 456 | return tensor::getMixedSize(builder, getLoc(), getOperand(), 0); |
| 457 | return failure(); |
| 458 | } |
| 459 | |
| 460 | //===----------------------------------------------------------------------===// |
| 461 | // SideEffectOp |
| 462 | //===----------------------------------------------------------------------===// |
| 463 | |
| 464 | namespace { |
| 465 | /// A test resource for side effects. |
| 466 | struct TestResource : public SideEffects::Resource::Base<TestResource> { |
| 467 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestResource) |
| 468 | |
| 469 | StringRef getName() const final { return "<Test>" ; } |
| 470 | }; |
| 471 | } // namespace |
| 472 | |
| 473 | void SideEffectOp::getEffects( |
| 474 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 475 | // Check for an effects attribute on the op instance. |
| 476 | ArrayAttr effectsAttr = |
| 477 | (*this)->getDiscardableAttrOfType<ArrayAttr>("effects" ); |
| 478 | if (!effectsAttr) |
| 479 | return; |
| 480 | |
| 481 | for (Attribute element : effectsAttr) { |
| 482 | DictionaryAttr effectElement = cast<DictionaryAttr>(element); |
| 483 | |
| 484 | // Get the specific memory effect. |
| 485 | MemoryEffects::Effect *effect = |
| 486 | StringSwitch<MemoryEffects::Effect *>( |
| 487 | cast<StringAttr>(effectElement.get("effect" )).getValue()) |
| 488 | .Case("allocate" , MemoryEffects::Allocate::get()) |
| 489 | .Case("free" , MemoryEffects::Free::get()) |
| 490 | .Case("read" , MemoryEffects::Read::get()) |
| 491 | .Case("write" , MemoryEffects::Write::get()); |
| 492 | |
| 493 | // Check for a non-default resource to use. |
| 494 | SideEffects::Resource *resource = SideEffects::DefaultResource::get(); |
| 495 | if (effectElement.get("test_resource" )) |
| 496 | resource = TestResource::get(); |
| 497 | else if (effectElement.get("test_nonaddressable_resource" )) |
| 498 | resource = TestNonAddressableResource::get(); |
| 499 | else if (effectElement.get("test_nonaddressable_resource_a" )) |
| 500 | resource = TestNonAddressableResourceA::get(); |
| 501 | else if (effectElement.get("test_nonaddressable_resource_b" )) |
| 502 | resource = TestNonAddressableResourceB::get(); |
| 503 | |
| 504 | // Check for a result to affect. |
| 505 | if (effectElement.get("on_result" )) |
| 506 | effects.emplace_back(effect, getOperation()->getOpResults()[0], resource); |
| 507 | else if (Attribute ref = effectElement.get("on_reference" )) |
| 508 | effects.emplace_back(effect, cast<SymbolRefAttr>(ref), resource); |
| 509 | else |
| 510 | effects.emplace_back(effect, resource); |
| 511 | } |
| 512 | } |
| 513 | |
| 514 | void SideEffectOp::getEffects( |
| 515 | SmallVectorImpl<TestEffects::EffectInstance> &effects) { |
| 516 | testSideEffectOpGetEffect(getOperation(), effects); |
| 517 | } |
| 518 | |
| 519 | //===----------------------------------------------------------------------===// |
| 520 | // ConditionalSideEffectOp |
| 521 | //===----------------------------------------------------------------------===// |
| 522 | |
| 523 | void ConditionalSideEffectOp::getEffects( |
| 524 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 525 | if (!getHasEffects()) |
| 526 | return; |
| 527 | |
| 528 | SideEffects::Resource *resource = SideEffects::DefaultResource::get(); |
| 529 | effects.emplace_back(MemoryEffects::Read::get(), resource); |
| 530 | effects.emplace_back(MemoryEffects::Write::get(), resource); |
| 531 | effects.emplace_back(MemoryEffects::Free::get(), resource); |
| 532 | effects.emplace_back(MemoryEffects::Allocate::get(), resource); |
| 533 | } |
| 534 | |
| 535 | void SideEffectWithRegionOp::getEffects( |
| 536 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 537 | // Check for an effects attribute on the op instance. |
| 538 | ArrayAttr effectsAttr = |
| 539 | (*this)->getDiscardableAttrOfType<ArrayAttr>("effects" ); |
| 540 | if (!effectsAttr) |
| 541 | return; |
| 542 | |
| 543 | for (Attribute element : effectsAttr) { |
| 544 | DictionaryAttr effectElement = cast<DictionaryAttr>(element); |
| 545 | |
| 546 | // Get the specific memory effect. |
| 547 | MemoryEffects::Effect *effect = |
| 548 | StringSwitch<MemoryEffects::Effect *>( |
| 549 | cast<StringAttr>(effectElement.get("effect" )).getValue()) |
| 550 | .Case("allocate" , MemoryEffects::Allocate::get()) |
| 551 | .Case("free" , MemoryEffects::Free::get()) |
| 552 | .Case("read" , MemoryEffects::Read::get()) |
| 553 | .Case("write" , MemoryEffects::Write::get()); |
| 554 | |
| 555 | // Check for a non-default resource to use. |
| 556 | SideEffects::Resource *resource = SideEffects::DefaultResource::get(); |
| 557 | if (effectElement.get("test_resource" )) |
| 558 | resource = TestResource::get(); |
| 559 | else if (effectElement.get("test_nonaddressable_resource" )) |
| 560 | resource = TestNonAddressableResource::get(); |
| 561 | else if (effectElement.get("test_nonaddressable_resource_a" )) |
| 562 | resource = TestNonAddressableResourceA::get(); |
| 563 | else if (effectElement.get("test_nonaddressable_resource_b" )) |
| 564 | resource = TestNonAddressableResourceB::get(); |
| 565 | |
| 566 | // Check for a result to affect. |
| 567 | if (effectElement.get("on_result" )) |
| 568 | effects.emplace_back(effect, getOperation()->getOpResults()[0], resource); |
| 569 | else if (effectElement.get("on_operand" )) |
| 570 | effects.emplace_back(effect, &getOperation()->getOpOperands()[0], |
| 571 | resource); |
| 572 | else if (effectElement.get("on_argument" )) |
| 573 | effects.emplace_back(effect, getOperation()->getRegion(0).getArgument(0), |
| 574 | resource); |
| 575 | else if (Attribute ref = effectElement.get("on_reference" )) |
| 576 | effects.emplace_back(effect, cast<SymbolRefAttr>(ref), resource); |
| 577 | else |
| 578 | effects.emplace_back(effect, resource); |
| 579 | } |
| 580 | } |
| 581 | |
| 582 | void SideEffectWithRegionOp::getEffects( |
| 583 | SmallVectorImpl<TestEffects::EffectInstance> &effects) { |
| 584 | testSideEffectOpGetEffect(getOperation(), effects); |
| 585 | } |
| 586 | |
| 587 | //===----------------------------------------------------------------------===// |
| 588 | // StringAttrPrettyNameOp |
| 589 | //===----------------------------------------------------------------------===// |
| 590 | |
| 591 | // This op has fancy handling of its SSA result name. |
| 592 | ParseResult StringAttrPrettyNameOp::parse(OpAsmParser &parser, |
| 593 | OperationState &result) { |
| 594 | // Add the result types. |
| 595 | for (size_t i = 0, e = parser.getNumResults(); i != e; ++i) |
| 596 | result.addTypes(parser.getBuilder().getIntegerType(32)); |
| 597 | |
| 598 | if (parser.parseOptionalAttrDictWithKeyword(result.attributes)) |
| 599 | return failure(); |
| 600 | |
| 601 | // If the attribute dictionary contains no 'names' attribute, infer it from |
| 602 | // the SSA name (if specified). |
| 603 | bool hadNames = llvm::any_of(result.attributes, [](NamedAttribute attr) { |
| 604 | return attr.getName() == "names" ; |
| 605 | }); |
| 606 | |
| 607 | // If there was no name specified, check to see if there was a useful name |
| 608 | // specified in the asm file. |
| 609 | if (hadNames || parser.getNumResults() == 0) |
| 610 | return success(); |
| 611 | |
| 612 | SmallVector<StringRef, 4> names; |
| 613 | auto *context = result.getContext(); |
| 614 | |
| 615 | for (size_t i = 0, e = parser.getNumResults(); i != e; ++i) { |
| 616 | auto resultName = parser.getResultName(i); |
| 617 | StringRef nameStr; |
| 618 | if (!resultName.first.empty() && !isdigit(resultName.first[0])) |
| 619 | nameStr = resultName.first; |
| 620 | |
| 621 | names.push_back(nameStr); |
| 622 | } |
| 623 | |
| 624 | auto namesAttr = parser.getBuilder().getStrArrayAttr(names); |
| 625 | result.attributes.push_back({StringAttr::get(context, "names" ), namesAttr}); |
| 626 | return success(); |
| 627 | } |
| 628 | |
| 629 | void StringAttrPrettyNameOp::print(OpAsmPrinter &p) { |
| 630 | // Note that we only need to print the "name" attribute if the asmprinter |
| 631 | // result name disagrees with it. This can happen in strange cases, e.g. |
| 632 | // when there are conflicts. |
| 633 | bool namesDisagree = getNames().size() != getNumResults(); |
| 634 | |
| 635 | SmallString<32> resultNameStr; |
| 636 | for (size_t i = 0, e = getNumResults(); i != e && !namesDisagree; ++i) { |
| 637 | resultNameStr.clear(); |
| 638 | llvm::raw_svector_ostream tmpStream(resultNameStr); |
| 639 | p.printOperand(getResult(i), tmpStream); |
| 640 | |
| 641 | auto expectedName = dyn_cast<StringAttr>(getNames()[i]); |
| 642 | if (!expectedName || |
| 643 | tmpStream.str().drop_front() != expectedName.getValue()) { |
| 644 | namesDisagree = true; |
| 645 | } |
| 646 | } |
| 647 | |
| 648 | if (namesDisagree) { |
| 649 | SmallVector<NamedAttribute> attrs((*this)->getDiscardableAttrs()); |
| 650 | attrs.emplace_back(getNamesAttrName(), getNamesAttr()); |
| 651 | llvm::sort(attrs); |
| 652 | p.printOptionalAttrDictWithKeyword(attrs); |
| 653 | } else { |
| 654 | p.printOptionalAttrDictWithKeyword( |
| 655 | (*this)->getDiscardableAttrDictionary().getValue(), {"names" }); |
| 656 | } |
| 657 | } |
| 658 | |
| 659 | // We set the SSA name in the asm syntax to the contents of the name |
| 660 | // attribute. |
| 661 | void StringAttrPrettyNameOp::getAsmResultNames( |
| 662 | function_ref<void(Value, StringRef)> setNameFn) { |
| 663 | |
| 664 | auto value = getNames(); |
| 665 | for (size_t i = 0, e = value.size(); i != e; ++i) |
| 666 | if (auto str = dyn_cast<StringAttr>(value[i])) |
| 667 | if (!str.getValue().empty()) |
| 668 | setNameFn(getResult(i), str.getValue()); |
| 669 | } |
| 670 | |
| 671 | //===----------------------------------------------------------------------===// |
| 672 | // CustomResultsNameOp |
| 673 | //===----------------------------------------------------------------------===// |
| 674 | |
| 675 | void CustomResultsNameOp::getAsmResultNames( |
| 676 | function_ref<void(Value, StringRef)> setNameFn) { |
| 677 | ArrayAttr value = getNames(); |
| 678 | for (size_t i = 0, e = value.size(); i != e; ++i) |
| 679 | if (auto str = dyn_cast<StringAttr>(value[i])) |
| 680 | if (!str.empty()) |
| 681 | setNameFn(getResult(i), str.getValue()); |
| 682 | } |
| 683 | |
| 684 | //===----------------------------------------------------------------------===// |
| 685 | // ResultNameFromTypeOp |
| 686 | //===----------------------------------------------------------------------===// |
| 687 | |
| 688 | void ResultNameFromTypeOp::getAsmResultNames( |
| 689 | function_ref<void(Value, StringRef)> setNameFn) { |
| 690 | auto result = getResult(); |
| 691 | auto setResultNameFn = [&](::llvm::StringRef name) { |
| 692 | setNameFn(result, name); |
| 693 | }; |
| 694 | auto opAsmTypeInterface = |
| 695 | ::mlir::cast<::mlir::OpAsmTypeInterface>(result.getType()); |
| 696 | opAsmTypeInterface.getAsmName(setResultNameFn); |
| 697 | } |
| 698 | |
| 699 | //===----------------------------------------------------------------------===// |
| 700 | // BlockArgumentNameFromTypeOp |
| 701 | //===----------------------------------------------------------------------===// |
| 702 | |
| 703 | void BlockArgumentNameFromTypeOp::getAsmBlockArgumentNames( |
| 704 | ::mlir::Region ®ion, ::mlir::OpAsmSetValueNameFn setNameFn) { |
| 705 | for (auto &block : region) { |
| 706 | for (auto arg : block.getArguments()) { |
| 707 | if (auto opAsmTypeInterface = |
| 708 | ::mlir::dyn_cast<::mlir::OpAsmTypeInterface>(arg.getType())) { |
| 709 | auto setArgNameFn = [&](StringRef name) { setNameFn(arg, name); }; |
| 710 | opAsmTypeInterface.getAsmName(setArgNameFn); |
| 711 | } |
| 712 | } |
| 713 | } |
| 714 | } |
| 715 | |
| 716 | //===----------------------------------------------------------------------===// |
| 717 | // ResultTypeWithTraitOp |
| 718 | //===----------------------------------------------------------------------===// |
| 719 | |
| 720 | LogicalResult ResultTypeWithTraitOp::verify() { |
| 721 | if ((*this)->getResultTypes()[0].hasTrait<TypeTrait::TestTypeTrait>()) |
| 722 | return success(); |
| 723 | return emitError("result type should have trait 'TestTypeTrait'" ); |
| 724 | } |
| 725 | |
| 726 | //===----------------------------------------------------------------------===// |
| 727 | // AttrWithTraitOp |
| 728 | //===----------------------------------------------------------------------===// |
| 729 | |
| 730 | LogicalResult AttrWithTraitOp::verify() { |
| 731 | if (getAttr().hasTrait<AttributeTrait::TestAttrTrait>()) |
| 732 | return success(); |
| 733 | return emitError("'attr' attribute should have trait 'TestAttrTrait'" ); |
| 734 | } |
| 735 | |
| 736 | //===----------------------------------------------------------------------===// |
| 737 | // RegionIfOp |
| 738 | //===----------------------------------------------------------------------===// |
| 739 | |
| 740 | void RegionIfOp::print(OpAsmPrinter &p) { |
| 741 | p << " " ; |
| 742 | p.printOperands(getOperands()); |
| 743 | p << ": " << getOperandTypes(); |
| 744 | p.printArrowTypeList(getResultTypes()); |
| 745 | p << " then " ; |
| 746 | p.printRegion(getThenRegion(), |
| 747 | /*printEntryBlockArgs=*/true, |
| 748 | /*printBlockTerminators=*/true); |
| 749 | p << " else " ; |
| 750 | p.printRegion(getElseRegion(), |
| 751 | /*printEntryBlockArgs=*/true, |
| 752 | /*printBlockTerminators=*/true); |
| 753 | p << " join " ; |
| 754 | p.printRegion(getJoinRegion(), |
| 755 | /*printEntryBlockArgs=*/true, |
| 756 | /*printBlockTerminators=*/true); |
| 757 | } |
| 758 | |
| 759 | ParseResult RegionIfOp::parse(OpAsmParser &parser, OperationState &result) { |
| 760 | SmallVector<OpAsmParser::UnresolvedOperand, 2> operandInfos; |
| 761 | SmallVector<Type, 2> operandTypes; |
| 762 | |
| 763 | result.regions.reserve(3); |
| 764 | Region *thenRegion = result.addRegion(); |
| 765 | Region *elseRegion = result.addRegion(); |
| 766 | Region *joinRegion = result.addRegion(); |
| 767 | |
| 768 | // Parse operand, type and arrow type lists. |
| 769 | if (parser.parseOperandList(operandInfos) || |
| 770 | parser.parseColonTypeList(operandTypes) || |
| 771 | parser.parseArrowTypeList(result.types)) |
| 772 | return failure(); |
| 773 | |
| 774 | // Parse all attached regions. |
| 775 | if (parser.parseKeyword("then" ) || parser.parseRegion(*thenRegion, {}, {}) || |
| 776 | parser.parseKeyword("else" ) || parser.parseRegion(*elseRegion, {}, {}) || |
| 777 | parser.parseKeyword("join" ) || parser.parseRegion(*joinRegion, {}, {})) |
| 778 | return failure(); |
| 779 | |
| 780 | return parser.resolveOperands(operandInfos, operandTypes, |
| 781 | parser.getCurrentLocation(), result.operands); |
| 782 | } |
| 783 | |
| 784 | OperandRange RegionIfOp::getEntrySuccessorOperands(RegionSuccessor successor) { |
| 785 | assert(llvm::is_contained({&getThenRegion(), &getElseRegion()}, |
| 786 | successor.getSuccessor()) && |
| 787 | "invalid region index" ); |
| 788 | return getOperands(); |
| 789 | } |
| 790 | |
| 791 | void RegionIfOp::getSuccessorRegions( |
| 792 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 793 | // We always branch to the join region. |
| 794 | if (!point.isParent()) { |
| 795 | if (point.getTerminatorPredecessorOrNull()->getParentRegion() != |
| 796 | &getJoinRegion()) |
| 797 | regions.push_back(RegionSuccessor(&getJoinRegion())); |
| 798 | else |
| 799 | regions.push_back(RegionSuccessor(getOperation())); |
| 800 | return; |
| 801 | } |
| 802 | |
| 803 | // The then and else regions are the entry regions of this op. |
| 804 | regions.push_back(RegionSuccessor(&getThenRegion())); |
| 805 | regions.push_back(RegionSuccessor(&getElseRegion())); |
| 806 | } |
| 807 | |
| 808 | ValueRange RegionIfOp::getSuccessorInputs(RegionSuccessor successor) { |
| 809 | if (successor.isOperation()) |
| 810 | return getResults(); |
| 811 | if (successor == &getThenRegion()) |
| 812 | return getThenArgs(); |
| 813 | if (successor == &getElseRegion()) |
| 814 | return getElseArgs(); |
| 815 | if (successor == &getJoinRegion()) |
| 816 | return getJoinArgs(); |
| 817 | llvm_unreachable("invalid region successor" ); |
| 818 | } |
| 819 | |
| 820 | void RegionIfOp::getRegionInvocationBounds( |
| 821 | ArrayRef<Attribute> operands, |
| 822 | SmallVectorImpl<InvocationBounds> &invocationBounds) { |
| 823 | // Each region is invoked at most once. |
| 824 | invocationBounds.assign(/*NumElts=*/3, /*Elt=*/{0, 1}); |
| 825 | } |
| 826 | |
| 827 | //===----------------------------------------------------------------------===// |
| 828 | // AnyCondOp |
| 829 | //===----------------------------------------------------------------------===// |
| 830 | |
| 831 | void AnyCondOp::getSuccessorRegions(RegionBranchPoint point, |
| 832 | SmallVectorImpl<RegionSuccessor> ®ions) { |
| 833 | // The parent op branches into the only region, and the region branches back |
| 834 | // to the parent op. |
| 835 | if (point.isParent()) |
| 836 | regions.emplace_back(&getRegion()); |
| 837 | else |
| 838 | regions.push_back(RegionSuccessor(getOperation())); |
| 839 | } |
| 840 | |
| 841 | ValueRange AnyCondOp::getSuccessorInputs(RegionSuccessor successor) { |
| 842 | return successor.isOperation() ? ValueRange(getResults()) : ValueRange(); |
| 843 | } |
| 844 | |
| 845 | void AnyCondOp::getRegionInvocationBounds( |
| 846 | ArrayRef<Attribute> operands, |
| 847 | SmallVectorImpl<InvocationBounds> &invocationBounds) { |
| 848 | invocationBounds.emplace_back(1, 1); |
| 849 | } |
| 850 | |
| 851 | //===----------------------------------------------------------------------===// |
| 852 | // SingleBlockImplicitTerminatorOp |
| 853 | //===----------------------------------------------------------------------===// |
| 854 | |
| 855 | /// Testing the correctness of some traits. |
| 856 | static_assert( |
| 857 | llvm::is_detected<OpTrait::has_implicit_terminator_t, |
| 858 | SingleBlockImplicitTerminatorOp>::value, |
| 859 | "has_implicit_terminator_t does not match SingleBlockImplicitTerminatorOp" ); |
| 860 | static_assert(OpTrait::hasSingleBlockImplicitTerminator< |
| 861 | SingleBlockImplicitTerminatorOp>::value, |
| 862 | "hasSingleBlockImplicitTerminator does not match " |
| 863 | "SingleBlockImplicitTerminatorOp" ); |
| 864 | |
| 865 | //===----------------------------------------------------------------------===// |
| 866 | // SingleNoTerminatorCustomAsmOp |
| 867 | //===----------------------------------------------------------------------===// |
| 868 | |
| 869 | ParseResult SingleNoTerminatorCustomAsmOp::parse(OpAsmParser &parser, |
| 870 | OperationState &state) { |
| 871 | Region *body = state.addRegion(); |
| 872 | if (parser.parseRegion(*body, /*arguments=*/{}, /*argTypes=*/{})) |
| 873 | return failure(); |
| 874 | return success(); |
| 875 | } |
| 876 | |
| 877 | void SingleNoTerminatorCustomAsmOp::print(OpAsmPrinter &printer) { |
| 878 | printer.printRegion( |
| 879 | getRegion(), /*printEntryBlockArgs=*/false, |
| 880 | // This op has a single block without terminators. But explicitly mark |
| 881 | // as not printing block terminators for testing. |
| 882 | /*printBlockTerminators=*/false); |
| 883 | } |
| 884 | |
| 885 | //===----------------------------------------------------------------------===// |
| 886 | // TestVerifiersOp |
| 887 | //===----------------------------------------------------------------------===// |
| 888 | |
| 889 | LogicalResult TestVerifiersOp::verify() { |
| 890 | if (!getRegion().hasOneBlock()) |
| 891 | return emitOpError("`hasOneBlock` trait hasn't been verified" ); |
| 892 | |
| 893 | Operation *definingOp = getInput().getDefiningOp(); |
| 894 | if (definingOp && failed(mlir::verify(definingOp))) |
| 895 | return emitOpError("operand hasn't been verified" ); |
| 896 | |
| 897 | // Avoid using `emitRemark(msg)` since that will trigger an infinite verifier |
| 898 | // loop. |
| 899 | mlir::emitRemark(getLoc(), "success run of verifier" ); |
| 900 | |
| 901 | return success(); |
| 902 | } |
| 903 | |
| 904 | LogicalResult TestVerifiersOp::verifyRegions() { |
| 905 | if (!getRegion().hasOneBlock()) |
| 906 | return emitOpError("`hasOneBlock` trait hasn't been verified" ); |
| 907 | |
| 908 | for (Block &block : getRegion()) |
| 909 | for (Operation &op : block) |
| 910 | if (failed(mlir::verify(&op))) |
| 911 | return emitOpError("nested op hasn't been verified" ); |
| 912 | |
| 913 | // Avoid using `emitRemark(msg)` since that will trigger an infinite verifier |
| 914 | // loop. |
| 915 | mlir::emitRemark(getLoc(), "success run of region verifier" ); |
| 916 | |
| 917 | return success(); |
| 918 | } |
| 919 | |
| 920 | //===----------------------------------------------------------------------===// |
| 921 | // Test InferIntRangeInterface |
| 922 | //===----------------------------------------------------------------------===// |
| 923 | |
| 924 | //===----------------------------------------------------------------------===// |
| 925 | // TestWithBoundsOp |
| 926 | //===----------------------------------------------------------------------===// |
| 927 | |
| 928 | LogicalResult TestWithBoundsOp::verify() { |
| 929 | Type type = getElementTypeOrSelf(getResult().getType()); |
| 930 | unsigned expectedWidth = 0; |
| 931 | if (type.isIndex()) |
| 932 | expectedWidth = IndexType::kInternalStorageBitWidth; |
| 933 | else if (auto intTy = llvm::dyn_cast<IntegerType>(type)) |
| 934 | expectedWidth = intTy.getWidth(); |
| 935 | if (expectedWidth != 0 && getUmin().getBitWidth() != expectedWidth) |
| 936 | return emitOpError("bound attribute width (" ) |
| 937 | << getUmin().getBitWidth() << ") does not match result type width (" |
| 938 | << expectedWidth << ")" ; |
| 939 | return success(); |
| 940 | } |
| 941 | |
| 942 | void TestWithBoundsOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges, |
| 943 | SetIntRangeFn setResultRanges) { |
| 944 | setResultRanges(getResult(), {getUmin(), getUmax(), getSmin(), getSmax()}); |
| 945 | } |
| 946 | |
| 947 | //===----------------------------------------------------------------------===// |
| 948 | // TestWithoutBoundsOp |
| 949 | //===----------------------------------------------------------------------===// |
| 950 | |
| 951 | void TestWithoutBoundsOp::inferResultRangesFromOptional( |
| 952 | ArrayRef<IntegerValueRange> argRanges, SetIntLatticeFn setResultRanges) { |
| 953 | // Mimic ops with uninitialized range. |
| 954 | setResultRanges(getResult(), IntegerValueRange{}); |
| 955 | } |
| 956 | |
| 957 | //===----------------------------------------------------------------------===// |
| 958 | // TestWithBoundsRegionOp |
| 959 | //===----------------------------------------------------------------------===// |
| 960 | |
| 961 | ParseResult TestWithBoundsRegionOp::parse(OpAsmParser &parser, |
| 962 | OperationState &result) { |
| 963 | if (parser.parseOptionalAttrDict(result.attributes)) |
| 964 | return failure(); |
| 965 | |
| 966 | // Parse the input argument |
| 967 | OpAsmParser::Argument argInfo; |
| 968 | if (failed(parser.parseArgument(argInfo, true))) |
| 969 | return failure(); |
| 970 | |
| 971 | // Parse the body region, and reuse the operand info as the argument info. |
| 972 | Region *body = result.addRegion(); |
| 973 | return parser.parseRegion(*body, argInfo, /*enableNameShadowing=*/false); |
| 974 | } |
| 975 | |
| 976 | void TestWithBoundsRegionOp::print(OpAsmPrinter &p) { |
| 977 | SmallVector<NamedAttribute> attrs((*this)->getDiscardableAttrs()); |
| 978 | attrs.emplace_back(getUminAttrName(), getUminAttr()); |
| 979 | attrs.emplace_back(getUmaxAttrName(), getUmaxAttr()); |
| 980 | attrs.emplace_back(getSminAttrName(), getSminAttr()); |
| 981 | attrs.emplace_back(getSmaxAttrName(), getSmaxAttr()); |
| 982 | llvm::sort(attrs); |
| 983 | p.printOptionalAttrDict(attrs); |
| 984 | p << ' '; |
| 985 | p.printRegionArgument(getRegion().getArgument(0), /*argAttrs=*/{}, |
| 986 | /*omitType=*/false); |
| 987 | p << ' '; |
| 988 | p.printRegion(getRegion(), /*printEntryBlockArgs=*/false); |
| 989 | } |
| 990 | |
| 991 | void TestWithBoundsRegionOp::inferResultRanges( |
| 992 | ArrayRef<ConstantIntRanges> argRanges, SetIntRangeFn setResultRanges) { |
| 993 | Value arg = getRegion().getArgument(0); |
| 994 | setResultRanges(arg, {getUmin(), getUmax(), getSmin(), getSmax()}); |
| 995 | } |
| 996 | |
| 997 | void TestWithBoundsRegionBranchOp::getSuccessorRegions( |
| 998 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 999 | if (point.isParent()) |
| 1000 | regions.emplace_back(&getBody()); |
| 1001 | else |
| 1002 | regions.emplace_back(getOperation()); |
| 1003 | } |
| 1004 | |
| 1005 | ValueRange |
| 1006 | TestWithBoundsRegionBranchOp::getSuccessorInputs(RegionSuccessor successor) { |
| 1007 | if (successor.isOperation()) |
| 1008 | return getResults().take_front(1); |
| 1009 | return getBody().getArguments().take_front(1); |
| 1010 | } |
| 1011 | |
| 1012 | OperandRange |
| 1013 | TestWithBoundsRegionBranchOp::getEntrySuccessorOperands(RegionSuccessor) { |
| 1014 | return getOperands().take_front(1); |
| 1015 | } |
| 1016 | |
| 1017 | void TestWithBoundsRegionBranchOp::inferResultRanges( |
| 1018 | ArrayRef<ConstantIntRanges> argRanges, SetIntRangeFn setResultRanges) { |
| 1019 | setResultRanges(getInferredResult(), argRanges[1]); |
| 1020 | setResultRanges(getBody().getArgument(1), argRanges[1]); |
| 1021 | } |
| 1022 | |
| 1023 | //===----------------------------------------------------------------------===// |
| 1024 | // TestIncrementOp |
| 1025 | //===----------------------------------------------------------------------===// |
| 1026 | |
| 1027 | void TestIncrementOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges, |
| 1028 | SetIntRangeFn setResultRanges) { |
| 1029 | const ConstantIntRanges &range = argRanges[0]; |
| 1030 | APInt one(range.umin().getBitWidth(), 1); |
| 1031 | setResultRanges(getResult(), |
| 1032 | {range.umin().uadd_sat(one), range.umax().uadd_sat(one), |
| 1033 | range.smin().sadd_sat(one), range.smax().sadd_sat(one)}); |
| 1034 | } |
| 1035 | |
| 1036 | //===----------------------------------------------------------------------===// |
| 1037 | // TestReflectBoundsOp |
| 1038 | //===----------------------------------------------------------------------===// |
| 1039 | |
| 1040 | void TestReflectBoundsOp::inferResultRanges( |
| 1041 | ArrayRef<ConstantIntRanges> argRanges, SetIntRangeFn setResultRanges) { |
| 1042 | const ConstantIntRanges &range = argRanges[0]; |
| 1043 | MLIRContext *ctx = getContext(); |
| 1044 | Builder b(ctx); |
| 1045 | Type sIntTy, uIntTy; |
| 1046 | // For plain `IntegerType`s, we can derive the appropriate signed and unsigned |
| 1047 | // Types for the Attributes. |
| 1048 | Type type = getElementTypeOrSelf(getType()); |
| 1049 | if (auto intTy = llvm::dyn_cast<IntegerType>(type)) { |
| 1050 | unsigned bitwidth = intTy.getWidth(); |
| 1051 | sIntTy = b.getIntegerType(bitwidth, /*isSigned=*/true); |
| 1052 | uIntTy = b.getIntegerType(bitwidth, /*isSigned=*/false); |
| 1053 | } else { |
| 1054 | sIntTy = uIntTy = type; |
| 1055 | } |
| 1056 | |
| 1057 | setUminAttr(b.getIntegerAttr(uIntTy, range.umin())); |
| 1058 | setUmaxAttr(b.getIntegerAttr(uIntTy, range.umax())); |
| 1059 | setSminAttr(b.getIntegerAttr(sIntTy, range.smin())); |
| 1060 | setSmaxAttr(b.getIntegerAttr(sIntTy, range.smax())); |
| 1061 | setResultRanges(getResult(), range); |
| 1062 | } |
| 1063 | |
| 1064 | //===----------------------------------------------------------------------===// |
| 1065 | // ConversionFuncOp |
| 1066 | //===----------------------------------------------------------------------===// |
| 1067 | |
| 1068 | ParseResult ConversionFuncOp::parse(OpAsmParser &parser, |
| 1069 | OperationState &result) { |
| 1070 | auto buildFuncType = |
| 1071 | [](Builder &builder, ArrayRef<Type> argTypes, ArrayRef<Type> results, |
| 1072 | function_interface_impl::VariadicFlag, |
| 1073 | std::string &) { return builder.getFunctionType(argTypes, results); }; |
| 1074 | |
| 1075 | return function_interface_impl::parseFunctionOp( |
| 1076 | parser, result, /*allowVariadic=*/false, |
| 1077 | getFunctionTypeAttrName(result.name), buildFuncType, |
| 1078 | getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name)); |
| 1079 | } |
| 1080 | |
| 1081 | void ConversionFuncOp::print(OpAsmPrinter &p) { |
| 1082 | function_interface_impl::printFunctionOp( |
| 1083 | p, *this, /*isVariadic=*/false, getFunctionTypeAttrName(), |
| 1084 | getArgAttrsAttrName(), getResAttrsAttrName()); |
| 1085 | } |
| 1086 | |
| 1087 | //===----------------------------------------------------------------------===// |
| 1088 | // TestValueWithBoundsOp |
| 1089 | //===----------------------------------------------------------------------===// |
| 1090 | |
| 1091 | void TestValueWithBoundsOp::populateBoundsForIndexValue( |
| 1092 | Value v, ValueBoundsConstraintSet &cstr) { |
| 1093 | cstr.bound(v) >= getMin().getSExtValue(); |
| 1094 | cstr.bound(v) <= getMax().getSExtValue(); |
| 1095 | } |
| 1096 | |
| 1097 | //===----------------------------------------------------------------------===// |
| 1098 | // ReifyBoundOp |
| 1099 | //===----------------------------------------------------------------------===// |
| 1100 | |
| 1101 | mlir::presburger::BoundType ReifyBoundOp::getBoundType() { |
| 1102 | if (getType() == "EQ" ) |
| 1103 | return mlir::presburger::BoundType::EQ; |
| 1104 | if (getType() == "LB" ) |
| 1105 | return mlir::presburger::BoundType::LB; |
| 1106 | if (getType() == "UB" ) |
| 1107 | return mlir::presburger::BoundType::UB; |
| 1108 | llvm_unreachable("invalid bound type" ); |
| 1109 | } |
| 1110 | |
| 1111 | LogicalResult ReifyBoundOp::verify() { |
| 1112 | if (getType() != "EQ" && getType() != "LB" && getType() != "UB" ) |
| 1113 | return emitOpError("invalid bound type '" ) |
| 1114 | << getType() << "', expected 'EQ', 'LB', or 'UB'" ; |
| 1115 | if (isa<ShapedType>(getVar().getType())) { |
| 1116 | if (!getDim().has_value()) |
| 1117 | return emitOpError("expected 'dim' attribute for shaped type variable" ); |
| 1118 | } else if (getVar().getType().isIndex() || getVar().getType().isInteger()) { |
| 1119 | if (getVar().getType().isInteger() && !getAllowIntegerType()) |
| 1120 | return emitOpError("integer variable requires 'allow_integer_type'" ); |
| 1121 | if (getDim().has_value()) |
| 1122 | return emitOpError( |
| 1123 | "unexpected 'dim' attribute for index/integer variable" ); |
| 1124 | } else { |
| 1125 | return emitOpError( |
| 1126 | "expected index-typed/integer-typed variable or shape type variable" ); |
| 1127 | } |
| 1128 | if (getConstant() && getScalable()) |
| 1129 | return emitOpError("'scalable' and 'constant' are mutually exlusive" ); |
| 1130 | if (getScalable() != getVscaleMin().has_value()) |
| 1131 | return emitOpError("expected 'vscale_min' if and only if 'scalable'" ); |
| 1132 | if (getScalable() != getVscaleMax().has_value()) |
| 1133 | return emitOpError("expected 'vscale_min' if and only if 'scalable'" ); |
| 1134 | return success(); |
| 1135 | } |
| 1136 | |
| 1137 | ValueBoundsConstraintSet::Variable ReifyBoundOp::getVariable() { |
| 1138 | if (getDim().has_value()) |
| 1139 | return ValueBoundsConstraintSet::Variable(getVar(), *getDim()); |
| 1140 | return ValueBoundsConstraintSet::Variable(getVar()); |
| 1141 | } |
| 1142 | |
| 1143 | //===----------------------------------------------------------------------===// |
| 1144 | // CompareOp |
| 1145 | //===----------------------------------------------------------------------===// |
| 1146 | |
| 1147 | ValueBoundsConstraintSet::ComparisonOperator |
| 1148 | CompareOp::getComparisonOperator() { |
| 1149 | if (getCmp() == "EQ" ) |
| 1150 | return ValueBoundsConstraintSet::ComparisonOperator::EQ; |
| 1151 | if (getCmp() == "LT" ) |
| 1152 | return ValueBoundsConstraintSet::ComparisonOperator::LT; |
| 1153 | if (getCmp() == "LE" ) |
| 1154 | return ValueBoundsConstraintSet::ComparisonOperator::LE; |
| 1155 | if (getCmp() == "GT" ) |
| 1156 | return ValueBoundsConstraintSet::ComparisonOperator::GT; |
| 1157 | if (getCmp() == "GE" ) |
| 1158 | return ValueBoundsConstraintSet::ComparisonOperator::GE; |
| 1159 | llvm_unreachable("invalid comparison operator" ); |
| 1160 | } |
| 1161 | |
| 1162 | mlir::ValueBoundsConstraintSet::Variable CompareOp::getLhs() { |
| 1163 | if (!getLhsMap()) |
| 1164 | return ValueBoundsConstraintSet::Variable(getVarOperands()[0]); |
| 1165 | SmallVector<Value> mapOperands( |
| 1166 | getVarOperands().slice(0, getLhsMap()->getNumInputs())); |
| 1167 | return ValueBoundsConstraintSet::Variable(*getLhsMap(), mapOperands); |
| 1168 | } |
| 1169 | |
| 1170 | mlir::ValueBoundsConstraintSet::Variable CompareOp::getRhs() { |
| 1171 | int64_t rhsOperandsBegin = getLhsMap() ? getLhsMap()->getNumInputs() : 1; |
| 1172 | if (!getRhsMap()) |
| 1173 | return ValueBoundsConstraintSet::Variable( |
| 1174 | getVarOperands()[rhsOperandsBegin]); |
| 1175 | SmallVector<Value> mapOperands( |
| 1176 | getVarOperands().slice(rhsOperandsBegin, getRhsMap()->getNumInputs())); |
| 1177 | return ValueBoundsConstraintSet::Variable(*getRhsMap(), mapOperands); |
| 1178 | } |
| 1179 | |
| 1180 | LogicalResult CompareOp::verify() { |
| 1181 | if (getCompose() && (getLhsMap() || getRhsMap())) |
| 1182 | return emitOpError( |
| 1183 | "'compose' not supported when 'lhs_map' or 'rhs_map' is present" ); |
| 1184 | int64_t expectedNumOperands = getLhsMap() ? getLhsMap()->getNumInputs() : 1; |
| 1185 | expectedNumOperands += getRhsMap() ? getRhsMap()->getNumInputs() : 1; |
| 1186 | if (getVarOperands().size() != size_t(expectedNumOperands)) |
| 1187 | return emitOpError("expected " ) |
| 1188 | << expectedNumOperands << " operands, but got " |
| 1189 | << getVarOperands().size(); |
| 1190 | return success(); |
| 1191 | } |
| 1192 | |
| 1193 | //===----------------------------------------------------------------------===// |
| 1194 | // TestOpInPlaceSelfFold |
| 1195 | //===----------------------------------------------------------------------===// |
| 1196 | |
| 1197 | OpFoldResult TestOpInPlaceSelfFold::fold(FoldAdaptor adaptor) { |
| 1198 | if (!getFolded()) { |
| 1199 | // The folder adds the "folded" if not present. |
| 1200 | setFolded(true); |
| 1201 | return getResult(); |
| 1202 | } |
| 1203 | return {}; |
| 1204 | } |
| 1205 | |
| 1206 | //===----------------------------------------------------------------------===// |
| 1207 | // TestOpFoldWithFoldAdaptor |
| 1208 | //===----------------------------------------------------------------------===// |
| 1209 | |
| 1210 | OpFoldResult TestOpFoldWithFoldAdaptor::fold(FoldAdaptor adaptor) { |
| 1211 | int64_t sum = 0; |
| 1212 | if (auto value = dyn_cast_or_null<IntegerAttr>(adaptor.getOp())) |
| 1213 | sum += value.getValue().getSExtValue(); |
| 1214 | |
| 1215 | for (Attribute attr : adaptor.getVariadic()) |
| 1216 | if (auto value = dyn_cast_or_null<IntegerAttr>(attr)) |
| 1217 | sum += 2 * value.getValue().getSExtValue(); |
| 1218 | |
| 1219 | for (ArrayRef<Attribute> attrs : adaptor.getVarOfVar()) |
| 1220 | for (Attribute attr : attrs) |
| 1221 | if (auto value = dyn_cast_or_null<IntegerAttr>(attr)) |
| 1222 | sum += 3 * value.getValue().getSExtValue(); |
| 1223 | |
| 1224 | sum += 4 * std::distance(adaptor.getBody().begin(), adaptor.getBody().end()); |
| 1225 | |
| 1226 | return IntegerAttr::get(getType(), sum); |
| 1227 | } |
| 1228 | |
| 1229 | //===----------------------------------------------------------------------===// |
| 1230 | // OpWithInferTypeAdaptorInterfaceOp |
| 1231 | //===----------------------------------------------------------------------===// |
| 1232 | |
| 1233 | LogicalResult OpWithInferTypeAdaptorInterfaceOp::inferReturnTypes( |
| 1234 | MLIRContext *, std::optional<Location> location, |
| 1235 | OpWithInferTypeAdaptorInterfaceOp::Adaptor adaptor, |
| 1236 | SmallVectorImpl<Type> &inferredReturnTypes) { |
| 1237 | if (adaptor.getX().getType() != adaptor.getY().getType()) { |
| 1238 | return emitOptionalError(location, "operand type mismatch " , |
| 1239 | adaptor.getX().getType(), " vs " , |
| 1240 | adaptor.getY().getType()); |
| 1241 | } |
| 1242 | inferredReturnTypes.assign({adaptor.getX().getType()}); |
| 1243 | return success(); |
| 1244 | } |
| 1245 | |
| 1246 | //===----------------------------------------------------------------------===// |
| 1247 | // OpWithRefineTypeInterfaceOp |
| 1248 | //===----------------------------------------------------------------------===// |
| 1249 | |
| 1250 | // TODO: We should be able to only define either inferReturnType or |
| 1251 | // refineReturnType, currently only refineReturnType can be omitted. |
| 1252 | LogicalResult OpWithRefineTypeInterfaceOp::inferReturnTypes( |
| 1253 | MLIRContext *context, std::optional<Location> location, ValueRange operands, |
| 1254 | DictionaryAttr attributes, PropertyRef properties, RegionRange regions, |
| 1255 | SmallVectorImpl<Type> &returnTypes) { |
| 1256 | returnTypes.clear(); |
| 1257 | return OpWithRefineTypeInterfaceOp::refineReturnTypes( |
| 1258 | context, location, operands, attributes, properties, regions, |
| 1259 | returnTypes); |
| 1260 | } |
| 1261 | |
| 1262 | LogicalResult OpWithRefineTypeInterfaceOp::refineReturnTypes( |
| 1263 | MLIRContext *, std::optional<Location> location, ValueRange operands, |
| 1264 | DictionaryAttr attributes, PropertyRef properties, RegionRange regions, |
| 1265 | SmallVectorImpl<Type> &returnTypes) { |
| 1266 | if (operands[0].getType() != operands[1].getType()) { |
| 1267 | return emitOptionalError(location, "operand type mismatch " , |
| 1268 | operands[0].getType(), " vs " , |
| 1269 | operands[1].getType()); |
| 1270 | } |
| 1271 | // TODO: Add helper to make this more concise to write. |
| 1272 | if (returnTypes.empty()) |
| 1273 | returnTypes.resize(1, nullptr); |
| 1274 | if (returnTypes[0] && returnTypes[0] != operands[0].getType()) |
| 1275 | return emitOptionalError(location, |
| 1276 | "required first operand and result to match" ); |
| 1277 | returnTypes[0] = operands[0].getType(); |
| 1278 | return success(); |
| 1279 | } |
| 1280 | |
| 1281 | //===----------------------------------------------------------------------===// |
| 1282 | // TilingNoDpsOp |
| 1283 | //===----------------------------------------------------------------------===// |
| 1284 | |
| 1285 | SmallVector<Range> TilingNoDpsOp::getIterationDomain(OpBuilder &builder) { |
| 1286 | return {}; |
| 1287 | } |
| 1288 | |
| 1289 | SmallVector<utils::IteratorType> TilingNoDpsOp::getLoopIteratorTypes() { |
| 1290 | return {}; |
| 1291 | } |
| 1292 | |
| 1293 | FailureOr<TilingResult> TilingNoDpsOp::getTiledImplementation( |
| 1294 | OpBuilder &builder, ArrayRef<OpFoldResult> offsets, |
| 1295 | ArrayRef<OpFoldResult> sizes, ArrayRef<InnerTileAlignment>) { |
| 1296 | return getTiledImplementation(builder, offsets, sizes); |
| 1297 | } |
| 1298 | |
| 1299 | FailureOr<TilingResult> |
| 1300 | TilingNoDpsOp::getTiledImplementation(OpBuilder &builder, |
| 1301 | ArrayRef<OpFoldResult> offsets, |
| 1302 | ArrayRef<OpFoldResult> sizes) { |
| 1303 | return failure(); |
| 1304 | } |
| 1305 | |
| 1306 | LogicalResult TilingNoDpsOp::getResultTilePosition( |
| 1307 | OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets, |
| 1308 | ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets, |
| 1309 | SmallVector<OpFoldResult> &resultSizes) { |
| 1310 | return failure(); |
| 1311 | } |
| 1312 | |
| 1313 | //===----------------------------------------------------------------------===// |
| 1314 | // OpWithShapedTypeInferTypeAdaptorInterfaceOp |
| 1315 | //===----------------------------------------------------------------------===// |
| 1316 | |
| 1317 | LogicalResult |
| 1318 | OpWithShapedTypeInferTypeAdaptorInterfaceOp::inferReturnTypeComponents( |
| 1319 | MLIRContext *context, std::optional<Location> location, |
| 1320 | OpWithShapedTypeInferTypeAdaptorInterfaceOp::Adaptor adaptor, |
| 1321 | SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) { |
| 1322 | // Create return type consisting of the last element of the first operand. |
| 1323 | auto operandType = adaptor.getOperand1().getType(); |
| 1324 | auto sval = dyn_cast<ShapedType>(operandType); |
| 1325 | if (!sval) |
| 1326 | return emitOptionalError(location, "only shaped type operands allowed" ); |
| 1327 | int64_t dim = sval.hasRank() ? sval.getShape().front() : ShapedType::kDynamic; |
| 1328 | auto type = IntegerType::get(context, 17); |
| 1329 | |
| 1330 | Attribute encoding; |
| 1331 | if (auto rankedTy = dyn_cast<RankedTensorType>(sval)) |
| 1332 | encoding = rankedTy.getEncoding(); |
| 1333 | inferredReturnShapes.push_back(ShapedTypeComponents({dim}, type, encoding)); |
| 1334 | return success(); |
| 1335 | } |
| 1336 | |
| 1337 | LogicalResult |
| 1338 | OpWithShapedTypeInferTypeAdaptorInterfaceOp::reifyReturnTypeShapes( |
| 1339 | OpBuilder &builder, ValueRange operands, |
| 1340 | llvm::SmallVectorImpl<Value> &shapes) { |
| 1341 | shapes = SmallVector<Value, 1>{ |
| 1342 | builder.createOrFold<tensor::DimOp>(getLoc(), operands.front(), 0)}; |
| 1343 | return success(); |
| 1344 | } |
| 1345 | |
| 1346 | //===----------------------------------------------------------------------===// |
| 1347 | // TestOpWithPropertiesAndInferredType |
| 1348 | //===----------------------------------------------------------------------===// |
| 1349 | |
| 1350 | LogicalResult TestOpWithPropertiesAndInferredType::inferReturnTypes( |
| 1351 | MLIRContext *context, std::optional<Location>, ValueRange operands, |
| 1352 | DictionaryAttr attributes, PropertyRef properties, RegionRange regions, |
| 1353 | SmallVectorImpl<Type> &inferredReturnTypes) { |
| 1354 | |
| 1355 | Adaptor adaptor(operands, attributes, properties, regions); |
| 1356 | inferredReturnTypes.push_back(IntegerType::get( |
| 1357 | context, adaptor.getLhs() + adaptor.getProperties().rhs)); |
| 1358 | return success(); |
| 1359 | } |
| 1360 | |
| 1361 | //===----------------------------------------------------------------------===// |
| 1362 | // SegmentedRegionBranchOp / SegmentedRegionBranchTerminatorOp |
| 1363 | //===----------------------------------------------------------------------===// |
| 1364 | |
| 1365 | void SegmentedRegionBranchOp::getCanonicalizationPatterns( |
| 1366 | RewritePatternSet &patterns, MLIRContext *context) { |
| 1367 | populateRegionBranchOpInterfaceCanonicalizationPatterns(patterns, |
| 1368 | getOperationName()); |
| 1369 | } |
| 1370 | |
| 1371 | void SegmentedRegionBranchOp::getSuccessorRegions( |
| 1372 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 1373 | regions.emplace_back(&getFirstRegion()); |
| 1374 | regions.emplace_back(&getSecondRegion()); |
| 1375 | if (!point.isParent()) |
| 1376 | regions.emplace_back(getOperation()); |
| 1377 | } |
| 1378 | |
| 1379 | ValueRange |
| 1380 | SegmentedRegionBranchOp::getSuccessorInputs(RegionSuccessor successor) { |
| 1381 | if (successor.isOperation()) |
| 1382 | return {}; |
| 1383 | return successor.getSuccessor()->front().getArguments(); |
| 1384 | } |
| 1385 | |
| 1386 | OperandRange |
| 1387 | SegmentedRegionBranchOp::getEntrySuccessorOperands(RegionSuccessor successor) { |
| 1388 | return successor.getSuccessor() == &getFirstRegion() ? getFirst() |
| 1389 | : getSecond(); |
| 1390 | } |
| 1391 | |
| 1392 | MutableOperandRange |
| 1393 | SegmentedRegionBranchTerminatorOp::getMutableSuccessorOperands( |
| 1394 | RegionSuccessor successor) { |
| 1395 | if (successor.isOperation()) |
| 1396 | return MutableOperandRange(getOperation(), getNumOperands(), 0); |
| 1397 | auto parent = cast<SegmentedRegionBranchOp>((*this)->getParentOp()); |
| 1398 | return successor.getSuccessor() == &parent.getFirstRegion() |
| 1399 | ? getFirstMutable() |
| 1400 | : getSecondMutable(); |
| 1401 | } |
| 1402 | |
| 1403 | //===----------------------------------------------------------------------===// |
| 1404 | // IsolatedRegionBranchOp |
| 1405 | //===----------------------------------------------------------------------===// |
| 1406 | |
| 1407 | void IsolatedRegionBranchOp::getSuccessorRegions( |
| 1408 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 1409 | if (!point.isParent()) { |
| 1410 | regions.emplace_back(getOperation()); |
| 1411 | return; |
| 1412 | } |
| 1413 | for (Region ®ion : getBranches()) |
| 1414 | regions.emplace_back(®ion); |
| 1415 | } |
| 1416 | |
| 1417 | ValueRange |
| 1418 | IsolatedRegionBranchOp::getSuccessorInputs(RegionSuccessor successor) { |
| 1419 | return successor.isOperation() |
| 1420 | ? ValueRange(getOutputs()) |
| 1421 | : ValueRange(successor.getSuccessor()->getArguments()); |
| 1422 | } |
| 1423 | |
| 1424 | OperandRange |
| 1425 | IsolatedRegionBranchOp::getEntrySuccessorOperands(RegionSuccessor successor) { |
| 1426 | return getInputs(); |
| 1427 | } |
| 1428 | |
| 1429 | void IsolatedRegionBranchOp::getCanonicalizationPatterns( |
| 1430 | RewritePatternSet &patterns, MLIRContext *context) { |
| 1431 | populateRegionBranchOpInterfaceCanonicalizationPatterns(patterns, |
| 1432 | getOperationName()); |
| 1433 | } |
| 1434 | |
| 1435 | MutableOperandRange |
| 1436 | IsolatedRegionYieldOp::getMutableSuccessorOperands(RegionSuccessor successor) { |
| 1437 | return getValuesMutable(); |
| 1438 | } |
| 1439 | |
| 1440 | //===----------------------------------------------------------------------===// |
| 1441 | // TypeChangingRegionBranchOp |
| 1442 | //===----------------------------------------------------------------------===// |
| 1443 | |
| 1444 | void TypeChangingRegionBranchOp::getSuccessorRegions( |
| 1445 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 1446 | if (!point.isParent()) { |
| 1447 | regions.emplace_back(getOperation()); |
| 1448 | return; |
| 1449 | } |
| 1450 | for (Region ®ion : getBranches()) |
| 1451 | regions.emplace_back(®ion); |
| 1452 | } |
| 1453 | |
| 1454 | ValueRange |
| 1455 | TypeChangingRegionBranchOp::getSuccessorInputs(RegionSuccessor successor) { |
| 1456 | return successor.isOperation() |
| 1457 | ? ValueRange(getOutputs()) |
| 1458 | : ValueRange(successor.getSuccessor()->getArguments()); |
| 1459 | } |
| 1460 | |
| 1461 | OperandRange TypeChangingRegionBranchOp::getEntrySuccessorOperands( |
| 1462 | RegionSuccessor successor) { |
| 1463 | return getInputs(); |
| 1464 | } |
| 1465 | |
| 1466 | bool TypeChangingRegionBranchOp::areTypesCompatible(Type lhs, Type rhs) { |
| 1467 | return true; |
| 1468 | } |
| 1469 | |
| 1470 | void TypeChangingRegionBranchOp::getCanonicalizationPatterns( |
| 1471 | RewritePatternSet &patterns, MLIRContext *context) { |
| 1472 | populateRegionBranchOpInterfaceCanonicalizationPatterns(patterns, |
| 1473 | getOperationName()); |
| 1474 | } |
| 1475 | |
| 1476 | MutableOperandRange |
| 1477 | TypeChangingYieldOp::getMutableSuccessorOperands(RegionSuccessor successor) { |
| 1478 | return getValuesMutable(); |
| 1479 | } |
| 1480 | |
| 1481 | //===----------------------------------------------------------------------===// |
| 1482 | // LoopBlockOp |
| 1483 | //===----------------------------------------------------------------------===// |
| 1484 | |
| 1485 | void LoopBlockOp::getSuccessorRegions( |
| 1486 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 1487 | regions.emplace_back(&getBody()); |
| 1488 | if (point.isParent()) |
| 1489 | return; |
| 1490 | |
| 1491 | regions.push_back(RegionSuccessor(getOperation())); |
| 1492 | } |
| 1493 | |
| 1494 | ValueRange LoopBlockOp::getSuccessorInputs(RegionSuccessor successor) { |
| 1495 | return successor.isOperation() ? ValueRange(getOperation()->getResults()) |
| 1496 | : ValueRange(getBody().getArguments()); |
| 1497 | } |
| 1498 | |
| 1499 | OperandRange LoopBlockOp::getEntrySuccessorOperands(RegionSuccessor successor) { |
| 1500 | assert(successor.getSuccessor() == &getBody()); |
| 1501 | return MutableOperandRange(getInitMutable()); |
| 1502 | } |
| 1503 | |
| 1504 | //===----------------------------------------------------------------------===// |
| 1505 | // LoopBlockTerminatorOp |
| 1506 | //===----------------------------------------------------------------------===// |
| 1507 | |
| 1508 | MutableOperandRange |
| 1509 | LoopBlockTerminatorOp::getMutableSuccessorOperands(RegionSuccessor successor) { |
| 1510 | if (successor.isOperation()) |
| 1511 | return getExitArgMutable(); |
| 1512 | return getNextIterArgMutable(); |
| 1513 | } |
| 1514 | |
| 1515 | //===----------------------------------------------------------------------===// |
| 1516 | // LoopWithExtraResultOp / LoopWithExtraResultYieldOp |
| 1517 | //===----------------------------------------------------------------------===// |
| 1518 | |
| 1519 | void LoopWithExtraResultOp::getSuccessorRegions( |
| 1520 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 1521 | // Parent always enters the body; the body can loop back or exit to parent. |
| 1522 | regions.emplace_back(&getBody()); |
| 1523 | if (!point.isParent()) |
| 1524 | regions.push_back(RegionSuccessor(getOperation())); |
| 1525 | } |
| 1526 | |
| 1527 | ValueRange |
| 1528 | LoopWithExtraResultOp::getSuccessorInputs(RegionSuccessor successor) { |
| 1529 | // When branching to the parent, only iterResult (result #1) is a successor |
| 1530 | // input; extraResult (#0) is not. Similaly when branching to the body, only |
| 1531 | // body block arg #1 (iterArg) is a successor input; arg #0 is not. |
| 1532 | if (successor.isOperation()) |
| 1533 | return getResults().drop_front(1); |
| 1534 | return getBody().getArguments().drop_front(1); |
| 1535 | } |
| 1536 | |
| 1537 | OperandRange LoopWithExtraResultOp::getEntrySuccessorOperands(RegionSuccessor) { |
| 1538 | return MutableOperandRange(getInitMutable()); |
| 1539 | } |
| 1540 | |
| 1541 | MutableOperandRange |
| 1542 | LoopWithExtraResultYieldOp::getMutableSuccessorOperands(RegionSuccessor) { |
| 1543 | return getIterArgMutable(); |
| 1544 | } |
| 1545 | |
| 1546 | //===----------------------------------------------------------------------===// |
| 1547 | // TestCrashingReturnOp |
| 1548 | //===----------------------------------------------------------------------===// |
| 1549 | |
| 1550 | LogicalResult TestCrashingReturnOp::verify() { |
| 1551 | if (!getValid()) |
| 1552 | return emitOpError("requires 'valid' unit attribute" ); |
| 1553 | return success(); |
| 1554 | } |
| 1555 | |
| 1556 | MutableOperandRange |
| 1557 | TestCrashingReturnOp::getMutableSuccessorOperands(RegionSuccessor successor) { |
| 1558 | if (!getValid()) |
| 1559 | llvm::report_fatal_error("getMutableSuccessorOperands called on unverified " |
| 1560 | "test.crashing_return" ); |
| 1561 | return getArgsMutable(); |
| 1562 | } |
| 1563 | |
| 1564 | //===----------------------------------------------------------------------===// |
| 1565 | // TestReturnWithIgnoredValueOp |
| 1566 | //===----------------------------------------------------------------------===// |
| 1567 | |
| 1568 | MutableOperandRange TestReturnWithIgnoredValueOp::getMutableSuccessorOperands( |
| 1569 | RegionSuccessor /*successor*/) { |
| 1570 | return getValuesMutable(); |
| 1571 | } |
| 1572 | |
| 1573 | //===----------------------------------------------------------------------===// |
| 1574 | // SwitchWithNoBreakOp |
| 1575 | //===----------------------------------------------------------------------===// |
| 1576 | |
| 1577 | void TestNoTerminatorOp::getSuccessorRegions( |
| 1578 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) {} |
| 1579 | |
| 1580 | //===----------------------------------------------------------------------===// |
| 1581 | // Test InferIntRangeInterface |
| 1582 | //===----------------------------------------------------------------------===// |
| 1583 | |
| 1584 | OpFoldResult ManualCppOpWithFold::fold(ArrayRef<Attribute> attributes) { |
| 1585 | // Just a simple fold for testing purposes that reads an operands constant |
| 1586 | // value and returns it. |
| 1587 | if (!attributes.empty()) |
| 1588 | return attributes.front(); |
| 1589 | return nullptr; |
| 1590 | } |
| 1591 | |
| 1592 | //===----------------------------------------------------------------------===// |
| 1593 | // Tensor/Buffer Ops |
| 1594 | //===----------------------------------------------------------------------===// |
| 1595 | |
| 1596 | void ReadBufferOp::getEffects( |
| 1597 | SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>> |
| 1598 | &effects) { |
| 1599 | // The buffer operand is read. |
| 1600 | effects.emplace_back(MemoryEffects::Read::get(), &getBufferMutable(), |
| 1601 | SideEffects::DefaultResource::get()); |
| 1602 | // The buffer contents are dumped. |
| 1603 | effects.emplace_back(MemoryEffects::Write::get(), |
| 1604 | SideEffects::DefaultResource::get()); |
| 1605 | } |
| 1606 | |
| 1607 | //===----------------------------------------------------------------------===// |
| 1608 | // Test Dataflow |
| 1609 | //===----------------------------------------------------------------------===// |
| 1610 | |
| 1611 | //===----------------------------------------------------------------------===// |
| 1612 | // TestCallAndStoreOp |
| 1613 | //===----------------------------------------------------------------------===// |
| 1614 | |
| 1615 | CallInterfaceCallable TestCallAndStoreOp::getCallableForCallee() { |
| 1616 | return getCallee(); |
| 1617 | } |
| 1618 | |
| 1619 | void TestCallAndStoreOp::setCalleeFromCallable(CallInterfaceCallable callee) { |
| 1620 | setCalleeAttr(cast<SymbolRefAttr>(callee)); |
| 1621 | } |
| 1622 | |
| 1623 | Operation::operand_range TestCallAndStoreOp::getArgOperands() { |
| 1624 | return getCalleeOperands(); |
| 1625 | } |
| 1626 | |
| 1627 | MutableOperandRange TestCallAndStoreOp::getArgOperandsMutable() { |
| 1628 | return getCalleeOperandsMutable(); |
| 1629 | } |
| 1630 | |
| 1631 | //===----------------------------------------------------------------------===// |
| 1632 | // TestCallOnDeviceOp |
| 1633 | //===----------------------------------------------------------------------===// |
| 1634 | |
| 1635 | CallInterfaceCallable TestCallOnDeviceOp::getCallableForCallee() { |
| 1636 | return getCallee(); |
| 1637 | } |
| 1638 | |
| 1639 | void TestCallOnDeviceOp::setCalleeFromCallable(CallInterfaceCallable callee) { |
| 1640 | setCalleeAttr(cast<SymbolRefAttr>(callee)); |
| 1641 | } |
| 1642 | |
| 1643 | Operation::operand_range TestCallOnDeviceOp::getArgOperands() { |
| 1644 | return getForwardedOperands(); |
| 1645 | } |
| 1646 | |
| 1647 | MutableOperandRange TestCallOnDeviceOp::getArgOperandsMutable() { |
| 1648 | return getForwardedOperandsMutable(); |
| 1649 | } |
| 1650 | |
| 1651 | //===----------------------------------------------------------------------===// |
| 1652 | // TestCallAndProduceOp |
| 1653 | //===----------------------------------------------------------------------===// |
| 1654 | |
| 1655 | CallInterfaceCallable TestCallAndProduceOp::getCallableForCallee() { |
| 1656 | return getCallee(); |
| 1657 | } |
| 1658 | |
| 1659 | void TestCallAndProduceOp::setCalleeFromCallable(CallInterfaceCallable callee) { |
| 1660 | setCalleeAttr(cast<SymbolRefAttr>(callee)); |
| 1661 | } |
| 1662 | |
| 1663 | Operation::operand_range TestCallAndProduceOp::getArgOperands() { |
| 1664 | return getForwardedOperands(); |
| 1665 | } |
| 1666 | |
| 1667 | MutableOperandRange TestCallAndProduceOp::getArgOperandsMutable() { |
| 1668 | return getForwardedOperandsMutable(); |
| 1669 | } |
| 1670 | |
| 1671 | Operation::result_range TestCallAndProduceOp::getForwardedResults() { |
| 1672 | // The first result (`produced_status`) is produced by this operation and is |
| 1673 | // not forwarded from the callee. |
| 1674 | return getForwarded(); |
| 1675 | } |
| 1676 | |
| 1677 | LogicalResult |
| 1678 | TestCallAndProduceOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 1679 | return call_interface_impl::verifyCallOpInterface(*this, symbolTable); |
| 1680 | } |
| 1681 | |
| 1682 | //===----------------------------------------------------------------------===// |
| 1683 | // TestStoreWithARegion |
| 1684 | //===----------------------------------------------------------------------===// |
| 1685 | |
| 1686 | void TestStoreWithARegion::getSuccessorRegions( |
| 1687 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 1688 | if (point.isParent()) |
| 1689 | regions.emplace_back(&getBody()); |
| 1690 | else |
| 1691 | regions.push_back(RegionSuccessor(getOperation())); |
| 1692 | } |
| 1693 | |
| 1694 | ValueRange TestStoreWithARegion::getSuccessorInputs(RegionSuccessor successor) { |
| 1695 | return successor.isOperation() ? ValueRange(getOperation()->getResults()) |
| 1696 | : ValueRange(getBody().front().getArguments()); |
| 1697 | } |
| 1698 | |
| 1699 | //===----------------------------------------------------------------------===// |
| 1700 | // TestStoreWithALoopRegion |
| 1701 | //===----------------------------------------------------------------------===// |
| 1702 | |
| 1703 | void TestStoreWithALoopRegion::getSuccessorRegions( |
| 1704 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 1705 | // Both the operation itself and the region may be branching into the body or |
| 1706 | // back into the operation itself. It is possible for the operation not to |
| 1707 | // enter the body. |
| 1708 | regions.emplace_back(&getBody()); |
| 1709 | regions.push_back(RegionSuccessor(getOperation())); |
| 1710 | } |
| 1711 | |
| 1712 | ValueRange |
| 1713 | TestStoreWithALoopRegion::getSuccessorInputs(RegionSuccessor successor) { |
| 1714 | return successor.isOperation() ? ValueRange(getOperation()->getResults()) |
| 1715 | : ValueRange(getBody().front().getArguments()); |
| 1716 | } |
| 1717 | |
| 1718 | //===----------------------------------------------------------------------===// |
| 1719 | // TestRegionTypesCompatOp |
| 1720 | //===----------------------------------------------------------------------===// |
| 1721 | |
| 1722 | void TestRegionTypesCompatOp::getSuccessorRegions( |
| 1723 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 1724 | if (point.isParent()) |
| 1725 | regions.emplace_back(&getBody()); |
| 1726 | else |
| 1727 | regions.push_back(RegionSuccessor(getOperation())); |
| 1728 | } |
| 1729 | |
| 1730 | OperandRange |
| 1731 | TestRegionTypesCompatOp::getEntrySuccessorOperands(RegionSuccessor) { |
| 1732 | return getEntries(); |
| 1733 | } |
| 1734 | |
| 1735 | ValueRange |
| 1736 | TestRegionTypesCompatOp::getSuccessorInputs(RegionSuccessor successor) { |
| 1737 | if (successor.isOperation()) |
| 1738 | return getResults(); |
| 1739 | return getBody().getArguments(); |
| 1740 | } |
| 1741 | |
| 1742 | bool TestRegionTypesCompatOp::areTypesCompatible(Type lhs, Type rhs) { |
| 1743 | return lhs == rhs || (isa<IntegerType>(lhs) && isa<IntegerType>(rhs)); |
| 1744 | } |
| 1745 | |
| 1746 | //===----------------------------------------------------------------------===// |
| 1747 | // TestLoopTypesCompatOp |
| 1748 | //===----------------------------------------------------------------------===// |
| 1749 | |
| 1750 | void TestLoopTypesCompatOp::getSuccessorRegions( |
| 1751 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 1752 | regions.emplace_back(&getBody()); |
| 1753 | if (!point.isParent()) |
| 1754 | regions.push_back(RegionSuccessor(getOperation())); |
| 1755 | } |
| 1756 | |
| 1757 | OperandRange TestLoopTypesCompatOp::getEntrySuccessorOperands(RegionSuccessor) { |
| 1758 | return getInitArgs(); |
| 1759 | } |
| 1760 | |
| 1761 | ValueRange |
| 1762 | TestLoopTypesCompatOp::getSuccessorInputs(RegionSuccessor successor) { |
| 1763 | if (successor.isOperation()) |
| 1764 | return getResults(); |
| 1765 | return getBody().getArguments(); |
| 1766 | } |
| 1767 | |
| 1768 | MutableArrayRef<OpOperand> TestLoopTypesCompatOp::getInitsMutable() { |
| 1769 | return getInitArgsMutable(); |
| 1770 | } |
| 1771 | |
| 1772 | Block::BlockArgListType TestLoopTypesCompatOp::getRegionIterArgs() { |
| 1773 | return getBody().getArguments(); |
| 1774 | } |
| 1775 | |
| 1776 | std::optional<MutableArrayRef<OpOperand>> |
| 1777 | TestLoopTypesCompatOp::getYieldedValuesMutable() { |
| 1778 | return cast<TestTypesCompatYieldOp>(getBody().front().getTerminator()) |
| 1779 | .getArgsMutable(); |
| 1780 | } |
| 1781 | |
| 1782 | std::optional<ResultRange> TestLoopTypesCompatOp::getLoopResults() { |
| 1783 | return getResults(); |
| 1784 | } |
| 1785 | |
| 1786 | SmallVector<Region *> TestLoopTypesCompatOp::getLoopRegions() { |
| 1787 | return {&getBody()}; |
| 1788 | } |
| 1789 | |
| 1790 | bool TestLoopTypesCompatOp::areTypesCompatible(Type lhs, Type rhs) { |
| 1791 | return lhs == rhs || (isa<IntegerType>(lhs) && isa<IntegerType>(rhs)); |
| 1792 | } |
| 1793 | |
| 1794 | void TestRegionTypeChangerOp::getSuccessorRegions( |
| 1795 | RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 1796 | if (point.isParent()) |
| 1797 | regions.emplace_back(&getBody()); |
| 1798 | else |
| 1799 | regions.push_back(RegionSuccessor(getOperation())); |
| 1800 | } |
| 1801 | |
| 1802 | OperandRange |
| 1803 | TestRegionTypeChangerOp::getEntrySuccessorOperands(RegionSuccessor) { |
| 1804 | return getEntries(); |
| 1805 | } |
| 1806 | |
| 1807 | ValueRange |
| 1808 | TestRegionTypeChangerOp::getSuccessorInputs(RegionSuccessor successor) { |
| 1809 | if (successor.isOperation()) |
| 1810 | return getResults(); |
| 1811 | return getBody().getArguments(); |
| 1812 | } |
| 1813 | |
| 1814 | bool TestRegionTypeChangerOp::areTypesCompatible(Type lhs, Type rhs) { |
| 1815 | return true; |
| 1816 | } |
| 1817 | |
| 1818 | //===----------------------------------------------------------------------===// |
| 1819 | // TestVersionedOpA |
| 1820 | //===----------------------------------------------------------------------===// |
| 1821 | |
| 1822 | LogicalResult |
| 1823 | TestVersionedOpA::readProperties(mlir::DialectBytecodeReader &reader, |
| 1824 | mlir::OperationState &state) { |
| 1825 | auto &prop = state.getOrAddProperties<Properties>(); |
| 1826 | if (mlir::failed(reader.readAttribute(prop.dims))) |
| 1827 | return mlir::failure(); |
| 1828 | |
| 1829 | // Check if we have a version. If not, assume we are parsing the current |
| 1830 | // version. |
| 1831 | auto maybeVersion = reader.getDialectVersion<test::TestDialect>(); |
| 1832 | if (succeeded(maybeVersion)) { |
| 1833 | // If version is less than 2.0, there is no additional attribute to parse. |
| 1834 | // We can materialize missing properties post parsing before verification. |
| 1835 | const auto *version = |
| 1836 | reinterpret_cast<const TestDialectVersion *>(*maybeVersion); |
| 1837 | if ((version->major_ < 2)) { |
| 1838 | return success(); |
| 1839 | } |
| 1840 | } |
| 1841 | |
| 1842 | if (mlir::failed(reader.readAttribute(prop.modifier))) |
| 1843 | return mlir::failure(); |
| 1844 | return mlir::success(); |
| 1845 | } |
| 1846 | |
| 1847 | void TestVersionedOpA::writeProperties(mlir::DialectBytecodeWriter &writer) { |
| 1848 | auto &prop = getProperties(); |
| 1849 | writer.writeAttribute(prop.dims); |
| 1850 | |
| 1851 | auto maybeVersion = writer.getDialectVersion<test::TestDialect>(); |
| 1852 | if (succeeded(maybeVersion)) { |
| 1853 | // If version is less than 2.0, there is no additional attribute to write. |
| 1854 | const auto *version = |
| 1855 | reinterpret_cast<const TestDialectVersion *>(*maybeVersion); |
| 1856 | if ((version->major_ < 2)) { |
| 1857 | return; |
| 1858 | } |
| 1859 | } |
| 1860 | writer.writeAttribute(prop.modifier); |
| 1861 | } |
| 1862 | |
| 1863 | //===----------------------------------------------------------------------===// |
| 1864 | // TestOpWithVersionedProperties |
| 1865 | //===----------------------------------------------------------------------===// |
| 1866 | |
| 1867 | llvm::LogicalResult TestOpWithVersionedProperties::readFromMlirBytecode( |
| 1868 | mlir::DialectBytecodeReader &reader, test::VersionedProperties &prop) { |
| 1869 | uint64_t value1, value2 = 0; |
| 1870 | if (failed(reader.readVarInt(value1))) |
| 1871 | return failure(); |
| 1872 | |
| 1873 | // Check if we have a version. If not, assume we are parsing the current |
| 1874 | // version. |
| 1875 | auto maybeVersion = reader.getDialectVersion<test::TestDialect>(); |
| 1876 | bool needToParseAnotherInt = true; |
| 1877 | if (succeeded(maybeVersion)) { |
| 1878 | // If version is less than 2.0, there is no additional attribute to parse. |
| 1879 | // We can materialize missing properties post parsing before verification. |
| 1880 | const auto *version = |
| 1881 | reinterpret_cast<const TestDialectVersion *>(*maybeVersion); |
| 1882 | if ((version->major_ < 2)) |
| 1883 | needToParseAnotherInt = false; |
| 1884 | } |
| 1885 | if (needToParseAnotherInt && failed(reader.readVarInt(value2))) |
| 1886 | return failure(); |
| 1887 | |
| 1888 | prop.value1 = value1; |
| 1889 | prop.value2 = value2; |
| 1890 | return success(); |
| 1891 | } |
| 1892 | |
| 1893 | void TestOpWithVersionedProperties::writeToMlirBytecode( |
| 1894 | mlir::DialectBytecodeWriter &writer, |
| 1895 | const test::VersionedProperties &prop) { |
| 1896 | writer.writeVarInt(prop.value1); |
| 1897 | auto version = writer.getDialectVersion<test::TestDialect>(); |
| 1898 | if (succeeded(version) && |
| 1899 | static_cast<const TestDialectVersion *>(*version)->major_ < 2) |
| 1900 | return; |
| 1901 | writer.writeVarInt(prop.value2); |
| 1902 | } |
| 1903 | |
| 1904 | //===----------------------------------------------------------------------===// |
| 1905 | // TestMultiSlotAlloca |
| 1906 | //===----------------------------------------------------------------------===// |
| 1907 | |
| 1908 | llvm::SmallVector<MemorySlot> TestMultiSlotAlloca::getPromotableSlots() { |
| 1909 | SmallVector<MemorySlot> slots; |
| 1910 | for (Value result : getResults()) { |
| 1911 | slots.push_back(MemorySlot{ |
| 1912 | result, cast<MemRefType>(result.getType()).getElementType()}); |
| 1913 | } |
| 1914 | return slots; |
| 1915 | } |
| 1916 | |
| 1917 | Value TestMultiSlotAlloca::getDefaultValue(const MemorySlot &slot, |
| 1918 | OpBuilder &builder) { |
| 1919 | return TestOpConstant::create(builder, getLoc(), slot.valueType, |
| 1920 | builder.getI32IntegerAttr(42)); |
| 1921 | } |
| 1922 | |
| 1923 | void TestMultiSlotAlloca::handleBlockArgument(const MemorySlot &slot, |
| 1924 | BlockArgument argument, |
| 1925 | OpBuilder &builder) { |
| 1926 | // Not relevant for testing. |
| 1927 | } |
| 1928 | |
| 1929 | /// Creates a new TestMultiSlotAlloca operation, just without the `slot`. |
| 1930 | static std::optional<TestMultiSlotAlloca> |
| 1931 | createNewMultiAllocaWithoutSlot(const MemorySlot &slot, OpBuilder &builder, |
| 1932 | TestMultiSlotAlloca oldOp) { |
| 1933 | |
| 1934 | if (oldOp.getNumResults() == 1) { |
| 1935 | oldOp.erase(); |
| 1936 | return std::nullopt; |
| 1937 | } |
| 1938 | |
| 1939 | SmallVector<Type> newTypes; |
| 1940 | SmallVector<Value> remainingValues; |
| 1941 | |
| 1942 | for (Value oldResult : oldOp.getResults()) { |
| 1943 | if (oldResult == slot.ptr) |
| 1944 | continue; |
| 1945 | remainingValues.push_back(oldResult); |
| 1946 | newTypes.push_back(oldResult.getType()); |
| 1947 | } |
| 1948 | |
| 1949 | OpBuilder::InsertionGuard guard(builder); |
| 1950 | builder.setInsertionPoint(oldOp); |
| 1951 | auto replacement = |
| 1952 | TestMultiSlotAlloca::create(builder, oldOp->getLoc(), newTypes); |
| 1953 | for (auto [oldResult, newResult] : |
| 1954 | llvm::zip_equal(remainingValues, replacement.getResults())) |
| 1955 | oldResult.replaceAllUsesWith(newResult); |
| 1956 | |
| 1957 | oldOp.erase(); |
| 1958 | return replacement; |
| 1959 | } |
| 1960 | |
| 1961 | std::optional<PromotableAllocationOpInterface> |
| 1962 | TestMultiSlotAlloca::handlePromotionComplete(const MemorySlot &slot, |
| 1963 | Value defaultValue, |
| 1964 | OpBuilder &builder) { |
| 1965 | if (defaultValue && defaultValue.use_empty()) |
| 1966 | defaultValue.getDefiningOp()->erase(); |
| 1967 | return createNewMultiAllocaWithoutSlot(slot, builder, *this); |
| 1968 | } |
| 1969 | |
| 1970 | SmallVector<DestructurableMemorySlot> |
| 1971 | TestMultiSlotAlloca::getDestructurableSlots() { |
| 1972 | SmallVector<DestructurableMemorySlot> slots; |
| 1973 | for (Value result : getResults()) { |
| 1974 | auto memrefType = cast<MemRefType>(result.getType()); |
| 1975 | auto destructurable = dyn_cast<DestructurableTypeInterface>(memrefType); |
| 1976 | if (!destructurable) |
| 1977 | continue; |
| 1978 | |
| 1979 | std::optional<DenseMap<Attribute, Type>> destructuredType = |
| 1980 | destructurable.getSubelementIndexMap(); |
| 1981 | if (!destructuredType) |
| 1982 | continue; |
| 1983 | slots.emplace_back( |
| 1984 | DestructurableMemorySlot{{result, memrefType}, *destructuredType}); |
| 1985 | } |
| 1986 | return slots; |
| 1987 | } |
| 1988 | |
| 1989 | DenseMap<Attribute, MemorySlot> TestMultiSlotAlloca::destructure( |
| 1990 | const DestructurableMemorySlot &slot, |
| 1991 | const SmallPtrSetImpl<Attribute> &usedIndices, OpBuilder &builder, |
| 1992 | SmallVectorImpl<DestructurableAllocationOpInterface> &newAllocators) { |
| 1993 | OpBuilder::InsertionGuard guard(builder); |
| 1994 | builder.setInsertionPointAfter(*this); |
| 1995 | |
| 1996 | DenseMap<Attribute, MemorySlot> slotMap; |
| 1997 | |
| 1998 | for (Attribute usedIndex : usedIndices) { |
| 1999 | Type elemType = slot.subelementTypes.lookup(usedIndex); |
| 2000 | MemRefType elemPtr = MemRefType::get({}, elemType); |
| 2001 | auto subAlloca = TestMultiSlotAlloca::create(builder, getLoc(), elemPtr); |
| 2002 | newAllocators.push_back(subAlloca); |
| 2003 | slotMap.try_emplace<MemorySlot>(usedIndex, |
| 2004 | {subAlloca.getResult(0), elemType}); |
| 2005 | } |
| 2006 | |
| 2007 | return slotMap; |
| 2008 | } |
| 2009 | |
| 2010 | std::optional<DestructurableAllocationOpInterface> |
| 2011 | TestMultiSlotAlloca::handleDestructuringComplete( |
| 2012 | const DestructurableMemorySlot &slot, OpBuilder &builder) { |
| 2013 | return createNewMultiAllocaWithoutSlot(slot, builder, *this); |
| 2014 | } |
| 2015 | |
| 2016 | //===----------------------------------------------------------------------===// |
| 2017 | // TestTransparentAlias |
| 2018 | //===----------------------------------------------------------------------===// |
| 2019 | |
| 2020 | void TestTransparentAlias::getPromotableSlotAliases( |
| 2021 | OpOperand &aliasedSlotPointerOperand, const MemorySlot & /*parentSlot*/, |
| 2022 | SmallVectorImpl<MemorySlot> &newMemorySlots) { |
| 2023 | if (aliasedSlotPointerOperand.get() != getSource()) |
| 2024 | return; |
| 2025 | Type elemType = cast<MemRefType>(getResult().getType()).getElementType(); |
| 2026 | newMemorySlots.push_back(MemorySlot{getResult(), elemType}); |
| 2027 | } |
| 2028 | |
| 2029 | bool TestTransparentAlias::canUsesBeRemoved( |
| 2030 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 2031 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 2032 | const DataLayout &dataLayout) { |
| 2033 | for (OpOperand &use : getResult().getUses()) |
| 2034 | newBlockingUses.push_back(&use); |
| 2035 | return true; |
| 2036 | } |
| 2037 | |
| 2038 | DeletionKind TestTransparentAlias::removeBlockingUses( |
| 2039 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 2040 | return DeletionKind::Delete; |
| 2041 | } |
| 2042 | |
| 2043 | //===----------------------------------------------------------------------===// |
| 2044 | // TestTransparentCastAlias |
| 2045 | //===----------------------------------------------------------------------===// |
| 2046 | |
| 2047 | void TestTransparentCastAlias::getPromotableSlotAliases( |
| 2048 | OpOperand &aliasedSlotPointerOperand, const MemorySlot & /*parentSlot*/, |
| 2049 | SmallVectorImpl<MemorySlot> &newMemorySlots) { |
| 2050 | if (aliasedSlotPointerOperand.get() != getSource()) |
| 2051 | return; |
| 2052 | Type elemType = cast<MemRefType>(getResult().getType()).getElementType(); |
| 2053 | newMemorySlots.push_back(MemorySlot{getResult(), elemType}); |
| 2054 | } |
| 2055 | |
| 2056 | bool TestTransparentCastAlias::canUsesBeRemoved( |
| 2057 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 2058 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 2059 | const DataLayout &dataLayout) { |
| 2060 | for (OpOperand &use : getResult().getUses()) |
| 2061 | newBlockingUses.push_back(&use); |
| 2062 | return true; |
| 2063 | } |
| 2064 | |
| 2065 | DeletionKind TestTransparentCastAlias::removeBlockingUses( |
| 2066 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 2067 | return DeletionKind::Delete; |
| 2068 | } |
| 2069 | |
| 2070 | Value TestTransparentCastAlias::projectSlotValueToAliasValue( |
| 2071 | OpOperand & /*aliasedSlotPointerOperand*/, |
| 2072 | const MemorySlot & /*parentSlot*/, const MemorySlot &aliasSlot, |
| 2073 | Value slotValue, OpBuilder &builder) { |
| 2074 | if (slotValue.getType() == aliasSlot.valueType) |
| 2075 | return slotValue; |
| 2076 | return UnrealizedConversionCastOp::create(builder, getLoc(), |
| 2077 | aliasSlot.valueType, slotValue) |
| 2078 | .getResult(0); |
| 2079 | } |
| 2080 | |
| 2081 | Value TestTransparentCastAlias::projectAliasValueToSlotValue( |
| 2082 | OpOperand & /*aliasedSlotPointerOperand*/, const MemorySlot &parentSlot, |
| 2083 | const MemorySlot & /*aliasSlot*/, Value aliasValue, Value /*reachingDef*/, |
| 2084 | OpBuilder &builder) { |
| 2085 | if (aliasValue.getType() == parentSlot.valueType) |
| 2086 | return aliasValue; |
| 2087 | return UnrealizedConversionCastOp::create(builder, getLoc(), |
| 2088 | parentSlot.valueType, aliasValue) |
| 2089 | .getResult(0); |
| 2090 | } |
| 2091 | |
| 2092 | //===----------------------------------------------------------------------===// |
| 2093 | // TestTransparentDualAlias |
| 2094 | //===----------------------------------------------------------------------===// |
| 2095 | |
| 2096 | void TestTransparentDualAlias::getPromotableSlotAliases( |
| 2097 | OpOperand &aliasedSlotPointerOperand, const MemorySlot & /*parentSlot*/, |
| 2098 | SmallVectorImpl<MemorySlot> &newMemorySlots) { |
| 2099 | if (aliasedSlotPointerOperand.get() != getSource()) |
| 2100 | return; |
| 2101 | // Expose both results as aliases of the same parent at their own |
| 2102 | // signedness (same bit width as the parent's signless i32). |
| 2103 | newMemorySlots.push_back(MemorySlot{ |
| 2104 | getResultSigned(), |
| 2105 | cast<MemRefType>(getResultSigned().getType()).getElementType()}); |
| 2106 | newMemorySlots.push_back(MemorySlot{ |
| 2107 | getResultUnsigned(), |
| 2108 | cast<MemRefType>(getResultUnsigned().getType()).getElementType()}); |
| 2109 | } |
| 2110 | |
| 2111 | bool TestTransparentDualAlias::canUsesBeRemoved( |
| 2112 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 2113 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 2114 | const DataLayout &dataLayout) { |
| 2115 | for (Value result : getResults()) |
| 2116 | for (OpOperand &use : result.getUses()) |
| 2117 | newBlockingUses.push_back(&use); |
| 2118 | return true; |
| 2119 | } |
| 2120 | |
| 2121 | DeletionKind TestTransparentDualAlias::removeBlockingUses( |
| 2122 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 2123 | return DeletionKind::Delete; |
| 2124 | } |
| 2125 | |
| 2126 | Value TestTransparentDualAlias::projectSlotValueToAliasValue( |
| 2127 | OpOperand & /*aliasedSlotPointerOperand*/, |
| 2128 | const MemorySlot & /*parentSlot*/, const MemorySlot &aliasSlot, |
| 2129 | Value slotValue, OpBuilder &builder) { |
| 2130 | if (slotValue.getType() == aliasSlot.valueType) |
| 2131 | return slotValue; |
| 2132 | return UnrealizedConversionCastOp::create(builder, getLoc(), |
| 2133 | aliasSlot.valueType, slotValue) |
| 2134 | .getResult(0); |
| 2135 | } |
| 2136 | |
| 2137 | Value TestTransparentDualAlias::projectAliasValueToSlotValue( |
| 2138 | OpOperand & /*aliasedSlotPointerOperand*/, const MemorySlot &parentSlot, |
| 2139 | const MemorySlot & /*aliasSlot*/, Value aliasValue, Value /*reachingDef*/, |
| 2140 | OpBuilder &builder) { |
| 2141 | if (aliasValue.getType() == parentSlot.valueType) |
| 2142 | return aliasValue; |
| 2143 | return UnrealizedConversionCastOp::create(builder, getLoc(), |
| 2144 | parentSlot.valueType, aliasValue) |
| 2145 | .getResult(0); |
| 2146 | } |
| 2147 | |
| 2148 | //===----------------------------------------------------------------------===// |
| 2149 | // TestPartialAlias |
| 2150 | //===----------------------------------------------------------------------===// |
| 2151 | |
| 2152 | void TestPartialAlias::getPromotableSlotAliases( |
| 2153 | OpOperand &aliasedSlotPointerOperand, const MemorySlot & /*parentSlot*/, |
| 2154 | SmallVectorImpl<MemorySlot> &newMemorySlots) { |
| 2155 | if (aliasedSlotPointerOperand.get() != getSource()) |
| 2156 | return; |
| 2157 | newMemorySlots.push_back(MemorySlot{ |
| 2158 | getResult(), cast<MemRefType>(getResult().getType()).getElementType()}); |
| 2159 | } |
| 2160 | |
| 2161 | bool TestPartialAlias::canUsesBeRemoved( |
| 2162 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 2163 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 2164 | const DataLayout &dataLayout) { |
| 2165 | for (OpOperand &use : getResult().getUses()) |
| 2166 | newBlockingUses.push_back(&use); |
| 2167 | return true; |
| 2168 | } |
| 2169 | |
| 2170 | DeletionKind TestPartialAlias::removeBlockingUses( |
| 2171 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 2172 | return DeletionKind::Delete; |
| 2173 | } |
| 2174 | |
| 2175 | Value TestPartialAlias::projectSlotValueToAliasValue( |
| 2176 | OpOperand & /*aliasedSlotPointerOperand*/, |
| 2177 | const MemorySlot & /*parentSlot*/, const MemorySlot &aliasSlot, |
| 2178 | Value slotValue, OpBuilder &builder) { |
| 2179 | // Sub-value extraction: 1-input cast. |
| 2180 | return UnrealizedConversionCastOp::create(builder, getLoc(), |
| 2181 | aliasSlot.valueType, slotValue) |
| 2182 | .getResult(0); |
| 2183 | } |
| 2184 | |
| 2185 | Value TestPartialAlias::projectAliasValueToSlotValue( |
| 2186 | OpOperand & /*aliasedSlotPointerOperand*/, const MemorySlot &parentSlot, |
| 2187 | const MemorySlot & /*aliasSlot*/, Value aliasValue, Value reachingDef, |
| 2188 | OpBuilder &builder) { |
| 2189 | // Sub-value insertion into the current reaching definition: emit a 2-input |
| 2190 | // cast taking both the new alias value and the existing parent value. |
| 2191 | return UnrealizedConversionCastOp::create(builder, getLoc(), |
| 2192 | parentSlot.valueType, |
| 2193 | ValueRange{aliasValue, reachingDef}) |
| 2194 | .getResult(0); |
| 2195 | } |
| 2196 | |
| 2197 | //===----------------------------------------------------------------------===// |
| 2198 | // TestOwningAccess |
| 2199 | //===----------------------------------------------------------------------===// |
| 2200 | |
| 2201 | /// Returns the memory operation wrapped by `op`, if it is the only operation |
| 2202 | /// before the terminator. |
| 2203 | static PromotableMemOpInterface getOwnedAccess(TestOwningAccess op) { |
| 2204 | Block &body = op.getBody().front(); |
| 2205 | if (body.getOperations().size() != 2) |
| 2206 | return {}; |
| 2207 | return dyn_cast<PromotableMemOpInterface>(Val&: body.front()); |
| 2208 | } |
| 2209 | |
| 2210 | bool TestOwningAccess::ownsWrappedBlockingUses() { |
| 2211 | return !getDisown() && getOwnedAccess(*this); |
| 2212 | } |
| 2213 | |
| 2214 | bool TestOwningAccess::loadsFrom(const MemorySlot &slot) { |
| 2215 | PromotableMemOpInterface access = getOwnedAccess(*this); |
| 2216 | return access && access.loadsFrom(slot); |
| 2217 | } |
| 2218 | |
| 2219 | bool TestOwningAccess::storesTo(const MemorySlot &slot) { |
| 2220 | PromotableMemOpInterface access = getOwnedAccess(*this); |
| 2221 | return access && access.storesTo(slot); |
| 2222 | } |
| 2223 | |
| 2224 | Value TestOwningAccess::getStored(const MemorySlot &slot, OpBuilder &builder, |
| 2225 | Value reachingDef, |
| 2226 | const DataLayout &dataLayout) { |
| 2227 | return getOwnedAccess(*this).getStored(slot, builder, reachingDef, |
| 2228 | dataLayout); |
| 2229 | } |
| 2230 | |
| 2231 | bool TestOwningAccess::canUsesBeRemoved( |
| 2232 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 2233 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 2234 | const DataLayout &dataLayout) { |
| 2235 | if (getDecline()) |
| 2236 | return false; |
| 2237 | PromotableMemOpInterface access = getOwnedAccess(*this); |
| 2238 | if (!access) |
| 2239 | return false; |
| 2240 | // The wrapped access can only remove its own uses. |
| 2241 | for (OpOperand *use : blockingUses) |
| 2242 | if (use->getOwner() != access) |
| 2243 | return false; |
| 2244 | return access.canUsesBeRemoved(slot, blockingUses, newBlockingUses, |
| 2245 | dataLayout); |
| 2246 | } |
| 2247 | |
| 2248 | DeletionKind TestOwningAccess::removeBlockingUses( |
| 2249 | const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 2250 | OpBuilder &builder, Value reachingDefinition, |
| 2251 | const DataLayout &dataLayout) { |
| 2252 | getOwnedAccess(*this).removeBlockingUses(slot, blockingUses, builder, |
| 2253 | reachingDefinition, dataLayout); |
| 2254 | // The wrapped load's result has been replaced with the reaching definition. |
| 2255 | // Replace the wrapper's results with the updated yield operands. |
| 2256 | getOperation()->replaceAllUsesWith( |
| 2257 | getBody().front().getTerminator()->getOperands()); |
| 2258 | return DeletionKind::Delete; |
| 2259 | } |
| 2260 | |
| 2261 | //===----------------------------------------------------------------------===// |
| 2262 | // TestSlotTracker |
| 2263 | //===----------------------------------------------------------------------===// |
| 2264 | |
| 2265 | bool TestSlotTracker::canUsesBeRemoved( |
| 2266 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 2267 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 2268 | const DataLayout &dataLayout) { |
| 2269 | if (blockingUses.size() != 1) |
| 2270 | return false; |
| 2271 | return (*blockingUses.begin())->get() == getSource(); |
| 2272 | } |
| 2273 | |
| 2274 | DeletionKind TestSlotTracker::removeBlockingUses( |
| 2275 | const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) { |
| 2276 | return DeletionKind::Delete; |
| 2277 | } |
| 2278 | |
| 2279 | bool TestSlotTracker::requiresReplacedValues() { return true; } |
| 2280 | |
| 2281 | void TestSlotTracker::visitReplacedValues( |
| 2282 | ArrayRef<std::pair<Operation *, Value>> definitions, OpBuilder &builder) { |
| 2283 | for (auto [op, value] : definitions) { |
| 2284 | builder.setInsertionPointAfter(op); |
| 2285 | TestTrackedValue::create(builder, getLoc(), value, getNameAttr()); |
| 2286 | } |
| 2287 | } |
| 2288 | |
| 2289 | ::mlir::LogicalResult test::TestDummyTensorOp::bufferize( |
| 2290 | ::mlir::RewriterBase &rewriter, |
| 2291 | const ::mlir::bufferization::BufferizationOptions &options, |
| 2292 | ::mlir::bufferization::BufferizationState &state) { |
| 2293 | auto buffer = |
| 2294 | mlir::bufferization::getBuffer(rewriter, getInput(), options, state); |
| 2295 | if (mlir::failed(buffer)) |
| 2296 | return failure(); |
| 2297 | |
| 2298 | const auto bufferizedOutType = |
| 2299 | mlir::bufferization::getBufferType(getOutput(), options, state); |
| 2300 | if (mlir::failed(bufferizedOutType)) |
| 2301 | return failure(); |
| 2302 | |
| 2303 | // replace op with memref analogy |
| 2304 | auto dummyMemrefOp = test::TestDummyMemrefOp::create( |
| 2305 | rewriter, getLoc(), *bufferizedOutType, *buffer); |
| 2306 | |
| 2307 | mlir::bufferization::replaceOpWithBufferizedValues(rewriter, getOperation(), |
| 2308 | dummyMemrefOp.getResult()); |
| 2309 | |
| 2310 | return mlir::success(); |
| 2311 | } |
| 2312 | |
| 2313 | mlir::FailureOr<mlir::bufferization::BufferLikeType> |
| 2314 | test::TestDummyTensorOp::getBufferType( |
| 2315 | mlir::Value value, const mlir::bufferization::BufferizationOptions &options, |
| 2316 | const mlir::bufferization::BufferizationState &, |
| 2317 | llvm::SmallVector<::mlir::Value> &) { |
| 2318 | // let unknown type converter handle the types - assume test operations work |
| 2319 | // with test-one-shot-module-bufferize pass. |
| 2320 | return options.unknownTypeConverterFn( |
| 2321 | cast<mlir::bufferization::TensorLikeType>(value.getType()), nullptr, |
| 2322 | options); |
| 2323 | } |
| 2324 | |
| 2325 | ::mlir::LogicalResult test::TestCreateTensorOp::bufferize( |
| 2326 | ::mlir::RewriterBase &rewriter, |
| 2327 | const ::mlir::bufferization::BufferizationOptions &options, |
| 2328 | ::mlir::bufferization::BufferizationState &state) { |
| 2329 | // Note: mlir::bufferization::getBufferType() would internally call |
| 2330 | // TestCreateTensorOp::getBufferType() |
| 2331 | const auto bufferizedOutType = |
| 2332 | mlir::bufferization::getBufferType(getOutput(), options, state); |
| 2333 | if (mlir::failed(bufferizedOutType)) |
| 2334 | return failure(); |
| 2335 | |
| 2336 | // replace op with memref analogy |
| 2337 | auto createMemrefOp = |
| 2338 | test::TestCreateMemrefOp::create(rewriter, getLoc(), *bufferizedOutType); |
| 2339 | |
| 2340 | mlir::bufferization::replaceOpWithBufferizedValues( |
| 2341 | rewriter, getOperation(), createMemrefOp.getResult()); |
| 2342 | |
| 2343 | return mlir::success(); |
| 2344 | } |
| 2345 | |
| 2346 | mlir::FailureOr<mlir::bufferization::BufferLikeType> |
| 2347 | test::TestCreateTensorOp::getBufferType( |
| 2348 | mlir::Value value, const mlir::bufferization::BufferizationOptions &options, |
| 2349 | const mlir::bufferization::BufferizationState &, |
| 2350 | llvm::SmallVector<::mlir::Value> &) { |
| 2351 | // let unknown type converter handle the types - assume test operations work |
| 2352 | // with test-one-shot-module-bufferize pass. |
| 2353 | return options.unknownTypeConverterFn( |
| 2354 | cast<mlir::bufferization::TensorLikeType>(value.getType()), nullptr, |
| 2355 | options); |
| 2356 | } |
| 2357 | |
| 2358 | ::mlir::LogicalResult test::TestTensorWithFutureLayoutOp::bufferize( |
| 2359 | ::mlir::RewriterBase &rewriter, |
| 2360 | const ::mlir::bufferization::BufferizationOptions &options, |
| 2361 | ::mlir::bufferization::BufferizationState &state) { |
| 2362 | const auto bufferizedOutType = |
| 2363 | mlir::bufferization::getBufferType(getOutput(), options, state); |
| 2364 | if (mlir::failed(bufferizedOutType)) |
| 2365 | return failure(); |
| 2366 | |
| 2367 | auto createMemrefOp = |
| 2368 | test::TestCreateMemrefOp::create(rewriter, getLoc(), *bufferizedOutType); |
| 2369 | mlir::bufferization::replaceOpWithBufferizedValues( |
| 2370 | rewriter, getOperation(), createMemrefOp.getResult()); |
| 2371 | return mlir::success(); |
| 2372 | } |
| 2373 | |
| 2374 | mlir::FailureOr<mlir::bufferization::BufferLikeType> |
| 2375 | test::TestTensorWithFutureLayoutOp::getBufferType( |
| 2376 | mlir::Value value, const mlir::bufferization::BufferizationOptions &, |
| 2377 | const mlir::bufferization::BufferizationState &, |
| 2378 | llvm::SmallVector<::mlir::Value> &) { |
| 2379 | return llvm::TypeSwitch<mlir::Type, |
| 2380 | mlir::FailureOr<mlir::bufferization::BufferLikeType>>( |
| 2381 | value.getType()) |
| 2382 | .Case([&](RankedTensorType tensorType) { |
| 2383 | // Set the memref layout to the op's 'layout' attribute, ignoring |
| 2384 | // any pre-existing tensor encoding. This is what lets two |
| 2385 | // `test.tensor_with_layout` ops produce *bufferized* memrefs with |
| 2386 | // different layouts while keeping their *tensor* result types identical |
| 2387 | // -- which is required to construct SCF iter_arg/branch mismatches that |
| 2388 | // the verifier still accepts. |
| 2389 | auto layout = cast<MemRefLayoutAttrInterface>(getLayout()); |
| 2390 | return cast<bufferization::BufferLikeType>(MemRefType::get( |
| 2391 | tensorType.getShape(), tensorType.getElementType(), layout)); |
| 2392 | }) |
| 2393 | .Case([&](TestTensorType tensorType) { |
| 2394 | auto layout = cast<MemRefLayoutAttrInterface>(getLayout()); |
| 2395 | return cast<bufferization::BufferLikeType>( |
| 2396 | TestMemrefType::get(tensorType.getContext(), tensorType.getShape(), |
| 2397 | tensorType.getElementType(), layout)); |
| 2398 | }) |
| 2399 | .Default([&](Type) { return emitError("unknown type" ); }); |
| 2400 | } |
| 2401 | |
| 2402 | LogicalResult test::TestForceNewLayoutOp::bufferize( |
| 2403 | RewriterBase &rewriter, const bufferization::BufferizationOptions &options, |
| 2404 | bufferization::BufferizationState &state) { |
| 2405 | auto buffer = bufferization::getBuffer(rewriter, getInput(), options, state); |
| 2406 | if (failed(buffer)) |
| 2407 | return failure(); |
| 2408 | |
| 2409 | const auto bufferizedOutType = |
| 2410 | bufferization::getBufferType(getOutput(), options, state); |
| 2411 | if (failed(bufferizedOutType)) |
| 2412 | return failure(); |
| 2413 | |
| 2414 | auto dummyMemrefOp = test::TestDummyMemrefOp::create( |
| 2415 | rewriter, getLoc(), *bufferizedOutType, *buffer); |
| 2416 | bufferization::replaceOpWithBufferizedValues(rewriter, getOperation(), |
| 2417 | dummyMemrefOp.getResult()); |
| 2418 | return success(); |
| 2419 | } |
| 2420 | |
| 2421 | FailureOr<bufferization::BufferLikeType> |
| 2422 | test::TestForceNewLayoutOp::getBufferType( |
| 2423 | mlir::Value value, const bufferization::BufferizationOptions &options, |
| 2424 | const bufferization::BufferizationState &, |
| 2425 | llvm::SmallVector<::mlir::Value> &) { |
| 2426 | return llvm::TypeSwitch<mlir::Type, |
| 2427 | mlir::FailureOr<mlir::bufferization::BufferLikeType>>( |
| 2428 | value.getType()) |
| 2429 | .Case([&](RankedTensorType tensorType) { |
| 2430 | // Set the memref layout to the op's 'layout' attribute, ignoring any |
| 2431 | // pre-existing tensor encoding. |
| 2432 | auto layout = cast<MemRefLayoutAttrInterface>(getLayout()); |
| 2433 | return cast<bufferization::BufferLikeType>(MemRefType::get( |
| 2434 | tensorType.getShape(), tensorType.getElementType(), layout)); |
| 2435 | }) |
| 2436 | .Case([&](TestTensorType tensorType) { |
| 2437 | auto layout = cast<MemRefLayoutAttrInterface>(getLayout()); |
| 2438 | return cast<bufferization::BufferLikeType>( |
| 2439 | TestMemrefType::get(tensorType.getContext(), tensorType.getShape(), |
| 2440 | tensorType.getElementType(), layout)); |
| 2441 | }) |
| 2442 | .Default([&](Type) { return emitError("unknown type" ); }); |
| 2443 | } |
| 2444 | |
| 2445 | // Define a custom builder for ManyRegionsOp declared in TestOps.td. |
| 2446 | // OpBuilder<(ins "::std::unique_ptr<::mlir::Region>":$firstRegion, |
| 2447 | // "::std::unique_ptr<::mlir::Region>":$secondRegion)> |
| 2448 | void test::ManyRegionsOp::build( |
| 2449 | mlir::OpBuilder &builder, mlir::OperationState &state, |
| 2450 | llvm::SmallVectorImpl<std::unique_ptr<mlir::Region>> &®ions) { |
| 2451 | for (auto &®ionPtr : std::move(regions)) |
| 2452 | state.addRegion(std::move(regionPtr)); |
| 2453 | ManyRegionsOp::build(builder, state, {}, regions.size()); |
| 2454 | } |
| 2455 | |