| 1 | //===- SwapExtractSliceWithProducerPatterns.cpp ---------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Swap a `tensor.extract_slice` with the producer of the source if the producer |
| 10 | // implements the `TilingInterface`. When used in conjunction with tiling this |
| 11 | // effectively tiles + fuses the producer with its consumer. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 16 | #include "mlir/Dialect/Tensor/Transforms/Transforms.h" |
| 17 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 18 | #include "mlir/Interfaces/TilingInterface.h" |
| 19 | #include "llvm/Support/Debug.h" |
| 20 | |
| 21 | #define DEBUG_TYPE "tensor-swap-slices" |
| 22 | |
| 23 | using namespace mlir; |
| 24 | |
| 25 | FailureOr<TilingResult> tensor::( |
| 26 | OpBuilder &builder, tensor::ExtractSliceOp sliceOp, OpResult producer, |
| 27 | ArrayRef<InnerTileAlignment> innerTileAlignments) { |
| 28 | auto producerOp = dyn_cast<TilingInterface>(Val: producer.getOwner()); |
| 29 | if (!producerOp) |
| 30 | return failure(); |
| 31 | |
| 32 | // `TilingInterface` currently only supports strides being 1. |
| 33 | if (!llvm::all_of(Range: sliceOp.getMixedStrides(), P: isOneInteger)) |
| 34 | return failure(); |
| 35 | |
| 36 | FailureOr<TilingResult> tiledResult = producerOp.generateResultTileValue( |
| 37 | b&: builder, resultNumber: producer.getResultNumber(), offsets: sliceOp.getMixedOffsets(), |
| 38 | sizes: sliceOp.getMixedSizes(), innerTileAlignments); |
| 39 | if (failed(Result: tiledResult)) |
| 40 | return failure(); |
| 41 | |
| 42 | // For cases where the slice was rank-reducing, create a rank-reducing slice |
| 43 | // to get the same type back. |
| 44 | llvm::SmallBitVector droppedDims = sliceOp.getDroppedDims(); |
| 45 | if (droppedDims.any()) { |
| 46 | assert(tiledResult->tiledValues.size() == 1 && |
| 47 | "expected only a single tiled result value to replace the extract " |
| 48 | "slice" ); |
| 49 | SmallVector<OpFoldResult> offsets(sliceOp.getSourceType().getRank(), |
| 50 | builder.getIndexAttr(value: 0)); |
| 51 | SmallVector<OpFoldResult> strides(sliceOp.getSourceType().getRank(), |
| 52 | builder.getIndexAttr(value: 1)); |
| 53 | auto newSliceOp = tensor::ExtractSliceOp::create( |
| 54 | builder, location: sliceOp.getLoc(), resultType: sliceOp.getType(), |
| 55 | source: tiledResult->tiledValues[0], offsets, sizes: sliceOp.getMixedSizes(), strides); |
| 56 | tiledResult->tiledValues[0] = newSliceOp; |
| 57 | tiledResult->generatedSlices.push_back(Elt: newSliceOp); |
| 58 | } |
| 59 | |
| 60 | return *tiledResult; |
| 61 | } |
| 62 | |
| 63 | FailureOr<TilingResult> tensor::replaceInsertSlicesWithTiledConsumer( |
| 64 | OpBuilder &builder, ArrayRef<tensor::InsertSliceOp> sliceOps, |
| 65 | ArrayRef<OpOperand *> consumerOperands, |
| 66 | ArrayRef<InnerTileAlignment> innerTileAlignments) { |
| 67 | if (sliceOps.empty()) { |
| 68 | LLVM_DEBUG( |
| 69 | { llvm::dbgs() << "expected candidate slices list to be non-empty" ; }); |
| 70 | return failure(); |
| 71 | } |
| 72 | if (sliceOps.size() != consumerOperands.size()) { |
| 73 | LLVM_DEBUG({ |
| 74 | llvm::dbgs() |
| 75 | << "expected as many operands as the number of slices passed" ; |
| 76 | }); |
| 77 | return failure(); |
| 78 | } |
| 79 | auto consumerOp = |
| 80 | dyn_cast<TilingInterface>(Val: consumerOperands.front()->getOwner()); |
| 81 | if (!consumerOp) |
| 82 | return failure(); |
| 83 | for (auto *opOperand : consumerOperands.drop_front()) { |
| 84 | if (opOperand->getOwner() != consumerOp) { |
| 85 | LLVM_DEBUG({ |
| 86 | llvm::dbgs() |
| 87 | << "expected all consumer operands to be from the same operation" ; |
| 88 | }); |
| 89 | return failure(); |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | auto consumerOperandNums = llvm::map_to_vector( |
| 94 | C&: consumerOperands, F: [](OpOperand *opOperand) -> unsigned { |
| 95 | return opOperand->getOperandNumber(); |
| 96 | }); |
| 97 | SmallVector<SmallVector<OpFoldResult>> allOffsets; |
| 98 | SmallVector<SmallVector<OpFoldResult>> allSizes; |
| 99 | for (auto sliceOp : sliceOps) { |
| 100 | |
| 101 | // `TilingInterface` currently only supports strides being 1. |
| 102 | if (!llvm::all_of(Range: sliceOp.getMixedStrides(), P: isOneInteger)) |
| 103 | return failure(); |
| 104 | |
| 105 | SmallVector<OpFoldResult> offsets = sliceOp.getMixedOffsets(); |
| 106 | SmallVector<OpFoldResult> sizes = sliceOp.getMixedSizes(); |
| 107 | allOffsets.emplace_back(Args: std::move(offsets)); |
| 108 | allSizes.emplace_back(Args: std::move(sizes)); |
| 109 | } |
| 110 | FailureOr<TilingResult> tiledResult = |
| 111 | consumerOp.getTiledImplementationFromOperandTiles( |
| 112 | b&: builder, operandNumbers: consumerOperandNums, allOffsets, allSizes, |
| 113 | innerTileAlignments); |
| 114 | if (failed(Result: tiledResult)) |
| 115 | return failure(); |
| 116 | |
| 117 | return *tiledResult; |
| 118 | } |
| 119 | |