| 1 | //===- ElementwiseToLinalg.cpp - conversion of elementwise to linalg ------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Linalg/Passes.h" |
| 10 | |
| 11 | #include "mlir/Dialect/Linalg/IR/Linalg.h" |
| 12 | #include "mlir/Dialect/Linalg/Transforms/Transforms.h" |
| 13 | #include "mlir/Dialect/Linalg/Utils/Utils.h" |
| 14 | #include "mlir/Transforms/DialectConversion.h" |
| 15 | #include "llvm/ADT/SmallVectorExtras.h" |
| 16 | |
| 17 | namespace mlir { |
| 18 | #define GEN_PASS_DEF_CONVERTELEMENTWISETOLINALGPASS |
| 19 | #include "mlir/Dialect/Linalg/Passes.h.inc" |
| 20 | } // namespace mlir |
| 21 | |
| 22 | using namespace mlir; |
| 23 | |
| 24 | static inline bool isScalarLike(Type t) { |
| 25 | return isa<IntegerType, FloatType, IndexType, ComplexType>(Val: t); |
| 26 | } |
| 27 | |
| 28 | static bool isElementwiseMappableOpOnRankedTensors(Operation *op) { |
| 29 | if (!OpTrait::hasElementwiseMappableTraits(op)) |
| 30 | return false; |
| 31 | |
| 32 | auto types = op->getOperandTypes(); |
| 33 | |
| 34 | // We want at least one ranked tensor. |
| 35 | bool anyRankedTensor = llvm::any_of(Range&: types, P: llvm::IsaPred<RankedTensorType>); |
| 36 | |
| 37 | // No invalid operands (i.e., every operand is a ranked tensor or |
| 38 | // scalar-like). |
| 39 | bool noneInvalid = llvm::none_of(Range&: types, P: [](Type t) { |
| 40 | return !(isa<RankedTensorType>(Val: t) || isScalarLike(t)); |
| 41 | }); |
| 42 | |
| 43 | return anyRankedTensor && noneInvalid; |
| 44 | } |
| 45 | |
| 46 | /// Given `op` assumed `isElementwiseMappableOpOnRankedTensors`, iterate over |
| 47 | /// the result types and return a list of values such that, for each result type |
| 48 | /// `t` and value `v` at the same index `idx`: |
| 49 | /// 1. `v.getType() == t` |
| 50 | /// 2. If an operand of `op` has type `t`, let `operand_first` be the first |
| 51 | /// such operand. Then`v == operand_first`. |
| 52 | /// 3. Otherwise, v is a newly created `tensor::EmptyOp` with: |
| 53 | /// a. Static and dynamic dims extracted from the first operand of `op`. |
| 54 | /// b. Elemental type equal to the elemental type of `t`. |
| 55 | /// |
| 56 | /// This is sufficient because ElementwiseMappable guarantees that "The static |
| 57 | /// types of all vector (resp. tensor) operands and results must have the same |
| 58 | /// shape". |
| 59 | static SmallVector<Value, 4> |
| 60 | getOrCreateOperandsMatchingResultTypes(OpBuilder &b, Operation *op) { |
| 61 | assert(isElementwiseMappableOpOnRankedTensors(op)); |
| 62 | Location loc = op->getLoc(); |
| 63 | ValueRange operands = op->getOperands(); |
| 64 | TypeRange rankedTensorTypes = op->getResultTypes(); |
| 65 | SmallVector<Value, 4> res; |
| 66 | res.reserve(N: rankedTensorTypes.size()); |
| 67 | for (Type t : rankedTensorTypes) { |
| 68 | // Try to find an operand with type matching the result tensor. |
| 69 | bool found = false; |
| 70 | for (Value v : operands) { |
| 71 | if (v.getType() == t) { |
| 72 | found = true; |
| 73 | res.push_back(Elt: v); |
| 74 | break; |
| 75 | } |
| 76 | } |
| 77 | if (found) |
| 78 | continue; |
| 79 | |
| 80 | // Extract static / dynamic shape mix from the first operand. |
| 81 | res.push_back(Elt: tensor::EmptyOp::create( |
| 82 | builder&: b, location: loc, sizes: tensor::getMixedSizes(builder&: b, loc, value: operands.front()), |
| 83 | elementType: cast<RankedTensorType>(Val&: t).getElementType())); |
| 84 | } |
| 85 | return res; |
| 86 | } |
| 87 | |
| 88 | namespace { |
| 89 | struct ConvertAnyElementwiseMappableOpOnRankedTensors : public RewritePattern { |
| 90 | ConvertAnyElementwiseMappableOpOnRankedTensors(MLIRContext *context) |
| 91 | : RewritePattern(MatchAnyOpTypeTag(), /*benefit=*/1, context) {} |
| 92 | LogicalResult matchAndRewrite(Operation *op, |
| 93 | PatternRewriter &rewriter) const final { |
| 94 | if (!isElementwiseMappableOpOnRankedTensors(op)) |
| 95 | return rewriter.notifyMatchFailure( |
| 96 | arg&: op, msg: "requires elementwise op on ranked tensors" ); |
| 97 | |
| 98 | auto resTy = cast<RankedTensorType>(Val: op->getResult(idx: 0).getType()); |
| 99 | auto rank = resTy.getRank(); |
| 100 | |
| 101 | // Maps: identity for tensors (rank > 0), scalar map for scalars. |
| 102 | AffineMap scalarMap = AffineMap::get(/*dimCount=*/rank, /*symbolCount=*/0, |
| 103 | /*results=*/{}, context: rewriter.getContext()); |
| 104 | AffineMap idMap = rewriter.getMultiDimIdentityMap(rank); |
| 105 | |
| 106 | // Match phase. |
| 107 | SmallVector<bool> isScalarOperand; |
| 108 | isScalarOperand.reserve(N: op->getNumOperands()); |
| 109 | for (Type ty : op->getOperandTypes()) { |
| 110 | if (isScalarLike(t: ty)) |
| 111 | isScalarOperand.push_back(Elt: true); |
| 112 | else if (auto rt = dyn_cast<RankedTensorType>(Val&: ty)) |
| 113 | isScalarOperand.push_back(Elt: false); |
| 114 | else |
| 115 | return rewriter.notifyMatchFailure( |
| 116 | arg&: op, |
| 117 | msg: "unsupported operand type (expected scalar-like or ranked tensor)" ); |
| 118 | } |
| 119 | |
| 120 | // Create indexing maps. |
| 121 | SmallVector<AffineMap> indexingMaps; |
| 122 | indexingMaps.reserve(N: op->getNumOperands() + op->getNumResults()); |
| 123 | |
| 124 | for (bool isScalar : isScalarOperand) |
| 125 | indexingMaps.push_back(Elt: isScalar ? scalarMap : idMap); |
| 126 | |
| 127 | indexingMaps.append(NumInputs: op->getNumResults(), Elt: idMap); |
| 128 | |
| 129 | SmallVector<utils::IteratorType> iteratorTypes( |
| 130 | rank, utils::IteratorType::parallel); |
| 131 | SmallVector<Value> outputs = |
| 132 | getOrCreateOperandsMatchingResultTypes(b&: rewriter, op); |
| 133 | rewriter.replaceOpWithNewOp<linalg::GenericOp>( |
| 134 | op, /*resultTensorTypes=*/args: op->getResultTypes(), |
| 135 | /*inputs=*/args: op->getOperands(), |
| 136 | /*outputs=*/args&: outputs, |
| 137 | /*indexingMaps=*/args&: indexingMaps, |
| 138 | /*iteratorTypes=*/args&: iteratorTypes, |
| 139 | /*bodyBuilder=*/ |
| 140 | args: [&](OpBuilder &builder, Location loc, ValueRange regionArgs) { |
| 141 | SmallVector<Type> resultEltTys = |
| 142 | llvm::map_to_vector<6>(C: op->getResultTypes(), F: [](Type type) { |
| 143 | return cast<TensorType>(Val&: type).getElementType(); |
| 144 | }); |
| 145 | Operation *scalarOp = builder.insert(op: Operation::create( |
| 146 | location: loc, name: op->getName(), resultTypes: resultEltTys, |
| 147 | operands: regionArgs.take_front(n: op->getNumOperands()), |
| 148 | attributes: op->getDiscardableAttrDictionary(), properties: op->getPropertiesStorage(), |
| 149 | /*successors=*/{}, /*numRegions=*/0)); |
| 150 | linalg::YieldOp::create(builder, location: loc, values: scalarOp->getResults()); |
| 151 | }); |
| 152 | return success(); |
| 153 | } |
| 154 | }; |
| 155 | } // namespace |
| 156 | |
| 157 | void mlir::linalg::populateElementwiseToLinalgConversionPatterns( |
| 158 | RewritePatternSet &patterns) { |
| 159 | patterns.add<ConvertAnyElementwiseMappableOpOnRankedTensors>( |
| 160 | arg: patterns.getContext()); |
| 161 | } |
| 162 | |
| 163 | namespace { |
| 164 | class ConvertElementwiseToLinalgPass |
| 165 | : public impl::ConvertElementwiseToLinalgPassBase< |
| 166 | ConvertElementwiseToLinalgPass> { |
| 167 | using impl::ConvertElementwiseToLinalgPassBase< |
| 168 | ConvertElementwiseToLinalgPass>::ConvertElementwiseToLinalgPassBase; |
| 169 | |
| 170 | void runOnOperation() final { |
| 171 | auto *func = getOperation(); |
| 172 | auto *context = &getContext(); |
| 173 | ConversionTarget target(*context); |
| 174 | RewritePatternSet patterns(context); |
| 175 | |
| 176 | mlir::linalg::populateElementwiseToLinalgConversionPatterns(patterns); |
| 177 | target.markUnknownOpDynamicallyLegal(fn: [](Operation *op) { |
| 178 | return !isElementwiseMappableOpOnRankedTensors(op); |
| 179 | }); |
| 180 | |
| 181 | if (failed(Result: applyPartialConversion(op: func, target, patterns: std::move(patterns)))) |
| 182 | signalPassFailure(); |
| 183 | } |
| 184 | }; |
| 185 | } // namespace |
| 186 | |