1//===- ElementwiseToLinalg.cpp - conversion of elementwise to linalg ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Linalg/Passes.h"
10
11#include "mlir/Dialect/Linalg/IR/Linalg.h"
12#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
13#include "mlir/Dialect/Linalg/Utils/Utils.h"
14#include "mlir/Transforms/DialectConversion.h"
15#include "llvm/ADT/SmallVectorExtras.h"
16
17namespace mlir {
18#define GEN_PASS_DEF_CONVERTELEMENTWISETOLINALGPASS
19#include "mlir/Dialect/Linalg/Passes.h.inc"
20} // namespace mlir
21
22using namespace mlir;
23
24static inline bool isScalarLike(Type t) {
25 return isa<IntegerType, FloatType, IndexType, ComplexType>(Val: t);
26}
27
28static bool isElementwiseMappableOpOnRankedTensors(Operation *op) {
29 if (!OpTrait::hasElementwiseMappableTraits(op))
30 return false;
31
32 auto types = op->getOperandTypes();
33
34 // We want at least one ranked tensor.
35 bool anyRankedTensor = llvm::any_of(Range&: types, P: llvm::IsaPred<RankedTensorType>);
36
37 // No invalid operands (i.e., every operand is a ranked tensor or
38 // scalar-like).
39 bool noneInvalid = llvm::none_of(Range&: types, P: [](Type t) {
40 return !(isa<RankedTensorType>(Val: t) || isScalarLike(t));
41 });
42
43 return anyRankedTensor && noneInvalid;
44}
45
46/// Given `op` assumed `isElementwiseMappableOpOnRankedTensors`, iterate over
47/// the result types and return a list of values such that, for each result type
48/// `t` and value `v` at the same index `idx`:
49/// 1. `v.getType() == t`
50/// 2. If an operand of `op` has type `t`, let `operand_first` be the first
51/// such operand. Then`v == operand_first`.
52/// 3. Otherwise, v is a newly created `tensor::EmptyOp` with:
53/// a. Static and dynamic dims extracted from the first operand of `op`.
54/// b. Elemental type equal to the elemental type of `t`.
55///
56/// This is sufficient because ElementwiseMappable guarantees that "The static
57/// types of all vector (resp. tensor) operands and results must have the same
58/// shape".
59static SmallVector<Value, 4>
60getOrCreateOperandsMatchingResultTypes(OpBuilder &b, Operation *op) {
61 assert(isElementwiseMappableOpOnRankedTensors(op));
62 Location loc = op->getLoc();
63 ValueRange operands = op->getOperands();
64 TypeRange rankedTensorTypes = op->getResultTypes();
65 SmallVector<Value, 4> res;
66 res.reserve(N: rankedTensorTypes.size());
67 for (Type t : rankedTensorTypes) {
68 // Try to find an operand with type matching the result tensor.
69 bool found = false;
70 for (Value v : operands) {
71 if (v.getType() == t) {
72 found = true;
73 res.push_back(Elt: v);
74 break;
75 }
76 }
77 if (found)
78 continue;
79
80 // Extract static / dynamic shape mix from the first operand.
81 res.push_back(Elt: tensor::EmptyOp::create(
82 builder&: b, location: loc, sizes: tensor::getMixedSizes(builder&: b, loc, value: operands.front()),
83 elementType: cast<RankedTensorType>(Val&: t).getElementType()));
84 }
85 return res;
86}
87
88namespace {
89struct ConvertAnyElementwiseMappableOpOnRankedTensors : public RewritePattern {
90 ConvertAnyElementwiseMappableOpOnRankedTensors(MLIRContext *context)
91 : RewritePattern(MatchAnyOpTypeTag(), /*benefit=*/1, context) {}
92 LogicalResult matchAndRewrite(Operation *op,
93 PatternRewriter &rewriter) const final {
94 if (!isElementwiseMappableOpOnRankedTensors(op))
95 return rewriter.notifyMatchFailure(
96 arg&: op, msg: "requires elementwise op on ranked tensors");
97
98 auto resTy = cast<RankedTensorType>(Val: op->getResult(idx: 0).getType());
99 auto rank = resTy.getRank();
100
101 // Maps: identity for tensors (rank > 0), scalar map for scalars.
102 AffineMap scalarMap = AffineMap::get(/*dimCount=*/rank, /*symbolCount=*/0,
103 /*results=*/{}, context: rewriter.getContext());
104 AffineMap idMap = rewriter.getMultiDimIdentityMap(rank);
105
106 // Match phase.
107 SmallVector<bool> isScalarOperand;
108 isScalarOperand.reserve(N: op->getNumOperands());
109 for (Type ty : op->getOperandTypes()) {
110 if (isScalarLike(t: ty))
111 isScalarOperand.push_back(Elt: true);
112 else if (auto rt = dyn_cast<RankedTensorType>(Val&: ty))
113 isScalarOperand.push_back(Elt: false);
114 else
115 return rewriter.notifyMatchFailure(
116 arg&: op,
117 msg: "unsupported operand type (expected scalar-like or ranked tensor)");
118 }
119
120 // Create indexing maps.
121 SmallVector<AffineMap> indexingMaps;
122 indexingMaps.reserve(N: op->getNumOperands() + op->getNumResults());
123
124 for (bool isScalar : isScalarOperand)
125 indexingMaps.push_back(Elt: isScalar ? scalarMap : idMap);
126
127 indexingMaps.append(NumInputs: op->getNumResults(), Elt: idMap);
128
129 SmallVector<utils::IteratorType> iteratorTypes(
130 rank, utils::IteratorType::parallel);
131 SmallVector<Value> outputs =
132 getOrCreateOperandsMatchingResultTypes(b&: rewriter, op);
133 rewriter.replaceOpWithNewOp<linalg::GenericOp>(
134 op, /*resultTensorTypes=*/args: op->getResultTypes(),
135 /*inputs=*/args: op->getOperands(),
136 /*outputs=*/args&: outputs,
137 /*indexingMaps=*/args&: indexingMaps,
138 /*iteratorTypes=*/args&: iteratorTypes,
139 /*bodyBuilder=*/
140 args: [&](OpBuilder &builder, Location loc, ValueRange regionArgs) {
141 SmallVector<Type> resultEltTys =
142 llvm::map_to_vector<6>(C: op->getResultTypes(), F: [](Type type) {
143 return cast<TensorType>(Val&: type).getElementType();
144 });
145 Operation *scalarOp = builder.insert(op: Operation::create(
146 location: loc, name: op->getName(), resultTypes: resultEltTys,
147 operands: regionArgs.take_front(n: op->getNumOperands()),
148 attributes: op->getDiscardableAttrDictionary(), properties: op->getPropertiesStorage(),
149 /*successors=*/{}, /*numRegions=*/0));
150 linalg::YieldOp::create(builder, location: loc, values: scalarOp->getResults());
151 });
152 return success();
153 }
154};
155} // namespace
156
157void mlir::linalg::populateElementwiseToLinalgConversionPatterns(
158 RewritePatternSet &patterns) {
159 patterns.add<ConvertAnyElementwiseMappableOpOnRankedTensors>(
160 arg: patterns.getContext());
161}
162
163namespace {
164class ConvertElementwiseToLinalgPass
165 : public impl::ConvertElementwiseToLinalgPassBase<
166 ConvertElementwiseToLinalgPass> {
167 using impl::ConvertElementwiseToLinalgPassBase<
168 ConvertElementwiseToLinalgPass>::ConvertElementwiseToLinalgPassBase;
169
170 void runOnOperation() final {
171 auto *func = getOperation();
172 auto *context = &getContext();
173 ConversionTarget target(*context);
174 RewritePatternSet patterns(context);
175
176 mlir::linalg::populateElementwiseToLinalgConversionPatterns(patterns);
177 target.markUnknownOpDynamicallyLegal(fn: [](Operation *op) {
178 return !isElementwiseMappableOpOnRankedTensors(op);
179 });
180
181 if (failed(Result: applyPartialConversion(op: func, target, patterns: std::move(patterns))))
182 signalPassFailure();
183 }
184};
185} // namespace
186