| 1 | //===- Specialize.cpp - linalg generic ops to named ops ------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements a method to specialize generic operations to named |
| 10 | // operations. Conceptually it is the opposite of generalize.cpp. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Analysis/SliceAnalysis.h" |
| 15 | #include "mlir/Dialect/Complex/IR/Complex.h" |
| 16 | #include "mlir/Dialect/Linalg/IR/Linalg.h" |
| 17 | #include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h" |
| 18 | #include "mlir/Dialect/Linalg/Passes.h" |
| 19 | #include "mlir/Dialect/Linalg/Transforms/Transforms.h" |
| 20 | #include "mlir/Dialect/Math/IR/Math.h" |
| 21 | #include "mlir/IR/PatternMatch.h" |
| 22 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 23 | #include "llvm/ADT/TypeSwitch.h" |
| 24 | |
| 25 | namespace mlir { |
| 26 | #define GEN_PASS_DEF_LINALGSPECIALIZEGENERICOPSPASS |
| 27 | #include "mlir/Dialect/Linalg/Passes.h.inc" |
| 28 | } // namespace mlir |
| 29 | |
| 30 | #define DEBUG_TYPE "linalg-specialization" |
| 31 | |
| 32 | using namespace mlir; |
| 33 | using namespace mlir::linalg; |
| 34 | |
| 35 | //===----------------------------------------------------------------------===// |
| 36 | // Specialize linalg generic to elementwise ops. |
| 37 | //===----------------------------------------------------------------------===// |
| 38 | |
| 39 | // Given an elementwise single binary linalg generic op, checks whether the |
| 40 | // binary op accesses operands as swapped. e.g. |
| 41 | // this differentiates between a linalg-generic body that contains: |
| 42 | // ^bb0(%a: f32, %b: f32, %c : f32): |
| 43 | // %0 = arith.subf %a, %b : f32 |
| 44 | // linalg.yield %0: f32 |
| 45 | // against: |
| 46 | // ^bb0(%a: f32, %b: f32, %c : f32): |
| 47 | // %0 = arith.subf %b, %a : f32 |
| 48 | // linalg.yield %0: f32 |
| 49 | // Former is linalg.sub(a,b), latter is linalg.sub(b,a). |
| 50 | static bool areBinOpsSwapped(GenericOp genericOp, bool isTernary) { |
| 51 | Block *body = genericOp.getBody(); |
| 52 | Operation *op = &body->front(); |
| 53 | bool swapped = false; |
| 54 | if (op->getOpOperand(idx: 0 + isTernary).get() != |
| 55 | body->getArgument(i: 0 + isTernary)) { |
| 56 | swapped = true; |
| 57 | assert(op->getOpOperand(0 + isTernary).get() == |
| 58 | body->getArgument(1 + isTernary) && |
| 59 | op->getOpOperand(1 + isTernary).get() == |
| 60 | body->getArgument(0 + isTernary) && |
| 61 | "binary op uses just one block arg" ); |
| 62 | } |
| 63 | return swapped; |
| 64 | } |
| 65 | |
| 66 | // Given an elementwise single unary linalg generic op whose body operation is a |
| 67 | // binary operation, check if one of its operands is a scalar value defined |
| 68 | // outside the generic op, set its index, and return true. Otherwise return |
| 69 | // false. The index is unique because the block argument is used at |
| 70 | // least by one operand, as checked in `isaElemwiseSingleUnaryOpInterface`. |
| 71 | // |
| 72 | // Example: |
| 73 | // %cst = arith.constant 3.14 : f32 |
| 74 | // %0 = linalg.generic { indexing_maps = [#mapA, #mapRes], ... } |
| 75 | // ins(%A : tensor<?xf32>) outs(...) { |
| 76 | // ^bb0(%a: f32, %out : f32): |
| 77 | // %0 = arith.mulf %a, %cst : f32 |
| 78 | // linalg.yield %0: f32 |
| 79 | // } -> tensor<?xf32> |
| 80 | // Here, the returned index is 1, and the generic op can be represented as |
| 81 | // %0 = linalg.elementwise <mul> |
| 82 | // indexing_maps = [#mapA, affine_map<(d0) -> ()>, #mapRes] |
| 83 | // ins(%A, %cst : tensor<?xf32>, f32) outs(...) -> tensor<?xf32> |
| 84 | static bool findIndexOfScalarOperand(GenericOp genericOp, int &index) { |
| 85 | Block *body = genericOp.getBody(); |
| 86 | Operation *op = &body->front(); |
| 87 | for (auto [i, v] : llvm::enumerate(First: op->getOperands())) { |
| 88 | if (auto blockArg = dyn_cast<BlockArgument>(Val&: v); |
| 89 | blockArg && blockArg.getOwner() == body) |
| 90 | continue; // not an outside value... |
| 91 | index = i; |
| 92 | return true; |
| 93 | } |
| 94 | return false; |
| 95 | } |
| 96 | |
| 97 | // Maps a linalg.generic body operation to its corresponding elementwise kind, |
| 98 | // if one exists. Handles the ops with a straightforward one-to-one mapping; |
| 99 | // ops needing extra context (e.g. boolean add/mul, reciprocal, square) are |
| 100 | // handled by the caller. |
| 101 | static std::optional<ElementwiseKind> getElementwiseKind(Operation *op) { |
| 102 | return llvm::TypeSwitch<Operation *, std::optional<ElementwiseKind>>(op) |
| 103 | .Case<arith::AddFOp, arith::AddIOp, complex::AddOp>( |
| 104 | caseFn: [](Operation *) { return ElementwiseKind::add; }) |
| 105 | .Case<arith::MulIOp, arith::MulFOp, complex::MulOp>( |
| 106 | caseFn: [](Operation *) { return ElementwiseKind::mul; }) |
| 107 | .Case(caseFn: [](math::ExpOp) { return ElementwiseKind::exp; }) |
| 108 | .Case(caseFn: [](math::AbsFOp) { return ElementwiseKind::abs; }) |
| 109 | .Case(caseFn: [](math::CeilOp) { return ElementwiseKind::ceil; }) |
| 110 | .Case(caseFn: [](math::FloorOp) { return ElementwiseKind::floor; }) |
| 111 | .Case(caseFn: [](arith::NegFOp) { return ElementwiseKind::negf; }) |
| 112 | .Case(caseFn: [](math::RoundOp) { return ElementwiseKind::round; }) |
| 113 | .Case(caseFn: [](math::SqrtOp) { return ElementwiseKind::sqrt; }) |
| 114 | .Case(caseFn: [](math::RsqrtOp) { return ElementwiseKind::rsqrt; }) |
| 115 | .Case(caseFn: [](math::TanhOp) { return ElementwiseKind::tanh; }) |
| 116 | .Case(caseFn: [](math::ErfOp) { return ElementwiseKind::erf; }) |
| 117 | .Case(caseFn: [](math::SinOp) { return ElementwiseKind::sin; }) |
| 118 | .Case(caseFn: [](math::CosOp) { return ElementwiseKind::cos; }) |
| 119 | .Case(caseFn: [](math::TanOp) { return ElementwiseKind::tan; }) |
| 120 | .Case(caseFn: [](math::AcosOp) { return ElementwiseKind::acos; }) |
| 121 | .Case(caseFn: [](math::AcoshOp) { return ElementwiseKind::acosh; }) |
| 122 | .Case(caseFn: [](math::AsinOp) { return ElementwiseKind::asin; }) |
| 123 | .Case(caseFn: [](math::AsinhOp) { return ElementwiseKind::asinh; }) |
| 124 | .Case(caseFn: [](math::AtanOp) { return ElementwiseKind::atan; }) |
| 125 | .Case(caseFn: [](math::AtanhOp) { return ElementwiseKind::atanh; }) |
| 126 | .Case(caseFn: [](math::LogOp) { return ElementwiseKind::log; }) |
| 127 | .Case(caseFn: [](math::Log10Op) { return ElementwiseKind::log10; }) |
| 128 | .Case(caseFn: [](math::Log1pOp) { return ElementwiseKind::log1p; }) |
| 129 | .Case(caseFn: [](math::Log2Op) { return ElementwiseKind::log2; }) |
| 130 | .Case<arith::SubIOp, arith::SubFOp, complex::SubOp>( |
| 131 | caseFn: [](Operation *) { return ElementwiseKind::sub; }) |
| 132 | .Case<arith::DivSIOp, arith::DivFOp, complex::DivOp>( |
| 133 | caseFn: [](Operation *) { return ElementwiseKind::div; }) |
| 134 | .Case(caseFn: [](arith::DivUIOp) { return ElementwiseKind::div_unsigned; }) |
| 135 | .Case<arith::MaxSIOp, arith::MaximumFOp>( |
| 136 | caseFn: [](Operation *) { return ElementwiseKind::max_signed; }) |
| 137 | .Case<arith::MinSIOp, arith::MinimumFOp>( |
| 138 | caseFn: [](Operation *) { return ElementwiseKind::min_signed; }) |
| 139 | .Case(caseFn: [](math::PowFOp) { return ElementwiseKind::powf; }) |
| 140 | .Case(caseFn: [](arith::MaxUIOp) { return ElementwiseKind::max_unsigned; }) |
| 141 | .Case(caseFn: [](arith::MinUIOp) { return ElementwiseKind::min_unsigned; }) |
| 142 | .Case(caseFn: [](arith::SelectOp) { return ElementwiseKind::select; }) |
| 143 | .Default(defaultFn: [](Operation *) { return std::nullopt; }); |
| 144 | } |
| 145 | |
| 146 | // Attempt to specialize unary/binary/ternary linalg.generic ops |
| 147 | // to linalg.elementwise. |
| 148 | // |
| 149 | // Example: |
| 150 | // %0 = linalg.generic { |
| 151 | // indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, |
| 152 | // affine_map<(d0, d1) -> (d0, d1)>], |
| 153 | // iterator_types = ["parallel", "parallel"] |
| 154 | // } ins(%In : tensor<?x?xf32>) outs(%Out : tensor<?x?xf32>) { |
| 155 | // ^bb0(%in: f32, %out: f32): |
| 156 | // %1 = math.exp %in : f32 |
| 157 | // linalg.yield %1 : f32 |
| 158 | // } -> tensor<?x?xf32> |
| 159 | // |
| 160 | // is specialized to |
| 161 | // linalg.elementwise <exp> ... |
| 162 | // |
| 163 | // The category op can carry non-identity indexing maps; these are |
| 164 | // transferred verbatim from the `genericOp`. |
| 165 | // |
| 166 | // In addition to the canonical forms used by the generalization path, this |
| 167 | // function can handle the following variations: |
| 168 | // |
| 169 | // 1) Swapped operands in binary ops (see the `areBinOpsSwapped` helper) |
| 170 | // 2) Unary generic ops with a binary body op (see the |
| 171 | // `findIndexOfScalarOperand` helper) |
| 172 | static FailureOr<LinalgOp> specializeLinalgElementwise(RewriterBase &rewriter, |
| 173 | GenericOp genericOp) { |
| 174 | // Classify the generic op. |
| 175 | unsigned arity = genericOp.getNumDpsInputs(); |
| 176 | bool isUnary = arity == 1; |
| 177 | bool isBinary = arity == 2; |
| 178 | bool isTernary = arity == 3; |
| 179 | |
| 180 | // Will inspect the body operation to determine named op or elementwise kind. |
| 181 | Operation *op = &genericOp.getBody()->front(); |
| 182 | |
| 183 | // Detect variations from canonical forms. |
| 184 | bool hasSwappedOperands = |
| 185 | (isBinary || isTernary) && areBinOpsSwapped(genericOp, isTernary); |
| 186 | int scalarOprIdx = -1; |
| 187 | bool hasScalarOperand = isUnary && op->getNumOperands() == 2 && |
| 188 | findIndexOfScalarOperand(genericOp, index&: scalarOprIdx); |
| 189 | |
| 190 | // Helper to dispatch between named op and `linalg.elementwise`. |
| 191 | // Lambdas with explicit template parameter list are a C++20 feature, hence |
| 192 | // the dummy op object. |
| 193 | auto replaceOp = [&](ElementwiseKind kind, |
| 194 | bool mayHoistScalarOperand = true) -> LinalgOp { |
| 195 | SmallVector<Value> inputs = genericOp.getDpsInputs(); |
| 196 | SmallVector<AffineMap> indexingMaps = genericOp.getIndexingMapsArray(); |
| 197 | if (hasSwappedOperands) { |
| 198 | // Ternary indices are +1, since the first is the boolean mask. |
| 199 | // If new ternary with non-booleans as first argument are created, |
| 200 | // we may need to calculate all combinations possible. |
| 201 | std::swap(a&: inputs[0 + isTernary], b&: inputs[1 + isTernary]); |
| 202 | std::swap(a&: indexingMaps[0 + isTernary], b&: indexingMaps[1 + isTernary]); |
| 203 | } |
| 204 | |
| 205 | if (hasScalarOperand && mayHoistScalarOperand) { |
| 206 | // Adjust inputs and indexing maps accordingly. |
| 207 | inputs.insert(I: inputs.begin() + scalarOprIdx, |
| 208 | Elt: op->getOperand(idx: scalarOprIdx)); |
| 209 | auto scalarBroadcastMap = |
| 210 | AffineMap::get(dimCount: genericOp.getNumParallelLoops(), /*symbolCount=*/0, |
| 211 | context: rewriter.getContext()); |
| 212 | indexingMaps.insert(I: indexingMaps.begin() + scalarOprIdx, |
| 213 | Elt: scalarBroadcastMap); |
| 214 | } |
| 215 | auto newOp = ElementwiseOp::create( |
| 216 | builder&: rewriter, location: genericOp.getLoc(), inputs, outputs: genericOp.getDpsInits(), kind, |
| 217 | indexingMaps: rewriter.getAffineMapArrayAttr(values: indexingMaps)); |
| 218 | |
| 219 | rewriter.replaceOp(op: genericOp, newOp); |
| 220 | return newOp; |
| 221 | }; |
| 222 | |
| 223 | // Reciprocal |
| 224 | if (auto divOp = dyn_cast<arith::DivFOp>(Val: op)) { |
| 225 | if (auto constOp = dyn_cast_if_present<arith::ConstantOp>( |
| 226 | Val: divOp.getLhs().getDefiningOp())) |
| 227 | if (cast<FloatAttr>(Val: constOp.getValue()).getValue().isExactlyValue(V: 1.0)) |
| 228 | return replaceOp(ElementwiseKind::reciprocal, |
| 229 | /*mayHoistScalarOperand=*/false); |
| 230 | } |
| 231 | |
| 232 | // Square |
| 233 | if (auto mulOp = dyn_cast<arith::MulFOp>(Val: op)) |
| 234 | if (mulOp.getLhs() == mulOp.getRhs()) |
| 235 | return replaceOp(ElementwiseKind::square); |
| 236 | |
| 237 | // Boolean-typed `add` and `mul`. |
| 238 | if (isBinary && llvm::all_of(Range: op->getOperands(), P: [](Value v) { |
| 239 | return v.getType().isInteger(width: 1); |
| 240 | })) { |
| 241 | if (isa<arith::OrIOp>(Val: op)) |
| 242 | return replaceOp(ElementwiseKind::add); |
| 243 | if (isa<arith::AndIOp>(Val: op)) |
| 244 | return replaceOp(ElementwiseKind::mul); |
| 245 | } |
| 246 | |
| 247 | // Table driven |
| 248 | if (std::optional<ElementwiseKind> kind = getElementwiseKind(op)) { |
| 249 | auto arityGroupAndKind = linalg::getArityGroupAndKind(kind: *kind); |
| 250 | // A hoisted scalar operand adds one input to the elementwise op. |
| 251 | unsigned numInputs = arity + (hasScalarOperand ? 1 : 0); |
| 252 | if (numInputs == static_cast<unsigned>(arityGroupAndKind.arityGroup)) |
| 253 | return replaceOp(*kind); |
| 254 | } |
| 255 | |
| 256 | return rewriter.notifyMatchFailure( |
| 257 | arg&: genericOp, msg: "elementwise operation cannot be specialized to category op" ); |
| 258 | } |
| 259 | |
| 260 | //===----------------------------------------------------------------------===// |
| 261 | // Specialize linalg generic to reduce op. |
| 262 | //===----------------------------------------------------------------------===// |
| 263 | |
| 264 | // Specialize a `linalg.generic` that is semantically a reduction into a |
| 265 | // `linalg.reduce`, moving the generic's payload into the reduce's combiner. |
| 266 | static FailureOr<LinalgOp> |
| 267 | specializeLinalgReductions(RewriterBase &rewriter, GenericOp genericOp, |
| 268 | ArrayRef<int64_t> reductionDims) { |
| 269 | auto reduceOp = ReduceOp::create( |
| 270 | builder&: rewriter, location: genericOp.getLoc(), inputs: genericOp.getDpsInputs(), |
| 271 | inits: genericOp.getDpsInits(), dimensions: reductionDims, /*bodyBuild=*/odsArg3: nullptr); |
| 272 | |
| 273 | // Move the generic's payload region into the reduce's combiner. |
| 274 | rewriter.inlineRegionBefore(region&: genericOp.getRegion(), parent&: reduceOp.getCombiner(), |
| 275 | before: reduceOp.getCombiner().end()); |
| 276 | rewriter.replaceOp(op: genericOp, newOp: reduceOp); |
| 277 | return cast<LinalgOp>(Val: reduceOp.getOperation()); |
| 278 | } |
| 279 | |
| 280 | //===----------------------------------------------------------------------===// |
| 281 | // Specialize linalg generic to matmul variants. |
| 282 | //===----------------------------------------------------------------------===// |
| 283 | /// Identifies linalg.generic that is essentially named op of the form: |
| 284 | // ` linalg.{batch_}?matmul{_transpose_a | _transpose_b}? ` |
| 285 | // |
| 286 | // It is possible that a linalg.generic may be implementing a matmul but not |
| 287 | // in a straight-forward way e.g. below is matrix multiply over some slice |
| 288 | // ``` |
| 289 | // %0 = linalg.generic { |
| 290 | // indexing_maps = [affine_map<(d0, d1, d2) -> (3, d1, d0)>, |
| 291 | // affine_map<(d0, d1, d2) -> (d0, 5, d2)>, |
| 292 | // affine_map<(d0, d1, d2) -> (d2, d1, 13)>], |
| 293 | // iterator_types = ["parallel", "parallel", "parallel"]} |
| 294 | // ins(%A, %B : tensor<20x20x20xf32>, tensor<20x20x20xf32>) |
| 295 | // outs(%C : tensor<20x20x20xf32>) { |
| 296 | // ^bb0(%a: f32, %b: f32, %c : f32): |
| 297 | // %mul = arith.mulf %a, %b : f32 |
| 298 | // %add = arith.addf %mul, %c : f32 |
| 299 | // linalg.yield %add : f32 |
| 300 | // } -> tensor<20x20x20xf32> |
| 301 | // ``` |
| 302 | // It is not possible to represent above as named op. |
| 303 | // e.g. linalg.batch_matmul(%A, %B : tensor<20x20x20xf32>, ...) is |
| 304 | // not the same as linalg.generic above. |
| 305 | namespace { |
| 306 | enum class IndexMatchResult { |
| 307 | Match = 0, // identity map. |
| 308 | Transposed, // transposed map. |
| 309 | Mismatch // none of the above. |
| 310 | }; |
| 311 | |
| 312 | // Checks whether the input Affine `map` contains two consecutive dims that |
| 313 | // can be interpreted as accessing a 2D matrix. It is assumed that the row |
| 314 | // column dimension are adjacent axis (in this order) and start at |
| 315 | // `rowDimIdx` in the input map. |
| 316 | // |
| 317 | // e.g. consider A matrix in `C[M,N] = A[M,K] * B[K,N]`. We will check |
| 318 | // whether the map of A is identity (match), transposed, or something |
| 319 | // completely different (mis-match). Similar for B and C. |
| 320 | static IndexMatchResult matchOperandMap(AffineMap map, unsigned rowDimIdx, |
| 321 | unsigned expectedPosOfRowDim, |
| 322 | unsigned expectedPosOfColDim) { |
| 323 | // Get the matrix multiply indices. They are past the batch indices. |
| 324 | auto exprOfRowDim = map.getResults()[rowDimIdx]; |
| 325 | auto exprOfColDim = map.getResults()[rowDimIdx + 1]; |
| 326 | |
| 327 | // They should be pure dimension ids. |
| 328 | if (exprOfRowDim.getKind() != AffineExprKind::DimId || |
| 329 | exprOfColDim.getKind() != AffineExprKind::DimId) |
| 330 | return IndexMatchResult::Mismatch; |
| 331 | |
| 332 | auto posRowDim = cast<AffineDimExpr>(Val&: exprOfRowDim).getPosition(); |
| 333 | auto posColDim = cast<AffineDimExpr>(Val&: exprOfColDim).getPosition(); |
| 334 | |
| 335 | if (expectedPosOfRowDim == posRowDim && expectedPosOfColDim == posColDim) |
| 336 | return IndexMatchResult::Match; |
| 337 | |
| 338 | if (expectedPosOfRowDim == posColDim && expectedPosOfColDim == posRowDim) |
| 339 | return IndexMatchResult::Transposed; |
| 340 | |
| 341 | return IndexMatchResult::Mismatch; |
| 342 | } |
| 343 | |
| 344 | // Replaces genericOp with `NamedOpTy` op, supplied as a template arg. |
| 345 | // All the variants expressed as pseudo regular expression: |
| 346 | // `linalg.{batch_}?matmul` have same number of ins/out, so it's easy to |
| 347 | // stamp different versions. |
| 348 | // `castTy` is an optional type function that indicates whether (and which) cast |
| 349 | // attribute is needed for the named matmul op variant. |
| 350 | template <typename NamedOpTy> |
| 351 | static LinalgOp replaceWithMatmulVariant(RewriterBase &rewriter, GenericOp op, |
| 352 | std::optional<TypeFn> castTy, |
| 353 | ArrayRef<AffineMap> indexingMaps) { |
| 354 | SmallVector<NamedAttribute> attributes; |
| 355 | // Only explicitly specify the cast attribute for unsigned cast; signed is |
| 356 | // the default for linalg.matmul/linalg.batch_matmul. |
| 357 | if (castTy.has_value() && *castTy == TypeFn::cast_unsigned) { |
| 358 | auto castAttr = rewriter.getNamedAttr( |
| 359 | name: "cast" , val: TypeFnAttr::get(context: rewriter.getContext(), value: *castTy)); |
| 360 | attributes.push_back(Elt: castAttr); |
| 361 | } |
| 362 | |
| 363 | // Set the original generic's maps to preserve operand indexing semantics like |
| 364 | // transposition. |
| 365 | SmallVector<Attribute, 3> indexingMapsAttrVal = |
| 366 | llvm::map_to_vector(indexingMaps, [](AffineMap map) -> Attribute { |
| 367 | return AffineMapAttr::get(value: map); |
| 368 | }); |
| 369 | auto indexingMapsAttr = rewriter.getNamedAttr( |
| 370 | name: "indexing_maps" , val: rewriter.getArrayAttr(value: indexingMapsAttrVal)); |
| 371 | attributes.push_back(Elt: indexingMapsAttr); |
| 372 | |
| 373 | LinalgOp namedOp = rewriter.replaceOpWithNewOp<NamedOpTy>( |
| 374 | op, ValueRange{op.getDpsInputs()[0], op.getDpsInputs()[1]}, |
| 375 | ValueRange{op.getDpsInits()[0]}, attributes); |
| 376 | |
| 377 | return namedOp; |
| 378 | } |
| 379 | |
| 380 | // Returns the cast type to use for a matmul-like named op. If the generic |
| 381 | // contains casts that cannot be represented (e.g. output casts or mixed |
| 382 | // signedness), return std::nullopt. |
| 383 | static std::optional<TypeFn> getCastTypeForMatmulLikeOp(GenericOp genericOp) { |
| 384 | bool foundCastForMatmulOutput = false; |
| 385 | SmallVector<TypeFn> castTyFns; |
| 386 | genericOp.getBody()->walk(callback: [&](CastOpInterface castOp) { |
| 387 | // Collect forward slice of the cast op to check if it is for the matmul |
| 388 | // output. |
| 389 | SetVector<Operation *> forwardSlice; |
| 390 | getForwardSlice(op: castOp, forwardSlice: &forwardSlice); |
| 391 | |
| 392 | // If there is no multiplication op in the forward slice, then this cast |
| 393 | // op is for the matmul output. Cast ops on matmul output cannot be |
| 394 | // expressed by the matmul op variant. |
| 395 | if (!llvm::any_of(Range&: forwardSlice, P: [](Operation *op) { |
| 396 | // We check explicitly for these multiplication ops in |
| 397 | // `specializeLinalgContractions()` to infer matmul-like ops. |
| 398 | return isa<arith::MulIOp, arith::MulFOp, complex::MulOp>(Val: op); |
| 399 | })) { |
| 400 | foundCastForMatmulOutput = true; |
| 401 | return WalkResult::interrupt(); |
| 402 | } |
| 403 | |
| 404 | // Determine the cast type. |
| 405 | if (isa<arith::ExtUIOp, arith::UIToFPOp, arith::FPToUIOp>(Val: castOp)) |
| 406 | castTyFns.push_back(Elt: TypeFn::cast_unsigned); |
| 407 | else if (isa<arith::ExtSIOp, arith::SIToFPOp, arith::FPToSIOp>(Val: castOp)) |
| 408 | castTyFns.push_back(Elt: TypeFn::cast_signed); |
| 409 | |
| 410 | return WalkResult::advance(); |
| 411 | }); |
| 412 | |
| 413 | if (foundCastForMatmulOutput) |
| 414 | return std::nullopt; |
| 415 | |
| 416 | if (!castTyFns.empty()) { |
| 417 | // If there were multiple different cast types found, then we can't express |
| 418 | // them using matmul-like ops. They only allow a single cast type for all |
| 419 | // inputs. |
| 420 | if (!llvm::all_equal(Range&: castTyFns)) |
| 421 | return std::nullopt; |
| 422 | return castTyFns.front(); |
| 423 | } |
| 424 | |
| 425 | // Default to signed cast for matmul-like ops. |
| 426 | return TypeFn::cast_signed; |
| 427 | } |
| 428 | |
| 429 | static FailureOr<LinalgOp> specializeLinalgMmt4D(RewriterBase &rewriter, |
| 430 | GenericOp genericOp, |
| 431 | std::optional<TypeFn> castTy, |
| 432 | ContractionDimensions &dims) { |
| 433 | // Should all be rank 4 and dim 6 |
| 434 | auto indexingMaps = genericOp.getIndexingMapsArray(); |
| 435 | if (llvm::any_of(Range&: indexingMaps, P: [](AffineMap m) { |
| 436 | return m.getResults().size() != 4 || m.getNumDims() != 6; |
| 437 | })) |
| 438 | return failure(); |
| 439 | |
| 440 | auto aOuter = matchOperandMap(map: indexingMaps[0], rowDimIdx: 0, expectedPosOfRowDim: dims.m[0], expectedPosOfColDim: dims.k[0]); |
| 441 | auto aInner = matchOperandMap(map: indexingMaps[0], rowDimIdx: 2, expectedPosOfRowDim: dims.m[1], expectedPosOfColDim: dims.k[1]); |
| 442 | |
| 443 | auto bOuter = matchOperandMap(map: indexingMaps[1], rowDimIdx: 0, expectedPosOfRowDim: dims.k[0], expectedPosOfColDim: dims.n[0]); |
| 444 | auto bInner = matchOperandMap(map: indexingMaps[1], rowDimIdx: 2, expectedPosOfRowDim: dims.k[1], expectedPosOfColDim: dims.n[1]); |
| 445 | |
| 446 | auto cOuter = matchOperandMap(map: indexingMaps[2], rowDimIdx: 0, expectedPosOfRowDim: dims.m[0], expectedPosOfColDim: dims.n[0]); |
| 447 | auto cInner = matchOperandMap(map: indexingMaps[2], rowDimIdx: 2, expectedPosOfRowDim: dims.m[1], expectedPosOfColDim: dims.n[1]); |
| 448 | |
| 449 | if (llvm::is_contained(Set: {aOuter, bOuter, cOuter}, Element: IndexMatchResult::Mismatch)) |
| 450 | return failure(); |
| 451 | if (llvm::is_contained(Set: {aInner, bInner, cInner}, Element: IndexMatchResult::Mismatch)) |
| 452 | return failure(); |
| 453 | |
| 454 | SmallVector<AffineMap> namedOpMaps = {indexingMaps[0], indexingMaps[1], |
| 455 | indexingMaps[2]}; |
| 456 | |
| 457 | return replaceWithMatmulVariant<Mmt4DOp>(rewriter, op: genericOp, castTy, |
| 458 | indexingMaps: namedOpMaps); |
| 459 | } |
| 460 | |
| 461 | // Attempts to specialize `genericOp` to a specific named matmul variant |
| 462 | // (`matmul`, `batch_matmul`, or `mmt4d`). Returns failure without modifying the |
| 463 | // IR if no named variant matches. `castTy` is the cast type inferred for the |
| 464 | // contraction body. |
| 465 | static FailureOr<LinalgOp> |
| 466 | specializeToNamedContraction(RewriterBase &rewriter, GenericOp genericOp, |
| 467 | std::optional<TypeFn> castTy) { |
| 468 | // Linalg generic contraction can be across multiple axis e.g. |
| 469 | // ``` |
| 470 | // linalg.generic |
| 471 | // {indexing_maps = [affine_map<(m, n, k1, k2) -> (m, k1, k2)>, |
| 472 | // affine_map<(m, n, k1, k2) -> (k2, k1, n)>, |
| 473 | // affine_map<(m, n, k1, k2) -> (m, n)>], |
| 474 | // iterator_types = ["parallel", "parallel", |
| 475 | // "reduction", "reduction"]} |
| 476 | // ins(%A, %B : tensor<10x20x30xf32>, tensor<30x20x40xf32>) |
| 477 | // outs(%C : tensor<10x40xf32>) { |
| 478 | // ^bb0(%a: f32, %b: f32, %c: f32): |
| 479 | // %1 = arith.mulf %a, %b : f32 |
| 480 | // %2 = arith.addf %c, %1 : f32 |
| 481 | // linalg.yield %2 : f32 |
| 482 | // } -> tensor<10x40xf32> |
| 483 | // ``` |
| 484 | // In above contraction, there are two reduction dimensions {k1, k2} |
| 485 | // and although a valid linalg contraction, it is not a named-op |
| 486 | // matrix multiply kind. Therefore, reject multi-dim reduction. |
| 487 | auto res = inferContractionDims(linalgOp: genericOp); |
| 488 | if (!succeeded(Result: res)) |
| 489 | return failure(); |
| 490 | auto dims = *res; |
| 491 | if (dims.m.size() == 2 && dims.n.size() == 2 && dims.k.size() == 2) |
| 492 | return specializeLinalgMmt4D(rewriter, genericOp, castTy, dims); |
| 493 | if (dims.m.size() != 1 || dims.n.size() != 1 || dims.k.size() != 1) |
| 494 | return failure(); |
| 495 | |
| 496 | // Check rank of operands |
| 497 | auto indexingMaps = genericOp.getIndexingMapsArray(); |
| 498 | if (llvm::any_of(Range&: indexingMaps, P: [&dims](AffineMap m) { |
| 499 | return m.getResults().size() != |
| 500 | dims.batch.size() + 2 /* any two of {m,n,k} */; |
| 501 | })) |
| 502 | return failure(); |
| 503 | |
| 504 | auto numOfBatchDims = dims.batch.size(); |
| 505 | if (indexingMaps[0].getNumDims() != numOfBatchDims + 3) |
| 506 | return failure(); |
| 507 | |
| 508 | if (numOfBatchDims) { |
| 509 | // Each operand in a linalg generic contraction could express different |
| 510 | // permutations for its batch dimension. But for named op it must be |
| 511 | // identity since separate maps are not specified. |
| 512 | if (llvm::any_of(Range&: indexingMaps, P: [numOfBatchDims](AffineMap m) { |
| 513 | for (unsigned i = 0; i < numOfBatchDims; ++i) { |
| 514 | auto expr = m.getResults()[i]; |
| 515 | if (expr.getKind() != AffineExprKind::DimId || |
| 516 | cast<AffineDimExpr>(Val&: expr).getPosition() != i) |
| 517 | return true; |
| 518 | } |
| 519 | return false; |
| 520 | })) |
| 521 | return failure(); |
| 522 | } |
| 523 | |
| 524 | auto a = |
| 525 | matchOperandMap(map: indexingMaps[0], rowDimIdx: numOfBatchDims, expectedPosOfRowDim: dims.m[0], expectedPosOfColDim: dims.k[0]); |
| 526 | auto b = |
| 527 | matchOperandMap(map: indexingMaps[1], rowDimIdx: numOfBatchDims, expectedPosOfRowDim: dims.k[0], expectedPosOfColDim: dims.n[0]); |
| 528 | auto c = |
| 529 | matchOperandMap(map: indexingMaps[2], rowDimIdx: numOfBatchDims, expectedPosOfRowDim: dims.m[0], expectedPosOfColDim: dims.n[0]); |
| 530 | |
| 531 | if (llvm::is_contained(Set: {a, b, c}, Element: IndexMatchResult::Mismatch)) |
| 532 | return failure(); |
| 533 | |
| 534 | // Build indexing maps for the named op in its canonical dimension ordering |
| 535 | auto *ctx = genericOp.getContext(); |
| 536 | unsigned numLoopDims = numOfBatchDims + 3; |
| 537 | unsigned mIdx = numOfBatchDims; |
| 538 | unsigned nIdx = mIdx + 1; |
| 539 | unsigned kIdx = mIdx + 2; |
| 540 | |
| 541 | // TODO: add support for indexing_maps with broadcasts. |
| 542 | auto makeMap = [&](IndexMatchResult match, unsigned rowIdx, unsigned colIdx) { |
| 543 | SmallVector<unsigned> tensorDims; |
| 544 | for (unsigned i = 0; i < numOfBatchDims; ++i) |
| 545 | tensorDims.push_back(Elt: i); |
| 546 | if (match == IndexMatchResult::Transposed) |
| 547 | llvm::append_values(C&: tensorDims, Values&: colIdx, Values&: rowIdx); |
| 548 | else |
| 549 | llvm::append_values(C&: tensorDims, Values&: rowIdx, Values&: colIdx); |
| 550 | return AffineMap::getMultiDimMapWithTargets(numDims: numLoopDims, targets: tensorDims, context: ctx); |
| 551 | }; |
| 552 | |
| 553 | auto mapA = makeMap(a, mIdx, kIdx); |
| 554 | auto mapB = makeMap(b, kIdx, nIdx); |
| 555 | auto mapC = makeMap(c, mIdx, nIdx); |
| 556 | |
| 557 | SmallVector<AffineMap> namedOpMaps = {mapA, mapB, mapC}; |
| 558 | |
| 559 | // Codegen the different matmul variants. |
| 560 | if (numOfBatchDims) { |
| 561 | return replaceWithMatmulVariant<BatchMatmulOp>(rewriter, op: genericOp, castTy, |
| 562 | indexingMaps: namedOpMaps); |
| 563 | } |
| 564 | return replaceWithMatmulVariant<MatmulOp>(rewriter, op: genericOp, castTy, |
| 565 | indexingMaps: namedOpMaps); |
| 566 | } |
| 567 | |
| 568 | // Converts linalg.generic to named linalg.*matmul* where possible, falling |
| 569 | // back to the generic `linalg.contract` op otherwise. |
| 570 | static FailureOr<LinalgOp> specializeLinalgContractions(RewriterBase &rewriter, |
| 571 | GenericOp genericOp, |
| 572 | bool emitCategoryOp) { |
| 573 | // Early exit if not projected permutations. |
| 574 | auto mapRange = genericOp.getIndexingMapsArray(); |
| 575 | if (llvm::any_of(Range&: mapRange, |
| 576 | P: [](AffineMap m) { return !m.isProjectedPermutation(); })) |
| 577 | return failure(); |
| 578 | |
| 579 | // Determine the cast type for the named matmul op, or bail out if casts |
| 580 | // cannot be represented by the named op. |
| 581 | std::optional<TypeFn> castTy = getCastTypeForMatmulLikeOp(genericOp); |
| 582 | if (!castTy) |
| 583 | return rewriter.notifyMatchFailure( |
| 584 | arg&: genericOp, msg: "contains invalid cast ops for the named matmul op" ); |
| 585 | |
| 586 | // TODO: When `emitCategoryOp` is set, skip the named-variant matching below |
| 587 | // and go straight to the `linalg.contract` category op. |
| 588 | |
| 589 | // Try to specialize to a specific named matmul variant first. |
| 590 | if (!emitCategoryOp) { |
| 591 | FailureOr<LinalgOp> namedOp = |
| 592 | specializeToNamedContraction(rewriter, genericOp, castTy); |
| 593 | if (succeeded(Result: namedOp)) |
| 594 | return namedOp; |
| 595 | } |
| 596 | |
| 597 | // No named variant matched; fall back to the generic `linalg.contract` op, |
| 598 | // which supports a wider range of variants. |
| 599 | return replaceWithMatmulVariant<ContractOp>(rewriter, op: genericOp, castTy, |
| 600 | indexingMaps: genericOp.getIndexingMapsArray()); |
| 601 | } |
| 602 | |
| 603 | /// Utility to specialize a `genericOp` with a convolution op of type `ConvOpTy` |
| 604 | /// with `dilations` and `strides`. |
| 605 | template <typename ConvOpTy> |
| 606 | static FailureOr<LinalgOp> |
| 607 | specializeToConvOp(RewriterBase &rewriter, GenericOp genericOp, |
| 608 | ArrayRef<int64_t> dilations, ArrayRef<int64_t> strides) { |
| 609 | SmallVector<Value> inputs = genericOp.getDpsInputs(); |
| 610 | ValueRange outputs = genericOp.getDpsInits(); |
| 611 | SmallVector<Type> resultTypes = genericOp.hasPureTensorSemantics() |
| 612 | ? TypeRange(ValueRange(outputs)) |
| 613 | : TypeRange{}; |
| 614 | LinalgOp namedOp; |
| 615 | // Ops with no dilations and no strides. |
| 616 | if constexpr (std::is_same_v<ConvOpTy, linalg::Conv1DOp> || |
| 617 | std::is_same_v<ConvOpTy, linalg::Conv2DOp> || |
| 618 | std::is_same_v<ConvOpTy, linalg::Conv3DOp>) { |
| 619 | namedOp = rewriter.replaceOpWithNewOp<ConvOpTy>(genericOp, resultTypes, |
| 620 | inputs, outputs); |
| 621 | } else { |
| 622 | Attribute stridesAttr = rewriter.getI64TensorAttr(values: strides); |
| 623 | Attribute dilationsAttr = rewriter.getI64TensorAttr(values: dilations); |
| 624 | namedOp = rewriter.replaceOpWithNewOp<ConvOpTy>( |
| 625 | genericOp, resultTypes, inputs, outputs, stridesAttr, dilationsAttr); |
| 626 | } |
| 627 | return namedOp; |
| 628 | } |
| 629 | |
| 630 | /// Converts linalg.generic to named linalg.*conv/pooling* where possible. |
| 631 | static FailureOr<LinalgOp> specializeLinalgConvolutions(RewriterBase &rewriter, |
| 632 | GenericOp genericOp) { |
| 633 | #define CONV_OP_SPECIALIZER(ConvOpTy) \ |
| 634 | if (std::optional<DilationsAndStrides> convParams = \ |
| 635 | matchConvolutionOpOfType<ConvOpTy>(genericOp)) \ |
| 636 | return specializeToConvOp<ConvOpTy>( \ |
| 637 | rewriter, genericOp, convParams->dilations, convParams->strides); \ |
| 638 | // ----------------------------- |
| 639 | // Convolution ops. |
| 640 | // ----------------------------- |
| 641 | CONV_OP_SPECIALIZER(linalg::Conv1DOp); |
| 642 | CONV_OP_SPECIALIZER(linalg::Conv1DNwcWcfOp); |
| 643 | CONV_OP_SPECIALIZER(linalg::Conv1DNcwFcwOp); |
| 644 | CONV_OP_SPECIALIZER(linalg::Conv2DOp); |
| 645 | CONV_OP_SPECIALIZER(linalg::Conv2DNhwcHwcfOp); |
| 646 | CONV_OP_SPECIALIZER(linalg::Conv2DNhwcHwcfQOp); |
| 647 | CONV_OP_SPECIALIZER(linalg::Conv2DNhwcFhwcOp); |
| 648 | CONV_OP_SPECIALIZER(linalg::Conv2DNhwcFhwcQOp); |
| 649 | CONV_OP_SPECIALIZER(linalg::Conv2DNchwFchwOp); |
| 650 | CONV_OP_SPECIALIZER(linalg::Conv2DNchwFchwQOp); |
| 651 | CONV_OP_SPECIALIZER(linalg::Conv2DNgchwFgchwOp); |
| 652 | CONV_OP_SPECIALIZER(linalg::Conv2DNgchwGfchwOp); |
| 653 | CONV_OP_SPECIALIZER(linalg::Conv2DNgchwGfchwQOp); |
| 654 | CONV_OP_SPECIALIZER(linalg::Conv2DNhwgcGfhwcOp); |
| 655 | CONV_OP_SPECIALIZER(linalg::Conv2DNhwgcGfhwcQOp); |
| 656 | CONV_OP_SPECIALIZER(linalg::Conv3DOp); |
| 657 | CONV_OP_SPECIALIZER(linalg::Conv3DNdhwcDhwcfOp); |
| 658 | CONV_OP_SPECIALIZER(linalg::Conv3DNdhwcDhwcfQOp); |
| 659 | CONV_OP_SPECIALIZER(linalg::Conv3DNcdhwFcdhwOp); |
| 660 | // ----------------------------- |
| 661 | // Depthwise Convolution ops. |
| 662 | // ----------------------------- |
| 663 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv1DNcwCwOp); |
| 664 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv1DNwcWcOp); |
| 665 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv1DNwcWcmOp); |
| 666 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv2DNchwChwOp); |
| 667 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv2DNhwcHwcOp); |
| 668 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv2DNhwcHwcQOp); |
| 669 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv2DNhwcHwcmOp); |
| 670 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv2DNhwcHwcmQOp); |
| 671 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv3DNdhwcDhwcOp); |
| 672 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv3DNcdhwCdhwOp); |
| 673 | CONV_OP_SPECIALIZER(linalg::DepthwiseConv3DNdhwcDhwcmOp); |
| 674 | // ----------------------------- |
| 675 | // Pooling ops. |
| 676 | // ----------------------------- |
| 677 | CONV_OP_SPECIALIZER(linalg::PoolingNhwcMaxOp); |
| 678 | CONV_OP_SPECIALIZER(linalg::PoolingNhwcMinOp); |
| 679 | CONV_OP_SPECIALIZER(linalg::PoolingNhwcSumOp); |
| 680 | CONV_OP_SPECIALIZER(linalg::PoolingNhwcMaxUnsignedOp); |
| 681 | CONV_OP_SPECIALIZER(linalg::PoolingNhwcMinUnsignedOp); |
| 682 | CONV_OP_SPECIALIZER(linalg::PoolingNchwSumOp); |
| 683 | CONV_OP_SPECIALIZER(linalg::PoolingNchwMaxOp); |
| 684 | CONV_OP_SPECIALIZER(linalg::PoolingNwcSumOp); |
| 685 | CONV_OP_SPECIALIZER(linalg::PoolingNcwSumOp); |
| 686 | CONV_OP_SPECIALIZER(linalg::PoolingNwcMaxOp); |
| 687 | CONV_OP_SPECIALIZER(linalg::PoolingNwcMaxUnsignedOp); |
| 688 | CONV_OP_SPECIALIZER(linalg::PoolingNcwMaxOp); |
| 689 | CONV_OP_SPECIALIZER(linalg::PoolingNwcMinOp); |
| 690 | CONV_OP_SPECIALIZER(linalg::PoolingNwcMinUnsignedOp); |
| 691 | CONV_OP_SPECIALIZER(linalg::PoolingNdhwcSumOp); |
| 692 | CONV_OP_SPECIALIZER(linalg::PoolingNdhwcMaxOp); |
| 693 | CONV_OP_SPECIALIZER(linalg::PoolingNdhwcMinOp); |
| 694 | #undef CONV_OP_SPECIALIZER |
| 695 | return failure(); |
| 696 | } |
| 697 | |
| 698 | } // namespace |
| 699 | |
| 700 | //===----------------------------------------------------------------------===// |
| 701 | // Categorize linalg generic to named op where possible. |
| 702 | //===----------------------------------------------------------------------===// |
| 703 | FailureOr<LinalgOp> mlir::linalg::specializeGenericOp(RewriterBase &rewriter, |
| 704 | GenericOp genericOp, |
| 705 | bool emitCategoryOps) { |
| 706 | // Elementwise - e.g. exp, add are always category ops |
| 707 | if (isaElemwiseSingleUnaryOpInterface(genericOp) || |
| 708 | isaElemwiseSingleBinaryOpInterface(genericOp) || |
| 709 | isaElemwiseSingleTernaryOpInterface(genericOp)) { |
| 710 | return specializeLinalgElementwise(rewriter, genericOp); |
| 711 | } |
| 712 | |
| 713 | // Contraction - e.g. matmul |
| 714 | if (isaContractionOpInterface(linalgOp: genericOp)) { |
| 715 | return specializeLinalgContractions(rewriter, genericOp, emitCategoryOp: emitCategoryOps); |
| 716 | } |
| 717 | |
| 718 | // Early exit in case of category specialization. |
| 719 | // TODO: Remove when matches for other ops account for both named and |
| 720 | // category. |
| 721 | if (emitCategoryOps) |
| 722 | return rewriter.notifyMatchFailure( |
| 723 | arg&: genericOp, msg: "no matching category op specialization" ); |
| 724 | |
| 725 | // Copy |
| 726 | if (isaCopyOpInterface(linalgOp: genericOp)) { |
| 727 | LinalgOp namedOp = rewriter.replaceOpWithNewOp<CopyOp>( |
| 728 | op: genericOp, args&: genericOp.getDpsInputs()[0], args: genericOp.getDpsInits()[0]); |
| 729 | return namedOp; |
| 730 | } |
| 731 | |
| 732 | // Fill |
| 733 | if (std::optional<Value> fillValue = isaFillOpInterface(genericOp)) { |
| 734 | // Always use the detected fill value, regardless of pattern |
| 735 | LinalgOp namedOp = rewriter.replaceOpWithNewOp<FillOp>( |
| 736 | op: genericOp, args&: *fillValue, args: genericOp.getDpsInits()[0]); |
| 737 | return namedOp; |
| 738 | } |
| 739 | |
| 740 | // Broadcast |
| 741 | std::optional<SmallVector<int64_t>> equivalentToBroadcast = |
| 742 | isaBroadcastOpInterface(linalgOp: genericOp); |
| 743 | if (equivalentToBroadcast) { |
| 744 | auto dims = *equivalentToBroadcast; |
| 745 | LinalgOp namedOp = rewriter.replaceOpWithNewOp<BroadcastOp>( |
| 746 | op: genericOp, args&: genericOp.getDpsInputs()[0], args: genericOp.getDpsInits()[0], |
| 747 | args&: dims); |
| 748 | return namedOp; |
| 749 | } |
| 750 | |
| 751 | // Transpose |
| 752 | std::optional<SmallVector<int64_t>> equivalentToTranspose = |
| 753 | isaTransposeOpInterface(genericOp); |
| 754 | if (equivalentToTranspose) { |
| 755 | auto permutation = *equivalentToTranspose; |
| 756 | LinalgOp namedOp = rewriter.replaceOpWithNewOp<TransposeOp>( |
| 757 | op: genericOp, args&: genericOp.getDpsInputs()[0], args: genericOp.getDpsInits()[0], |
| 758 | args&: permutation); |
| 759 | return namedOp; |
| 760 | } |
| 761 | |
| 762 | // Reduction - e.g. linalg.reduce |
| 763 | if (std::optional<SmallVector<int64_t>> reductionDims = |
| 764 | isaReductionOpInterface(genericOp)) { |
| 765 | return specializeLinalgReductions(rewriter, genericOp, reductionDims: *reductionDims); |
| 766 | } |
| 767 | |
| 768 | // Convolution - e.g. *conv/pooling* |
| 769 | if (isaConvolutionOpInterface(linalgOp: genericOp)) |
| 770 | return specializeLinalgConvolutions(rewriter, genericOp); |
| 771 | |
| 772 | return rewriter.notifyMatchFailure(arg&: genericOp, |
| 773 | msg: "no matching named op specialization" ); |
| 774 | } |
| 775 | |
| 776 | namespace { |
| 777 | struct LinalgSpecializeGenericOpsPass |
| 778 | : public impl::LinalgSpecializeGenericOpsPassBase< |
| 779 | LinalgSpecializeGenericOpsPass> { |
| 780 | |
| 781 | using impl::LinalgSpecializeGenericOpsPassBase< |
| 782 | LinalgSpecializeGenericOpsPass>::LinalgSpecializeGenericOpsPassBase; |
| 783 | void runOnOperation() override; |
| 784 | }; |
| 785 | } // namespace |
| 786 | |
| 787 | void LinalgSpecializeGenericOpsPass::runOnOperation() { |
| 788 | RewritePatternSet patterns(&getContext()); |
| 789 | populateLinalgGenericOpsSpecializationPatterns(patterns); |
| 790 | populateDecomposeProjectedPermutationPatterns(patterns); |
| 791 | |
| 792 | if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns)))) |
| 793 | signalPassFailure(); |
| 794 | } |
| 795 | |
| 796 | void mlir::linalg::populateLinalgGenericOpsSpecializationPatterns( |
| 797 | RewritePatternSet &patterns, bool emitCategoryOps) { |
| 798 | patterns.add<LinalgSpecializationPattern>(arg: patterns.getContext(), |
| 799 | args&: emitCategoryOps); |
| 800 | } |
| 801 | |