1//===- VectorContractToFMA.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Vector/IR/VectorOps.h"
10#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
11#include "mlir/Dialect/X86/Transforms.h"
12#include "mlir/Dialect/X86/X86Dialect.h"
13
14#include "mlir/IR/BuiltinAttributes.h"
15#include "mlir/IR/Dominance.h"
16#include "mlir/IR/PatternMatch.h"
17
18#include "mlir/Pass/Pass.h"
19#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
20
21using namespace mlir;
22using namespace mlir::vector;
23using namespace mlir::x86;
24
25namespace {
26
27// Implements outer product contraction as a sequence of broadcast and
28// FMA operations.
29//
30// For example - for F32 type:
31// ```
32// vector.contract <1x1xf32>, <1x16xf32> into <1x16xf32>
33// ```
34// to
35// ```
36// vector.broadcast %lhs to <16xf32>
37// vector.fma vector<16xf32>
38// ```
39struct VectorContractToFMA : public AddContractionOpRewritePattern {
40 using Base::Base;
41
42 LogicalResult
43 matchAndRewriteAddContractOp(vector::ContractionOp contractOp,
44 PatternRewriter &rewriter) const override {
45
46 VectorType lhsTy = contractOp.getLhsType();
47 if (!lhsTy.getElementType().isF32())
48 return rewriter.notifyMatchFailure(arg&: contractOp,
49 msg: "Only F32 lowering is supported.");
50
51 ArrayRef<int64_t> lhsShape = lhsTy.getShape();
52 llvm::SmallVector<int64_t> nonUnitDimLhs;
53 llvm::copy_if(Range&: lhsShape, Out: std::back_inserter(x&: nonUnitDimLhs),
54 P: [](int64_t dim) { return dim != 1; });
55
56 VectorType rhsTy = contractOp.getRhsType();
57 ArrayRef<int64_t> rhsShape = rhsTy.getShape();
58 llvm::SmallVector<int64_t> nonUnitDimRhs;
59 llvm::copy_if(Range&: rhsShape, Out: std::back_inserter(x&: nonUnitDimRhs),
60 P: [](int64_t dim) { return dim != 1; });
61
62 if (nonUnitDimLhs.size() > 0 && nonUnitDimRhs.size() > 0)
63 return rewriter.notifyMatchFailure(
64 arg&: contractOp, msg: "Excepts unit dimensions for either LHS or RHS shape.");
65
66 if (nonUnitDimLhs.size() != 1 && nonUnitDimRhs.size() != 1)
67 return rewriter.notifyMatchFailure(
68 arg&: contractOp,
69 msg: "Excepts a one non-unit A/B dimension for either LHS or RHS shape.");
70
71 VectorType accTy = dyn_cast<VectorType>(Val: contractOp.getAccType());
72 if (!accTy)
73 return rewriter.notifyMatchFailure(arg&: contractOp,
74 msg: "Accmulator is not a vector type");
75
76 if (!accTy.getElementType().isF32())
77 return rewriter.notifyMatchFailure(arg&: contractOp,
78 msg: "Accmulator should be F32 type.");
79
80 ArrayRef<int64_t> accShape = accTy.getShape();
81 llvm::SmallVector<int64_t> nonUnitDimAcc;
82 llvm::copy_if(Range&: accShape, Out: std::back_inserter(x&: nonUnitDimAcc),
83 P: [](int64_t dim) { return dim != 1; });
84 if (nonUnitDimAcc.size() != 1)
85 return rewriter.notifyMatchFailure(
86 arg&: contractOp, msg: "A or B dimension should be non-unit.");
87
88 // Lowers vector.contract into a broadcast+FMA sequence.
89 auto loc = contractOp.getLoc();
90 auto castAcc = vector::ShapeCastOp::create(
91 builder&: rewriter, location: loc,
92 result: VectorType::get(shape: nonUnitDimAcc.front(), elementType: accTy.getElementType()),
93 source: contractOp.getAcc());
94
95 vector::FMAOp fma;
96
97 // Broadcast the unit-dimension LHS or RHS to match the vector length of the
98 // corresponding non-unit dimension on the other operand. For example,
99 // if LHS has type vector<1x1xf32> and RHS has type vector<1x16xf32>, we
100 // broadcast the LHS to vector<1x16xf32>. In the opposite case (non-unit
101 // dimension on the LHS), we broadcast the RHS instead.
102 if (nonUnitDimRhs.size() > 0) {
103 auto castLhs = vector::ShapeCastOp::create(
104 builder&: rewriter, location: loc, result: VectorType::get(shape: 1, elementType: lhsTy.getElementType()),
105 source: contractOp.getLhs());
106 auto castRhs = vector::ShapeCastOp::create(
107 builder&: rewriter, location: loc,
108 result: VectorType::get(shape: nonUnitDimRhs.front(), elementType: rhsTy.getElementType()),
109 source: contractOp.getRhs());
110 auto broadcastLhs = vector::BroadcastOp::create(
111 builder&: rewriter, location: loc, result: castRhs.getResult().getType(), source: castLhs);
112 fma =
113 vector::FMAOp::create(builder&: rewriter, location: loc, lhs: broadcastLhs, rhs: castRhs, acc: castAcc);
114 } else {
115 auto castLhs = vector::ShapeCastOp::create(
116 builder&: rewriter, location: loc,
117 result: VectorType::get(shape: nonUnitDimLhs.front(), elementType: lhsTy.getElementType()),
118 source: contractOp.getLhs());
119 auto castRhs = vector::ShapeCastOp::create(
120 builder&: rewriter, location: loc, result: VectorType::get(shape: 1, elementType: rhsTy.getElementType()),
121 source: contractOp.getRhs());
122 auto broadcastRhs = vector::BroadcastOp::create(
123 builder&: rewriter, location: loc, result: castLhs.getResult().getType(), source: castRhs);
124 fma =
125 vector::FMAOp::create(builder&: rewriter, location: loc, lhs: castLhs, rhs: broadcastRhs, acc: castAcc);
126 }
127
128 auto castFma = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accTy, source: fma);
129 rewriter.replaceOp(op: contractOp, newOp: castFma);
130
131 return success();
132 }
133};
134} // namespace
135
136void x86::populateVectorContractToFMAPatterns(RewritePatternSet &patterns) {
137 patterns.add<VectorContractToFMA>(arg: patterns.getContext());
138}
139