1//===- MoveAccumulatorForContractLoop.cpp ---------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/SCF/IR/SCF.h"
10#include "mlir/Dialect/Vector/IR/VectorOps.h"
11#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
12#include "mlir/Dialect/X86/Transforms.h"
13#include "mlir/Dialect/X86/Utils/X86Utils.h"
14#include "mlir/Dialect/X86/X86Dialect.h"
15
16#include "mlir/IR/BuiltinAttributes.h"
17#include "mlir/IR/Dominance.h"
18#include "mlir/IR/PatternMatch.h"
19#include "llvm/Support/Casting.h"
20
21#include "mlir/Pass/Pass.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23
24using namespace mlir;
25using namespace mlir::vector;
26using namespace mlir::x86;
27
28namespace {
29// Transforms vector.contract(A, B, Acc) into vector.contract(A, B, 0) + Acc
30// to decouple the contraction computation from the accumulator update.
31struct MoveAccumulatorForContractLoop : public AddContractionOpRewritePattern {
32 using AddContractionOpRewritePattern::AddContractionOpRewritePattern;
33
34 LogicalResult
35 matchAndRewriteAddContractOp(vector::ContractionOp contractOp,
36 PatternRewriter &rewriter) const override {
37
38 Operation *accReadOp =
39 traceToVectorReadLikeParentOperation(v: contractOp.getAcc());
40
41 Value contractValue = contractionUsersAfterYield(v: contractOp.getResult());
42
43 if (!contractValue)
44 return rewriter.notifyMatchFailure(
45 arg&: contractOp, msg: "Final acc write might have multiple users.");
46
47 Operation *resultUserOp = *contractValue.getUsers().begin();
48
49 if (!accReadOp || !resultUserOp)
50 return rewriter.notifyMatchFailure(
51 arg&: contractOp, msg: "Read from acc matrix is not by "
52 "transfer_read/load/constant_zero or multiple users of "
53 "contract operation.");
54
55 if (isa<arith::ConstantOp>(Val: accReadOp))
56 return rewriter.notifyMatchFailure(
57 arg&: contractOp,
58 msg: "The input acc to contract is already a constant vector.");
59
60 if ((accReadOp->getBlock() == contractOp->getBlock()) ||
61 (resultUserOp->getBlock() == contractOp->getBlock()))
62 return rewriter.notifyMatchFailure(
63 arg&: contractOp, msg: "Acc read/write should be in a separate block.");
64
65 // Replace acc of a contraction operation with vector constant.
66 Value accValue = accReadOp->getResult(idx: 0);
67 if (!accValue.hasOneUse())
68 return rewriter.notifyMatchFailure(
69 arg&: contractOp, msg: "The input accumulator has multiple users.");
70
71 Operation *firstUser = *accValue.getUsers().begin();
72 rewriter.setInsertionPoint(firstUser);
73
74 auto vecTy = dyn_cast<VectorType>(Val: accValue.getType());
75 if (!vecTy)
76 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Expects vector type.");
77
78 Location loc = accReadOp->getLoc();
79 Type elemTy = vecTy.getElementType();
80
81 Value zeroVec = arith::ConstantOp::create(
82 builder&: rewriter, location: loc,
83 value: DenseElementsAttr::get(type: vecTy, values: rewriter.getZeroAttr(type: elemTy)));
84
85 rewriter.replaceAllUsesWith(from: accValue, to: zeroVec);
86
87 // Adds the initial acc value with contract results before storing to acc
88 // matrix.
89 rewriter.setInsertionPoint(resultUserOp);
90 Location locUser = resultUserOp->getLoc();
91
92 Value addition;
93
94 if (isa<FloatType>(Val: elemTy)) {
95 addition =
96 arith::AddFOp::create(builder&: rewriter, location: locUser, lhs: contractValue, rhs: accValue);
97 } else if (isa<IntegerType>(Val: elemTy)) {
98 addition =
99 arith::AddIOp::create(builder&: rewriter, location: locUser, lhs: contractValue, rhs: accValue);
100 } else {
101 llvm_unreachable("expected floating-point or integer element type");
102 }
103
104 rewriter.modifyOpInPlace(root: resultUserOp, callable: [&]() {
105 resultUserOp->replaceUsesOfWith(from: contractValue, to: addition);
106 });
107 return success();
108 }
109};
110} // namespace
111
112void x86::populateMoveAccumulatorForContractLoopPatterns(
113 RewritePatternSet &patterns) {
114 patterns.add<MoveAccumulatorForContractLoop>(arg: patterns.getContext());
115}
116