1//===- SincosFusion.cpp - Fuse sin/cos into sincos -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Math/IR/Math.h"
10#include "mlir/Dialect/Math/Transforms/Passes.h"
11#include "mlir/IR/PatternMatch.h"
12#include "llvm/ADT/DenseSet.h"
13#include "llvm/ADT/SmallVector.h"
14
15using namespace mlir;
16using namespace mlir::math;
17
18namespace mlir::math {
19#define GEN_PASS_DEF_MATHSINCOSFUSIONPASS
20#include "mlir/Dialect/Math/Transforms/Passes.h.inc"
21} // namespace mlir::math
22
23namespace {
24
25/// A math.sin and a math.cos in the same block, on the same operand and with
26/// identical fastmath flags, that can be replaced by a single math.sincos.
27struct SincosPair {
28 math::SinOp sinOp;
29 math::CosOp cosOp;
30 /// Whichever of the two comes first in the block; the math.sincos is
31 /// inserted there so that its results dominate both uses.
32 Operation *firstOp;
33};
34
35/// Find the math.cos that should be fused with `sinOp`: the earliest one in the
36/// same block that uses the same operand with the same fastmath flags and has
37/// not already been paired with another math.sin.
38static math::CosOp
39findFusionCandidate(math::SinOp sinOp,
40 const llvm::DenseSet<Operation *> &pairedCosOps) {
41 Value operand = sinOp.getOperand();
42 arith::FastMathFlags sinFastMathFlags = sinOp.getFastmath();
43 Block *block = sinOp->getBlock();
44
45 math::CosOp candidate = nullptr;
46 for (Operation *user : operand.getUsers()) {
47 auto cosOp = dyn_cast<math::CosOp>(Val: user);
48 if (!cosOp || cosOp->getBlock() != block)
49 continue;
50 if (cosOp.getFastmath() != sinFastMathFlags)
51 continue;
52 if (pairedCosOps.contains(V: cosOp))
53 continue;
54 // The operand use list is not in program order, so keep the earliest
55 // candidate to make the choice independent of use list order.
56 if (!candidate || cosOp->isBeforeInBlock(other: candidate))
57 candidate = cosOp;
58 }
59 return candidate;
60}
61
62struct MathSincosFusionPass final
63 : math::impl::MathSincosFusionPassBase<MathSincosFusionPass> {
64 using MathSincosFusionPassBase::MathSincosFusionPassBase;
65
66 void runOnOperation() override {
67 // Collect the pairs before touching the IR: fusing erases the math.cos,
68 // which may be the operation the walk is about to visit next.
69 llvm::SmallVector<SincosPair> pairs;
70 llvm::DenseSet<Operation *> pairedCosOps;
71 getOperation()->walk(callback: [&](math::SinOp sinOp) {
72 math::CosOp cosOp = findFusionCandidate(sinOp, pairedCosOps);
73 if (!cosOp)
74 return;
75 pairedCosOps.insert(V: cosOp);
76 Operation *firstOp = sinOp->isBeforeInBlock(other: cosOp) ? sinOp.getOperation()
77 : cosOp.getOperation();
78 pairs.push_back(Elt: {.sinOp: sinOp, .cosOp: cosOp, .firstOp: firstOp});
79 });
80
81 IRRewriter rewriter(&getContext());
82 for (SincosPair &pair : pairs) {
83 rewriter.setInsertionPoint(pair.firstOp);
84 Type elemType = pair.sinOp.getType();
85 auto sincos = math::SincosOp::create(
86 builder&: rewriter, location: pair.firstOp->getLoc(), resultTypes: TypeRange{elemType, elemType},
87 operand: pair.sinOp.getOperand(), fastmath: pair.sinOp.getFastmathAttr());
88 rewriter.replaceOp(op: pair.sinOp, newValues: sincos.getSin());
89 rewriter.replaceOp(op: pair.cosOp, newValues: sincos.getCos());
90 }
91 }
92};
93
94} // namespace
95