1//===-- MathToLibm.cpp - conversion from Math to libm calls ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/MathToLibm/MathToLibm.h"
10
11#include "mlir/Dialect/Arith/IR/Arith.h"
12#include "mlir/Dialect/Func/IR/FuncOps.h"
13#include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h"
14#include "mlir/Dialect/Math/IR/Math.h"
15#include "mlir/Dialect/Utils/IndexingUtils.h"
16#include "mlir/Dialect/Vector/IR/VectorOps.h"
17#include "mlir/IR/BuiltinDialect.h"
18#include "mlir/IR/PatternMatch.h"
19#include "mlir/Transforms/DialectConversion.h"
20#include "llvm/ADT/SmallVectorExtras.h"
21
22namespace mlir {
23#define GEN_PASS_DEF_CONVERTMATHTOLIBMPASS
24#include "mlir/Conversion/Passes.h.inc"
25} // namespace mlir
26
27using namespace mlir;
28
29namespace {
30// Pattern to convert vector operations to scalar operations. This is needed as
31// libm calls require scalars.
32template <typename Op>
33struct VecOpToScalarOp : public OpRewritePattern<Op> {
34public:
35 using OpRewritePattern<Op>::OpRewritePattern;
36
37 LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final;
38};
39// Pattern to promote an op of a smaller floating point type to F32.
40template <typename Op>
41struct PromoteOpToF32 : public OpRewritePattern<Op> {
42public:
43 using OpRewritePattern<Op>::OpRewritePattern;
44
45 LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final;
46};
47// Pattern to convert scalar math operations to calls to libm functions.
48// Additionally the libm function signatures are declared.
49template <typename Op>
50struct ScalarOpToLibmCall : public OpRewritePattern<Op> {
51public:
52 using OpRewritePattern<Op>::OpRewritePattern;
53 ScalarOpToLibmCall(MLIRContext *context, PatternBenefit benefit,
54 StringRef floatFunc, StringRef doubleFunc)
55 : OpRewritePattern<Op>(context, benefit), floatFunc(floatFunc),
56 doubleFunc(doubleFunc) {};
57
58 LogicalResult matchAndRewrite(Op op, PatternRewriter &rewriter) const final;
59
60private:
61 std::string floatFunc, doubleFunc;
62};
63
64template <typename OpTy>
65void populatePatternsForOp(RewritePatternSet &patterns, PatternBenefit benefit,
66 MLIRContext *ctx, StringRef floatFunc,
67 StringRef doubleFunc) {
68 patterns.add<VecOpToScalarOp<OpTy>, PromoteOpToF32<OpTy>>(ctx, benefit);
69 patterns.add<ScalarOpToLibmCall<OpTy>>(ctx, benefit, floatFunc, doubleFunc);
70}
71
72} // namespace
73
74template <typename Op>
75LogicalResult
76VecOpToScalarOp<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
77 auto opType = op.getType();
78 auto loc = op.getLoc();
79 auto vecType = dyn_cast<VectorType>(opType);
80
81 if (!vecType)
82 return failure();
83 if (!vecType.hasRank())
84 return failure();
85 auto shape = vecType.getShape();
86 int64_t numElements = vecType.getNumElements();
87
88 Value result = arith::ConstantOp::create(
89 rewriter, loc,
90 DenseElementsAttr::get(vecType,
91 FloatAttr::get(vecType.getElementType(), 0.0)));
92 SmallVector<int64_t> strides = computeStrides(shape);
93 for (auto linearIndex = 0; linearIndex < numElements; ++linearIndex) {
94 SmallVector<int64_t> positions = delinearize(linearIndex, strides);
95 SmallVector<Value> operands;
96 for (auto input : op->getOperands())
97 operands.push_back(
98 Elt: vector::ExtractOp::create(rewriter, loc, input, positions));
99 Value scalarOp = Op::create(
100 rewriter, loc, TypeRange{vecType.getElementType()}, operands,
101 op.getProperties(), op->getDiscardableAttrDictionary().getValue());
102 result =
103 vector::InsertOp::create(rewriter, loc, scalarOp, result, positions);
104 }
105 rewriter.replaceOp(op, {result});
106 return success();
107}
108
109template <typename Op>
110LogicalResult
111PromoteOpToF32<Op>::matchAndRewrite(Op op, PatternRewriter &rewriter) const {
112 auto opType = op.getType();
113 if (!isa<Float16Type, BFloat16Type>(opType))
114 return failure();
115
116 auto loc = op.getLoc();
117 auto f32 = rewriter.getF32Type();
118 auto extendedOperands =
119 llvm::map_to_vector(op->getOperands(), [&](Value operand) -> Value {
120 return arith::ExtFOp::create(rewriter, loc, TypeRange{f32},
121 ValueRange{operand},
122 arith::ExtFOp::Properties{});
123 });
124 auto newOp = Op::create(rewriter, loc, TypeRange{f32}, extendedOperands,
125 op.getProperties(),
126 op->getDiscardableAttrDictionary().getValue());
127 rewriter.replaceOpWithNewOp<arith::TruncFOp>(op, opType, newOp);
128 return success();
129}
130
131template <typename Op>
132LogicalResult
133ScalarOpToLibmCall<Op>::matchAndRewrite(Op op,
134 PatternRewriter &rewriter) const {
135 auto module = SymbolTable::getNearestSymbolTable(from: op);
136 auto type = op.getType();
137 if (!isa<Float32Type, Float64Type>(type))
138 return failure();
139
140 auto name = type.getIntOrFloatBitWidth() == 64 ? doubleFunc : floatFunc;
141 auto opFunc = dyn_cast_or_null<SymbolOpInterface>(
142 SymbolTable::lookupSymbolIn(module, name));
143 // Forward declare function if it hasn't already been
144 if (!opFunc) {
145 OpBuilder::InsertionGuard guard(rewriter);
146 rewriter.setInsertionPointToStart(&module->getRegion(0).front());
147 auto opFunctionTy = FunctionType::get(
148 context: rewriter.getContext(), inputs: op->getOperandTypes(), results: op->getResultTypes());
149 opFunc = func::FuncOp::create(rewriter, rewriter.getUnknownLoc(), name,
150 opFunctionTy);
151 opFunc.setPrivate();
152
153 // By definition Math dialect operations imply LLVM's "readnone"
154 // function attribute, so we can set it here to provide more
155 // optimization opportunities (e.g. LICM) for backends targeting LLVM IR.
156 // This will have to be changed, when strict FP behavior is supported
157 // by Math dialect.
158 opFunc->setDiscardableAttr(LLVM::LLVMDialect::getReadnoneAttrName(),
159 UnitAttr::get(context: rewriter.getContext()));
160 }
161 assert(isa<FunctionOpInterface>(SymbolTable::lookupSymbolIn(module, name)));
162
163 rewriter.replaceOpWithNewOp<func::CallOp>(op, name, op.getType(),
164 op->getOperands());
165
166 return success();
167}
168
169void mlir::populateMathToLibmConversionPatterns(RewritePatternSet &patterns,
170 PatternBenefit benefit) {
171 MLIRContext *ctx = patterns.getContext();
172
173 populatePatternsForOp<math::AbsFOp>(patterns, benefit, ctx, floatFunc: "fabsf", doubleFunc: "fabs");
174 populatePatternsForOp<math::AcosOp>(patterns, benefit, ctx, floatFunc: "acosf", doubleFunc: "acos");
175 populatePatternsForOp<math::AcoshOp>(patterns, benefit, ctx, floatFunc: "acoshf",
176 doubleFunc: "acosh");
177 populatePatternsForOp<math::AsinOp>(patterns, benefit, ctx, floatFunc: "asinf", doubleFunc: "asin");
178 populatePatternsForOp<math::AsinhOp>(patterns, benefit, ctx, floatFunc: "asinhf",
179 doubleFunc: "asinh");
180 populatePatternsForOp<math::Atan2Op>(patterns, benefit, ctx, floatFunc: "atan2f",
181 doubleFunc: "atan2");
182 populatePatternsForOp<math::AtanOp>(patterns, benefit, ctx, floatFunc: "atanf", doubleFunc: "atan");
183 populatePatternsForOp<math::AtanhOp>(patterns, benefit, ctx, floatFunc: "atanhf",
184 doubleFunc: "atanh");
185 populatePatternsForOp<math::CbrtOp>(patterns, benefit, ctx, floatFunc: "cbrtf", doubleFunc: "cbrt");
186 populatePatternsForOp<math::CeilOp>(patterns, benefit, ctx, floatFunc: "ceilf", doubleFunc: "ceil");
187 populatePatternsForOp<math::CosOp>(patterns, benefit, ctx, floatFunc: "cosf", doubleFunc: "cos");
188 populatePatternsForOp<math::CoshOp>(patterns, benefit, ctx, floatFunc: "coshf", doubleFunc: "cosh");
189 populatePatternsForOp<math::ErfOp>(patterns, benefit, ctx, floatFunc: "erff", doubleFunc: "erf");
190 populatePatternsForOp<math::ErfcOp>(patterns, benefit, ctx, floatFunc: "erfcf", doubleFunc: "erfc");
191 populatePatternsForOp<math::ExpOp>(patterns, benefit, ctx, floatFunc: "expf", doubleFunc: "exp");
192 populatePatternsForOp<math::Exp2Op>(patterns, benefit, ctx, floatFunc: "exp2f", doubleFunc: "exp2");
193 populatePatternsForOp<math::ExpM1Op>(patterns, benefit, ctx, floatFunc: "expm1f",
194 doubleFunc: "expm1");
195 populatePatternsForOp<math::FloorOp>(patterns, benefit, ctx, floatFunc: "floorf",
196 doubleFunc: "floor");
197 populatePatternsForOp<math::FmaOp>(patterns, benefit, ctx, floatFunc: "fmaf", doubleFunc: "fma");
198 populatePatternsForOp<math::LogOp>(patterns, benefit, ctx, floatFunc: "logf", doubleFunc: "log");
199 populatePatternsForOp<math::Log2Op>(patterns, benefit, ctx, floatFunc: "log2f", doubleFunc: "log2");
200 populatePatternsForOp<math::Log10Op>(patterns, benefit, ctx, floatFunc: "log10f",
201 doubleFunc: "log10");
202 populatePatternsForOp<math::Log1pOp>(patterns, benefit, ctx, floatFunc: "log1pf",
203 doubleFunc: "log1p");
204 populatePatternsForOp<math::PowFOp>(patterns, benefit, ctx, floatFunc: "powf", doubleFunc: "pow");
205 populatePatternsForOp<math::RoundEvenOp>(patterns, benefit, ctx, floatFunc: "roundevenf",
206 doubleFunc: "roundeven");
207 populatePatternsForOp<math::RoundOp>(patterns, benefit, ctx, floatFunc: "roundf",
208 doubleFunc: "round");
209 populatePatternsForOp<math::SinOp>(patterns, benefit, ctx, floatFunc: "sinf", doubleFunc: "sin");
210 populatePatternsForOp<math::SinhOp>(patterns, benefit, ctx, floatFunc: "sinhf", doubleFunc: "sinh");
211 populatePatternsForOp<math::SqrtOp>(patterns, benefit, ctx, floatFunc: "sqrtf", doubleFunc: "sqrt");
212 populatePatternsForOp<math::RsqrtOp>(patterns, benefit, ctx, floatFunc: "rsqrtf",
213 doubleFunc: "rsqrt");
214 populatePatternsForOp<math::TanOp>(patterns, benefit, ctx, floatFunc: "tanf", doubleFunc: "tan");
215 populatePatternsForOp<math::TanhOp>(patterns, benefit, ctx, floatFunc: "tanhf", doubleFunc: "tanh");
216 populatePatternsForOp<math::TruncOp>(patterns, benefit, ctx, floatFunc: "truncf",
217 doubleFunc: "trunc");
218}
219
220namespace {
221struct ConvertMathToLibmPass
222 : public impl::ConvertMathToLibmPassBase<ConvertMathToLibmPass> {
223 void runOnOperation() override;
224};
225} // namespace
226
227void ConvertMathToLibmPass::runOnOperation() {
228 auto module = getOperation();
229
230 RewritePatternSet patterns(&getContext());
231 populateMathToLibmConversionPatterns(patterns);
232
233 ConversionTarget target(getContext());
234 target.addLegalDialect<arith::ArithDialect, BuiltinDialect, func::FuncDialect,
235 vector::VectorDialect>();
236 target.addIllegalDialect<math::MathDialect>();
237 if (failed(Result: applyPartialConversion(op: module, target, patterns: std::move(patterns))))
238 signalPassFailure();
239}
240