1//===- MathToLLVM.cpp - Math to LLVM dialect conversion -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/MathToLLVM/MathToLLVM.h"
10
11#include "mlir/Conversion/ArithAttrToLLVM/AttrToLLVMConverter.h"
12#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
13#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
14#include "mlir/Conversion/LLVMCommon/Pattern.h"
15#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
16#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMConstrainedIntrinsicOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMIntegerIntrinsicOps.h"
22#include "mlir/Dialect/Math/IR/Math.h"
23#include "mlir/IR/Matchers.h"
24#include "mlir/IR/TypeUtilities.h"
25#include "mlir/Pass/Pass.h"
26
27#include "llvm/ADT/FloatingPointMode.h"
28
29namespace mlir {
30#define GEN_PASS_DEF_CONVERTMATHTOLLVMPASS
31#include "mlir/Conversion/Passes.h.inc"
32} // namespace mlir
33
34using namespace mlir;
35
36namespace {
37
38template <typename SourceOp, typename TargetOp>
39using ConvertFastMath = arith::AttrConvertFastMathToLLVM<SourceOp, TargetOp>;
40
41template <typename SourceOp, typename TargetOp, bool FailOnUnsupportedFP = true>
42using ConvertFMFMathToLLVMPattern =
43 VectorConvertToLLVMPattern<SourceOp, TargetOp, ConvertFastMath,
44 FailOnUnsupportedFP>;
45
46/// Lowering pattern that matches only when the source op's rounding mode
47/// presence agrees with `HasRoundingMode`. Mirrors the helper of the same
48/// name in `mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp`. This lets us
49/// register two patterns for one math op: an unconstrained one that lowers
50/// to a regular LLVM op, and a constrained one (rounding mode present) that
51/// lowers to an `llvm.intr.experimental.constrained.*` intrinsic.
52template <typename SourceOp, typename TargetOp, bool HasRoundingMode,
53 template <typename, typename> typename AttrConvert =
54 AttrConvertPassThrough,
55 bool FailOnUnsupportedFP = true>
56struct ConstrainedVectorConvertToLLVMPattern
57 : public VectorConvertToLLVMPattern<SourceOp, TargetOp, AttrConvert,
58 FailOnUnsupportedFP> {
59 using VectorConvertToLLVMPattern<
60 SourceOp, TargetOp, AttrConvert,
61 FailOnUnsupportedFP>::VectorConvertToLLVMPattern;
62
63 LogicalResult
64 matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor,
65 ConversionPatternRewriter &rewriter) const override {
66 if (HasRoundingMode != static_cast<bool>(op.getRoundingModeAttr()))
67 return failure();
68 return VectorConvertToLLVMPattern<
69 SourceOp, TargetOp, AttrConvert,
70 FailOnUnsupportedFP>::matchAndRewrite(op, adaptor, rewriter);
71 }
72};
73
74using AbsFOpLowering =
75 ConvertFMFMathToLLVMPattern<math::AbsFOp, LLVM::FAbsOp,
76 /*FailOnUnsupportedFP=*/true>;
77using CeilOpLowering = ConvertFMFMathToLLVMPattern<math::CeilOp, LLVM::FCeilOp>;
78using CopySignOpLowering =
79 ConvertFMFMathToLLVMPattern<math::CopySignOp, LLVM::CopySignOp>;
80using CosOpLowering = ConvertFMFMathToLLVMPattern<math::CosOp, LLVM::CosOp>;
81using CoshOpLowering = ConvertFMFMathToLLVMPattern<math::CoshOp, LLVM::CoshOp>;
82using AcosOpLowering = ConvertFMFMathToLLVMPattern<math::AcosOp, LLVM::ACosOp>;
83using CtPopFOpLowering =
84 VectorConvertToLLVMPattern<math::CtPopOp, LLVM::CtPopOp,
85 AttrConvertPassThrough,
86 /*FailOnUnsupportedFP=*/true>;
87using Exp2OpLowering = ConvertFMFMathToLLVMPattern<math::Exp2Op, LLVM::Exp2Op>;
88using ExpOpLowering = ConvertFMFMathToLLVMPattern<math::ExpOp, LLVM::ExpOp>;
89using FloorOpLowering =
90 ConvertFMFMathToLLVMPattern<math::FloorOp, LLVM::FFloorOp>;
91using FmaOpLowering =
92 ConstrainedVectorConvertToLLVMPattern<math::FmaOp, LLVM::FMAOp,
93 /*HasRoundingMode=*/false,
94 ConvertFastMath,
95 /*FailOnUnsupportedFP=*/true>;
96using ConstrainedFmaOpLowering = ConstrainedVectorConvertToLLVMPattern<
97 math::FmaOp, LLVM::ConstrainedFMAIntr, /*HasRoundingMode=*/true,
98 arith::AttrConverterConstrainedFPToLLVM, /*FailOnUnsupportedFP=*/true>;
99using Log10OpLowering =
100 ConvertFMFMathToLLVMPattern<math::Log10Op, LLVM::Log10Op>;
101using Log2OpLowering = ConvertFMFMathToLLVMPattern<math::Log2Op, LLVM::Log2Op>;
102using LogOpLowering = ConvertFMFMathToLLVMPattern<math::LogOp, LLVM::LogOp>;
103using PowFOpLowering = ConvertFMFMathToLLVMPattern<math::PowFOp, LLVM::PowOp>;
104using RoundEvenOpLowering =
105 ConvertFMFMathToLLVMPattern<math::RoundEvenOp, LLVM::RoundEvenOp>;
106using RoundOpLowering =
107 ConvertFMFMathToLLVMPattern<math::RoundOp, LLVM::RoundOp>;
108using SinOpLowering = ConvertFMFMathToLLVMPattern<math::SinOp, LLVM::SinOp>;
109using SinhOpLowering = ConvertFMFMathToLLVMPattern<math::SinhOp, LLVM::SinhOp>;
110using ASinOpLowering = ConvertFMFMathToLLVMPattern<math::AsinOp, LLVM::ASinOp>;
111using SqrtOpLowering = ConvertFMFMathToLLVMPattern<math::SqrtOp, LLVM::SqrtOp>;
112using FTruncOpLowering =
113 ConvertFMFMathToLLVMPattern<math::TruncOp, LLVM::FTruncOp>;
114using TanOpLowering = ConvertFMFMathToLLVMPattern<math::TanOp, LLVM::TanOp>;
115using TanhOpLowering = ConvertFMFMathToLLVMPattern<math::TanhOp, LLVM::TanhOp>;
116using ATanOpLowering = ConvertFMFMathToLLVMPattern<math::AtanOp, LLVM::ATanOp>;
117using ATan2OpLowering =
118 ConvertFMFMathToLLVMPattern<math::Atan2Op, LLVM::ATan2Op>;
119// A `CtLz/CtTz/absi(a)` is converted into `CtLz/CtTz/absi(a, false)`.
120// TODO: Result and operand types match for `absi` as opposed to `ct*z`, so it
121// may be better to separate the patterns.
122template <typename MathOp, typename LLVMOp>
123struct IntOpWithFlagLowering
124 : public ConvertOpToLLVMPattern<MathOp, /*FailOnUnsupportedFP=*/true> {
125 using ConvertOpToLLVMPattern<
126 MathOp, /*FailOnUnsupportedFP=*/true>::ConvertOpToLLVMPattern;
127 using Super = IntOpWithFlagLowering<MathOp, LLVMOp>;
128
129 LogicalResult
130 matchAndRewrite(MathOp op, typename MathOp::Adaptor adaptor,
131 ConversionPatternRewriter &rewriter) const override {
132 const auto &typeConverter = *this->getTypeConverter();
133 auto operandType = adaptor.getOperand().getType();
134 auto llvmOperandType = typeConverter.convertType(operandType);
135 if (!llvmOperandType)
136 return failure();
137
138 auto loc = op.getLoc();
139 auto resultType = op.getResult().getType();
140 auto llvmResultType = typeConverter.convertType(resultType);
141 if (!llvmResultType)
142 return failure();
143
144 if (!isa<LLVM::LLVMArrayType>(llvmOperandType)) {
145 rewriter.replaceOpWithNewOp<LLVMOp>(op, llvmResultType,
146 adaptor.getOperand(), false);
147 return success();
148 }
149
150 if (!isa<VectorType>(resultType))
151 return failure();
152
153 return LLVM::detail::handleMultidimensionalVectors(
154 op: op.getOperation(), operands: adaptor.getOperands(), typeConverter,
155 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) {
156 return LLVMOp::create(rewriter, loc, llvm1DVectorTy, operands[0],
157 false);
158 },
159 rewriter);
160 }
161};
162
163using CountLeadingZerosOpLowering =
164 IntOpWithFlagLowering<math::CountLeadingZerosOp, LLVM::CountLeadingZerosOp>;
165using CountTrailingZerosOpLowering =
166 IntOpWithFlagLowering<math::CountTrailingZerosOp,
167 LLVM::CountTrailingZerosOp>;
168using AbsIOpLowering = IntOpWithFlagLowering<math::AbsIOp, LLVM::AbsOp>;
169
170// A `sincos` is converted into `llvm.intr.sincos` followed by extractvalue ops.
171struct SincosOpLowering
172 : public ConvertOpToLLVMPattern<math::SincosOp,
173 /*FailOnUnsupportedFP=*/true> {
174 using ConvertOpToLLVMPattern<
175 math::SincosOp, /*FailOnUnsupportedFP=*/true>::ConvertOpToLLVMPattern;
176
177 LogicalResult
178 matchAndRewrite(math::SincosOp op, OpAdaptor adaptor,
179 ConversionPatternRewriter &rewriter) const override {
180 const LLVMTypeConverter &typeConverter = *this->getTypeConverter();
181 mlir::Location loc = op.getLoc();
182 mlir::Type operandType = adaptor.getOperand().getType();
183 mlir::Type llvmOperandType = typeConverter.convertType(t: operandType);
184 mlir::Type sinType = typeConverter.convertType(t: op.getSin().getType());
185 mlir::Type cosType = typeConverter.convertType(t: op.getCos().getType());
186 if (!llvmOperandType || !sinType || !cosType)
187 return failure();
188
189 ConvertFastMath<math::SincosOp, LLVM::SincosOp> attrs(op);
190
191 auto structType = LLVM::LLVMStructType::getLiteral(
192 context: rewriter.getContext(), types: {llvmOperandType, llvmOperandType});
193
194 auto sincosOp = LLVM::SincosOp::create(
195 builder&: rewriter, location: loc, resultTypes: TypeRange{structType}, operands: ValueRange{adaptor.getOperand()},
196 properties: attrs.getProperties(), discardableAttributes: attrs.getDiscardableAttrs());
197
198 auto sinValue = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: sincosOp, position: 0);
199 auto cosValue = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: sincosOp, position: 1);
200
201 rewriter.replaceOp(op, newValues: {sinValue, cosValue});
202 return success();
203 }
204};
205
206// A `expm1` is converted into `exp - 1`.
207struct ExpM1OpLowering
208 : public ConvertOpToLLVMPattern<math::ExpM1Op,
209 /*FailOnUnsupportedFP=*/true> {
210 using ConvertOpToLLVMPattern<
211 math::ExpM1Op, /*FailOnUnsupportedFP=*/true>::ConvertOpToLLVMPattern;
212
213 LogicalResult
214 matchAndRewrite(math::ExpM1Op op, OpAdaptor adaptor,
215 ConversionPatternRewriter &rewriter) const override {
216 const auto &typeConverter = *this->getTypeConverter();
217 auto operandType = adaptor.getOperand().getType();
218 auto llvmOperandType = typeConverter.convertType(t: operandType);
219 if (!llvmOperandType)
220 return failure();
221
222 auto loc = op.getLoc();
223 auto resultType = op.getResult().getType();
224 auto floatType = cast<FloatType>(
225 Val: typeConverter.convertType(t: getElementTypeOrSelf(type: resultType)));
226 auto floatOne = rewriter.getFloatAttr(type: floatType, value: 1.0);
227 ConvertFastMath<math::ExpM1Op, LLVM::ExpOp> expAttrs(op);
228 ConvertFastMath<math::ExpM1Op, LLVM::FSubOp> subAttrs(op);
229
230 if (!isa<LLVM::LLVMArrayType>(Val: llvmOperandType)) {
231 LLVM::ConstantOp one;
232 if (LLVM::isCompatibleVectorType(type: llvmOperandType)) {
233 one = LLVM::ConstantOp::create(
234 builder&: rewriter, location: loc, res: llvmOperandType,
235 value: SplatElementsAttr::get(type: cast<ShapedType>(Val&: llvmOperandType),
236 values: floatOne));
237 } else {
238 one =
239 LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmOperandType, value: floatOne);
240 }
241 auto exp = LLVM::ExpOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{llvmOperandType},
242 operands: ValueRange{adaptor.getOperand()},
243 properties: expAttrs.getProperties(),
244 discardableAttributes: expAttrs.getDiscardableAttrs());
245 rewriter.replaceOpWithNewOp<LLVM::FSubOp>(
246 op, args: TypeRange{llvmOperandType}, args: ValueRange{exp, one},
247 args: subAttrs.getProperties(), args: subAttrs.getDiscardableAttrs());
248 return success();
249 }
250
251 if (!isa<VectorType>(Val: resultType))
252 return rewriter.notifyMatchFailure(arg&: op, msg: "expected vector result type");
253
254 return LLVM::detail::handleMultidimensionalVectors(
255 op: op.getOperation(), operands: adaptor.getOperands(), typeConverter,
256 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) {
257 auto numElements = LLVM::getVectorNumElements(type: llvm1DVectorTy);
258 auto splatAttr = SplatElementsAttr::get(
259 type: mlir::VectorType::get(shape: {numElements.getKnownMinValue()}, elementType: floatType,
260 scalableDims: {numElements.isScalable()}),
261 values: floatOne);
262 auto one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvm1DVectorTy,
263 value: splatAttr);
264 auto exp = LLVM::ExpOp::create(
265 builder&: rewriter, location: loc, resultTypes: TypeRange{llvm1DVectorTy}, operands: ValueRange{operands[0]},
266 properties: expAttrs.getProperties(), discardableAttributes: expAttrs.getDiscardableAttrs());
267 return LLVM::FSubOp::create(
268 builder&: rewriter, location: loc, resultTypes: TypeRange{llvm1DVectorTy}, operands: ValueRange{exp, one},
269 properties: subAttrs.getProperties(), discardableAttributes: subAttrs.getDiscardableAttrs());
270 },
271 rewriter);
272 }
273};
274
275// A `log1p` is converted into `log(1 + ...)`.
276struct Log1pOpLowering
277 : public ConvertOpToLLVMPattern<math::Log1pOp,
278 /*FailOnUnsupportedFP=*/true> {
279 using ConvertOpToLLVMPattern<
280 math::Log1pOp, /*FailOnUnsupportedFP=*/true>::ConvertOpToLLVMPattern;
281
282 LogicalResult
283 matchAndRewrite(math::Log1pOp op, OpAdaptor adaptor,
284 ConversionPatternRewriter &rewriter) const override {
285 const auto &typeConverter = *this->getTypeConverter();
286 auto operandType = adaptor.getOperand().getType();
287 auto llvmOperandType = typeConverter.convertType(t: operandType);
288 if (!llvmOperandType)
289 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported operand type");
290
291 auto loc = op.getLoc();
292 auto resultType = op.getResult().getType();
293 auto floatType = cast<FloatType>(
294 Val: typeConverter.convertType(t: getElementTypeOrSelf(type: resultType)));
295 auto floatOne = rewriter.getFloatAttr(type: floatType, value: 1.0);
296 ConvertFastMath<math::Log1pOp, LLVM::FAddOp> addAttrs(op);
297 ConvertFastMath<math::Log1pOp, LLVM::LogOp> logAttrs(op);
298
299 if (!isa<LLVM::LLVMArrayType>(Val: llvmOperandType)) {
300 LLVM::ConstantOp one =
301 isa<VectorType>(Val: llvmOperandType)
302 ? LLVM::ConstantOp::create(
303 builder&: rewriter, location: loc, res: llvmOperandType,
304 value: SplatElementsAttr::get(type: cast<ShapedType>(Val&: llvmOperandType),
305 values: floatOne))
306 : LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmOperandType,
307 value: floatOne);
308
309 auto add = LLVM::FAddOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{llvmOperandType},
310 operands: ValueRange{one, adaptor.getOperand()},
311 properties: addAttrs.getProperties(),
312 discardableAttributes: addAttrs.getDiscardableAttrs());
313 rewriter.replaceOpWithNewOp<LLVM::LogOp>(
314 op, args: TypeRange{llvmOperandType}, args: ValueRange{add},
315 args: logAttrs.getProperties(), args: logAttrs.getDiscardableAttrs());
316 return success();
317 }
318
319 if (!isa<VectorType>(Val: resultType))
320 return rewriter.notifyMatchFailure(arg&: op, msg: "expected vector result type");
321
322 return LLVM::detail::handleMultidimensionalVectors(
323 op: op.getOperation(), operands: adaptor.getOperands(), typeConverter,
324 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) {
325 auto numElements = LLVM::getVectorNumElements(type: llvm1DVectorTy);
326 auto splatAttr = SplatElementsAttr::get(
327 type: mlir::VectorType::get(shape: {numElements.getKnownMinValue()}, elementType: floatType,
328 scalableDims: {numElements.isScalable()}),
329 values: floatOne);
330 auto one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvm1DVectorTy,
331 value: splatAttr);
332 auto add = LLVM::FAddOp::create(
333 builder&: rewriter, location: loc, resultTypes: TypeRange{llvm1DVectorTy},
334 operands: ValueRange{one, operands[0]}, properties: addAttrs.getProperties(),
335 discardableAttributes: addAttrs.getDiscardableAttrs());
336 return LLVM::LogOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{llvm1DVectorTy},
337 operands: ValueRange{add}, properties: logAttrs.getProperties(),
338 discardableAttributes: logAttrs.getDiscardableAttrs());
339 },
340 rewriter);
341 }
342};
343
344// A `rsqrt` is converted into `1 / sqrt`.
345struct RsqrtOpLowering
346 : public ConvertOpToLLVMPattern<math::RsqrtOp,
347 /*FailOnUnsupportedFP=*/true> {
348 using ConvertOpToLLVMPattern<
349 math::RsqrtOp, /*FailOnUnsupportedFP=*/true>::ConvertOpToLLVMPattern;
350
351 LogicalResult
352 matchAndRewrite(math::RsqrtOp op, OpAdaptor adaptor,
353 ConversionPatternRewriter &rewriter) const override {
354 const auto &typeConverter = *this->getTypeConverter();
355 auto operandType = adaptor.getOperand().getType();
356 auto llvmOperandType = typeConverter.convertType(t: operandType);
357 if (!llvmOperandType)
358 return failure();
359
360 auto loc = op.getLoc();
361 auto resultType = op.getResult().getType();
362 auto floatType = cast<FloatType>(
363 Val: typeConverter.convertType(t: getElementTypeOrSelf(type: resultType)));
364 auto floatOne = rewriter.getFloatAttr(type: floatType, value: 1.0);
365 ConvertFastMath<math::RsqrtOp, LLVM::SqrtOp> sqrtAttrs(op);
366 ConvertFastMath<math::RsqrtOp, LLVM::FDivOp> divAttrs(op);
367
368 if (!isa<LLVM::LLVMArrayType>(Val: llvmOperandType)) {
369 LLVM::ConstantOp one;
370 if (isa<VectorType>(Val: llvmOperandType)) {
371 one = LLVM::ConstantOp::create(
372 builder&: rewriter, location: loc, res: llvmOperandType,
373 value: SplatElementsAttr::get(type: cast<ShapedType>(Val&: llvmOperandType),
374 values: floatOne));
375 } else {
376 one =
377 LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmOperandType, value: floatOne);
378 }
379 auto sqrt = LLVM::SqrtOp::create(
380 builder&: rewriter, location: loc, resultTypes: TypeRange{llvmOperandType},
381 operands: ValueRange{adaptor.getOperand()}, properties: sqrtAttrs.getProperties(),
382 discardableAttributes: sqrtAttrs.getDiscardableAttrs());
383 rewriter.replaceOpWithNewOp<LLVM::FDivOp>(
384 op, args: TypeRange{llvmOperandType}, args: ValueRange{one, sqrt},
385 args: divAttrs.getProperties(), args: divAttrs.getDiscardableAttrs());
386 return success();
387 }
388
389 if (!isa<VectorType>(Val: resultType))
390 return failure();
391
392 return LLVM::detail::handleMultidimensionalVectors(
393 op: op.getOperation(), operands: adaptor.getOperands(), typeConverter,
394 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) {
395 auto numElements = LLVM::getVectorNumElements(type: llvm1DVectorTy);
396 auto splatAttr = SplatElementsAttr::get(
397 type: mlir::VectorType::get(shape: {numElements.getKnownMinValue()}, elementType: floatType,
398 scalableDims: {numElements.isScalable()}),
399 values: floatOne);
400 auto one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvm1DVectorTy,
401 value: splatAttr);
402 auto sqrt = LLVM::SqrtOp::create(
403 builder&: rewriter, location: loc, resultTypes: TypeRange{llvm1DVectorTy}, operands: ValueRange{operands[0]},
404 properties: sqrtAttrs.getProperties(), discardableAttributes: sqrtAttrs.getDiscardableAttrs());
405 return LLVM::FDivOp::create(
406 builder&: rewriter, location: loc, resultTypes: TypeRange{llvm1DVectorTy}, operands: ValueRange{one, sqrt},
407 properties: divAttrs.getProperties(), discardableAttributes: divAttrs.getDiscardableAttrs());
408 },
409 rewriter);
410 }
411};
412
413struct FPowIOpLowering
414 : public ConvertOpToLLVMPattern<math::FPowIOp,
415 /*FailOnUnsupportedFP=*/true> {
416 using ConvertOpToLLVMPattern<
417 math::FPowIOp, /*FailOnUnsupportedFP=*/true>::ConvertOpToLLVMPattern;
418
419 LogicalResult
420 matchAndRewrite(math::FPowIOp op, OpAdaptor adaptor,
421 ConversionPatternRewriter &rewriter) const override {
422 const auto &typeConverter = *this->getTypeConverter();
423 auto llvmOperandType = typeConverter.convertType(t: op.getLhs().getType());
424 if (!llvmOperandType)
425 return failure();
426
427 auto loc = op.getLoc();
428 Value exponent = adaptor.getRhs();
429 if (isa<VectorType>(Val: op.getRhs().getType())) {
430 SplatElementsAttr splatAttr;
431 if (!matchPattern(value: op.getRhs(), pattern: m_Constant(bind_value: &splatAttr)))
432 return rewriter.notifyMatchFailure(arg&: op, msg: "expected a splat exponent");
433
434 auto exponentType = typeConverter.convertType(
435 t: getElementTypeOrSelf(type: op.getRhs().getType()));
436 if (!exponentType)
437 return failure();
438 exponent = LLVM::ConstantOp::create(
439 builder&: rewriter, location: loc, res: exponentType,
440 value: rewriter.getIntegerAttr(type: exponentType,
441 value: splatAttr.getSplatValue<APInt>()));
442 }
443
444 ConvertFastMath<math::FPowIOp, LLVM::PowIOp> attrs(op);
445
446 if (!isa<LLVM::LLVMArrayType>(Val: llvmOperandType)) {
447 rewriter.replaceOpWithNewOp<LLVM::PowIOp>(
448 op, args: TypeRange{llvmOperandType},
449 args: ValueRange{adaptor.getLhs(), exponent}, args: attrs.getProperties(),
450 args: attrs.getDiscardableAttrs());
451 return success();
452 }
453
454 return LLVM::detail::handleMultidimensionalVectors(
455 op: op.getOperation(), operands: ValueRange{adaptor.getLhs()}, typeConverter,
456 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) {
457 return LLVM::PowIOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{llvm1DVectorTy},
458 operands: ValueRange{operands[0], exponent},
459 properties: attrs.getProperties(),
460 discardableAttributes: attrs.getDiscardableAttrs());
461 },
462 rewriter);
463 }
464};
465
466struct IsNaNOpLowering
467 : public ConvertOpToLLVMPattern<math::IsNaNOp,
468 /*FailOnUnsupportedFP=*/true> {
469 using ConvertOpToLLVMPattern<
470 math::IsNaNOp, /*FailOnUnsupportedFP=*/true>::ConvertOpToLLVMPattern;
471
472 LogicalResult
473 matchAndRewrite(math::IsNaNOp op, OpAdaptor adaptor,
474 ConversionPatternRewriter &rewriter) const override {
475 const auto &typeConverter = *this->getTypeConverter();
476 auto operandType =
477 typeConverter.convertType(t: adaptor.getOperand().getType());
478 auto resultType = typeConverter.convertType(t: op.getResult().getType());
479 if (!operandType || !resultType)
480 return failure();
481
482 rewriter.replaceOpWithNewOp<LLVM::IsFPClass>(
483 op, args&: resultType, args: adaptor.getOperand(), args: llvm::fcNan);
484 return success();
485 }
486};
487
488struct IsFiniteOpLowering
489 : public ConvertOpToLLVMPattern<math::IsFiniteOp,
490 /*FailOnUnsupportedFP=*/true> {
491 using ConvertOpToLLVMPattern<
492 math::IsFiniteOp, /*FailOnUnsupportedFP=*/true>::ConvertOpToLLVMPattern;
493
494 LogicalResult
495 matchAndRewrite(math::IsFiniteOp op, OpAdaptor adaptor,
496 ConversionPatternRewriter &rewriter) const override {
497 const auto &typeConverter = *this->getTypeConverter();
498 auto operandType =
499 typeConverter.convertType(t: adaptor.getOperand().getType());
500 auto resultType = typeConverter.convertType(t: op.getResult().getType());
501 if (!operandType || !resultType)
502 return failure();
503
504 rewriter.replaceOpWithNewOp<LLVM::IsFPClass>(
505 op, args&: resultType, args: adaptor.getOperand(), args: llvm::fcFinite);
506 return success();
507 }
508};
509
510struct ConvertMathToLLVMPass
511 : public impl::ConvertMathToLLVMPassBase<ConvertMathToLLVMPass> {
512 using Base::Base;
513
514 void runOnOperation() override {
515 RewritePatternSet patterns(&getContext());
516 LLVMTypeConverter converter(&getContext());
517 populateMathToLLVMConversionPatterns(converter, patterns, approximateLog1p);
518 LLVMConversionTarget target(getContext());
519 if (failed(Result: applyPartialConversion(op: getOperation(), target,
520 patterns: std::move(patterns))))
521 signalPassFailure();
522 }
523};
524} // namespace
525
526void mlir::populateMathToLLVMConversionPatterns(
527 const LLVMTypeConverter &converter, RewritePatternSet &patterns,
528 bool approximateLog1p, PatternBenefit benefit) {
529 if (approximateLog1p)
530 patterns.add<Log1pOpLowering>(arg: converter, args&: benefit);
531 // clang-format off
532 patterns.add<
533 IsNaNOpLowering,
534 IsFiniteOpLowering,
535 AbsFOpLowering,
536 AbsIOpLowering,
537 CeilOpLowering,
538 CopySignOpLowering,
539 CosOpLowering,
540 CoshOpLowering,
541 AcosOpLowering,
542 CountLeadingZerosOpLowering,
543 CountTrailingZerosOpLowering,
544 CtPopFOpLowering,
545 Exp2OpLowering,
546 ExpM1OpLowering,
547 ExpOpLowering,
548 FPowIOpLowering,
549 FloorOpLowering,
550 FmaOpLowering,
551 ConstrainedFmaOpLowering,
552 Log10OpLowering,
553 Log2OpLowering,
554 LogOpLowering,
555 PowFOpLowering,
556 RoundEvenOpLowering,
557 RoundOpLowering,
558 RsqrtOpLowering,
559 SincosOpLowering,
560 SinOpLowering,
561 SinhOpLowering,
562 ASinOpLowering,
563 SqrtOpLowering,
564 FTruncOpLowering,
565 TanOpLowering,
566 TanhOpLowering,
567 ATanOpLowering,
568 ATan2OpLowering
569 >(arg: converter, args&: benefit);
570 // clang-format on
571}
572
573//===----------------------------------------------------------------------===//
574// ConvertToLLVMPatternInterface implementation
575//===----------------------------------------------------------------------===//
576
577namespace {
578/// Implement the interface to convert Math to LLVM.
579struct MathToLLVMDialectInterface : public ConvertToLLVMPatternInterface {
580 MathToLLVMDialectInterface(Dialect *dialect)
581 : ConvertToLLVMPatternInterface(dialect) {}
582
583 void loadDependentDialects(MLIRContext *context) const final {
584 context->loadDialect<LLVM::LLVMDialect>();
585 }
586
587 /// Hook for derived dialect interface to provide conversion patterns
588 /// and mark dialect legal for the conversion target.
589 void populateConvertToLLVMConversionPatterns(
590 ConversionTarget &target, LLVMTypeConverter &typeConverter,
591 RewritePatternSet &patterns) const final {
592 populateMathToLLVMConversionPatterns(converter: typeConverter, patterns);
593 }
594};
595} // namespace
596
597void mlir::registerConvertMathToLLVMInterface(DialectRegistry &registry) {
598 registry.addExtension(extensionFn: +[](MLIRContext *ctx, math::MathDialect *dialect) {
599 dialect->addInterfaces<MathToLLVMDialectInterface>();
600 });
601}
602