1//===-- MathToNVVM.cpp - conversion from Math to CUDA libdevice calls ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/MathToNVVM/MathToNVVM.h"
10#include "mlir/Dialect/Arith/IR/Arith.h"
11#include "mlir/Dialect/Func/IR/FuncDialect.h"
12#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
13#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
14#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
15#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
19#include "mlir/Dialect/LLVMIR/NVVMDialectDecl.h"
20#include "mlir/Dialect/Math/IR/Math.h"
21#include "mlir/Dialect/Vector/IR/VectorDialect.h"
22#include "mlir/IR/BuiltinDialect.h"
23#include "mlir/Pass/Pass.h"
24
25#include "../GPUCommon/GPUOpsLowering.h"
26#include "../GPUCommon/OpToFuncCallLowering.h"
27
28namespace mlir {
29#define GEN_PASS_DEF_CONVERTMATHTONVVM
30#include "mlir/Conversion/Passes.h.inc"
31} // namespace mlir
32
33using namespace mlir;
34
35#define DEBUG_TYPE "math-to-nvvm"
36
37template <typename OpTy>
38static void populateOpPatterns(const LLVMTypeConverter &converter,
39 RewritePatternSet &patterns,
40 PatternBenefit benefit, StringRef f32Func,
41 StringRef f64Func, StringRef f32ApproxFunc = "",
42 StringRef f16Func = "") {
43 patterns.add<ScalarizeVectorOpLowering<OpTy>>(converter, benefit);
44 patterns.add<OpToFuncCallLowering<OpTy>>(converter, f32Func, f64Func,
45 f32ApproxFunc, f16Func,
46 /*i32Func=*/"", benefit);
47}
48
49template <typename OpTy>
50static void populateIntOpPatterns(const LLVMTypeConverter &converter,
51 RewritePatternSet &patterns,
52 PatternBenefit benefit, StringRef i32Func) {
53 patterns.add<ScalarizeVectorOpLowering<OpTy>>(converter, benefit);
54 patterns.add<OpToFuncCallLowering<OpTy>>(converter, "", "", "", "", i32Func,
55 benefit);
56}
57
58/// libdevice's __nv_powi and __nv_powif take an i32 exponent, so math.fpowi
59/// with a wider or index exponent is not lowered.
60static bool exponentFitsInI32(math::FPowIOp op) {
61 auto type =
62 dyn_cast<IntegerType>(Val: getElementTypeOrSelf(type: op.getRhs().getType()));
63 return type && type.getWidth() <= 32;
64}
65
66namespace {
67struct FPowIScalarizeLowering
68 : public ScalarizeVectorOpLowering<math::FPowIOp> {
69 using ScalarizeVectorOpLowering::ScalarizeVectorOpLowering;
70
71 LogicalResult
72 matchAndRewrite(math::FPowIOp op, OpAdaptor adaptor,
73 ConversionPatternRewriter &rewriter) const override {
74 if (!exponentFitsInI32(op))
75 return rewriter.notifyMatchFailure(arg&: op, msg: "exponent does not fit in i32");
76 return ScalarizeVectorOpLowering::matchAndRewrite(op, adaptor, rewriter);
77 }
78};
79
80struct FPowIOpLowering : public OpToFuncCallLowering<math::FPowIOp> {
81 using OpToFuncCallLowering::OpToFuncCallLowering;
82
83 LogicalResult
84 matchAndRewrite(math::FPowIOp op, OpAdaptor adaptor,
85 ConversionPatternRewriter &rewriter) const override {
86 if (!exponentFitsInI32(op))
87 return rewriter.notifyMatchFailure(arg&: op, msg: "exponent does not fit in i32");
88 // Return before creating anything in the cases OpToFuncCallLowering
89 // does not handle.
90 if (!op->getParentOfType<FunctionOpInterface>() ||
91 !isa<Float16Type, BFloat16Type, Float32Type, Float64Type>(
92 Val: adaptor.getLhs().getType()))
93 return rewriter.notifyMatchFailure(arg&: op, msg: "cannot be lowered to libdevice");
94 // Sign-extend a narrower exponent.
95 Value exponent = adaptor.getRhs();
96 if (!exponent.getType().isInteger(width: 32))
97 exponent = LLVM::SExtOp::create(builder&: rewriter, location: op.getLoc(),
98 res: rewriter.getI32Type(), arg: exponent);
99 Value operands[] = {adaptor.getLhs(), exponent};
100 return OpToFuncCallLowering::matchAndRewrite(
101 op, adaptor: OpAdaptor(ValueRange(operands), op), rewriter);
102 }
103};
104} // namespace
105
106// Custom pattern for sincos since it returns two values
107struct SincosOpLowering : public ConvertOpToLLVMPattern<math::SincosOp> {
108 using ConvertOpToLLVMPattern<math::SincosOp>::ConvertOpToLLVMPattern;
109
110 LogicalResult
111 matchAndRewrite(math::SincosOp op, OpAdaptor adaptor,
112 ConversionPatternRewriter &rewriter) const override {
113 Location loc = op.getLoc();
114 Value input = adaptor.getOperand();
115 Type inputType = input.getType();
116 auto convertedInput = maybeExt(operand: input, rewriter);
117 auto computeType = convertedInput.getType();
118
119 StringRef sincosFunc;
120 if (isa<Float32Type>(Val: computeType)) {
121 const arith::FastMathFlags flag = op.getFastmath();
122 const bool useApprox =
123 mlir::arith::bitEnumContainsAny(bits: flag, bit: arith::FastMathFlags::afn);
124 sincosFunc = useApprox ? "__nv_fast_sincosf" : "__nv_sincosf";
125 } else if (isa<Float64Type>(Val: computeType)) {
126 sincosFunc = "__nv_sincos";
127 } else {
128 return rewriter.notifyMatchFailure(arg&: op,
129 msg: "unsupported operand type for sincos");
130 }
131
132 auto ptrType = LLVM::LLVMPointerType::get(context: rewriter.getContext());
133
134 Value sinPtr, cosPtr;
135 {
136 OpBuilder::InsertionGuard guard(rewriter);
137 auto *scope =
138 op->getParentWithTrait<mlir::OpTrait::AutomaticAllocationScope>();
139 assert(scope && "Expected op to be inside automatic allocation scope");
140 rewriter.setInsertionPointToStart(&scope->getRegion(index: 0).front());
141 auto one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: rewriter.getI32Type(),
142 value: rewriter.getI32IntegerAttr(value: 1));
143 sinPtr =
144 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: computeType, arraySize: one, alignment: 0);
145 cosPtr =
146 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: computeType, arraySize: one, alignment: 0);
147 }
148
149 createSincosCall(rewriter, loc, funcName: sincosFunc, input: convertedInput, sinPtr, cosPtr,
150 op);
151
152 auto sinResult = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: computeType, addr: sinPtr);
153 auto cosResult = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: computeType, addr: cosPtr);
154
155 rewriter.replaceOp(op, newValues: {maybeTrunc(operand: sinResult, type: inputType, rewriter),
156 maybeTrunc(operand: cosResult, type: inputType, rewriter)});
157 return success();
158 }
159
160private:
161 Value maybeExt(Value operand, PatternRewriter &rewriter) const {
162 if (isa<Float16Type, BFloat16Type>(Val: operand.getType()))
163 return LLVM::FPExtOp::create(builder&: rewriter, location: operand.getLoc(),
164 res: Float32Type::get(context: rewriter.getContext()),
165 arg: operand);
166 return operand;
167 }
168
169 Value maybeTrunc(Value operand, Type type, PatternRewriter &rewriter) const {
170 if (operand.getType() != type)
171 return LLVM::FPTruncOp::create(builder&: rewriter, location: operand.getLoc(), res: type, arg: operand);
172 return operand;
173 }
174
175 void createSincosCall(ConversionPatternRewriter &rewriter, Location loc,
176 StringRef funcName, Value input, Value sinPtr,
177 Value cosPtr, Operation *op) const {
178 auto voidType = LLVM::LLVMVoidType::get(ctx: rewriter.getContext());
179 auto ptrType = sinPtr.getType();
180
181 SmallVector<Type> operandTypes = {input.getType(), ptrType, ptrType};
182 auto funcType = LLVM::LLVMFunctionType::get(result: voidType, arguments: operandTypes);
183
184 auto funcAttr = StringAttr::get(context: op->getContext(), bytes: funcName);
185 auto funcOp =
186 SymbolTable::lookupNearestSymbolFrom<LLVM::LLVMFuncOp>(from: op, symbol: funcAttr);
187
188 if (!funcOp) {
189 auto parentFunc = op->getParentOfType<FunctionOpInterface>();
190 assert(parentFunc && "expected there to be a parent function");
191 OpBuilder b(parentFunc);
192
193 auto globalloc = loc->findInstanceOfOrUnknown<FileLineColLoc>();
194 funcOp = LLVM::LLVMFuncOp::create(builder&: b, location: globalloc, name: funcName, type: funcType);
195 }
196
197 SmallVector<Value> callOperands = {input, sinPtr, cosPtr};
198 LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcOp, args: callOperands);
199 }
200};
201
202void mlir::populateLibDeviceConversionPatterns(
203 const LLVMTypeConverter &converter, RewritePatternSet &patterns,
204 PatternBenefit benefit) {
205 populateOpPatterns<arith::RemFOp>(converter, patterns, benefit, f32Func: "__nv_fmodf",
206 f64Func: "__nv_fmod");
207 populateOpPatterns<arith::MaxNumFOp>(converter, patterns, benefit,
208 f32Func: "__nv_fmaxf", f64Func: "__nv_fmax");
209 populateOpPatterns<arith::MinNumFOp>(converter, patterns, benefit,
210 f32Func: "__nv_fminf", f64Func: "__nv_fmin");
211
212 populateIntOpPatterns<math::AbsIOp>(converter, patterns, benefit, i32Func: "__nv_abs");
213 populateOpPatterns<math::AbsFOp>(converter, patterns, benefit, f32Func: "__nv_fabsf",
214 f64Func: "__nv_fabs");
215 populateOpPatterns<math::AcosOp>(converter, patterns, benefit, f32Func: "__nv_acosf",
216 f64Func: "__nv_acos");
217 populateOpPatterns<math::AcoshOp>(converter, patterns, benefit, f32Func: "__nv_acoshf",
218 f64Func: "__nv_acosh");
219 populateOpPatterns<math::AsinOp>(converter, patterns, benefit, f32Func: "__nv_asinf",
220 f64Func: "__nv_asin");
221 populateOpPatterns<math::AsinhOp>(converter, patterns, benefit, f32Func: "__nv_asinhf",
222 f64Func: "__nv_asinh");
223 populateOpPatterns<math::AtanOp>(converter, patterns, benefit, f32Func: "__nv_atanf",
224 f64Func: "__nv_atan");
225 populateOpPatterns<math::Atan2Op>(converter, patterns, benefit, f32Func: "__nv_atan2f",
226 f64Func: "__nv_atan2");
227 populateOpPatterns<math::AtanhOp>(converter, patterns, benefit, f32Func: "__nv_atanhf",
228 f64Func: "__nv_atanh");
229 populateOpPatterns<math::CbrtOp>(converter, patterns, benefit, f32Func: "__nv_cbrtf",
230 f64Func: "__nv_cbrt");
231 populateOpPatterns<math::CeilOp>(converter, patterns, benefit, f32Func: "__nv_ceilf",
232 f64Func: "__nv_ceil");
233 populateOpPatterns<math::CopySignOp>(converter, patterns, benefit,
234 f32Func: "__nv_copysignf", f64Func: "__nv_copysign");
235 populateOpPatterns<math::CosOp>(converter, patterns, benefit, f32Func: "__nv_cosf",
236 f64Func: "__nv_cos", f32ApproxFunc: "__nv_fast_cosf");
237 populateOpPatterns<math::CoshOp>(converter, patterns, benefit, f32Func: "__nv_coshf",
238 f64Func: "__nv_cosh");
239 populateOpPatterns<math::ErfOp>(converter, patterns, benefit, f32Func: "__nv_erff",
240 f64Func: "__nv_erf");
241 populateOpPatterns<math::ErfcOp>(converter, patterns, benefit, f32Func: "__nv_erfcf",
242 f64Func: "__nv_erfc");
243 populateOpPatterns<math::ExpOp>(converter, patterns, benefit, f32Func: "__nv_expf",
244 f64Func: "__nv_exp", f32ApproxFunc: "__nv_fast_expf");
245 populateOpPatterns<math::Exp2Op>(converter, patterns, benefit, f32Func: "__nv_exp2f",
246 f64Func: "__nv_exp2");
247 populateOpPatterns<math::ExpM1Op>(converter, patterns, benefit, f32Func: "__nv_expm1f",
248 f64Func: "__nv_expm1");
249 populateOpPatterns<math::FloorOp>(converter, patterns, benefit, f32Func: "__nv_floorf",
250 f64Func: "__nv_floor");
251 populateOpPatterns<math::FmaOp>(converter, patterns, benefit, f32Func: "__nv_fmaf",
252 f64Func: "__nv_fma");
253 // Note: libdevice uses a different name for 32-bit finite checking
254 populateOpPatterns<math::IsFiniteOp>(converter, patterns, benefit,
255 f32Func: "__nv_finitef", f64Func: "__nv_isfinited");
256 populateOpPatterns<math::IsInfOp>(converter, patterns, benefit, f32Func: "__nv_isinff",
257 f64Func: "__nv_isinfd");
258 populateOpPatterns<math::IsNaNOp>(converter, patterns, benefit, f32Func: "__nv_isnanf",
259 f64Func: "__nv_isnand");
260 populateOpPatterns<math::LogOp>(converter, patterns, benefit, f32Func: "__nv_logf",
261 f64Func: "__nv_log", f32ApproxFunc: "__nv_fast_logf");
262 populateOpPatterns<math::Log10Op>(converter, patterns, benefit, f32Func: "__nv_log10f",
263 f64Func: "__nv_log10", f32ApproxFunc: "__nv_fast_log10f");
264 populateOpPatterns<math::Log1pOp>(converter, patterns, benefit, f32Func: "__nv_log1pf",
265 f64Func: "__nv_log1p");
266 populateOpPatterns<math::Log2Op>(converter, patterns, benefit, f32Func: "__nv_log2f",
267 f64Func: "__nv_log2", f32ApproxFunc: "__nv_fast_log2f");
268 populateOpPatterns<math::PowFOp>(converter, patterns, benefit, f32Func: "__nv_powf",
269 f64Func: "__nv_pow", f32ApproxFunc: "__nv_fast_powf");
270 patterns.add<FPowIScalarizeLowering>(arg: converter, args&: benefit);
271 patterns.add<FPowIOpLowering>(arg: converter, args: "__nv_powif", args: "__nv_powi", args: "", args: "",
272 /*i32Func=*/args: "", args&: benefit);
273 populateOpPatterns<math::RoundOp>(converter, patterns, benefit, f32Func: "__nv_roundf",
274 f64Func: "__nv_round");
275 populateOpPatterns<math::RoundEvenOp>(converter, patterns, benefit,
276 f32Func: "__nv_rintf", f64Func: "__nv_rint");
277 populateOpPatterns<math::RsqrtOp>(converter, patterns, benefit, f32Func: "__nv_rsqrtf",
278 f64Func: "__nv_rsqrt");
279 populateOpPatterns<math::SinOp>(converter, patterns, benefit, f32Func: "__nv_sinf",
280 f64Func: "__nv_sin", f32ApproxFunc: "__nv_fast_sinf");
281 populateOpPatterns<math::SinhOp>(converter, patterns, benefit, f32Func: "__nv_sinhf",
282 f64Func: "__nv_sinh");
283 populateOpPatterns<math::SqrtOp>(converter, patterns, benefit, f32Func: "__nv_sqrtf",
284 f64Func: "__nv_sqrt");
285 populateOpPatterns<math::TanOp>(converter, patterns, benefit, f32Func: "__nv_tanf",
286 f64Func: "__nv_tan", f32ApproxFunc: "__nv_fast_tanf");
287 populateOpPatterns<math::TanhOp>(converter, patterns, benefit, f32Func: "__nv_tanhf",
288 f64Func: "__nv_tanh");
289
290 // Custom pattern for sincos since it returns two values
291 patterns.add<SincosOpLowering>(arg: converter, args&: benefit);
292}
293
294namespace {
295struct ConvertMathToNVVMPass final
296 : impl::ConvertMathToNVVMBase<ConvertMathToNVVMPass> {
297 using impl::ConvertMathToNVVMBase<
298 ConvertMathToNVVMPass>::ConvertMathToNVVMBase;
299
300 void runOnOperation() override;
301};
302} // namespace
303
304void ConvertMathToNVVMPass::runOnOperation() {
305 auto m = getOperation();
306 MLIRContext *ctx = m.getContext();
307
308 RewritePatternSet patterns(&getContext());
309 LowerToLLVMOptions options(ctx, DataLayout(m));
310 LLVMTypeConverter converter(ctx, options);
311
312 populateLibDeviceConversionPatterns(converter, patterns, /*benefit=*/1);
313
314 ConversionTarget target(getContext());
315 target
316 .addLegalDialect<BuiltinDialect, func::FuncDialect, vector::VectorDialect,
317 LLVM::LLVMDialect, NVVM::NVVMDialect>();
318 target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, LLVM::FAbsOp,
319 LLVM::FCeilOp, LLVM::FFloorOp, LLVM::FRemOp, LLVM::LogOp,
320 LLVM::Log10Op, LLVM::Log2Op, LLVM::PowOp, LLVM::SinOp,
321 LLVM::SqrtOp>();
322 if (failed(Result: applyPartialConversion(op: m, target, patterns: std::move(patterns))))
323 signalPassFailure();
324}
325