1//===- OpToFuncCallLowering.h - GPU ops lowering to custom calls *- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#ifndef MLIR_CONVERSION_GPUCOMMON_OPTOFUNCCALLLOWERING_H_
9#define MLIR_CONVERSION_GPUCOMMON_OPTOFUNCCALLLOWERING_H_
10
11#include "mlir/Conversion/LLVMCommon/Pattern.h"
12#include "mlir/Dialect/Arith/IR/ArithAttributes.h"
13#include "mlir/Dialect/GPU/IR/GPUDialect.h"
14#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
15#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
19#include "mlir/IR/Builders.h"
20
21namespace mlir {
22
23namespace {
24/// Detection trait tor the `getFastmath` instance method.
25template <typename T>
26using has_get_fastmath_t = decltype(std::declval<T>().getFastmath());
27} // namespace
28
29/// Rewriting that replaces SourceOp with a CallOp to `f32Func` or `f64Func` or
30/// `f32ApproxFunc` or `f16Func` or `i32Type` depending on the element type and
31/// the fastMathFlag of that Op, if present. The function declaration is added
32/// in case it was not added before.
33///
34/// If the input values are of bf16 type (or f16 type if f16Func is empty), the
35/// value is first casted to f32, the function called and then the result casted
36/// back.
37///
38/// Example with NVVM:
39/// %exp_f32 = math.exp %arg_f32 : f32
40///
41/// will be transformed into
42/// llvm.call @__nv_expf(%arg_f32) : (f32) -> f32
43///
44/// If the fastMathFlag attribute of SourceOp is `afn` or `fast`, this Op lowers
45/// to the approximate calculation function.
46///
47/// Also example with NVVM:
48/// %exp_f32 = math.exp %arg_f32 fastmath<afn> : f32
49///
50/// will be transformed into
51/// llvm.call @__nv_fast_expf(%arg_f32) : (f32) -> f32
52///
53/// Final example with NVVM:
54/// %pow_f32 = math.fpowi %arg_f32, %arg_i32
55///
56/// will be transformed into
57/// llvm.call @__nv_powif(%arg_f32, %arg_i32) : (f32, i32) -> f32
58template <typename SourceOp>
59struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
60public:
61 explicit OpToFuncCallLowering(
62 const LLVMTypeConverter &lowering, StringRef f32Func, StringRef f64Func,
63 StringRef f32ApproxFunc, StringRef f16Func, StringRef i32Func = "",
64 PatternBenefit benefit = 1,
65 LLVM::cconv::CConv cconv = LLVM::cconv::CConv::C)
66 : ConvertOpToLLVMPattern<SourceOp>(lowering, benefit), f32Func(f32Func),
67 f64Func(f64Func), f32ApproxFunc(f32ApproxFunc), f16Func(f16Func),
68 i32Func(i32Func), cconv(cconv) {}
69
70 LogicalResult
71 matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor,
72 ConversionPatternRewriter &rewriter) const override {
73 using LLVM::LLVMFuncOp;
74
75 static_assert(
76 std::is_base_of<OpTrait::OneResult<SourceOp>, SourceOp>::value,
77 "expected single result op");
78
79 // This pattern only handles scalar ops. Ops with shaped (e.g. vector)
80 // result types, such as `math.isinf` on `vector<Nxf32>`, are expected to be
81 // scalarized first by `ScalarizeVectorOpLowering`, which is co-registered
82 // for these ops; bail out so that pattern can take over.
83 Type opResultType = op->getResultTypes().front();
84 if (!opResultType.isIntOrIndexOrFloat())
85 return rewriter.notifyMatchFailure(op, "expected scalar result type");
86
87 bool isResultBool = opResultType.isInteger(width: 1);
88 if constexpr (!std::is_base_of<OpTrait::SameOperandsAndResultType<SourceOp>,
89 SourceOp>::value) {
90 assert(op->getNumOperands() > 0 &&
91 "expected op to take at least one operand");
92 assert((op->getResultTypes().front() == op->getOperand(0).getType() ||
93 isResultBool) &&
94 "expected op with same operand and result types");
95 }
96
97 if (!op->template getParentOfType<FunctionOpInterface>()) {
98 return rewriter.notifyMatchFailure(
99 op, "expected op to be within a function region");
100 }
101
102 SmallVector<Value, 1> castedOperands;
103 for (Value operand : adaptor.getOperands())
104 castedOperands.push_back(Elt: maybeCast(operand, rewriter));
105
106 Type castedOperandType = castedOperands.front().getType();
107
108 // At ABI level, booleans are treated as i32.
109 Type resultType =
110 isResultBool ? rewriter.getIntegerType(width: 32) : castedOperandType;
111 Type funcType = getFunctionType(resultType, operands: castedOperands);
112 StringRef funcName = getFunctionName(type: castedOperandType, op);
113 if (funcName.empty())
114 return failure();
115
116 LLVMFuncOp funcOp = appendOrGetFuncOp(funcName, funcType, op);
117 auto callOp =
118 LLVM::CallOp::create(rewriter, op->getLoc(), funcOp, castedOperands);
119 callOp.setCConv(cconv);
120
121 if (resultType == adaptor.getOperands().front().getType()) {
122 rewriter.replaceOp(op, {callOp.getResult()});
123 return success();
124 }
125
126 // Boolean result are mapping to i32 at the ABI level with zero values being
127 // interpreted as false and non-zero values being interpreted as true. Since
128 // there is no guarantee of a specific value being used to indicate true,
129 // compare for inequality with zero (rather than truncate or shift).
130 if (isResultBool) {
131 Value zero = LLVM::ConstantOp::create(rewriter, op->getLoc(),
132 rewriter.getIntegerType(width: 32),
133 rewriter.getI32IntegerAttr(value: 0));
134 Value truncated =
135 LLVM::ICmpOp::create(rewriter, op->getLoc(), LLVM::ICmpPredicate::ne,
136 callOp.getResult(), zero);
137 rewriter.replaceOp(op, {truncated});
138 return success();
139 }
140
141 assert(callOp.getResult().getType().isF32() &&
142 "only f32 types are supposed to be truncated back");
143 Value truncated = LLVM::FPTruncOp::create(
144 rewriter, op->getLoc(), adaptor.getOperands().front().getType(),
145 callOp.getResult());
146 rewriter.replaceOp(op, {truncated});
147 return success();
148 }
149
150 Value maybeCast(Value operand, PatternRewriter &rewriter) const {
151 Type type = operand.getType();
152 if (!isa<Float16Type, BFloat16Type>(Val: type))
153 return operand;
154
155 // If there's an f16 function, no need to cast f16 values.
156 if (!f16Func.empty() && isa<Float16Type>(Val: type))
157 return operand;
158
159 return LLVM::FPExtOp::create(builder&: rewriter, location: operand.getLoc(),
160 res: Float32Type::get(context: rewriter.getContext()),
161 arg: operand);
162 }
163
164 Type getFunctionType(Type resultType, ValueRange operands) const {
165 SmallVector<Type> operandTypes(operands.getTypes());
166 return LLVM::LLVMFunctionType::get(result: resultType, arguments: operandTypes);
167 }
168
169 LLVM::LLVMFuncOp appendOrGetFuncOp(StringRef funcName, Type funcType,
170 Operation *op) const {
171 using LLVM::LLVMFuncOp;
172
173 auto funcAttr = StringAttr::get(context: op->getContext(), bytes: funcName);
174 auto funcOp =
175 SymbolTable::lookupNearestSymbolFrom<LLVMFuncOp>(from: op, symbol: funcAttr);
176 if (funcOp)
177 return funcOp;
178
179 auto parentFunc = op->getParentOfType<FunctionOpInterface>();
180 assert(parentFunc && "expected there to be a parent function");
181 OpBuilder b(parentFunc);
182
183 // Create a valid global location removing any metadata attached to the
184 // location as debug info metadata inside of a function cannot be used
185 // outside of that function.
186 auto globalloc = op->getLoc()->findInstanceOfOrUnknown<FileLineColLoc>();
187 auto newFuncOp = LLVMFuncOp::create(builder&: b, location: globalloc, name: funcName, type: funcType);
188 newFuncOp.setCConv(cconv);
189 return newFuncOp;
190 }
191
192 StringRef getFunctionName(Type type, SourceOp op) const {
193 bool useApprox = false;
194 if constexpr (llvm::is_detected<has_get_fastmath_t, SourceOp>::value) {
195 arith::FastMathFlags flag = op.getFastmath();
196 useApprox = ((uint32_t)arith::FastMathFlags::afn & (uint32_t)flag) &&
197 !f32ApproxFunc.empty();
198 }
199
200 if (isa<Float16Type>(Val: type))
201 return f16Func;
202 if (isa<Float32Type>(Val: type)) {
203 if (useApprox)
204 return f32ApproxFunc;
205 return f32Func;
206 }
207 if (isa<Float64Type>(Val: type))
208 return f64Func;
209
210 if (type.isInteger(width: 32))
211 return i32Func;
212 return "";
213 }
214
215 const std::string f32Func;
216 const std::string f64Func;
217 const std::string f32ApproxFunc;
218 const std::string f16Func;
219 const std::string i32Func;
220 const LLVM::cconv::CConv cconv;
221};
222
223} // namespace mlir
224
225#endif // MLIR_CONVERSION_GPUCOMMON_OPTOFUNCCALLLOWERING_H_
226