1//===-- MathToROCDL.cpp - conversion from Math to rocdl calls -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/MathToROCDL/MathToROCDL.h"
10#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
11#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
12#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
13#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
14#include "mlir/Dialect/Arith/IR/Arith.h"
15#include "mlir/Dialect/Func/IR/FuncDialect.h"
16#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h"
18#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
19#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
20#include "mlir/Dialect/Math/IR/Math.h"
21#include "mlir/Dialect/Vector/IR/VectorOps.h"
22#include "mlir/IR/BuiltinDialect.h"
23#include "mlir/IR/PatternMatch.h"
24#include "mlir/Pass/Pass.h"
25#include "mlir/Transforms/DialectConversion.h"
26#include "llvm/Support/DebugLog.h"
27
28#include "../GPUCommon/GPUOpsLowering.h"
29#include "../GPUCommon/OpToFuncCallLowering.h"
30
31namespace mlir {
32#define GEN_PASS_DEF_CONVERTMATHTOROCDL
33#include "mlir/Conversion/Passes.h.inc"
34} // namespace mlir
35
36using namespace mlir;
37
38#define DEBUG_TYPE "math-to-rocdl"
39
40template <typename OpTy>
41static void populateOpPatterns(const LLVMTypeConverter &converter,
42 RewritePatternSet &patterns, StringRef f32Func,
43 StringRef f64Func, StringRef f16Func,
44 StringRef f32ApproxFunc = "") {
45 patterns.add<ScalarizeVectorOpLowering<OpTy>>(converter);
46 patterns.add<OpToFuncCallLowering<OpTy>>(converter, f32Func, f64Func,
47 f32ApproxFunc, f16Func);
48}
49
50struct ClampFOpConversion final
51 : public ConvertOpToLLVMPattern<math::ClampFOp> {
52 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
53
54 LogicalResult
55 matchAndRewrite(math::ClampFOp op, OpAdaptor adaptor,
56 ConversionPatternRewriter &rewriter) const override {
57 // Only f16 and f32 types are supported by fmed3
58 Type opTy = op.getType();
59 Type resultType = getTypeConverter()->convertType(t: opTy);
60
61 if (auto vectorType = dyn_cast<VectorType>(Val&: opTy))
62 opTy = vectorType.getElementType();
63
64 if (!isa<Float16Type, Float32Type>(Val: opTy))
65 return rewriter.notifyMatchFailure(
66 arg&: op, msg: "fmed3 only supports f16 and f32 types");
67
68 // Handle multi-dimensional vectors (converted to LLVM arrays)
69 if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(Val&: resultType))
70 return LLVM::detail::handleMultidimensionalVectors(
71 op: op.getOperation(), operands: adaptor.getOperands(), typeConverter: *getTypeConverter(),
72 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
73 math::ClampFOp::Adaptor adaptor(operands);
74 return ROCDL::FMed3Op::create(builder&: rewriter, location: op.getLoc(), res: llvm1DVectorTy,
75 src0: adaptor.getValue(), src1: adaptor.getMin(),
76 src2: adaptor.getMax());
77 },
78 rewriter);
79
80 // Handle 1D vectors and scalars directly
81 rewriter.replaceOpWithNewOp<ROCDL::FMed3Op>(op, args: op.getType(), args: op.getValue(),
82 args: op.getMin(), args: op.getMax());
83 return success();
84 }
85};
86
87void mlir::populateMathToROCDLConversionPatterns(
88 const LLVMTypeConverter &converter, RewritePatternSet &patterns,
89 std::optional<ROCDL::TargetInfo> target) {
90 // Handled by mathToLLVM: math::AbsIOp
91 // Handled by mathToLLVM: math::AbsFOp
92 // Handled by mathToLLVM: math::CopySignOp
93 // Handled by mathToLLVM: math::CountLeadingZerosOp
94 // Handled by mathToLLVM: math::CountTrailingZerosOp
95 // Handled by mathToLLVM: math::CgPopOp
96 // Handled by mathToLLVM: math::ExpOp (32-bit only)
97 // Handled by mathToLLVM: math::FmaOp
98 // Handled by mathToLLVM: math::LogOp (32-bit only)
99 // FIXME: math::IPowIOp
100 // Handled by mathToLLVM: math::RoundEvenOp
101 // Handled by mathToLLVM: math::RoundOp
102 // Handled by mathToLLVM: math::SqrtOp
103 // Handled by mathToLLVM: math::TruncOp
104 populateOpPatterns<math::AcosOp>(converter, patterns, f32Func: "__ocml_acos_f32",
105 f64Func: "__ocml_acos_f64", f16Func: "__ocml_acos_f16");
106 populateOpPatterns<math::AcoshOp>(converter, patterns, f32Func: "__ocml_acosh_f32",
107 f64Func: "__ocml_acosh_f64", f16Func: "__ocml_acosh_f16");
108 populateOpPatterns<math::AsinOp>(converter, patterns, f32Func: "__ocml_asin_f32",
109 f64Func: "__ocml_asin_f64", f16Func: "__ocml_asin_f16");
110 populateOpPatterns<math::AsinhOp>(converter, patterns, f32Func: "__ocml_asinh_f32",
111 f64Func: "__ocml_asinh_f64", f16Func: "__ocml_asinh_f16");
112 populateOpPatterns<math::AtanOp>(converter, patterns, f32Func: "__ocml_atan_f32",
113 f64Func: "__ocml_atan_f64", f16Func: "__ocml_atan_f16");
114 populateOpPatterns<math::AtanhOp>(converter, patterns, f32Func: "__ocml_atanh_f32",
115 f64Func: "__ocml_atanh_f64", f16Func: "__ocml_atanh_f16");
116 populateOpPatterns<math::Atan2Op>(converter, patterns, f32Func: "__ocml_atan2_f32",
117 f64Func: "__ocml_atan2_f64", f16Func: "__ocml_atan2_f16");
118 populateOpPatterns<math::CbrtOp>(converter, patterns, f32Func: "__ocml_cbrt_f32",
119 f64Func: "__ocml_cbrt_f64", f16Func: "__ocml_cbrt_f16");
120 populateOpPatterns<math::CeilOp>(converter, patterns, f32Func: "__ocml_ceil_f32",
121 f64Func: "__ocml_ceil_f64", f16Func: "__ocml_ceil_f16");
122 populateOpPatterns<math::CosOp>(converter, patterns, f32Func: "__ocml_cos_f32",
123 f64Func: "__ocml_cos_f64", f16Func: "__ocml_cos_f16");
124 populateOpPatterns<math::CoshOp>(converter, patterns, f32Func: "__ocml_cosh_f32",
125 f64Func: "__ocml_cosh_f64", f16Func: "__ocml_cosh_f16");
126 populateOpPatterns<math::SinhOp>(converter, patterns, f32Func: "__ocml_sinh_f32",
127 f64Func: "__ocml_sinh_f64", f16Func: "__ocml_sinh_f16");
128 populateOpPatterns<math::ExpOp>(converter, patterns, f32Func: "", f64Func: "__ocml_exp_f64",
129 f16Func: "__ocml_exp_f16");
130 populateOpPatterns<math::Exp2Op>(converter, patterns, f32Func: "__ocml_exp2_f32",
131 f64Func: "__ocml_exp2_f64", f16Func: "__ocml_exp2_f16");
132 populateOpPatterns<math::ExpM1Op>(converter, patterns, f32Func: "__ocml_expm1_f32",
133 f64Func: "__ocml_expm1_f64", f16Func: "__ocml_expm1_f16");
134 populateOpPatterns<math::FloorOp>(converter, patterns, f32Func: "__ocml_floor_f32",
135 f64Func: "__ocml_floor_f64", f16Func: "__ocml_floor_f16");
136 populateOpPatterns<math::LogOp>(converter, patterns, f32Func: "", f64Func: "__ocml_log_f64",
137 f16Func: "__ocml_log_f16");
138 populateOpPatterns<math::Log10Op>(converter, patterns, f32Func: "__ocml_log10_f32",
139 f64Func: "__ocml_log10_f64", f16Func: "__ocml_log10_f16");
140 populateOpPatterns<math::Log1pOp>(converter, patterns, f32Func: "__ocml_log1p_f32",
141 f64Func: "__ocml_log1p_f64", f16Func: "__ocml_log1p_f16");
142 populateOpPatterns<math::Log2Op>(converter, patterns, f32Func: "__ocml_log2_f32",
143 f64Func: "__ocml_log2_f64", f16Func: "__ocml_log2_f16");
144 populateOpPatterns<math::PowFOp>(converter, patterns, f32Func: "__ocml_pow_f32",
145 f64Func: "__ocml_pow_f64", f16Func: "__ocml_pow_f16");
146 populateOpPatterns<math::RsqrtOp>(converter, patterns, f32Func: "__ocml_rsqrt_f32",
147 f64Func: "__ocml_rsqrt_f64", f16Func: "__ocml_rsqrt_f16");
148 populateOpPatterns<math::SinOp>(converter, patterns, f32Func: "__ocml_sin_f32",
149 f64Func: "__ocml_sin_f64", f16Func: "__ocml_sin_f16");
150 populateOpPatterns<math::TanhOp>(converter, patterns, f32Func: "__ocml_tanh_f32",
151 f64Func: "__ocml_tanh_f64", f16Func: "__ocml_tanh_f16");
152 populateOpPatterns<math::TanOp>(converter, patterns, f32Func: "__ocml_tan_f32",
153 f64Func: "__ocml_tan_f64", f16Func: "__ocml_tan_f16");
154 populateOpPatterns<math::ErfOp>(converter, patterns, f32Func: "__ocml_erf_f32",
155 f64Func: "__ocml_erf_f64", f16Func: "__ocml_erf_f16");
156 populateOpPatterns<math::ErfcOp>(converter, patterns, f32Func: "__ocml_erfc_f32",
157 f64Func: "__ocml_erfc_f64", f16Func: "__ocml_erfc_f16");
158 populateOpPatterns<math::FPowIOp>(converter, patterns, f32Func: "__ocml_pown_f32",
159 f64Func: "__ocml_pown_f64", f16Func: "__ocml_pown_f16");
160 // Single arith pattern that needs a ROCDL call, probably not
161 // worth creating a separate pass for it.
162 populateOpPatterns<arith::RemFOp>(converter, patterns, f32Func: "__ocml_fmod_f32",
163 f64Func: "__ocml_fmod_f64", f16Func: "__ocml_fmod_f16");
164
165 if (target && target->has(feature: llvm::AMDGPU::FEAT_GFX9_INSTS)) {
166 patterns.add<ClampFOpConversion>(arg: converter);
167 } else {
168 LDBG() << "Target dependent patterns were not added";
169 }
170}
171
172struct ConvertMathToROCDLPass final
173 : impl::ConvertMathToROCDLBase<ConvertMathToROCDLPass> {
174 using impl::ConvertMathToROCDLBase<
175 ConvertMathToROCDLPass>::ConvertMathToROCDLBase;
176
177 void runOnOperation() override;
178};
179
180void ConvertMathToROCDLPass::runOnOperation() {
181 auto m = getOperation();
182 MLIRContext *ctx = m.getContext();
183
184 RewritePatternSet patterns(&getContext());
185 LowerToLLVMOptions options(ctx, DataLayout(m));
186 LLVMTypeConverter converter(ctx, options);
187
188 // An empty architecture means "no target", in which case the
189 // target-dependent patterns are simply not added.
190 std::optional<ROCDL::TargetInfo> resolved;
191 StringRef resolvedArch = ROCDL::resolveArchOption(arch, deprecatedAlias: chipset);
192 if (!resolvedArch.empty()) {
193 FailureOr<ROCDL::TargetInfo> targetInfo =
194 ROCDL::TargetInfo::get(arch: resolvedArch, /*waveSize=*/0, emitError: [&] {
195 return emitError(loc: UnknownLoc::get(context: &getContext()));
196 });
197 if (failed(Result: targetInfo))
198 return signalPassFailure();
199 resolved = *targetInfo;
200 }
201 populateMathToROCDLConversionPatterns(converter, patterns, target: resolved);
202
203 ConversionTarget target(getContext());
204 target
205 .addLegalDialect<BuiltinDialect, func::FuncDialect, vector::VectorDialect,
206 LLVM::LLVMDialect, ROCDL::ROCDLDialect>();
207 target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, LLVM::FAbsOp,
208 LLVM::FCeilOp, LLVM::FFloorOp, LLVM::FRemOp, LLVM::LogOp,
209 LLVM::Log10Op, LLVM::Log2Op, LLVM::PowOp, LLVM::SinOp,
210 LLVM::SqrtOp>();
211 if (failed(Result: applyPartialConversion(op: m, target, patterns: std::move(patterns))))
212 signalPassFailure();
213}
214