1//===-- MathToXeVM.cpp - conversion from Math to XeVM ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/MathToXeVM/MathToXeVM.h"
10#include "mlir/Analysis/DataLayoutAnalysis.h"
11#include "mlir/Conversion/ArithAttrToLLVM/AttrToLLVMConverter.h"
12#include "mlir/Dialect/Arith/IR/Arith.h"
13#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
14#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
15#include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
17#include "mlir/Dialect/Math/IR/Math.h"
18#include "mlir/Dialect/Math/Transforms/Passes.h"
19#include "mlir/Dialect/Vector/IR/VectorOps.h"
20#include "mlir/IR/BuiltinDialect.h"
21#include "mlir/Pass/Pass.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23#include "llvm/Support/FormatVariadic.h"
24
25#include "../GPUCommon/GPUOpsLowering.h"
26#include "../GPUCommon/OpToFuncCallLowering.h"
27
28namespace mlir {
29#define GEN_PASS_DEF_CONVERTMATHTOXEVM
30#include "mlir/Conversion/Passes.h.inc"
31} // namespace mlir
32
33using namespace mlir;
34
35#define DEBUG_TYPE "math-to-xevm"
36
37static bool isSizeOneVector(Type type) {
38 auto vecType = dyn_cast<VectorType>(Val&: type);
39 return vecType && vecType.getShape().size() == 1 &&
40 vecType.getShape()[0] == 1 && vecType.getElementType().isFloat();
41}
42
43static bool isSPIRVCompatibleFloatOrVec(Type type) {
44 if (type.isFloat())
45 return true;
46 if (auto vecType = dyn_cast<VectorType>(Val&: type)) {
47 if (!vecType.getElementType().isFloat())
48 return false;
49 // SPIRV distinguishes between vectors and matrices: OpenCL native math
50 // intrsinics are not compatible with matrices.
51 ArrayRef<int64_t> shape = vecType.getShape();
52 if (shape.size() != 1)
53 return false;
54 // SPIRV has no size-1 vector type; such degenerate vectors are handled
55 // by unwrapping to the scalar intrinsic (see matchAndRewrite).
56 if (shape[0] == 1)
57 return true;
58 // SPIRV only allows vectors of size 2, 3, 4, 8, 16.
59 if (shape[0] == 2 || shape[0] == 3 || shape[0] == 4 || shape[0] == 8 ||
60 shape[0] == 16)
61 return true;
62 }
63 return false;
64}
65
66/// Convert math ops marked with `fast` (`afn`) to native OpenCL intrinsics.
67template <typename Op>
68struct ConvertNativeFuncPattern final : public OpConversionPattern<Op> {
69
70 ConvertNativeFuncPattern(MLIRContext *context, StringRef nativeFunc,
71 PatternBenefit benefit = 1)
72 : OpConversionPattern<Op>(context, benefit), nativeFunc(nativeFunc) {}
73
74 inline std::string
75 getMangledNativeFuncName(const ArrayRef<Type> operandTypes) const {
76 std::string mangledFuncName =
77 "_Z" + std::to_string(val: nativeFunc.size()) + nativeFunc.str();
78
79 auto appendFloatToMangledFunc = [&mangledFuncName](Type type) {
80 if (type.isF32())
81 mangledFuncName += "f";
82 else if (type.isF16())
83 mangledFuncName += "Dh";
84 else if (type.isF64())
85 mangledFuncName += "d";
86 };
87
88 for (auto type : operandTypes) {
89 if (auto vecType = dyn_cast<VectorType>(Val&: type)) {
90 mangledFuncName += "Dv" + std::to_string(val: vecType.getShape()[0]) + "_";
91 appendFloatToMangledFunc(vecType.getElementType());
92 } else
93 appendFloatToMangledFunc(type);
94 }
95
96 return mangledFuncName;
97 }
98
99 LogicalResult
100 matchAndRewrite(Op op, typename Op::Adaptor adaptor,
101 ConversionPatternRewriter &rewriter) const override {
102 if (!isSPIRVCompatibleFloatOrVec(op.getType()))
103 return failure();
104
105 arith::FastMathFlags fastFlags = op.getFastmath();
106 if (!arith::bitEnumContainsAll(bits: fastFlags, bit: arith::FastMathFlags::afn))
107 return rewriter.notifyMatchFailure(op, "not a fastmath `afn` operation");
108
109 Location loc = op.getLoc();
110
111 // SPIRV has no size-1 vector type: such vectors are the degenerate result
112 // of distributing/linearizing larger vectors down to a single element (e.g.
113 // by the XeGPU lowering pipeline). They have no OpenCL vector intrinsic, so
114 // unwrap them to the scalar element type and use the scalar intrinsic.
115 SmallVector<Value, 1> operands(adaptor.getOperands());
116 SmallVector<Type, 1> operandTypes;
117 bool unwrapSizeOneVec = isSizeOneVector(op.getType());
118 for (Value &operand : operands) {
119 Type opTy = operand.getType();
120 // This pass only supports operations on vectors that are already in SPIRV
121 // supported vector sizes: Distributing unsupported vector sizes to SPIRV
122 // supported vector sizes are done in other blocking optimization passes.
123 if (!isSPIRVCompatibleFloatOrVec(type: opTy))
124 return rewriter.notifyMatchFailure(
125 op, llvm::formatv(Fmt: "incompatible operand type: '{0}'", Vals&: opTy));
126 if (unwrapSizeOneVec) {
127 assert(isSizeOneVector(opTy) &&
128 "expected all operands to be size-1 vectors");
129 opTy = cast<VectorType>(Val&: opTy).getElementType();
130 operand = vector::ExtractOp::create(builder&: rewriter, location: loc, source: operand,
131 position: ArrayRef<int64_t>{0});
132 }
133 operandTypes.push_back(Elt: opTy);
134 }
135
136 Type resultType = unwrapSizeOneVec
137 ? cast<VectorType>(op.getType()).getElementType()
138 : op.getType();
139
140 auto moduleOp = op->template getParentWithTrait<OpTrait::SymbolTable>();
141 auto funcOpRes = LLVM::lookupOrCreateFn(
142 b&: rewriter, moduleOp, name: getMangledNativeFuncName(operandTypes),
143 paramTypes: operandTypes, resultType);
144 assert(!failed(funcOpRes));
145 LLVM::LLVMFuncOp funcOp = funcOpRes.value();
146
147 auto callOp = LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcOp, args: operands);
148 // Preserve fastmath flags in our MLIR op when converting to llvm function
149 // calls, in order to allow further fastmath optimizations: We thus need to
150 // convert arith fastmath attrs into attrs recognized by llvm.
151 arith::AttrConvertFastMathToLLVM<Op, LLVM::CallOp> fastAttrConverter(op);
152 callOp.setFastmathFlagsAttr(
153 fastAttrConverter.getProperties().getFastmathFlags());
154 callOp->setDiscardableAttrs(fastAttrConverter.getDiscardableAttrs());
155
156 if (unwrapSizeOneVec) {
157 // Re-wrap the scalar result back into a size-1 vector to preserve types.
158 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, op.getType(),
159 callOp.getResult());
160 } else {
161 rewriter.replaceOp(op, callOp);
162 }
163 return success();
164 }
165
166 const StringRef nativeFunc;
167};
168
169template <typename OpTy>
170static void populateOCLExtSetOpPatterns(const LLVMTypeConverter &converter,
171 RewritePatternSet &patterns,
172 PatternBenefit benefit,
173 StringRef opName) {
174 std::string prefix = "__spirv_ocl_";
175 std::string mangledName = "_Z" +
176 std::to_string(val: prefix.size() + opName.size()) +
177 prefix + opName.str();
178
179 patterns.add<ScalarizeVectorOpLowering<OpTy>>(converter, benefit);
180 patterns.add<OpToFuncCallLowering<OpTy>>(
181 converter, mangledName + "f", mangledName + "d",
182 /*f32ApproxFunc=*/"", /*f16Func=*/"",
183 /*i32Func=*/"", benefit, LLVM::cconv::CConv::SPIR_FUNC);
184}
185
186void mlir::populateMathToScalarOCLExtSetConversionPatterns(
187 const LLVMTypeConverter &converter, RewritePatternSet &patterns,
188 PatternBenefit benefit) {
189 populateOCLExtSetOpPatterns<math::AcosOp>(converter, patterns, benefit,
190 opName: "acos");
191 populateOCLExtSetOpPatterns<math::AcoshOp>(converter, patterns, benefit,
192 opName: "acosh");
193 populateOCLExtSetOpPatterns<math::AsinOp>(converter, patterns, benefit,
194 opName: "asin");
195 populateOCLExtSetOpPatterns<math::AsinhOp>(converter, patterns, benefit,
196 opName: "asinh");
197 populateOCLExtSetOpPatterns<math::AtanOp>(converter, patterns, benefit,
198 opName: "atan");
199 populateOCLExtSetOpPatterns<math::Atan2Op>(converter, patterns, benefit,
200 opName: "atan2");
201 populateOCLExtSetOpPatterns<math::AtanhOp>(converter, patterns, benefit,
202 opName: "atanh");
203 populateOCLExtSetOpPatterns<math::CbrtOp>(converter, patterns, benefit,
204 opName: "cbrt");
205 populateOCLExtSetOpPatterns<math::CopySignOp>(converter, patterns, benefit,
206 opName: "copysign");
207 populateOCLExtSetOpPatterns<math::CosOp>(converter, patterns, benefit, opName: "cos");
208 populateOCLExtSetOpPatterns<math::CoshOp>(converter, patterns, benefit,
209 opName: "cosh");
210 populateOCLExtSetOpPatterns<math::ErfOp>(converter, patterns, benefit, opName: "erf");
211 populateOCLExtSetOpPatterns<math::ErfcOp>(converter, patterns, benefit,
212 opName: "erfc");
213 populateOCLExtSetOpPatterns<math::ExpOp>(converter, patterns, benefit, opName: "exp");
214 populateOCLExtSetOpPatterns<math::Exp2Op>(converter, patterns, benefit,
215 opName: "exp2");
216 populateOCLExtSetOpPatterns<math::ExpM1Op>(converter, patterns, benefit,
217 opName: "expm1");
218 populateOCLExtSetOpPatterns<math::LogOp>(converter, patterns, benefit, opName: "log");
219 populateOCLExtSetOpPatterns<math::Log10Op>(converter, patterns, benefit,
220 opName: "log10");
221 populateOCLExtSetOpPatterns<math::Log1pOp>(converter, patterns, benefit,
222 opName: "log1p");
223 populateOCLExtSetOpPatterns<math::Log2Op>(converter, patterns, benefit,
224 opName: "log2");
225 populateOCLExtSetOpPatterns<math::PowFOp>(converter, patterns, benefit,
226 opName: "pow");
227 populateOCLExtSetOpPatterns<math::RsqrtOp>(converter, patterns, benefit,
228 opName: "rsqrt");
229 populateOCLExtSetOpPatterns<math::SinOp>(converter, patterns, benefit, opName: "sin");
230 populateOCLExtSetOpPatterns<math::SinhOp>(converter, patterns, benefit,
231 opName: "sinh");
232 populateOCLExtSetOpPatterns<math::SqrtOp>(converter, patterns, benefit,
233 opName: "sqrt");
234 populateOCLExtSetOpPatterns<math::TanOp>(converter, patterns, benefit, opName: "tan");
235 populateOCLExtSetOpPatterns<math::TanhOp>(converter, patterns, benefit,
236 opName: "tanh");
237}
238
239void mlir::populateMathToXeVMConversionPatterns(RewritePatternSet &patterns,
240 bool convertArith,
241 PatternBenefit benefit) {
242 patterns.add<ConvertNativeFuncPattern<math::ExpOp>>(
243 arg: patterns.getContext(), args: "__spirv_ocl_native_exp", args&: benefit);
244 patterns.add<ConvertNativeFuncPattern<math::CosOp>>(
245 arg: patterns.getContext(), args: "__spirv_ocl_native_cos", args&: benefit);
246 patterns.add<ConvertNativeFuncPattern<math::Exp2Op>>(
247 arg: patterns.getContext(), args: "__spirv_ocl_native_exp2", args&: benefit);
248 patterns.add<ConvertNativeFuncPattern<math::LogOp>>(
249 arg: patterns.getContext(), args: "__spirv_ocl_native_log", args&: benefit);
250 patterns.add<ConvertNativeFuncPattern<math::Log2Op>>(
251 arg: patterns.getContext(), args: "__spirv_ocl_native_log2", args&: benefit);
252 patterns.add<ConvertNativeFuncPattern<math::Log10Op>>(
253 arg: patterns.getContext(), args: "__spirv_ocl_native_log10", args&: benefit);
254 patterns.add<ConvertNativeFuncPattern<math::PowFOp>>(
255 arg: patterns.getContext(), args: "__spirv_ocl_native_powr", args&: benefit);
256 patterns.add<ConvertNativeFuncPattern<math::RsqrtOp>>(
257 arg: patterns.getContext(), args: "__spirv_ocl_native_rsqrt", args&: benefit);
258 patterns.add<ConvertNativeFuncPattern<math::SinOp>>(
259 arg: patterns.getContext(), args: "__spirv_ocl_native_sin", args&: benefit);
260 patterns.add<ConvertNativeFuncPattern<math::SqrtOp>>(
261 arg: patterns.getContext(), args: "__spirv_ocl_native_sqrt", args&: benefit);
262 patterns.add<ConvertNativeFuncPattern<math::TanOp>>(
263 arg: patterns.getContext(), args: "__spirv_ocl_native_tan", args&: benefit);
264 if (convertArith)
265 patterns.add<ConvertNativeFuncPattern<arith::DivFOp>>(
266 arg: patterns.getContext(), args: "__spirv_ocl_native_divide", args&: benefit);
267}
268
269namespace {
270struct ConvertMathToXeVMPass
271 : public impl::ConvertMathToXeVMBase<ConvertMathToXeVMPass> {
272 using Base::Base;
273 void runOnOperation() override;
274};
275} // namespace
276
277void ConvertMathToXeVMPass::runOnOperation() {
278 Operation *op = getOperation();
279 MLIRContext *ctx = op->getContext();
280
281 // Simplify first, so the cheaper form is what the conversion below lowers. A
282 // simplification rewrites a whole expression, so it must run before the
283 // lowering turns the parts of that expression into calls. Only the ops these
284 // patterns can match are given to the driver, and folding is off, so nothing
285 // else is touched.
286 {
287 RewritePatternSet simplifications(ctx);
288 populateMathAlgebraicSimplificationPatterns(patterns&: simplifications);
289 FrozenRewritePatternSet frozen(std::move(simplifications));
290 SmallVector<Operation *> candidates;
291 op->walk(callback: [&](Operation *nested) {
292 if (frozen.getOpSpecificNativePatterns().contains(Val: nested->getName()))
293 candidates.push_back(Elt: nested);
294 });
295 GreedyRewriteConfig config;
296 config.enableFolding(enable: false);
297 if (failed(Result: applyOpPatternsGreedily(ops: candidates, patterns: frozen, config)))
298 return signalPassFailure();
299 }
300
301 const auto &dl = getAnalysis<DataLayoutAnalysis>();
302
303 RewritePatternSet patterns(&getContext());
304 LowerToLLVMOptions options(ctx, dl.getAtOrAbove(operation: op));
305 LLVMTypeConverter converter(ctx, options);
306 ConversionTarget target(getContext());
307
308 // The native (`afn`) patterns must outrank the precise OCL patterns: an op
309 // marked `afn` gets the native intrinsic, and every other op falls through to
310 // the precise OCL intrinsic.
311 constexpr unsigned oclBenefit = 1;
312 populateMathToXeVMConversionPatterns(patterns, convertArith, benefit: oclBenefit + 1);
313 if (convertToOCL) {
314 populateMathToScalarOCLExtSetConversionPatterns(converter, patterns,
315 benefit: oclBenefit);
316 target
317 .addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, LLVM::LogOp,
318 LLVM::Log10Op, LLVM::Log2Op, LLVM::SinOp, LLVM::SqrtOp>();
319 }
320 target.addLegalDialect<BuiltinDialect, LLVM::LLVMDialect>();
321 // The size-1-vector patterns unwrap to the scalar intrinsic via
322 // vector.extract / vector.broadcast; these must be legal for the partial
323 // conversion to succeed.
324 target.addLegalOp<vector::ExtractOp, vector::BroadcastOp>();
325 if (failed(
326 Result: applyPartialConversion(op: getOperation(), target, patterns: std::move(patterns))))
327 signalPassFailure();
328}
329