| 1 | //===-- MathToXeVM.cpp - conversion from Math to XeVM ---------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/MathToXeVM/MathToXeVM.h" |
| 10 | #include "mlir/Analysis/DataLayoutAnalysis.h" |
| 11 | #include "mlir/Conversion/ArithAttrToLLVM/AttrToLLVMConverter.h" |
| 12 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 13 | #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 17 | #include "mlir/Dialect/Math/IR/Math.h" |
| 18 | #include "mlir/Dialect/Math/Transforms/Passes.h" |
| 19 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 20 | #include "mlir/IR/BuiltinDialect.h" |
| 21 | #include "mlir/Pass/Pass.h" |
| 22 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 23 | #include "llvm/Support/FormatVariadic.h" |
| 24 | |
| 25 | #include "../GPUCommon/GPUOpsLowering.h" |
| 26 | #include "../GPUCommon/OpToFuncCallLowering.h" |
| 27 | |
| 28 | namespace mlir { |
| 29 | #define GEN_PASS_DEF_CONVERTMATHTOXEVM |
| 30 | #include "mlir/Conversion/Passes.h.inc" |
| 31 | } // namespace mlir |
| 32 | |
| 33 | using namespace mlir; |
| 34 | |
| 35 | #define DEBUG_TYPE "math-to-xevm" |
| 36 | |
| 37 | static bool isSizeOneVector(Type type) { |
| 38 | auto vecType = dyn_cast<VectorType>(Val&: type); |
| 39 | return vecType && vecType.getShape().size() == 1 && |
| 40 | vecType.getShape()[0] == 1 && vecType.getElementType().isFloat(); |
| 41 | } |
| 42 | |
| 43 | static bool isSPIRVCompatibleFloatOrVec(Type type) { |
| 44 | if (type.isFloat()) |
| 45 | return true; |
| 46 | if (auto vecType = dyn_cast<VectorType>(Val&: type)) { |
| 47 | if (!vecType.getElementType().isFloat()) |
| 48 | return false; |
| 49 | // SPIRV distinguishes between vectors and matrices: OpenCL native math |
| 50 | // intrsinics are not compatible with matrices. |
| 51 | ArrayRef<int64_t> shape = vecType.getShape(); |
| 52 | if (shape.size() != 1) |
| 53 | return false; |
| 54 | // SPIRV has no size-1 vector type; such degenerate vectors are handled |
| 55 | // by unwrapping to the scalar intrinsic (see matchAndRewrite). |
| 56 | if (shape[0] == 1) |
| 57 | return true; |
| 58 | // SPIRV only allows vectors of size 2, 3, 4, 8, 16. |
| 59 | if (shape[0] == 2 || shape[0] == 3 || shape[0] == 4 || shape[0] == 8 || |
| 60 | shape[0] == 16) |
| 61 | return true; |
| 62 | } |
| 63 | return false; |
| 64 | } |
| 65 | |
| 66 | /// Convert math ops marked with `fast` (`afn`) to native OpenCL intrinsics. |
| 67 | template <typename Op> |
| 68 | struct ConvertNativeFuncPattern final : public OpConversionPattern<Op> { |
| 69 | |
| 70 | ConvertNativeFuncPattern(MLIRContext *context, StringRef nativeFunc, |
| 71 | PatternBenefit benefit = 1) |
| 72 | : OpConversionPattern<Op>(context, benefit), nativeFunc(nativeFunc) {} |
| 73 | |
| 74 | inline std::string |
| 75 | getMangledNativeFuncName(const ArrayRef<Type> operandTypes) const { |
| 76 | std::string mangledFuncName = |
| 77 | "_Z" + std::to_string(val: nativeFunc.size()) + nativeFunc.str(); |
| 78 | |
| 79 | auto appendFloatToMangledFunc = [&mangledFuncName](Type type) { |
| 80 | if (type.isF32()) |
| 81 | mangledFuncName += "f" ; |
| 82 | else if (type.isF16()) |
| 83 | mangledFuncName += "Dh" ; |
| 84 | else if (type.isF64()) |
| 85 | mangledFuncName += "d" ; |
| 86 | }; |
| 87 | |
| 88 | for (auto type : operandTypes) { |
| 89 | if (auto vecType = dyn_cast<VectorType>(Val&: type)) { |
| 90 | mangledFuncName += "Dv" + std::to_string(val: vecType.getShape()[0]) + "_" ; |
| 91 | appendFloatToMangledFunc(vecType.getElementType()); |
| 92 | } else |
| 93 | appendFloatToMangledFunc(type); |
| 94 | } |
| 95 | |
| 96 | return mangledFuncName; |
| 97 | } |
| 98 | |
| 99 | LogicalResult |
| 100 | matchAndRewrite(Op op, typename Op::Adaptor adaptor, |
| 101 | ConversionPatternRewriter &rewriter) const override { |
| 102 | if (!isSPIRVCompatibleFloatOrVec(op.getType())) |
| 103 | return failure(); |
| 104 | |
| 105 | arith::FastMathFlags fastFlags = op.getFastmath(); |
| 106 | if (!arith::bitEnumContainsAll(bits: fastFlags, bit: arith::FastMathFlags::afn)) |
| 107 | return rewriter.notifyMatchFailure(op, "not a fastmath `afn` operation" ); |
| 108 | |
| 109 | Location loc = op.getLoc(); |
| 110 | |
| 111 | // SPIRV has no size-1 vector type: such vectors are the degenerate result |
| 112 | // of distributing/linearizing larger vectors down to a single element (e.g. |
| 113 | // by the XeGPU lowering pipeline). They have no OpenCL vector intrinsic, so |
| 114 | // unwrap them to the scalar element type and use the scalar intrinsic. |
| 115 | SmallVector<Value, 1> operands(adaptor.getOperands()); |
| 116 | SmallVector<Type, 1> operandTypes; |
| 117 | bool unwrapSizeOneVec = isSizeOneVector(op.getType()); |
| 118 | for (Value &operand : operands) { |
| 119 | Type opTy = operand.getType(); |
| 120 | // This pass only supports operations on vectors that are already in SPIRV |
| 121 | // supported vector sizes: Distributing unsupported vector sizes to SPIRV |
| 122 | // supported vector sizes are done in other blocking optimization passes. |
| 123 | if (!isSPIRVCompatibleFloatOrVec(type: opTy)) |
| 124 | return rewriter.notifyMatchFailure( |
| 125 | op, llvm::formatv(Fmt: "incompatible operand type: '{0}'" , Vals&: opTy)); |
| 126 | if (unwrapSizeOneVec) { |
| 127 | assert(isSizeOneVector(opTy) && |
| 128 | "expected all operands to be size-1 vectors" ); |
| 129 | opTy = cast<VectorType>(Val&: opTy).getElementType(); |
| 130 | operand = vector::ExtractOp::create(builder&: rewriter, location: loc, source: operand, |
| 131 | position: ArrayRef<int64_t>{0}); |
| 132 | } |
| 133 | operandTypes.push_back(Elt: opTy); |
| 134 | } |
| 135 | |
| 136 | Type resultType = unwrapSizeOneVec |
| 137 | ? cast<VectorType>(op.getType()).getElementType() |
| 138 | : op.getType(); |
| 139 | |
| 140 | auto moduleOp = op->template getParentWithTrait<OpTrait::SymbolTable>(); |
| 141 | auto funcOpRes = LLVM::lookupOrCreateFn( |
| 142 | b&: rewriter, moduleOp, name: getMangledNativeFuncName(operandTypes), |
| 143 | paramTypes: operandTypes, resultType); |
| 144 | assert(!failed(funcOpRes)); |
| 145 | LLVM::LLVMFuncOp funcOp = funcOpRes.value(); |
| 146 | |
| 147 | auto callOp = LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcOp, args: operands); |
| 148 | // Preserve fastmath flags in our MLIR op when converting to llvm function |
| 149 | // calls, in order to allow further fastmath optimizations: We thus need to |
| 150 | // convert arith fastmath attrs into attrs recognized by llvm. |
| 151 | arith::AttrConvertFastMathToLLVM<Op, LLVM::CallOp> fastAttrConverter(op); |
| 152 | callOp.setFastmathFlagsAttr( |
| 153 | fastAttrConverter.getProperties().getFastmathFlags()); |
| 154 | callOp->setDiscardableAttrs(fastAttrConverter.getDiscardableAttrs()); |
| 155 | |
| 156 | if (unwrapSizeOneVec) { |
| 157 | // Re-wrap the scalar result back into a size-1 vector to preserve types. |
| 158 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, op.getType(), |
| 159 | callOp.getResult()); |
| 160 | } else { |
| 161 | rewriter.replaceOp(op, callOp); |
| 162 | } |
| 163 | return success(); |
| 164 | } |
| 165 | |
| 166 | const StringRef nativeFunc; |
| 167 | }; |
| 168 | |
| 169 | template <typename OpTy> |
| 170 | static void populateOCLExtSetOpPatterns(const LLVMTypeConverter &converter, |
| 171 | RewritePatternSet &patterns, |
| 172 | PatternBenefit benefit, |
| 173 | StringRef opName) { |
| 174 | std::string prefix = "__spirv_ocl_" ; |
| 175 | std::string mangledName = "_Z" + |
| 176 | std::to_string(val: prefix.size() + opName.size()) + |
| 177 | prefix + opName.str(); |
| 178 | |
| 179 | patterns.add<ScalarizeVectorOpLowering<OpTy>>(converter, benefit); |
| 180 | patterns.add<OpToFuncCallLowering<OpTy>>( |
| 181 | converter, mangledName + "f" , mangledName + "d" , |
| 182 | /*f32ApproxFunc=*/"" , /*f16Func=*/"" , |
| 183 | /*i32Func=*/"" , benefit, LLVM::cconv::CConv::SPIR_FUNC); |
| 184 | } |
| 185 | |
| 186 | void mlir::populateMathToScalarOCLExtSetConversionPatterns( |
| 187 | const LLVMTypeConverter &converter, RewritePatternSet &patterns, |
| 188 | PatternBenefit benefit) { |
| 189 | populateOCLExtSetOpPatterns<math::AcosOp>(converter, patterns, benefit, |
| 190 | opName: "acos" ); |
| 191 | populateOCLExtSetOpPatterns<math::AcoshOp>(converter, patterns, benefit, |
| 192 | opName: "acosh" ); |
| 193 | populateOCLExtSetOpPatterns<math::AsinOp>(converter, patterns, benefit, |
| 194 | opName: "asin" ); |
| 195 | populateOCLExtSetOpPatterns<math::AsinhOp>(converter, patterns, benefit, |
| 196 | opName: "asinh" ); |
| 197 | populateOCLExtSetOpPatterns<math::AtanOp>(converter, patterns, benefit, |
| 198 | opName: "atan" ); |
| 199 | populateOCLExtSetOpPatterns<math::Atan2Op>(converter, patterns, benefit, |
| 200 | opName: "atan2" ); |
| 201 | populateOCLExtSetOpPatterns<math::AtanhOp>(converter, patterns, benefit, |
| 202 | opName: "atanh" ); |
| 203 | populateOCLExtSetOpPatterns<math::CbrtOp>(converter, patterns, benefit, |
| 204 | opName: "cbrt" ); |
| 205 | populateOCLExtSetOpPatterns<math::CopySignOp>(converter, patterns, benefit, |
| 206 | opName: "copysign" ); |
| 207 | populateOCLExtSetOpPatterns<math::CosOp>(converter, patterns, benefit, opName: "cos" ); |
| 208 | populateOCLExtSetOpPatterns<math::CoshOp>(converter, patterns, benefit, |
| 209 | opName: "cosh" ); |
| 210 | populateOCLExtSetOpPatterns<math::ErfOp>(converter, patterns, benefit, opName: "erf" ); |
| 211 | populateOCLExtSetOpPatterns<math::ErfcOp>(converter, patterns, benefit, |
| 212 | opName: "erfc" ); |
| 213 | populateOCLExtSetOpPatterns<math::ExpOp>(converter, patterns, benefit, opName: "exp" ); |
| 214 | populateOCLExtSetOpPatterns<math::Exp2Op>(converter, patterns, benefit, |
| 215 | opName: "exp2" ); |
| 216 | populateOCLExtSetOpPatterns<math::ExpM1Op>(converter, patterns, benefit, |
| 217 | opName: "expm1" ); |
| 218 | populateOCLExtSetOpPatterns<math::LogOp>(converter, patterns, benefit, opName: "log" ); |
| 219 | populateOCLExtSetOpPatterns<math::Log10Op>(converter, patterns, benefit, |
| 220 | opName: "log10" ); |
| 221 | populateOCLExtSetOpPatterns<math::Log1pOp>(converter, patterns, benefit, |
| 222 | opName: "log1p" ); |
| 223 | populateOCLExtSetOpPatterns<math::Log2Op>(converter, patterns, benefit, |
| 224 | opName: "log2" ); |
| 225 | populateOCLExtSetOpPatterns<math::PowFOp>(converter, patterns, benefit, |
| 226 | opName: "pow" ); |
| 227 | populateOCLExtSetOpPatterns<math::RsqrtOp>(converter, patterns, benefit, |
| 228 | opName: "rsqrt" ); |
| 229 | populateOCLExtSetOpPatterns<math::SinOp>(converter, patterns, benefit, opName: "sin" ); |
| 230 | populateOCLExtSetOpPatterns<math::SinhOp>(converter, patterns, benefit, |
| 231 | opName: "sinh" ); |
| 232 | populateOCLExtSetOpPatterns<math::SqrtOp>(converter, patterns, benefit, |
| 233 | opName: "sqrt" ); |
| 234 | populateOCLExtSetOpPatterns<math::TanOp>(converter, patterns, benefit, opName: "tan" ); |
| 235 | populateOCLExtSetOpPatterns<math::TanhOp>(converter, patterns, benefit, |
| 236 | opName: "tanh" ); |
| 237 | } |
| 238 | |
| 239 | void mlir::populateMathToXeVMConversionPatterns(RewritePatternSet &patterns, |
| 240 | bool convertArith, |
| 241 | PatternBenefit benefit) { |
| 242 | patterns.add<ConvertNativeFuncPattern<math::ExpOp>>( |
| 243 | arg: patterns.getContext(), args: "__spirv_ocl_native_exp" , args&: benefit); |
| 244 | patterns.add<ConvertNativeFuncPattern<math::CosOp>>( |
| 245 | arg: patterns.getContext(), args: "__spirv_ocl_native_cos" , args&: benefit); |
| 246 | patterns.add<ConvertNativeFuncPattern<math::Exp2Op>>( |
| 247 | arg: patterns.getContext(), args: "__spirv_ocl_native_exp2" , args&: benefit); |
| 248 | patterns.add<ConvertNativeFuncPattern<math::LogOp>>( |
| 249 | arg: patterns.getContext(), args: "__spirv_ocl_native_log" , args&: benefit); |
| 250 | patterns.add<ConvertNativeFuncPattern<math::Log2Op>>( |
| 251 | arg: patterns.getContext(), args: "__spirv_ocl_native_log2" , args&: benefit); |
| 252 | patterns.add<ConvertNativeFuncPattern<math::Log10Op>>( |
| 253 | arg: patterns.getContext(), args: "__spirv_ocl_native_log10" , args&: benefit); |
| 254 | patterns.add<ConvertNativeFuncPattern<math::PowFOp>>( |
| 255 | arg: patterns.getContext(), args: "__spirv_ocl_native_powr" , args&: benefit); |
| 256 | patterns.add<ConvertNativeFuncPattern<math::RsqrtOp>>( |
| 257 | arg: patterns.getContext(), args: "__spirv_ocl_native_rsqrt" , args&: benefit); |
| 258 | patterns.add<ConvertNativeFuncPattern<math::SinOp>>( |
| 259 | arg: patterns.getContext(), args: "__spirv_ocl_native_sin" , args&: benefit); |
| 260 | patterns.add<ConvertNativeFuncPattern<math::SqrtOp>>( |
| 261 | arg: patterns.getContext(), args: "__spirv_ocl_native_sqrt" , args&: benefit); |
| 262 | patterns.add<ConvertNativeFuncPattern<math::TanOp>>( |
| 263 | arg: patterns.getContext(), args: "__spirv_ocl_native_tan" , args&: benefit); |
| 264 | if (convertArith) |
| 265 | patterns.add<ConvertNativeFuncPattern<arith::DivFOp>>( |
| 266 | arg: patterns.getContext(), args: "__spirv_ocl_native_divide" , args&: benefit); |
| 267 | } |
| 268 | |
| 269 | namespace { |
| 270 | struct ConvertMathToXeVMPass |
| 271 | : public impl::ConvertMathToXeVMBase<ConvertMathToXeVMPass> { |
| 272 | using Base::Base; |
| 273 | void runOnOperation() override; |
| 274 | }; |
| 275 | } // namespace |
| 276 | |
| 277 | void ConvertMathToXeVMPass::runOnOperation() { |
| 278 | Operation *op = getOperation(); |
| 279 | MLIRContext *ctx = op->getContext(); |
| 280 | |
| 281 | // Simplify first, so the cheaper form is what the conversion below lowers. A |
| 282 | // simplification rewrites a whole expression, so it must run before the |
| 283 | // lowering turns the parts of that expression into calls. Only the ops these |
| 284 | // patterns can match are given to the driver, and folding is off, so nothing |
| 285 | // else is touched. |
| 286 | { |
| 287 | RewritePatternSet simplifications(ctx); |
| 288 | populateMathAlgebraicSimplificationPatterns(patterns&: simplifications); |
| 289 | FrozenRewritePatternSet frozen(std::move(simplifications)); |
| 290 | SmallVector<Operation *> candidates; |
| 291 | op->walk(callback: [&](Operation *nested) { |
| 292 | if (frozen.getOpSpecificNativePatterns().contains(Val: nested->getName())) |
| 293 | candidates.push_back(Elt: nested); |
| 294 | }); |
| 295 | GreedyRewriteConfig config; |
| 296 | config.enableFolding(enable: false); |
| 297 | if (failed(Result: applyOpPatternsGreedily(ops: candidates, patterns: frozen, config))) |
| 298 | return signalPassFailure(); |
| 299 | } |
| 300 | |
| 301 | const auto &dl = getAnalysis<DataLayoutAnalysis>(); |
| 302 | |
| 303 | RewritePatternSet patterns(&getContext()); |
| 304 | LowerToLLVMOptions options(ctx, dl.getAtOrAbove(operation: op)); |
| 305 | LLVMTypeConverter converter(ctx, options); |
| 306 | ConversionTarget target(getContext()); |
| 307 | |
| 308 | // The native (`afn`) patterns must outrank the precise OCL patterns: an op |
| 309 | // marked `afn` gets the native intrinsic, and every other op falls through to |
| 310 | // the precise OCL intrinsic. |
| 311 | constexpr unsigned oclBenefit = 1; |
| 312 | populateMathToXeVMConversionPatterns(patterns, convertArith, benefit: oclBenefit + 1); |
| 313 | if (convertToOCL) { |
| 314 | populateMathToScalarOCLExtSetConversionPatterns(converter, patterns, |
| 315 | benefit: oclBenefit); |
| 316 | target |
| 317 | .addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, LLVM::LogOp, |
| 318 | LLVM::Log10Op, LLVM::Log2Op, LLVM::SinOp, LLVM::SqrtOp>(); |
| 319 | } |
| 320 | target.addLegalDialect<BuiltinDialect, LLVM::LLVMDialect>(); |
| 321 | // The size-1-vector patterns unwrap to the scalar intrinsic via |
| 322 | // vector.extract / vector.broadcast; these must be legal for the partial |
| 323 | // conversion to succeed. |
| 324 | target.addLegalOp<vector::ExtractOp, vector::BroadcastOp>(); |
| 325 | if (failed( |
| 326 | Result: applyPartialConversion(op: getOperation(), target, patterns: std::move(patterns)))) |
| 327 | signalPassFailure(); |
| 328 | } |
| 329 | |