| 1 | //===-- MathToNVVM.cpp - conversion from Math to CUDA libdevice calls ----===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/MathToNVVM/MathToNVVM.h" |
| 10 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 11 | #include "mlir/Dialect/Func/IR/FuncDialect.h" |
| 12 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 13 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 19 | #include "mlir/Dialect/LLVMIR/NVVMDialectDecl.h" |
| 20 | #include "mlir/Dialect/Math/IR/Math.h" |
| 21 | #include "mlir/Dialect/Vector/IR/VectorDialect.h" |
| 22 | #include "mlir/IR/BuiltinDialect.h" |
| 23 | #include "mlir/Pass/Pass.h" |
| 24 | |
| 25 | #include "../GPUCommon/GPUOpsLowering.h" |
| 26 | #include "../GPUCommon/OpToFuncCallLowering.h" |
| 27 | |
| 28 | namespace mlir { |
| 29 | #define GEN_PASS_DEF_CONVERTMATHTONVVM |
| 30 | #include "mlir/Conversion/Passes.h.inc" |
| 31 | } // namespace mlir |
| 32 | |
| 33 | using namespace mlir; |
| 34 | |
| 35 | #define DEBUG_TYPE "math-to-nvvm" |
| 36 | |
| 37 | template <typename OpTy> |
| 38 | static void populateOpPatterns(const LLVMTypeConverter &converter, |
| 39 | RewritePatternSet &patterns, |
| 40 | PatternBenefit benefit, StringRef f32Func, |
| 41 | StringRef f64Func, StringRef f32ApproxFunc = "" , |
| 42 | StringRef f16Func = "" ) { |
| 43 | patterns.add<ScalarizeVectorOpLowering<OpTy>>(converter, benefit); |
| 44 | patterns.add<OpToFuncCallLowering<OpTy>>(converter, f32Func, f64Func, |
| 45 | f32ApproxFunc, f16Func, |
| 46 | /*i32Func=*/"" , benefit); |
| 47 | } |
| 48 | |
| 49 | template <typename OpTy> |
| 50 | static void populateIntOpPatterns(const LLVMTypeConverter &converter, |
| 51 | RewritePatternSet &patterns, |
| 52 | PatternBenefit benefit, StringRef i32Func) { |
| 53 | patterns.add<ScalarizeVectorOpLowering<OpTy>>(converter, benefit); |
| 54 | patterns.add<OpToFuncCallLowering<OpTy>>(converter, "" , "" , "" , "" , i32Func, |
| 55 | benefit); |
| 56 | } |
| 57 | |
| 58 | /// libdevice's __nv_powi and __nv_powif take an i32 exponent, so math.fpowi |
| 59 | /// with a wider or index exponent is not lowered. |
| 60 | static bool exponentFitsInI32(math::FPowIOp op) { |
| 61 | auto type = |
| 62 | dyn_cast<IntegerType>(Val: getElementTypeOrSelf(type: op.getRhs().getType())); |
| 63 | return type && type.getWidth() <= 32; |
| 64 | } |
| 65 | |
| 66 | namespace { |
| 67 | struct FPowIScalarizeLowering |
| 68 | : public ScalarizeVectorOpLowering<math::FPowIOp> { |
| 69 | using ScalarizeVectorOpLowering::ScalarizeVectorOpLowering; |
| 70 | |
| 71 | LogicalResult |
| 72 | matchAndRewrite(math::FPowIOp op, OpAdaptor adaptor, |
| 73 | ConversionPatternRewriter &rewriter) const override { |
| 74 | if (!exponentFitsInI32(op)) |
| 75 | return rewriter.notifyMatchFailure(arg&: op, msg: "exponent does not fit in i32" ); |
| 76 | return ScalarizeVectorOpLowering::matchAndRewrite(op, adaptor, rewriter); |
| 77 | } |
| 78 | }; |
| 79 | |
| 80 | struct FPowIOpLowering : public OpToFuncCallLowering<math::FPowIOp> { |
| 81 | using OpToFuncCallLowering::OpToFuncCallLowering; |
| 82 | |
| 83 | LogicalResult |
| 84 | matchAndRewrite(math::FPowIOp op, OpAdaptor adaptor, |
| 85 | ConversionPatternRewriter &rewriter) const override { |
| 86 | if (!exponentFitsInI32(op)) |
| 87 | return rewriter.notifyMatchFailure(arg&: op, msg: "exponent does not fit in i32" ); |
| 88 | // Return before creating anything in the cases OpToFuncCallLowering |
| 89 | // does not handle. |
| 90 | if (!op->getParentOfType<FunctionOpInterface>() || |
| 91 | !isa<Float16Type, BFloat16Type, Float32Type, Float64Type>( |
| 92 | Val: adaptor.getLhs().getType())) |
| 93 | return rewriter.notifyMatchFailure(arg&: op, msg: "cannot be lowered to libdevice" ); |
| 94 | // Sign-extend a narrower exponent. |
| 95 | Value exponent = adaptor.getRhs(); |
| 96 | if (!exponent.getType().isInteger(width: 32)) |
| 97 | exponent = LLVM::SExtOp::create(builder&: rewriter, location: op.getLoc(), |
| 98 | res: rewriter.getI32Type(), arg: exponent); |
| 99 | Value operands[] = {adaptor.getLhs(), exponent}; |
| 100 | return OpToFuncCallLowering::matchAndRewrite( |
| 101 | op, adaptor: OpAdaptor(ValueRange(operands), op), rewriter); |
| 102 | } |
| 103 | }; |
| 104 | } // namespace |
| 105 | |
| 106 | // Custom pattern for sincos since it returns two values |
| 107 | struct SincosOpLowering : public ConvertOpToLLVMPattern<math::SincosOp> { |
| 108 | using ConvertOpToLLVMPattern<math::SincosOp>::ConvertOpToLLVMPattern; |
| 109 | |
| 110 | LogicalResult |
| 111 | matchAndRewrite(math::SincosOp op, OpAdaptor adaptor, |
| 112 | ConversionPatternRewriter &rewriter) const override { |
| 113 | Location loc = op.getLoc(); |
| 114 | Value input = adaptor.getOperand(); |
| 115 | Type inputType = input.getType(); |
| 116 | auto convertedInput = maybeExt(operand: input, rewriter); |
| 117 | auto computeType = convertedInput.getType(); |
| 118 | |
| 119 | StringRef sincosFunc; |
| 120 | if (isa<Float32Type>(Val: computeType)) { |
| 121 | const arith::FastMathFlags flag = op.getFastmath(); |
| 122 | const bool useApprox = |
| 123 | mlir::arith::bitEnumContainsAny(bits: flag, bit: arith::FastMathFlags::afn); |
| 124 | sincosFunc = useApprox ? "__nv_fast_sincosf" : "__nv_sincosf" ; |
| 125 | } else if (isa<Float64Type>(Val: computeType)) { |
| 126 | sincosFunc = "__nv_sincos" ; |
| 127 | } else { |
| 128 | return rewriter.notifyMatchFailure(arg&: op, |
| 129 | msg: "unsupported operand type for sincos" ); |
| 130 | } |
| 131 | |
| 132 | auto ptrType = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 133 | |
| 134 | Value sinPtr, cosPtr; |
| 135 | { |
| 136 | OpBuilder::InsertionGuard guard(rewriter); |
| 137 | auto *scope = |
| 138 | op->getParentWithTrait<mlir::OpTrait::AutomaticAllocationScope>(); |
| 139 | assert(scope && "Expected op to be inside automatic allocation scope" ); |
| 140 | rewriter.setInsertionPointToStart(&scope->getRegion(index: 0).front()); |
| 141 | auto one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: rewriter.getI32Type(), |
| 142 | value: rewriter.getI32IntegerAttr(value: 1)); |
| 143 | sinPtr = |
| 144 | LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: computeType, arraySize: one, alignment: 0); |
| 145 | cosPtr = |
| 146 | LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: computeType, arraySize: one, alignment: 0); |
| 147 | } |
| 148 | |
| 149 | createSincosCall(rewriter, loc, funcName: sincosFunc, input: convertedInput, sinPtr, cosPtr, |
| 150 | op); |
| 151 | |
| 152 | auto sinResult = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: computeType, addr: sinPtr); |
| 153 | auto cosResult = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: computeType, addr: cosPtr); |
| 154 | |
| 155 | rewriter.replaceOp(op, newValues: {maybeTrunc(operand: sinResult, type: inputType, rewriter), |
| 156 | maybeTrunc(operand: cosResult, type: inputType, rewriter)}); |
| 157 | return success(); |
| 158 | } |
| 159 | |
| 160 | private: |
| 161 | Value maybeExt(Value operand, PatternRewriter &rewriter) const { |
| 162 | if (isa<Float16Type, BFloat16Type>(Val: operand.getType())) |
| 163 | return LLVM::FPExtOp::create(builder&: rewriter, location: operand.getLoc(), |
| 164 | res: Float32Type::get(context: rewriter.getContext()), |
| 165 | arg: operand); |
| 166 | return operand; |
| 167 | } |
| 168 | |
| 169 | Value maybeTrunc(Value operand, Type type, PatternRewriter &rewriter) const { |
| 170 | if (operand.getType() != type) |
| 171 | return LLVM::FPTruncOp::create(builder&: rewriter, location: operand.getLoc(), res: type, arg: operand); |
| 172 | return operand; |
| 173 | } |
| 174 | |
| 175 | void createSincosCall(ConversionPatternRewriter &rewriter, Location loc, |
| 176 | StringRef funcName, Value input, Value sinPtr, |
| 177 | Value cosPtr, Operation *op) const { |
| 178 | auto voidType = LLVM::LLVMVoidType::get(ctx: rewriter.getContext()); |
| 179 | auto ptrType = sinPtr.getType(); |
| 180 | |
| 181 | SmallVector<Type> operandTypes = {input.getType(), ptrType, ptrType}; |
| 182 | auto funcType = LLVM::LLVMFunctionType::get(result: voidType, arguments: operandTypes); |
| 183 | |
| 184 | auto funcAttr = StringAttr::get(context: op->getContext(), bytes: funcName); |
| 185 | auto funcOp = |
| 186 | SymbolTable::lookupNearestSymbolFrom<LLVM::LLVMFuncOp>(from: op, symbol: funcAttr); |
| 187 | |
| 188 | if (!funcOp) { |
| 189 | auto parentFunc = op->getParentOfType<FunctionOpInterface>(); |
| 190 | assert(parentFunc && "expected there to be a parent function" ); |
| 191 | OpBuilder b(parentFunc); |
| 192 | |
| 193 | auto globalloc = loc->findInstanceOfOrUnknown<FileLineColLoc>(); |
| 194 | funcOp = LLVM::LLVMFuncOp::create(builder&: b, location: globalloc, name: funcName, type: funcType); |
| 195 | } |
| 196 | |
| 197 | SmallVector<Value> callOperands = {input, sinPtr, cosPtr}; |
| 198 | LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcOp, args: callOperands); |
| 199 | } |
| 200 | }; |
| 201 | |
| 202 | void mlir::populateLibDeviceConversionPatterns( |
| 203 | const LLVMTypeConverter &converter, RewritePatternSet &patterns, |
| 204 | PatternBenefit benefit) { |
| 205 | populateOpPatterns<arith::RemFOp>(converter, patterns, benefit, f32Func: "__nv_fmodf" , |
| 206 | f64Func: "__nv_fmod" ); |
| 207 | populateOpPatterns<arith::MaxNumFOp>(converter, patterns, benefit, |
| 208 | f32Func: "__nv_fmaxf" , f64Func: "__nv_fmax" ); |
| 209 | populateOpPatterns<arith::MinNumFOp>(converter, patterns, benefit, |
| 210 | f32Func: "__nv_fminf" , f64Func: "__nv_fmin" ); |
| 211 | |
| 212 | populateIntOpPatterns<math::AbsIOp>(converter, patterns, benefit, i32Func: "__nv_abs" ); |
| 213 | populateOpPatterns<math::AbsFOp>(converter, patterns, benefit, f32Func: "__nv_fabsf" , |
| 214 | f64Func: "__nv_fabs" ); |
| 215 | populateOpPatterns<math::AcosOp>(converter, patterns, benefit, f32Func: "__nv_acosf" , |
| 216 | f64Func: "__nv_acos" ); |
| 217 | populateOpPatterns<math::AcoshOp>(converter, patterns, benefit, f32Func: "__nv_acoshf" , |
| 218 | f64Func: "__nv_acosh" ); |
| 219 | populateOpPatterns<math::AsinOp>(converter, patterns, benefit, f32Func: "__nv_asinf" , |
| 220 | f64Func: "__nv_asin" ); |
| 221 | populateOpPatterns<math::AsinhOp>(converter, patterns, benefit, f32Func: "__nv_asinhf" , |
| 222 | f64Func: "__nv_asinh" ); |
| 223 | populateOpPatterns<math::AtanOp>(converter, patterns, benefit, f32Func: "__nv_atanf" , |
| 224 | f64Func: "__nv_atan" ); |
| 225 | populateOpPatterns<math::Atan2Op>(converter, patterns, benefit, f32Func: "__nv_atan2f" , |
| 226 | f64Func: "__nv_atan2" ); |
| 227 | populateOpPatterns<math::AtanhOp>(converter, patterns, benefit, f32Func: "__nv_atanhf" , |
| 228 | f64Func: "__nv_atanh" ); |
| 229 | populateOpPatterns<math::CbrtOp>(converter, patterns, benefit, f32Func: "__nv_cbrtf" , |
| 230 | f64Func: "__nv_cbrt" ); |
| 231 | populateOpPatterns<math::CeilOp>(converter, patterns, benefit, f32Func: "__nv_ceilf" , |
| 232 | f64Func: "__nv_ceil" ); |
| 233 | populateOpPatterns<math::CopySignOp>(converter, patterns, benefit, |
| 234 | f32Func: "__nv_copysignf" , f64Func: "__nv_copysign" ); |
| 235 | populateOpPatterns<math::CosOp>(converter, patterns, benefit, f32Func: "__nv_cosf" , |
| 236 | f64Func: "__nv_cos" , f32ApproxFunc: "__nv_fast_cosf" ); |
| 237 | populateOpPatterns<math::CoshOp>(converter, patterns, benefit, f32Func: "__nv_coshf" , |
| 238 | f64Func: "__nv_cosh" ); |
| 239 | populateOpPatterns<math::ErfOp>(converter, patterns, benefit, f32Func: "__nv_erff" , |
| 240 | f64Func: "__nv_erf" ); |
| 241 | populateOpPatterns<math::ErfcOp>(converter, patterns, benefit, f32Func: "__nv_erfcf" , |
| 242 | f64Func: "__nv_erfc" ); |
| 243 | populateOpPatterns<math::ExpOp>(converter, patterns, benefit, f32Func: "__nv_expf" , |
| 244 | f64Func: "__nv_exp" , f32ApproxFunc: "__nv_fast_expf" ); |
| 245 | populateOpPatterns<math::Exp2Op>(converter, patterns, benefit, f32Func: "__nv_exp2f" , |
| 246 | f64Func: "__nv_exp2" ); |
| 247 | populateOpPatterns<math::ExpM1Op>(converter, patterns, benefit, f32Func: "__nv_expm1f" , |
| 248 | f64Func: "__nv_expm1" ); |
| 249 | populateOpPatterns<math::FloorOp>(converter, patterns, benefit, f32Func: "__nv_floorf" , |
| 250 | f64Func: "__nv_floor" ); |
| 251 | populateOpPatterns<math::FmaOp>(converter, patterns, benefit, f32Func: "__nv_fmaf" , |
| 252 | f64Func: "__nv_fma" ); |
| 253 | // Note: libdevice uses a different name for 32-bit finite checking |
| 254 | populateOpPatterns<math::IsFiniteOp>(converter, patterns, benefit, |
| 255 | f32Func: "__nv_finitef" , f64Func: "__nv_isfinited" ); |
| 256 | populateOpPatterns<math::IsInfOp>(converter, patterns, benefit, f32Func: "__nv_isinff" , |
| 257 | f64Func: "__nv_isinfd" ); |
| 258 | populateOpPatterns<math::IsNaNOp>(converter, patterns, benefit, f32Func: "__nv_isnanf" , |
| 259 | f64Func: "__nv_isnand" ); |
| 260 | populateOpPatterns<math::LogOp>(converter, patterns, benefit, f32Func: "__nv_logf" , |
| 261 | f64Func: "__nv_log" , f32ApproxFunc: "__nv_fast_logf" ); |
| 262 | populateOpPatterns<math::Log10Op>(converter, patterns, benefit, f32Func: "__nv_log10f" , |
| 263 | f64Func: "__nv_log10" , f32ApproxFunc: "__nv_fast_log10f" ); |
| 264 | populateOpPatterns<math::Log1pOp>(converter, patterns, benefit, f32Func: "__nv_log1pf" , |
| 265 | f64Func: "__nv_log1p" ); |
| 266 | populateOpPatterns<math::Log2Op>(converter, patterns, benefit, f32Func: "__nv_log2f" , |
| 267 | f64Func: "__nv_log2" , f32ApproxFunc: "__nv_fast_log2f" ); |
| 268 | populateOpPatterns<math::PowFOp>(converter, patterns, benefit, f32Func: "__nv_powf" , |
| 269 | f64Func: "__nv_pow" , f32ApproxFunc: "__nv_fast_powf" ); |
| 270 | patterns.add<FPowIScalarizeLowering>(arg: converter, args&: benefit); |
| 271 | patterns.add<FPowIOpLowering>(arg: converter, args: "__nv_powif" , args: "__nv_powi" , args: "" , args: "" , |
| 272 | /*i32Func=*/args: "" , args&: benefit); |
| 273 | populateOpPatterns<math::RoundOp>(converter, patterns, benefit, f32Func: "__nv_roundf" , |
| 274 | f64Func: "__nv_round" ); |
| 275 | populateOpPatterns<math::RoundEvenOp>(converter, patterns, benefit, |
| 276 | f32Func: "__nv_rintf" , f64Func: "__nv_rint" ); |
| 277 | populateOpPatterns<math::RsqrtOp>(converter, patterns, benefit, f32Func: "__nv_rsqrtf" , |
| 278 | f64Func: "__nv_rsqrt" ); |
| 279 | populateOpPatterns<math::SinOp>(converter, patterns, benefit, f32Func: "__nv_sinf" , |
| 280 | f64Func: "__nv_sin" , f32ApproxFunc: "__nv_fast_sinf" ); |
| 281 | populateOpPatterns<math::SinhOp>(converter, patterns, benefit, f32Func: "__nv_sinhf" , |
| 282 | f64Func: "__nv_sinh" ); |
| 283 | populateOpPatterns<math::SqrtOp>(converter, patterns, benefit, f32Func: "__nv_sqrtf" , |
| 284 | f64Func: "__nv_sqrt" ); |
| 285 | populateOpPatterns<math::TanOp>(converter, patterns, benefit, f32Func: "__nv_tanf" , |
| 286 | f64Func: "__nv_tan" , f32ApproxFunc: "__nv_fast_tanf" ); |
| 287 | populateOpPatterns<math::TanhOp>(converter, patterns, benefit, f32Func: "__nv_tanhf" , |
| 288 | f64Func: "__nv_tanh" ); |
| 289 | |
| 290 | // Custom pattern for sincos since it returns two values |
| 291 | patterns.add<SincosOpLowering>(arg: converter, args&: benefit); |
| 292 | } |
| 293 | |
| 294 | namespace { |
| 295 | struct ConvertMathToNVVMPass final |
| 296 | : impl::ConvertMathToNVVMBase<ConvertMathToNVVMPass> { |
| 297 | using impl::ConvertMathToNVVMBase< |
| 298 | ConvertMathToNVVMPass>::ConvertMathToNVVMBase; |
| 299 | |
| 300 | void runOnOperation() override; |
| 301 | }; |
| 302 | } // namespace |
| 303 | |
| 304 | void ConvertMathToNVVMPass::runOnOperation() { |
| 305 | auto m = getOperation(); |
| 306 | MLIRContext *ctx = m.getContext(); |
| 307 | |
| 308 | RewritePatternSet patterns(&getContext()); |
| 309 | LowerToLLVMOptions options(ctx, DataLayout(m)); |
| 310 | LLVMTypeConverter converter(ctx, options); |
| 311 | |
| 312 | populateLibDeviceConversionPatterns(converter, patterns, /*benefit=*/1); |
| 313 | |
| 314 | ConversionTarget target(getContext()); |
| 315 | target |
| 316 | .addLegalDialect<BuiltinDialect, func::FuncDialect, vector::VectorDialect, |
| 317 | LLVM::LLVMDialect, NVVM::NVVMDialect>(); |
| 318 | target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op, LLVM::FAbsOp, |
| 319 | LLVM::FCeilOp, LLVM::FFloorOp, LLVM::FRemOp, LLVM::LogOp, |
| 320 | LLVM::Log10Op, LLVM::Log2Op, LLVM::PowOp, LLVM::SinOp, |
| 321 | LLVM::SqrtOp>(); |
| 322 | if (failed(Result: applyPartialConversion(op: m, target, patterns: std::move(patterns)))) |
| 323 | signalPassFailure(); |
| 324 | } |
| 325 | |