1//===- OptimizeForNVVM.cpp - Optimize LLVM IR for NVVM --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/NVVM/Transforms/OptimizeForNVVM.h"
10
11#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
12#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
13#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
14#include "mlir/Dialect/LLVMIR/LLVMFloatIntrinsicOps.h"
15#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
16#include "mlir/IR/Builders.h"
17#include "mlir/IR/PatternMatch.h"
18#include "mlir/Pass/Pass.h"
19#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
20
21namespace mlir {
22namespace NVVM {
23#define GEN_PASS_DEF_NVVMOPTIMIZEFORTARGETPASS
24#include "mlir/Dialect/NVVM/Transforms/Passes.h.inc"
25} // namespace NVVM
26} // namespace mlir
27
28using namespace mlir;
29
30namespace {
31// Replaces fdiv on fp16 with fp32 multiplication with reciprocal plus one
32// (conditional) Newton iteration.
33//
34// This as accurate as promoting the division to fp32 in the NVPTX backend, but
35// faster because it performs less Newton iterations, avoids the slow path
36// for e.g. denormals, and allows reuse of the reciprocal for multiple divisions
37// by the same divisor.
38struct ExpandDivF16 : public OpRewritePattern<LLVM::FDivOp> {
39 using OpRewritePattern<LLVM::FDivOp>::OpRewritePattern;
40
41private:
42 LogicalResult matchAndRewrite(LLVM::FDivOp op,
43 PatternRewriter &rewriter) const override;
44};
45
46struct NVVMOptimizeForTarget
47 : public NVVM::impl::NVVMOptimizeForTargetPassBase<NVVMOptimizeForTarget> {
48 void runOnOperation() override;
49
50 void getDependentDialects(DialectRegistry &registry) const override {
51 registry.insert<NVVM::NVVMDialect>();
52 }
53};
54} // namespace
55
56LogicalResult ExpandDivF16::matchAndRewrite(LLVM::FDivOp op,
57 PatternRewriter &rewriter) const {
58 if (!op.getType().isF16())
59 return rewriter.notifyMatchFailure(arg&: op, msg: "not f16");
60 Location loc = op.getLoc();
61
62 Type f32Type = rewriter.getF32Type();
63 Type i32Type = rewriter.getI32Type();
64
65 // Extend lhs and rhs to fp32.
66 Value lhs = LLVM::FPExtOp::create(builder&: rewriter, location: loc, res: f32Type, arg: op.getLhs());
67 Value rhs = LLVM::FPExtOp::create(builder&: rewriter, location: loc, res: f32Type, arg: op.getRhs());
68
69 // float rcp = rcp.approx.ftz.f32(rhs), approx = lhs * rcp.
70 Value rcp = NVVM::RcpApproxFtzF32Op::create(builder&: rewriter, location: loc, res: f32Type, arg: rhs);
71 Value approx = LLVM::FMulOp::create(builder&: rewriter, location: loc, lhs, rhs: rcp);
72
73 // Refine the approximation with one Newton iteration:
74 // float refined = approx + (lhs - approx * rhs) * rcp;
75 Value err = LLVM::FMAOp::create(
76 builder&: rewriter, location: loc, a: approx, b: LLVM::FNegOp::create(builder&: rewriter, location: loc, operand: rhs), c: lhs);
77 Value refined = LLVM::FMAOp::create(builder&: rewriter, location: loc, a: err, b: rcp, c: approx);
78
79 // Use refined value if approx is normal (exponent neither all 0 or all 1).
80 Value mask = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: i32Type,
81 value: rewriter.getI32IntegerAttr(value: 0x7f800000));
82 Value cast = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: i32Type, arg: approx);
83 Value exp = LLVM::AndOp::create(builder&: rewriter, location: loc, res: i32Type, lhs: cast, rhs: mask);
84 Value zero = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: i32Type,
85 value: rewriter.getI32IntegerAttr(value: 0));
86 Value pred = LLVM::OrOp::create(
87 builder&: rewriter, location: loc,
88 lhs: LLVM::ICmpOp::create(builder&: rewriter, location: loc, predicate: LLVM::ICmpPredicate::eq, lhs: exp, rhs: zero),
89 rhs: LLVM::ICmpOp::create(builder&: rewriter, location: loc, predicate: LLVM::ICmpPredicate::eq, lhs: exp, rhs: mask));
90 Value result =
91 LLVM::SelectOp::create(builder&: rewriter, location: loc, res: f32Type, condition: pred, trueValue: approx, falseValue: refined);
92
93 // Replace with trucation back to fp16.
94 rewriter.replaceOpWithNewOp<LLVM::FPTruncOp>(op, args: op.getType(), args&: result);
95
96 return success();
97}
98
99void NVVMOptimizeForTarget::runOnOperation() {
100 MLIRContext *ctx = getOperation()->getContext();
101 RewritePatternSet patterns(ctx);
102 patterns.add<ExpandDivF16>(arg&: ctx);
103 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
104 return signalPassFailure();
105}
106