| 1 | //===- MathToAPFloat.cpp - Mathmetic to APFloat Conversion ----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "Utils.h" |
| 10 | |
| 11 | #include "mlir/Conversion/ArithAndMathToAPFloat/MathToAPFloat.h" |
| 12 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 13 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 14 | #include "mlir/Dialect/Func/Utils/Utils.h" |
| 15 | #include "mlir/Dialect/Math/IR/Math.h" |
| 16 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 17 | #include "mlir/IR/PatternMatch.h" |
| 18 | #include "mlir/IR/Verifier.h" |
| 19 | #include "mlir/Pass/Pass.h" |
| 20 | #include "mlir/Transforms/WalkPatternRewriteDriver.h" |
| 21 | |
| 22 | namespace mlir { |
| 23 | #define GEN_PASS_DEF_MATHTOAPFLOATCONVERSIONPASS |
| 24 | #include "mlir/Conversion/Passes.h.inc" |
| 25 | } // namespace mlir |
| 26 | |
| 27 | using namespace mlir; |
| 28 | using namespace mlir::func; |
| 29 | |
| 30 | struct AbsFOpToAPFloatConversion final : OpRewritePattern<math::AbsFOp> { |
| 31 | AbsFOpToAPFloatConversion(MLIRContext *context, SymbolOpInterface symTable, |
| 32 | PatternBenefit benefit = 1) |
| 33 | : OpRewritePattern<math::AbsFOp>(context, benefit), symTable(symTable) {} |
| 34 | |
| 35 | LogicalResult matchAndRewrite(math::AbsFOp op, |
| 36 | PatternRewriter &rewriter) const override { |
| 37 | if (failed(Result: checkPreconditions(rewriter, op))) |
| 38 | return failure(); |
| 39 | // Get APFloat function from runtime library. |
| 40 | auto i32Type = IntegerType::get(context: symTable->getContext(), width: 32); |
| 41 | auto i64Type = IntegerType::get(context: symTable->getContext(), width: 64); |
| 42 | FailureOr<FuncOp> fn = lookupOrCreateFnDecl( |
| 43 | b&: rewriter, symTable, name: "_mlir_apfloat_abs" , paramTypes: {i32Type, i64Type}); |
| 44 | if (failed(Result: fn)) |
| 45 | return fn; |
| 46 | Location loc = op.getLoc(); |
| 47 | rewriter.setInsertionPoint(op); |
| 48 | // Scalarize and convert to APFloat runtime calls. |
| 49 | Value repl = forEachScalarValue( |
| 50 | rewriter, loc, operand1: op.getOperand(), /*operand2=*/Value(), resultType: op.getType(), |
| 51 | fn: [&](Value operand, Value, Type resultType) { |
| 52 | auto floatTy = cast<FloatType>(Val: operand.getType()); |
| 53 | auto intWType = rewriter.getIntegerType(width: floatTy.getWidth()); |
| 54 | Value operandBits = arith::ExtUIOp::create( |
| 55 | builder&: rewriter, location: loc, out: i64Type, |
| 56 | in: arith::BitcastOp::create(builder&: rewriter, location: loc, out: intWType, in: operand)); |
| 57 | // Call APFloat function. |
| 58 | Value semValue = getAPFloatSemanticsValue(b&: rewriter, loc, floatTy); |
| 59 | SmallVector<Value> params = {semValue, operandBits}; |
| 60 | Value negatedBits = |
| 61 | func::CallOp::create(builder&: rewriter, location: loc, results: TypeRange(i64Type), |
| 62 | callee: SymbolRefAttr::get(symbol: *fn), operands: params) |
| 63 | ->getResult(idx: 0); |
| 64 | // Truncate result to the original width. |
| 65 | auto truncatedBits = |
| 66 | arith::TruncIOp::create(builder&: rewriter, location: loc, out: intWType, in: negatedBits); |
| 67 | return arith::BitcastOp::create(builder&: rewriter, location: loc, out: floatTy, |
| 68 | in: truncatedBits); |
| 69 | }); |
| 70 | |
| 71 | rewriter.replaceOp(op, newValues: repl); |
| 72 | return success(); |
| 73 | } |
| 74 | |
| 75 | SymbolOpInterface symTable; |
| 76 | }; |
| 77 | |
| 78 | template <typename OpTy> |
| 79 | struct IsOpToAPFloatConversion final : OpRewritePattern<OpTy> { |
| 80 | IsOpToAPFloatConversion(MLIRContext *context, const char *APFloatName, |
| 81 | SymbolOpInterface symTable, |
| 82 | PatternBenefit benefit = 1) |
| 83 | : OpRewritePattern<OpTy>(context, benefit), symTable(symTable), |
| 84 | APFloatName(APFloatName) {}; |
| 85 | |
| 86 | LogicalResult matchAndRewrite(OpTy op, |
| 87 | PatternRewriter &rewriter) const override { |
| 88 | if (failed(checkPreconditions(rewriter, op))) |
| 89 | return failure(); |
| 90 | // Get APFloat function from runtime library. |
| 91 | auto i1 = IntegerType::get(context: symTable->getContext(), width: 1); |
| 92 | auto i32Type = IntegerType::get(context: symTable->getContext(), width: 32); |
| 93 | auto i64Type = IntegerType::get(context: symTable->getContext(), width: 64); |
| 94 | std::string funcName = |
| 95 | (llvm::Twine("_mlir_apfloat_is" ) + APFloatName).str(); |
| 96 | FailureOr<FuncOp> fn = lookupOrCreateFnDecl( |
| 97 | b&: rewriter, symTable, name: funcName, paramTypes: {i32Type, i64Type}, symbolTables: nullptr, resultType: i1); |
| 98 | if (failed(Result: fn)) |
| 99 | return fn; |
| 100 | Location loc = op.getLoc(); |
| 101 | rewriter.setInsertionPoint(op); |
| 102 | // Scalarize and convert to APFloat runtime calls. |
| 103 | Value repl = forEachScalarValue( |
| 104 | rewriter, loc, op.getOperand(), /*operand2=*/Value(), op.getType(), |
| 105 | [&](Value operand, Value, Type resultType) { |
| 106 | auto floatTy = cast<FloatType>(Val: operand.getType()); |
| 107 | auto intWType = rewriter.getIntegerType(width: floatTy.getWidth()); |
| 108 | Value operandBits = arith::ExtUIOp::create( |
| 109 | builder&: rewriter, location: loc, out: i64Type, |
| 110 | in: arith::BitcastOp::create(builder&: rewriter, location: loc, out: intWType, in: operand)); |
| 111 | |
| 112 | // Call APFloat function. |
| 113 | Value semValue = getAPFloatSemanticsValue(b&: rewriter, loc, floatTy); |
| 114 | Value params[] = {semValue, operandBits}; |
| 115 | return func::CallOp::create(builder&: rewriter, location: loc, results: TypeRange(i1), |
| 116 | callee: SymbolRefAttr::get(symbol: *fn), operands: params) |
| 117 | .getResult(i: 0); |
| 118 | }); |
| 119 | rewriter.replaceOp(op, repl); |
| 120 | return success(); |
| 121 | } |
| 122 | |
| 123 | SymbolOpInterface symTable; |
| 124 | const char *APFloatName; |
| 125 | }; |
| 126 | |
| 127 | struct FmaOpToAPFloatConversion final : OpRewritePattern<math::FmaOp> { |
| 128 | FmaOpToAPFloatConversion(MLIRContext *context, SymbolOpInterface symTable, |
| 129 | PatternBenefit benefit = 1) |
| 130 | : OpRewritePattern<math::FmaOp>(context, benefit), symTable(symTable) {}; |
| 131 | |
| 132 | LogicalResult matchAndRewrite(math::FmaOp op, |
| 133 | PatternRewriter &rewriter) const override { |
| 134 | if (failed(Result: checkPreconditions(rewriter, op))) |
| 135 | return failure(); |
| 136 | // Cast operands to 64-bit integers. |
| 137 | mlir::Type resType = op.getResult().getType(); |
| 138 | auto floatTy = dyn_cast<FloatType>(Val&: resType); |
| 139 | if (!floatTy) { |
| 140 | auto vecTy1 = cast<VectorType>(Val&: resType); |
| 141 | floatTy = llvm::cast<FloatType>(Val: vecTy1.getElementType()); |
| 142 | } |
| 143 | auto i32Type = IntegerType::get(context: symTable->getContext(), width: 32); |
| 144 | auto i64Type = IntegerType::get(context: symTable->getContext(), width: 64); |
| 145 | FailureOr<FuncOp> fn = lookupOrCreateFnDecl( |
| 146 | b&: rewriter, symTable, name: "_mlir_apfloat_fused_multiply_add" , |
| 147 | paramTypes: {i32Type, i64Type, i64Type, i64Type}); |
| 148 | if (failed(Result: fn)) |
| 149 | return fn; |
| 150 | Location loc = op.getLoc(); |
| 151 | rewriter.setInsertionPoint(op); |
| 152 | |
| 153 | IntegerType intWType = rewriter.getIntegerType(width: floatTy.getWidth()); |
| 154 | IntegerType int64Type = rewriter.getI64Type(); |
| 155 | |
| 156 | auto scalarFMA = [&rewriter, &loc, &floatTy, &fn, &intWType, |
| 157 | &int64Type](Value a, Value b, Value c) { |
| 158 | Value operand = arith::ExtUIOp::create( |
| 159 | builder&: rewriter, location: loc, out: int64Type, |
| 160 | in: arith::BitcastOp::create(builder&: rewriter, location: loc, out: intWType, in: a)); |
| 161 | Value multiplicand = arith::ExtUIOp::create( |
| 162 | builder&: rewriter, location: loc, out: int64Type, |
| 163 | in: arith::BitcastOp::create(builder&: rewriter, location: loc, out: intWType, in: b)); |
| 164 | Value addend = arith::ExtUIOp::create( |
| 165 | builder&: rewriter, location: loc, out: int64Type, |
| 166 | in: arith::BitcastOp::create(builder&: rewriter, location: loc, out: intWType, in: c)); |
| 167 | // Call APFloat function. |
| 168 | Value semValue = getAPFloatSemanticsValue(b&: rewriter, loc, floatTy); |
| 169 | SmallVector<Value> params = {semValue, operand, multiplicand, addend}; |
| 170 | auto resultOp = |
| 171 | func::CallOp::create(builder&: rewriter, location: loc, results: TypeRange(rewriter.getI64Type()), |
| 172 | callee: SymbolRefAttr::get(symbol: *fn), operands: params); |
| 173 | |
| 174 | // Truncate result to the original width. |
| 175 | auto trunc = arith::TruncIOp::create(builder&: rewriter, location: loc, out: intWType, |
| 176 | in: resultOp->getResult(idx: 0)); |
| 177 | return arith::BitcastOp::create(builder&: rewriter, location: loc, out: floatTy, in: trunc); |
| 178 | }; |
| 179 | |
| 180 | if (auto vecTy1 = dyn_cast<VectorType>(Val: op.getA().getType())) { |
| 181 | // Sanity check: Operand types must match. |
| 182 | assert(vecTy1 == dyn_cast<VectorType>(op.getB().getType()) && |
| 183 | "expected same vector types" ); |
| 184 | assert(vecTy1 == dyn_cast<VectorType>(op.getC().getType()) && |
| 185 | "expected same vector types" ); |
| 186 | // Prepare scalar operands. |
| 187 | ResultRange scalarOperands = |
| 188 | vector::ToElementsOp::create(builder&: rewriter, location: loc, source: op.getA())->getResults(); |
| 189 | ResultRange scalarMultiplicands = |
| 190 | vector::ToElementsOp::create(builder&: rewriter, location: loc, source: op.getB())->getResults(); |
| 191 | ResultRange scalarAddends = |
| 192 | vector::ToElementsOp::create(builder&: rewriter, location: loc, source: op.getC())->getResults(); |
| 193 | // Call the function for each pair of scalar operands. |
| 194 | SmallVector<Value> results; |
| 195 | for (auto [operand, multiplicand, addend] : llvm::zip_equal( |
| 196 | t&: scalarOperands, u&: scalarMultiplicands, args&: scalarAddends)) { |
| 197 | results.push_back(Elt: scalarFMA(operand, multiplicand, addend)); |
| 198 | } |
| 199 | // Package the results into a vector. |
| 200 | auto fromElements = vector::FromElementsOp::create( |
| 201 | builder&: rewriter, location: loc, |
| 202 | result: vecTy1.cloneWith(/*shape=*/std::nullopt, elementType: results.front().getType()), |
| 203 | elements: results); |
| 204 | rewriter.replaceOp(op, newOp: fromElements); |
| 205 | return success(); |
| 206 | } |
| 207 | |
| 208 | Value repl = scalarFMA(op.getA(), op.getB(), op.getC()); |
| 209 | rewriter.replaceOp(op, newValues: repl); |
| 210 | return success(); |
| 211 | } |
| 212 | |
| 213 | SymbolOpInterface symTable; |
| 214 | }; |
| 215 | |
| 216 | namespace { |
| 217 | struct MathToAPFloatConversionPass final |
| 218 | : impl::MathToAPFloatConversionPassBase<MathToAPFloatConversionPass> { |
| 219 | using Base::Base; |
| 220 | |
| 221 | void runOnOperation() override; |
| 222 | }; |
| 223 | |
| 224 | void MathToAPFloatConversionPass::runOnOperation() { |
| 225 | MLIRContext *context = &getContext(); |
| 226 | RewritePatternSet patterns(context); |
| 227 | |
| 228 | patterns.add<AbsFOpToAPFloatConversion>(arg&: context, args: getOperation()); |
| 229 | patterns.add<IsOpToAPFloatConversion<math::IsFiniteOp>>(arg&: context, args: "finite" , |
| 230 | args: getOperation()); |
| 231 | patterns.add<IsOpToAPFloatConversion<math::IsInfOp>>(arg&: context, args: "infinite" , |
| 232 | args: getOperation()); |
| 233 | patterns.add<IsOpToAPFloatConversion<math::IsNaNOp>>(arg&: context, args: "nan" , |
| 234 | args: getOperation()); |
| 235 | patterns.add<IsOpToAPFloatConversion<math::IsNormalOp>>(arg&: context, args: "normal" , |
| 236 | args: getOperation()); |
| 237 | patterns.add<FmaOpToAPFloatConversion>(arg&: context, args: getOperation()); |
| 238 | |
| 239 | LogicalResult result = success(); |
| 240 | ScopedDiagnosticHandler scopedHandler(context, [&result](Diagnostic &diag) { |
| 241 | if (diag.getSeverity() == DiagnosticSeverity::Error) { |
| 242 | result = failure(); |
| 243 | } |
| 244 | // NB: if you don't return failure, no other diag handlers will fire (see |
| 245 | // mlir/lib/IR/Diagnostics.cpp:DiagnosticEngineImpl::emit). |
| 246 | return failure(); |
| 247 | }); |
| 248 | walkAndApplyPatterns(op: getOperation(), patterns: std::move(patterns)); |
| 249 | if (failed(Result: result)) |
| 250 | return signalPassFailure(); |
| 251 | } |
| 252 | } // namespace |
| 253 | |