| 1 | //===- ExpandOps.cpp - Pass to legalize Arith ops for LLVM lowering --===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/ArithCommon/CeilFloorDivExpansion.h" |
| 10 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 11 | #include "mlir/Dialect/Arith/Transforms/Passes.h" |
| 12 | #include "mlir/Dialect/Vector/IR/VectorDialect.h" |
| 13 | #include "mlir/IR/BuiltinTypeInterfaces.h" |
| 14 | #include "mlir/IR/Location.h" |
| 15 | #include "mlir/IR/TypeUtilities.h" |
| 16 | #include "mlir/Transforms/DialectConversion.h" |
| 17 | |
| 18 | namespace mlir { |
| 19 | namespace arith { |
| 20 | #define GEN_PASS_DEF_ARITHEXPANDOPSPASS |
| 21 | #include "mlir/Dialect/Arith/Transforms/Passes.h.inc" |
| 22 | } // namespace arith |
| 23 | } // namespace mlir |
| 24 | |
| 25 | using namespace mlir; |
| 26 | |
| 27 | /// Create an integer or index constant. |
| 28 | static Value createConst(Location loc, Type type, int value, |
| 29 | PatternRewriter &rewriter) { |
| 30 | auto attr = rewriter.getIntegerAttr(type: getElementTypeOrSelf(type), value); |
| 31 | if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) { |
| 32 | return arith::ConstantOp::create(builder&: rewriter, location: loc, |
| 33 | value: DenseElementsAttr::get(type: shapedTy, values: attr)); |
| 34 | } |
| 35 | return arith::ConstantOp::create(builder&: rewriter, location: loc, value: attr); |
| 36 | } |
| 37 | |
| 38 | /// Create an integer constant from an APInt. |
| 39 | static Value createAPIntConst(Location loc, Type type, const APInt &value, |
| 40 | PatternRewriter &rewriter) { |
| 41 | auto attr = IntegerAttr::get(type: getElementTypeOrSelf(type), value); |
| 42 | if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) { |
| 43 | return arith::ConstantOp::create(builder&: rewriter, location: loc, |
| 44 | value: DenseElementsAttr::get(type: shapedTy, values: attr)); |
| 45 | } |
| 46 | return arith::ConstantOp::create(builder&: rewriter, location: loc, value: attr); |
| 47 | } |
| 48 | |
| 49 | /// Create a float constant. |
| 50 | static Value createFloatConst(Location loc, Type type, const APFloat &value, |
| 51 | PatternRewriter &rewriter) { |
| 52 | auto attr = rewriter.getFloatAttr(type: getElementTypeOrSelf(type), value); |
| 53 | if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) { |
| 54 | return arith::ConstantOp::create(builder&: rewriter, location: loc, |
| 55 | value: DenseElementsAttr::get(type: shapedTy, values: attr)); |
| 56 | } |
| 57 | |
| 58 | return arith::ConstantOp::create(builder&: rewriter, location: loc, value: attr); |
| 59 | } |
| 60 | |
| 61 | /// Creates shapedType using shape from cloneFrom and base type from cloneTo |
| 62 | static Type cloneToShapedType(Type cloneFrom, Type cloneTo) { |
| 63 | if (auto shapedTy = dyn_cast<ShapedType>(Val&: cloneFrom)) { |
| 64 | return shapedTy.clone(elementType: cloneTo); |
| 65 | } |
| 66 | return cloneTo; |
| 67 | } |
| 68 | |
| 69 | /// Returns only the nnan flag from fastmath, or a null attribute if it is absent. |
| 70 | static arith::FastMathFlagsAttr getNNaNOnly(MLIRContext *ctx, |
| 71 | arith::FastMathFlagsAttr fastmath) { |
| 72 | if (fastmath && arith::bitEnumContainsAll(bits: fastmath.getValue(), |
| 73 | bit: arith::FastMathFlags::nnan)) |
| 74 | return arith::FastMathFlagsAttr::get(context: ctx, value: arith::FastMathFlags::nnan); |
| 75 | return {}; |
| 76 | } |
| 77 | |
| 78 | namespace { |
| 79 | |
| 80 | template <typename OpTy, arith::CmpIPredicate pred> |
| 81 | struct MaxMinIOpConverter : public OpRewritePattern<OpTy> { |
| 82 | public: |
| 83 | using OpRewritePattern<OpTy>::OpRewritePattern; |
| 84 | |
| 85 | LogicalResult matchAndRewrite(OpTy op, |
| 86 | PatternRewriter &rewriter) const final { |
| 87 | Value lhs = op.getLhs(); |
| 88 | Value rhs = op.getRhs(); |
| 89 | |
| 90 | Value cmp = arith::CmpIOp::create(rewriter, op.getLoc(), pred, lhs, rhs); |
| 91 | rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cmp, lhs, rhs); |
| 92 | return success(); |
| 93 | } |
| 94 | }; |
| 95 | |
| 96 | template <typename OpTy, arith::CmpFPredicate pred> |
| 97 | struct MaximumMinimumFOpConverter : public OpRewritePattern<OpTy> { |
| 98 | public: |
| 99 | using OpRewritePattern<OpTy>::OpRewritePattern; |
| 100 | |
| 101 | LogicalResult matchAndRewrite(OpTy op, |
| 102 | PatternRewriter &rewriter) const final { |
| 103 | Value lhs = op.getLhs(); |
| 104 | Value rhs = op.getRhs(); |
| 105 | |
| 106 | Location loc = op.getLoc(); |
| 107 | // If any operand is NaN, 'cmp' will be true (and 'select' returns 'lhs'). |
| 108 | static_assert(pred == arith::CmpFPredicate::UGT || |
| 109 | pred == arith::CmpFPredicate::ULT, |
| 110 | "pred must be either UGT or ULT" ); |
| 111 | Value cmp = arith::CmpFOp::create(builder&: rewriter, location: loc, predicate: pred, lhs, rhs); |
| 112 | Value select = arith::SelectOp::create(builder&: rewriter, location: loc, condition: cmp, true_value: lhs, false_value: rhs); |
| 113 | |
| 114 | // Handle the case where rhs is NaN: 'isNaN(rhs) ? rhs : select'. |
| 115 | Value isNaN = arith::CmpFOp::create(builder&: rewriter, location: loc, |
| 116 | predicate: arith::CmpFPredicate::UNO, lhs: rhs, rhs); |
| 117 | rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select); |
| 118 | return success(); |
| 119 | } |
| 120 | }; |
| 121 | |
| 122 | template <typename OpTy, arith::CmpFPredicate pred> |
| 123 | struct MaxNumMinNumFOpConverter : public OpRewritePattern<OpTy> { |
| 124 | public: |
| 125 | using OpRewritePattern<OpTy>::OpRewritePattern; |
| 126 | |
| 127 | LogicalResult matchAndRewrite(OpTy op, |
| 128 | PatternRewriter &rewriter) const final { |
| 129 | Value lhs = op.getLhs(); |
| 130 | Value rhs = op.getRhs(); |
| 131 | |
| 132 | Location loc = op.getLoc(); |
| 133 | // If any operand is NaN, 'cmp' will be true (and 'select' returns 'lhs'). |
| 134 | static_assert(pred == arith::CmpFPredicate::UGT || |
| 135 | pred == arith::CmpFPredicate::ULT, |
| 136 | "pred must be either UGT or ULT" ); |
| 137 | Value cmp = arith::CmpFOp::create(builder&: rewriter, location: loc, predicate: pred, lhs, rhs); |
| 138 | Value select = arith::SelectOp::create(builder&: rewriter, location: loc, condition: cmp, true_value: lhs, false_value: rhs); |
| 139 | |
| 140 | // Handle the case where lhs is NaN: 'isNaN(lhs) ? rhs : select'. |
| 141 | Value isNaN = arith::CmpFOp::create(builder&: rewriter, location: loc, |
| 142 | predicate: arith::CmpFPredicate::UNO, lhs, rhs: lhs); |
| 143 | rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select); |
| 144 | return success(); |
| 145 | } |
| 146 | }; |
| 147 | |
| 148 | struct BFloat16ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> { |
| 149 | using Base::Base; |
| 150 | LogicalResult matchAndRewrite(arith::ExtFOp op, |
| 151 | PatternRewriter &rewriter) const final { |
| 152 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 153 | auto operand = op.getOperand(); |
| 154 | Type operandTy = operand.getType(); |
| 155 | Type resultTy = op.getType(); |
| 156 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 157 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 158 | |
| 159 | if (!operandETy.isBF16() || !resultETy.isF32()) { |
| 160 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a ext of bf16 to f32." ); |
| 161 | } |
| 162 | |
| 163 | Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type()); |
| 164 | Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type()); |
| 165 | |
| 166 | Value bitcast = arith::BitcastOp::create(builder&: b, out: i16Ty, in: operand); |
| 167 | Value exti = arith::ExtUIOp::create(builder&: b, out: i32Ty, in: bitcast); |
| 168 | |
| 169 | Value c16 = createConst(loc: op.getLoc(), type: i32Ty, value: 16, rewriter); |
| 170 | Value shl = arith::ShLIOp::create(builder&: b, lhs: exti, rhs: c16); |
| 171 | Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: shl); |
| 172 | |
| 173 | rewriter.replaceOp(op, newValues: result); |
| 174 | return success(); |
| 175 | } |
| 176 | }; |
| 177 | |
| 178 | struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> { |
| 179 | using Base::Base; |
| 180 | LogicalResult matchAndRewrite(arith::TruncFOp op, |
| 181 | PatternRewriter &rewriter) const final { |
| 182 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 183 | auto operand = op.getOperand(); |
| 184 | Type operandTy = operand.getType(); |
| 185 | Type resultTy = op.getType(); |
| 186 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 187 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 188 | |
| 189 | if (!operandETy.isF32() || !resultETy.isBF16()) { |
| 190 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a trunc of f32 to bf16." ); |
| 191 | } |
| 192 | |
| 193 | if (op.getRoundingmodeAttr()) { |
| 194 | return rewriter.notifyMatchFailure( |
| 195 | arg&: op, msg: "only applicable to default rounding mode." ); |
| 196 | } |
| 197 | |
| 198 | Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type()); |
| 199 | Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type()); |
| 200 | |
| 201 | // Algorithm borrowed from this excellent code: |
| 202 | // https://github.com/pytorch/pytorch/blob/e1502c0cdbfd17548c612f25d5a65b1e4b86224d/c10/util/BFloat16.h#L60-L79 |
| 203 | // There is a magic idea there, to let the addition of the rounding_bias to |
| 204 | // the mantissa simply overflow into the exponent bits. It's a bit of an |
| 205 | // aggressive, obfuscating optimization, but it is well-tested code, and it |
| 206 | // results in more concise and efficient IR. |
| 207 | // The case of NaN is handled separately (see isNaN and the final select). |
| 208 | // The case of infinities is NOT handled separately, which deserves an |
| 209 | // explanation. As the encoding of infinities has zero mantissa, the |
| 210 | // rounding-bias addition never carries into the exponent so that just gets |
| 211 | // truncated away, and as bfloat16 and float32 have the same number of |
| 212 | // exponent bits, that simple truncation is the desired outcome for |
| 213 | // infinities. |
| 214 | Value isNan = |
| 215 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNE, lhs: operand, rhs: operand); |
| 216 | // Constant used to make the rounding bias. |
| 217 | Value c7FFF = createConst(loc: op.getLoc(), type: i32Ty, value: 0x7fff, rewriter); |
| 218 | // Constant used to generate a quiet NaN. |
| 219 | Value c7FC0I16 = createConst(loc: op.getLoc(), type: i16Ty, value: 0x7fc0, rewriter); |
| 220 | // Small constants used to address bits. |
| 221 | Value c16 = createConst(loc: op.getLoc(), type: i32Ty, value: 16, rewriter); |
| 222 | Value c1 = createConst(loc: op.getLoc(), type: i32Ty, value: 1, rewriter); |
| 223 | // Reinterpret the input f32 value as bits. |
| 224 | Value bitcast = arith::BitcastOp::create(builder&: b, out: i32Ty, in: operand); |
| 225 | // Read bit 16 as a value in {0,1}. |
| 226 | Value bit16 = |
| 227 | arith::AndIOp::create(builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: bitcast, rhs: c16), rhs: c1); |
| 228 | // Determine the rounding bias to add as either 0x7fff or 0x8000 depending |
| 229 | // on bit 16, implementing the tie-breaking "to nearest even". |
| 230 | Value roundingBias = arith::AddIOp::create(builder&: b, lhs: bit16, rhs: c7FFF); |
| 231 | // Add the rounding bias. Generally we want this to be added to the |
| 232 | // mantissa, but nothing prevents this to from carrying into the exponent |
| 233 | // bits, which would feel like a bug, but this is the magic trick here: |
| 234 | // when that happens, the mantissa gets reset to zero and the exponent |
| 235 | // gets incremented by the carry... which is actually exactly what we |
| 236 | // want. |
| 237 | Value biased = arith::AddIOp::create(builder&: b, lhs: bitcast, rhs: roundingBias); |
| 238 | // Now that the rounding-bias has been added, truncating the low bits |
| 239 | // yields the correctly rounded result. |
| 240 | Value biasedAndShifted = arith::ShRUIOp::create(builder&: b, lhs: biased, rhs: c16); |
| 241 | Value normalCaseResultI16 = |
| 242 | arith::TruncIOp::create(builder&: b, out: i16Ty, in: biasedAndShifted); |
| 243 | // Select either the above-computed result, or a quiet NaN constant |
| 244 | // if the input was NaN. |
| 245 | Value select = |
| 246 | arith::SelectOp::create(builder&: b, condition: isNan, true_value: c7FC0I16, false_value: normalCaseResultI16); |
| 247 | Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: select); |
| 248 | rewriter.replaceOp(op, newValues: result); |
| 249 | return success(); |
| 250 | } |
| 251 | }; |
| 252 | |
| 253 | /// In this implementation of extf we take advantage of some key patterns we |
| 254 | /// notice between the binary representation of an F4E2M1 value and its |
| 255 | /// corresponding value in F32. |
| 256 | /// |
| 257 | /// Note: x is sign bit |
| 258 | /// | Binary | F4E2M1 | f32[23:32] |
| 259 | /// | x000 | 0.0 | x000 0000 00 |
| 260 | /// | x001 | 0.5 | x011 1111 00 |
| 261 | /// | x010 | 1.0 | x011 1111 10 |
| 262 | /// | x011 | 1.5 | x011 1111 11 |
| 263 | /// | x100 | 2.0 | x010 0000 00 |
| 264 | /// | x101 | 3.0 | x010 0000 01 |
| 265 | /// | x110 | 4.0 | x010 0000 10 |
| 266 | /// | x111 | 6.0 | x010 0000 11 |
| 267 | /// |
| 268 | /// 1) There are only two versions of bits [25:31] in the f32 result |
| 269 | /// F4E2M1 bits[2:3] decide whether: |
| 270 | /// - F32 bits[25:31] = 0011 1111 |
| 271 | /// - F32 bits[25:31] = 0010 0000 |
| 272 | /// Exception is zero where |
| 273 | /// - F32 bits[25:31] = 0000 0000 |
| 274 | /// |
| 275 | /// 2) F4E2M1 bits[1:2] = F32 bits[23:24] |
| 276 | /// Exception is 0.5 where |
| 277 | /// - F4E2M1 bits[1:2] = 01, F32 bits[23:24] = 00 |
| 278 | /// |
| 279 | /// 3) F4E2M1 bits[4] = F32 bits[32] (sign bits are equal) |
| 280 | /// |
| 281 | /// 4) F32 bits[1:22] = 0 |
| 282 | struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> { |
| 283 | using Base::Base; |
| 284 | LogicalResult matchAndRewrite(arith::ExtFOp op, |
| 285 | PatternRewriter &rewriter) const final { |
| 286 | Location loc = op.getLoc(); |
| 287 | ImplicitLocOpBuilder b(loc, rewriter); |
| 288 | Value operand = op.getOperand(); |
| 289 | Type operandTy = operand.getType(); |
| 290 | Type resultTy = op.getType(); |
| 291 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 292 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 293 | |
| 294 | if (!isa<Float4E2M1FNType>(Val: operandETy)) |
| 295 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a ext of F4E2M1FN" ); |
| 296 | |
| 297 | Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type()); |
| 298 | Type i4Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI4Type()); |
| 299 | Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type()); |
| 300 | Value i4Bits = arith::BitcastOp::create(builder&: b, out: i4Ty, in: operand); |
| 301 | |
| 302 | Value c0x0 = createConst(loc, type: i4Ty, value: 0x0, rewriter); |
| 303 | Value c0x1 = createConst(loc, type: i4Ty, value: 0x1, rewriter); |
| 304 | Value c0x2 = createConst(loc, type: i4Ty, value: 0x2, rewriter); |
| 305 | Value c0x4 = createConst(loc, type: i4Ty, value: 0x4, rewriter); |
| 306 | Value c0x7 = createConst(loc, type: i4Ty, value: 0x7, rewriter); |
| 307 | |
| 308 | Value i4BitsNoSign = arith::AndIOp::create(builder&: b, lhs: i4Bits, rhs: c0x7); |
| 309 | |
| 310 | // Set last Exponent bit and Mantissa. |
| 311 | Value c0x00000014 = createConst(loc, type: i32Ty, value: 0x14, rewriter); |
| 312 | Value bits1To24 = arith::ShLIOp::create(builder&: b, lhs: i4BitsNoSign, rhs: c0x2); |
| 313 | Value isHalf = |
| 314 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: i4BitsNoSign, rhs: c0x1); |
| 315 | bits1To24 = arith::SelectOp::create(builder&: b, condition: isHalf, true_value: c0x0, false_value: bits1To24); |
| 316 | bits1To24 = arith::ExtUIOp::create(builder&: b, out: i32Ty, in: bits1To24); |
| 317 | bits1To24 = arith::ShLIOp::create(builder&: b, lhs: bits1To24, rhs: c0x00000014); |
| 318 | |
| 319 | // Set first 7 bits of Exponent. |
| 320 | Value zeroExpBits = createConst(loc, type: i32Ty, value: 0x00000000, rewriter); |
| 321 | Value highExpBits = createConst(loc, type: i32Ty, value: 0x40000000, rewriter); |
| 322 | Value lowExpBits = createConst(loc, type: i32Ty, value: 0x3f000000, rewriter); |
| 323 | Value useLargerExp = |
| 324 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::uge, lhs: i4BitsNoSign, rhs: c0x4); |
| 325 | Value bits25To31 = |
| 326 | arith::SelectOp::create(builder&: b, condition: useLargerExp, true_value: highExpBits, false_value: lowExpBits); |
| 327 | Value zeroExp = |
| 328 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: i4BitsNoSign, rhs: c0x0); |
| 329 | bits25To31 = arith::SelectOp::create(builder&: b, condition: zeroExp, true_value: zeroExpBits, false_value: bits25To31); |
| 330 | |
| 331 | // Set sign. |
| 332 | Value c0x80000000 = createConst(loc, type: i32Ty, value: 0x80000000, rewriter); |
| 333 | Value c0x8 = createConst(loc, type: i4Ty, value: 0x8, rewriter); |
| 334 | Value negative = |
| 335 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::uge, lhs: i4Bits, rhs: c0x8); |
| 336 | Value bit32 = |
| 337 | arith::SelectOp::create(builder&: b, condition: negative, true_value: c0x80000000, false_value: zeroExpBits); |
| 338 | |
| 339 | // Add segments together. |
| 340 | Value bits1To31 = arith::AddIOp::create(builder&: b, lhs: bits1To24, rhs: bits25To31); |
| 341 | Value bits1To32 = arith::AddIOp::create(builder&: b, lhs: bits1To31, rhs: bit32); |
| 342 | Value result = arith::BitcastOp::create(builder&: b, out: f32Ty, in: bits1To32); |
| 343 | if (!isa<Float32Type>(Val: resultETy)) |
| 344 | result = arith::TruncFOp::create(builder&: b, out: resultTy, in: result); |
| 345 | |
| 346 | rewriter.replaceOp(op, newValues: result); |
| 347 | return success(); |
| 348 | } |
| 349 | }; |
| 350 | |
| 351 | struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> { |
| 352 | using Base::Base; |
| 353 | LogicalResult matchAndRewrite(arith::ExtFOp op, |
| 354 | PatternRewriter &rewriter) const final { |
| 355 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 356 | Value operand = op.getOperand(); |
| 357 | Type operandTy = operand.getType(); |
| 358 | Type resultTy = op.getType(); |
| 359 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 360 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 361 | |
| 362 | if (!llvm::isa<Float8E8M0FNUType>(Val: operandETy)) { |
| 363 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a ext of F8E8M0FNU" ); |
| 364 | } |
| 365 | |
| 366 | Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type()); |
| 367 | Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type()); |
| 368 | Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type()); |
| 369 | |
| 370 | Value bitcast = arith::BitcastOp::create(builder&: b, out: i8Ty, in: operand); |
| 371 | Value cF32MantissaWidth = createConst(loc: op->getLoc(), type: i32Ty, value: 23, rewriter); |
| 372 | Value exti = arith::ExtUIOp::create(builder&: b, out: i32Ty, in: bitcast); |
| 373 | Value f32Bits = arith::ShLIOp::create(builder&: b, lhs: exti, rhs: cF32MantissaWidth); |
| 374 | |
| 375 | // F8E8M0 has no zero representation. |
| 376 | // All zeros encoding represents 2^-127 which is an f32 subnormal. |
| 377 | Value cF8Min = createConst(loc: op.getLoc(), type: i8Ty, value: 0, rewriter); |
| 378 | Value cF32Min = createConst(loc: op.getLoc(), type: i32Ty, value: 0x00400000, rewriter); |
| 379 | Value isMin = |
| 380 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: bitcast, rhs: cF8Min); |
| 381 | f32Bits = arith::SelectOp::create(builder&: b, condition: isMin, true_value: cF32Min, false_value: f32Bits); |
| 382 | |
| 383 | // If FastMathFlag allows no NaN checks, skip it |
| 384 | auto fastMath = op.getFastmathAttr(); |
| 385 | bool NoNaN = fastMath |
| 386 | ? (fastMath.getValue() & arith::FastMathFlags::nnan) == |
| 387 | arith::FastMathFlags::nnan |
| 388 | : false; |
| 389 | if (!NoNaN) { |
| 390 | Value cF8NaN = createConst(loc: op.getLoc(), type: i8Ty, value: 0xff, rewriter); |
| 391 | Value cF32NaN = createConst(loc: op.getLoc(), type: i32Ty, value: 0x7fc00000, rewriter); |
| 392 | Value isNan = |
| 393 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: bitcast, rhs: cF8NaN); |
| 394 | // select for NaNs: the quiet NaN, as the constant folder gives |
| 395 | f32Bits = arith::SelectOp::create(builder&: b, condition: isNan, true_value: cF32NaN, false_value: f32Bits); |
| 396 | } |
| 397 | |
| 398 | Value result = arith::BitcastOp::create(builder&: b, out: f32Ty, in: f32Bits); |
| 399 | if (resultETy.getIntOrFloatBitWidth() < 32) { |
| 400 | result = arith::TruncFOp::create(builder&: b, out: resultTy, in: result, roundingmode: nullptr, |
| 401 | fastmath: op.getFastmathAttr()); |
| 402 | } else if (resultETy.getIntOrFloatBitWidth() > 32) { |
| 403 | result = arith::ExtFOp::create(builder&: b, out: resultTy, in: result, fastmath: op.getFastmathAttr()); |
| 404 | } |
| 405 | rewriter.replaceOp(op, newValues: result); |
| 406 | return success(); |
| 407 | } |
| 408 | }; |
| 409 | |
| 410 | /// Conversion from F32 to F4E2M1 according to the OCP Spec: |
| 411 | /// www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf |
| 412 | /// |
| 413 | /// The spec requiers us to perform Round to Nearest, Ties to Even. |
| 414 | /// |
| 415 | /// This means that after rounding, we should break ties by choosing the option |
| 416 | /// which results in a mantissa of 0 in the least significant digit. |
| 417 | /// |
| 418 | /// Table of representable values in F4E2M1: |
| 419 | /// |
| 420 | /// Note: x is sign bit |
| 421 | /// | Binary | F4E2M1 | F32[23:32] |
| 422 | /// | x000 | 0.0 | x000 0000 00 |
| 423 | /// | x001 | 0.5 | x011 1111 00 |
| 424 | /// | x010 | 1.0 | x011 1111 10 |
| 425 | /// | x011 | 1.5 | x011 1111 11 |
| 426 | /// | x100 | 2.0 | x010 0000 00 |
| 427 | /// | x101 | 3.0 | x010 0000 01 |
| 428 | /// | x110 | 4.0 | x010 0000 10 |
| 429 | /// | x111 | 6.0 | x010 0000 11 |
| 430 | /// |
| 431 | /// Conversion procedure: |
| 432 | /// Step 1: Clamp to representable bounds. |
| 433 | /// Step 2: Convert exponent by adjusting bias. |
| 434 | /// Step 3: Set mantissa to first bit. |
| 435 | /// Step 4: Special consideration for subnormal and zero exponent. |
| 436 | /// Step 5: Round up if necessary, if mantissa[1:] greater than 1000000 or |
| 437 | /// subnormal. |
| 438 | struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> { |
| 439 | using Base::Base; |
| 440 | LogicalResult matchAndRewrite(arith::TruncFOp op, |
| 441 | PatternRewriter &rewriter) const final { |
| 442 | Location loc = op.getLoc(); |
| 443 | ImplicitLocOpBuilder b(loc, rewriter); |
| 444 | Value operand = op.getOperand(); |
| 445 | Type operandTy = operand.getType(); |
| 446 | Type resultTy = op.getType(); |
| 447 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 448 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 449 | |
| 450 | Type i4Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI4Type()); |
| 451 | Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type()); |
| 452 | Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type()); |
| 453 | Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type()); |
| 454 | |
| 455 | if (!isa<Float4E2M1FNType>(Val: resultETy)) |
| 456 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a trunc of F4E2M1FN" ); |
| 457 | if (!isa<Float32Type>(Val: operandETy)) |
| 458 | operand = |
| 459 | arith::ExtFOp::create(builder&: b, out: f32Ty, in: operand, fastmath: arith::FastMathFlagsAttr{}); |
| 460 | |
| 461 | Value c0x1 = createConst(loc, type: i4Ty, value: 1, rewriter); |
| 462 | Value c0x3 = createConst(loc, type: i4Ty, value: 3, rewriter); |
| 463 | Value c0x00000016 = createConst(loc, type: i32Ty, value: 22, rewriter); |
| 464 | Value c0x00 = createConst(loc, type: i8Ty, value: 0x00, rewriter); |
| 465 | Value c0xff = createConst(loc, type: i8Ty, value: 0xff, rewriter); |
| 466 | Value zeroExpBits = createConst(loc, type: i32Ty, value: 0, rewriter); |
| 467 | |
| 468 | // Step 0: Clamp to bounds. |
| 469 | Value cHigherBound = createFloatConst(loc, type: f32Ty, value: APFloat(6.0f), rewriter); |
| 470 | Value cLowerBound = createFloatConst(loc, type: f32Ty, value: APFloat(-6.0f), rewriter); |
| 471 | Value operandClamped = arith::MinNumFOp::create(builder&: b, lhs: cHigherBound, rhs: operand); |
| 472 | operandClamped = arith::MaxNumFOp::create(builder&: b, lhs: cLowerBound, rhs: operandClamped); |
| 473 | Value f32Bits = arith::BitcastOp::create(builder&: b, out: i32Ty, in: operandClamped); |
| 474 | |
| 475 | // Step 1: Set sign bit. |
| 476 | Value cF32ExpManWidth = createConst(loc, type: i32Ty, value: 31, rewriter); // 23 |
| 477 | Value f32Sign = arith::ShRUIOp::create(builder&: b, lhs: f32Bits, rhs: cF32ExpManWidth); |
| 478 | Value f4Sign = arith::TruncIOp::create(builder&: b, out: i4Ty, in: f32Sign); |
| 479 | Value f4Bits = arith::ShLIOp::create(builder&: b, lhs: f4Sign, rhs: c0x3); |
| 480 | |
| 481 | // Step 2: Convert exponent by adjusting bias. |
| 482 | Value biasAdjustment = createConst(loc, type: i32Ty, value: 0x7e, rewriter); |
| 483 | Value cF4MantissaWidth = c0x1; // 1 |
| 484 | Value cF32MantissaWidth = createConst(loc, type: i32Ty, value: 23, rewriter); // 23 |
| 485 | Value f32SignExp = arith::ShRUIOp::create(builder&: b, lhs: f32Bits, rhs: cF32MantissaWidth); |
| 486 | Value biasAdjustedSignExp = |
| 487 | arith::SubIOp::create(builder&: b, lhs: f32SignExp, rhs: biasAdjustment); |
| 488 | Value f4Exp = arith::TruncIOp::create(builder&: b, out: i4Ty, in: biasAdjustedSignExp); |
| 489 | f4Exp = arith::ShLIOp::create(builder&: b, lhs: f4Exp, rhs: cF4MantissaWidth); |
| 490 | f4Bits = arith::AddIOp::create(builder&: b, lhs: f4Bits, rhs: f4Exp); |
| 491 | |
| 492 | // Step 3: Set mantissa to first bit. |
| 493 | Value cF32FirstBitMask = createConst(loc, type: i32Ty, value: 0x400000, rewriter); |
| 494 | Value man1Bit = arith::AndIOp::create(builder&: b, lhs: f32Bits, rhs: cF32FirstBitMask); |
| 495 | man1Bit = arith::ShRUIOp::create(builder&: b, lhs: man1Bit, rhs: c0x00000016); |
| 496 | Value f4Man = arith::TruncIOp::create(builder&: b, out: i4Ty, in: man1Bit); |
| 497 | f4Bits = arith::AddIOp::create(builder&: b, lhs: f4Bits, rhs: f4Man); |
| 498 | |
| 499 | // Step 4: Special consideration for conversion to 0.5. |
| 500 | Value cF32MantissaMask = createConst(loc, type: i32Ty, value: 0x7fffff, rewriter); |
| 501 | Value f8Exp = arith::TruncIOp::create(builder&: b, out: i8Ty, in: biasAdjustedSignExp); |
| 502 | Value isSubnormal = |
| 503 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::sle, lhs: f8Exp, rhs: c0x00); |
| 504 | Value isNegOneExp = |
| 505 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: f8Exp, rhs: c0xff); |
| 506 | Value man23Bits = arith::AndIOp::create(builder&: b, lhs: f32Bits, rhs: cF32MantissaMask); |
| 507 | Value isNonZeroMan = arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::ugt, |
| 508 | lhs: man23Bits, rhs: zeroExpBits); |
| 509 | Value roundToHalf = arith::AndIOp::create(builder&: b, lhs: isNegOneExp, rhs: isNonZeroMan); |
| 510 | Value isZeroExp = |
| 511 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: f8Exp, rhs: c0x00); |
| 512 | Value subnormalF4Bits = createConst(loc, type: i4Ty, value: 0xf, rewriter); |
| 513 | Value halfF4Bits = createConst(loc, type: i4Ty, value: 0x0, rewriter); |
| 514 | Value subResult = |
| 515 | arith::SelectOp::create(builder&: b, condition: isSubnormal, true_value: subnormalF4Bits, false_value: f4Bits); |
| 516 | subResult = arith::SelectOp::create(builder&: b, condition: roundToHalf, true_value: halfF4Bits, false_value: subResult); |
| 517 | f4Bits = arith::SelectOp::create(builder&: b, condition: isZeroExp, true_value: f4Bits, false_value: subResult); |
| 518 | |
| 519 | // Step 5: Round up if necessary. |
| 520 | Value cF32Last22BitMask = createConst(loc, type: i32Ty, value: 0x3fffff, rewriter); |
| 521 | Value cRound = createConst(loc, type: i32Ty, value: 0x200000, rewriter); // 010 0000... |
| 522 | Value man22Bits = arith::AndIOp::create(builder&: b, lhs: f32Bits, rhs: cF32Last22BitMask); |
| 523 | Value shouldRound = |
| 524 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::uge, lhs: man22Bits, rhs: cRound); |
| 525 | shouldRound = arith::OrIOp::create(builder&: b, lhs: shouldRound, rhs: isSubnormal); |
| 526 | Value roundedF4Bits = arith::AddIOp::create(builder&: b, lhs: f4Bits, rhs: c0x1); |
| 527 | f4Bits = arith::SelectOp::create(builder&: b, condition: shouldRound, true_value: roundedF4Bits, false_value: f4Bits); |
| 528 | |
| 529 | Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: f4Bits); |
| 530 | rewriter.replaceOp(op, newValues: result); |
| 531 | return success(); |
| 532 | } |
| 533 | }; |
| 534 | |
| 535 | /* |
| 536 | TruncF to F8E8M0 is expected to extract exponent bits out of F32 type |
| 537 | Since All kinds of Infs and NaNs are mapped to same exponent bits in F32 type, |
| 538 | they all map to NaN in F8E8M0 Type. |
| 539 | */ |
| 540 | struct F8E8M0TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> { |
| 541 | using Base::Base; |
| 542 | LogicalResult matchAndRewrite(arith::TruncFOp op, |
| 543 | PatternRewriter &rewriter) const final { |
| 544 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 545 | Value operand = op.getOperand(); |
| 546 | Type operandTy = operand.getType(); |
| 547 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 548 | Type resultTy = op.getType(); |
| 549 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 550 | if (!llvm::isa<Float8E8M0FNUType>(Val: resultETy)) { |
| 551 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a truncf to f8E8M0FNU" ); |
| 552 | } |
| 553 | |
| 554 | if (op.getRoundingmodeAttr()) { |
| 555 | return rewriter.notifyMatchFailure( |
| 556 | arg&: op, msg: "only applicable to default rounding mode." ); |
| 557 | } |
| 558 | |
| 559 | Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type()); |
| 560 | Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type()); |
| 561 | Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type()); |
| 562 | |
| 563 | if (operandETy.getIntOrFloatBitWidth() < 32) { |
| 564 | // Propagating contract would let ExtFOp::fold cancel an earlier contract |
| 565 | // truncf, skipping its rounding before we extract the exponent bits. |
| 566 | operand = arith::ExtFOp::create( |
| 567 | builder&: b, out: f32Ty, in: operand, fastmath: getNNaNOnly(ctx: b.getContext(), fastmath: op.getFastmathAttr())); |
| 568 | } else if (operandETy.getIntOrFloatBitWidth() > 32) { |
| 569 | operand = arith::TruncFOp::create( |
| 570 | builder&: b, out: f32Ty, in: operand, roundingmode: op.getRoundingmodeAttr(), fastmath: op.getFastmathAttr()); |
| 571 | } |
| 572 | Value f32Bits = arith::BitcastOp::create(builder&: b, out: i32Ty, in: operand); |
| 573 | Value cF32MantissaWidth = createConst(loc: op->getLoc(), type: i32Ty, value: 23, rewriter); |
| 574 | Value f32SignExp = arith::ShRUIOp::create(builder&: b, lhs: f32Bits, rhs: cF32MantissaWidth); |
| 575 | Value exp8Bits = arith::TruncIOp::create(builder&: b, out: i8Ty, in: f32SignExp); |
| 576 | Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: exp8Bits); |
| 577 | rewriter.replaceOp(op, newValues: result); |
| 578 | return success(); |
| 579 | } |
| 580 | }; |
| 581 | |
| 582 | /// Expand an ExtF from F8E5M2. F8E5M2 uses a 5-bit exponent with bias 15 and a |
| 583 | /// 2-bit mantissa, i.e. it is bit-for-bit the high byte of an IEEE F16 value |
| 584 | /// (same exponent field and bias), including infinities and NaNs. The exact |
| 585 | /// F16 value is therefore obtained by placing the 8 F8E5M2 bits into the high |
| 586 | /// byte of an i16. The F16 value is then converted to the requested result |
| 587 | /// type with the native (LLVM-lowerable) extf/truncf. |
| 588 | struct F8E5M2ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> { |
| 589 | using Base::Base; |
| 590 | LogicalResult matchAndRewrite(arith::ExtFOp op, |
| 591 | PatternRewriter &rewriter) const final { |
| 592 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 593 | Value operand = op.getOperand(); |
| 594 | Type operandTy = operand.getType(); |
| 595 | Type resultTy = op.getType(); |
| 596 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 597 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 598 | |
| 599 | if (!llvm::isa<Float8E5M2Type>(Val: operandETy)) |
| 600 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a ext of F8E5M2" ); |
| 601 | |
| 602 | Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type()); |
| 603 | Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type()); |
| 604 | Type f16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF16Type()); |
| 605 | |
| 606 | Value bitcast = arith::BitcastOp::create(builder&: b, out: i8Ty, in: operand); |
| 607 | Value exti = arith::ExtUIOp::create(builder&: b, out: i16Ty, in: bitcast); |
| 608 | Value c8 = createConst(loc: op.getLoc(), type: i16Ty, value: 8, rewriter); |
| 609 | Value f16Bits = arith::ShLIOp::create(builder&: b, lhs: exti, rhs: c8); |
| 610 | Value f16 = arith::BitcastOp::create(builder&: b, out: f16Ty, in: f16Bits); |
| 611 | |
| 612 | Value result = f16; |
| 613 | if (!resultETy.isF16()) { |
| 614 | if (resultETy.getIntOrFloatBitWidth() < 16) |
| 615 | result = arith::TruncFOp::create(builder&: b, out: resultTy, in: f16, roundingmode: nullptr, |
| 616 | fastmath: op.getFastmathAttr()); |
| 617 | else |
| 618 | result = arith::ExtFOp::create(builder&: b, out: resultTy, in: f16, fastmath: op.getFastmathAttr()); |
| 619 | } |
| 620 | rewriter.replaceOp(op, newValues: result); |
| 621 | return success(); |
| 622 | } |
| 623 | }; |
| 624 | |
| 625 | /// Expand a TruncF to F8E5M2. The input is first reduced to F16 (which shares |
| 626 | /// the F8E5M2 exponent layout and bias) using the native truncf, then the low |
| 627 | /// 8 mantissa bits of the F16 value are dropped with round-to-nearest-even. |
| 628 | /// The rounding-bias trick is borrowed from the BF16 converter: adding the |
| 629 | /// bias may carry into the exponent field, which is exactly the desired |
| 630 | /// behavior since F16 and F8E5M2 share the same exponent bias. NaN is handled |
| 631 | /// separately. |
| 632 | struct F8E5M2TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> { |
| 633 | using Base::Base; |
| 634 | LogicalResult matchAndRewrite(arith::TruncFOp op, |
| 635 | PatternRewriter &rewriter) const final { |
| 636 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 637 | Value operand = op.getOperand(); |
| 638 | Type operandTy = operand.getType(); |
| 639 | Type resultTy = op.getType(); |
| 640 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 641 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 642 | |
| 643 | if (!llvm::isa<Float8E5M2Type>(Val: resultETy)) |
| 644 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a trunc to F8E5M2" ); |
| 645 | if (op.getRoundingmodeAttr()) |
| 646 | return rewriter.notifyMatchFailure( |
| 647 | arg&: op, msg: "only applicable to default rounding mode." ); |
| 648 | |
| 649 | Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type()); |
| 650 | Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type()); |
| 651 | Type f16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF16Type()); |
| 652 | |
| 653 | Value h16 = operand; |
| 654 | if (!operandETy.isF16()) |
| 655 | h16 = arith::TruncFOp::create(builder&: b, out: f16Ty, in: operand, roundingmode: nullptr, |
| 656 | fastmath: op.getFastmathAttr()); |
| 657 | |
| 658 | Value isNan = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNE, lhs: h16, rhs: h16); |
| 659 | Value h16Bits = arith::BitcastOp::create(builder&: b, out: i16Ty, in: h16); |
| 660 | // Rounding bias constants for dropping the low 8 mantissa bits. |
| 661 | Value c7F = createConst(loc: op.getLoc(), type: i16Ty, value: 0x7f, rewriter); |
| 662 | Value c8 = createConst(loc: op.getLoc(), type: i16Ty, value: 8, rewriter); |
| 663 | Value c1 = createConst(loc: op.getLoc(), type: i16Ty, value: 1, rewriter); |
| 664 | Value bit8 = |
| 665 | arith::AndIOp::create(builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: h16Bits, rhs: c8), rhs: c1); |
| 666 | Value roundingBias = arith::AddIOp::create(builder&: b, lhs: bit8, rhs: c7F); |
| 667 | Value biased = arith::AddIOp::create(builder&: b, lhs: h16Bits, rhs: roundingBias); |
| 668 | Value biasedAndShifted = arith::ShRUIOp::create(builder&: b, lhs: biased, rhs: c8); |
| 669 | Value normalCaseResult = arith::TruncIOp::create(builder&: b, out: i8Ty, in: biasedAndShifted); |
| 670 | // Quiet NaN for F8E5M2 (exponent all ones, mantissa MSB set). |
| 671 | Value cNan = createConst(loc: op.getLoc(), type: i8Ty, value: 0x7e, rewriter); |
| 672 | Value select = arith::SelectOp::create(builder&: b, condition: isNan, true_value: cNan, false_value: normalCaseResult); |
| 673 | Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: select); |
| 674 | rewriter.replaceOp(op, newValues: result); |
| 675 | return success(); |
| 676 | } |
| 677 | }; |
| 678 | |
| 679 | /// Expand an ExtF from F8E4M3FN. F8E4M3FN uses a 4-bit exponent with bias 7, a |
| 680 | /// 3-bit mantissa, no infinities and a single NaN encoding (S.1111.111). The |
| 681 | /// 7 magnitude bits (EEEE.MMM) are placed into the high mantissa/exponent bits |
| 682 | /// of an F16 by shifting left by 7, producing an F16 whose value equals the |
| 683 | /// desired magnitude scaled by 2^-8 (the F16 bias is 15 while F8E4M3FN's is |
| 684 | /// 7). Multiplying by 256 in F32 recovers the true magnitude for both normal |
| 685 | /// and subnormal inputs. The sign bit and the NaN encoding are re-applied |
| 686 | /// explicitly. |
| 687 | struct F8E4M3FNExtFOpConverter : public OpRewritePattern<arith::ExtFOp> { |
| 688 | using Base::Base; |
| 689 | LogicalResult matchAndRewrite(arith::ExtFOp op, |
| 690 | PatternRewriter &rewriter) const final { |
| 691 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 692 | Value operand = op.getOperand(); |
| 693 | Type operandTy = operand.getType(); |
| 694 | Type resultTy = op.getType(); |
| 695 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 696 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 697 | |
| 698 | if (!llvm::isa<Float8E4M3FNType>(Val: operandETy)) |
| 699 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a ext of F8E4M3FN" ); |
| 700 | |
| 701 | Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type()); |
| 702 | Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type()); |
| 703 | Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type()); |
| 704 | Type f16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF16Type()); |
| 705 | Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type()); |
| 706 | |
| 707 | Value bits = arith::BitcastOp::create(builder&: b, out: i8Ty, in: operand); |
| 708 | Value c7F8 = createConst(loc: op.getLoc(), type: i8Ty, value: 0x7f, rewriter); |
| 709 | Value mag8 = arith::AndIOp::create(builder&: b, lhs: bits, rhs: c7F8); |
| 710 | // Build an F16 equal to the magnitude times 2^-8. |
| 711 | Value mag16 = arith::ExtUIOp::create(builder&: b, out: i16Ty, in: mag8); |
| 712 | Value c7 = createConst(loc: op.getLoc(), type: i16Ty, value: 7, rewriter); |
| 713 | Value g16Bits = arith::ShLIOp::create(builder&: b, lhs: mag16, rhs: c7); |
| 714 | Value g16 = arith::BitcastOp::create(builder&: b, out: f16Ty, in: g16Bits); |
| 715 | Value gF32 = arith::ExtFOp::create(builder&: b, out: f32Ty, in: g16, fastmath: op.getFastmathAttr()); |
| 716 | Value c256 = |
| 717 | createFloatConst(loc: op.getLoc(), type: f32Ty, value: APFloat(256.0f), rewriter); |
| 718 | Value magF32 = arith::MulFOp::create(builder&: b, lhs: gF32, rhs: c256, fastmath: op.getFastmathAttr()); |
| 719 | // Re-apply the sign bit into the F32 result. |
| 720 | Value magI32 = arith::BitcastOp::create(builder&: b, out: i32Ty, in: magF32); |
| 721 | Value c80I8 = createConst(loc: op.getLoc(), type: i8Ty, value: 0x80, rewriter); |
| 722 | Value sign8 = arith::AndIOp::create(builder&: b, lhs: bits, rhs: c80I8); |
| 723 | Value sign32 = arith::ExtUIOp::create(builder&: b, out: i32Ty, in: sign8); |
| 724 | Value c24 = createConst(loc: op.getLoc(), type: i32Ty, value: 24, rewriter); |
| 725 | Value signBit = arith::ShLIOp::create(builder&: b, lhs: sign32, rhs: c24); |
| 726 | Value signedI32 = arith::OrIOp::create(builder&: b, lhs: magI32, rhs: signBit); |
| 727 | Value signedF32 = arith::BitcastOp::create(builder&: b, out: f32Ty, in: signedI32); |
| 728 | // NaN encoding is magnitude == 0x7f. |
| 729 | Value isNan = |
| 730 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: mag8, rhs: c7F8); |
| 731 | Value cNan32 = createConst(loc: op.getLoc(), type: i32Ty, value: 0x7fc00000, rewriter); |
| 732 | Value nanSigned = arith::OrIOp::create(builder&: b, lhs: cNan32, rhs: signBit); |
| 733 | Value nanF32 = arith::BitcastOp::create(builder&: b, out: f32Ty, in: nanSigned); |
| 734 | Value resultF32 = arith::SelectOp::create(builder&: b, condition: isNan, true_value: nanF32, false_value: signedF32); |
| 735 | |
| 736 | Value result = resultF32; |
| 737 | if (!resultETy.isF32()) { |
| 738 | if (resultETy.getIntOrFloatBitWidth() < 32) |
| 739 | result = arith::TruncFOp::create(builder&: b, out: resultTy, in: resultF32, roundingmode: nullptr, |
| 740 | fastmath: op.getFastmathAttr()); |
| 741 | else |
| 742 | result = |
| 743 | arith::ExtFOp::create(builder&: b, out: resultTy, in: resultF32, fastmath: op.getFastmathAttr()); |
| 744 | } |
| 745 | rewriter.replaceOp(op, newValues: result); |
| 746 | return success(); |
| 747 | } |
| 748 | }; |
| 749 | |
| 750 | /// Expand a TruncF to F8E4M3FN. The magnitude is scaled by 2^-8 and reduced to |
| 751 | /// F16 (undoing the bias difference so the F16 value equals the magnitude times |
| 752 | /// 2^-8). The low 7 bits of the F16 encoding are then dropped with |
| 753 | /// round-to-nearest-even to recover the 7 magnitude bits. F8E4M3FN has no |
| 754 | /// infinity, so any input that overflows the maximum representable magnitude |
| 755 | /// (448), as well as infinities and NaNs, maps to the F8E4M3FN NaN encoding to |
| 756 | /// match the LLVM APFloat NanOnly overflow behavior and the OCP FP8 (E4M3) |
| 757 | /// spec. |
| 758 | struct F8E4M3FNTruncFOpConverter : public OpRewritePattern<arith::TruncFOp> { |
| 759 | using Base::Base; |
| 760 | LogicalResult matchAndRewrite(arith::TruncFOp op, |
| 761 | PatternRewriter &rewriter) const final { |
| 762 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 763 | Value operand = op.getOperand(); |
| 764 | Type operandTy = operand.getType(); |
| 765 | Type resultTy = op.getType(); |
| 766 | Type operandETy = getElementTypeOrSelf(type: operandTy); |
| 767 | Type resultETy = getElementTypeOrSelf(type: resultTy); |
| 768 | |
| 769 | if (!llvm::isa<Float8E4M3FNType>(Val: resultETy)) |
| 770 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a trunc to F8E4M3FN" ); |
| 771 | if (op.getRoundingmodeAttr()) |
| 772 | return rewriter.notifyMatchFailure( |
| 773 | arg&: op, msg: "only applicable to default rounding mode." ); |
| 774 | |
| 775 | Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type()); |
| 776 | Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type()); |
| 777 | Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type()); |
| 778 | Type f16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF16Type()); |
| 779 | Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type()); |
| 780 | |
| 781 | Value f32 = operand; |
| 782 | if (!operandETy.isF32()) { |
| 783 | if (operandETy.getIntOrFloatBitWidth() < 32) |
| 784 | f32 = arith::ExtFOp::create(builder&: b, out: f32Ty, in: operand, fastmath: op.getFastmathAttr()); |
| 785 | else |
| 786 | f32 = arith::TruncFOp::create(builder&: b, out: f32Ty, in: operand, roundingmode: nullptr, |
| 787 | fastmath: op.getFastmathAttr()); |
| 788 | } |
| 789 | |
| 790 | Value isNan = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNE, lhs: f32, rhs: f32); |
| 791 | // Split sign and magnitude. |
| 792 | Value f32Bits = arith::BitcastOp::create(builder&: b, out: i32Ty, in: f32); |
| 793 | Value cSignMask = createConst(loc: op.getLoc(), type: i32Ty, value: 0x80000000, rewriter); |
| 794 | Value cAbsMask = createConst(loc: op.getLoc(), type: i32Ty, value: 0x7fffffff, rewriter); |
| 795 | Value signBits = arith::AndIOp::create(builder&: b, lhs: f32Bits, rhs: cSignMask); |
| 796 | Value absBits = arith::AndIOp::create(builder&: b, lhs: f32Bits, rhs: cAbsMask); |
| 797 | Value absF32 = arith::BitcastOp::create(builder&: b, out: f32Ty, in: absBits); |
| 798 | // F8E4M3FN has no infinity: a magnitude above the round-to-nearest-even |
| 799 | // overflow boundary (464 = 448 + half an ulp) or an infinity maps to NaN. |
| 800 | Value cOverflow = |
| 801 | createFloatConst(loc: op.getLoc(), type: f32Ty, value: APFloat(464.0f), rewriter); |
| 802 | Value isOverflow = |
| 803 | arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OGT, lhs: absF32, rhs: cOverflow); |
| 804 | // Clamp to the F8E4M3FN maximum magnitude (448) so the finite path stays |
| 805 | // well-defined; overflowing inputs are replaced by NaN below. |
| 806 | Value cMax = |
| 807 | createFloatConst(loc: op.getLoc(), type: f32Ty, value: APFloat(448.0f), rewriter); |
| 808 | absF32 = arith::MinNumFOp::create(builder&: b, lhs: absF32, rhs: cMax); |
| 809 | Value cInv256 = |
| 810 | createFloatConst(loc: op.getLoc(), type: f32Ty, value: APFloat(0.00390625f), rewriter); |
| 811 | Value scaled = arith::MulFOp::create(builder&: b, lhs: absF32, rhs: cInv256, fastmath: nullptr); |
| 812 | Value h16 = arith::TruncFOp::create(builder&: b, out: f16Ty, in: scaled, roundingmode: nullptr, |
| 813 | fastmath: op.getFastmathAttr()); |
| 814 | Value h16Bits = arith::BitcastOp::create(builder&: b, out: i16Ty, in: h16); |
| 815 | // Drop the low 7 bits with round-to-nearest-even. |
| 816 | Value c3F = createConst(loc: op.getLoc(), type: i16Ty, value: 0x3f, rewriter); |
| 817 | Value c7 = createConst(loc: op.getLoc(), type: i16Ty, value: 7, rewriter); |
| 818 | Value c1 = createConst(loc: op.getLoc(), type: i16Ty, value: 1, rewriter); |
| 819 | Value bit7 = |
| 820 | arith::AndIOp::create(builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: h16Bits, rhs: c7), rhs: c1); |
| 821 | Value roundingBias = arith::AddIOp::create(builder&: b, lhs: bit7, rhs: c3F); |
| 822 | Value biased = arith::AddIOp::create(builder&: b, lhs: h16Bits, rhs: roundingBias); |
| 823 | Value shifted = arith::ShRUIOp::create(builder&: b, lhs: biased, rhs: c7); |
| 824 | Value mag8 = arith::TruncIOp::create(builder&: b, out: i8Ty, in: shifted); |
| 825 | Value c7F8 = createConst(loc: op.getLoc(), type: i8Ty, value: 0x7f, rewriter); |
| 826 | mag8 = arith::AndIOp::create(builder&: b, lhs: mag8, rhs: c7F8); |
| 827 | // Re-apply the sign. |
| 828 | Value c24 = createConst(loc: op.getLoc(), type: i32Ty, value: 24, rewriter); |
| 829 | Value sign8 = arith::TruncIOp::create( |
| 830 | builder&: b, out: i8Ty, in: arith::ShRUIOp::create(builder&: b, lhs: signBits, rhs: c24)); |
| 831 | Value res8 = arith::OrIOp::create(builder&: b, lhs: mag8, rhs: sign8); |
| 832 | // NaN input or an overflowing/infinite magnitude maps to the NaN encoding. |
| 833 | Value isNanOrOverflow = arith::OrIOp::create(builder&: b, lhs: isNan, rhs: isOverflow); |
| 834 | Value cNan8 = createConst(loc: op.getLoc(), type: i8Ty, value: 0x7f, rewriter); |
| 835 | Value res = arith::SelectOp::create(builder&: b, condition: isNanOrOverflow, true_value: cNan8, false_value: res8); |
| 836 | Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: res); |
| 837 | rewriter.replaceOp(op, newValues: result); |
| 838 | return success(); |
| 839 | } |
| 840 | }; |
| 841 | |
| 842 | struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> { |
| 843 | using Base::Base; |
| 844 | LogicalResult matchAndRewrite(arith::ScalingExtFOp op, |
| 845 | PatternRewriter &rewriter) const final { |
| 846 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 847 | Value inputOperand = op.getIn(); |
| 848 | Value scaleOperand = op.getScale(); |
| 849 | Type scaleTy = scaleOperand.getType(); |
| 850 | Type scaleETy = getElementTypeOrSelf(val: scaleOperand); |
| 851 | // allow implicit exponent extraction from 16/32 bits floats |
| 852 | if (scaleETy.getIntOrFloatBitWidth() >= 16) { |
| 853 | scaleETy = b.getF8E8M0Type(); |
| 854 | scaleTy = cloneToShapedType(cloneFrom: scaleTy, cloneTo: scaleETy); |
| 855 | scaleOperand = arith::TruncFOp::create(builder&: b, out: scaleTy, in: scaleOperand, roundingmode: nullptr, |
| 856 | fastmath: op.getFastmathAttr()); |
| 857 | } |
| 858 | // Catch scale types like f8E5M2. |
| 859 | if (!llvm::isa<Float8E8M0FNUType>(Val: scaleETy)) { |
| 860 | return rewriter.notifyMatchFailure( |
| 861 | arg&: op, msg: "scaling_extf is using scales of type which can not be converted " |
| 862 | "to f8E8M0FNU" ); |
| 863 | } |
| 864 | Type resultTy = op.getType(); |
| 865 | // extf on scale will essentially create floating point number |
| 866 | // of type resulTy that is 2^scale and will also propagate NaNs |
| 867 | Value scaleExt = |
| 868 | arith::ExtFOp::create(builder&: b, out: resultTy, in: scaleOperand, fastmath: op.getFastmathAttr()); |
| 869 | Value inputExt = |
| 870 | arith::ExtFOp::create(builder&: b, out: resultTy, in: inputOperand, fastmath: op.getFastmathAttr()); |
| 871 | Value result = |
| 872 | arith::MulFOp::create(builder&: b, lhs: inputExt, rhs: scaleExt, fastmath: op.getFastmathAttr()); |
| 873 | rewriter.replaceOp(op, newValues: result); |
| 874 | return success(); |
| 875 | } |
| 876 | }; |
| 877 | |
| 878 | /* |
| 879 | Expands arith.ScalingTruncFOp(in, scale) into |
| 880 | scale = arith.truncf(scale) : scaleTy -> f8E8M0FNU |
| 881 | result = arith.truncf(in / (2^scale)) |
| 882 | */ |
| 883 | struct ScalingTruncFOpConverter |
| 884 | : public OpRewritePattern<arith::ScalingTruncFOp> { |
| 885 | using Base::Base; |
| 886 | LogicalResult matchAndRewrite(arith::ScalingTruncFOp op, |
| 887 | PatternRewriter &rewriter) const final { |
| 888 | ImplicitLocOpBuilder b(op.getLoc(), rewriter); |
| 889 | Value inputOperand = op.getIn(); |
| 890 | Value scaleOperand = op.getScale(); |
| 891 | Type scaleTy = scaleOperand.getType(); |
| 892 | Type scaleETy = getElementTypeOrSelf(val: scaleOperand); |
| 893 | // allow implicit exponent extraction from 16/32 bits floats |
| 894 | if (scaleETy.getIntOrFloatBitWidth() >= 16) { |
| 895 | scaleETy = b.getF8E8M0Type(); |
| 896 | scaleTy = cloneToShapedType(cloneFrom: scaleTy, cloneTo: scaleETy); |
| 897 | scaleOperand = arith::TruncFOp::create(builder&: b, out: scaleTy, in: scaleOperand, roundingmode: nullptr, |
| 898 | fastmath: op.getFastmathAttr()); |
| 899 | } |
| 900 | if (!llvm::isa<Float8E8M0FNUType>(Val: scaleETy)) { |
| 901 | return rewriter.notifyMatchFailure( |
| 902 | arg&: op, msg: "scaling_truncf is using scales type which can not be converted " |
| 903 | "to f8E8M0FNU" ); |
| 904 | } |
| 905 | Type resultTy = op.getType(); |
| 906 | Type inputTy = inputOperand.getType(); |
| 907 | // this will create a floating point number of type |
| 908 | // inputTy that is 2^scale and will also propagate NaNs |
| 909 | scaleOperand = |
| 910 | arith::ExtFOp::create(builder&: b, out: inputTy, in: scaleOperand, fastmath: op.getFastmathAttr()); |
| 911 | Value result = arith::DivFOp::create(builder&: b, lhs: inputOperand, rhs: scaleOperand, |
| 912 | fastmath: op.getFastmathAttr()); |
| 913 | Value resultCast = arith::TruncFOp::create( |
| 914 | builder&: b, out: resultTy, in: result, roundingmode: op.getRoundingmodeAttr(), fastmath: op.getFastmathAttr()); |
| 915 | rewriter.replaceOp(op, newValues: resultCast); |
| 916 | return success(); |
| 917 | } |
| 918 | }; |
| 919 | |
| 920 | /// Expands `arith.flush_denormals` into integer arithmetic. |
| 921 | /// |
| 922 | /// For an IEEE-like floating-point value with a sign|exponent|mantissa bit |
| 923 | /// layout, a value is denormal iff its biased exponent field is zero and its |
| 924 | /// stored mantissa is non-zero. When the exponent field is zero, the value is |
| 925 | /// either pos/neg 0 (mantissa = 0) or a denormal (mantissa != 0); in both |
| 926 | /// cases, clearing the mantissa bits produces the desired sign-preserved zero |
| 927 | /// (a no-op for pos/neg 0, a flush for denormals). When the exponent field is |
| 928 | /// non-zero, the value passes through unchanged. |
| 929 | /// |
| 930 | /// Pseudocode: |
| 931 | /// bits = bitcast(x, iN) |
| 932 | /// expIsZero = (bits & expMask) == 0 |
| 933 | /// cleared = bits & ~manMask |
| 934 | /// resultBits = select(expIsZero, cleared, bits) |
| 935 | /// result = bitcast(resultBits, floatTy) |
| 936 | struct FlushDenormalsOpConverter |
| 937 | : public OpRewritePattern<arith::FlushDenormalsOp> { |
| 938 | using Base::Base; |
| 939 | LogicalResult matchAndRewrite(arith::FlushDenormalsOp op, |
| 940 | PatternRewriter &rewriter) const final { |
| 941 | Location loc = op.getLoc(); |
| 942 | ImplicitLocOpBuilder b(loc, rewriter); |
| 943 | Value operand = op.getOperand(); |
| 944 | Type operandTy = operand.getType(); |
| 945 | auto floatTy = dyn_cast<FloatType>(Val: getElementTypeOrSelf(type: operandTy)); |
| 946 | if (!floatTy) |
| 947 | return rewriter.notifyMatchFailure(arg&: op, msg: "operand is not a float type" ); |
| 948 | |
| 949 | const llvm::fltSemantics &sem = floatTy.getFloatSemantics(); |
| 950 | // Restrict to IEEE-like encodings, where the sign bit is the MSB and |
| 951 | // denormals are exactly "biased exponent == 0 and non-zero mantissa". |
| 952 | if (!llvm::APFloatBase::isIEEELikeFP(sem)) |
| 953 | return rewriter.notifyMatchFailure( |
| 954 | arg&: op, msg: "only IEEE-like floating-point types are supported" ); |
| 955 | |
| 956 | unsigned totalBits = llvm::APFloatBase::semanticsSizeInBits(sem); |
| 957 | unsigned precision = llvm::APFloatBase::semanticsPrecision(sem); |
| 958 | // Stored mantissa bits = precision - 1 (implicit leading bit not stored). |
| 959 | // Exponent field bits = totalBits - 1 (sign) - storedMantissa. |
| 960 | if (precision < 1 || precision > totalBits) |
| 961 | return rewriter.notifyMatchFailure(arg&: op, msg: "unexpected float semantics" ); |
| 962 | unsigned mantissaBits = precision - 1; |
| 963 | unsigned expBits = totalBits - 1 - mantissaBits; |
| 964 | if (expBits == 0 || mantissaBits == 0) |
| 965 | return rewriter.notifyMatchFailure( |
| 966 | arg&: op, msg: "degenerate float encoding has no exponent or mantissa" ); |
| 967 | |
| 968 | Type intTy = |
| 969 | cloneToShapedType(cloneFrom: operandTy, cloneTo: rewriter.getIntegerType(width: totalBits)); |
| 970 | Value bits = arith::BitcastOp::create(builder&: b, out: intTy, in: operand); |
| 971 | APInt expMaskVal = |
| 972 | APInt::getBitsSet(numBits: totalBits, loBit: mantissaBits, hiBit: mantissaBits + expBits); |
| 973 | APInt clearMantissaMaskVal = ~APInt::getLowBitsSet(numBits: totalBits, loBitsSet: mantissaBits); |
| 974 | APInt zeroVal = APInt::getZero(numBits: totalBits); |
| 975 | Value expMask = createAPIntConst(loc, type: intTy, value: expMaskVal, rewriter); |
| 976 | Value clearMantissaMask = |
| 977 | createAPIntConst(loc, type: intTy, value: clearMantissaMaskVal, rewriter); |
| 978 | Value zero = createAPIntConst(loc, type: intTy, value: zeroVal, rewriter); |
| 979 | |
| 980 | // expField == 0 |
| 981 | Value expField = arith::AndIOp::create(builder&: b, lhs: bits, rhs: expMask); |
| 982 | Value expIsZero = |
| 983 | arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: expField, rhs: zero); |
| 984 | |
| 985 | // Clear mantissa bits: when exp == 0, this produces pos/neg 0.0. |
| 986 | Value cleared = arith::AndIOp::create(builder&: b, lhs: bits, rhs: clearMantissaMask); |
| 987 | Value resultBits = arith::SelectOp::create(builder&: b, condition: expIsZero, true_value: cleared, false_value: bits); |
| 988 | Value result = arith::BitcastOp::create(builder&: b, out: operandTy, in: resultBits); |
| 989 | |
| 990 | rewriter.replaceOp(op, newValues: result); |
| 991 | return success(); |
| 992 | } |
| 993 | }; |
| 994 | |
| 995 | struct ArithExpandOpsPass |
| 996 | : public arith::impl::ArithExpandOpsPassBase<ArithExpandOpsPass> { |
| 997 | using ArithExpandOpsPassBase::ArithExpandOpsPassBase; |
| 998 | |
| 999 | void runOnOperation() override { |
| 1000 | RewritePatternSet patterns(&getContext()); |
| 1001 | ConversionTarget target(getContext()); |
| 1002 | |
| 1003 | arith::populateCeilFloorDivExpandOpsPatterns(patterns); |
| 1004 | arith::populateExpandScalingExtTruncPatterns(patterns); |
| 1005 | |
| 1006 | target.addLegalDialect<arith::ArithDialect>(); |
| 1007 | target.addLegalDialect<vector::VectorDialect>(); |
| 1008 | |
| 1009 | // clang-format off |
| 1010 | target.addIllegalOp< |
| 1011 | arith::CeilDivSIOp, |
| 1012 | arith::CeilDivUIOp, |
| 1013 | arith::FloorDivSIOp, |
| 1014 | arith::ScalingExtFOp, |
| 1015 | arith::ScalingTruncFOp |
| 1016 | >(); |
| 1017 | // clang-format on |
| 1018 | |
| 1019 | // The min/max ops also have a direct arith-to-llvm lowering to the |
| 1020 | // `llvm.intr.maximum`/`minimum`/... (and smax/umax/...) intrinsics, which |
| 1021 | // are a single hardware instruction on many targets. Only expand them into |
| 1022 | // cmpf/cmpi + select when requested, so pipelines that run arith-to-llvm |
| 1023 | // can keep the intrinsic lowering. The float and integer ops are gated |
| 1024 | // separately. |
| 1025 | if (includeMinMaxF) { |
| 1026 | arith::populateExpandMinMaxFPatterns(patterns); |
| 1027 | // clang-format off |
| 1028 | target.addIllegalOp< |
| 1029 | arith::MaximumFOp, |
| 1030 | arith::MinimumFOp, |
| 1031 | arith::MaxNumFOp, |
| 1032 | arith::MinNumFOp |
| 1033 | >(); |
| 1034 | // clang-format on |
| 1035 | } |
| 1036 | if (includeMinMaxI) { |
| 1037 | arith::populateExpandMinMaxIPatterns(patterns); |
| 1038 | // clang-format off |
| 1039 | target.addIllegalOp< |
| 1040 | arith::MaxSIOp, |
| 1041 | arith::MaxUIOp, |
| 1042 | arith::MinSIOp, |
| 1043 | arith::MinUIOp |
| 1044 | >(); |
| 1045 | // clang-format on |
| 1046 | } |
| 1047 | |
| 1048 | if (includeBf16) |
| 1049 | arith::populateExpandBFloat16Patterns(patterns); |
| 1050 | if (includeF8E8M0) |
| 1051 | arith::populateExpandF8E8M0Patterns(patterns); |
| 1052 | if (includeF4E2M1) |
| 1053 | arith::populateExpandF4E2M1Patterns(patterns); |
| 1054 | if (includeF8E5M2) |
| 1055 | arith::populateExpandF8E5M2Patterns(patterns); |
| 1056 | if (includeF8E4M3FN) |
| 1057 | arith::populateExpandF8E4M3FNPatterns(patterns); |
| 1058 | if (includeFlushDenormals) { |
| 1059 | arith::populateExpandFlushDenormalsPatterns(patterns); |
| 1060 | // Only IEEE-like floating-point types are expanded by the pattern; |
| 1061 | // leave `arith.flush_denormals` on other types alone. |
| 1062 | target.addDynamicallyLegalOp<arith::FlushDenormalsOp>( |
| 1063 | callback: [](arith::FlushDenormalsOp op) { |
| 1064 | auto floatTy = |
| 1065 | dyn_cast<FloatType>(Val: getElementTypeOrSelf(type: op.getType())); |
| 1066 | if (!floatTy) |
| 1067 | return true; |
| 1068 | return !llvm::APFloatBase::isIEEELikeFP( |
| 1069 | floatTy.getFloatSemantics()); |
| 1070 | }); |
| 1071 | } |
| 1072 | |
| 1073 | target.addDynamicallyLegalOp<arith::ExtFOp>(callback: [=](arith::ExtFOp op) { |
| 1074 | Type inETy = getElementTypeOrSelf(type: op.getOperand().getType()); |
| 1075 | Type outETy = getElementTypeOrSelf(type: op.getType()); |
| 1076 | bool legalTypes = true; |
| 1077 | if (includeBf16) |
| 1078 | legalTypes &= !(inETy.isBF16() && outETy.isF32()); |
| 1079 | if (includeF8E8M0) |
| 1080 | legalTypes &= !llvm::isa<Float8E8M0FNUType>(Val: inETy); |
| 1081 | if (includeF4E2M1) |
| 1082 | legalTypes &= !llvm::isa<Float4E2M1FNType>(Val: inETy); |
| 1083 | if (includeF8E5M2) |
| 1084 | legalTypes &= !llvm::isa<Float8E5M2Type>(Val: inETy); |
| 1085 | if (includeF8E4M3FN) |
| 1086 | legalTypes &= !llvm::isa<Float8E4M3FNType>(Val: inETy); |
| 1087 | return legalTypes; |
| 1088 | }); |
| 1089 | |
| 1090 | target.addDynamicallyLegalOp<arith::TruncFOp>(callback: [=](arith::TruncFOp op) { |
| 1091 | Type inETy = getElementTypeOrSelf(type: op.getOperand().getType()); |
| 1092 | Type outETy = getElementTypeOrSelf(type: op.getType()); |
| 1093 | bool legalTypes = true; |
| 1094 | if (includeBf16) |
| 1095 | legalTypes &= !(inETy.isF32() && outETy.isBF16()); |
| 1096 | if (includeF8E8M0) |
| 1097 | legalTypes &= !(llvm::isa<Float8E8M0FNUType>(Val: outETy)); |
| 1098 | if (includeF4E2M1) |
| 1099 | legalTypes &= !llvm::isa<Float4E2M1FNType>(Val: outETy); |
| 1100 | if (includeF8E5M2) |
| 1101 | legalTypes &= !llvm::isa<Float8E5M2Type>(Val: outETy); |
| 1102 | if (includeF8E4M3FN) |
| 1103 | legalTypes &= !llvm::isa<Float8E4M3FNType>(Val: outETy); |
| 1104 | return legalTypes; |
| 1105 | }); |
| 1106 | |
| 1107 | // clang-format on |
| 1108 | if (failed(Result: applyPartialConversion(op: getOperation(), target, |
| 1109 | patterns: std::move(patterns)))) |
| 1110 | signalPassFailure(); |
| 1111 | } |
| 1112 | }; |
| 1113 | |
| 1114 | } // namespace |
| 1115 | |
| 1116 | void mlir::arith::populateExpandBFloat16Patterns(RewritePatternSet &patterns) { |
| 1117 | patterns.add<BFloat16ExtFOpConverter, BFloat16TruncFOpConverter>( |
| 1118 | arg: patterns.getContext()); |
| 1119 | } |
| 1120 | |
| 1121 | void mlir::arith::populateExpandF4E2M1Patterns(RewritePatternSet &patterns) { |
| 1122 | patterns.add<F4E2M1ExtFOpConverter, F4E2M1TruncFOpConverter>( |
| 1123 | arg: patterns.getContext()); |
| 1124 | } |
| 1125 | |
| 1126 | void mlir::arith::populateExpandF8E5M2Patterns(RewritePatternSet &patterns) { |
| 1127 | patterns.add<F8E5M2ExtFOpConverter, F8E5M2TruncFOpConverter>( |
| 1128 | arg: patterns.getContext()); |
| 1129 | } |
| 1130 | |
| 1131 | void mlir::arith::populateExpandF8E4M3FNPatterns(RewritePatternSet &patterns) { |
| 1132 | patterns.add<F8E4M3FNExtFOpConverter, F8E4M3FNTruncFOpConverter>( |
| 1133 | arg: patterns.getContext()); |
| 1134 | } |
| 1135 | |
| 1136 | void mlir::arith::populateExpandF8E8M0Patterns(RewritePatternSet &patterns) { |
| 1137 | patterns.add<F8E8M0ExtFOpConverter, F8E8M0TruncFOpConverter>( |
| 1138 | arg: patterns.getContext()); |
| 1139 | } |
| 1140 | |
| 1141 | void mlir::arith::populateExpandScalingExtTruncPatterns( |
| 1142 | RewritePatternSet &patterns) { |
| 1143 | patterns.add<ScalingExtFOpConverter, ScalingTruncFOpConverter>( |
| 1144 | arg: patterns.getContext()); |
| 1145 | } |
| 1146 | |
| 1147 | void mlir::arith::populateExpandFlushDenormalsPatterns( |
| 1148 | RewritePatternSet &patterns) { |
| 1149 | patterns.add<FlushDenormalsOpConverter>(arg: patterns.getContext()); |
| 1150 | } |
| 1151 | |
| 1152 | void mlir::arith::populateExpandMinMaxFPatterns(RewritePatternSet &patterns) { |
| 1153 | // clang-format off |
| 1154 | patterns.add< |
| 1155 | MaximumMinimumFOpConverter<MaximumFOp, arith::CmpFPredicate::UGT>, |
| 1156 | MaximumMinimumFOpConverter<MinimumFOp, arith::CmpFPredicate::ULT>, |
| 1157 | MaxNumMinNumFOpConverter<MaxNumFOp, arith::CmpFPredicate::UGT>, |
| 1158 | MaxNumMinNumFOpConverter<MinNumFOp, arith::CmpFPredicate::ULT> |
| 1159 | >(arg: patterns.getContext()); |
| 1160 | // clang-format on |
| 1161 | } |
| 1162 | |
| 1163 | void mlir::arith::populateExpandMinMaxIPatterns(RewritePatternSet &patterns) { |
| 1164 | // clang-format off |
| 1165 | patterns.add< |
| 1166 | MaxMinIOpConverter<MaxSIOp, arith::CmpIPredicate::sgt>, |
| 1167 | MaxMinIOpConverter<MaxUIOp, arith::CmpIPredicate::ugt>, |
| 1168 | MaxMinIOpConverter<MinSIOp, arith::CmpIPredicate::slt>, |
| 1169 | MaxMinIOpConverter<MinUIOp, arith::CmpIPredicate::ult> |
| 1170 | >(arg: patterns.getContext()); |
| 1171 | // clang-format on |
| 1172 | } |
| 1173 | |
| 1174 | void mlir::arith::populateExpandMinMaxPatterns(RewritePatternSet &patterns) { |
| 1175 | populateExpandMinMaxFPatterns(patterns); |
| 1176 | populateExpandMinMaxIPatterns(patterns); |
| 1177 | } |
| 1178 | |
| 1179 | void mlir::arith::populateArithExpandOpsPatterns(RewritePatternSet &patterns) { |
| 1180 | populateCeilFloorDivExpandOpsPatterns(patterns); |
| 1181 | populateExpandScalingExtTruncPatterns(patterns); |
| 1182 | populateExpandMinMaxPatterns(patterns); |
| 1183 | } |
| 1184 | |