1//===- ExpandOps.cpp - Pass to legalize Arith ops for LLVM lowering --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/ArithCommon/CeilFloorDivExpansion.h"
10#include "mlir/Dialect/Arith/IR/Arith.h"
11#include "mlir/Dialect/Arith/Transforms/Passes.h"
12#include "mlir/Dialect/Vector/IR/VectorDialect.h"
13#include "mlir/IR/BuiltinTypeInterfaces.h"
14#include "mlir/IR/Location.h"
15#include "mlir/IR/TypeUtilities.h"
16#include "mlir/Transforms/DialectConversion.h"
17
18namespace mlir {
19namespace arith {
20#define GEN_PASS_DEF_ARITHEXPANDOPSPASS
21#include "mlir/Dialect/Arith/Transforms/Passes.h.inc"
22} // namespace arith
23} // namespace mlir
24
25using namespace mlir;
26
27/// Create an integer or index constant.
28static Value createConst(Location loc, Type type, int value,
29 PatternRewriter &rewriter) {
30 auto attr = rewriter.getIntegerAttr(type: getElementTypeOrSelf(type), value);
31 if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) {
32 return arith::ConstantOp::create(builder&: rewriter, location: loc,
33 value: DenseElementsAttr::get(type: shapedTy, values: attr));
34 }
35 return arith::ConstantOp::create(builder&: rewriter, location: loc, value: attr);
36}
37
38/// Create an integer constant from an APInt.
39static Value createAPIntConst(Location loc, Type type, const APInt &value,
40 PatternRewriter &rewriter) {
41 auto attr = IntegerAttr::get(type: getElementTypeOrSelf(type), value);
42 if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) {
43 return arith::ConstantOp::create(builder&: rewriter, location: loc,
44 value: DenseElementsAttr::get(type: shapedTy, values: attr));
45 }
46 return arith::ConstantOp::create(builder&: rewriter, location: loc, value: attr);
47}
48
49/// Create a float constant.
50static Value createFloatConst(Location loc, Type type, const APFloat &value,
51 PatternRewriter &rewriter) {
52 auto attr = rewriter.getFloatAttr(type: getElementTypeOrSelf(type), value);
53 if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) {
54 return arith::ConstantOp::create(builder&: rewriter, location: loc,
55 value: DenseElementsAttr::get(type: shapedTy, values: attr));
56 }
57
58 return arith::ConstantOp::create(builder&: rewriter, location: loc, value: attr);
59}
60
61/// Creates shapedType using shape from cloneFrom and base type from cloneTo
62static Type cloneToShapedType(Type cloneFrom, Type cloneTo) {
63 if (auto shapedTy = dyn_cast<ShapedType>(Val&: cloneFrom)) {
64 return shapedTy.clone(elementType: cloneTo);
65 }
66 return cloneTo;
67}
68
69/// Returns only the nnan flag from fastmath, or a null attribute if it is absent.
70static arith::FastMathFlagsAttr getNNaNOnly(MLIRContext *ctx,
71 arith::FastMathFlagsAttr fastmath) {
72 if (fastmath && arith::bitEnumContainsAll(bits: fastmath.getValue(),
73 bit: arith::FastMathFlags::nnan))
74 return arith::FastMathFlagsAttr::get(context: ctx, value: arith::FastMathFlags::nnan);
75 return {};
76}
77
78namespace {
79
80template <typename OpTy, arith::CmpIPredicate pred>
81struct MaxMinIOpConverter : public OpRewritePattern<OpTy> {
82public:
83 using OpRewritePattern<OpTy>::OpRewritePattern;
84
85 LogicalResult matchAndRewrite(OpTy op,
86 PatternRewriter &rewriter) const final {
87 Value lhs = op.getLhs();
88 Value rhs = op.getRhs();
89
90 Value cmp = arith::CmpIOp::create(rewriter, op.getLoc(), pred, lhs, rhs);
91 rewriter.replaceOpWithNewOp<arith::SelectOp>(op, cmp, lhs, rhs);
92 return success();
93 }
94};
95
96template <typename OpTy, arith::CmpFPredicate pred>
97struct MaximumMinimumFOpConverter : public OpRewritePattern<OpTy> {
98public:
99 using OpRewritePattern<OpTy>::OpRewritePattern;
100
101 LogicalResult matchAndRewrite(OpTy op,
102 PatternRewriter &rewriter) const final {
103 Value lhs = op.getLhs();
104 Value rhs = op.getRhs();
105
106 Location loc = op.getLoc();
107 // If any operand is NaN, 'cmp' will be true (and 'select' returns 'lhs').
108 static_assert(pred == arith::CmpFPredicate::UGT ||
109 pred == arith::CmpFPredicate::ULT,
110 "pred must be either UGT or ULT");
111 Value cmp = arith::CmpFOp::create(builder&: rewriter, location: loc, predicate: pred, lhs, rhs);
112 Value select = arith::SelectOp::create(builder&: rewriter, location: loc, condition: cmp, true_value: lhs, false_value: rhs);
113
114 // Handle the case where rhs is NaN: 'isNaN(rhs) ? rhs : select'.
115 Value isNaN = arith::CmpFOp::create(builder&: rewriter, location: loc,
116 predicate: arith::CmpFPredicate::UNO, lhs: rhs, rhs);
117 rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
118 return success();
119 }
120};
121
122template <typename OpTy, arith::CmpFPredicate pred>
123struct MaxNumMinNumFOpConverter : public OpRewritePattern<OpTy> {
124public:
125 using OpRewritePattern<OpTy>::OpRewritePattern;
126
127 LogicalResult matchAndRewrite(OpTy op,
128 PatternRewriter &rewriter) const final {
129 Value lhs = op.getLhs();
130 Value rhs = op.getRhs();
131
132 Location loc = op.getLoc();
133 // If any operand is NaN, 'cmp' will be true (and 'select' returns 'lhs').
134 static_assert(pred == arith::CmpFPredicate::UGT ||
135 pred == arith::CmpFPredicate::ULT,
136 "pred must be either UGT or ULT");
137 Value cmp = arith::CmpFOp::create(builder&: rewriter, location: loc, predicate: pred, lhs, rhs);
138 Value select = arith::SelectOp::create(builder&: rewriter, location: loc, condition: cmp, true_value: lhs, false_value: rhs);
139
140 // Handle the case where lhs is NaN: 'isNaN(lhs) ? rhs : select'.
141 Value isNaN = arith::CmpFOp::create(builder&: rewriter, location: loc,
142 predicate: arith::CmpFPredicate::UNO, lhs, rhs: lhs);
143 rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, rhs, select);
144 return success();
145 }
146};
147
148struct BFloat16ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
149 using Base::Base;
150 LogicalResult matchAndRewrite(arith::ExtFOp op,
151 PatternRewriter &rewriter) const final {
152 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
153 auto operand = op.getOperand();
154 Type operandTy = operand.getType();
155 Type resultTy = op.getType();
156 Type operandETy = getElementTypeOrSelf(type: operandTy);
157 Type resultETy = getElementTypeOrSelf(type: resultTy);
158
159 if (!operandETy.isBF16() || !resultETy.isF32()) {
160 return rewriter.notifyMatchFailure(arg&: op, msg: "not a ext of bf16 to f32.");
161 }
162
163 Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type());
164 Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type());
165
166 Value bitcast = arith::BitcastOp::create(builder&: b, out: i16Ty, in: operand);
167 Value exti = arith::ExtUIOp::create(builder&: b, out: i32Ty, in: bitcast);
168
169 Value c16 = createConst(loc: op.getLoc(), type: i32Ty, value: 16, rewriter);
170 Value shl = arith::ShLIOp::create(builder&: b, lhs: exti, rhs: c16);
171 Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: shl);
172
173 rewriter.replaceOp(op, newValues: result);
174 return success();
175 }
176};
177
178struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
179 using Base::Base;
180 LogicalResult matchAndRewrite(arith::TruncFOp op,
181 PatternRewriter &rewriter) const final {
182 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
183 auto operand = op.getOperand();
184 Type operandTy = operand.getType();
185 Type resultTy = op.getType();
186 Type operandETy = getElementTypeOrSelf(type: operandTy);
187 Type resultETy = getElementTypeOrSelf(type: resultTy);
188
189 if (!operandETy.isF32() || !resultETy.isBF16()) {
190 return rewriter.notifyMatchFailure(arg&: op, msg: "not a trunc of f32 to bf16.");
191 }
192
193 if (op.getRoundingmodeAttr()) {
194 return rewriter.notifyMatchFailure(
195 arg&: op, msg: "only applicable to default rounding mode.");
196 }
197
198 Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type());
199 Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type());
200
201 // Algorithm borrowed from this excellent code:
202 // https://github.com/pytorch/pytorch/blob/e1502c0cdbfd17548c612f25d5a65b1e4b86224d/c10/util/BFloat16.h#L60-L79
203 // There is a magic idea there, to let the addition of the rounding_bias to
204 // the mantissa simply overflow into the exponent bits. It's a bit of an
205 // aggressive, obfuscating optimization, but it is well-tested code, and it
206 // results in more concise and efficient IR.
207 // The case of NaN is handled separately (see isNaN and the final select).
208 // The case of infinities is NOT handled separately, which deserves an
209 // explanation. As the encoding of infinities has zero mantissa, the
210 // rounding-bias addition never carries into the exponent so that just gets
211 // truncated away, and as bfloat16 and float32 have the same number of
212 // exponent bits, that simple truncation is the desired outcome for
213 // infinities.
214 Value isNan =
215 arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNE, lhs: operand, rhs: operand);
216 // Constant used to make the rounding bias.
217 Value c7FFF = createConst(loc: op.getLoc(), type: i32Ty, value: 0x7fff, rewriter);
218 // Constant used to generate a quiet NaN.
219 Value c7FC0I16 = createConst(loc: op.getLoc(), type: i16Ty, value: 0x7fc0, rewriter);
220 // Small constants used to address bits.
221 Value c16 = createConst(loc: op.getLoc(), type: i32Ty, value: 16, rewriter);
222 Value c1 = createConst(loc: op.getLoc(), type: i32Ty, value: 1, rewriter);
223 // Reinterpret the input f32 value as bits.
224 Value bitcast = arith::BitcastOp::create(builder&: b, out: i32Ty, in: operand);
225 // Read bit 16 as a value in {0,1}.
226 Value bit16 =
227 arith::AndIOp::create(builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: bitcast, rhs: c16), rhs: c1);
228 // Determine the rounding bias to add as either 0x7fff or 0x8000 depending
229 // on bit 16, implementing the tie-breaking "to nearest even".
230 Value roundingBias = arith::AddIOp::create(builder&: b, lhs: bit16, rhs: c7FFF);
231 // Add the rounding bias. Generally we want this to be added to the
232 // mantissa, but nothing prevents this to from carrying into the exponent
233 // bits, which would feel like a bug, but this is the magic trick here:
234 // when that happens, the mantissa gets reset to zero and the exponent
235 // gets incremented by the carry... which is actually exactly what we
236 // want.
237 Value biased = arith::AddIOp::create(builder&: b, lhs: bitcast, rhs: roundingBias);
238 // Now that the rounding-bias has been added, truncating the low bits
239 // yields the correctly rounded result.
240 Value biasedAndShifted = arith::ShRUIOp::create(builder&: b, lhs: biased, rhs: c16);
241 Value normalCaseResultI16 =
242 arith::TruncIOp::create(builder&: b, out: i16Ty, in: biasedAndShifted);
243 // Select either the above-computed result, or a quiet NaN constant
244 // if the input was NaN.
245 Value select =
246 arith::SelectOp::create(builder&: b, condition: isNan, true_value: c7FC0I16, false_value: normalCaseResultI16);
247 Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: select);
248 rewriter.replaceOp(op, newValues: result);
249 return success();
250 }
251};
252
253/// In this implementation of extf we take advantage of some key patterns we
254/// notice between the binary representation of an F4E2M1 value and its
255/// corresponding value in F32.
256///
257/// Note: x is sign bit
258/// | Binary | F4E2M1 | f32[23:32]
259/// | x000 | 0.0 | x000 0000 00
260/// | x001 | 0.5 | x011 1111 00
261/// | x010 | 1.0 | x011 1111 10
262/// | x011 | 1.5 | x011 1111 11
263/// | x100 | 2.0 | x010 0000 00
264/// | x101 | 3.0 | x010 0000 01
265/// | x110 | 4.0 | x010 0000 10
266/// | x111 | 6.0 | x010 0000 11
267///
268/// 1) There are only two versions of bits [25:31] in the f32 result
269/// F4E2M1 bits[2:3] decide whether:
270/// - F32 bits[25:31] = 0011 1111
271/// - F32 bits[25:31] = 0010 0000
272/// Exception is zero where
273/// - F32 bits[25:31] = 0000 0000
274///
275/// 2) F4E2M1 bits[1:2] = F32 bits[23:24]
276/// Exception is 0.5 where
277/// - F4E2M1 bits[1:2] = 01, F32 bits[23:24] = 00
278///
279/// 3) F4E2M1 bits[4] = F32 bits[32] (sign bits are equal)
280///
281/// 4) F32 bits[1:22] = 0
282struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
283 using Base::Base;
284 LogicalResult matchAndRewrite(arith::ExtFOp op,
285 PatternRewriter &rewriter) const final {
286 Location loc = op.getLoc();
287 ImplicitLocOpBuilder b(loc, rewriter);
288 Value operand = op.getOperand();
289 Type operandTy = operand.getType();
290 Type resultTy = op.getType();
291 Type operandETy = getElementTypeOrSelf(type: operandTy);
292 Type resultETy = getElementTypeOrSelf(type: resultTy);
293
294 if (!isa<Float4E2M1FNType>(Val: operandETy))
295 return rewriter.notifyMatchFailure(arg&: op, msg: "not a ext of F4E2M1FN");
296
297 Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type());
298 Type i4Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI4Type());
299 Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type());
300 Value i4Bits = arith::BitcastOp::create(builder&: b, out: i4Ty, in: operand);
301
302 Value c0x0 = createConst(loc, type: i4Ty, value: 0x0, rewriter);
303 Value c0x1 = createConst(loc, type: i4Ty, value: 0x1, rewriter);
304 Value c0x2 = createConst(loc, type: i4Ty, value: 0x2, rewriter);
305 Value c0x4 = createConst(loc, type: i4Ty, value: 0x4, rewriter);
306 Value c0x7 = createConst(loc, type: i4Ty, value: 0x7, rewriter);
307
308 Value i4BitsNoSign = arith::AndIOp::create(builder&: b, lhs: i4Bits, rhs: c0x7);
309
310 // Set last Exponent bit and Mantissa.
311 Value c0x00000014 = createConst(loc, type: i32Ty, value: 0x14, rewriter);
312 Value bits1To24 = arith::ShLIOp::create(builder&: b, lhs: i4BitsNoSign, rhs: c0x2);
313 Value isHalf =
314 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: i4BitsNoSign, rhs: c0x1);
315 bits1To24 = arith::SelectOp::create(builder&: b, condition: isHalf, true_value: c0x0, false_value: bits1To24);
316 bits1To24 = arith::ExtUIOp::create(builder&: b, out: i32Ty, in: bits1To24);
317 bits1To24 = arith::ShLIOp::create(builder&: b, lhs: bits1To24, rhs: c0x00000014);
318
319 // Set first 7 bits of Exponent.
320 Value zeroExpBits = createConst(loc, type: i32Ty, value: 0x00000000, rewriter);
321 Value highExpBits = createConst(loc, type: i32Ty, value: 0x40000000, rewriter);
322 Value lowExpBits = createConst(loc, type: i32Ty, value: 0x3f000000, rewriter);
323 Value useLargerExp =
324 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::uge, lhs: i4BitsNoSign, rhs: c0x4);
325 Value bits25To31 =
326 arith::SelectOp::create(builder&: b, condition: useLargerExp, true_value: highExpBits, false_value: lowExpBits);
327 Value zeroExp =
328 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: i4BitsNoSign, rhs: c0x0);
329 bits25To31 = arith::SelectOp::create(builder&: b, condition: zeroExp, true_value: zeroExpBits, false_value: bits25To31);
330
331 // Set sign.
332 Value c0x80000000 = createConst(loc, type: i32Ty, value: 0x80000000, rewriter);
333 Value c0x8 = createConst(loc, type: i4Ty, value: 0x8, rewriter);
334 Value negative =
335 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::uge, lhs: i4Bits, rhs: c0x8);
336 Value bit32 =
337 arith::SelectOp::create(builder&: b, condition: negative, true_value: c0x80000000, false_value: zeroExpBits);
338
339 // Add segments together.
340 Value bits1To31 = arith::AddIOp::create(builder&: b, lhs: bits1To24, rhs: bits25To31);
341 Value bits1To32 = arith::AddIOp::create(builder&: b, lhs: bits1To31, rhs: bit32);
342 Value result = arith::BitcastOp::create(builder&: b, out: f32Ty, in: bits1To32);
343 if (!isa<Float32Type>(Val: resultETy))
344 result = arith::TruncFOp::create(builder&: b, out: resultTy, in: result);
345
346 rewriter.replaceOp(op, newValues: result);
347 return success();
348 }
349};
350
351struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
352 using Base::Base;
353 LogicalResult matchAndRewrite(arith::ExtFOp op,
354 PatternRewriter &rewriter) const final {
355 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
356 Value operand = op.getOperand();
357 Type operandTy = operand.getType();
358 Type resultTy = op.getType();
359 Type operandETy = getElementTypeOrSelf(type: operandTy);
360 Type resultETy = getElementTypeOrSelf(type: resultTy);
361
362 if (!llvm::isa<Float8E8M0FNUType>(Val: operandETy)) {
363 return rewriter.notifyMatchFailure(arg&: op, msg: "not a ext of F8E8M0FNU");
364 }
365
366 Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type());
367 Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type());
368 Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type());
369
370 Value bitcast = arith::BitcastOp::create(builder&: b, out: i8Ty, in: operand);
371 Value cF32MantissaWidth = createConst(loc: op->getLoc(), type: i32Ty, value: 23, rewriter);
372 Value exti = arith::ExtUIOp::create(builder&: b, out: i32Ty, in: bitcast);
373 Value f32Bits = arith::ShLIOp::create(builder&: b, lhs: exti, rhs: cF32MantissaWidth);
374
375 // F8E8M0 has no zero representation.
376 // All zeros encoding represents 2^-127 which is an f32 subnormal.
377 Value cF8Min = createConst(loc: op.getLoc(), type: i8Ty, value: 0, rewriter);
378 Value cF32Min = createConst(loc: op.getLoc(), type: i32Ty, value: 0x00400000, rewriter);
379 Value isMin =
380 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: bitcast, rhs: cF8Min);
381 f32Bits = arith::SelectOp::create(builder&: b, condition: isMin, true_value: cF32Min, false_value: f32Bits);
382
383 // If FastMathFlag allows no NaN checks, skip it
384 auto fastMath = op.getFastmathAttr();
385 bool NoNaN = fastMath
386 ? (fastMath.getValue() & arith::FastMathFlags::nnan) ==
387 arith::FastMathFlags::nnan
388 : false;
389 if (!NoNaN) {
390 Value cF8NaN = createConst(loc: op.getLoc(), type: i8Ty, value: 0xff, rewriter);
391 Value cF32NaN = createConst(loc: op.getLoc(), type: i32Ty, value: 0x7fc00000, rewriter);
392 Value isNan =
393 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: bitcast, rhs: cF8NaN);
394 // select for NaNs: the quiet NaN, as the constant folder gives
395 f32Bits = arith::SelectOp::create(builder&: b, condition: isNan, true_value: cF32NaN, false_value: f32Bits);
396 }
397
398 Value result = arith::BitcastOp::create(builder&: b, out: f32Ty, in: f32Bits);
399 if (resultETy.getIntOrFloatBitWidth() < 32) {
400 result = arith::TruncFOp::create(builder&: b, out: resultTy, in: result, roundingmode: nullptr,
401 fastmath: op.getFastmathAttr());
402 } else if (resultETy.getIntOrFloatBitWidth() > 32) {
403 result = arith::ExtFOp::create(builder&: b, out: resultTy, in: result, fastmath: op.getFastmathAttr());
404 }
405 rewriter.replaceOp(op, newValues: result);
406 return success();
407 }
408};
409
410/// Conversion from F32 to F4E2M1 according to the OCP Spec:
411/// www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
412///
413/// The spec requiers us to perform Round to Nearest, Ties to Even.
414///
415/// This means that after rounding, we should break ties by choosing the option
416/// which results in a mantissa of 0 in the least significant digit.
417///
418/// Table of representable values in F4E2M1:
419///
420/// Note: x is sign bit
421/// | Binary | F4E2M1 | F32[23:32]
422/// | x000 | 0.0 | x000 0000 00
423/// | x001 | 0.5 | x011 1111 00
424/// | x010 | 1.0 | x011 1111 10
425/// | x011 | 1.5 | x011 1111 11
426/// | x100 | 2.0 | x010 0000 00
427/// | x101 | 3.0 | x010 0000 01
428/// | x110 | 4.0 | x010 0000 10
429/// | x111 | 6.0 | x010 0000 11
430///
431/// Conversion procedure:
432/// Step 1: Clamp to representable bounds.
433/// Step 2: Convert exponent by adjusting bias.
434/// Step 3: Set mantissa to first bit.
435/// Step 4: Special consideration for subnormal and zero exponent.
436/// Step 5: Round up if necessary, if mantissa[1:] greater than 1000000 or
437/// subnormal.
438struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
439 using Base::Base;
440 LogicalResult matchAndRewrite(arith::TruncFOp op,
441 PatternRewriter &rewriter) const final {
442 Location loc = op.getLoc();
443 ImplicitLocOpBuilder b(loc, rewriter);
444 Value operand = op.getOperand();
445 Type operandTy = operand.getType();
446 Type resultTy = op.getType();
447 Type operandETy = getElementTypeOrSelf(type: operandTy);
448 Type resultETy = getElementTypeOrSelf(type: resultTy);
449
450 Type i4Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI4Type());
451 Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type());
452 Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type());
453 Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type());
454
455 if (!isa<Float4E2M1FNType>(Val: resultETy))
456 return rewriter.notifyMatchFailure(arg&: op, msg: "not a trunc of F4E2M1FN");
457 if (!isa<Float32Type>(Val: operandETy))
458 operand =
459 arith::ExtFOp::create(builder&: b, out: f32Ty, in: operand, fastmath: arith::FastMathFlagsAttr{});
460
461 Value c0x1 = createConst(loc, type: i4Ty, value: 1, rewriter);
462 Value c0x3 = createConst(loc, type: i4Ty, value: 3, rewriter);
463 Value c0x00000016 = createConst(loc, type: i32Ty, value: 22, rewriter);
464 Value c0x00 = createConst(loc, type: i8Ty, value: 0x00, rewriter);
465 Value c0xff = createConst(loc, type: i8Ty, value: 0xff, rewriter);
466 Value zeroExpBits = createConst(loc, type: i32Ty, value: 0, rewriter);
467
468 // Step 0: Clamp to bounds.
469 Value cHigherBound = createFloatConst(loc, type: f32Ty, value: APFloat(6.0f), rewriter);
470 Value cLowerBound = createFloatConst(loc, type: f32Ty, value: APFloat(-6.0f), rewriter);
471 Value operandClamped = arith::MinNumFOp::create(builder&: b, lhs: cHigherBound, rhs: operand);
472 operandClamped = arith::MaxNumFOp::create(builder&: b, lhs: cLowerBound, rhs: operandClamped);
473 Value f32Bits = arith::BitcastOp::create(builder&: b, out: i32Ty, in: operandClamped);
474
475 // Step 1: Set sign bit.
476 Value cF32ExpManWidth = createConst(loc, type: i32Ty, value: 31, rewriter); // 23
477 Value f32Sign = arith::ShRUIOp::create(builder&: b, lhs: f32Bits, rhs: cF32ExpManWidth);
478 Value f4Sign = arith::TruncIOp::create(builder&: b, out: i4Ty, in: f32Sign);
479 Value f4Bits = arith::ShLIOp::create(builder&: b, lhs: f4Sign, rhs: c0x3);
480
481 // Step 2: Convert exponent by adjusting bias.
482 Value biasAdjustment = createConst(loc, type: i32Ty, value: 0x7e, rewriter);
483 Value cF4MantissaWidth = c0x1; // 1
484 Value cF32MantissaWidth = createConst(loc, type: i32Ty, value: 23, rewriter); // 23
485 Value f32SignExp = arith::ShRUIOp::create(builder&: b, lhs: f32Bits, rhs: cF32MantissaWidth);
486 Value biasAdjustedSignExp =
487 arith::SubIOp::create(builder&: b, lhs: f32SignExp, rhs: biasAdjustment);
488 Value f4Exp = arith::TruncIOp::create(builder&: b, out: i4Ty, in: biasAdjustedSignExp);
489 f4Exp = arith::ShLIOp::create(builder&: b, lhs: f4Exp, rhs: cF4MantissaWidth);
490 f4Bits = arith::AddIOp::create(builder&: b, lhs: f4Bits, rhs: f4Exp);
491
492 // Step 3: Set mantissa to first bit.
493 Value cF32FirstBitMask = createConst(loc, type: i32Ty, value: 0x400000, rewriter);
494 Value man1Bit = arith::AndIOp::create(builder&: b, lhs: f32Bits, rhs: cF32FirstBitMask);
495 man1Bit = arith::ShRUIOp::create(builder&: b, lhs: man1Bit, rhs: c0x00000016);
496 Value f4Man = arith::TruncIOp::create(builder&: b, out: i4Ty, in: man1Bit);
497 f4Bits = arith::AddIOp::create(builder&: b, lhs: f4Bits, rhs: f4Man);
498
499 // Step 4: Special consideration for conversion to 0.5.
500 Value cF32MantissaMask = createConst(loc, type: i32Ty, value: 0x7fffff, rewriter);
501 Value f8Exp = arith::TruncIOp::create(builder&: b, out: i8Ty, in: biasAdjustedSignExp);
502 Value isSubnormal =
503 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::sle, lhs: f8Exp, rhs: c0x00);
504 Value isNegOneExp =
505 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: f8Exp, rhs: c0xff);
506 Value man23Bits = arith::AndIOp::create(builder&: b, lhs: f32Bits, rhs: cF32MantissaMask);
507 Value isNonZeroMan = arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::ugt,
508 lhs: man23Bits, rhs: zeroExpBits);
509 Value roundToHalf = arith::AndIOp::create(builder&: b, lhs: isNegOneExp, rhs: isNonZeroMan);
510 Value isZeroExp =
511 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: f8Exp, rhs: c0x00);
512 Value subnormalF4Bits = createConst(loc, type: i4Ty, value: 0xf, rewriter);
513 Value halfF4Bits = createConst(loc, type: i4Ty, value: 0x0, rewriter);
514 Value subResult =
515 arith::SelectOp::create(builder&: b, condition: isSubnormal, true_value: subnormalF4Bits, false_value: f4Bits);
516 subResult = arith::SelectOp::create(builder&: b, condition: roundToHalf, true_value: halfF4Bits, false_value: subResult);
517 f4Bits = arith::SelectOp::create(builder&: b, condition: isZeroExp, true_value: f4Bits, false_value: subResult);
518
519 // Step 5: Round up if necessary.
520 Value cF32Last22BitMask = createConst(loc, type: i32Ty, value: 0x3fffff, rewriter);
521 Value cRound = createConst(loc, type: i32Ty, value: 0x200000, rewriter); // 010 0000...
522 Value man22Bits = arith::AndIOp::create(builder&: b, lhs: f32Bits, rhs: cF32Last22BitMask);
523 Value shouldRound =
524 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::uge, lhs: man22Bits, rhs: cRound);
525 shouldRound = arith::OrIOp::create(builder&: b, lhs: shouldRound, rhs: isSubnormal);
526 Value roundedF4Bits = arith::AddIOp::create(builder&: b, lhs: f4Bits, rhs: c0x1);
527 f4Bits = arith::SelectOp::create(builder&: b, condition: shouldRound, true_value: roundedF4Bits, false_value: f4Bits);
528
529 Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: f4Bits);
530 rewriter.replaceOp(op, newValues: result);
531 return success();
532 }
533};
534
535/*
536TruncF to F8E8M0 is expected to extract exponent bits out of F32 type
537Since All kinds of Infs and NaNs are mapped to same exponent bits in F32 type,
538they all map to NaN in F8E8M0 Type.
539*/
540struct F8E8M0TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
541 using Base::Base;
542 LogicalResult matchAndRewrite(arith::TruncFOp op,
543 PatternRewriter &rewriter) const final {
544 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
545 Value operand = op.getOperand();
546 Type operandTy = operand.getType();
547 Type operandETy = getElementTypeOrSelf(type: operandTy);
548 Type resultTy = op.getType();
549 Type resultETy = getElementTypeOrSelf(type: resultTy);
550 if (!llvm::isa<Float8E8M0FNUType>(Val: resultETy)) {
551 return rewriter.notifyMatchFailure(arg&: op, msg: "not a truncf to f8E8M0FNU");
552 }
553
554 if (op.getRoundingmodeAttr()) {
555 return rewriter.notifyMatchFailure(
556 arg&: op, msg: "only applicable to default rounding mode.");
557 }
558
559 Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type());
560 Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type());
561 Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type());
562
563 if (operandETy.getIntOrFloatBitWidth() < 32) {
564 // Propagating contract would let ExtFOp::fold cancel an earlier contract
565 // truncf, skipping its rounding before we extract the exponent bits.
566 operand = arith::ExtFOp::create(
567 builder&: b, out: f32Ty, in: operand, fastmath: getNNaNOnly(ctx: b.getContext(), fastmath: op.getFastmathAttr()));
568 } else if (operandETy.getIntOrFloatBitWidth() > 32) {
569 operand = arith::TruncFOp::create(
570 builder&: b, out: f32Ty, in: operand, roundingmode: op.getRoundingmodeAttr(), fastmath: op.getFastmathAttr());
571 }
572 Value f32Bits = arith::BitcastOp::create(builder&: b, out: i32Ty, in: operand);
573 Value cF32MantissaWidth = createConst(loc: op->getLoc(), type: i32Ty, value: 23, rewriter);
574 Value f32SignExp = arith::ShRUIOp::create(builder&: b, lhs: f32Bits, rhs: cF32MantissaWidth);
575 Value exp8Bits = arith::TruncIOp::create(builder&: b, out: i8Ty, in: f32SignExp);
576 Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: exp8Bits);
577 rewriter.replaceOp(op, newValues: result);
578 return success();
579 }
580};
581
582/// Expand an ExtF from F8E5M2. F8E5M2 uses a 5-bit exponent with bias 15 and a
583/// 2-bit mantissa, i.e. it is bit-for-bit the high byte of an IEEE F16 value
584/// (same exponent field and bias), including infinities and NaNs. The exact
585/// F16 value is therefore obtained by placing the 8 F8E5M2 bits into the high
586/// byte of an i16. The F16 value is then converted to the requested result
587/// type with the native (LLVM-lowerable) extf/truncf.
588struct F8E5M2ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
589 using Base::Base;
590 LogicalResult matchAndRewrite(arith::ExtFOp op,
591 PatternRewriter &rewriter) const final {
592 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
593 Value operand = op.getOperand();
594 Type operandTy = operand.getType();
595 Type resultTy = op.getType();
596 Type operandETy = getElementTypeOrSelf(type: operandTy);
597 Type resultETy = getElementTypeOrSelf(type: resultTy);
598
599 if (!llvm::isa<Float8E5M2Type>(Val: operandETy))
600 return rewriter.notifyMatchFailure(arg&: op, msg: "not a ext of F8E5M2");
601
602 Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type());
603 Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type());
604 Type f16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF16Type());
605
606 Value bitcast = arith::BitcastOp::create(builder&: b, out: i8Ty, in: operand);
607 Value exti = arith::ExtUIOp::create(builder&: b, out: i16Ty, in: bitcast);
608 Value c8 = createConst(loc: op.getLoc(), type: i16Ty, value: 8, rewriter);
609 Value f16Bits = arith::ShLIOp::create(builder&: b, lhs: exti, rhs: c8);
610 Value f16 = arith::BitcastOp::create(builder&: b, out: f16Ty, in: f16Bits);
611
612 Value result = f16;
613 if (!resultETy.isF16()) {
614 if (resultETy.getIntOrFloatBitWidth() < 16)
615 result = arith::TruncFOp::create(builder&: b, out: resultTy, in: f16, roundingmode: nullptr,
616 fastmath: op.getFastmathAttr());
617 else
618 result = arith::ExtFOp::create(builder&: b, out: resultTy, in: f16, fastmath: op.getFastmathAttr());
619 }
620 rewriter.replaceOp(op, newValues: result);
621 return success();
622 }
623};
624
625/// Expand a TruncF to F8E5M2. The input is first reduced to F16 (which shares
626/// the F8E5M2 exponent layout and bias) using the native truncf, then the low
627/// 8 mantissa bits of the F16 value are dropped with round-to-nearest-even.
628/// The rounding-bias trick is borrowed from the BF16 converter: adding the
629/// bias may carry into the exponent field, which is exactly the desired
630/// behavior since F16 and F8E5M2 share the same exponent bias. NaN is handled
631/// separately.
632struct F8E5M2TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
633 using Base::Base;
634 LogicalResult matchAndRewrite(arith::TruncFOp op,
635 PatternRewriter &rewriter) const final {
636 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
637 Value operand = op.getOperand();
638 Type operandTy = operand.getType();
639 Type resultTy = op.getType();
640 Type operandETy = getElementTypeOrSelf(type: operandTy);
641 Type resultETy = getElementTypeOrSelf(type: resultTy);
642
643 if (!llvm::isa<Float8E5M2Type>(Val: resultETy))
644 return rewriter.notifyMatchFailure(arg&: op, msg: "not a trunc to F8E5M2");
645 if (op.getRoundingmodeAttr())
646 return rewriter.notifyMatchFailure(
647 arg&: op, msg: "only applicable to default rounding mode.");
648
649 Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type());
650 Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type());
651 Type f16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF16Type());
652
653 Value h16 = operand;
654 if (!operandETy.isF16())
655 h16 = arith::TruncFOp::create(builder&: b, out: f16Ty, in: operand, roundingmode: nullptr,
656 fastmath: op.getFastmathAttr());
657
658 Value isNan = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNE, lhs: h16, rhs: h16);
659 Value h16Bits = arith::BitcastOp::create(builder&: b, out: i16Ty, in: h16);
660 // Rounding bias constants for dropping the low 8 mantissa bits.
661 Value c7F = createConst(loc: op.getLoc(), type: i16Ty, value: 0x7f, rewriter);
662 Value c8 = createConst(loc: op.getLoc(), type: i16Ty, value: 8, rewriter);
663 Value c1 = createConst(loc: op.getLoc(), type: i16Ty, value: 1, rewriter);
664 Value bit8 =
665 arith::AndIOp::create(builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: h16Bits, rhs: c8), rhs: c1);
666 Value roundingBias = arith::AddIOp::create(builder&: b, lhs: bit8, rhs: c7F);
667 Value biased = arith::AddIOp::create(builder&: b, lhs: h16Bits, rhs: roundingBias);
668 Value biasedAndShifted = arith::ShRUIOp::create(builder&: b, lhs: biased, rhs: c8);
669 Value normalCaseResult = arith::TruncIOp::create(builder&: b, out: i8Ty, in: biasedAndShifted);
670 // Quiet NaN for F8E5M2 (exponent all ones, mantissa MSB set).
671 Value cNan = createConst(loc: op.getLoc(), type: i8Ty, value: 0x7e, rewriter);
672 Value select = arith::SelectOp::create(builder&: b, condition: isNan, true_value: cNan, false_value: normalCaseResult);
673 Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: select);
674 rewriter.replaceOp(op, newValues: result);
675 return success();
676 }
677};
678
679/// Expand an ExtF from F8E4M3FN. F8E4M3FN uses a 4-bit exponent with bias 7, a
680/// 3-bit mantissa, no infinities and a single NaN encoding (S.1111.111). The
681/// 7 magnitude bits (EEEE.MMM) are placed into the high mantissa/exponent bits
682/// of an F16 by shifting left by 7, producing an F16 whose value equals the
683/// desired magnitude scaled by 2^-8 (the F16 bias is 15 while F8E4M3FN's is
684/// 7). Multiplying by 256 in F32 recovers the true magnitude for both normal
685/// and subnormal inputs. The sign bit and the NaN encoding are re-applied
686/// explicitly.
687struct F8E4M3FNExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
688 using Base::Base;
689 LogicalResult matchAndRewrite(arith::ExtFOp op,
690 PatternRewriter &rewriter) const final {
691 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
692 Value operand = op.getOperand();
693 Type operandTy = operand.getType();
694 Type resultTy = op.getType();
695 Type operandETy = getElementTypeOrSelf(type: operandTy);
696 Type resultETy = getElementTypeOrSelf(type: resultTy);
697
698 if (!llvm::isa<Float8E4M3FNType>(Val: operandETy))
699 return rewriter.notifyMatchFailure(arg&: op, msg: "not a ext of F8E4M3FN");
700
701 Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type());
702 Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type());
703 Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type());
704 Type f16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF16Type());
705 Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type());
706
707 Value bits = arith::BitcastOp::create(builder&: b, out: i8Ty, in: operand);
708 Value c7F8 = createConst(loc: op.getLoc(), type: i8Ty, value: 0x7f, rewriter);
709 Value mag8 = arith::AndIOp::create(builder&: b, lhs: bits, rhs: c7F8);
710 // Build an F16 equal to the magnitude times 2^-8.
711 Value mag16 = arith::ExtUIOp::create(builder&: b, out: i16Ty, in: mag8);
712 Value c7 = createConst(loc: op.getLoc(), type: i16Ty, value: 7, rewriter);
713 Value g16Bits = arith::ShLIOp::create(builder&: b, lhs: mag16, rhs: c7);
714 Value g16 = arith::BitcastOp::create(builder&: b, out: f16Ty, in: g16Bits);
715 Value gF32 = arith::ExtFOp::create(builder&: b, out: f32Ty, in: g16, fastmath: op.getFastmathAttr());
716 Value c256 =
717 createFloatConst(loc: op.getLoc(), type: f32Ty, value: APFloat(256.0f), rewriter);
718 Value magF32 = arith::MulFOp::create(builder&: b, lhs: gF32, rhs: c256, fastmath: op.getFastmathAttr());
719 // Re-apply the sign bit into the F32 result.
720 Value magI32 = arith::BitcastOp::create(builder&: b, out: i32Ty, in: magF32);
721 Value c80I8 = createConst(loc: op.getLoc(), type: i8Ty, value: 0x80, rewriter);
722 Value sign8 = arith::AndIOp::create(builder&: b, lhs: bits, rhs: c80I8);
723 Value sign32 = arith::ExtUIOp::create(builder&: b, out: i32Ty, in: sign8);
724 Value c24 = createConst(loc: op.getLoc(), type: i32Ty, value: 24, rewriter);
725 Value signBit = arith::ShLIOp::create(builder&: b, lhs: sign32, rhs: c24);
726 Value signedI32 = arith::OrIOp::create(builder&: b, lhs: magI32, rhs: signBit);
727 Value signedF32 = arith::BitcastOp::create(builder&: b, out: f32Ty, in: signedI32);
728 // NaN encoding is magnitude == 0x7f.
729 Value isNan =
730 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: mag8, rhs: c7F8);
731 Value cNan32 = createConst(loc: op.getLoc(), type: i32Ty, value: 0x7fc00000, rewriter);
732 Value nanSigned = arith::OrIOp::create(builder&: b, lhs: cNan32, rhs: signBit);
733 Value nanF32 = arith::BitcastOp::create(builder&: b, out: f32Ty, in: nanSigned);
734 Value resultF32 = arith::SelectOp::create(builder&: b, condition: isNan, true_value: nanF32, false_value: signedF32);
735
736 Value result = resultF32;
737 if (!resultETy.isF32()) {
738 if (resultETy.getIntOrFloatBitWidth() < 32)
739 result = arith::TruncFOp::create(builder&: b, out: resultTy, in: resultF32, roundingmode: nullptr,
740 fastmath: op.getFastmathAttr());
741 else
742 result =
743 arith::ExtFOp::create(builder&: b, out: resultTy, in: resultF32, fastmath: op.getFastmathAttr());
744 }
745 rewriter.replaceOp(op, newValues: result);
746 return success();
747 }
748};
749
750/// Expand a TruncF to F8E4M3FN. The magnitude is scaled by 2^-8 and reduced to
751/// F16 (undoing the bias difference so the F16 value equals the magnitude times
752/// 2^-8). The low 7 bits of the F16 encoding are then dropped with
753/// round-to-nearest-even to recover the 7 magnitude bits. F8E4M3FN has no
754/// infinity, so any input that overflows the maximum representable magnitude
755/// (448), as well as infinities and NaNs, maps to the F8E4M3FN NaN encoding to
756/// match the LLVM APFloat NanOnly overflow behavior and the OCP FP8 (E4M3)
757/// spec.
758struct F8E4M3FNTruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
759 using Base::Base;
760 LogicalResult matchAndRewrite(arith::TruncFOp op,
761 PatternRewriter &rewriter) const final {
762 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
763 Value operand = op.getOperand();
764 Type operandTy = operand.getType();
765 Type resultTy = op.getType();
766 Type operandETy = getElementTypeOrSelf(type: operandTy);
767 Type resultETy = getElementTypeOrSelf(type: resultTy);
768
769 if (!llvm::isa<Float8E4M3FNType>(Val: resultETy))
770 return rewriter.notifyMatchFailure(arg&: op, msg: "not a trunc to F8E4M3FN");
771 if (op.getRoundingmodeAttr())
772 return rewriter.notifyMatchFailure(
773 arg&: op, msg: "only applicable to default rounding mode.");
774
775 Type i8Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI8Type());
776 Type i16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI16Type());
777 Type i32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getI32Type());
778 Type f16Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF16Type());
779 Type f32Ty = cloneToShapedType(cloneFrom: operandTy, cloneTo: b.getF32Type());
780
781 Value f32 = operand;
782 if (!operandETy.isF32()) {
783 if (operandETy.getIntOrFloatBitWidth() < 32)
784 f32 = arith::ExtFOp::create(builder&: b, out: f32Ty, in: operand, fastmath: op.getFastmathAttr());
785 else
786 f32 = arith::TruncFOp::create(builder&: b, out: f32Ty, in: operand, roundingmode: nullptr,
787 fastmath: op.getFastmathAttr());
788 }
789
790 Value isNan = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::UNE, lhs: f32, rhs: f32);
791 // Split sign and magnitude.
792 Value f32Bits = arith::BitcastOp::create(builder&: b, out: i32Ty, in: f32);
793 Value cSignMask = createConst(loc: op.getLoc(), type: i32Ty, value: 0x80000000, rewriter);
794 Value cAbsMask = createConst(loc: op.getLoc(), type: i32Ty, value: 0x7fffffff, rewriter);
795 Value signBits = arith::AndIOp::create(builder&: b, lhs: f32Bits, rhs: cSignMask);
796 Value absBits = arith::AndIOp::create(builder&: b, lhs: f32Bits, rhs: cAbsMask);
797 Value absF32 = arith::BitcastOp::create(builder&: b, out: f32Ty, in: absBits);
798 // F8E4M3FN has no infinity: a magnitude above the round-to-nearest-even
799 // overflow boundary (464 = 448 + half an ulp) or an infinity maps to NaN.
800 Value cOverflow =
801 createFloatConst(loc: op.getLoc(), type: f32Ty, value: APFloat(464.0f), rewriter);
802 Value isOverflow =
803 arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OGT, lhs: absF32, rhs: cOverflow);
804 // Clamp to the F8E4M3FN maximum magnitude (448) so the finite path stays
805 // well-defined; overflowing inputs are replaced by NaN below.
806 Value cMax =
807 createFloatConst(loc: op.getLoc(), type: f32Ty, value: APFloat(448.0f), rewriter);
808 absF32 = arith::MinNumFOp::create(builder&: b, lhs: absF32, rhs: cMax);
809 Value cInv256 =
810 createFloatConst(loc: op.getLoc(), type: f32Ty, value: APFloat(0.00390625f), rewriter);
811 Value scaled = arith::MulFOp::create(builder&: b, lhs: absF32, rhs: cInv256, fastmath: nullptr);
812 Value h16 = arith::TruncFOp::create(builder&: b, out: f16Ty, in: scaled, roundingmode: nullptr,
813 fastmath: op.getFastmathAttr());
814 Value h16Bits = arith::BitcastOp::create(builder&: b, out: i16Ty, in: h16);
815 // Drop the low 7 bits with round-to-nearest-even.
816 Value c3F = createConst(loc: op.getLoc(), type: i16Ty, value: 0x3f, rewriter);
817 Value c7 = createConst(loc: op.getLoc(), type: i16Ty, value: 7, rewriter);
818 Value c1 = createConst(loc: op.getLoc(), type: i16Ty, value: 1, rewriter);
819 Value bit7 =
820 arith::AndIOp::create(builder&: b, lhs: arith::ShRUIOp::create(builder&: b, lhs: h16Bits, rhs: c7), rhs: c1);
821 Value roundingBias = arith::AddIOp::create(builder&: b, lhs: bit7, rhs: c3F);
822 Value biased = arith::AddIOp::create(builder&: b, lhs: h16Bits, rhs: roundingBias);
823 Value shifted = arith::ShRUIOp::create(builder&: b, lhs: biased, rhs: c7);
824 Value mag8 = arith::TruncIOp::create(builder&: b, out: i8Ty, in: shifted);
825 Value c7F8 = createConst(loc: op.getLoc(), type: i8Ty, value: 0x7f, rewriter);
826 mag8 = arith::AndIOp::create(builder&: b, lhs: mag8, rhs: c7F8);
827 // Re-apply the sign.
828 Value c24 = createConst(loc: op.getLoc(), type: i32Ty, value: 24, rewriter);
829 Value sign8 = arith::TruncIOp::create(
830 builder&: b, out: i8Ty, in: arith::ShRUIOp::create(builder&: b, lhs: signBits, rhs: c24));
831 Value res8 = arith::OrIOp::create(builder&: b, lhs: mag8, rhs: sign8);
832 // NaN input or an overflowing/infinite magnitude maps to the NaN encoding.
833 Value isNanOrOverflow = arith::OrIOp::create(builder&: b, lhs: isNan, rhs: isOverflow);
834 Value cNan8 = createConst(loc: op.getLoc(), type: i8Ty, value: 0x7f, rewriter);
835 Value res = arith::SelectOp::create(builder&: b, condition: isNanOrOverflow, true_value: cNan8, false_value: res8);
836 Value result = arith::BitcastOp::create(builder&: b, out: resultTy, in: res);
837 rewriter.replaceOp(op, newValues: result);
838 return success();
839 }
840};
841
842struct ScalingExtFOpConverter : public OpRewritePattern<arith::ScalingExtFOp> {
843 using Base::Base;
844 LogicalResult matchAndRewrite(arith::ScalingExtFOp op,
845 PatternRewriter &rewriter) const final {
846 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
847 Value inputOperand = op.getIn();
848 Value scaleOperand = op.getScale();
849 Type scaleTy = scaleOperand.getType();
850 Type scaleETy = getElementTypeOrSelf(val: scaleOperand);
851 // allow implicit exponent extraction from 16/32 bits floats
852 if (scaleETy.getIntOrFloatBitWidth() >= 16) {
853 scaleETy = b.getF8E8M0Type();
854 scaleTy = cloneToShapedType(cloneFrom: scaleTy, cloneTo: scaleETy);
855 scaleOperand = arith::TruncFOp::create(builder&: b, out: scaleTy, in: scaleOperand, roundingmode: nullptr,
856 fastmath: op.getFastmathAttr());
857 }
858 // Catch scale types like f8E5M2.
859 if (!llvm::isa<Float8E8M0FNUType>(Val: scaleETy)) {
860 return rewriter.notifyMatchFailure(
861 arg&: op, msg: "scaling_extf is using scales of type which can not be converted "
862 "to f8E8M0FNU");
863 }
864 Type resultTy = op.getType();
865 // extf on scale will essentially create floating point number
866 // of type resulTy that is 2^scale and will also propagate NaNs
867 Value scaleExt =
868 arith::ExtFOp::create(builder&: b, out: resultTy, in: scaleOperand, fastmath: op.getFastmathAttr());
869 Value inputExt =
870 arith::ExtFOp::create(builder&: b, out: resultTy, in: inputOperand, fastmath: op.getFastmathAttr());
871 Value result =
872 arith::MulFOp::create(builder&: b, lhs: inputExt, rhs: scaleExt, fastmath: op.getFastmathAttr());
873 rewriter.replaceOp(op, newValues: result);
874 return success();
875 }
876};
877
878/*
879Expands arith.ScalingTruncFOp(in, scale) into
880 scale = arith.truncf(scale) : scaleTy -> f8E8M0FNU
881 result = arith.truncf(in / (2^scale))
882 */
883struct ScalingTruncFOpConverter
884 : public OpRewritePattern<arith::ScalingTruncFOp> {
885 using Base::Base;
886 LogicalResult matchAndRewrite(arith::ScalingTruncFOp op,
887 PatternRewriter &rewriter) const final {
888 ImplicitLocOpBuilder b(op.getLoc(), rewriter);
889 Value inputOperand = op.getIn();
890 Value scaleOperand = op.getScale();
891 Type scaleTy = scaleOperand.getType();
892 Type scaleETy = getElementTypeOrSelf(val: scaleOperand);
893 // allow implicit exponent extraction from 16/32 bits floats
894 if (scaleETy.getIntOrFloatBitWidth() >= 16) {
895 scaleETy = b.getF8E8M0Type();
896 scaleTy = cloneToShapedType(cloneFrom: scaleTy, cloneTo: scaleETy);
897 scaleOperand = arith::TruncFOp::create(builder&: b, out: scaleTy, in: scaleOperand, roundingmode: nullptr,
898 fastmath: op.getFastmathAttr());
899 }
900 if (!llvm::isa<Float8E8M0FNUType>(Val: scaleETy)) {
901 return rewriter.notifyMatchFailure(
902 arg&: op, msg: "scaling_truncf is using scales type which can not be converted "
903 "to f8E8M0FNU");
904 }
905 Type resultTy = op.getType();
906 Type inputTy = inputOperand.getType();
907 // this will create a floating point number of type
908 // inputTy that is 2^scale and will also propagate NaNs
909 scaleOperand =
910 arith::ExtFOp::create(builder&: b, out: inputTy, in: scaleOperand, fastmath: op.getFastmathAttr());
911 Value result = arith::DivFOp::create(builder&: b, lhs: inputOperand, rhs: scaleOperand,
912 fastmath: op.getFastmathAttr());
913 Value resultCast = arith::TruncFOp::create(
914 builder&: b, out: resultTy, in: result, roundingmode: op.getRoundingmodeAttr(), fastmath: op.getFastmathAttr());
915 rewriter.replaceOp(op, newValues: resultCast);
916 return success();
917 }
918};
919
920/// Expands `arith.flush_denormals` into integer arithmetic.
921///
922/// For an IEEE-like floating-point value with a sign|exponent|mantissa bit
923/// layout, a value is denormal iff its biased exponent field is zero and its
924/// stored mantissa is non-zero. When the exponent field is zero, the value is
925/// either pos/neg 0 (mantissa = 0) or a denormal (mantissa != 0); in both
926/// cases, clearing the mantissa bits produces the desired sign-preserved zero
927/// (a no-op for pos/neg 0, a flush for denormals). When the exponent field is
928/// non-zero, the value passes through unchanged.
929///
930/// Pseudocode:
931/// bits = bitcast(x, iN)
932/// expIsZero = (bits & expMask) == 0
933/// cleared = bits & ~manMask
934/// resultBits = select(expIsZero, cleared, bits)
935/// result = bitcast(resultBits, floatTy)
936struct FlushDenormalsOpConverter
937 : public OpRewritePattern<arith::FlushDenormalsOp> {
938 using Base::Base;
939 LogicalResult matchAndRewrite(arith::FlushDenormalsOp op,
940 PatternRewriter &rewriter) const final {
941 Location loc = op.getLoc();
942 ImplicitLocOpBuilder b(loc, rewriter);
943 Value operand = op.getOperand();
944 Type operandTy = operand.getType();
945 auto floatTy = dyn_cast<FloatType>(Val: getElementTypeOrSelf(type: operandTy));
946 if (!floatTy)
947 return rewriter.notifyMatchFailure(arg&: op, msg: "operand is not a float type");
948
949 const llvm::fltSemantics &sem = floatTy.getFloatSemantics();
950 // Restrict to IEEE-like encodings, where the sign bit is the MSB and
951 // denormals are exactly "biased exponent == 0 and non-zero mantissa".
952 if (!llvm::APFloatBase::isIEEELikeFP(sem))
953 return rewriter.notifyMatchFailure(
954 arg&: op, msg: "only IEEE-like floating-point types are supported");
955
956 unsigned totalBits = llvm::APFloatBase::semanticsSizeInBits(sem);
957 unsigned precision = llvm::APFloatBase::semanticsPrecision(sem);
958 // Stored mantissa bits = precision - 1 (implicit leading bit not stored).
959 // Exponent field bits = totalBits - 1 (sign) - storedMantissa.
960 if (precision < 1 || precision > totalBits)
961 return rewriter.notifyMatchFailure(arg&: op, msg: "unexpected float semantics");
962 unsigned mantissaBits = precision - 1;
963 unsigned expBits = totalBits - 1 - mantissaBits;
964 if (expBits == 0 || mantissaBits == 0)
965 return rewriter.notifyMatchFailure(
966 arg&: op, msg: "degenerate float encoding has no exponent or mantissa");
967
968 Type intTy =
969 cloneToShapedType(cloneFrom: operandTy, cloneTo: rewriter.getIntegerType(width: totalBits));
970 Value bits = arith::BitcastOp::create(builder&: b, out: intTy, in: operand);
971 APInt expMaskVal =
972 APInt::getBitsSet(numBits: totalBits, loBit: mantissaBits, hiBit: mantissaBits + expBits);
973 APInt clearMantissaMaskVal = ~APInt::getLowBitsSet(numBits: totalBits, loBitsSet: mantissaBits);
974 APInt zeroVal = APInt::getZero(numBits: totalBits);
975 Value expMask = createAPIntConst(loc, type: intTy, value: expMaskVal, rewriter);
976 Value clearMantissaMask =
977 createAPIntConst(loc, type: intTy, value: clearMantissaMaskVal, rewriter);
978 Value zero = createAPIntConst(loc, type: intTy, value: zeroVal, rewriter);
979
980 // expField == 0
981 Value expField = arith::AndIOp::create(builder&: b, lhs: bits, rhs: expMask);
982 Value expIsZero =
983 arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::eq, lhs: expField, rhs: zero);
984
985 // Clear mantissa bits: when exp == 0, this produces pos/neg 0.0.
986 Value cleared = arith::AndIOp::create(builder&: b, lhs: bits, rhs: clearMantissaMask);
987 Value resultBits = arith::SelectOp::create(builder&: b, condition: expIsZero, true_value: cleared, false_value: bits);
988 Value result = arith::BitcastOp::create(builder&: b, out: operandTy, in: resultBits);
989
990 rewriter.replaceOp(op, newValues: result);
991 return success();
992 }
993};
994
995struct ArithExpandOpsPass
996 : public arith::impl::ArithExpandOpsPassBase<ArithExpandOpsPass> {
997 using ArithExpandOpsPassBase::ArithExpandOpsPassBase;
998
999 void runOnOperation() override {
1000 RewritePatternSet patterns(&getContext());
1001 ConversionTarget target(getContext());
1002
1003 arith::populateCeilFloorDivExpandOpsPatterns(patterns);
1004 arith::populateExpandScalingExtTruncPatterns(patterns);
1005
1006 target.addLegalDialect<arith::ArithDialect>();
1007 target.addLegalDialect<vector::VectorDialect>();
1008
1009 // clang-format off
1010 target.addIllegalOp<
1011 arith::CeilDivSIOp,
1012 arith::CeilDivUIOp,
1013 arith::FloorDivSIOp,
1014 arith::ScalingExtFOp,
1015 arith::ScalingTruncFOp
1016 >();
1017 // clang-format on
1018
1019 // The min/max ops also have a direct arith-to-llvm lowering to the
1020 // `llvm.intr.maximum`/`minimum`/... (and smax/umax/...) intrinsics, which
1021 // are a single hardware instruction on many targets. Only expand them into
1022 // cmpf/cmpi + select when requested, so pipelines that run arith-to-llvm
1023 // can keep the intrinsic lowering. The float and integer ops are gated
1024 // separately.
1025 if (includeMinMaxF) {
1026 arith::populateExpandMinMaxFPatterns(patterns);
1027 // clang-format off
1028 target.addIllegalOp<
1029 arith::MaximumFOp,
1030 arith::MinimumFOp,
1031 arith::MaxNumFOp,
1032 arith::MinNumFOp
1033 >();
1034 // clang-format on
1035 }
1036 if (includeMinMaxI) {
1037 arith::populateExpandMinMaxIPatterns(patterns);
1038 // clang-format off
1039 target.addIllegalOp<
1040 arith::MaxSIOp,
1041 arith::MaxUIOp,
1042 arith::MinSIOp,
1043 arith::MinUIOp
1044 >();
1045 // clang-format on
1046 }
1047
1048 if (includeBf16)
1049 arith::populateExpandBFloat16Patterns(patterns);
1050 if (includeF8E8M0)
1051 arith::populateExpandF8E8M0Patterns(patterns);
1052 if (includeF4E2M1)
1053 arith::populateExpandF4E2M1Patterns(patterns);
1054 if (includeF8E5M2)
1055 arith::populateExpandF8E5M2Patterns(patterns);
1056 if (includeF8E4M3FN)
1057 arith::populateExpandF8E4M3FNPatterns(patterns);
1058 if (includeFlushDenormals) {
1059 arith::populateExpandFlushDenormalsPatterns(patterns);
1060 // Only IEEE-like floating-point types are expanded by the pattern;
1061 // leave `arith.flush_denormals` on other types alone.
1062 target.addDynamicallyLegalOp<arith::FlushDenormalsOp>(
1063 callback: [](arith::FlushDenormalsOp op) {
1064 auto floatTy =
1065 dyn_cast<FloatType>(Val: getElementTypeOrSelf(type: op.getType()));
1066 if (!floatTy)
1067 return true;
1068 return !llvm::APFloatBase::isIEEELikeFP(
1069 floatTy.getFloatSemantics());
1070 });
1071 }
1072
1073 target.addDynamicallyLegalOp<arith::ExtFOp>(callback: [=](arith::ExtFOp op) {
1074 Type inETy = getElementTypeOrSelf(type: op.getOperand().getType());
1075 Type outETy = getElementTypeOrSelf(type: op.getType());
1076 bool legalTypes = true;
1077 if (includeBf16)
1078 legalTypes &= !(inETy.isBF16() && outETy.isF32());
1079 if (includeF8E8M0)
1080 legalTypes &= !llvm::isa<Float8E8M0FNUType>(Val: inETy);
1081 if (includeF4E2M1)
1082 legalTypes &= !llvm::isa<Float4E2M1FNType>(Val: inETy);
1083 if (includeF8E5M2)
1084 legalTypes &= !llvm::isa<Float8E5M2Type>(Val: inETy);
1085 if (includeF8E4M3FN)
1086 legalTypes &= !llvm::isa<Float8E4M3FNType>(Val: inETy);
1087 return legalTypes;
1088 });
1089
1090 target.addDynamicallyLegalOp<arith::TruncFOp>(callback: [=](arith::TruncFOp op) {
1091 Type inETy = getElementTypeOrSelf(type: op.getOperand().getType());
1092 Type outETy = getElementTypeOrSelf(type: op.getType());
1093 bool legalTypes = true;
1094 if (includeBf16)
1095 legalTypes &= !(inETy.isF32() && outETy.isBF16());
1096 if (includeF8E8M0)
1097 legalTypes &= !(llvm::isa<Float8E8M0FNUType>(Val: outETy));
1098 if (includeF4E2M1)
1099 legalTypes &= !llvm::isa<Float4E2M1FNType>(Val: outETy);
1100 if (includeF8E5M2)
1101 legalTypes &= !llvm::isa<Float8E5M2Type>(Val: outETy);
1102 if (includeF8E4M3FN)
1103 legalTypes &= !llvm::isa<Float8E4M3FNType>(Val: outETy);
1104 return legalTypes;
1105 });
1106
1107 // clang-format on
1108 if (failed(Result: applyPartialConversion(op: getOperation(), target,
1109 patterns: std::move(patterns))))
1110 signalPassFailure();
1111 }
1112};
1113
1114} // namespace
1115
1116void mlir::arith::populateExpandBFloat16Patterns(RewritePatternSet &patterns) {
1117 patterns.add<BFloat16ExtFOpConverter, BFloat16TruncFOpConverter>(
1118 arg: patterns.getContext());
1119}
1120
1121void mlir::arith::populateExpandF4E2M1Patterns(RewritePatternSet &patterns) {
1122 patterns.add<F4E2M1ExtFOpConverter, F4E2M1TruncFOpConverter>(
1123 arg: patterns.getContext());
1124}
1125
1126void mlir::arith::populateExpandF8E5M2Patterns(RewritePatternSet &patterns) {
1127 patterns.add<F8E5M2ExtFOpConverter, F8E5M2TruncFOpConverter>(
1128 arg: patterns.getContext());
1129}
1130
1131void mlir::arith::populateExpandF8E4M3FNPatterns(RewritePatternSet &patterns) {
1132 patterns.add<F8E4M3FNExtFOpConverter, F8E4M3FNTruncFOpConverter>(
1133 arg: patterns.getContext());
1134}
1135
1136void mlir::arith::populateExpandF8E8M0Patterns(RewritePatternSet &patterns) {
1137 patterns.add<F8E8M0ExtFOpConverter, F8E8M0TruncFOpConverter>(
1138 arg: patterns.getContext());
1139}
1140
1141void mlir::arith::populateExpandScalingExtTruncPatterns(
1142 RewritePatternSet &patterns) {
1143 patterns.add<ScalingExtFOpConverter, ScalingTruncFOpConverter>(
1144 arg: patterns.getContext());
1145}
1146
1147void mlir::arith::populateExpandFlushDenormalsPatterns(
1148 RewritePatternSet &patterns) {
1149 patterns.add<FlushDenormalsOpConverter>(arg: patterns.getContext());
1150}
1151
1152void mlir::arith::populateExpandMinMaxFPatterns(RewritePatternSet &patterns) {
1153 // clang-format off
1154 patterns.add<
1155 MaximumMinimumFOpConverter<MaximumFOp, arith::CmpFPredicate::UGT>,
1156 MaximumMinimumFOpConverter<MinimumFOp, arith::CmpFPredicate::ULT>,
1157 MaxNumMinNumFOpConverter<MaxNumFOp, arith::CmpFPredicate::UGT>,
1158 MaxNumMinNumFOpConverter<MinNumFOp, arith::CmpFPredicate::ULT>
1159 >(arg: patterns.getContext());
1160 // clang-format on
1161}
1162
1163void mlir::arith::populateExpandMinMaxIPatterns(RewritePatternSet &patterns) {
1164 // clang-format off
1165 patterns.add<
1166 MaxMinIOpConverter<MaxSIOp, arith::CmpIPredicate::sgt>,
1167 MaxMinIOpConverter<MaxUIOp, arith::CmpIPredicate::ugt>,
1168 MaxMinIOpConverter<MinSIOp, arith::CmpIPredicate::slt>,
1169 MaxMinIOpConverter<MinUIOp, arith::CmpIPredicate::ult>
1170 >(arg: patterns.getContext());
1171 // clang-format on
1172}
1173
1174void mlir::arith::populateExpandMinMaxPatterns(RewritePatternSet &patterns) {
1175 populateExpandMinMaxFPatterns(patterns);
1176 populateExpandMinMaxIPatterns(patterns);
1177}
1178
1179void mlir::arith::populateArithExpandOpsPatterns(RewritePatternSet &patterns) {
1180 populateCeilFloorDivExpandOpsPatterns(patterns);
1181 populateExpandScalingExtTruncPatterns(patterns);
1182 populateExpandMinMaxPatterns(patterns);
1183}
1184