| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Helper for Fixed point code generation. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "CIRGenBuilder.h" |
| 14 | |
| 15 | #include "mlir/IR/Value.h" |
| 16 | |
| 17 | namespace clang::CIRGen { |
| 18 | // A CIR-specific generating version of the llvm::FixedPointBuilder. |
| 19 | struct CIRGenFixedPointBuilder { |
| 20 | CIRGenFixedPointBuilder(CIRGenBuilderTy &builder, mlir::Location loc) |
| 21 | : builder(builder), loc(loc) {} |
| 22 | |
| 23 | mlir::Value createFixedToFloating(mlir::Value src, |
| 24 | const llvm::FixedPointSemantics &srcSema, |
| 25 | mlir::Type dstTy) { |
| 26 | mlir::Type opTy = getAccommodatingFloatType(ty: dstTy, sema: srcSema); |
| 27 | // Convert the raw fixed-point value directly to floating point. If the |
| 28 | // value is too large to fit, it will be rounded, not truncated. |
| 29 | mlir::Value result = |
| 30 | builder.createCast(loc, kind: cir::CastKind::int_to_float, src, newTy: opTy); |
| 31 | // Rescale the integral-in-floating point by the scaling factor. This is |
| 32 | // lossless, except for overflow to infinity which is unlikely. |
| 33 | const llvm::fltSemantics &opSemantics = |
| 34 | mlir::cast<cir::FPTypeInterface>(Val&: opTy).getFloatSemantics(); |
| 35 | llvm::APFloat scaleVal( |
| 36 | std::pow(x: 2.0, y: -static_cast<int>(srcSema.getScale()))); |
| 37 | bool losesInfo; |
| 38 | scaleVal.convert(ToSemantics: opSemantics, RM: llvm::APFloat::rmNearestTiesToEven, |
| 39 | losesInfo: &losesInfo); |
| 40 | (void)losesInfo; |
| 41 | |
| 42 | cir::ConstantOp fpConst = builder.getConstFP(loc, t: opTy, fpVal: scaleVal); |
| 43 | result = builder.createFMul(loc, lhs: result, rhs: fpConst); |
| 44 | |
| 45 | if (opTy != dstTy) |
| 46 | result = builder.createFloatingCast(v: result, destType: dstTy); |
| 47 | return result; |
| 48 | } |
| 49 | |
| 50 | mlir::Value createFloatingToFixed(mlir::Value src, |
| 51 | const llvm::FixedPointSemantics &dstSema) { |
| 52 | mlir::Value result = src; |
| 53 | mlir::Type opTy = getAccommodatingFloatType(ty: src.getType(), sema: dstSema); |
| 54 | |
| 55 | if (opTy != src.getType()) |
| 56 | result = builder.createFloatingCast(v: result, destType: opTy); |
| 57 | |
| 58 | // Rescale the floating point value so that its significant bits (for the |
| 59 | // purposes of the conversion) are in the integral range. |
| 60 | const llvm::fltSemantics &opSemantics = |
| 61 | mlir::cast<cir::FPTypeInterface>(Val&: opTy).getFloatSemantics(); |
| 62 | llvm::APFloat scaleVal(std::pow(x: 2.0, y: dstSema.getScale())); |
| 63 | bool losesInfo; |
| 64 | scaleVal.convert(ToSemantics: opSemantics, RM: llvm::APFloat::rmNearestTiesToEven, |
| 65 | losesInfo: &losesInfo); |
| 66 | (void)losesInfo; |
| 67 | |
| 68 | cir::ConstantOp fpConst = builder.getConstFP(loc, t: opTy, fpVal: scaleVal); |
| 69 | result = builder.createFMul(loc, lhs: result, rhs: fpConst); |
| 70 | |
| 71 | cir::IntType opIntTy = getOpIntType(sema: dstSema); |
| 72 | if (dstSema.isSaturated()) { |
| 73 | result = builder.createFPToIntSat(loc, src: result, dstTy: opIntTy); |
| 74 | if (dstSema.hasUnsignedPadding()) |
| 75 | result = clampNegativeToZero(value: result); |
| 76 | } else { |
| 77 | result = |
| 78 | builder.createCast(loc, kind: cir::CastKind::float_to_int, src: result, newTy: opIntTy); |
| 79 | } |
| 80 | |
| 81 | return builder.createIntCast(src: result, newTy: getIntType(sema: dstSema)); |
| 82 | } |
| 83 | |
| 84 | mlir::Value createFixedToInteger(mlir::Value src, |
| 85 | const llvm::FixedPointSemantics &srcSema, |
| 86 | unsigned dstWidth, bool dstIsSigned) { |
| 87 | return convert( |
| 88 | src, srcSema, |
| 89 | dstSema: llvm::FixedPointSemantics::GetIntegerSemantics(Width: dstWidth, IsSigned: dstIsSigned), |
| 90 | /*dstIsInteger=*/dstIsInteger: true); |
| 91 | } |
| 92 | |
| 93 | mlir::Value createIntegerToFixed(mlir::Value src, unsigned srcIsSigned, |
| 94 | const llvm::FixedPointSemantics &dstSema) { |
| 95 | unsigned srcWidth; |
| 96 | if (mlir::isa<cir::BoolType>(Val: src.getType())) { |
| 97 | assert(!srcIsSigned); |
| 98 | srcWidth = 1; |
| 99 | src = builder.createBoolToInt( |
| 100 | src, newTy: cir::IntType::get(context: builder.getContext(), width: 1, /*isSigned=*/is_signed: false)); |
| 101 | } else { |
| 102 | srcWidth = mlir::cast<cir::IntType>(Val: src.getType()).getWidth(); |
| 103 | } |
| 104 | return convert( |
| 105 | src, |
| 106 | srcSema: llvm::FixedPointSemantics::GetIntegerSemantics(Width: srcWidth, IsSigned: srcIsSigned), |
| 107 | dstSema, /*dstIsInteger=*/dstIsInteger: false); |
| 108 | } |
| 109 | |
| 110 | mlir::Value createFixedToFixed(mlir::Value src, |
| 111 | const llvm::FixedPointSemantics &srcSema, |
| 112 | const llvm::FixedPointSemantics &dstSema) { |
| 113 | return convert(src, srcSema, dstSema, /*dstIsInteger=*/dstIsInteger: false); |
| 114 | } |
| 115 | |
| 116 | mlir::Value createAdd(mlir::Value lhs, |
| 117 | const llvm::FixedPointSemantics &lhsSema, |
| 118 | mlir::Value rhs, |
| 119 | const llvm::FixedPointSemantics &rhsSema) { |
| 120 | auto commonSema = getCommonBinopSemantic(lhsSema, rhsSema); |
| 121 | |
| 122 | mlir::Value wideLhs = createFixedToFixed(src: lhs, srcSema: lhsSema, dstSema: commonSema); |
| 123 | mlir::Value wideRhs = createFixedToFixed(src: rhs, srcSema: rhsSema, dstSema: commonSema); |
| 124 | |
| 125 | mlir::Value result = |
| 126 | commonSema.isSaturated() |
| 127 | ? createSaturatingAddSub(lhs: wideLhs, rhs: wideRhs, sema: commonSema, |
| 128 | /*isSub=*/isSub: false) |
| 129 | : builder.createAdd(loc, lhs: wideLhs, rhs: wideRhs); |
| 130 | |
| 131 | return createFixedToFixed(src: result, srcSema: commonSema, |
| 132 | dstSema: lhsSema.getCommonSemantics(Other: rhsSema)); |
| 133 | } |
| 134 | |
| 135 | mlir::Value createSub(mlir::Value lhs, |
| 136 | const llvm::FixedPointSemantics &lhsSema, |
| 137 | mlir::Value rhs, |
| 138 | const llvm::FixedPointSemantics &rhsSema) { |
| 139 | auto commonSema = getCommonBinopSemantic(lhsSema, rhsSema); |
| 140 | |
| 141 | mlir::Value wideLhs = createFixedToFixed(src: lhs, srcSema: lhsSema, dstSema: commonSema); |
| 142 | mlir::Value wideRhs = createFixedToFixed(src: rhs, srcSema: rhsSema, dstSema: commonSema); |
| 143 | |
| 144 | mlir::Value result = |
| 145 | commonSema.isSaturated() |
| 146 | ? createSaturatingAddSub(lhs: wideLhs, rhs: wideRhs, sema: commonSema, |
| 147 | /*isSub=*/isSub: true) |
| 148 | : builder.createSub(loc, lhs: wideLhs, rhs: wideRhs); |
| 149 | |
| 150 | return createFixedToFixed(src: result, srcSema: commonSema, |
| 151 | dstSema: lhsSema.getCommonSemantics(Other: rhsSema)); |
| 152 | } |
| 153 | |
| 154 | /// Negates \p value as 0 - value, so that saturating types saturate. |
| 155 | mlir::Value createNeg(mlir::Value value, |
| 156 | const llvm::FixedPointSemantics &sema) { |
| 157 | return createSub(lhs: builder.getNullValue(ty: value.getType(), loc), lhsSema: sema, rhs: value, |
| 158 | rhsSema: sema); |
| 159 | } |
| 160 | |
| 161 | mlir::Value createMul(mlir::Value lhs, |
| 162 | const llvm::FixedPointSemantics &lhsSema, |
| 163 | mlir::Value rhs, |
| 164 | const llvm::FixedPointSemantics &rhsSema) { |
| 165 | auto commonSema = getCommonBinopSemantic(lhsSema, rhsSema); |
| 166 | bool useSigned = commonSema.isSigned() || commonSema.hasUnsignedPadding(); |
| 167 | |
| 168 | mlir::Value wideLhs = createFixedToFixed(src: lhs, srcSema: lhsSema, dstSema: commonSema); |
| 169 | mlir::Value wideRhs = createFixedToFixed(src: rhs, srcSema: rhsSema, dstSema: commonSema); |
| 170 | |
| 171 | llvm::SmallString<13> intrinId; |
| 172 | cir::ConstantOp scale; |
| 173 | |
| 174 | if (useSigned) { |
| 175 | intrinId = "smul.fix" ; |
| 176 | scale = builder.getSInt32(c: commonSema.getScale(), loc); |
| 177 | } else { |
| 178 | intrinId = "umul.fix" ; |
| 179 | scale = builder.getUInt32(c: commonSema.getScale(), loc); |
| 180 | } |
| 181 | |
| 182 | if (commonSema.isSaturated()) |
| 183 | intrinId += ".sat" ; |
| 184 | |
| 185 | mlir::Value result = |
| 186 | builder.emitIntrinsicCallOp(loc, str: intrinId, resTy: wideLhs.getType(), |
| 187 | op: mlir::ValueRange{wideLhs, wideRhs, scale}); |
| 188 | |
| 189 | return createFixedToFixed(src: result, srcSema: commonSema, |
| 190 | dstSema: lhsSema.getCommonSemantics(Other: rhsSema)); |
| 191 | } |
| 192 | |
| 193 | mlir::Value createDiv(mlir::Value lhs, |
| 194 | const llvm::FixedPointSemantics &lhsSema, |
| 195 | mlir::Value rhs, |
| 196 | const llvm::FixedPointSemantics &rhsSema) { |
| 197 | auto commonSema = getCommonBinopSemantic(lhsSema, rhsSema); |
| 198 | bool useSigned = commonSema.isSigned() || commonSema.hasUnsignedPadding(); |
| 199 | |
| 200 | mlir::Value wideLhs = createFixedToFixed(src: lhs, srcSema: lhsSema, dstSema: commonSema); |
| 201 | mlir::Value wideRhs = createFixedToFixed(src: rhs, srcSema: rhsSema, dstSema: commonSema); |
| 202 | |
| 203 | llvm::SmallString<13> intrinId; |
| 204 | cir::ConstantOp scale; |
| 205 | |
| 206 | if (useSigned) { |
| 207 | intrinId = "sdiv.fix" ; |
| 208 | scale = builder.getSInt32(c: commonSema.getScale(), loc); |
| 209 | } else { |
| 210 | intrinId = "udiv.fix" ; |
| 211 | scale = builder.getUInt32(c: commonSema.getScale(), loc); |
| 212 | } |
| 213 | |
| 214 | if (commonSema.isSaturated()) |
| 215 | intrinId += ".sat" ; |
| 216 | |
| 217 | mlir::Value result = |
| 218 | builder.emitIntrinsicCallOp(loc, str: intrinId, resTy: wideLhs.getType(), |
| 219 | op: mlir::ValueRange{wideLhs, wideRhs, scale}); |
| 220 | |
| 221 | return createFixedToFixed(src: result, srcSema: commonSema, |
| 222 | dstSema: lhsSema.getCommonSemantics(Other: rhsSema)); |
| 223 | } |
| 224 | |
| 225 | mlir::Value createCmp(mlir::Value lhs, |
| 226 | const llvm::FixedPointSemantics &lhsSema, |
| 227 | mlir::Value rhs, |
| 228 | const llvm::FixedPointSemantics &rhsSema, |
| 229 | cir::CmpOpKind kind) { |
| 230 | auto commonSema = getCommonBinopSemantic(lhsSema, rhsSema); |
| 231 | |
| 232 | mlir::Value wideLhs = createFixedToFixed(src: lhs, srcSema: lhsSema, dstSema: commonSema); |
| 233 | mlir::Value wideRhs = createFixedToFixed(src: rhs, srcSema: rhsSema, dstSema: commonSema); |
| 234 | |
| 235 | return builder.createCompare(loc, kind, lhs: wideLhs, rhs: wideRhs); |
| 236 | } |
| 237 | |
| 238 | mlir::Value createShl(mlir::Value lhs, |
| 239 | const llvm::FixedPointSemantics &lhsSema, |
| 240 | mlir::Value rhs) { |
| 241 | mlir::Value result; |
| 242 | if (lhsSema.isSaturated()) { |
| 243 | // We have to cast the RHS to the matching int type, but we have to do so |
| 244 | // through unsigned so we can ensure we get zext. |
| 245 | auto rhsIntTy = mlir::cast<cir::IntType>(Val: rhs.getType()); |
| 246 | auto rhsUnsignedTy = cir::IntType::get( |
| 247 | context: builder.getContext(), width: rhsIntTy.getWidth(), /*isSigned=*/is_signed: false); |
| 248 | |
| 249 | mlir::Value rhsUnsigned = |
| 250 | builder.createCast(kind: cir::CastKind::integral, src: rhs, newTy: rhsUnsignedTy); |
| 251 | mlir::Value rhsResized = builder.createCast(kind: cir::CastKind::integral, |
| 252 | src: rhsUnsigned, newTy: lhs.getType()); |
| 253 | |
| 254 | bool useSigned = lhsSema.isSigned() || lhsSema.hasUnsignedPadding(); |
| 255 | result = builder.emitIntrinsicCallOp( |
| 256 | loc, str: useSigned ? "sshl.sat" : "ushl.sat" , resTy: lhs.getType(), |
| 257 | op: mlir::ValueRange{lhs, rhsResized}); |
| 258 | } else { |
| 259 | result = builder.createShiftLeft(loc, lhs, rhs); |
| 260 | } |
| 261 | |
| 262 | return result; |
| 263 | } |
| 264 | |
| 265 | mlir::Value createShr(mlir::Value lhs, mlir::Value rhs) { |
| 266 | return builder.createShiftRight(loc, lhs, rhs); |
| 267 | } |
| 268 | |
| 269 | private: |
| 270 | mlir::Value convert(mlir::Value src, const llvm::FixedPointSemantics &srcSema, |
| 271 | const llvm::FixedPointSemantics &dstSema, |
| 272 | bool dstIsInteger) { |
| 273 | unsigned srcWidth = srcSema.getWidth(); |
| 274 | unsigned dstWidth = dstSema.getWidth(); |
| 275 | unsigned srcScale = srcSema.getScale(); |
| 276 | unsigned dstScale = dstSema.getScale(); |
| 277 | bool srcIsSigned = srcSema.isSigned(); |
| 278 | bool dstIsSigned = dstSema.isSigned(); |
| 279 | |
| 280 | mlir::Value result = src; |
| 281 | unsigned resultWidth = srcWidth; |
| 282 | |
| 283 | // Downscale. |
| 284 | if (dstScale < srcScale) { |
| 285 | // When converting to integers, we round towards zero. For negative |
| 286 | // numbers, right shifting rounds towards negative infinity. In this case, |
| 287 | // we can just round up before shifting. |
| 288 | if (dstIsInteger && srcIsSigned) { |
| 289 | mlir::Value zero = builder.getNullValue(ty: result.getType(), loc); |
| 290 | mlir::Value isNegative = |
| 291 | builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: result, rhs: zero); |
| 292 | mlir::Value lowBits = builder.getConstAPInt( |
| 293 | loc, typ: result.getType(), |
| 294 | val: llvm::APInt::getLowBitsSet(numBits: srcWidth, loBitsSet: srcScale)); |
| 295 | mlir::Value rounded = builder.createAdd(loc, lhs: result, rhs: lowBits); |
| 296 | result = builder.createSelect(loc, condition: isNegative, trueValue: rounded, falseValue: result); |
| 297 | } |
| 298 | result = builder.createShiftRight(loc, lhs: result, bits: srcScale - dstScale); |
| 299 | } |
| 300 | |
| 301 | cir::IntType dstIntTy = getIntType(sema: dstSema); |
| 302 | |
| 303 | if (!dstSema.isSaturated()) { |
| 304 | // Resize. |
| 305 | result = builder.createIntCast(src: result, newTy: dstIntTy); |
| 306 | // Upscale. |
| 307 | if (dstScale > srcScale) |
| 308 | result = builder.createShiftLeft(loc, lhs: result, bits: dstScale - srcScale); |
| 309 | } else { |
| 310 | // Adjust the number of fractional bits. |
| 311 | if (dstScale > srcScale) { |
| 312 | // Compare to DstWidth to prevent resizing twice. |
| 313 | resultWidth = std::max(a: srcWidth + dstScale - srcScale, b: dstWidth); |
| 314 | cir::IntType upscaledTy = |
| 315 | cir::IntType::get(context: builder.getContext(), width: resultWidth, is_signed: srcIsSigned); |
| 316 | result = builder.createIntCast(src: result, newTy: upscaledTy); |
| 317 | result = builder.createShiftLeft(loc, lhs: result, bits: dstScale - srcScale); |
| 318 | } |
| 319 | |
| 320 | // Handle saturation. |
| 321 | bool fewerIntBits = dstSema.getIntegralBits() < srcSema.getIntegralBits(); |
| 322 | if (fewerIntBits) { |
| 323 | mlir::Value max = builder.getConstAPInt( |
| 324 | loc, typ: result.getType(), |
| 325 | val: llvm::APFixedPoint::getMax(Sema: dstSema).getValue().extOrTrunc( |
| 326 | width: resultWidth)); |
| 327 | |
| 328 | mlir::Value tooHigh = |
| 329 | builder.createCompare(loc, kind: cir::CmpOpKind::gt, lhs: result, rhs: max); |
| 330 | result = builder.createSelect(loc, condition: tooHigh, trueValue: max, falseValue: result); |
| 331 | } |
| 332 | |
| 333 | // Cannot overflow min to dest type if src is unsigned since all fixed |
| 334 | // point types can cover the unsigned min of 0. |
| 335 | if (srcIsSigned && (fewerIntBits || !dstIsSigned)) { |
| 336 | mlir::Value min = builder.getConstAPInt( |
| 337 | loc, typ: result.getType(), |
| 338 | val: llvm::APFixedPoint::getMin(Sema: dstSema).getValue().extOrTrunc( |
| 339 | width: resultWidth)); |
| 340 | mlir::Value tooLow = |
| 341 | builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: result, rhs: min); |
| 342 | result = builder.createSelect(loc, condition: tooLow, trueValue: min, falseValue: result); |
| 343 | } |
| 344 | |
| 345 | // Unlike LLVM integers, CIR integers carry signedness, so the cast is |
| 346 | // needed even when only the signedness differs. |
| 347 | result = builder.createIntCast(src: result, newTy: dstIntTy); |
| 348 | } |
| 349 | return result; |
| 350 | } |
| 351 | |
| 352 | mlir::Type getAccommodatingFloatType(mlir::Type ty, |
| 353 | const llvm::FixedPointSemantics &sema) { |
| 354 | const llvm::fltSemantics *floatSema = |
| 355 | &mlir::cast<cir::FPTypeInterface>(Val&: ty).getFloatSemantics(); |
| 356 | while (!sema.fitsInFloatSemantics(FloatSema: *floatSema)) |
| 357 | floatSema = llvm::APFixedPoint::promoteFloatSemantics(S: floatSema); |
| 358 | cir::FPTypeInterface accommodating = |
| 359 | cir::getFloatingPointType(sem: *floatSema, ctx: builder.getContext()); |
| 360 | assert(accommodating && "no float type for semantics?" ); |
| 361 | return accommodating; |
| 362 | } |
| 363 | /// Get the common semantic for two semantics, with the added imposition that |
| 364 | /// saturated padded types retain the padding bit. |
| 365 | llvm::FixedPointSemantics |
| 366 | getCommonBinopSemantic(const llvm::FixedPointSemantics &lhsSema, |
| 367 | const llvm::FixedPointSemantics &rhsSema) { |
| 368 | auto c = lhsSema.getCommonSemantics(Other: rhsSema); |
| 369 | bool bothPadded = |
| 370 | lhsSema.hasUnsignedPadding() && rhsSema.hasUnsignedPadding(); |
| 371 | return llvm::FixedPointSemantics( |
| 372 | c.getWidth() + static_cast<unsigned>(bothPadded && c.isSaturated()), |
| 373 | c.getScale(), c.isSigned(), c.isSaturated(), bothPadded); |
| 374 | } |
| 375 | |
| 376 | /// The CIR integer type that holds values of \p sema. |
| 377 | cir::IntType getIntType(const llvm::FixedPointSemantics &sema) { |
| 378 | return cir::IntType::get(context: builder.getContext(), width: sema.getWidth(), |
| 379 | is_signed: sema.isSigned()); |
| 380 | } |
| 381 | |
| 382 | /// Padded unsigned semantics compute as signed, as llvm::FixedPointBuilder |
| 383 | /// does: the padding bit keeps values at or below the signed maximum. CIR |
| 384 | /// integers carry signedness, so operands are cast to this type and back. |
| 385 | cir::IntType getOpIntType(const llvm::FixedPointSemantics &sema) { |
| 386 | return cir::IntType::get(context: builder.getContext(), width: sema.getWidth(), |
| 387 | is_signed: sema.isSigned() || sema.hasUnsignedPadding()); |
| 388 | } |
| 389 | |
| 390 | /// A padded unsigned result computed as signed may have gone negative. |
| 391 | mlir::Value clampNegativeToZero(mlir::Value value) { |
| 392 | mlir::Value zero = builder.getNullValue(ty: value.getType(), loc); |
| 393 | mlir::Value isNeg = |
| 394 | builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: value, rhs: zero); |
| 395 | return builder.createSelect(loc, condition: isNeg, trueValue: zero, falseValue: value); |
| 396 | } |
| 397 | |
| 398 | mlir::Value createSaturatingAddSub(mlir::Value lhs, mlir::Value rhs, |
| 399 | const llvm::FixedPointSemantics &sema, |
| 400 | bool isSub) { |
| 401 | cir::IntType opTy = getOpIntType(sema); |
| 402 | mlir::Value opLhs = builder.createIntCast(src: lhs, newTy: opTy); |
| 403 | mlir::Value opRhs = builder.createIntCast(src: rhs, newTy: opTy); |
| 404 | mlir::Value result = isSub ? builder.createSubSat(loc, lhs: opLhs, rhs: opRhs) |
| 405 | : builder.createAddSat(loc, lhs: opLhs, rhs: opRhs); |
| 406 | if (isSub && sema.hasUnsignedPadding()) |
| 407 | result = clampNegativeToZero(value: result); |
| 408 | return builder.createIntCast(src: result, newTy: lhs.getType()); |
| 409 | } |
| 410 | |
| 411 | CIRGenBuilderTy &builder; |
| 412 | mlir::Location loc; |
| 413 | }; |
| 414 | } // namespace clang::CIRGen |
| 415 | |