1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Helper for Fixed point code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CIRGenBuilder.h"
14
15#include "mlir/IR/Value.h"
16
17namespace clang::CIRGen {
18// A CIR-specific generating version of the llvm::FixedPointBuilder.
19struct CIRGenFixedPointBuilder {
20 CIRGenFixedPointBuilder(CIRGenBuilderTy &builder, mlir::Location loc)
21 : builder(builder), loc(loc) {}
22
23 mlir::Value createFixedToFloating(mlir::Value src,
24 const llvm::FixedPointSemantics &srcSema,
25 mlir::Type dstTy) {
26 mlir::Type opTy = getAccommodatingFloatType(ty: dstTy, sema: srcSema);
27 // Convert the raw fixed-point value directly to floating point. If the
28 // value is too large to fit, it will be rounded, not truncated.
29 mlir::Value result =
30 builder.createCast(loc, kind: cir::CastKind::int_to_float, src, newTy: opTy);
31 // Rescale the integral-in-floating point by the scaling factor. This is
32 // lossless, except for overflow to infinity which is unlikely.
33 const llvm::fltSemantics &opSemantics =
34 mlir::cast<cir::FPTypeInterface>(Val&: opTy).getFloatSemantics();
35 llvm::APFloat scaleVal(
36 std::pow(x: 2.0, y: -static_cast<int>(srcSema.getScale())));
37 bool losesInfo;
38 scaleVal.convert(ToSemantics: opSemantics, RM: llvm::APFloat::rmNearestTiesToEven,
39 losesInfo: &losesInfo);
40 (void)losesInfo;
41
42 cir::ConstantOp fpConst = builder.getConstFP(loc, t: opTy, fpVal: scaleVal);
43 result = builder.createFMul(loc, lhs: result, rhs: fpConst);
44
45 if (opTy != dstTy)
46 result = builder.createFloatingCast(v: result, destType: dstTy);
47 return result;
48 }
49
50 mlir::Value createFloatingToFixed(mlir::Value src,
51 const llvm::FixedPointSemantics &dstSema) {
52 mlir::Value result = src;
53 mlir::Type opTy = getAccommodatingFloatType(ty: src.getType(), sema: dstSema);
54
55 if (opTy != src.getType())
56 result = builder.createFloatingCast(v: result, destType: opTy);
57
58 // Rescale the floating point value so that its significant bits (for the
59 // purposes of the conversion) are in the integral range.
60 const llvm::fltSemantics &opSemantics =
61 mlir::cast<cir::FPTypeInterface>(Val&: opTy).getFloatSemantics();
62 llvm::APFloat scaleVal(std::pow(x: 2.0, y: dstSema.getScale()));
63 bool losesInfo;
64 scaleVal.convert(ToSemantics: opSemantics, RM: llvm::APFloat::rmNearestTiesToEven,
65 losesInfo: &losesInfo);
66 (void)losesInfo;
67
68 cir::ConstantOp fpConst = builder.getConstFP(loc, t: opTy, fpVal: scaleVal);
69 result = builder.createFMul(loc, lhs: result, rhs: fpConst);
70
71 cir::IntType opIntTy = getOpIntType(sema: dstSema);
72 if (dstSema.isSaturated()) {
73 result = builder.createFPToIntSat(loc, src: result, dstTy: opIntTy);
74 if (dstSema.hasUnsignedPadding())
75 result = clampNegativeToZero(value: result);
76 } else {
77 result =
78 builder.createCast(loc, kind: cir::CastKind::float_to_int, src: result, newTy: opIntTy);
79 }
80
81 return builder.createIntCast(src: result, newTy: getIntType(sema: dstSema));
82 }
83
84 mlir::Value createFixedToInteger(mlir::Value src,
85 const llvm::FixedPointSemantics &srcSema,
86 unsigned dstWidth, bool dstIsSigned) {
87 return convert(
88 src, srcSema,
89 dstSema: llvm::FixedPointSemantics::GetIntegerSemantics(Width: dstWidth, IsSigned: dstIsSigned),
90 /*dstIsInteger=*/dstIsInteger: true);
91 }
92
93 mlir::Value createIntegerToFixed(mlir::Value src, unsigned srcIsSigned,
94 const llvm::FixedPointSemantics &dstSema) {
95 unsigned srcWidth;
96 if (mlir::isa<cir::BoolType>(Val: src.getType())) {
97 assert(!srcIsSigned);
98 srcWidth = 1;
99 src = builder.createBoolToInt(
100 src, newTy: cir::IntType::get(context: builder.getContext(), width: 1, /*isSigned=*/is_signed: false));
101 } else {
102 srcWidth = mlir::cast<cir::IntType>(Val: src.getType()).getWidth();
103 }
104 return convert(
105 src,
106 srcSema: llvm::FixedPointSemantics::GetIntegerSemantics(Width: srcWidth, IsSigned: srcIsSigned),
107 dstSema, /*dstIsInteger=*/dstIsInteger: false);
108 }
109
110 mlir::Value createFixedToFixed(mlir::Value src,
111 const llvm::FixedPointSemantics &srcSema,
112 const llvm::FixedPointSemantics &dstSema) {
113 return convert(src, srcSema, dstSema, /*dstIsInteger=*/dstIsInteger: false);
114 }
115
116 mlir::Value createAdd(mlir::Value lhs,
117 const llvm::FixedPointSemantics &lhsSema,
118 mlir::Value rhs,
119 const llvm::FixedPointSemantics &rhsSema) {
120 auto commonSema = getCommonBinopSemantic(lhsSema, rhsSema);
121
122 mlir::Value wideLhs = createFixedToFixed(src: lhs, srcSema: lhsSema, dstSema: commonSema);
123 mlir::Value wideRhs = createFixedToFixed(src: rhs, srcSema: rhsSema, dstSema: commonSema);
124
125 mlir::Value result =
126 commonSema.isSaturated()
127 ? createSaturatingAddSub(lhs: wideLhs, rhs: wideRhs, sema: commonSema,
128 /*isSub=*/isSub: false)
129 : builder.createAdd(loc, lhs: wideLhs, rhs: wideRhs);
130
131 return createFixedToFixed(src: result, srcSema: commonSema,
132 dstSema: lhsSema.getCommonSemantics(Other: rhsSema));
133 }
134
135 mlir::Value createSub(mlir::Value lhs,
136 const llvm::FixedPointSemantics &lhsSema,
137 mlir::Value rhs,
138 const llvm::FixedPointSemantics &rhsSema) {
139 auto commonSema = getCommonBinopSemantic(lhsSema, rhsSema);
140
141 mlir::Value wideLhs = createFixedToFixed(src: lhs, srcSema: lhsSema, dstSema: commonSema);
142 mlir::Value wideRhs = createFixedToFixed(src: rhs, srcSema: rhsSema, dstSema: commonSema);
143
144 mlir::Value result =
145 commonSema.isSaturated()
146 ? createSaturatingAddSub(lhs: wideLhs, rhs: wideRhs, sema: commonSema,
147 /*isSub=*/isSub: true)
148 : builder.createSub(loc, lhs: wideLhs, rhs: wideRhs);
149
150 return createFixedToFixed(src: result, srcSema: commonSema,
151 dstSema: lhsSema.getCommonSemantics(Other: rhsSema));
152 }
153
154 /// Negates \p value as 0 - value, so that saturating types saturate.
155 mlir::Value createNeg(mlir::Value value,
156 const llvm::FixedPointSemantics &sema) {
157 return createSub(lhs: builder.getNullValue(ty: value.getType(), loc), lhsSema: sema, rhs: value,
158 rhsSema: sema);
159 }
160
161 mlir::Value createMul(mlir::Value lhs,
162 const llvm::FixedPointSemantics &lhsSema,
163 mlir::Value rhs,
164 const llvm::FixedPointSemantics &rhsSema) {
165 auto commonSema = getCommonBinopSemantic(lhsSema, rhsSema);
166 bool useSigned = commonSema.isSigned() || commonSema.hasUnsignedPadding();
167
168 mlir::Value wideLhs = createFixedToFixed(src: lhs, srcSema: lhsSema, dstSema: commonSema);
169 mlir::Value wideRhs = createFixedToFixed(src: rhs, srcSema: rhsSema, dstSema: commonSema);
170
171 llvm::SmallString<13> intrinId;
172 cir::ConstantOp scale;
173
174 if (useSigned) {
175 intrinId = "smul.fix";
176 scale = builder.getSInt32(c: commonSema.getScale(), loc);
177 } else {
178 intrinId = "umul.fix";
179 scale = builder.getUInt32(c: commonSema.getScale(), loc);
180 }
181
182 if (commonSema.isSaturated())
183 intrinId += ".sat";
184
185 mlir::Value result =
186 builder.emitIntrinsicCallOp(loc, str: intrinId, resTy: wideLhs.getType(),
187 op: mlir::ValueRange{wideLhs, wideRhs, scale});
188
189 return createFixedToFixed(src: result, srcSema: commonSema,
190 dstSema: lhsSema.getCommonSemantics(Other: rhsSema));
191 }
192
193 mlir::Value createDiv(mlir::Value lhs,
194 const llvm::FixedPointSemantics &lhsSema,
195 mlir::Value rhs,
196 const llvm::FixedPointSemantics &rhsSema) {
197 auto commonSema = getCommonBinopSemantic(lhsSema, rhsSema);
198 bool useSigned = commonSema.isSigned() || commonSema.hasUnsignedPadding();
199
200 mlir::Value wideLhs = createFixedToFixed(src: lhs, srcSema: lhsSema, dstSema: commonSema);
201 mlir::Value wideRhs = createFixedToFixed(src: rhs, srcSema: rhsSema, dstSema: commonSema);
202
203 llvm::SmallString<13> intrinId;
204 cir::ConstantOp scale;
205
206 if (useSigned) {
207 intrinId = "sdiv.fix";
208 scale = builder.getSInt32(c: commonSema.getScale(), loc);
209 } else {
210 intrinId = "udiv.fix";
211 scale = builder.getUInt32(c: commonSema.getScale(), loc);
212 }
213
214 if (commonSema.isSaturated())
215 intrinId += ".sat";
216
217 mlir::Value result =
218 builder.emitIntrinsicCallOp(loc, str: intrinId, resTy: wideLhs.getType(),
219 op: mlir::ValueRange{wideLhs, wideRhs, scale});
220
221 return createFixedToFixed(src: result, srcSema: commonSema,
222 dstSema: lhsSema.getCommonSemantics(Other: rhsSema));
223 }
224
225 mlir::Value createCmp(mlir::Value lhs,
226 const llvm::FixedPointSemantics &lhsSema,
227 mlir::Value rhs,
228 const llvm::FixedPointSemantics &rhsSema,
229 cir::CmpOpKind kind) {
230 auto commonSema = getCommonBinopSemantic(lhsSema, rhsSema);
231
232 mlir::Value wideLhs = createFixedToFixed(src: lhs, srcSema: lhsSema, dstSema: commonSema);
233 mlir::Value wideRhs = createFixedToFixed(src: rhs, srcSema: rhsSema, dstSema: commonSema);
234
235 return builder.createCompare(loc, kind, lhs: wideLhs, rhs: wideRhs);
236 }
237
238 mlir::Value createShl(mlir::Value lhs,
239 const llvm::FixedPointSemantics &lhsSema,
240 mlir::Value rhs) {
241 mlir::Value result;
242 if (lhsSema.isSaturated()) {
243 // We have to cast the RHS to the matching int type, but we have to do so
244 // through unsigned so we can ensure we get zext.
245 auto rhsIntTy = mlir::cast<cir::IntType>(Val: rhs.getType());
246 auto rhsUnsignedTy = cir::IntType::get(
247 context: builder.getContext(), width: rhsIntTy.getWidth(), /*isSigned=*/is_signed: false);
248
249 mlir::Value rhsUnsigned =
250 builder.createCast(kind: cir::CastKind::integral, src: rhs, newTy: rhsUnsignedTy);
251 mlir::Value rhsResized = builder.createCast(kind: cir::CastKind::integral,
252 src: rhsUnsigned, newTy: lhs.getType());
253
254 bool useSigned = lhsSema.isSigned() || lhsSema.hasUnsignedPadding();
255 result = builder.emitIntrinsicCallOp(
256 loc, str: useSigned ? "sshl.sat" : "ushl.sat", resTy: lhs.getType(),
257 op: mlir::ValueRange{lhs, rhsResized});
258 } else {
259 result = builder.createShiftLeft(loc, lhs, rhs);
260 }
261
262 return result;
263 }
264
265 mlir::Value createShr(mlir::Value lhs, mlir::Value rhs) {
266 return builder.createShiftRight(loc, lhs, rhs);
267 }
268
269private:
270 mlir::Value convert(mlir::Value src, const llvm::FixedPointSemantics &srcSema,
271 const llvm::FixedPointSemantics &dstSema,
272 bool dstIsInteger) {
273 unsigned srcWidth = srcSema.getWidth();
274 unsigned dstWidth = dstSema.getWidth();
275 unsigned srcScale = srcSema.getScale();
276 unsigned dstScale = dstSema.getScale();
277 bool srcIsSigned = srcSema.isSigned();
278 bool dstIsSigned = dstSema.isSigned();
279
280 mlir::Value result = src;
281 unsigned resultWidth = srcWidth;
282
283 // Downscale.
284 if (dstScale < srcScale) {
285 // When converting to integers, we round towards zero. For negative
286 // numbers, right shifting rounds towards negative infinity. In this case,
287 // we can just round up before shifting.
288 if (dstIsInteger && srcIsSigned) {
289 mlir::Value zero = builder.getNullValue(ty: result.getType(), loc);
290 mlir::Value isNegative =
291 builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: result, rhs: zero);
292 mlir::Value lowBits = builder.getConstAPInt(
293 loc, typ: result.getType(),
294 val: llvm::APInt::getLowBitsSet(numBits: srcWidth, loBitsSet: srcScale));
295 mlir::Value rounded = builder.createAdd(loc, lhs: result, rhs: lowBits);
296 result = builder.createSelect(loc, condition: isNegative, trueValue: rounded, falseValue: result);
297 }
298 result = builder.createShiftRight(loc, lhs: result, bits: srcScale - dstScale);
299 }
300
301 cir::IntType dstIntTy = getIntType(sema: dstSema);
302
303 if (!dstSema.isSaturated()) {
304 // Resize.
305 result = builder.createIntCast(src: result, newTy: dstIntTy);
306 // Upscale.
307 if (dstScale > srcScale)
308 result = builder.createShiftLeft(loc, lhs: result, bits: dstScale - srcScale);
309 } else {
310 // Adjust the number of fractional bits.
311 if (dstScale > srcScale) {
312 // Compare to DstWidth to prevent resizing twice.
313 resultWidth = std::max(a: srcWidth + dstScale - srcScale, b: dstWidth);
314 cir::IntType upscaledTy =
315 cir::IntType::get(context: builder.getContext(), width: resultWidth, is_signed: srcIsSigned);
316 result = builder.createIntCast(src: result, newTy: upscaledTy);
317 result = builder.createShiftLeft(loc, lhs: result, bits: dstScale - srcScale);
318 }
319
320 // Handle saturation.
321 bool fewerIntBits = dstSema.getIntegralBits() < srcSema.getIntegralBits();
322 if (fewerIntBits) {
323 mlir::Value max = builder.getConstAPInt(
324 loc, typ: result.getType(),
325 val: llvm::APFixedPoint::getMax(Sema: dstSema).getValue().extOrTrunc(
326 width: resultWidth));
327
328 mlir::Value tooHigh =
329 builder.createCompare(loc, kind: cir::CmpOpKind::gt, lhs: result, rhs: max);
330 result = builder.createSelect(loc, condition: tooHigh, trueValue: max, falseValue: result);
331 }
332
333 // Cannot overflow min to dest type if src is unsigned since all fixed
334 // point types can cover the unsigned min of 0.
335 if (srcIsSigned && (fewerIntBits || !dstIsSigned)) {
336 mlir::Value min = builder.getConstAPInt(
337 loc, typ: result.getType(),
338 val: llvm::APFixedPoint::getMin(Sema: dstSema).getValue().extOrTrunc(
339 width: resultWidth));
340 mlir::Value tooLow =
341 builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: result, rhs: min);
342 result = builder.createSelect(loc, condition: tooLow, trueValue: min, falseValue: result);
343 }
344
345 // Unlike LLVM integers, CIR integers carry signedness, so the cast is
346 // needed even when only the signedness differs.
347 result = builder.createIntCast(src: result, newTy: dstIntTy);
348 }
349 return result;
350 }
351
352 mlir::Type getAccommodatingFloatType(mlir::Type ty,
353 const llvm::FixedPointSemantics &sema) {
354 const llvm::fltSemantics *floatSema =
355 &mlir::cast<cir::FPTypeInterface>(Val&: ty).getFloatSemantics();
356 while (!sema.fitsInFloatSemantics(FloatSema: *floatSema))
357 floatSema = llvm::APFixedPoint::promoteFloatSemantics(S: floatSema);
358 cir::FPTypeInterface accommodating =
359 cir::getFloatingPointType(sem: *floatSema, ctx: builder.getContext());
360 assert(accommodating && "no float type for semantics?");
361 return accommodating;
362 }
363 /// Get the common semantic for two semantics, with the added imposition that
364 /// saturated padded types retain the padding bit.
365 llvm::FixedPointSemantics
366 getCommonBinopSemantic(const llvm::FixedPointSemantics &lhsSema,
367 const llvm::FixedPointSemantics &rhsSema) {
368 auto c = lhsSema.getCommonSemantics(Other: rhsSema);
369 bool bothPadded =
370 lhsSema.hasUnsignedPadding() && rhsSema.hasUnsignedPadding();
371 return llvm::FixedPointSemantics(
372 c.getWidth() + static_cast<unsigned>(bothPadded && c.isSaturated()),
373 c.getScale(), c.isSigned(), c.isSaturated(), bothPadded);
374 }
375
376 /// The CIR integer type that holds values of \p sema.
377 cir::IntType getIntType(const llvm::FixedPointSemantics &sema) {
378 return cir::IntType::get(context: builder.getContext(), width: sema.getWidth(),
379 is_signed: sema.isSigned());
380 }
381
382 /// Padded unsigned semantics compute as signed, as llvm::FixedPointBuilder
383 /// does: the padding bit keeps values at or below the signed maximum. CIR
384 /// integers carry signedness, so operands are cast to this type and back.
385 cir::IntType getOpIntType(const llvm::FixedPointSemantics &sema) {
386 return cir::IntType::get(context: builder.getContext(), width: sema.getWidth(),
387 is_signed: sema.isSigned() || sema.hasUnsignedPadding());
388 }
389
390 /// A padded unsigned result computed as signed may have gone negative.
391 mlir::Value clampNegativeToZero(mlir::Value value) {
392 mlir::Value zero = builder.getNullValue(ty: value.getType(), loc);
393 mlir::Value isNeg =
394 builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: value, rhs: zero);
395 return builder.createSelect(loc, condition: isNeg, trueValue: zero, falseValue: value);
396 }
397
398 mlir::Value createSaturatingAddSub(mlir::Value lhs, mlir::Value rhs,
399 const llvm::FixedPointSemantics &sema,
400 bool isSub) {
401 cir::IntType opTy = getOpIntType(sema);
402 mlir::Value opLhs = builder.createIntCast(src: lhs, newTy: opTy);
403 mlir::Value opRhs = builder.createIntCast(src: rhs, newTy: opTy);
404 mlir::Value result = isSub ? builder.createSubSat(loc, lhs: opLhs, rhs: opRhs)
405 : builder.createAddSat(loc, lhs: opLhs, rhs: opRhs);
406 if (isSub && sema.hasUnsignedPadding())
407 result = clampNegativeToZero(value: result);
408 return builder.createIntCast(src: result, newTy: lhs.getType());
409 }
410
411 CIRGenBuilderTy &builder;
412 mlir::Location loc;
413};
414} // namespace clang::CIRGen
415