1//===--- ExpandIRInsts.cpp - Expand IR instructions -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass expands certain instructions at the IR level.
9//
10// The following expansions are implemented:
11// - Expansion of ‘fptoui .. to’, ‘fptosi .. to’, ‘uitofp .. to’, ‘sitofp
12// .. to’ instructions with a bitwidth above a threshold. This is
13// useful for targets like x86_64 that cannot lower fp convertions
14// with more than 128 bits.
15//
16// - Expansion of ‘frem‘ for types MVT::f16, MVT::f32, and MVT::f64 for
17// targets which use "Expand" as the legalization action for the
18// corresponding type.
19//
20// - Expansion of ‘udiv‘, ‘sdiv‘, ‘urem‘, and ‘srem‘ instructions with
21// a bitwidth above a threshold into a call to auto-generated
22// functions. This is useful for targets like x86_64 that cannot
23// lower divisions with more than 128 bits or targets like x86_32 that
24// cannot lower divisions with more than 64 bits.
25//
26// Instructions with vector types are scalarized first if their scalar
27// types can be expanded. Scalable vector types are not supported.
28//===----------------------------------------------------------------------===//
29
30#include "llvm/CodeGen/ExpandIRInsts.h"
31#include "llvm/ADT/SmallVector.h"
32#include "llvm/Analysis/AssumptionCache.h"
33#include "llvm/Analysis/GlobalsModRef.h"
34#include "llvm/Analysis/SimplifyQuery.h"
35#include "llvm/Analysis/ValueTracking.h"
36#include "llvm/CodeGen/ISDOpcodes.h"
37#include "llvm/CodeGen/Passes.h"
38#include "llvm/CodeGen/RuntimeLibcallUtil.h"
39#include "llvm/CodeGen/TargetLowering.h"
40#include "llvm/CodeGen/TargetPassConfig.h"
41#include "llvm/CodeGen/TargetSubtargetInfo.h"
42#include "llvm/IR/IRBuilder.h"
43#include "llvm/IR/InstIterator.h"
44#include "llvm/IR/Instruction.h"
45#include "llvm/IR/Instructions.h"
46#include "llvm/IR/IntrinsicInst.h"
47#include "llvm/IR/MDBuilder.h"
48#include "llvm/IR/Module.h"
49#include "llvm/IR/PassManager.h"
50#include "llvm/IR/ProfDataUtils.h"
51#include "llvm/InitializePasses.h"
52#include "llvm/Pass.h"
53#include "llvm/Support/Casting.h"
54#include "llvm/Support/CommandLine.h"
55#include "llvm/Support/ErrorHandling.h"
56#include "llvm/Target/TargetMachine.h"
57#include "llvm/Transforms/Utils/BasicBlockUtils.h"
58#include "llvm/Transforms/Utils/IntegerDivision.h"
59#include <optional>
60
61#define DEBUG_TYPE "expand-ir-insts"
62
63using namespace llvm;
64
65namespace llvm {
66extern cl::opt<bool> ProfcheckDisableMetadataFixes;
67}
68
69static cl::opt<unsigned>
70 ExpandFpConvertBits("expand-fp-convert-bits", cl::Hidden,
71 cl::init(Val: IntegerType::MAX_INT_BITS),
72 cl::desc("fp convert instructions on integers with "
73 "more than <N> bits are expanded."));
74
75static cl::opt<unsigned>
76 ExpandDivRemBits("expand-div-rem-bits", cl::Hidden,
77 cl::init(Val: IntegerType::MAX_INT_BITS),
78 cl::desc("div and rem instructions on integers with "
79 "more than <N> bits are expanded."));
80
81static bool isConstantPowerOfTwo(Value *V, bool SignedOp) {
82 auto *C = dyn_cast<ConstantInt>(Val: V);
83 if (!C)
84 return false;
85
86 APInt Val = C->getValue();
87 if (SignedOp && Val.isNegative())
88 Val = -Val;
89 return Val.isPowerOf2();
90}
91
92static bool isSigned(unsigned Opcode) {
93 return Opcode == Instruction::SDiv || Opcode == Instruction::SRem;
94}
95
96/// For signed div/rem by a power of 2, compute the bias-adjusted dividend:
97/// Sign = ashr X, (BitWidth - 1) -- 0 or -1
98/// Bias = lshr Sign, (BitWidth - ShiftAmt) -- 0 or 2^ShiftAmt - 1
99/// Adjusted = add X, Bias
100/// The bias adds (2^ShiftAmt - 1) for negative X, correcting rounding towards
101/// zero (instead of towards -inf that a plain ashr would give).
102/// The lshr form is used instead of 'and' to avoid large immediate constants.
103static Value *addSignedBias(IRBuilder<> &Builder, Value *X, unsigned BitWidth,
104 unsigned ShiftAmt) {
105 assert(ShiftAmt > 0 && ShiftAmt < BitWidth &&
106 "ShiftAmt out of range; callers should handle ShiftAmt == 0");
107 Value *Sign = Builder.CreateAShr(LHS: X, RHS: BitWidth - 1, Name: "sign");
108 Value *Bias = Builder.CreateLShr(LHS: Sign, RHS: BitWidth - ShiftAmt, Name: "bias");
109 return Builder.CreateAdd(LHS: X, RHS: Bias, Name: "adjusted");
110}
111
112/// Expand division or remainder by a power-of-2 constant.
113/// Division (let C = log2(|divisor|)):
114/// udiv X, 2^C -> lshr X, C
115/// sdiv X, 2^C -> ashr (add X, Bias), C (Bias corrects rounding)
116/// sdiv exact X, 2^C -> ashr exact X, C (no bias needed)
117/// For negative power-of-2 divisors, the division result is negated.
118/// Remainder (let C = log2(|divisor|)):
119/// urem X, 2^C -> and X, (2^C - 1)
120/// srem X, 2^C -> sub X, (shl (ashr (add X, Bias), C), C)
121static void expandPow2DivRem(BinaryOperator *BO) {
122 LLVM_DEBUG(dbgs() << "Expanding instruction: " << *BO << '\n');
123
124 unsigned Opcode = BO->getOpcode();
125 bool IsDiv = (Opcode == Instruction::UDiv || Opcode == Instruction::SDiv);
126 bool IsSigned = isSigned(Opcode);
127 // isExact() is only valid for div.
128 bool IsExact = IsDiv && BO->isExact();
129
130 assert(isConstantPowerOfTwo(BO->getOperand(1), IsSigned) &&
131 "Expected power-of-2 constant divisor");
132
133 Value *X = BO->getOperand(i_nocapture: 0);
134 auto *C = cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1));
135 Type *Ty = BO->getType();
136 unsigned BitWidth = Ty->getIntegerBitWidth();
137
138 APInt DivisorVal = C->getValue();
139 bool IsNegativeDivisor = IsSigned && DivisorVal.isNegative();
140 // Use countr_zero() to get the shift amount directly from the bit pattern.
141 // This works correctly for both positive and negative powers of 2, including
142 // INT_MIN, without needing to negate the value first.
143 unsigned ShiftAmt = DivisorVal.countr_zero();
144
145 IRBuilder<> Builder(BO);
146 Value *Result;
147
148 if (ShiftAmt == 0) {
149 // Div by 1/-1: X / 1 = X, X / -1 = -X.
150 // Rem by 1/-1: always 0.
151 if (IsDiv)
152 Result = IsNegativeDivisor ? Builder.CreateNeg(V: X) : X;
153 else
154 Result = ConstantInt::get(Ty, V: 0);
155 } else if (IsSigned) {
156 // The signed expansion uses X multiple times (bias computation, shift,
157 // and sub for remainder). Freeze X to ensure consistent behavior if it is
158 // undef/poison. For exact division, no bias is needed and X is used only
159 // once, so freeze is unnecessary.
160 if (!IsExact && !isGuaranteedNotToBeUndefOrPoison(V: X))
161 X = Builder.CreateFreeze(V: X, Name: X->getName() + ".fr");
162 // For exact division, no bias is needed since there's no rounding.
163 Value *Dividend =
164 IsExact ? X : addSignedBias(Builder, X, BitWidth, ShiftAmt);
165 Value *Quotient = Builder.CreateAShr(
166 LHS: Dividend, RHS: ShiftAmt, Name: IsDiv && IsNegativeDivisor ? "pre.neg" : "shifted",
167 isExact: IsExact);
168 if (IsDiv) {
169 Result = IsNegativeDivisor ? Builder.CreateNeg(V: Quotient) : Quotient;
170 } else {
171 // Rem = X - (Quotient << ShiftAmt):
172 // clear lower ShiftAmt bits via round-trip shift, then subtract.
173 Value *Truncated = Builder.CreateShl(LHS: Quotient, RHS: ShiftAmt, Name: "truncated");
174 Result = Builder.CreateSub(LHS: X, RHS: Truncated);
175 }
176 } else {
177 if (IsDiv) {
178 Result = Builder.CreateLShr(LHS: X, RHS: ShiftAmt, Name: "", isExact: IsExact);
179 } else {
180 APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ShiftAmt);
181 Result = Builder.CreateAnd(LHS: X, RHS: ConstantInt::get(Ty, V: Mask));
182 }
183 }
184
185 BO->replaceAllUsesWith(V: Result);
186 if (Result != X)
187 if (auto *RI = dyn_cast<Instruction>(Val: Result))
188 RI->takeName(V: BO);
189 BO->dropAllReferences();
190 BO->eraseFromParent();
191}
192
193/// This class implements a precise expansion of the frem instruction.
194/// The generated code is based on the fmod implementation in the AMD device
195/// libs.
196namespace {
197class FRemExpander {
198 /// The IRBuilder to use for the expansion.
199 IRBuilder<> &B;
200
201 /// Floating point type of the return value and the arguments of the FRem
202 /// instructions that should be expanded.
203 Type *FremTy;
204
205 /// Floating point type to use for the computation. This may be
206 /// wider than the \p FremTy.
207 Type *ComputeFpTy;
208
209 /// Integer type used to hold the exponents returned by frexp.
210 Type *ExTy;
211
212 /// How many bits of the quotient to compute per iteration of the
213 /// algorithm, stored as a value of type \p ExTy.
214 Value *Bits;
215
216 /// Constant 1 of type \p ExTy.
217 Value *One;
218
219 /// The frem argument/return types that can be expanded by this class.
220 // TODO: The expansion could work for other floating point types
221 // as well, but this would require additional testing.
222 static constexpr std::array<MVT, 3> ExpandableTypes{MVT::f16, MVT::f32,
223 MVT::f64};
224
225public:
226 static bool canExpandType(Type *Ty) {
227 EVT VT = EVT::getEVT(Ty);
228 assert(VT.isSimple() && "Can expand only simple types");
229
230 return is_contained(Range: ExpandableTypes, Element: VT.getSimpleVT());
231 }
232
233 static bool shouldExpandFremType(const TargetLowering &TLI,
234 const LibcallLoweringInfo &Libcalls,
235 EVT VT) {
236 assert(!VT.isVector() && "Cannot handle vector type; must scalarize first");
237 switch (TLI.getOperationAction(Op: ISD::FREM, VT)) {
238 case TargetLowering::LegalizeAction::Expand:
239 return true;
240 case TargetLowering::LegalizeAction::LibCall:
241 // The target expects a libcall, but expand inline for the supported
242 // types when that libcall is unavailable.
243 return VT.isSimple() && is_contained(Range: ExpandableTypes, Element: VT.getSimpleVT()) &&
244 Libcalls.getLibcallImpl(Call: RTLIB::getREM(VT)) == RTLIB::Unsupported;
245 default:
246 return false;
247 }
248 }
249
250 static bool shouldExpandFremType(const TargetLowering &TLI,
251 const LibcallLoweringInfo &Libcalls,
252 Type *Ty) {
253 // Consider scalar type for simplicity. It seems unlikely that a
254 // vector type can be legalized without expansion if the scalar
255 // type cannot.
256 return shouldExpandFremType(TLI, Libcalls,
257 VT: EVT::getEVT(Ty: Ty->getScalarType()));
258 }
259
260 /// Return true if the pass should expand frem instructions of any type
261 /// for the target represented by \p TLI.
262 static bool shouldExpandAnyFremType(const TargetLowering &TLI,
263 const LibcallLoweringInfo &Libcalls) {
264 return any_of(Range: ExpandableTypes, P: [&](MVT V) {
265 return shouldExpandFremType(TLI, Libcalls, VT: EVT(V));
266 });
267 }
268
269 static FRemExpander create(IRBuilder<> &B, Type *Ty) {
270 assert(canExpandType(Ty) && "Expected supported floating point type");
271
272 // The type to use for the computation of the remainder. This may be
273 // wider than the input/result type which affects the ...
274 Type *ComputeTy = Ty;
275 // ... maximum number of iterations of the remainder computation loop
276 // to use. This value is for the case in which the computation
277 // uses the same input/result type.
278 unsigned MaxIter = 2;
279
280 if (Ty->isHalfTy()) {
281 // Use the wider type and less iterations.
282 ComputeTy = B.getFloatTy();
283 MaxIter = 1;
284 }
285
286 unsigned Precision = APFloat::semanticsPrecision(Ty->getFltSemantics());
287 return FRemExpander{B, Ty, Precision / MaxIter, ComputeTy};
288 }
289
290 /// Build the FRem expansion for the numerator \p X and the
291 /// denumerator \p Y. The type of X and Y must match \p FremTy. The
292 /// code will be generated at the insertion point of \p B and the
293 /// insertion point will be reset at exit.
294 Value *buildFRem(Value *X, Value *Y, std::optional<SimplifyQuery> &SQ) const;
295
296 /// Build an approximate FRem expansion for the numerator \p X and
297 /// the denumerator \p Y at the insertion point of builder \p B.
298 /// The type of X and Y must match \p FremTy.
299 Value *buildApproxFRem(Value *X, Value *Y) const;
300
301private:
302 FRemExpander(IRBuilder<> &B, Type *FremTy, unsigned Bits, Type *ComputeFpTy)
303 : B(B), FremTy(FremTy), ComputeFpTy(ComputeFpTy), ExTy(B.getInt32Ty()),
304 Bits(ConstantInt::get(Ty: ExTy, V: Bits)), One(ConstantInt::get(Ty: ExTy, V: 1)) {}
305
306 Value *createRcp(Value *V, const Twine &Name) const {
307 // Leave it to later optimizations to turn this into an rcp
308 // instruction if available.
309 return B.CreateFDiv(L: ConstantFP::get(Ty: ComputeFpTy, V: 1.0), R: V, Name);
310 }
311
312 // Helper function to build the UPDATE_AX code which is common to the
313 // loop body and the "final iteration".
314 Value *buildUpdateAx(Value *Ax, Value *Ay, Value *Ayinv) const {
315 // Build:
316 // float q = rint(ax * ayinv);
317 // ax = fma(-q, ay, ax);
318 // int clt = ax < 0.0f;
319 // float axp = ax + ay;
320 // ax = clt ? axp : ax;
321 Value *Q = B.CreateUnaryIntrinsic(ID: Intrinsic::rint, Op: B.CreateFMul(L: Ax, R: Ayinv),
322 FMFSource: {}, Name: "q");
323 Value *AxUpdate = B.CreateFMA(Factor1: B.CreateFNeg(V: Q), Factor2: Ay, Summand: Ax, FMFSource: {}, Name: "ax");
324 Value *Clt = B.CreateFCmp(P: CmpInst::FCMP_OLT, LHS: AxUpdate,
325 RHS: ConstantFP::getZero(Ty: ComputeFpTy), Name: "clt");
326 Value *Axp = B.CreateFAdd(L: AxUpdate, R: Ay, Name: "axp");
327 return B.CreateSelect(C: Clt, True: Axp, False: AxUpdate, Name: "ax");
328 }
329
330 /// Build code to extract the exponent and mantissa of \p Src.
331 /// Return the exponent minus one for use as a loop bound and
332 /// the mantissa taken to the given \p NewExp power.
333 std::pair<Value *, Value *> buildExpAndPower(Value *Src, Value *NewExp,
334 const Twine &ExName,
335 const Twine &PowName) const {
336 // Build:
337 // ExName = frexp_exp(Src) - 1;
338 // PowName = fldexp(frexp_mant(ExName), NewExp);
339 Type *Ty = Src->getType();
340 Type *ExTy = B.getInt32Ty();
341 Value *Frexp = B.CreateIntrinsic(ID: Intrinsic::frexp, OverloadTypes: {Ty, ExTy}, Args: Src);
342 Value *Mant = B.CreateExtractValue(Agg: Frexp, Idxs: {0});
343 Value *Exp = B.CreateExtractValue(Agg: Frexp, Idxs: {1});
344
345 Exp = B.CreateSub(LHS: Exp, RHS: One, Name: ExName);
346 Value *Pow = B.CreateLdexp(Src: Mant, Exp: NewExp, FMFSource: {}, Name: PowName);
347
348 return {Pow, Exp};
349 }
350
351 /// Build the main computation of the remainder for the case in which
352 /// Ax > Ay, where Ax = |X|, Ay = |Y|, and X is the numerator and Y the
353 /// denumerator. Add the incoming edge from the computation result
354 /// to \p RetPhi.
355 void buildRemainderComputation(Value *AxInitial, Value *AyInitial, Value *X,
356 PHINode *RetPhi, FastMathFlags FMF) const {
357 IRBuilder<>::FastMathFlagGuard Guard(B);
358 B.setFastMathFlags(FMF);
359
360 // Build:
361 // ex = frexp_exp(ax) - 1;
362 // ax = fldexp(frexp_mant(ax), bits);
363 // ey = frexp_exp(ay) - 1;
364 // ay = fledxp(frexp_mant(ay), 1);
365 auto [Ax, Ex] = buildExpAndPower(Src: AxInitial, NewExp: Bits, ExName: "ex", PowName: "ax");
366 auto [Ay, Ey] = buildExpAndPower(Src: AyInitial, NewExp: One, ExName: "ey", PowName: "ay");
367
368 // Build:
369 // int nb = ex - ey;
370 // float ayinv = 1.0/ay;
371 Value *Nb = B.CreateSub(LHS: Ex, RHS: Ey, Name: "nb");
372 Value *Ayinv = createRcp(V: Ay, Name: "ayinv");
373
374 // Build: while (nb > bits)
375 BasicBlock *PreheaderBB = B.GetInsertBlock();
376 Function *Fun = PreheaderBB->getParent();
377 auto *LoopBB = BasicBlock::Create(Context&: B.getContext(), Name: "frem.loop_body", Parent: Fun);
378 auto *ExitBB = BasicBlock::Create(Context&: B.getContext(), Name: "frem.loop_exit", Parent: Fun);
379
380 B.CreateCondBr(Cond: B.CreateICmp(P: CmpInst::ICMP_SGT, LHS: Nb, RHS: Bits), True: LoopBB, False: ExitBB);
381
382 // Build loop body:
383 // UPDATE_AX
384 // ax = fldexp(ax, bits);
385 // nb -= bits;
386 // One iteration of the loop is factored out. The code shared by
387 // the loop and this "iteration" is denoted by UPDATE_AX.
388 B.SetInsertPoint(LoopBB);
389 PHINode *NbIv = B.CreatePHI(Ty: Nb->getType(), NumReservedValues: 2, Name: "nb_iv");
390 NbIv->addIncoming(V: Nb, BB: PreheaderBB);
391
392 auto *AxPhi = B.CreatePHI(Ty: ComputeFpTy, NumReservedValues: 2, Name: "ax_loop_phi");
393 AxPhi->addIncoming(V: Ax, BB: PreheaderBB);
394
395 Value *AxPhiUpdate = buildUpdateAx(Ax: AxPhi, Ay, Ayinv);
396 AxPhiUpdate = B.CreateLdexp(Src: AxPhiUpdate, Exp: Bits, FMFSource: {}, Name: "ax_update");
397 AxPhi->addIncoming(V: AxPhiUpdate, BB: LoopBB);
398 NbIv->addIncoming(V: B.CreateSub(LHS: NbIv, RHS: Bits, Name: "nb_update"), BB: LoopBB);
399
400 B.CreateCondBr(Cond: B.CreateICmp(P: CmpInst::ICMP_SGT, LHS: NbIv, RHS: Bits), True: LoopBB, False: ExitBB);
401
402 // Build final iteration
403 // ax = fldexp(ax, nb - bits + 1);
404 // UPDATE_AX
405 B.SetInsertPoint(ExitBB);
406
407 auto *AxPhiExit = B.CreatePHI(Ty: ComputeFpTy, NumReservedValues: 2, Name: "ax_exit_phi");
408 AxPhiExit->addIncoming(V: Ax, BB: PreheaderBB);
409 AxPhiExit->addIncoming(V: AxPhi, BB: LoopBB);
410 auto *NbExitPhi = B.CreatePHI(Ty: Nb->getType(), NumReservedValues: 2, Name: "nb_exit_phi");
411 NbExitPhi->addIncoming(V: NbIv, BB: LoopBB);
412 NbExitPhi->addIncoming(V: Nb, BB: PreheaderBB);
413
414 Value *AxFinal = B.CreateLdexp(
415 Src: AxPhiExit, Exp: B.CreateAdd(LHS: B.CreateSub(LHS: NbExitPhi, RHS: Bits), RHS: One), FMFSource: {}, Name: "ax");
416 AxFinal = buildUpdateAx(Ax: AxFinal, Ay, Ayinv);
417
418 // Build:
419 // ax = fldexp(ax, ey);
420 // ret = copysign(ax,x);
421 AxFinal = B.CreateLdexp(Src: AxFinal, Exp: Ey, FMFSource: {}, Name: "ax");
422 if (ComputeFpTy != FremTy)
423 AxFinal = B.CreateFPTrunc(V: AxFinal, DestTy: FremTy);
424 Value *Ret = B.CreateCopySign(LHS: AxFinal, RHS: X);
425
426 RetPhi->addIncoming(V: Ret, BB: ExitBB);
427 }
428
429 /// Build the else-branch of the conditional in the FRem
430 /// expansion, i.e. the case in wich Ax <= Ay, where Ax = |X|, Ay
431 /// = |Y|, and X is the numerator and Y the denumerator. Add the
432 /// incoming edge from the result to \p RetPhi.
433 void buildElseBranch(Value *Ax, Value *Ay, Value *X, PHINode *RetPhi) const {
434 // Build:
435 // ret = ax == ay ? copysign(0.0f, x) : x;
436 Value *ZeroWithXSign = B.CreateCopySign(LHS: ConstantFP::getZero(Ty: FremTy), RHS: X);
437 Value *Ret = B.CreateSelect(C: B.CreateFCmpOEQ(LHS: Ax, RHS: Ay), True: ZeroWithXSign, False: X);
438
439 RetPhi->addIncoming(V: Ret, BB: B.GetInsertBlock());
440 }
441
442 /// Return a value that is NaN if one of the corner cases concerning
443 /// the inputs \p X and \p Y is detected, and \p Ret otherwise.
444 Value *handleInputCornerCases(Value *Ret, Value *X, Value *Y,
445 std::optional<SimplifyQuery> &SQ,
446 bool NoInfs) const {
447 // Build:
448 // ret = (y == 0.0f || isnan(y)) ? QNAN : ret;
449 // ret = isfinite(x) ? ret : QNAN;
450 Value *Nan = ConstantFP::getQNaN(Ty: FremTy);
451 Ret = B.CreateSelect(C: B.CreateFCmpUEQ(LHS: Y, RHS: ConstantFP::getZero(Ty: FremTy)), True: Nan,
452 False: Ret);
453 Value *XFinite =
454 NoInfs || (SQ && isKnownNeverInfinity(V: X, SQ: *SQ))
455 ? B.getTrue()
456 : B.CreateFCmpULT(LHS: B.CreateFAbs(V: X), RHS: ConstantFP::getInfinity(Ty: FremTy));
457 Ret = B.CreateSelect(C: XFinite, True: Ret, False: Nan);
458
459 return Ret;
460 }
461};
462} // namespace
463
464Value *FRemExpander::buildApproxFRem(Value *X, Value *Y) const {
465 IRBuilder<>::FastMathFlagGuard Guard(B);
466 // Propagating the approximate functions flag to the
467 // division leads to an unacceptable drop in precision
468 // on AMDGPU.
469 // TODO Find out if any flags might be worth propagating.
470 B.clearFastMathFlags();
471
472 Value *Quot = B.CreateFDiv(L: X, R: Y);
473 Value *Trunc = B.CreateUnaryIntrinsic(ID: Intrinsic::trunc, Op: Quot, FMFSource: {});
474 Value *Neg = B.CreateFNeg(V: Trunc);
475
476 return B.CreateFMA(Factor1: Neg, Factor2: Y, Summand: X);
477}
478
479Value *FRemExpander::buildFRem(Value *X, Value *Y,
480 std::optional<SimplifyQuery> &SQ) const {
481 assert(X->getType() == FremTy && Y->getType() == FremTy);
482
483 FastMathFlags FMF = B.getFastMathFlags();
484
485 // This function generates the following code structure:
486 // if (abs(x) > abs(y))
487 // { ret = compute remainder }
488 // else
489 // { ret = x or 0 with sign of x }
490 // Adjust ret to NaN/inf in input
491 // return ret
492 Value *Ax = B.CreateFAbs(V: X, FMFSource: {}, Name: "ax");
493 Value *Ay = B.CreateFAbs(V: Y, FMFSource: {}, Name: "ay");
494 if (ComputeFpTy != X->getType()) {
495 Ax = B.CreateFPExt(V: Ax, DestTy: ComputeFpTy, Name: "ax");
496 Ay = B.CreateFPExt(V: Ay, DestTy: ComputeFpTy, Name: "ay");
497 }
498 Value *AxAyCmp = B.CreateFCmpOGT(LHS: Ax, RHS: Ay);
499
500 PHINode *RetPhi = B.CreatePHI(Ty: FremTy, NumReservedValues: 2, Name: "ret");
501 Value *Ret = RetPhi;
502
503 // We would return NaN in all corner cases handled here.
504 // Hence, if NaNs are excluded, keep the result as it is.
505 if (!FMF.noNaNs())
506 Ret = handleInputCornerCases(Ret, X, Y, SQ, NoInfs: FMF.noInfs());
507
508 Function *Fun = B.GetInsertBlock()->getParent();
509 auto *ThenBB = BasicBlock::Create(Context&: B.getContext(), Name: "frem.compute", Parent: Fun);
510 auto *ElseBB = BasicBlock::Create(Context&: B.getContext(), Name: "frem.else", Parent: Fun);
511 SplitBlockAndInsertIfThenElse(Cond: AxAyCmp, SplitBefore: RetPhi, ThenBlock: &ThenBB, ElseBlock: &ElseBB);
512
513 auto SavedInsertPt = B.GetInsertPoint();
514
515 // Build remainder computation for "then" branch
516 //
517 // The ordered comparison ensures that ax and ay are not NaNs
518 // in the then-branch. Furthermore, y cannot be an infinity and the
519 // check at the end of the function ensures that the result will not
520 // be used if x is an infinity.
521 FastMathFlags ComputeFMF = FMF;
522 ComputeFMF.setNoInfs();
523 ComputeFMF.setNoNaNs();
524
525 B.SetInsertPoint(ThenBB);
526 buildRemainderComputation(AxInitial: Ax, AyInitial: Ay, X, RetPhi, FMF);
527 B.CreateBr(Dest: RetPhi->getParent());
528
529 // Build "else"-branch
530 B.SetInsertPoint(ElseBB);
531 buildElseBranch(Ax, Ay, X, RetPhi);
532 B.CreateBr(Dest: RetPhi->getParent());
533
534 B.SetInsertPoint(SavedInsertPt);
535
536 return Ret;
537}
538
539static bool expandFRem(BinaryOperator &I, std::optional<SimplifyQuery> &SQ) {
540 LLVM_DEBUG(dbgs() << "Expanding instruction: " << I << '\n');
541
542 Type *Ty = I.getType();
543 assert(FRemExpander::canExpandType(Ty) &&
544 "Expected supported floating point type");
545
546 FastMathFlags FMF = I.getFastMathFlags();
547 // TODO Make use of those flags for optimization?
548 FMF.setAllowReciprocal(false);
549 FMF.setAllowContract(false);
550
551 IRBuilder<> B(&I);
552 B.setFastMathFlags(FMF);
553 B.SetCurrentDebugLocation(I.getDebugLoc());
554
555 const FRemExpander Expander = FRemExpander::create(B, Ty);
556 Value *Ret = FMF.approxFunc()
557 ? Expander.buildApproxFRem(X: I.getOperand(i_nocapture: 0), Y: I.getOperand(i_nocapture: 1))
558 : Expander.buildFRem(X: I.getOperand(i_nocapture: 0), Y: I.getOperand(i_nocapture: 1), SQ);
559
560 I.replaceAllUsesWith(V: Ret);
561 Ret->takeName(V: &I);
562 I.eraseFromParent();
563
564 return true;
565}
566// clang-format off: preserve formatting of the following example
567
568/// Generate code to convert a fp number to integer, replacing FPToS(U)I with
569/// the generated code. This currently generates code similarly to compiler-rt's
570/// implementations.
571///
572/// An example IR generated from compiler-rt/fixsfdi.c looks like below:
573/// define dso_local i64 @foo(float noundef %a) local_unnamed_addr #0 {
574/// entry:
575/// %0 = bitcast float %a to i32
576/// %conv.i = zext i32 %0 to i64
577/// %tobool.not = icmp sgt i32 %0, -1
578/// %conv = select i1 %tobool.not, i64 1, i64 -1
579/// %and = lshr i64 %conv.i, 23
580/// %shr = and i64 %and, 255
581/// %and2 = and i64 %conv.i, 8388607
582/// %or = or i64 %and2, 8388608
583/// %cmp = icmp ult i64 %shr, 127
584/// br i1 %cmp, label %cleanup, label %if.end
585///
586/// if.end: ; preds = %entry
587/// %sub = add nuw nsw i64 %shr, 4294967169
588/// %conv5 = and i64 %sub, 4294967232
589/// %cmp6.not = icmp eq i64 %conv5, 0
590/// br i1 %cmp6.not, label %if.end12, label %if.then8
591///
592/// if.then8: ; preds = %if.end
593/// %cond11 = select i1 %tobool.not, i64 9223372036854775807, i64
594/// -9223372036854775808 br label %cleanup
595///
596/// if.end12: ; preds = %if.end
597/// %cmp13 = icmp ult i64 %shr, 150
598/// br i1 %cmp13, label %if.then15, label %if.else
599///
600/// if.then15: ; preds = %if.end12
601/// %sub16 = sub nuw nsw i64 150, %shr
602/// %shr17 = lshr i64 %or, %sub16
603/// %mul = mul nsw i64 %shr17, %conv
604/// br label %cleanup
605///
606/// if.else: ; preds = %if.end12
607/// %sub18 = add nsw i64 %shr, -150
608/// %shl = shl i64 %or, %sub18
609/// %mul19 = mul nsw i64 %shl, %conv
610/// br label %cleanup
611///
612/// cleanup: ; preds = %entry,
613/// %if.else, %if.then15, %if.then8
614/// %retval.0 = phi i64 [ %cond11, %if.then8 ], [ %mul, %if.then15 ], [
615/// %mul19, %if.else ], [ 0, %entry ] ret i64 %retval.0
616/// }
617///
618/// Replace fp to integer with generated code.
619static void expandFPToI(Instruction *FPToI, bool IsSaturating, bool IsSigned) {
620 // clang-format on
621 IRBuilder<> Builder(FPToI);
622 auto *FloatVal = FPToI->getOperand(i: 0);
623 IntegerType *IntTy = cast<IntegerType>(Val: FPToI->getType());
624
625 unsigned BitWidth = FPToI->getType()->getIntegerBitWidth();
626 unsigned FPMantissaWidth = FloatVal->getType()->getFPMantissaWidth() - 1;
627
628 // FIXME: fp16's range is covered by i32. So `fptoi half` can convert
629 // to i32 first following a sext/zext to target integer type.
630 Value *A1 = nullptr;
631 if (FloatVal->getType()->isHalfTy() && BitWidth >= 32) {
632 if (FPToI->getOpcode() == Instruction::FPToUI) {
633 Value *A0 = Builder.CreateFPToUI(V: FloatVal, DestTy: Builder.getInt32Ty());
634 A1 = Builder.CreateZExt(V: A0, DestTy: IntTy);
635 } else { // FPToSI
636 Value *A0 = Builder.CreateFPToSI(V: FloatVal, DestTy: Builder.getInt32Ty());
637 A1 = Builder.CreateSExt(V: A0, DestTy: IntTy);
638 }
639 FPToI->replaceAllUsesWith(V: A1);
640 FPToI->dropAllReferences();
641 FPToI->eraseFromParent();
642 return;
643 }
644
645 // fp80 conversion is implemented by fpext to fp128 first then do the
646 // conversion.
647 FPMantissaWidth = FPMantissaWidth == 63 ? 112 : FPMantissaWidth;
648 unsigned FloatWidth =
649 PowerOf2Ceil(A: FloatVal->getType()->getScalarSizeInBits());
650 unsigned ExponentWidth = FloatWidth - FPMantissaWidth - 1;
651 unsigned ExponentBias = (1 << (ExponentWidth - 1)) - 1;
652 IntegerType *FloatIntTy = Builder.getIntNTy(N: FloatWidth);
653 Value *ImplicitBit = ConstantInt::get(
654 Ty: FloatIntTy, V: APInt::getOneBitSet(numBits: FloatWidth, BitNo: FPMantissaWidth));
655 Value *SignificandMask = ConstantInt::get(
656 Ty: FloatIntTy, V: APInt::getLowBitsSet(numBits: FloatWidth, loBitsSet: FPMantissaWidth));
657
658 BasicBlock *Entry = Builder.GetInsertBlock();
659 Function *F = Entry->getParent();
660 Entry->setName(Twine(Entry->getName(), "fp-to-i-entry"));
661 BasicBlock *CheckSaturateBB, *SaturateBB;
662 BasicBlock *End =
663 Entry->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "fp-to-i-cleanup");
664 if (IsSaturating) {
665 CheckSaturateBB = BasicBlock::Create(Context&: Builder.getContext(),
666 Name: "fp-to-i-if-check.saturate", Parent: F, InsertBefore: End);
667 SaturateBB =
668 BasicBlock::Create(Context&: Builder.getContext(), Name: "fp-to-i-if-saturate", Parent: F, InsertBefore: End);
669 }
670 BasicBlock *CheckExpSizeBB = BasicBlock::Create(
671 Context&: Builder.getContext(), Name: "fp-to-i-if-check.exp.size", Parent: F, InsertBefore: End);
672 BasicBlock *ExpSmallBB =
673 BasicBlock::Create(Context&: Builder.getContext(), Name: "fp-to-i-if-exp.small", Parent: F, InsertBefore: End);
674 BasicBlock *ExpLargeBB =
675 BasicBlock::Create(Context&: Builder.getContext(), Name: "fp-to-i-if-exp.large", Parent: F, InsertBefore: End);
676
677 Entry->getTerminator()->eraseFromParent();
678
679 // entry:
680 Builder.SetInsertPoint(Entry);
681 // We're going to introduce branches on the value, so freeze it.
682 if (!isGuaranteedNotToBeUndefOrPoison(V: FloatVal))
683 FloatVal = Builder.CreateFreeze(V: FloatVal);
684 // fp80 conversion is implemented by fpext to fp128 first then do the
685 // conversion.
686 if (FloatVal->getType()->isX86_FP80Ty())
687 FloatVal =
688 Builder.CreateFPExt(V: FloatVal, DestTy: Type::getFP128Ty(C&: Builder.getContext()));
689 Value *ARep = Builder.CreateBitCast(V: FloatVal, DestTy: FloatIntTy);
690 Value *PosOrNeg, *Sign;
691 if (IsSigned) {
692 PosOrNeg =
693 Builder.CreateICmpSGT(LHS: ARep, RHS: ConstantInt::getSigned(Ty: FloatIntTy, V: -1));
694 Sign = Builder.CreateSelectWithUnknownProfile(
695 C: PosOrNeg, True: ConstantInt::getSigned(Ty: IntTy, V: 1),
696 False: ConstantInt::getSigned(Ty: IntTy, V: -1), PassName: "sign");
697 }
698 Value *And =
699 Builder.CreateLShr(LHS: ARep, RHS: Builder.getIntN(N: FloatWidth, C: FPMantissaWidth));
700 Value *BiasedExp = Builder.CreateAnd(
701 LHS: And, RHS: Builder.getIntN(N: FloatWidth, C: (1 << ExponentWidth) - 1), Name: "biased.exp");
702 Value *Abs = Builder.CreateAnd(LHS: ARep, RHS: SignificandMask);
703 Value *Significand = Builder.CreateOr(LHS: Abs, RHS: ImplicitBit, Name: "significand");
704 Value *ZeroResultCond = Builder.CreateICmpULT(
705 LHS: BiasedExp, RHS: Builder.getIntN(N: FloatWidth, C: ExponentBias), Name: "exp.is.negative");
706 if (IsSaturating) {
707 Value *IsNaN = Builder.CreateFCmpUNO(LHS: FloatVal, RHS: FloatVal, Name: "is.nan");
708 ZeroResultCond = Builder.CreateOr(LHS: ZeroResultCond, RHS: IsNaN);
709 if (!IsSigned) {
710 Value *IsNeg = Builder.CreateIsNeg(Arg: ARep);
711 ZeroResultCond = Builder.CreateOr(LHS: ZeroResultCond, RHS: IsNeg);
712 }
713 }
714 Instruction *CondBr = Builder.CreateCondBr(
715 Cond: ZeroResultCond, True: End, False: IsSaturating ? CheckSaturateBB : CheckExpSizeBB);
716 // We do not have any information on the value of the exponent, so mark the
717 // branch weights as unkown.
718 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CondBr, DEBUG_TYPE, F);
719
720 Value *Saturated;
721 if (IsSaturating) {
722 // check.saturate:
723 Builder.SetInsertPoint(CheckSaturateBB);
724 uint64_t SaturatingBiasedExp =
725 static_cast<uint64_t>(ExponentBias) + BitWidth - IsSigned;
726 // Clamp to the all-ones (inf/NaN) exponent. Without this, when the integer
727 // is wide enough to hold every finite float the threshold exceeds any
728 // possible biased exponent, so +/-inf would never saturate.
729 uint64_t MaxBiasedExp = (1ULL << ExponentWidth) - 1;
730 if (SaturatingBiasedExp > MaxBiasedExp)
731 SaturatingBiasedExp = MaxBiasedExp;
732 Value *Cmp3 = Builder.CreateICmpUGE(
733 LHS: BiasedExp, RHS: ConstantInt::get(Ty: FloatIntTy, V: SaturatingBiasedExp));
734 Value *CondBrSat = Builder.CreateCondBr(Cond: Cmp3, True: SaturateBB, False: CheckExpSizeBB);
735 // Saturation is considered an unlikely event.
736 applyProfMetadataIfEnabled(V: CondBrSat, setMetadataCallback: [&](Instruction *Inst) {
737 Inst->setMetadata(
738 KindID: LLVMContext::MD_prof,
739 Node: MDBuilder(Inst->getContext()).createUnlikelyBranchWeights());
740 });
741
742 // saturate:
743 Builder.SetInsertPoint(SaturateBB);
744 if (IsSigned) {
745 Value *SignedMax =
746 ConstantInt::get(Ty: IntTy, V: APInt::getSignedMaxValue(numBits: BitWidth));
747 Value *SignedMin =
748 ConstantInt::get(Ty: IntTy, V: APInt::getSignedMinValue(numBits: BitWidth));
749 // Select between the signed max and min values for saturation.
750 Saturated = Builder.CreateSelectWithUnknownProfile(
751 C: PosOrNeg, True: SignedMax, False: SignedMin, PassName: "saturated");
752 } else {
753 Saturated = ConstantInt::getAllOnesValue(Ty: IntTy);
754 }
755 Builder.CreateBr(Dest: End);
756 }
757
758 // if.end9:
759 Builder.SetInsertPoint(CheckExpSizeBB);
760 Value *ExpSmallerMantissaWidth = Builder.CreateICmpULT(
761 LHS: BiasedExp, RHS: Builder.getIntN(N: FloatWidth, C: ExponentBias + FPMantissaWidth),
762 Name: "exp.smaller.mantissa.width");
763 // We cannot determine whether this is a left shift or a right shift,
764 // so we mark the branch weights as unknown.
765 Value *CondBr2 =
766 Builder.CreateCondBr(Cond: ExpSmallerMantissaWidth, True: ExpSmallBB, False: ExpLargeBB);
767 applyProfMetadataIfEnabled(V: CondBr2, setMetadataCallback: [&](Instruction *Inst) {
768 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *Inst, DEBUG_TYPE, F);
769 });
770
771 // exp.small:
772 Builder.SetInsertPoint(ExpSmallBB);
773 Value *Sub13 = Builder.CreateSub(
774 LHS: Builder.getIntN(N: FloatWidth, C: ExponentBias + FPMantissaWidth), RHS: BiasedExp);
775 Value *ExpSmallRes =
776 Builder.CreateZExtOrTrunc(V: Builder.CreateLShr(LHS: Significand, RHS: Sub13), DestTy: IntTy);
777 if (IsSigned)
778 ExpSmallRes = Builder.CreateMul(LHS: ExpSmallRes, RHS: Sign);
779 Builder.CreateBr(Dest: End);
780
781 // exp.large:
782 Builder.SetInsertPoint(ExpLargeBB);
783 Value *Sub15 = Builder.CreateAdd(
784 LHS: BiasedExp,
785 RHS: ConstantInt::getSigned(
786 Ty: FloatIntTy, V: -static_cast<int64_t>(ExponentBias + FPMantissaWidth)));
787 Value *SignificandCast = Builder.CreateZExtOrTrunc(V: Significand, DestTy: IntTy);
788 Value *ExpLargeRes = Builder.CreateShl(
789 LHS: SignificandCast, RHS: Builder.CreateZExtOrTrunc(V: Sub15, DestTy: IntTy));
790 if (IsSigned)
791 ExpLargeRes = Builder.CreateMul(LHS: ExpLargeRes, RHS: Sign);
792 Builder.CreateBr(Dest: End);
793
794 // cleanup:
795 Builder.SetInsertPoint(TheBB: End, IP: End->begin());
796 PHINode *Retval0 = Builder.CreatePHI(Ty: FPToI->getType(), NumReservedValues: 3 + IsSaturating);
797
798 if (IsSaturating)
799 Retval0->addIncoming(V: Saturated, BB: SaturateBB);
800 Retval0->addIncoming(V: ExpSmallRes, BB: ExpSmallBB);
801 Retval0->addIncoming(V: ExpLargeRes, BB: ExpLargeBB);
802 Retval0->addIncoming(V: Builder.getIntN(N: BitWidth, C: 0), BB: Entry);
803
804 FPToI->replaceAllUsesWith(V: Retval0);
805 FPToI->dropAllReferences();
806 FPToI->eraseFromParent();
807}
808
809// clang-format off: preserve formatting of the following example
810
811/// Generate code to convert a fp number to integer, replacing S(U)IToFP with
812/// the generated code. This currently generates code similarly to compiler-rt's
813/// implementations. This implementation has an implicit assumption that integer
814/// width is larger than fp.
815///
816/// An example IR generated from compiler-rt/floatdisf.c looks like below:
817/// define dso_local float @__floatdisf(i64 noundef %a) local_unnamed_addr #0 {
818/// entry:
819/// %cmp = icmp eq i64 %a, 0
820/// br i1 %cmp, label %return, label %if.end
821///
822/// if.end: ; preds = %entry
823/// %shr = ashr i64 %a, 63
824/// %xor = xor i64 %shr, %a
825/// %sub = sub nsw i64 %xor, %shr
826/// %0 = tail call i64 @llvm.ctlz.i64(i64 %sub, i1 true), !range !5
827/// %cast = trunc i64 %0 to i32
828/// %sub1 = sub nuw nsw i32 64, %cast
829/// %sub2 = xor i32 %cast, 63
830/// %cmp3 = icmp ult i32 %cast, 40
831/// br i1 %cmp3, label %if.then4, label %if.else
832///
833/// if.then4: ; preds = %if.end
834/// switch i32 %sub1, label %sw.default [
835/// i32 25, label %sw.bb
836/// i32 26, label %sw.epilog
837/// ]
838///
839/// sw.bb: ; preds = %if.then4
840/// %shl = shl i64 %sub, 1
841/// br label %sw.epilog
842///
843/// sw.default: ; preds = %if.then4
844/// %sub5 = sub nsw i64 38, %0
845/// %sh_prom = and i64 %sub5, 4294967295
846/// %shr6 = lshr i64 %sub, %sh_prom
847/// %shr9 = lshr i64 274877906943, %0
848/// %and = and i64 %shr9, %sub
849/// %cmp10 = icmp ne i64 %and, 0
850/// %conv11 = zext i1 %cmp10 to i64
851/// %or = or i64 %shr6, %conv11
852/// br label %sw.epilog
853///
854/// sw.epilog: ; preds = %sw.default,
855/// %if.then4, %sw.bb
856/// %a.addr.0 = phi i64 [ %or, %sw.default ], [ %sub, %if.then4 ], [ %shl,
857/// %sw.bb ] %1 = lshr i64 %a.addr.0, 2 %2 = and i64 %1, 1 %or16 = or i64 %2,
858/// %a.addr.0 %inc = add nsw i64 %or16, 1 %3 = and i64 %inc, 67108864
859/// %tobool.not = icmp eq i64 %3, 0
860/// %spec.select.v = select i1 %tobool.not, i64 2, i64 3
861/// %spec.select = ashr i64 %inc, %spec.select.v
862/// %spec.select56 = select i1 %tobool.not, i32 %sub2, i32 %sub1
863/// br label %if.end26
864///
865/// if.else: ; preds = %if.end
866/// %sub23 = add nuw nsw i64 %0, 4294967256
867/// %sh_prom24 = and i64 %sub23, 4294967295
868/// %shl25 = shl i64 %sub, %sh_prom24
869/// br label %if.end26
870///
871/// if.end26: ; preds = %sw.epilog,
872/// %if.else
873/// %a.addr.1 = phi i64 [ %shl25, %if.else ], [ %spec.select, %sw.epilog ]
874/// %e.0 = phi i32 [ %sub2, %if.else ], [ %spec.select56, %sw.epilog ]
875/// %conv27 = trunc i64 %shr to i32
876/// %and28 = and i32 %conv27, -2147483648
877/// %add = shl nuw nsw i32 %e.0, 23
878/// %shl29 = add nuw nsw i32 %add, 1065353216
879/// %conv31 = trunc i64 %a.addr.1 to i32
880/// %and32 = and i32 %conv31, 8388607
881/// %or30 = or i32 %and32, %and28
882/// %or33 = or i32 %or30, %shl29
883/// %4 = bitcast i32 %or33 to float
884/// br label %return
885///
886/// return: ; preds = %entry,
887/// %if.end26
888/// %retval.0 = phi float [ %4, %if.end26 ], [ 0.000000e+00, %entry ]
889/// ret float %retval.0
890/// }
891///
892/// Replace integer to fp with generated code.
893static void expandIToFP(Instruction *IToFP) {
894 // clang-format on
895 IRBuilder<> Builder(IToFP);
896 auto *IntVal = IToFP->getOperand(i: 0);
897 IntegerType *IntTy = cast<IntegerType>(Val: IntVal->getType());
898
899 unsigned BitWidth = IntVal->getType()->getIntegerBitWidth();
900 unsigned FPMantissaWidth = IToFP->getType()->getFPMantissaWidth() - 1;
901 // fp80 conversion is implemented by conversion tp fp128 first following
902 // a fptrunc to fp80.
903 FPMantissaWidth = FPMantissaWidth == 63 ? 112 : FPMantissaWidth;
904 // FIXME: As there is no related builtins added in compliler-rt,
905 // here currently utilized the fp32 <-> fp16 lib calls to implement.
906 FPMantissaWidth = FPMantissaWidth == 10 ? 23 : FPMantissaWidth;
907 FPMantissaWidth = FPMantissaWidth == 7 ? 23 : FPMantissaWidth;
908 unsigned FloatWidth = PowerOf2Ceil(A: FPMantissaWidth);
909 bool IsSigned = IToFP->getOpcode() == Instruction::SIToFP;
910
911 // We're going to introduce branches on the value, so freeze it.
912 if (!isGuaranteedNotToBeUndefOrPoison(V: IntVal))
913 IntVal = Builder.CreateFreeze(V: IntVal);
914
915 // The expansion below assumes that int width >= float width. Zero or sign
916 // extend the integer accordingly.
917 if (BitWidth < FloatWidth) {
918 BitWidth = FloatWidth;
919 IntTy = Builder.getIntNTy(N: BitWidth);
920 IntVal = Builder.CreateIntCast(V: IntVal, DestTy: IntTy, isSigned: IsSigned);
921 }
922
923 Value *Temp1 =
924 Builder.CreateShl(LHS: Builder.getIntN(N: BitWidth, C: 1),
925 RHS: Builder.getIntN(N: BitWidth, C: FPMantissaWidth + 3));
926
927 BasicBlock *Entry = Builder.GetInsertBlock();
928 Function *F = Entry->getParent();
929 Entry->setName(Twine(Entry->getName(), "itofp-entry"));
930 BasicBlock *End =
931 Entry->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "itofp-return");
932 BasicBlock *IfEnd =
933 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-if-end", Parent: F, InsertBefore: End);
934 BasicBlock *IfThen4 =
935 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-if-then4", Parent: F, InsertBefore: End);
936 BasicBlock *SwBB =
937 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-sw-bb", Parent: F, InsertBefore: End);
938 BasicBlock *SwDefault =
939 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-sw-default", Parent: F, InsertBefore: End);
940 BasicBlock *SwEpilog =
941 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-sw-epilog", Parent: F, InsertBefore: End);
942 BasicBlock *IfThen20 =
943 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-if-then20", Parent: F, InsertBefore: End);
944 BasicBlock *IfElse =
945 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-if-else", Parent: F, InsertBefore: End);
946 BasicBlock *IfEnd26 =
947 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-if-end26", Parent: F, InsertBefore: End);
948
949 Entry->getTerminator()->eraseFromParent();
950
951 Function *CTLZ =
952 Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: Intrinsic::ctlz, OverloadTys: IntTy);
953 ConstantInt *True = Builder.getTrue();
954
955 // entry:
956 Builder.SetInsertPoint(Entry);
957 // We assume that the zero is an unlikely input case, so the branch to 'End'
958 // is the unlikely path.
959 Value *Cmp = Builder.CreateICmpEQ(LHS: IntVal, RHS: ConstantInt::getSigned(Ty: IntTy, V: 0));
960 Value *CondBrEntry = Builder.CreateCondBr(Cond: Cmp, True: End, False: IfEnd);
961 applyProfMetadataIfEnabled(V: CondBrEntry, setMetadataCallback: [&](Instruction *Inst) {
962 Inst->setMetadata(
963 KindID: LLVMContext::MD_prof,
964 Node: MDBuilder(Inst->getContext()).createUnlikelyBranchWeights());
965 });
966
967 // if.end:
968 Builder.SetInsertPoint(IfEnd);
969 Value *Shr =
970 Builder.CreateAShr(LHS: IntVal, RHS: Builder.getIntN(N: BitWidth, C: BitWidth - 1));
971 Value *Xor = Builder.CreateXor(LHS: Shr, RHS: IntVal);
972 Value *Sub = Builder.CreateSub(LHS: Xor, RHS: Shr);
973 Value *Call = Builder.CreateCall(Callee: CTLZ, Args: {IsSigned ? Sub : IntVal, True});
974 Value *Cast = Builder.CreateTrunc(V: Call, DestTy: Builder.getInt32Ty());
975 int BitWidthNew = FloatWidth == 128 ? BitWidth : 32;
976 Value *Sub1 = Builder.CreateSub(LHS: Builder.getIntN(N: BitWidthNew, C: BitWidth),
977 RHS: FloatWidth == 128 ? Call : Cast);
978 Value *Sub2 = Builder.CreateSub(LHS: Builder.getIntN(N: BitWidthNew, C: BitWidth - 1),
979 RHS: FloatWidth == 128 ? Call : Cast);
980 Value *Cmp3 = Builder.CreateICmpSGT(
981 LHS: Sub1, RHS: Builder.getIntN(N: BitWidthNew, C: FPMantissaWidth + 1));
982 // This branch handles the rare case where rounding the mantissa causes a
983 // carry-out at the most significant bit, necessitating an increment of the
984 // exponent. This is rare case, so the True path is mared as likely.
985 Value *CondBrIfEnd = Builder.CreateCondBr(Cond: Cmp3, True: IfThen4, False: IfElse);
986 applyProfMetadataIfEnabled(V: CondBrIfEnd, setMetadataCallback: [&](Instruction *Inst) {
987 Inst->setMetadata(
988 KindID: LLVMContext::MD_prof,
989 Node: MDBuilder(Inst->getContext()).createLikelyBranchWeights());
990 });
991
992 // if.then4:
993 Builder.SetInsertPoint(IfThen4);
994 SwitchInst *SI = Builder.CreateSwitch(V: Sub1, Dest: SwDefault);
995 SI->addCase(OnVal: Builder.getIntN(N: BitWidthNew, C: FPMantissaWidth + 2), Dest: SwBB);
996 SI->addCase(OnVal: Builder.getIntN(N: BitWidthNew, C: FPMantissaWidth + 3), Dest: SwEpilog);
997 // Add branch weights to the SwitchInst. The weights are provided for the
998 // default case first (SwDefault), followed by each explicit case in the
999 // order they were added (SwBB, then SwEpilog). Because the following cases
1000 // are rare, the defalut case is given a likely weight.
1001 if (!ProfcheckDisableMetadataFixes) {
1002 SI->setMetadata(
1003 KindID: LLVMContext::MD_prof,
1004 Node: MDBuilder(SI->getContext())
1005 .createBranchWeights(Weights: {llvm::MDBuilder::kLikelyBranchWeight,
1006 llvm::MDBuilder::kUnlikelyBranchWeight,
1007 llvm::MDBuilder::kUnlikelyBranchWeight}));
1008 }
1009
1010 // sw.bb:
1011 Builder.SetInsertPoint(SwBB);
1012 Value *Shl =
1013 Builder.CreateShl(LHS: IsSigned ? Sub : IntVal, RHS: Builder.getIntN(N: BitWidth, C: 1));
1014 Builder.CreateBr(Dest: SwEpilog);
1015
1016 // sw.default:
1017 Builder.SetInsertPoint(SwDefault);
1018 Value *Sub5 = Builder.CreateSub(
1019 LHS: Builder.getIntN(N: BitWidthNew, C: BitWidth - FPMantissaWidth - 3),
1020 RHS: FloatWidth == 128 ? Call : Cast);
1021 Value *ShProm = Builder.CreateZExt(V: Sub5, DestTy: IntTy);
1022 Value *Shr6 = Builder.CreateLShr(LHS: IsSigned ? Sub : IntVal,
1023 RHS: FloatWidth == 128 ? Sub5 : ShProm);
1024 Value *Sub8 =
1025 Builder.CreateAdd(LHS: FloatWidth == 128 ? Call : Cast,
1026 RHS: Builder.getIntN(N: BitWidthNew, C: FPMantissaWidth + 3));
1027 Value *ShProm9 = Builder.CreateZExt(V: Sub8, DestTy: IntTy);
1028 Value *Shr9 = Builder.CreateLShr(LHS: ConstantInt::getSigned(Ty: IntTy, V: -1),
1029 RHS: FloatWidth == 128 ? Sub8 : ShProm9);
1030 Value *And = Builder.CreateAnd(LHS: Shr9, RHS: IsSigned ? Sub : IntVal);
1031 Value *Cmp10 = Builder.CreateICmpNE(LHS: And, RHS: Builder.getIntN(N: BitWidth, C: 0));
1032 Value *Conv11 = Builder.CreateZExt(V: Cmp10, DestTy: IntTy);
1033 Value *Or = Builder.CreateOr(LHS: Shr6, RHS: Conv11);
1034 Builder.CreateBr(Dest: SwEpilog);
1035
1036 // sw.epilog:
1037 Builder.SetInsertPoint(SwEpilog);
1038 PHINode *AAddr0 = Builder.CreatePHI(Ty: IntTy, NumReservedValues: 3);
1039 AAddr0->addIncoming(V: Or, BB: SwDefault);
1040 AAddr0->addIncoming(V: IsSigned ? Sub : IntVal, BB: IfThen4);
1041 AAddr0->addIncoming(V: Shl, BB: SwBB);
1042 Value *A0 = Builder.CreateTrunc(V: AAddr0, DestTy: Builder.getInt32Ty());
1043 Value *A1 = Builder.CreateLShr(LHS: A0, RHS: Builder.getInt32(C: 2));
1044 Value *A2 = Builder.CreateAnd(LHS: A1, RHS: Builder.getInt32(C: 1));
1045 Value *Conv16 = Builder.CreateZExt(V: A2, DestTy: IntTy);
1046 Value *Or17 = Builder.CreateOr(LHS: AAddr0, RHS: Conv16);
1047 Value *Inc = Builder.CreateAdd(LHS: Or17, RHS: Builder.getIntN(N: BitWidth, C: 1));
1048 Value *Shr18 = nullptr;
1049 if (IsSigned)
1050 Shr18 = Builder.CreateAShr(LHS: Inc, RHS: Builder.getIntN(N: BitWidth, C: 2));
1051 else
1052 Shr18 = Builder.CreateLShr(LHS: Inc, RHS: Builder.getIntN(N: BitWidth, C: 2));
1053 Value *A3 = Builder.CreateAnd(LHS: Inc, RHS: Temp1, Name: "a3");
1054 Value *PosOrNeg = Builder.CreateICmpEQ(LHS: A3, RHS: Builder.getIntN(N: BitWidth, C: 0));
1055 Value *ExtractT60 = Builder.CreateTrunc(V: Shr18, DestTy: Builder.getIntNTy(N: FloatWidth));
1056 Value *Extract63 = Builder.CreateLShr(LHS: Shr18, RHS: Builder.getIntN(N: BitWidth, C: 32));
1057 Value *ExtractT64 = nullptr;
1058 if (FloatWidth > 80)
1059 ExtractT64 = Builder.CreateTrunc(V: Sub2, DestTy: Builder.getInt64Ty());
1060 else
1061 ExtractT64 = Builder.CreateTrunc(V: Extract63, DestTy: Builder.getInt32Ty());
1062 // Rounding usually keeps the exponent within its current magnitude and
1063 // overflow is rare. The False path is unlikely to be taken.
1064 Value *CondBrSwEpilog = Builder.CreateCondBr(Cond: PosOrNeg, True: IfEnd26, False: IfThen20);
1065 applyProfMetadataIfEnabled(V: CondBrSwEpilog, setMetadataCallback: [&](Instruction *Inst) {
1066 Inst->setMetadata(
1067 KindID: LLVMContext::MD_prof,
1068 Node: MDBuilder(Inst->getContext()).createLikelyBranchWeights());
1069 });
1070
1071 // if.then20
1072 Builder.SetInsertPoint(IfThen20);
1073 Value *Shr21 = nullptr;
1074 if (IsSigned)
1075 Shr21 = Builder.CreateAShr(LHS: Inc, RHS: Builder.getIntN(N: BitWidth, C: 3));
1076 else
1077 Shr21 = Builder.CreateLShr(LHS: Inc, RHS: Builder.getIntN(N: BitWidth, C: 3));
1078 Value *ExtractT = Builder.CreateTrunc(V: Shr21, DestTy: Builder.getIntNTy(N: FloatWidth));
1079 Value *Extract = Builder.CreateLShr(LHS: Shr21, RHS: Builder.getIntN(N: BitWidth, C: 32));
1080 Value *ExtractT62 = nullptr;
1081 if (FloatWidth > 80)
1082 ExtractT62 = Builder.CreateTrunc(V: Sub1, DestTy: Builder.getInt64Ty());
1083 else
1084 ExtractT62 = Builder.CreateTrunc(V: Extract, DestTy: Builder.getInt32Ty());
1085 Builder.CreateBr(Dest: IfEnd26);
1086
1087 // if.else:
1088 Builder.SetInsertPoint(IfElse);
1089 Value *Sub24 = Builder.CreateAdd(
1090 LHS: FloatWidth == 128 ? Call : Cast,
1091 RHS: ConstantInt::getSigned(Ty: Builder.getIntNTy(N: BitWidthNew),
1092 V: -(int)(BitWidth - FPMantissaWidth - 1)));
1093 Value *ShProm25 = Builder.CreateZExt(V: Sub24, DestTy: IntTy);
1094 Value *Shl26 = Builder.CreateShl(LHS: IsSigned ? Sub : IntVal,
1095 RHS: FloatWidth == 128 ? Sub24 : ShProm25);
1096 Value *ExtractT61 = Builder.CreateTrunc(V: Shl26, DestTy: Builder.getIntNTy(N: FloatWidth));
1097 Value *Extract65 = Builder.CreateLShr(LHS: Shl26, RHS: Builder.getIntN(N: BitWidth, C: 32));
1098 Value *ExtractT66 = nullptr;
1099 if (FloatWidth > 80)
1100 ExtractT66 = Builder.CreateTrunc(V: Sub2, DestTy: Builder.getInt64Ty());
1101 else
1102 ExtractT66 = Builder.CreateTrunc(V: Extract65, DestTy: Builder.getInt32Ty());
1103 Builder.CreateBr(Dest: IfEnd26);
1104
1105 // if.end26:
1106 Builder.SetInsertPoint(IfEnd26);
1107 PHINode *AAddr1Off0 = Builder.CreatePHI(Ty: Builder.getIntNTy(N: FloatWidth), NumReservedValues: 3);
1108 AAddr1Off0->addIncoming(V: ExtractT, BB: IfThen20);
1109 AAddr1Off0->addIncoming(V: ExtractT60, BB: SwEpilog);
1110 AAddr1Off0->addIncoming(V: ExtractT61, BB: IfElse);
1111 PHINode *AAddr1Off32 = nullptr;
1112 if (FloatWidth > 32) {
1113 AAddr1Off32 =
1114 Builder.CreatePHI(Ty: Builder.getIntNTy(N: FloatWidth > 80 ? 64 : 32), NumReservedValues: 3);
1115 AAddr1Off32->addIncoming(V: ExtractT62, BB: IfThen20);
1116 AAddr1Off32->addIncoming(V: ExtractT64, BB: SwEpilog);
1117 AAddr1Off32->addIncoming(V: ExtractT66, BB: IfElse);
1118 }
1119 PHINode *E0 = nullptr;
1120 if (FloatWidth <= 80) {
1121 E0 = Builder.CreatePHI(Ty: Builder.getIntNTy(N: BitWidthNew), NumReservedValues: 3);
1122 E0->addIncoming(V: Sub1, BB: IfThen20);
1123 E0->addIncoming(V: Sub2, BB: SwEpilog);
1124 E0->addIncoming(V: Sub2, BB: IfElse);
1125 }
1126 Value *And29 = nullptr;
1127 if (FloatWidth > 80) {
1128 Value *Temp2 = Builder.CreateShl(LHS: Builder.getIntN(N: BitWidth, C: 1),
1129 RHS: Builder.getIntN(N: BitWidth, C: 63));
1130 And29 = Builder.CreateAnd(LHS: Shr, RHS: Temp2, Name: "and29");
1131 } else {
1132 Value *Conv28 = Builder.CreateTrunc(V: Shr, DestTy: Builder.getInt32Ty());
1133 And29 = Builder.CreateAnd(
1134 LHS: Conv28, RHS: ConstantInt::get(Context&: Builder.getContext(), V: APInt::getSignMask(BitWidth: 32)));
1135 }
1136 unsigned TempMod = FPMantissaWidth % 32;
1137 Value *And34 = nullptr;
1138 Value *Shl30 = nullptr;
1139 if (FloatWidth > 80) {
1140 TempMod += 32;
1141 Value *Add = Builder.CreateShl(LHS: AAddr1Off32, RHS: Builder.getInt64(C: TempMod));
1142 Shl30 = Builder.CreateAdd(
1143 LHS: Add, RHS: Builder.getInt64(C: ((1ull << (62ull - TempMod)) - 1ull) << TempMod));
1144 And34 = Builder.CreateZExt(V: Shl30, DestTy: Builder.getInt128Ty());
1145 } else {
1146 Value *Add = Builder.CreateShl(LHS: E0, RHS: Builder.getInt32(C: TempMod));
1147 Shl30 = Builder.CreateAdd(
1148 LHS: Add, RHS: Builder.getInt32(C: ((1 << (30 - TempMod)) - 1) << TempMod));
1149 And34 = Builder.CreateAnd(LHS: FloatWidth > 32 ? AAddr1Off32 : AAddr1Off0,
1150 RHS: Builder.getInt32(C: (1 << TempMod) - 1));
1151 }
1152 Value *Or35 = nullptr;
1153 if (FloatWidth > 80) {
1154 Value *And29Trunc = Builder.CreateTrunc(V: And29, DestTy: Builder.getInt128Ty());
1155 Value *Or31 = Builder.CreateOr(LHS: And29Trunc, RHS: And34);
1156 Value *Or34 = Builder.CreateShl(LHS: Or31, RHS: Builder.getIntN(N: 128, C: 64));
1157 Value *Temp3 = Builder.CreateShl(LHS: Builder.getIntN(N: 128, C: 1),
1158 RHS: Builder.getIntN(N: 128, C: FPMantissaWidth));
1159 Value *Temp4 = Builder.CreateSub(LHS: Temp3, RHS: Builder.getIntN(N: 128, C: 1));
1160 Value *A6 = Builder.CreateAnd(LHS: AAddr1Off0, RHS: Temp4);
1161 Or35 = Builder.CreateOr(LHS: Or34, RHS: A6);
1162 } else {
1163 Value *Or31 = Builder.CreateOr(LHS: And34, RHS: And29);
1164 Or35 = Builder.CreateOr(LHS: IsSigned ? Or31 : And34, RHS: Shl30);
1165 }
1166 Value *A4 = nullptr;
1167 if (IToFP->getType()->isDoubleTy()) {
1168 Value *ZExt1 = Builder.CreateZExt(V: Or35, DestTy: Builder.getIntNTy(N: FloatWidth));
1169 Value *Shl1 = Builder.CreateShl(LHS: ZExt1, RHS: Builder.getIntN(N: FloatWidth, C: 32));
1170 Value *And1 =
1171 Builder.CreateAnd(LHS: AAddr1Off0, RHS: Builder.getIntN(N: FloatWidth, C: 0xFFFFFFFF));
1172 Value *Or1 = Builder.CreateOr(LHS: Shl1, RHS: And1);
1173 A4 = Builder.CreateBitCast(V: Or1, DestTy: IToFP->getType());
1174 } else if (IToFP->getType()->isX86_FP80Ty()) {
1175 Value *A40 =
1176 Builder.CreateBitCast(V: Or35, DestTy: Type::getFP128Ty(C&: Builder.getContext()));
1177 A4 = Builder.CreateFPTrunc(V: A40, DestTy: IToFP->getType());
1178 } else if (IToFP->getType()->isHalfTy() || IToFP->getType()->isBFloatTy()) {
1179 // Deal with "half" situation. This is a workaround since we don't have
1180 // floattihf.c currently as referring.
1181 Value *A40 =
1182 Builder.CreateBitCast(V: Or35, DestTy: Type::getFloatTy(C&: Builder.getContext()));
1183 A4 = Builder.CreateFPTrunc(V: A40, DestTy: IToFP->getType());
1184 } else // float type
1185 A4 = Builder.CreateBitCast(V: Or35, DestTy: IToFP->getType());
1186
1187 // Sub2 is the unbiased exponent (the index of the top set bit in the input).
1188 // The exponent arithmetic above wraps to garbage instead of inf once it
1189 // overflows the exponent field, so saturate to a correctly-signed infinity
1190 // when Sub2 reaches 1 << (ExponentWidth - 1). Sub2 is at most BitWidth - 1,
1191 // so skip the check entirely when even that can't reach the threshold.
1192 // (Values that round *up* into inf, e.g. 2^n - 1, keep Sub2 = BitWidth - 1;
1193 // these are handled by the conversion's own rounding, not by this
1194 // saturation.)
1195 unsigned ExponentWidth = FloatWidth - FPMantissaWidth - 1;
1196 uint64_t MinInfExp = 1ULL << (ExponentWidth - 1);
1197 if (BitWidth - 1 >= MinInfExp) {
1198 Value *MinInfExpVal = Builder.getIntN(N: BitWidthNew, C: MinInfExp);
1199 Value *Overflow = Builder.CreateICmpUGE(LHS: Sub2, RHS: MinInfExpVal);
1200 Value *Inf = ConstantFP::getInfinity(Ty: IToFP->getType(), /*Negative=*/false);
1201 if (IsSigned) {
1202 Value *NegInf =
1203 ConstantFP::getInfinity(Ty: IToFP->getType(), /*Negative=*/true);
1204 Value *IsNeg =
1205 Builder.CreateICmpSLT(LHS: IntVal, RHS: ConstantInt::getNullValue(Ty: IntTy));
1206 Inf = Builder.CreateSelectWithUnknownProfile(C: IsNeg, True: NegInf, False: Inf,
1207 DEBUG_TYPE);
1208 }
1209 A4 = Builder.CreateSelect(C: Overflow, True: Inf, False: A4);
1210 // We consider overflow to be an unlikely case.
1211 applyProfMetadataIfEnabled(V: A4, setMetadataCallback: [&](Instruction *Inst) {
1212 Inst->setMetadata(
1213 KindID: LLVMContext::MD_prof,
1214 Node: MDBuilder(Inst->getContext()).createUnlikelyBranchWeights());
1215 });
1216 }
1217 Builder.CreateBr(Dest: End);
1218
1219 // return:
1220 Builder.SetInsertPoint(TheBB: End, IP: End->begin());
1221 PHINode *Retval0 = Builder.CreatePHI(Ty: IToFP->getType(), NumReservedValues: 2);
1222 Retval0->addIncoming(V: A4, BB: IfEnd26);
1223 Retval0->addIncoming(V: ConstantFP::getZero(Ty: IToFP->getType(), Negative: false), BB: Entry);
1224
1225 IToFP->replaceAllUsesWith(V: Retval0);
1226 IToFP->dropAllReferences();
1227 IToFP->eraseFromParent();
1228}
1229
1230static void scalarize(Instruction *I,
1231 SmallVectorImpl<Instruction *> &Worklist) {
1232 VectorType *VTy = cast<FixedVectorType>(Val: I->getType());
1233
1234 IRBuilder<> Builder(I);
1235
1236 unsigned NumElements = VTy->getElementCount().getFixedValue();
1237 Value *Result = PoisonValue::get(T: VTy);
1238 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
1239 Value *Ext = Builder.CreateExtractElement(Vec: I->getOperand(i: 0), Idx);
1240
1241 Value *NewOp = nullptr;
1242 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I))
1243 NewOp = Builder.CreateBinOp(
1244 Opc: BinOp->getOpcode(), LHS: Ext,
1245 RHS: Builder.CreateExtractElement(Vec: I->getOperand(i: 1), Idx));
1246 else if (auto *CastI = dyn_cast<CastInst>(Val: I))
1247 NewOp = Builder.CreateCast(Op: CastI->getOpcode(), V: Ext,
1248 DestTy: I->getType()->getScalarType());
1249 else if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
1250 assert(II->getIntrinsicID() == Intrinsic::fptoui_sat ||
1251 II->getIntrinsicID() == Intrinsic::fptosi_sat);
1252 NewOp = Builder.CreateIntrinsic(RetTy: I->getType()->getScalarType(),
1253 ID: II->getIntrinsicID(), Args: {Ext});
1254 } else
1255 llvm_unreachable("Unsupported instruction type");
1256
1257 Result = Builder.CreateInsertElement(Vec: Result, NewElt: NewOp, Idx);
1258 if (auto *ScalarizedI = dyn_cast<Instruction>(Val: NewOp)) {
1259 ScalarizedI->copyIRFlags(V: I, IncludeWrapFlags: true);
1260 Worklist.push_back(Elt: ScalarizedI);
1261 }
1262 }
1263
1264 I->replaceAllUsesWith(V: Result);
1265 I->dropAllReferences();
1266 I->eraseFromParent();
1267}
1268
1269static void addToWorklist(Instruction &I,
1270 SmallVector<Instruction *, 4> &Worklist) {
1271 if (I.getOperand(i: 0)->getType()->isVectorTy())
1272 scalarize(I: &I, Worklist);
1273 else
1274 Worklist.push_back(Elt: &I);
1275}
1276
1277static bool runImpl(Function &F, const TargetLowering &TLI,
1278 const LibcallLoweringInfo &Libcalls, AssumptionCache *AC) {
1279 SmallVector<Instruction *, 4> Worklist;
1280
1281 unsigned MaxLegalFpConvertBitWidth =
1282 TLI.getMaxLargeFPConvertBitWidthSupported();
1283 if (ExpandFpConvertBits != IntegerType::MAX_INT_BITS)
1284 MaxLegalFpConvertBitWidth = ExpandFpConvertBits;
1285
1286 unsigned MaxLegalDivRemBitWidth = TLI.getMaxDivRemBitWidthSupported();
1287 if (ExpandDivRemBits != IntegerType::MAX_INT_BITS)
1288 MaxLegalDivRemBitWidth = ExpandDivRemBits;
1289
1290 bool DisableExpandLargeFp =
1291 MaxLegalFpConvertBitWidth >= IntegerType::MAX_INT_BITS;
1292 bool DisableExpandLargeDivRem =
1293 MaxLegalDivRemBitWidth >= IntegerType::MAX_INT_BITS;
1294 bool DisableFrem = !FRemExpander::shouldExpandAnyFremType(TLI, Libcalls);
1295
1296 if (DisableExpandLargeFp && DisableFrem && DisableExpandLargeDivRem)
1297 return false;
1298
1299 auto ShouldHandleInst = [&](Instruction &I) {
1300 Type *Ty = I.getType();
1301 // TODO: This pass doesn't handle scalable vectors.
1302 if (Ty->isScalableTy())
1303 return false;
1304
1305 switch (I.getOpcode()) {
1306 case Instruction::FRem:
1307 return !DisableFrem &&
1308 FRemExpander::shouldExpandFremType(TLI, Libcalls, Ty);
1309 case Instruction::FPToUI:
1310 case Instruction::FPToSI:
1311 return !DisableExpandLargeFp &&
1312 cast<IntegerType>(Val: Ty->getScalarType())->getIntegerBitWidth() >
1313 MaxLegalFpConvertBitWidth;
1314 case Instruction::UIToFP:
1315 case Instruction::SIToFP:
1316 return !DisableExpandLargeFp &&
1317 cast<IntegerType>(Val: I.getOperand(i: 0)->getType()->getScalarType())
1318 ->getIntegerBitWidth() > MaxLegalFpConvertBitWidth;
1319 case Instruction::UDiv:
1320 case Instruction::SDiv:
1321 case Instruction::URem:
1322 case Instruction::SRem:
1323 // Power-of-2 divisors are handled inside the expansion (via efficient
1324 // shift/mask sequences) rather than being excluded here, so that
1325 // backends that cannot lower wide div/rem even for powers of two
1326 // (e.g. when DAGCombiner is disabled) still get valid lowered code.
1327 return !DisableExpandLargeDivRem &&
1328 cast<IntegerType>(Val: Ty->getScalarType())->getIntegerBitWidth() >
1329 MaxLegalDivRemBitWidth;
1330 case Instruction::Call: {
1331 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
1332 if (II && (II->getIntrinsicID() == Intrinsic::fptoui_sat ||
1333 II->getIntrinsicID() == Intrinsic::fptosi_sat)) {
1334 return !DisableExpandLargeFp &&
1335 cast<IntegerType>(Val: Ty->getScalarType())->getIntegerBitWidth() >
1336 MaxLegalFpConvertBitWidth;
1337 }
1338 return false;
1339 }
1340 }
1341
1342 return false;
1343 };
1344
1345 bool Modified = false;
1346 for (auto It = inst_begin(F: &F), End = inst_end(F); It != End;) {
1347 Instruction &I = *It++;
1348 if (!ShouldHandleInst(I))
1349 continue;
1350
1351 addToWorklist(I, Worklist);
1352 Modified = true;
1353 }
1354
1355 while (!Worklist.empty()) {
1356 Instruction *I = Worklist.pop_back_val();
1357
1358 switch (I->getOpcode()) {
1359 case Instruction::FRem: {
1360 auto SQ = [&]() -> std::optional<SimplifyQuery> {
1361 if (AC) {
1362 auto Res = std::make_optional<SimplifyQuery>(
1363 args: I->getModule()->getDataLayout(), args&: I);
1364 Res->AC = AC;
1365 return Res;
1366 }
1367 return {};
1368 }();
1369
1370 expandFRem(I&: cast<BinaryOperator>(Val&: *I), SQ);
1371 break;
1372 }
1373
1374 case Instruction::FPToUI:
1375 expandFPToI(FPToI: I, /*IsSaturating=*/false, /*IsSigned=*/false);
1376 break;
1377 case Instruction::FPToSI:
1378 expandFPToI(FPToI: I, /*IsSaturating=*/false, /*IsSigned=*/true);
1379 break;
1380
1381 case Instruction::UIToFP:
1382 case Instruction::SIToFP:
1383 expandIToFP(IToFP: I);
1384 break;
1385
1386 case Instruction::UDiv:
1387 case Instruction::SDiv:
1388 case Instruction::URem:
1389 case Instruction::SRem: {
1390 auto *BO = cast<BinaryOperator>(Val: I);
1391 // TODO: isConstantPowerOfTwo does not handle vector constants, so
1392 // vector div/rem by a power-of-2 splat goes through the generic path.
1393 if (isConstantPowerOfTwo(V: BO->getOperand(i_nocapture: 1), SignedOp: isSigned(Opcode: BO->getOpcode()))) {
1394 expandPow2DivRem(BO);
1395 } else {
1396 unsigned Opc = BO->getOpcode();
1397 if (Opc == Instruction::UDiv || Opc == Instruction::SDiv)
1398 expandDivision(Div: BO);
1399 else
1400 expandRemainder(Rem: BO);
1401 }
1402 break;
1403 }
1404 case Instruction::Call: {
1405 auto *II = cast<IntrinsicInst>(Val: I);
1406 assert(II->getIntrinsicID() == Intrinsic::fptoui_sat ||
1407 II->getIntrinsicID() == Intrinsic::fptosi_sat);
1408 expandFPToI(FPToI: I, /*IsSaturating=*/true,
1409 /*IsSigned=*/II->getIntrinsicID() == Intrinsic::fptosi_sat);
1410 break;
1411 }
1412 }
1413 }
1414
1415 return Modified;
1416}
1417
1418namespace {
1419class ExpandIRInstsLegacyPass : public FunctionPass {
1420 CodeGenOptLevel OptLevel;
1421
1422public:
1423 static char ID;
1424
1425 ExpandIRInstsLegacyPass(CodeGenOptLevel OptLevel)
1426 : FunctionPass(ID), OptLevel(OptLevel) {}
1427
1428 ExpandIRInstsLegacyPass() : ExpandIRInstsLegacyPass(CodeGenOptLevel::None) {}
1429
1430 bool runOnFunction(Function &F) override {
1431 auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
1432 const TargetSubtargetInfo *Subtarget = TM->getSubtargetImpl(F);
1433 auto *TLI = Subtarget->getTargetLowering();
1434 AssumptionCache *AC = nullptr;
1435
1436 const LibcallLoweringInfo &Libcalls =
1437 getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
1438 M: *F.getParent(), Subtarget: *Subtarget);
1439
1440 if (OptLevel != CodeGenOptLevel::None && !F.hasOptNone())
1441 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
1442 return runImpl(F, TLI: *TLI, Libcalls, AC);
1443 }
1444
1445 void getAnalysisUsage(AnalysisUsage &AU) const override {
1446 AU.addRequired<LibcallLoweringInfoWrapper>();
1447 AU.addRequired<TargetPassConfig>();
1448 if (OptLevel != CodeGenOptLevel::None)
1449 AU.addRequired<AssumptionCacheTracker>();
1450 AU.addPreserved<AAResultsWrapperPass>();
1451 AU.addPreserved<GlobalsAAWrapperPass>();
1452 AU.addRequired<LibcallLoweringInfoWrapper>();
1453 }
1454};
1455} // namespace
1456
1457ExpandIRInstsPass::ExpandIRInstsPass(const TargetMachine &TM,
1458 CodeGenOptLevel OptLevel)
1459 : TM(&TM), OptLevel(OptLevel) {}
1460
1461void ExpandIRInstsPass::printPipeline(
1462 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
1463 static_cast<PassInfoMixin<ExpandIRInstsPass> *>(this)->printPipeline(
1464 OS, MapClassName2PassName);
1465 OS << '<';
1466 OS << "O" << (int)OptLevel;
1467 OS << '>';
1468}
1469
1470PreservedAnalyses ExpandIRInstsPass::run(Function &F,
1471 FunctionAnalysisManager &FAM) {
1472 const TargetSubtargetInfo *STI = TM->getSubtargetImpl(F);
1473 auto &TLI = *STI->getTargetLowering();
1474 AssumptionCache *AC = nullptr;
1475 if (OptLevel != CodeGenOptLevel::None)
1476 AC = &FAM.getResult<AssumptionAnalysis>(IR&: F);
1477
1478 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
1479
1480 const ModuleLibcallLoweringInfo *LibcallLowering =
1481 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(IR&: *F.getParent());
1482
1483 if (!LibcallLowering) {
1484 F.getContext().emitError(ErrorStr: "'" + LibcallLoweringModuleAnalysis::name() +
1485 "' analysis required");
1486 return PreservedAnalyses::all();
1487 }
1488
1489 const LibcallLoweringInfo &Libcalls =
1490 getLibcallLowering(ModuleInfo: *LibcallLowering, Subtarget: *STI);
1491
1492 return runImpl(F, TLI, Libcalls, AC) ? PreservedAnalyses::none()
1493 : PreservedAnalyses::all();
1494}
1495
1496char ExpandIRInstsLegacyPass::ID = 0;
1497INITIALIZE_PASS_BEGIN(ExpandIRInstsLegacyPass, "expand-ir-insts",
1498 "Expand certain fp instructions", false, false)
1499INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
1500INITIALIZE_PASS_END(ExpandIRInstsLegacyPass, "expand-ir-insts",
1501 "Expand IR instructions", false, false)
1502
1503FunctionPass *llvm::createExpandIRInstsPass(CodeGenOptLevel OptLevel) {
1504 return new ExpandIRInstsLegacyPass(OptLevel);
1505}
1506