1//===--- ExpandIRInsts.cpp - Expand IR instructions -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass expands certain instructions at the IR level.
9//
10// The following expansions are implemented:
11// - Expansion of ‘fptoui .. to’, ‘fptosi .. to’, ‘uitofp .. to’, ‘sitofp
12// .. to’ instructions with a bitwidth above a threshold. This is
13// useful for targets like x86_64 that cannot lower fp convertions
14// with more than 128 bits.
15//
16// - Expansion of ‘frem‘ for types MVT::f16, MVT::bf16, MVT::f32, and MVT::f64
17// for targets which use "Expand" as the legalization action for the
18// corresponding type.
19//
20// - Expansion of ‘udiv‘, ‘sdiv‘, ‘urem‘, and ‘srem‘ instructions with
21// a bitwidth above a threshold into a call to auto-generated
22// functions. This is useful for targets like x86_64 that cannot
23// lower divisions with more than 128 bits or targets like x86_32 that
24// cannot lower divisions with more than 64 bits.
25//
26// Instructions with vector types are scalarized first if their scalar
27// types can be expanded. Scalable vector types are not supported.
28//===----------------------------------------------------------------------===//
29
30#include "llvm/CodeGen/ExpandIRInsts.h"
31#include "CodeGenOptions.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/Analysis/AssumptionCache.h"
34#include "llvm/Analysis/GlobalsModRef.h"
35#include "llvm/Analysis/SimplifyQuery.h"
36#include "llvm/Analysis/ValueTracking.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/Passes.h"
39#include "llvm/CodeGen/RuntimeLibcallUtil.h"
40#include "llvm/CodeGen/TargetLowering.h"
41#include "llvm/CodeGen/TargetPassConfig.h"
42#include "llvm/CodeGen/TargetSubtargetInfo.h"
43#include "llvm/IR/IRBuilder.h"
44#include "llvm/IR/InstIterator.h"
45#include "llvm/IR/Instruction.h"
46#include "llvm/IR/Instructions.h"
47#include "llvm/IR/IntrinsicInst.h"
48#include "llvm/IR/MDBuilder.h"
49#include "llvm/IR/Module.h"
50#include "llvm/IR/PassManager.h"
51#include "llvm/IR/ProfDataUtils.h"
52#include "llvm/InitializePasses.h"
53#include "llvm/Pass.h"
54#include "llvm/Support/Casting.h"
55#include "llvm/Support/CommandLine.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/Target/TargetMachine.h"
58#include "llvm/Transforms/Utils/BasicBlockUtils.h"
59#include "llvm/Transforms/Utils/IntegerDivision.h"
60#include <optional>
61
62#define DEBUG_TYPE "expand-ir-insts"
63
64using namespace llvm;
65
66namespace llvm {
67extern cl::opt<bool> ProfcheckDisableMetadataFixes;
68}
69
70static bool isConstantPowerOfTwo(Value *V, bool SignedOp) {
71 auto *C = dyn_cast<ConstantInt>(Val: V);
72 if (!C)
73 return false;
74
75 APInt Val = C->getValue();
76 if (SignedOp && Val.isNegative())
77 Val = -Val;
78 return Val.isPowerOf2();
79}
80
81static bool isSigned(unsigned Opcode) {
82 return Opcode == Instruction::SDiv || Opcode == Instruction::SRem;
83}
84
85/// For signed div/rem by a power of 2, compute the bias-adjusted dividend:
86/// Sign = ashr X, (BitWidth - 1) -- 0 or -1
87/// Bias = lshr Sign, (BitWidth - ShiftAmt) -- 0 or 2^ShiftAmt - 1
88/// Adjusted = add X, Bias
89/// The bias adds (2^ShiftAmt - 1) for negative X, correcting rounding towards
90/// zero (instead of towards -inf that a plain ashr would give).
91/// The lshr form is used instead of 'and' to avoid large immediate constants.
92static Value *addSignedBias(IRBuilder<> &Builder, Value *X, unsigned BitWidth,
93 unsigned ShiftAmt) {
94 assert(ShiftAmt > 0 && ShiftAmt < BitWidth &&
95 "ShiftAmt out of range; callers should handle ShiftAmt == 0");
96 Value *Sign = Builder.CreateAShr(LHS: X, RHS: BitWidth - 1, Name: "sign");
97 Value *Bias = Builder.CreateLShr(LHS: Sign, RHS: BitWidth - ShiftAmt, Name: "bias");
98 return Builder.CreateAdd(LHS: X, RHS: Bias, Name: "adjusted");
99}
100
101/// Expand division or remainder by a power-of-2 constant.
102/// Division (let C = log2(|divisor|)):
103/// udiv X, 2^C -> lshr X, C
104/// sdiv X, 2^C -> ashr (add X, Bias), C (Bias corrects rounding)
105/// sdiv exact X, 2^C -> ashr exact X, C (no bias needed)
106/// For negative power-of-2 divisors, the division result is negated.
107/// Remainder (let C = log2(|divisor|)):
108/// urem X, 2^C -> and X, (2^C - 1)
109/// srem X, 2^C -> sub X, (shl (ashr (add X, Bias), C), C)
110static void expandPow2DivRem(BinaryOperator *BO) {
111 LLVM_DEBUG(dbgs() << "Expanding instruction: " << *BO << '\n');
112
113 unsigned Opcode = BO->getOpcode();
114 bool IsDiv = (Opcode == Instruction::UDiv || Opcode == Instruction::SDiv);
115 bool IsSigned = isSigned(Opcode);
116 // isExact() is only valid for div.
117 bool IsExact = IsDiv && BO->isExact();
118
119 assert(isConstantPowerOfTwo(BO->getOperand(1), IsSigned) &&
120 "Expected power-of-2 constant divisor");
121
122 Value *X = BO->getOperand(i_nocapture: 0);
123 auto *C = cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1));
124 Type *Ty = BO->getType();
125 unsigned BitWidth = Ty->getIntegerBitWidth();
126
127 APInt DivisorVal = C->getValue();
128 bool IsNegativeDivisor = IsSigned && DivisorVal.isNegative();
129 // Use countr_zero() to get the shift amount directly from the bit pattern.
130 // This works correctly for both positive and negative powers of 2, including
131 // INT_MIN, without needing to negate the value first.
132 unsigned ShiftAmt = DivisorVal.countr_zero();
133
134 IRBuilder<> Builder(BO);
135 Value *Result;
136
137 if (ShiftAmt == 0) {
138 // Div by 1/-1: X / 1 = X, X / -1 = -X.
139 // Rem by 1/-1: always 0.
140 if (IsDiv)
141 Result = IsNegativeDivisor ? Builder.CreateNeg(V: X) : X;
142 else
143 Result = ConstantInt::get(Ty, V: 0);
144 } else if (IsSigned) {
145 // The signed expansion uses X multiple times (bias computation, shift,
146 // and sub for remainder). Freeze X to ensure consistent behavior if it is
147 // undef/poison. For exact division, no bias is needed and X is used only
148 // once, so freeze is unnecessary.
149 if (!IsExact && !isGuaranteedNotToBeUndefOrPoison(V: X))
150 X = Builder.CreateFreeze(V: X, Name: X->getName() + ".fr");
151 // For exact division, no bias is needed since there's no rounding.
152 Value *Dividend =
153 IsExact ? X : addSignedBias(Builder, X, BitWidth, ShiftAmt);
154 Value *Quotient = Builder.CreateAShr(
155 LHS: Dividend, RHS: ShiftAmt, Name: IsDiv && IsNegativeDivisor ? "pre.neg" : "shifted",
156 isExact: IsExact);
157 if (IsDiv) {
158 Result = IsNegativeDivisor ? Builder.CreateNeg(V: Quotient) : Quotient;
159 } else {
160 // Rem = X - (Quotient << ShiftAmt):
161 // clear lower ShiftAmt bits via round-trip shift, then subtract.
162 Value *Truncated = Builder.CreateShl(LHS: Quotient, RHS: ShiftAmt, Name: "truncated");
163 Result = Builder.CreateSub(LHS: X, RHS: Truncated);
164 }
165 } else {
166 if (IsDiv) {
167 Result = Builder.CreateLShr(LHS: X, RHS: ShiftAmt, Name: "", isExact: IsExact);
168 } else {
169 APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ShiftAmt);
170 Result = Builder.CreateAnd(LHS: X, RHS: ConstantInt::get(Ty, V: Mask));
171 }
172 }
173
174 BO->replaceAllUsesWith(V: Result);
175 if (Result != X)
176 if (auto *RI = dyn_cast<Instruction>(Val: Result))
177 RI->takeName(V: BO);
178 BO->dropAllReferences();
179 BO->eraseFromParent();
180}
181
182/// This class implements a precise expansion of the frem instruction.
183/// The generated code is based on the fmod implementation in the AMD device
184/// libs.
185namespace {
186class FRemExpander {
187 /// The IRBuilder to use for the expansion.
188 IRBuilder<> &B;
189
190 /// Floating point type of the return value and the arguments of the FRem
191 /// instructions that should be expanded.
192 Type *FremTy;
193
194 /// Floating point type to use for the computation. This may be
195 /// wider than the \p FremTy.
196 Type *ComputeFpTy;
197
198 /// Integer type used to hold the exponents returned by frexp.
199 Type *ExTy;
200
201 /// How many bits of the quotient to compute per iteration of the
202 /// algorithm, stored as a value of type \p ExTy.
203 Value *Bits;
204
205 /// Constant 1 of type \p ExTy.
206 Value *One;
207
208 /// The frem argument/return types that can be expanded by this class.
209 // TODO: The expansion could work for other floating point types
210 // as well, but this would require additional testing.
211 static constexpr std::array<MVT, 4> ExpandableTypes{MVT::f16, MVT::bf16,
212 MVT::f32, MVT::f64};
213
214public:
215 static bool canExpandType(Type *Ty) {
216 EVT VT = EVT::getEVT(Ty);
217 assert(VT.isSimple() && "Can expand only simple types");
218
219 return is_contained(Range: ExpandableTypes, Element: VT.getSimpleVT());
220 }
221
222 static bool shouldExpandFremType(const TargetLowering &TLI,
223 const LibcallLoweringInfo &Libcalls,
224 EVT VT) {
225 assert(!VT.isVector() && "Cannot handle vector type; must scalarize first");
226 switch (TLI.getOperationAction(Op: ISD::FREM, VT)) {
227 case TargetLowering::LegalizeAction::Expand:
228 return true;
229 case TargetLowering::LegalizeAction::LibCall:
230 // The target expects a libcall, but expand inline for the supported
231 // types when that libcall is unavailable.
232 return VT.isSimple() && is_contained(Range: ExpandableTypes, Element: VT.getSimpleVT()) &&
233 Libcalls.getLibcallImpl(Call: RTLIB::getREM(VT)) == RTLIB::Unsupported;
234 default:
235 return false;
236 }
237 }
238
239 static bool shouldExpandFremType(const TargetLowering &TLI,
240 const LibcallLoweringInfo &Libcalls,
241 Type *Ty) {
242 // Consider scalar type for simplicity. It seems unlikely that a
243 // vector type can be legalized without expansion if the scalar
244 // type cannot.
245 return shouldExpandFremType(TLI, Libcalls,
246 VT: EVT::getEVT(Ty: Ty->getScalarType()));
247 }
248
249 /// Return true if the pass should expand frem instructions of any type
250 /// for the target represented by \p TLI.
251 static bool shouldExpandAnyFremType(const TargetLowering &TLI,
252 const LibcallLoweringInfo &Libcalls) {
253 return any_of(Range: ExpandableTypes, P: [&](MVT V) {
254 return shouldExpandFremType(TLI, Libcalls, VT: EVT(V));
255 });
256 }
257
258 static FRemExpander create(IRBuilder<> &B, Type *Ty) {
259 assert(canExpandType(Ty) && "Expected supported floating point type");
260
261 // The type to use for the computation of the remainder. This may be
262 // wider than the input/result type which affects the ...
263 Type *ComputeTy = Ty;
264 // ... maximum number of iterations of the remainder computation loop
265 // to use. This value is for the case in which the computation
266 // uses the same input/result type.
267 unsigned MaxIter = 2;
268
269 if (Ty->is16bitFPTy()) {
270 // Use the wider type and less iterations.
271 ComputeTy = B.getFloatTy();
272 MaxIter = 1;
273 }
274
275 unsigned Precision = APFloat::semanticsPrecision(Ty->getFltSemantics());
276 return FRemExpander{B, Ty, Precision / MaxIter, ComputeTy};
277 }
278
279 /// Build the FRem expansion for the numerator \p X and the
280 /// denumerator \p Y. The type of X and Y must match \p FremTy. The
281 /// code will be generated at the insertion point of \p B and the
282 /// insertion point will be reset at exit.
283 Value *buildFRem(Value *X, Value *Y, std::optional<SimplifyQuery> &SQ) const;
284
285 /// Build an approximate FRem expansion for the numerator \p X and
286 /// the denumerator \p Y at the insertion point of builder \p B.
287 /// The type of X and Y must match \p FremTy.
288 Value *buildApproxFRem(Value *X, Value *Y) const;
289
290private:
291 FRemExpander(IRBuilder<> &B, Type *FremTy, unsigned Bits, Type *ComputeFpTy)
292 : B(B), FremTy(FremTy), ComputeFpTy(ComputeFpTy), ExTy(B.getInt32Ty()),
293 Bits(ConstantInt::get(Ty: ExTy, V: Bits)), One(ConstantInt::get(Ty: ExTy, V: 1)) {}
294
295 Value *createRcp(Value *V, const Twine &Name) const {
296 // Leave it to later optimizations to turn this into an rcp
297 // instruction if available.
298 return B.CreateFDiv(L: ConstantFP::get(Ty: ComputeFpTy, V: 1.0), R: V, Name);
299 }
300
301 // Helper function to build the UPDATE_AX code which is common to the
302 // loop body and the "final iteration".
303 Value *buildUpdateAx(Value *Ax, Value *Ay, Value *Ayinv) const {
304 // Build:
305 // float q = rint(ax * ayinv);
306 // ax = fma(-q, ay, ax);
307 // int clt = ax < 0.0f;
308 // float axp = ax + ay;
309 // ax = clt ? axp : ax;
310 Value *Q = B.CreateUnaryIntrinsic(ID: Intrinsic::rint, Op: B.CreateFMul(L: Ax, R: Ayinv),
311 FMFSource: {}, Name: "q");
312 Value *AxUpdate = B.CreateFMA(Factor1: B.CreateFNeg(V: Q), Factor2: Ay, Summand: Ax, FMFSource: {}, Name: "ax");
313 Value *Clt = B.CreateFCmp(P: CmpInst::FCMP_OLT, LHS: AxUpdate,
314 RHS: ConstantFP::getZero(Ty: ComputeFpTy), Name: "clt");
315 Value *Axp = B.CreateFAdd(L: AxUpdate, R: Ay, Name: "axp");
316 return B.CreateSelect(C: Clt, True: Axp, False: AxUpdate, Name: "ax");
317 }
318
319 /// Build code to extract the exponent and mantissa of \p Src.
320 /// Return the exponent minus one for use as a loop bound and
321 /// the mantissa taken to the given \p NewExp power.
322 std::pair<Value *, Value *> buildExpAndPower(Value *Src, Value *NewExp,
323 const Twine &ExName,
324 const Twine &PowName) const {
325 // Build:
326 // ExName = frexp_exp(Src) - 1;
327 // PowName = fldexp(frexp_mant(ExName), NewExp);
328 Type *Ty = Src->getType();
329 Type *ExTy = B.getInt32Ty();
330 Value *Frexp = B.CreateIntrinsic(ID: Intrinsic::frexp, OverloadTypes: {Ty, ExTy}, Args: Src);
331 Value *Mant = B.CreateExtractValue(Agg: Frexp, Idxs: {0});
332 Value *Exp = B.CreateExtractValue(Agg: Frexp, Idxs: {1});
333
334 Exp = B.CreateSub(LHS: Exp, RHS: One, Name: ExName);
335 Value *Pow = B.CreateLdexp(Src: Mant, Exp: NewExp, FMFSource: {}, Name: PowName);
336
337 return {Pow, Exp};
338 }
339
340 /// Build the main computation of the remainder for the case in which
341 /// Ax > Ay, where Ax = |X|, Ay = |Y|, and X is the numerator and Y the
342 /// denumerator. Add the incoming edge from the computation result
343 /// to \p RetPhi.
344 void buildRemainderComputation(Value *AxInitial, Value *AyInitial, Value *X,
345 PHINode *RetPhi, FastMathFlags FMF) const {
346 IRBuilder<>::FastMathFlagGuard Guard(B);
347 B.setFastMathFlags(FMF);
348
349 // Build:
350 // ex = frexp_exp(ax) - 1;
351 // ax = fldexp(frexp_mant(ax), bits);
352 // ey = frexp_exp(ay) - 1;
353 // ay = fledxp(frexp_mant(ay), 1);
354 auto [Ax, Ex] = buildExpAndPower(Src: AxInitial, NewExp: Bits, ExName: "ex", PowName: "ax");
355 auto [Ay, Ey] = buildExpAndPower(Src: AyInitial, NewExp: One, ExName: "ey", PowName: "ay");
356
357 // Build:
358 // int nb = ex - ey;
359 // float ayinv = 1.0/ay;
360 Value *Nb = B.CreateSub(LHS: Ex, RHS: Ey, Name: "nb");
361 Value *Ayinv = createRcp(V: Ay, Name: "ayinv");
362
363 // Build: while (nb > bits)
364 BasicBlock *PreheaderBB = B.GetInsertBlock();
365 Function *Fun = PreheaderBB->getParent();
366 auto *LoopBB = BasicBlock::Create(Context&: B.getContext(), Name: "frem.loop_body", Parent: Fun);
367 auto *ExitBB = BasicBlock::Create(Context&: B.getContext(), Name: "frem.loop_exit", Parent: Fun);
368
369 B.CreateCondBr(Cond: B.CreateICmp(P: CmpInst::ICMP_SGT, LHS: Nb, RHS: Bits), True: LoopBB, False: ExitBB);
370
371 // Build loop body:
372 // UPDATE_AX
373 // ax = fldexp(ax, bits);
374 // nb -= bits;
375 // One iteration of the loop is factored out. The code shared by
376 // the loop and this "iteration" is denoted by UPDATE_AX.
377 B.SetInsertPoint(LoopBB);
378 PHINode *NbIv = B.CreatePHI(Ty: Nb->getType(), NumReservedValues: 2, Name: "nb_iv");
379 NbIv->addIncoming(V: Nb, BB: PreheaderBB);
380
381 auto *AxPhi = B.CreatePHI(Ty: ComputeFpTy, NumReservedValues: 2, Name: "ax_loop_phi");
382 AxPhi->addIncoming(V: Ax, BB: PreheaderBB);
383
384 Value *AxPhiUpdate = buildUpdateAx(Ax: AxPhi, Ay, Ayinv);
385 AxPhiUpdate = B.CreateLdexp(Src: AxPhiUpdate, Exp: Bits, FMFSource: {}, Name: "ax_update");
386 AxPhi->addIncoming(V: AxPhiUpdate, BB: LoopBB);
387 NbIv->addIncoming(V: B.CreateSub(LHS: NbIv, RHS: Bits, Name: "nb_update"), BB: LoopBB);
388
389 B.CreateCondBr(Cond: B.CreateICmp(P: CmpInst::ICMP_SGT, LHS: NbIv, RHS: Bits), True: LoopBB, False: ExitBB);
390
391 // Build final iteration
392 // ax = fldexp(ax, nb - bits + 1);
393 // UPDATE_AX
394 B.SetInsertPoint(ExitBB);
395
396 auto *AxPhiExit = B.CreatePHI(Ty: ComputeFpTy, NumReservedValues: 2, Name: "ax_exit_phi");
397 AxPhiExit->addIncoming(V: Ax, BB: PreheaderBB);
398 AxPhiExit->addIncoming(V: AxPhi, BB: LoopBB);
399 auto *NbExitPhi = B.CreatePHI(Ty: Nb->getType(), NumReservedValues: 2, Name: "nb_exit_phi");
400 NbExitPhi->addIncoming(V: NbIv, BB: LoopBB);
401 NbExitPhi->addIncoming(V: Nb, BB: PreheaderBB);
402
403 Value *AxFinal = B.CreateLdexp(
404 Src: AxPhiExit, Exp: B.CreateAdd(LHS: B.CreateSub(LHS: NbExitPhi, RHS: Bits), RHS: One), FMFSource: {}, Name: "ax");
405 AxFinal = buildUpdateAx(Ax: AxFinal, Ay, Ayinv);
406
407 // Build:
408 // ax = fldexp(ax, ey);
409 // ret = copysign(ax,x);
410 AxFinal = B.CreateLdexp(Src: AxFinal, Exp: Ey, FMFSource: {}, Name: "ax");
411 if (ComputeFpTy != FremTy)
412 AxFinal = B.CreateFPTrunc(V: AxFinal, DestTy: FremTy);
413 Value *Ret = B.CreateCopySign(LHS: AxFinal, RHS: X);
414
415 RetPhi->addIncoming(V: Ret, BB: ExitBB);
416 }
417
418 /// Build the else-branch of the conditional in the FRem
419 /// expansion, i.e. the case in wich Ax <= Ay, where Ax = |X|, Ay
420 /// = |Y|, and X is the numerator and Y the denumerator. Add the
421 /// incoming edge from the result to \p RetPhi.
422 void buildElseBranch(Value *Ax, Value *Ay, Value *X, PHINode *RetPhi) const {
423 // Build:
424 // ret = ax == ay ? copysign(0.0f, x) : x;
425 Value *ZeroWithXSign = B.CreateCopySign(LHS: ConstantFP::getZero(Ty: FremTy), RHS: X);
426 Value *Ret = B.CreateSelect(C: B.CreateFCmpOEQ(LHS: Ax, RHS: Ay), True: ZeroWithXSign, False: X);
427
428 RetPhi->addIncoming(V: Ret, BB: B.GetInsertBlock());
429 }
430
431 /// Return a value that is NaN if one of the corner cases concerning
432 /// the inputs \p X and \p Y is detected, and \p Ret otherwise.
433 Value *handleInputCornerCases(Value *Ret, Value *X, Value *Y,
434 std::optional<SimplifyQuery> &SQ,
435 bool NoInfs) const {
436 // Build:
437 // ret = (y == 0.0f || isnan(y)) ? QNAN : ret;
438 // ret = isfinite(x) ? ret : QNAN;
439 Value *Nan = ConstantFP::getQNaN(Ty: FremTy);
440 Ret = B.CreateSelect(C: B.CreateFCmpUEQ(LHS: Y, RHS: ConstantFP::getZero(Ty: FremTy)), True: Nan,
441 False: Ret);
442 Value *XFinite =
443 NoInfs || (SQ && isKnownNeverInfinity(V: X, SQ: *SQ))
444 ? B.getTrue()
445 : B.CreateFCmpULT(LHS: B.CreateFAbs(V: X), RHS: ConstantFP::getInfinity(Ty: FremTy));
446 Ret = B.CreateSelect(C: XFinite, True: Ret, False: Nan);
447
448 return Ret;
449 }
450};
451} // namespace
452
453Value *FRemExpander::buildApproxFRem(Value *X, Value *Y) const {
454 IRBuilder<>::FastMathFlagGuard Guard(B);
455 // Propagating the approximate functions flag to the
456 // division leads to an unacceptable drop in precision
457 // on AMDGPU.
458 // TODO Find out if any flags might be worth propagating.
459 B.clearFastMathFlags();
460
461 Value *Quot = B.CreateFDiv(L: X, R: Y);
462 Value *Trunc = B.CreateUnaryIntrinsic(ID: Intrinsic::trunc, Op: Quot, FMFSource: {});
463 Value *Neg = B.CreateFNeg(V: Trunc);
464
465 return B.CreateFMA(Factor1: Neg, Factor2: Y, Summand: X);
466}
467
468Value *FRemExpander::buildFRem(Value *X, Value *Y,
469 std::optional<SimplifyQuery> &SQ) const {
470 assert(X->getType() == FremTy && Y->getType() == FremTy);
471
472 FastMathFlags FMF = B.getFastMathFlags();
473
474 // This function generates the following code structure:
475 // if (abs(x) > abs(y))
476 // { ret = compute remainder }
477 // else
478 // { ret = x or 0 with sign of x }
479 // Adjust ret to NaN/inf in input
480 // return ret
481 Value *Ax = B.CreateFAbs(V: X, FMFSource: {}, Name: "ax");
482 Value *Ay = B.CreateFAbs(V: Y, FMFSource: {}, Name: "ay");
483 if (ComputeFpTy != X->getType()) {
484 Ax = B.CreateFPExt(V: Ax, DestTy: ComputeFpTy, Name: "ax");
485 Ay = B.CreateFPExt(V: Ay, DestTy: ComputeFpTy, Name: "ay");
486 }
487 Value *AxAyCmp = B.CreateFCmpOGT(LHS: Ax, RHS: Ay);
488
489 PHINode *RetPhi = B.CreatePHI(Ty: FremTy, NumReservedValues: 2, Name: "ret");
490 Value *Ret = RetPhi;
491
492 // We would return NaN in all corner cases handled here.
493 // Hence, if NaNs are excluded, keep the result as it is.
494 if (!FMF.noNaNs())
495 Ret = handleInputCornerCases(Ret, X, Y, SQ, NoInfs: FMF.noInfs());
496
497 Function *Fun = B.GetInsertBlock()->getParent();
498 auto *ThenBB = BasicBlock::Create(Context&: B.getContext(), Name: "frem.compute", Parent: Fun);
499 auto *ElseBB = BasicBlock::Create(Context&: B.getContext(), Name: "frem.else", Parent: Fun);
500 SplitBlockAndInsertIfThenElse(Cond: AxAyCmp, SplitBefore: RetPhi, ThenBlock: &ThenBB, ElseBlock: &ElseBB);
501
502 auto SavedInsertPt = B.GetInsertPoint();
503
504 // Build remainder computation for "then" branch
505 //
506 // The ordered comparison ensures that ax and ay are not NaNs
507 // in the then-branch. Furthermore, y cannot be an infinity and the
508 // check at the end of the function ensures that the result will not
509 // be used if x is an infinity.
510 FastMathFlags ComputeFMF = FMF;
511 ComputeFMF.setNoInfs();
512 ComputeFMF.setNoNaNs();
513
514 B.SetInsertPoint(ThenBB);
515 buildRemainderComputation(AxInitial: Ax, AyInitial: Ay, X, RetPhi, FMF);
516 B.CreateBr(Dest: RetPhi->getParent());
517
518 // Build "else"-branch
519 B.SetInsertPoint(ElseBB);
520 buildElseBranch(Ax, Ay, X, RetPhi);
521 B.CreateBr(Dest: RetPhi->getParent());
522
523 B.SetInsertPoint(SavedInsertPt);
524
525 return Ret;
526}
527
528static bool expandFRem(BinaryOperator &I, std::optional<SimplifyQuery> &SQ) {
529 LLVM_DEBUG(dbgs() << "Expanding instruction: " << I << '\n');
530
531 Type *Ty = I.getType();
532 assert(FRemExpander::canExpandType(Ty) &&
533 "Expected supported floating point type");
534
535 FastMathFlags FMF = I.getFastMathFlags();
536 // TODO Make use of those flags for optimization?
537 FMF.setAllowReciprocal(false);
538 FMF.setAllowContract(false);
539
540 IRBuilder<> B(&I);
541 B.setFastMathFlags(FMF);
542 B.SetCurrentDebugLocation(I.getDebugLoc());
543
544 const FRemExpander Expander = FRemExpander::create(B, Ty);
545 Value *Ret = FMF.approxFunc()
546 ? Expander.buildApproxFRem(X: I.getOperand(i_nocapture: 0), Y: I.getOperand(i_nocapture: 1))
547 : Expander.buildFRem(X: I.getOperand(i_nocapture: 0), Y: I.getOperand(i_nocapture: 1), SQ);
548
549 I.replaceAllUsesWith(V: Ret);
550 Ret->takeName(V: &I);
551 I.eraseFromParent();
552
553 return true;
554}
555// clang-format off: preserve formatting of the following example
556
557/// Generate code to convert a fp number to integer, replacing FPToS(U)I with
558/// the generated code. This currently generates code similarly to compiler-rt's
559/// implementations.
560///
561/// An example IR generated from compiler-rt/fixsfdi.c looks like below:
562/// define dso_local i64 @foo(float noundef %a) local_unnamed_addr #0 {
563/// entry:
564/// %0 = bitcast float %a to i32
565/// %conv.i = zext i32 %0 to i64
566/// %tobool.not = icmp sgt i32 %0, -1
567/// %conv = select i1 %tobool.not, i64 1, i64 -1
568/// %and = lshr i64 %conv.i, 23
569/// %shr = and i64 %and, 255
570/// %and2 = and i64 %conv.i, 8388607
571/// %or = or i64 %and2, 8388608
572/// %cmp = icmp ult i64 %shr, 127
573/// br i1 %cmp, label %cleanup, label %if.end
574///
575/// if.end: ; preds = %entry
576/// %sub = add nuw nsw i64 %shr, 4294967169
577/// %conv5 = and i64 %sub, 4294967232
578/// %cmp6.not = icmp eq i64 %conv5, 0
579/// br i1 %cmp6.not, label %if.end12, label %if.then8
580///
581/// if.then8: ; preds = %if.end
582/// %cond11 = select i1 %tobool.not, i64 9223372036854775807, i64
583/// -9223372036854775808 br label %cleanup
584///
585/// if.end12: ; preds = %if.end
586/// %cmp13 = icmp ult i64 %shr, 150
587/// br i1 %cmp13, label %if.then15, label %if.else
588///
589/// if.then15: ; preds = %if.end12
590/// %sub16 = sub nuw nsw i64 150, %shr
591/// %shr17 = lshr i64 %or, %sub16
592/// %mul = mul nsw i64 %shr17, %conv
593/// br label %cleanup
594///
595/// if.else: ; preds = %if.end12
596/// %sub18 = add nsw i64 %shr, -150
597/// %shl = shl i64 %or, %sub18
598/// %mul19 = mul nsw i64 %shl, %conv
599/// br label %cleanup
600///
601/// cleanup: ; preds = %entry,
602/// %if.else, %if.then15, %if.then8
603/// %retval.0 = phi i64 [ %cond11, %if.then8 ], [ %mul, %if.then15 ], [
604/// %mul19, %if.else ], [ 0, %entry ] ret i64 %retval.0
605/// }
606///
607/// Replace fp to integer with generated code.
608static void expandFPToI(Instruction *FPToI, bool IsSaturating, bool IsSigned) {
609 // clang-format on
610 IRBuilder<> Builder(FPToI);
611 auto *FloatVal = FPToI->getOperand(i: 0);
612 IntegerType *IntTy = cast<IntegerType>(Val: FPToI->getType());
613
614 unsigned BitWidth = FPToI->getType()->getIntegerBitWidth();
615 unsigned FPMantissaWidth = FloatVal->getType()->getFPMantissaWidth() - 1;
616
617 // FIXME: fp16's range is covered by i32. So `fptoi half` can convert
618 // to i32 first following a sext/zext to target integer type.
619 Value *A1 = nullptr;
620 if (FloatVal->getType()->isHalfTy() && BitWidth >= 32 && !IsSaturating) {
621 if (FPToI->getOpcode() == Instruction::FPToUI) {
622 Value *A0 = Builder.CreateFPToUI(V: FloatVal, DestTy: Builder.getInt32Ty());
623 A1 = Builder.CreateZExt(V: A0, DestTy: IntTy);
624 } else { // FPToSI
625 Value *A0 = Builder.CreateFPToSI(V: FloatVal, DestTy: Builder.getInt32Ty());
626 A1 = Builder.CreateSExt(V: A0, DestTy: IntTy);
627 }
628 FPToI->replaceAllUsesWith(V: A1);
629 FPToI->dropAllReferences();
630 FPToI->eraseFromParent();
631 return;
632 }
633
634 // fp80 conversion is implemented by fpext to fp128 first then do the
635 // conversion.
636 FPMantissaWidth = FPMantissaWidth == 63 ? 112 : FPMantissaWidth;
637 unsigned FloatWidth =
638 PowerOf2Ceil(A: FloatVal->getType()->getScalarSizeInBits());
639 unsigned ExponentWidth = FloatWidth - FPMantissaWidth - 1;
640 unsigned ExponentBias = (1 << (ExponentWidth - 1)) - 1;
641 IntegerType *FloatIntTy = Builder.getIntNTy(N: FloatWidth);
642 Value *ImplicitBit = ConstantInt::get(
643 Ty: FloatIntTy, V: APInt::getOneBitSet(numBits: FloatWidth, BitNo: FPMantissaWidth));
644 Value *SignificandMask = ConstantInt::get(
645 Ty: FloatIntTy, V: APInt::getLowBitsSet(numBits: FloatWidth, loBitsSet: FPMantissaWidth));
646
647 BasicBlock *Entry = Builder.GetInsertBlock();
648 Function *F = Entry->getParent();
649 Entry->setName(Twine(Entry->getName(), "fp-to-i-entry"));
650 BasicBlock *CheckSaturateBB, *SaturateBB;
651 BasicBlock *End =
652 Entry->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "fp-to-i-cleanup");
653 if (IsSaturating) {
654 CheckSaturateBB = BasicBlock::Create(Context&: Builder.getContext(),
655 Name: "fp-to-i-if-check.saturate", Parent: F, InsertBefore: End);
656 SaturateBB =
657 BasicBlock::Create(Context&: Builder.getContext(), Name: "fp-to-i-if-saturate", Parent: F, InsertBefore: End);
658 }
659 BasicBlock *CheckExpSizeBB = BasicBlock::Create(
660 Context&: Builder.getContext(), Name: "fp-to-i-if-check.exp.size", Parent: F, InsertBefore: End);
661 BasicBlock *ExpSmallBB =
662 BasicBlock::Create(Context&: Builder.getContext(), Name: "fp-to-i-if-exp.small", Parent: F, InsertBefore: End);
663 BasicBlock *ExpLargeBB =
664 BasicBlock::Create(Context&: Builder.getContext(), Name: "fp-to-i-if-exp.large", Parent: F, InsertBefore: End);
665
666 Entry->getTerminator()->eraseFromParent();
667
668 // entry:
669 Builder.SetInsertPoint(Entry);
670 // We're going to introduce branches on the value, so freeze it.
671 if (!isGuaranteedNotToBeUndefOrPoison(V: FloatVal))
672 FloatVal = Builder.CreateFreeze(V: FloatVal);
673 // fp80 conversion is implemented by fpext to fp128 first then do the
674 // conversion.
675 if (FloatVal->getType()->isX86_FP80Ty())
676 FloatVal =
677 Builder.CreateFPExt(V: FloatVal, DestTy: Type::getFP128Ty(C&: Builder.getContext()));
678 Value *ARep = Builder.CreateBitCast(V: FloatVal, DestTy: FloatIntTy);
679 Value *PosOrNeg, *Sign;
680 if (IsSigned) {
681 PosOrNeg =
682 Builder.CreateICmpSGT(LHS: ARep, RHS: ConstantInt::getSigned(Ty: FloatIntTy, V: -1));
683 Sign = Builder.CreateSelectWithUnknownProfile(
684 C: PosOrNeg, True: ConstantInt::getSigned(Ty: IntTy, V: 1),
685 False: ConstantInt::getSigned(Ty: IntTy, V: -1), PassName: "sign");
686 }
687 Value *And =
688 Builder.CreateLShr(LHS: ARep, RHS: Builder.getIntN(N: FloatWidth, C: FPMantissaWidth));
689 Value *BiasedExp = Builder.CreateAnd(
690 LHS: And, RHS: Builder.getIntN(N: FloatWidth, C: (1 << ExponentWidth) - 1), Name: "biased.exp");
691 Value *Abs = Builder.CreateAnd(LHS: ARep, RHS: SignificandMask);
692 Value *Significand = Builder.CreateOr(LHS: Abs, RHS: ImplicitBit, Name: "significand");
693 Value *ZeroResultCond = Builder.CreateICmpULT(
694 LHS: BiasedExp, RHS: Builder.getIntN(N: FloatWidth, C: ExponentBias), Name: "exp.is.negative");
695 if (IsSaturating) {
696 Value *IsNaN = Builder.CreateFCmpUNO(LHS: FloatVal, RHS: FloatVal, Name: "is.nan");
697 ZeroResultCond = Builder.CreateOr(LHS: ZeroResultCond, RHS: IsNaN);
698 if (!IsSigned) {
699 Value *IsNeg = Builder.CreateIsNeg(Arg: ARep);
700 ZeroResultCond = Builder.CreateOr(LHS: ZeroResultCond, RHS: IsNeg);
701 }
702 }
703 Instruction *CondBr = Builder.CreateCondBr(
704 Cond: ZeroResultCond, True: End, False: IsSaturating ? CheckSaturateBB : CheckExpSizeBB);
705 // We do not have any information on the value of the exponent, so mark the
706 // branch weights as unkown.
707 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *CondBr, DEBUG_TYPE, F);
708
709 Value *Saturated;
710 if (IsSaturating) {
711 // check.saturate:
712 Builder.SetInsertPoint(CheckSaturateBB);
713 uint64_t SaturatingBiasedExp =
714 static_cast<uint64_t>(ExponentBias) + BitWidth - IsSigned;
715 // Clamp to the all-ones (inf/NaN) exponent. Without this, when the integer
716 // is wide enough to hold every finite float the threshold exceeds any
717 // possible biased exponent, so +/-inf would never saturate.
718 uint64_t MaxBiasedExp = (1ULL << ExponentWidth) - 1;
719 if (SaturatingBiasedExp > MaxBiasedExp)
720 SaturatingBiasedExp = MaxBiasedExp;
721 Value *Cmp3 = Builder.CreateICmpUGE(
722 LHS: BiasedExp, RHS: ConstantInt::get(Ty: FloatIntTy, V: SaturatingBiasedExp));
723 Value *CondBrSat = Builder.CreateCondBr(Cond: Cmp3, True: SaturateBB, False: CheckExpSizeBB);
724 // Saturation is considered an unlikely event.
725 applyProfMetadataIfEnabled(V: CondBrSat, setMetadataCallback: [&](Instruction *Inst) {
726 Inst->setMetadata(
727 KindID: LLVMContext::MD_prof,
728 Node: MDBuilder(Inst->getContext()).createUnlikelyBranchWeights());
729 });
730
731 // saturate:
732 Builder.SetInsertPoint(SaturateBB);
733 if (IsSigned) {
734 Value *SignedMax =
735 ConstantInt::get(Ty: IntTy, V: APInt::getSignedMaxValue(numBits: BitWidth));
736 Value *SignedMin =
737 ConstantInt::get(Ty: IntTy, V: APInt::getSignedMinValue(numBits: BitWidth));
738 // Select between the signed max and min values for saturation.
739 Saturated = Builder.CreateSelectWithUnknownProfile(
740 C: PosOrNeg, True: SignedMax, False: SignedMin, PassName: "saturated");
741 } else {
742 Saturated = ConstantInt::getAllOnesValue(Ty: IntTy);
743 }
744 Builder.CreateBr(Dest: End);
745 }
746
747 // if.end9:
748 Builder.SetInsertPoint(CheckExpSizeBB);
749 Value *ExpSmallerMantissaWidth = Builder.CreateICmpULT(
750 LHS: BiasedExp, RHS: Builder.getIntN(N: FloatWidth, C: ExponentBias + FPMantissaWidth),
751 Name: "exp.smaller.mantissa.width");
752 // We cannot determine whether this is a left shift or a right shift,
753 // so we mark the branch weights as unknown.
754 Value *CondBr2 =
755 Builder.CreateCondBr(Cond: ExpSmallerMantissaWidth, True: ExpSmallBB, False: ExpLargeBB);
756 applyProfMetadataIfEnabled(V: CondBr2, setMetadataCallback: [&](Instruction *Inst) {
757 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *Inst, DEBUG_TYPE, F);
758 });
759
760 // exp.small:
761 Builder.SetInsertPoint(ExpSmallBB);
762 Value *Sub13 = Builder.CreateSub(
763 LHS: Builder.getIntN(N: FloatWidth, C: ExponentBias + FPMantissaWidth), RHS: BiasedExp);
764 Value *ExpSmallRes =
765 Builder.CreateZExtOrTrunc(V: Builder.CreateLShr(LHS: Significand, RHS: Sub13), DestTy: IntTy);
766 if (IsSigned)
767 ExpSmallRes = Builder.CreateMul(LHS: ExpSmallRes, RHS: Sign);
768 Builder.CreateBr(Dest: End);
769
770 // exp.large:
771 Builder.SetInsertPoint(ExpLargeBB);
772 Value *Sub15 = Builder.CreateAdd(
773 LHS: BiasedExp,
774 RHS: ConstantInt::getSigned(
775 Ty: FloatIntTy, V: -static_cast<int64_t>(ExponentBias + FPMantissaWidth)));
776 Value *SignificandCast = Builder.CreateZExtOrTrunc(V: Significand, DestTy: IntTy);
777 Value *ExpLargeRes = Builder.CreateShl(
778 LHS: SignificandCast, RHS: Builder.CreateZExtOrTrunc(V: Sub15, DestTy: IntTy));
779 if (IsSigned)
780 ExpLargeRes = Builder.CreateMul(LHS: ExpLargeRes, RHS: Sign);
781 Builder.CreateBr(Dest: End);
782
783 // cleanup:
784 Builder.SetInsertPoint(End->begin());
785 PHINode *Retval0 = Builder.CreatePHI(Ty: FPToI->getType(), NumReservedValues: 3 + IsSaturating);
786
787 if (IsSaturating)
788 Retval0->addIncoming(V: Saturated, BB: SaturateBB);
789 Retval0->addIncoming(V: ExpSmallRes, BB: ExpSmallBB);
790 Retval0->addIncoming(V: ExpLargeRes, BB: ExpLargeBB);
791 Retval0->addIncoming(V: Builder.getIntN(N: BitWidth, C: 0), BB: Entry);
792
793 FPToI->replaceAllUsesWith(V: Retval0);
794 FPToI->dropAllReferences();
795 FPToI->eraseFromParent();
796}
797
798// clang-format off: preserve formatting of the following example
799
800/// Generate code to convert a fp number to integer, replacing S(U)IToFP with
801/// the generated code. This currently generates code similarly to compiler-rt's
802/// implementations. This implementation has an implicit assumption that integer
803/// width is larger than fp.
804///
805/// An example IR generated from compiler-rt/floatdisf.c looks like below:
806/// define dso_local float @__floatdisf(i64 noundef %a) local_unnamed_addr #0 {
807/// entry:
808/// %cmp = icmp eq i64 %a, 0
809/// br i1 %cmp, label %return, label %if.end
810///
811/// if.end: ; preds = %entry
812/// %shr = ashr i64 %a, 63
813/// %xor = xor i64 %shr, %a
814/// %sub = sub nsw i64 %xor, %shr
815/// %0 = tail call i64 @llvm.ctlz.i64(i64 %sub, i1 true), !range !5
816/// %cast = trunc i64 %0 to i32
817/// %sub1 = sub nuw nsw i32 64, %cast
818/// %sub2 = xor i32 %cast, 63
819/// %cmp3 = icmp ult i32 %cast, 40
820/// br i1 %cmp3, label %if.then4, label %if.else
821///
822/// if.then4: ; preds = %if.end
823/// switch i32 %sub1, label %sw.default [
824/// i32 25, label %sw.bb
825/// i32 26, label %sw.epilog
826/// ]
827///
828/// sw.bb: ; preds = %if.then4
829/// %shl = shl i64 %sub, 1
830/// br label %sw.epilog
831///
832/// sw.default: ; preds = %if.then4
833/// %sub5 = sub nsw i64 38, %0
834/// %sh_prom = and i64 %sub5, 4294967295
835/// %shr6 = lshr i64 %sub, %sh_prom
836/// %shr9 = lshr i64 274877906943, %0
837/// %and = and i64 %shr9, %sub
838/// %cmp10 = icmp ne i64 %and, 0
839/// %conv11 = zext i1 %cmp10 to i64
840/// %or = or i64 %shr6, %conv11
841/// br label %sw.epilog
842///
843/// sw.epilog: ; preds = %sw.default,
844/// %if.then4, %sw.bb
845/// %a.addr.0 = phi i64 [ %or, %sw.default ], [ %sub, %if.then4 ], [ %shl,
846/// %sw.bb ] %1 = lshr i64 %a.addr.0, 2 %2 = and i64 %1, 1 %or16 = or i64 %2,
847/// %a.addr.0 %inc = add nsw i64 %or16, 1 %3 = and i64 %inc, 67108864
848/// %tobool.not = icmp eq i64 %3, 0
849/// %spec.select.v = select i1 %tobool.not, i64 2, i64 3
850/// %spec.select = ashr i64 %inc, %spec.select.v
851/// %spec.select56 = select i1 %tobool.not, i32 %sub2, i32 %sub1
852/// br label %if.end26
853///
854/// if.else: ; preds = %if.end
855/// %sub23 = add nuw nsw i64 %0, 4294967256
856/// %sh_prom24 = and i64 %sub23, 4294967295
857/// %shl25 = shl i64 %sub, %sh_prom24
858/// br label %if.end26
859///
860/// if.end26: ; preds = %sw.epilog,
861/// %if.else
862/// %a.addr.1 = phi i64 [ %shl25, %if.else ], [ %spec.select, %sw.epilog ]
863/// %e.0 = phi i32 [ %sub2, %if.else ], [ %spec.select56, %sw.epilog ]
864/// %conv27 = trunc i64 %shr to i32
865/// %and28 = and i32 %conv27, -2147483648
866/// %add = shl nuw nsw i32 %e.0, 23
867/// %shl29 = add nuw nsw i32 %add, 1065353216
868/// %conv31 = trunc i64 %a.addr.1 to i32
869/// %and32 = and i32 %conv31, 8388607
870/// %or30 = or i32 %and32, %and28
871/// %or33 = or i32 %or30, %shl29
872/// %4 = bitcast i32 %or33 to float
873/// br label %return
874///
875/// return: ; preds = %entry,
876/// %if.end26
877/// %retval.0 = phi float [ %4, %if.end26 ], [ 0.000000e+00, %entry ]
878/// ret float %retval.0
879/// }
880///
881/// Replace integer to fp with generated code.
882static void expandIToFP(Instruction *IToFP) {
883 // clang-format on
884 IRBuilder<> Builder(IToFP);
885 auto *IntVal = IToFP->getOperand(i: 0);
886 IntegerType *IntTy = cast<IntegerType>(Val: IntVal->getType());
887
888 unsigned BitWidth = IntVal->getType()->getIntegerBitWidth();
889 unsigned FPMantissaWidth = IToFP->getType()->getFPMantissaWidth() - 1;
890 // fp80 conversion is implemented by conversion tp fp128 first following
891 // a fptrunc to fp80.
892 FPMantissaWidth = FPMantissaWidth == 63 ? 112 : FPMantissaWidth;
893 // FIXME: As there is no related builtins added in compliler-rt,
894 // here currently utilized the fp32 <-> fp16 lib calls to implement.
895 FPMantissaWidth = FPMantissaWidth == 10 ? 23 : FPMantissaWidth;
896 FPMantissaWidth = FPMantissaWidth == 7 ? 23 : FPMantissaWidth;
897 unsigned FloatWidth = PowerOf2Ceil(A: FPMantissaWidth);
898 bool IsSigned = IToFP->getOpcode() == Instruction::SIToFP;
899
900 // We're going to introduce branches on the value, so freeze it.
901 if (!isGuaranteedNotToBeUndefOrPoison(V: IntVal))
902 IntVal = Builder.CreateFreeze(V: IntVal);
903
904 // The expansion below assumes that int width >= float width. Zero or sign
905 // extend the integer accordingly.
906 if (BitWidth < FloatWidth) {
907 BitWidth = FloatWidth;
908 IntTy = Builder.getIntNTy(N: BitWidth);
909 IntVal = Builder.CreateIntCast(V: IntVal, DestTy: IntTy, isSigned: IsSigned);
910 }
911
912 Value *Temp1 =
913 Builder.CreateShl(LHS: Builder.getIntN(N: BitWidth, C: 1),
914 RHS: Builder.getIntN(N: BitWidth, C: FPMantissaWidth + 3));
915
916 BasicBlock *Entry = Builder.GetInsertBlock();
917 Function *F = Entry->getParent();
918 Entry->setName(Twine(Entry->getName(), "itofp-entry"));
919 BasicBlock *End =
920 Entry->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "itofp-return");
921 BasicBlock *IfEnd =
922 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-if-end", Parent: F, InsertBefore: End);
923 BasicBlock *IfThen4 =
924 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-if-then4", Parent: F, InsertBefore: End);
925 BasicBlock *SwBB =
926 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-sw-bb", Parent: F, InsertBefore: End);
927 BasicBlock *SwDefault =
928 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-sw-default", Parent: F, InsertBefore: End);
929 BasicBlock *SwEpilog =
930 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-sw-epilog", Parent: F, InsertBefore: End);
931 BasicBlock *IfThen20 =
932 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-if-then20", Parent: F, InsertBefore: End);
933 BasicBlock *IfElse =
934 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-if-else", Parent: F, InsertBefore: End);
935 BasicBlock *IfEnd26 =
936 BasicBlock::Create(Context&: Builder.getContext(), Name: "itofp-if-end26", Parent: F, InsertBefore: End);
937
938 Entry->getTerminator()->eraseFromParent();
939
940 Function *CTLZ =
941 Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: Intrinsic::ctlz, OverloadTys: IntTy);
942 ConstantInt *True = Builder.getTrue();
943
944 // entry:
945 Builder.SetInsertPoint(Entry);
946 // We assume that the zero is an unlikely input case, so the branch to 'End'
947 // is the unlikely path.
948 Value *Cmp = Builder.CreateICmpEQ(LHS: IntVal, RHS: ConstantInt::getSigned(Ty: IntTy, V: 0));
949 Value *CondBrEntry = Builder.CreateCondBr(Cond: Cmp, True: End, False: IfEnd);
950 applyProfMetadataIfEnabled(V: CondBrEntry, setMetadataCallback: [&](Instruction *Inst) {
951 Inst->setMetadata(
952 KindID: LLVMContext::MD_prof,
953 Node: MDBuilder(Inst->getContext()).createUnlikelyBranchWeights());
954 });
955
956 // if.end:
957 Builder.SetInsertPoint(IfEnd);
958 Value *Shr =
959 Builder.CreateAShr(LHS: IntVal, RHS: Builder.getIntN(N: BitWidth, C: BitWidth - 1));
960 Value *Xor = Builder.CreateXor(LHS: Shr, RHS: IntVal);
961 Value *Sub = Builder.CreateSub(LHS: Xor, RHS: Shr);
962 Value *Call = Builder.CreateCall(Callee: CTLZ, Args: {IsSigned ? Sub : IntVal, True});
963 Value *Cast = Builder.CreateTrunc(V: Call, DestTy: Builder.getInt32Ty());
964 int BitWidthNew = FloatWidth == 128 ? BitWidth : 32;
965 Value *Sub1 = Builder.CreateSub(LHS: Builder.getIntN(N: BitWidthNew, C: BitWidth),
966 RHS: FloatWidth == 128 ? Call : Cast);
967 Value *Sub2 = Builder.CreateSub(LHS: Builder.getIntN(N: BitWidthNew, C: BitWidth - 1),
968 RHS: FloatWidth == 128 ? Call : Cast);
969 Value *Cmp3 = Builder.CreateICmpSGT(
970 LHS: Sub1, RHS: Builder.getIntN(N: BitWidthNew, C: FPMantissaWidth + 1));
971 // This branch handles the rare case where rounding the mantissa causes a
972 // carry-out at the most significant bit, necessitating an increment of the
973 // exponent. This is rare case, so the True path is mared as likely.
974 Value *CondBrIfEnd = Builder.CreateCondBr(Cond: Cmp3, True: IfThen4, False: IfElse);
975 applyProfMetadataIfEnabled(V: CondBrIfEnd, setMetadataCallback: [&](Instruction *Inst) {
976 Inst->setMetadata(
977 KindID: LLVMContext::MD_prof,
978 Node: MDBuilder(Inst->getContext()).createLikelyBranchWeights());
979 });
980
981 // if.then4:
982 Builder.SetInsertPoint(IfThen4);
983 SwitchInst *SI = Builder.CreateSwitch(V: Sub1, Dest: SwDefault);
984 SI->addCase(OnVal: Builder.getIntN(N: BitWidthNew, C: FPMantissaWidth + 2), Dest: SwBB);
985 SI->addCase(OnVal: Builder.getIntN(N: BitWidthNew, C: FPMantissaWidth + 3), Dest: SwEpilog);
986 // Add branch weights to the SwitchInst. The weights are provided for the
987 // default case first (SwDefault), followed by each explicit case in the
988 // order they were added (SwBB, then SwEpilog). Because the following cases
989 // are rare, the defalut case is given a likely weight.
990 if (!ProfcheckDisableMetadataFixes) {
991 SI->setMetadata(
992 KindID: LLVMContext::MD_prof,
993 Node: MDBuilder(SI->getContext())
994 .createBranchWeights(Weights: {llvm::MDBuilder::kLikelyBranchWeight,
995 llvm::MDBuilder::kUnlikelyBranchWeight,
996 llvm::MDBuilder::kUnlikelyBranchWeight}));
997 }
998
999 // sw.bb:
1000 Builder.SetInsertPoint(SwBB);
1001 Value *Shl =
1002 Builder.CreateShl(LHS: IsSigned ? Sub : IntVal, RHS: Builder.getIntN(N: BitWidth, C: 1));
1003 Builder.CreateBr(Dest: SwEpilog);
1004
1005 // sw.default:
1006 Builder.SetInsertPoint(SwDefault);
1007 Value *Sub5 = Builder.CreateSub(
1008 LHS: Builder.getIntN(N: BitWidthNew, C: BitWidth - FPMantissaWidth - 3),
1009 RHS: FloatWidth == 128 ? Call : Cast);
1010 Value *ShProm = Builder.CreateZExt(V: Sub5, DestTy: IntTy);
1011 Value *Shr6 = Builder.CreateLShr(LHS: IsSigned ? Sub : IntVal,
1012 RHS: FloatWidth == 128 ? Sub5 : ShProm);
1013 Value *Sub8 =
1014 Builder.CreateAdd(LHS: FloatWidth == 128 ? Call : Cast,
1015 RHS: Builder.getIntN(N: BitWidthNew, C: FPMantissaWidth + 3));
1016 Value *ShProm9 = Builder.CreateZExt(V: Sub8, DestTy: IntTy);
1017 Value *Shr9 = Builder.CreateLShr(LHS: ConstantInt::getSigned(Ty: IntTy, V: -1),
1018 RHS: FloatWidth == 128 ? Sub8 : ShProm9);
1019 Value *And = Builder.CreateAnd(LHS: Shr9, RHS: IsSigned ? Sub : IntVal);
1020 Value *Cmp10 = Builder.CreateICmpNE(LHS: And, RHS: Builder.getIntN(N: BitWidth, C: 0));
1021 Value *Conv11 = Builder.CreateZExt(V: Cmp10, DestTy: IntTy);
1022 Value *Or = Builder.CreateOr(LHS: Shr6, RHS: Conv11);
1023 Builder.CreateBr(Dest: SwEpilog);
1024
1025 // sw.epilog:
1026 Builder.SetInsertPoint(SwEpilog);
1027 PHINode *AAddr0 = Builder.CreatePHI(Ty: IntTy, NumReservedValues: 3);
1028 AAddr0->addIncoming(V: Or, BB: SwDefault);
1029 AAddr0->addIncoming(V: IsSigned ? Sub : IntVal, BB: IfThen4);
1030 AAddr0->addIncoming(V: Shl, BB: SwBB);
1031 Value *A0 = Builder.CreateTrunc(V: AAddr0, DestTy: Builder.getInt32Ty());
1032 Value *A1 = Builder.CreateLShr(LHS: A0, RHS: Builder.getInt32(C: 2));
1033 Value *A2 = Builder.CreateAnd(LHS: A1, RHS: Builder.getInt32(C: 1));
1034 Value *Conv16 = Builder.CreateZExt(V: A2, DestTy: IntTy);
1035 Value *Or17 = Builder.CreateOr(LHS: AAddr0, RHS: Conv16);
1036 Value *Inc = Builder.CreateAdd(LHS: Or17, RHS: Builder.getIntN(N: BitWidth, C: 1));
1037 Value *Shr18 = nullptr;
1038 if (IsSigned)
1039 Shr18 = Builder.CreateAShr(LHS: Inc, RHS: Builder.getIntN(N: BitWidth, C: 2));
1040 else
1041 Shr18 = Builder.CreateLShr(LHS: Inc, RHS: Builder.getIntN(N: BitWidth, C: 2));
1042 Value *A3 = Builder.CreateAnd(LHS: Inc, RHS: Temp1, Name: "a3");
1043 Value *PosOrNeg = Builder.CreateICmpEQ(LHS: A3, RHS: Builder.getIntN(N: BitWidth, C: 0));
1044 Value *ExtractT60 = Builder.CreateTrunc(V: Shr18, DestTy: Builder.getIntNTy(N: FloatWidth));
1045 Value *Extract63 = Builder.CreateLShr(LHS: Shr18, RHS: Builder.getIntN(N: BitWidth, C: 32));
1046 Value *ExtractT64 = nullptr;
1047 if (FloatWidth > 80)
1048 ExtractT64 = Builder.CreateTrunc(V: Sub2, DestTy: Builder.getInt64Ty());
1049 else
1050 ExtractT64 = Builder.CreateTrunc(V: Extract63, DestTy: Builder.getInt32Ty());
1051 // Rounding usually keeps the exponent within its current magnitude and
1052 // overflow is rare. The False path is unlikely to be taken.
1053 Value *CondBrSwEpilog = Builder.CreateCondBr(Cond: PosOrNeg, True: IfEnd26, False: IfThen20);
1054 applyProfMetadataIfEnabled(V: CondBrSwEpilog, setMetadataCallback: [&](Instruction *Inst) {
1055 Inst->setMetadata(
1056 KindID: LLVMContext::MD_prof,
1057 Node: MDBuilder(Inst->getContext()).createLikelyBranchWeights());
1058 });
1059
1060 // if.then20
1061 Builder.SetInsertPoint(IfThen20);
1062 Value *Shr21 = nullptr;
1063 if (IsSigned)
1064 Shr21 = Builder.CreateAShr(LHS: Inc, RHS: Builder.getIntN(N: BitWidth, C: 3));
1065 else
1066 Shr21 = Builder.CreateLShr(LHS: Inc, RHS: Builder.getIntN(N: BitWidth, C: 3));
1067 Value *ExtractT = Builder.CreateTrunc(V: Shr21, DestTy: Builder.getIntNTy(N: FloatWidth));
1068 Value *Extract = Builder.CreateLShr(LHS: Shr21, RHS: Builder.getIntN(N: BitWidth, C: 32));
1069 Value *ExtractT62 = nullptr;
1070 if (FloatWidth > 80)
1071 ExtractT62 = Builder.CreateTrunc(V: Sub1, DestTy: Builder.getInt64Ty());
1072 else
1073 ExtractT62 = Builder.CreateTrunc(V: Extract, DestTy: Builder.getInt32Ty());
1074 Builder.CreateBr(Dest: IfEnd26);
1075
1076 // if.else:
1077 Builder.SetInsertPoint(IfElse);
1078 Value *Sub24 = Builder.CreateAdd(
1079 LHS: FloatWidth == 128 ? Call : Cast,
1080 RHS: ConstantInt::getSigned(Ty: Builder.getIntNTy(N: BitWidthNew),
1081 V: -(int)(BitWidth - FPMantissaWidth - 1)));
1082 Value *ShProm25 = Builder.CreateZExt(V: Sub24, DestTy: IntTy);
1083 Value *Shl26 = Builder.CreateShl(LHS: IsSigned ? Sub : IntVal,
1084 RHS: FloatWidth == 128 ? Sub24 : ShProm25);
1085 Value *ExtractT61 = Builder.CreateTrunc(V: Shl26, DestTy: Builder.getIntNTy(N: FloatWidth));
1086 Value *Extract65 = Builder.CreateLShr(LHS: Shl26, RHS: Builder.getIntN(N: BitWidth, C: 32));
1087 Value *ExtractT66 = nullptr;
1088 if (FloatWidth > 80)
1089 ExtractT66 = Builder.CreateTrunc(V: Sub2, DestTy: Builder.getInt64Ty());
1090 else
1091 ExtractT66 = Builder.CreateTrunc(V: Extract65, DestTy: Builder.getInt32Ty());
1092 Builder.CreateBr(Dest: IfEnd26);
1093
1094 // if.end26:
1095 Builder.SetInsertPoint(IfEnd26);
1096 PHINode *AAddr1Off0 = Builder.CreatePHI(Ty: Builder.getIntNTy(N: FloatWidth), NumReservedValues: 3);
1097 AAddr1Off0->addIncoming(V: ExtractT, BB: IfThen20);
1098 AAddr1Off0->addIncoming(V: ExtractT60, BB: SwEpilog);
1099 AAddr1Off0->addIncoming(V: ExtractT61, BB: IfElse);
1100 PHINode *AAddr1Off32 = nullptr;
1101 if (FloatWidth > 32) {
1102 AAddr1Off32 =
1103 Builder.CreatePHI(Ty: Builder.getIntNTy(N: FloatWidth > 80 ? 64 : 32), NumReservedValues: 3);
1104 AAddr1Off32->addIncoming(V: ExtractT62, BB: IfThen20);
1105 AAddr1Off32->addIncoming(V: ExtractT64, BB: SwEpilog);
1106 AAddr1Off32->addIncoming(V: ExtractT66, BB: IfElse);
1107 }
1108 PHINode *E0 = nullptr;
1109 if (FloatWidth <= 80) {
1110 E0 = Builder.CreatePHI(Ty: Builder.getIntNTy(N: BitWidthNew), NumReservedValues: 3);
1111 E0->addIncoming(V: Sub1, BB: IfThen20);
1112 E0->addIncoming(V: Sub2, BB: SwEpilog);
1113 E0->addIncoming(V: Sub2, BB: IfElse);
1114 }
1115 Value *And29 = nullptr;
1116 if (FloatWidth > 80) {
1117 Value *Temp2 = Builder.CreateShl(LHS: Builder.getIntN(N: BitWidth, C: 1),
1118 RHS: Builder.getIntN(N: BitWidth, C: 63));
1119 And29 = Builder.CreateAnd(LHS: Shr, RHS: Temp2, Name: "and29");
1120 } else {
1121 Value *Conv28 = Builder.CreateTrunc(V: Shr, DestTy: Builder.getInt32Ty());
1122 And29 = Builder.CreateAnd(
1123 LHS: Conv28, RHS: ConstantInt::get(Context&: Builder.getContext(), V: APInt::getSignMask(BitWidth: 32)));
1124 }
1125 unsigned TempMod = FPMantissaWidth % 32;
1126 Value *And34 = nullptr;
1127 Value *Shl30 = nullptr;
1128 if (FloatWidth > 80) {
1129 TempMod += 32;
1130 Value *Add = Builder.CreateShl(LHS: AAddr1Off32, RHS: Builder.getInt64(C: TempMod));
1131 Shl30 = Builder.CreateAdd(
1132 LHS: Add, RHS: Builder.getInt64(C: ((1ull << (62ull - TempMod)) - 1ull) << TempMod));
1133 And34 = Builder.CreateZExt(V: Shl30, DestTy: Builder.getInt128Ty());
1134 } else {
1135 Value *Add = Builder.CreateShl(LHS: E0, RHS: Builder.getInt32(C: TempMod));
1136 Shl30 = Builder.CreateAdd(
1137 LHS: Add, RHS: Builder.getInt32(C: ((1 << (30 - TempMod)) - 1) << TempMod));
1138 And34 = Builder.CreateAnd(LHS: FloatWidth > 32 ? AAddr1Off32 : AAddr1Off0,
1139 RHS: Builder.getInt32(C: (1 << TempMod) - 1));
1140 }
1141 Value *Or35 = nullptr;
1142 if (FloatWidth > 80) {
1143 Value *And29Trunc = Builder.CreateTrunc(V: And29, DestTy: Builder.getInt128Ty());
1144 Value *Or31 = Builder.CreateOr(LHS: And29Trunc, RHS: And34);
1145 Value *Or34 = Builder.CreateShl(LHS: Or31, RHS: Builder.getIntN(N: 128, C: 64));
1146 Value *Temp3 = Builder.CreateShl(LHS: Builder.getIntN(N: 128, C: 1),
1147 RHS: Builder.getIntN(N: 128, C: FPMantissaWidth));
1148 Value *Temp4 = Builder.CreateSub(LHS: Temp3, RHS: Builder.getIntN(N: 128, C: 1));
1149 Value *A6 = Builder.CreateAnd(LHS: AAddr1Off0, RHS: Temp4);
1150 Or35 = Builder.CreateOr(LHS: Or34, RHS: A6);
1151 } else {
1152 Value *Or31 = Builder.CreateOr(LHS: And34, RHS: And29);
1153 Or35 = Builder.CreateOr(LHS: IsSigned ? Or31 : And34, RHS: Shl30);
1154 }
1155 Value *A4 = nullptr;
1156 if (IToFP->getType()->isDoubleTy()) {
1157 Value *ZExt1 = Builder.CreateZExt(V: Or35, DestTy: Builder.getIntNTy(N: FloatWidth));
1158 Value *Shl1 = Builder.CreateShl(LHS: ZExt1, RHS: Builder.getIntN(N: FloatWidth, C: 32));
1159 Value *And1 =
1160 Builder.CreateAnd(LHS: AAddr1Off0, RHS: Builder.getIntN(N: FloatWidth, C: 0xFFFFFFFF));
1161 Value *Or1 = Builder.CreateOr(LHS: Shl1, RHS: And1);
1162 A4 = Builder.CreateBitCast(V: Or1, DestTy: IToFP->getType());
1163 } else if (IToFP->getType()->isX86_FP80Ty()) {
1164 Value *A40 =
1165 Builder.CreateBitCast(V: Or35, DestTy: Type::getFP128Ty(C&: Builder.getContext()));
1166 A4 = Builder.CreateFPTrunc(V: A40, DestTy: IToFP->getType());
1167 } else if (IToFP->getType()->isHalfTy() || IToFP->getType()->isBFloatTy()) {
1168 // Deal with "half" situation. This is a workaround since we don't have
1169 // floattihf.c currently as referring.
1170 Value *A40 =
1171 Builder.CreateBitCast(V: Or35, DestTy: Type::getFloatTy(C&: Builder.getContext()));
1172 A4 = Builder.CreateFPTrunc(V: A40, DestTy: IToFP->getType());
1173 } else // float type
1174 A4 = Builder.CreateBitCast(V: Or35, DestTy: IToFP->getType());
1175
1176 // Sub2 is the unbiased exponent (the index of the top set bit in the input).
1177 // The exponent arithmetic above wraps to garbage instead of inf once it
1178 // overflows the exponent field, so saturate to a correctly-signed infinity
1179 // when Sub2 reaches 1 << (ExponentWidth - 1). Sub2 is at most BitWidth - 1,
1180 // so skip the check entirely when even that can't reach the threshold.
1181 // (Values that round *up* into inf, e.g. 2^n - 1, keep Sub2 = BitWidth - 1;
1182 // these are handled by the conversion's own rounding, not by this
1183 // saturation.)
1184 unsigned ExponentWidth = FloatWidth - FPMantissaWidth - 1;
1185 uint64_t MinInfExp = 1ULL << (ExponentWidth - 1);
1186 if (BitWidth - 1 >= MinInfExp) {
1187 Value *MinInfExpVal = Builder.getIntN(N: BitWidthNew, C: MinInfExp);
1188 Value *Overflow = Builder.CreateICmpUGE(LHS: Sub2, RHS: MinInfExpVal);
1189 Value *Inf = ConstantFP::getInfinity(Ty: IToFP->getType(), /*Negative=*/false);
1190 if (IsSigned) {
1191 Value *NegInf =
1192 ConstantFP::getInfinity(Ty: IToFP->getType(), /*Negative=*/true);
1193 Value *IsNeg =
1194 Builder.CreateICmpSLT(LHS: IntVal, RHS: ConstantInt::getNullValue(Ty: IntTy));
1195 Inf = Builder.CreateSelectWithUnknownProfile(C: IsNeg, True: NegInf, False: Inf,
1196 DEBUG_TYPE);
1197 }
1198 A4 = Builder.CreateSelect(C: Overflow, True: Inf, False: A4);
1199 // We consider overflow to be an unlikely case.
1200 applyProfMetadataIfEnabled(V: A4, setMetadataCallback: [&](Instruction *Inst) {
1201 Inst->setMetadata(
1202 KindID: LLVMContext::MD_prof,
1203 Node: MDBuilder(Inst->getContext()).createUnlikelyBranchWeights());
1204 });
1205 }
1206 Builder.CreateBr(Dest: End);
1207
1208 // return:
1209 Builder.SetInsertPoint(End->begin());
1210 PHINode *Retval0 = Builder.CreatePHI(Ty: IToFP->getType(), NumReservedValues: 2);
1211 Retval0->addIncoming(V: A4, BB: IfEnd26);
1212 Retval0->addIncoming(V: ConstantFP::getZero(Ty: IToFP->getType(), Negative: false), BB: Entry);
1213
1214 IToFP->replaceAllUsesWith(V: Retval0);
1215 IToFP->dropAllReferences();
1216 IToFP->eraseFromParent();
1217}
1218
1219static void scalarize(Instruction *I,
1220 SmallVectorImpl<Instruction *> &Worklist) {
1221 VectorType *VTy = cast<FixedVectorType>(Val: I->getType());
1222
1223 IRBuilder<> Builder(I);
1224
1225 unsigned NumElements = VTy->getElementCount().getFixedValue();
1226 Value *Result = PoisonValue::get(T: VTy);
1227 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
1228 Value *Ext = Builder.CreateExtractElement(Vec: I->getOperand(i: 0), Idx);
1229
1230 Value *NewOp = nullptr;
1231 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I))
1232 NewOp = Builder.CreateBinOp(
1233 Opc: BinOp->getOpcode(), LHS: Ext,
1234 RHS: Builder.CreateExtractElement(Vec: I->getOperand(i: 1), Idx));
1235 else if (auto *CastI = dyn_cast<CastInst>(Val: I))
1236 NewOp = Builder.CreateCast(Op: CastI->getOpcode(), V: Ext,
1237 DestTy: I->getType()->getScalarType());
1238 else if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
1239 assert(II->getIntrinsicID() == Intrinsic::fptoui_sat ||
1240 II->getIntrinsicID() == Intrinsic::fptosi_sat);
1241 NewOp = Builder.CreateIntrinsic(RetTy: I->getType()->getScalarType(),
1242 ID: II->getIntrinsicID(), Args: {Ext});
1243 } else
1244 llvm_unreachable("Unsupported instruction type");
1245
1246 Result = Builder.CreateInsertElement(Vec: Result, NewElt: NewOp, Idx);
1247 if (auto *ScalarizedI = dyn_cast<Instruction>(Val: NewOp)) {
1248 ScalarizedI->copyIRFlags(V: I, IncludeWrapFlags: true);
1249 Worklist.push_back(Elt: ScalarizedI);
1250 }
1251 }
1252
1253 I->replaceAllUsesWith(V: Result);
1254 I->dropAllReferences();
1255 I->eraseFromParent();
1256}
1257
1258static void addToWorklist(Instruction &I,
1259 SmallVector<Instruction *, 4> &Worklist) {
1260 if (I.getOperand(i: 0)->getType()->isVectorTy())
1261 scalarize(I: &I, Worklist);
1262 else
1263 Worklist.push_back(Elt: &I);
1264}
1265
1266static bool runImpl(Function &F, const TargetLowering &TLI,
1267 const LibcallLoweringInfo &Libcalls, AssumptionCache *AC) {
1268 SmallVector<Instruction *, 4> Worklist;
1269
1270 const CodeGenOptions &Opts = getCodeGenOptions(F);
1271 unsigned MaxLegalFpConvertBitWidth = Opts.expand_fp_convert_bits.value_or(
1272 u: TLI.getMaxLargeFPConvertBitWidthSupported());
1273 unsigned MaxLegalDivRemBitWidth =
1274 Opts.expand_div_rem_bits.value_or(u: TLI.getMaxDivRemBitWidthSupported());
1275
1276 bool DisableExpandLargeFp =
1277 MaxLegalFpConvertBitWidth >= IntegerType::MAX_INT_BITS;
1278 bool DisableExpandLargeDivRem =
1279 MaxLegalDivRemBitWidth >= IntegerType::MAX_INT_BITS;
1280 bool DisableFrem = !FRemExpander::shouldExpandAnyFremType(TLI, Libcalls);
1281
1282 if (DisableExpandLargeFp && DisableFrem && DisableExpandLargeDivRem)
1283 return false;
1284
1285 auto ShouldHandleInst = [&](Instruction &I) {
1286 Type *Ty = I.getType();
1287 // TODO: This pass doesn't handle scalable vectors.
1288 if (Ty->isScalableTy())
1289 return false;
1290
1291 switch (I.getOpcode()) {
1292 case Instruction::FRem:
1293 return !DisableFrem &&
1294 FRemExpander::shouldExpandFremType(TLI, Libcalls, Ty);
1295 case Instruction::FPToUI:
1296 case Instruction::FPToSI:
1297 return !DisableExpandLargeFp &&
1298 cast<IntegerType>(Val: Ty->getScalarType())->getIntegerBitWidth() >
1299 MaxLegalFpConvertBitWidth;
1300 case Instruction::UIToFP:
1301 case Instruction::SIToFP:
1302 return !DisableExpandLargeFp &&
1303 cast<IntegerType>(Val: I.getOperand(i: 0)->getType()->getScalarType())
1304 ->getIntegerBitWidth() > MaxLegalFpConvertBitWidth;
1305 case Instruction::UDiv:
1306 case Instruction::SDiv:
1307 case Instruction::URem:
1308 case Instruction::SRem:
1309 // Power-of-2 divisors are handled inside the expansion (via efficient
1310 // shift/mask sequences) rather than being excluded here, so that
1311 // backends that cannot lower wide div/rem even for powers of two
1312 // (e.g. when DAGCombiner is disabled) still get valid lowered code.
1313 return !DisableExpandLargeDivRem &&
1314 cast<IntegerType>(Val: Ty->getScalarType())->getIntegerBitWidth() >
1315 MaxLegalDivRemBitWidth;
1316 case Instruction::Call: {
1317 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
1318 if (II && (II->getIntrinsicID() == Intrinsic::fptoui_sat ||
1319 II->getIntrinsicID() == Intrinsic::fptosi_sat)) {
1320 return !DisableExpandLargeFp &&
1321 cast<IntegerType>(Val: Ty->getScalarType())->getIntegerBitWidth() >
1322 MaxLegalFpConvertBitWidth;
1323 }
1324 return false;
1325 }
1326 }
1327
1328 return false;
1329 };
1330
1331 bool Modified = false;
1332 for (auto It = inst_begin(F: &F), End = inst_end(F); It != End;) {
1333 Instruction &I = *It++;
1334 if (!ShouldHandleInst(I))
1335 continue;
1336
1337 addToWorklist(I, Worklist);
1338 Modified = true;
1339 }
1340
1341 while (!Worklist.empty()) {
1342 Instruction *I = Worklist.pop_back_val();
1343
1344 switch (I->getOpcode()) {
1345 case Instruction::FRem: {
1346 auto SQ = [&]() -> std::optional<SimplifyQuery> {
1347 if (AC) {
1348 auto Res = std::make_optional<SimplifyQuery>(args: I->getDataLayout(), args&: I);
1349 Res->AC = AC;
1350 return Res;
1351 }
1352 return {};
1353 }();
1354
1355 expandFRem(I&: cast<BinaryOperator>(Val&: *I), SQ);
1356 break;
1357 }
1358
1359 case Instruction::FPToUI:
1360 expandFPToI(FPToI: I, /*IsSaturating=*/false, /*IsSigned=*/false);
1361 break;
1362 case Instruction::FPToSI:
1363 expandFPToI(FPToI: I, /*IsSaturating=*/false, /*IsSigned=*/true);
1364 break;
1365
1366 case Instruction::UIToFP:
1367 case Instruction::SIToFP:
1368 expandIToFP(IToFP: I);
1369 break;
1370
1371 case Instruction::UDiv:
1372 case Instruction::SDiv:
1373 case Instruction::URem:
1374 case Instruction::SRem: {
1375 auto *BO = cast<BinaryOperator>(Val: I);
1376 // TODO: isConstantPowerOfTwo does not handle vector constants, so
1377 // vector div/rem by a power-of-2 splat goes through the generic path.
1378 if (isConstantPowerOfTwo(V: BO->getOperand(i_nocapture: 1), SignedOp: isSigned(Opcode: BO->getOpcode()))) {
1379 expandPow2DivRem(BO);
1380 } else {
1381 unsigned Opc = BO->getOpcode();
1382 if (Opc == Instruction::UDiv || Opc == Instruction::SDiv)
1383 expandDivision(Div: BO);
1384 else
1385 expandRemainder(Rem: BO);
1386 }
1387 break;
1388 }
1389 case Instruction::Call: {
1390 auto *II = cast<IntrinsicInst>(Val: I);
1391 assert(II->getIntrinsicID() == Intrinsic::fptoui_sat ||
1392 II->getIntrinsicID() == Intrinsic::fptosi_sat);
1393 expandFPToI(FPToI: I, /*IsSaturating=*/true,
1394 /*IsSigned=*/II->getIntrinsicID() == Intrinsic::fptosi_sat);
1395 break;
1396 }
1397 }
1398 }
1399
1400 return Modified;
1401}
1402
1403namespace {
1404class ExpandIRInstsLegacyPass : public FunctionPass {
1405 CodeGenOptLevel OptLevel;
1406
1407public:
1408 static char ID;
1409
1410 ExpandIRInstsLegacyPass(CodeGenOptLevel OptLevel)
1411 : FunctionPass(ID), OptLevel(OptLevel) {}
1412
1413 ExpandIRInstsLegacyPass() : ExpandIRInstsLegacyPass(CodeGenOptLevel::None) {}
1414
1415 bool runOnFunction(Function &F) override {
1416 auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
1417 const TargetSubtargetInfo *Subtarget = TM->getSubtargetImpl(F);
1418 auto *TLI = Subtarget->getTargetLowering();
1419 AssumptionCache *AC = nullptr;
1420
1421 const LibcallLoweringInfo &Libcalls =
1422 getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
1423 M: *F.getParent(), Subtarget: *Subtarget);
1424
1425 if (OptLevel != CodeGenOptLevel::None && !F.hasOptNone())
1426 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
1427 return runImpl(F, TLI: *TLI, Libcalls, AC);
1428 }
1429
1430 void getAnalysisUsage(AnalysisUsage &AU) const override {
1431 AU.addRequired<LibcallLoweringInfoWrapper>();
1432 AU.addRequired<TargetPassConfig>();
1433 if (OptLevel != CodeGenOptLevel::None)
1434 AU.addRequired<AssumptionCacheTracker>();
1435 AU.addPreserved<AAResultsWrapperPass>();
1436 AU.addPreserved<GlobalsAAWrapperPass>();
1437 AU.addRequired<LibcallLoweringInfoWrapper>();
1438 }
1439};
1440} // namespace
1441
1442ExpandIRInstsPass::ExpandIRInstsPass(const TargetMachine &TM,
1443 CodeGenOptLevel OptLevel)
1444 : TM(&TM), OptLevel(OptLevel) {}
1445
1446void ExpandIRInstsPass::printPipeline(
1447 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
1448 static_cast<PassInfoMixin<ExpandIRInstsPass> *>(this)->printPipeline(
1449 OS, MapClassName2PassName);
1450 OS << '<';
1451 OS << "O" << (int)OptLevel;
1452 OS << '>';
1453}
1454
1455PreservedAnalyses ExpandIRInstsPass::run(Function &F,
1456 FunctionAnalysisManager &FAM) {
1457 const TargetSubtargetInfo *STI = TM->getSubtargetImpl(F);
1458 auto &TLI = *STI->getTargetLowering();
1459 AssumptionCache *AC = nullptr;
1460 if (OptLevel != CodeGenOptLevel::None)
1461 AC = &FAM.getResult<AssumptionAnalysis>(IR&: F);
1462
1463 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
1464
1465 const ModuleLibcallLoweringInfo *LibcallLowering =
1466 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(IR&: *F.getParent());
1467
1468 if (!LibcallLowering) {
1469 F.getContext().emitError(ErrorStr: "'" + LibcallLoweringModuleAnalysis::name() +
1470 "' analysis required");
1471 return PreservedAnalyses::all();
1472 }
1473
1474 const LibcallLoweringInfo &Libcalls =
1475 getLibcallLowering(ModuleInfo: *LibcallLowering, Subtarget: *STI);
1476
1477 return runImpl(F, TLI, Libcalls, AC) ? PreservedAnalyses::none()
1478 : PreservedAnalyses::all();
1479}
1480
1481char ExpandIRInstsLegacyPass::ID = 0;
1482INITIALIZE_PASS_BEGIN(ExpandIRInstsLegacyPass, "expand-ir-insts",
1483 "Expand certain fp instructions", false, false)
1484INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
1485INITIALIZE_PASS_END(ExpandIRInstsLegacyPass, "expand-ir-insts",
1486 "Expand IR instructions", false, false)
1487
1488FunctionPass *llvm::createExpandIRInstsPass(CodeGenOptLevel OptLevel) {
1489 return new ExpandIRInstsLegacyPass(OptLevel);
1490}
1491