1//===- InstCombineSelect.cpp ----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitSelect function.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/CmpInstAnalysis.h"
19#include "llvm/Analysis/InstructionSimplify.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OverflowInstAnalysis.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/IR/BasicBlock.h"
25#include "llvm/IR/Constant.h"
26#include "llvm/IR/ConstantRange.h"
27#include "llvm/IR/Constants.h"
28#include "llvm/IR/DerivedTypes.h"
29#include "llvm/IR/FMF.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstrTypes.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/Operator.h"
37#include "llvm/IR/PatternMatch.h"
38#include "llvm/IR/ProfDataUtils.h"
39#include "llvm/IR/Type.h"
40#include "llvm/IR/User.h"
41#include "llvm/IR/Value.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/KnownBits.h"
45#include "llvm/Support/MathExtras.h"
46#include "llvm/Transforms/InstCombine/InstCombiner.h"
47#include <cassert>
48#include <optional>
49#include <utility>
50
51#define DEBUG_TYPE "instcombine"
52#include "llvm/Transforms/Utils/InstructionWorklist.h"
53
54using namespace llvm;
55using namespace PatternMatch;
56
57namespace llvm {
58extern cl::opt<bool> ProfcheckDisableMetadataFixes;
59}
60
61/// Replace a select operand based on an equality comparison with the identity
62/// constant of a binop.
63static Instruction *foldSelectBinOpIdentity(SelectInst &Sel,
64 const TargetLibraryInfo &TLI,
65 InstCombinerImpl &IC) {
66 // The select condition must be an equality compare with a constant operand.
67 Value *X;
68 Constant *C;
69 CmpPredicate Pred;
70 if (!match(V: Sel.getCondition(), P: m_Cmp(Pred, L: m_Value(V&: X), R: m_Constant(C))))
71 return nullptr;
72
73 bool IsEq;
74 if (ICmpInst::isEquality(P: Pred))
75 IsEq = Pred == ICmpInst::ICMP_EQ;
76 else if (Pred == FCmpInst::FCMP_OEQ)
77 IsEq = true;
78 else if (Pred == FCmpInst::FCMP_UNE)
79 IsEq = false;
80 else
81 return nullptr;
82
83 // A select operand must be a binop.
84 BinaryOperator *BO;
85 if (!match(V: Sel.getOperand(i_nocapture: IsEq ? 1 : 2), P: m_BinOp(I&: BO)))
86 return nullptr;
87
88 // For absorbing values, we can fold to the compared value.
89 bool IsAbsorbingValue = false;
90
91 // Last, match the compare variable operand with a binop operand.
92 Value *Y;
93 if (BO->isCommutative()) {
94 // Recognized 0 as an absorbing value for fmul, but we need to be careful
95 // about the sign. This could be more aggressive, by handling arbitrary sign
96 // bit operations as long as we know the fmul sign matches (and handling
97 // arbitrary opcodes).
98 if (match(V: BO, P: m_c_FMul(L: m_FAbs(Op0: m_Specific(V: X)), R: m_Value(V&: Y))) &&
99 match(V: C, P: m_AnyZeroFP()) &&
100 IC.fmulByZeroIsZero(MulVal: Y, FMF: BO->getFastMathFlags(), CtxI: &Sel))
101 IsAbsorbingValue = true;
102 else if (!match(V: BO, P: m_c_BinOp(L: m_Value(V&: Y), R: m_Specific(V: X))))
103 return nullptr;
104 } else {
105 if (!match(V: BO, P: m_BinOp(L: m_Value(V&: Y), R: m_Specific(V: X))))
106 return nullptr;
107 }
108
109 // The compare constant must be the identity constant for that binop.
110 // If this a floating-point compare with 0.0, any zero constant will do.
111 Type *Ty = BO->getType();
112
113 Value *FoldedVal;
114 if (IsAbsorbingValue) {
115 FoldedVal = C;
116 } else {
117 Constant *IdC = ConstantExpr::getBinOpIdentity(Opcode: BO->getOpcode(), Ty, AllowRHSConstant: true);
118 if (IdC != C) {
119 if (!IdC || !CmpInst::isFPPredicate(P: Pred))
120 return nullptr;
121
122 if (!match(V: IdC, P: m_AnyZeroFP()) || !match(V: C, P: m_AnyZeroFP()))
123 return nullptr;
124 }
125
126 // +0.0 compares equal to -0.0, and so it does not behave as required for
127 // this transform. Bail out if we can not exclude that possibility.
128 if (const auto *FPO = dyn_cast<FPMathOperator>(Val: BO))
129 if (!FPO->hasNoSignedZeros() &&
130 !cannotBeNegativeZero(V: Y,
131 SQ: IC.getSimplifyQuery().getWithInstruction(I: &Sel)))
132 return nullptr;
133
134 FoldedVal = Y;
135 }
136
137 // BO = binop Y, X
138 // S = { select (cmp eq X, C), BO, ? } or { select (cmp ne X, C), ?, BO }
139 // =>
140 // S = { select (cmp eq X, C), Y, ? } or { select (cmp ne X, C), ?, Y }
141 return IC.replaceOperand(I&: Sel, OpNum: IsEq ? 1 : 2, V: FoldedVal);
142}
143
144/// This folds:
145/// select (icmp eq (and X, C1)), TC, FC
146/// iff C1 is a power 2 and the difference between TC and FC is a power-of-2.
147/// To something like:
148/// (shr (and (X, C1)), (log2(C1) - log2(TC-FC))) + FC
149/// Or:
150/// (shl (and (X, C1)), (log2(TC-FC) - log2(C1))) + FC
151/// With some variations depending if FC is larger than TC, or the shift
152/// isn't needed, or the bit widths don't match.
153static Value *foldSelectICmpAnd(SelectInst &Sel, Value *CondVal, Value *TrueVal,
154 Value *FalseVal, Value *V, const APInt &AndMask,
155 bool CreateAnd,
156 InstCombiner::BuilderTy &Builder) {
157 const APInt *SelTC, *SelFC;
158 if (!match(V: TrueVal, P: m_APInt(Res&: SelTC)) || !match(V: FalseVal, P: m_APInt(Res&: SelFC)))
159 return nullptr;
160
161 Type *SelType = Sel.getType();
162 // In general, when both constants are non-zero, we would need an offset to
163 // replace the select. This would require more instructions than we started
164 // with. But there's one special-case that we handle here because it can
165 // simplify/reduce the instructions.
166 const APInt &TC = *SelTC;
167 const APInt &FC = *SelFC;
168 if (!TC.isZero() && !FC.isZero()) {
169 if (TC.getBitWidth() != AndMask.getBitWidth())
170 return nullptr;
171 // If we have to create an 'and', then we must kill the cmp to not
172 // increase the instruction count.
173 if (CreateAnd && !CondVal->hasOneUse())
174 return nullptr;
175
176 // (V & AndMaskC) == 0 ? TC : FC --> TC | (V & AndMaskC)
177 // (V & AndMaskC) == 0 ? TC : FC --> TC ^ (V & AndMaskC)
178 // (V & AndMaskC) == 0 ? TC : FC --> TC + (V & AndMaskC)
179 // (V & AndMaskC) == 0 ? TC : FC --> TC - (V & AndMaskC)
180 Constant *TCC = ConstantInt::get(Ty: SelType, V: TC);
181 Constant *FCC = ConstantInt::get(Ty: SelType, V: FC);
182 Constant *MaskC = ConstantInt::get(Ty: SelType, V: AndMask);
183 for (auto Opc : {Instruction::Or, Instruction::Xor, Instruction::Add,
184 Instruction::Sub}) {
185 if (ConstantFoldBinaryOpOperands(Opcode: Opc, LHS: TCC, RHS: MaskC, DL: Sel.getDataLayout()) ==
186 FCC) {
187 if (CreateAnd)
188 V = Builder.CreateAnd(LHS: V, RHS: MaskC);
189 return Builder.CreateBinOp(Opc, LHS: TCC, RHS: V);
190 }
191 }
192
193 return nullptr;
194 }
195
196 // Make sure one of the select arms is a power-of-2.
197 if (!TC.isPowerOf2() && !FC.isPowerOf2())
198 return nullptr;
199
200 // Determine which shift is needed to transform result of the 'and' into the
201 // desired result.
202 const APInt &ValC = !TC.isZero() ? TC : FC;
203 unsigned ValZeros = ValC.logBase2();
204 unsigned AndZeros = AndMask.logBase2();
205 bool ShouldNotVal = !TC.isZero();
206 bool NeedShift = ValZeros != AndZeros;
207 bool NeedZExtTrunc =
208 SelType->getScalarSizeInBits() != V->getType()->getScalarSizeInBits();
209
210 // If we would need to create an 'and' + 'shift' + 'xor' + cast to replace
211 // a 'select' + 'icmp', then this transformation would result in more
212 // instructions and potentially interfere with other folding.
213 if (CreateAnd + ShouldNotVal + NeedShift + NeedZExtTrunc >
214 1 + CondVal->hasOneUse())
215 return nullptr;
216
217 // Insert the 'and' instruction on the input to the truncate.
218 if (CreateAnd)
219 V = Builder.CreateAnd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: AndMask));
220
221 // If types don't match, we can still convert the select by introducing a zext
222 // or a trunc of the 'and'.
223 if (ValZeros > AndZeros) {
224 V = Builder.CreateZExtOrTrunc(V, DestTy: SelType);
225 V = Builder.CreateShl(LHS: V, RHS: ValZeros - AndZeros);
226 } else if (ValZeros < AndZeros) {
227 V = Builder.CreateLShr(LHS: V, RHS: AndZeros - ValZeros);
228 V = Builder.CreateZExtOrTrunc(V, DestTy: SelType);
229 } else {
230 V = Builder.CreateZExtOrTrunc(V, DestTy: SelType);
231 }
232
233 // Okay, now we know that everything is set up, we just don't know whether we
234 // have a icmp_ne or icmp_eq and whether the true or false val is the zero.
235 if (ShouldNotVal)
236 V = Builder.CreateXor(LHS: V, RHS: ValC);
237
238 return V;
239}
240
241/// We want to turn code that looks like this:
242/// %C = or %A, %B
243/// %D = select %cond, %C, %A
244/// into:
245/// %C = select %cond, %B, 0
246/// %D = or %A, %C
247///
248/// Assuming that the specified instruction is an operand to the select, return
249/// a bitmask indicating which operands of this instruction are foldable if they
250/// equal the other incoming value of the select.
251static unsigned getSelectFoldableOperands(BinaryOperator *I) {
252 switch (I->getOpcode()) {
253 case Instruction::Add:
254 case Instruction::FAdd:
255 case Instruction::Mul:
256 case Instruction::FMul:
257 case Instruction::And:
258 case Instruction::Or:
259 case Instruction::Xor:
260 return 3; // Can fold through either operand.
261 case Instruction::Sub: // Can only fold on the amount subtracted.
262 case Instruction::FSub:
263 case Instruction::FDiv: // Can only fold on the divisor amount.
264 case Instruction::Shl: // Can only fold on the shift amount.
265 case Instruction::LShr:
266 case Instruction::AShr:
267 return 1;
268 default:
269 return 0; // Cannot fold
270 }
271}
272
273/// We have (select c, TI, FI), and we know that TI and FI have the same opcode.
274Instruction *InstCombinerImpl::foldSelectOpOp(SelectInst &SI, Instruction *TI,
275 Instruction *FI) {
276 // If this is a cast from the same type, merge.
277 Value *Cond = SI.getCondition();
278 Type *CondTy = Cond->getType();
279 if (TI->getNumOperands() == 1 && TI->isCast()) {
280 Type *FIOpndTy = FI->getOperand(i: 0)->getType();
281 if (TI->getOperand(i: 0)->getType() != FIOpndTy)
282 return nullptr;
283
284 // The select condition may be a vector. We may only change the operand
285 // type if the vector width remains the same (and matches the condition).
286 if (auto *CondVTy = dyn_cast<VectorType>(Val: CondTy)) {
287 if (!FIOpndTy->isVectorTy() ||
288 CondVTy->getElementCount() !=
289 cast<VectorType>(Val: FIOpndTy)->getElementCount())
290 return nullptr;
291
292 // TODO: If the backend knew how to deal with casts better, we could
293 // remove this limitation. For now, there's too much potential to create
294 // worse codegen by promoting the select ahead of size-altering casts
295 // (PR28160).
296 //
297 // Note that ValueTracking's matchSelectPattern() looks through casts
298 // without checking 'hasOneUse' when it matches min/max patterns, so this
299 // transform may end up happening anyway.
300 if (TI->getOpcode() != Instruction::BitCast &&
301 (!TI->hasOneUse() || !FI->hasOneUse()))
302 return nullptr;
303 } else if (!TI->hasOneUse() || !FI->hasOneUse()) {
304 // TODO: The one-use restrictions for a scalar select could be eased if
305 // the fold of a select in visitLoadInst() was enhanced to match a pattern
306 // that includes a cast.
307 return nullptr;
308 }
309
310 // Fold this by inserting a select from the input values.
311 Value *NewSI =
312 Builder.CreateSelect(C: Cond, True: TI->getOperand(i: 0), False: FI->getOperand(i: 0),
313 Name: SI.getName() + ".v", MDFrom: &SI);
314 return CastInst::Create(Instruction::CastOps(TI->getOpcode()), S: NewSI,
315 Ty: TI->getType());
316 }
317
318 Value *OtherOpT, *OtherOpF;
319 bool MatchIsOpZero;
320 auto getCommonOp = [&](Instruction *TI, Instruction *FI, bool Commute,
321 bool Swapped = false) -> Value * {
322 assert(!(Commute && Swapped) &&
323 "Commute and Swapped can't set at the same time");
324 if (!Swapped) {
325 if (TI->getOperand(i: 0) == FI->getOperand(i: 0)) {
326 OtherOpT = TI->getOperand(i: 1);
327 OtherOpF = FI->getOperand(i: 1);
328 MatchIsOpZero = true;
329 return TI->getOperand(i: 0);
330 } else if (TI->getOperand(i: 1) == FI->getOperand(i: 1)) {
331 OtherOpT = TI->getOperand(i: 0);
332 OtherOpF = FI->getOperand(i: 0);
333 MatchIsOpZero = false;
334 return TI->getOperand(i: 1);
335 }
336 }
337
338 if (!Commute && !Swapped)
339 return nullptr;
340
341 // If we are allowing commute or swap of operands, then
342 // allow a cross-operand match. In that case, MatchIsOpZero
343 // means that TI's operand 0 (FI's operand 1) is the common op.
344 if (TI->getOperand(i: 0) == FI->getOperand(i: 1)) {
345 OtherOpT = TI->getOperand(i: 1);
346 OtherOpF = FI->getOperand(i: 0);
347 MatchIsOpZero = true;
348 return TI->getOperand(i: 0);
349 } else if (TI->getOperand(i: 1) == FI->getOperand(i: 0)) {
350 OtherOpT = TI->getOperand(i: 0);
351 OtherOpF = FI->getOperand(i: 1);
352 MatchIsOpZero = false;
353 return TI->getOperand(i: 1);
354 }
355 return nullptr;
356 };
357
358 if (TI->hasOneUse() || FI->hasOneUse()) {
359 // Cond ? -X : -Y --> -(Cond ? X : Y)
360 Value *X, *Y;
361 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
362 if (match(V: TI, P: m_FNegOrFSubNegZero(X: m_Value(V&: X))) &&
363 match(V: FI, P: m_FNegOrFSubNegZero(X: m_Value(V&: Y)))) {
364 // Intersect FMF from the fneg instructions and union those with the
365 // select.
366 FastMathFlags FMF = TI->getFastMathFlags();
367 FMF &= FI->getFastMathFlags();
368 FMF |= SI.getFastMathFlags();
369 Value *NewSel =
370 Builder.CreateSelect(C: Cond, True: X, False: Y, Name: SI.getName() + ".v", MDFrom: &SI);
371 if (auto *NewSelI = dyn_cast<Instruction>(Val: NewSel))
372 NewSelI->setFastMathFlags(FMF);
373 Instruction *NewFNeg = UnaryOperator::CreateFNeg(V: NewSel);
374 NewFNeg->setFastMathFlags(FMF);
375 return NewFNeg;
376 }
377
378 // Min/max intrinsic with a common operand can have the common operand
379 // pulled after the select. This is the same transform as below for binops,
380 // but specialized for intrinsic matching and without the restrictive uses
381 // clause.
382 auto *TII = dyn_cast<IntrinsicInst>(Val: TI);
383 auto *FII = dyn_cast<IntrinsicInst>(Val: FI);
384 if (TII && FII && TII->getIntrinsicID() == FII->getIntrinsicID()) {
385 if (match(V: TII, P: m_MaxOrMin(Op0: m_Value(), Op1: m_Value()))) {
386 if (Value *MatchOp = getCommonOp(TI, FI, true)) {
387 Value *NewSel =
388 Builder.CreateSelect(C: Cond, True: OtherOpT, False: OtherOpF, Name: "minmaxop", MDFrom: &SI);
389 return CallInst::Create(Func: TII->getCalledFunction(), Args: {NewSel, MatchOp});
390 }
391 }
392
393 // select c, (ldexp v, e0), (ldexp v, e1) -> ldexp v, (select c, e0, e1)
394 // select c, (ldexp v0, e), (ldexp v1, e) -> ldexp (select c, v0, v1), e
395 //
396 // select c, (ldexp v0, e0), (ldexp v1, e1) ->
397 // ldexp (select c, v0, v1), (select c, e0, e1)
398 if (TII->getIntrinsicID() == Intrinsic::ldexp) {
399 Value *LdexpVal0 = TII->getArgOperand(i: 0);
400 Value *LdexpExp0 = TII->getArgOperand(i: 1);
401 Value *LdexpVal1 = FII->getArgOperand(i: 0);
402 Value *LdexpExp1 = FII->getArgOperand(i: 1);
403 if (LdexpExp0->getType() == LdexpExp1->getType()) {
404 FPMathOperator *SelectFPOp = cast<FPMathOperator>(Val: &SI);
405 FastMathFlags FMF = cast<FPMathOperator>(Val: TII)->getFastMathFlags();
406 FMF &= cast<FPMathOperator>(Val: FII)->getFastMathFlags();
407 FMF |= SelectFPOp->getFastMathFlags();
408
409 Value *SelectVal = Builder.CreateSelect(
410 C: Cond, True: LdexpVal0, False: LdexpVal1, Name: "",
411 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &SI);
412 Value *SelectExp = Builder.CreateSelect(
413 C: Cond, True: LdexpExp0, False: LdexpExp1, Name: "",
414 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &SI);
415
416 Value *NewLdexp = Builder.CreateIntrinsic(
417 RetTy: TII->getType(), ID: Intrinsic::ldexp, Args: {SelectVal, SelectExp}, FMFSource: FMF);
418 return replaceInstUsesWith(I&: SI, V: NewLdexp);
419 }
420 }
421 }
422
423 auto CreateCmpSel = [&](std::optional<CmpPredicate> P,
424 bool Swapped) -> CmpInst * {
425 if (!P)
426 return nullptr;
427 auto *MatchOp = getCommonOp(TI, FI, ICmpInst::isEquality(P: *P),
428 ICmpInst::isRelational(P: *P) && Swapped);
429 if (!MatchOp)
430 return nullptr;
431 Value *NewSel = Builder.CreateSelect(C: Cond, True: OtherOpT, False: OtherOpF,
432 Name: SI.getName() + ".v", MDFrom: &SI);
433 return new ICmpInst(MatchIsOpZero ? *P
434 : ICmpInst::getSwappedCmpPredicate(Pred: *P),
435 MatchOp, NewSel);
436 };
437
438 // icmp with a common operand also can have the common operand
439 // pulled after the select.
440 CmpPredicate TPred, FPred;
441 if (match(V: TI, P: m_ICmp(Pred&: TPred, L: m_Value(), R: m_Value())) &&
442 match(V: FI, P: m_ICmp(Pred&: FPred, L: m_Value(), R: m_Value()))) {
443 if (auto *R =
444 CreateCmpSel(CmpPredicate::getMatching(A: TPred, B: FPred), false))
445 return R;
446 if (auto *R =
447 CreateCmpSel(CmpPredicate::getMatching(
448 A: TPred, B: ICmpInst::getSwappedCmpPredicate(Pred: FPred)),
449 true))
450 return R;
451 }
452 }
453
454 // Only handle binary operators (including two-operand getelementptr) with
455 // one-use here. As with the cast case above, it may be possible to relax the
456 // one-use constraint, but that needs be examined carefully since it may not
457 // reduce the total number of instructions.
458 if (TI->getNumOperands() != 2 || FI->getNumOperands() != 2 ||
459 !TI->isSameOperationAs(I: FI) ||
460 (!isa<BinaryOperator>(Val: TI) && !isa<GetElementPtrInst>(Val: TI)) ||
461 !TI->hasOneUse() || !FI->hasOneUse())
462 return nullptr;
463
464 // Figure out if the operations have any operands in common.
465 Value *MatchOp = getCommonOp(TI, FI, TI->isCommutative());
466 if (!MatchOp)
467 return nullptr;
468
469 // If the select condition is a vector, the operands of the original select's
470 // operands also must be vectors. This may not be the case for getelementptr
471 // for example.
472 if (CondTy->isVectorTy() && (!OtherOpT->getType()->isVectorTy() ||
473 !OtherOpF->getType()->isVectorTy()))
474 return nullptr;
475
476 // If we are sinking div/rem after a select, we may need to freeze the
477 // condition because div/rem may induce immediate UB with a poison operand.
478 // For example, the following transform is not safe if Cond can ever be poison
479 // because we can replace poison with zero and then we have div-by-zero that
480 // didn't exist in the original code:
481 // Cond ? x/y : x/z --> x / (Cond ? y : z)
482 auto *BO = dyn_cast<BinaryOperator>(Val: TI);
483 if (BO && BO->isIntDivRem() && !isGuaranteedNotToBePoison(V: Cond)) {
484 // A udiv/urem with a common divisor is safe because UB can only occur with
485 // div-by-zero, and that would be present in the original code.
486 if (BO->getOpcode() == Instruction::SDiv ||
487 BO->getOpcode() == Instruction::SRem || MatchIsOpZero)
488 Cond = Builder.CreateFreeze(V: Cond);
489 }
490
491 // If we reach here, they do have operations in common.
492 Value *NewSI = Builder.CreateSelect(C: Cond, True: OtherOpT, False: OtherOpF,
493 Name: SI.getName() + ".v", MDFrom: &SI);
494 Value *Op0 = MatchIsOpZero ? MatchOp : NewSI;
495 Value *Op1 = MatchIsOpZero ? NewSI : MatchOp;
496 if (auto *BO = dyn_cast<BinaryOperator>(Val: TI)) {
497 BinaryOperator *NewBO = BinaryOperator::Create(Op: BO->getOpcode(), S1: Op0, S2: Op1);
498 NewBO->copyIRFlags(V: TI);
499 NewBO->andIRFlags(V: FI);
500 return NewBO;
501 }
502 if (auto *TGEP = dyn_cast<GetElementPtrInst>(Val: TI)) {
503 auto *FGEP = cast<GetElementPtrInst>(Val: FI);
504 Type *ElementType = TGEP->getSourceElementType();
505 return GetElementPtrInst::Create(
506 PointeeType: ElementType, Ptr: Op0, IdxList: Op1, NW: TGEP->getNoWrapFlags() & FGEP->getNoWrapFlags());
507 }
508 llvm_unreachable("Expected BinaryOperator or GEP");
509 return nullptr;
510}
511
512/// Return true if a constant in the given argument position of the given
513/// intrinsic may enable cheaper lowering, in which case a select should not
514/// be sunk into that argument (it would hide the constant behind the
515/// select).
516static bool isConstantBeneficialInIntrinsicArg(Intrinsic::ID IID,
517 unsigned ArgNo) {
518 switch (IID) {
519 case Intrinsic::copysign:
520 // Only the sign operand is constant-sensitive: copysign(y, +-C) folds to
521 // fabs/fnabs. A constant magnitude has no cheaper lowering, so sinking
522 // the select is profitable (and merges two calls into one).
523 return ArgNo == 1;
524 default:
525 // Conservatively assume that a constant may be beneficial.
526 return true;
527 }
528}
529
530/// select C, f(..., X, ...), f(..., Y, ...) --> f(..., select C, X, Y, ...)
531static Instruction *
532foldSelectIntoIntrinsicArg(SelectInst &SI, IntrinsicInst *TII,
533 IntrinsicInst *FII,
534 InstCombiner::BuilderTy &Builder) {
535 if (TII->getCalledFunction() != FII->getCalledFunction())
536 return nullptr;
537
538 std::optional<unsigned> DiffArgNo;
539 for (unsigned I = 0, E = TII->arg_size(); I != E; ++I) {
540 if (TII->getArgOperand(i: I) == FII->getArgOperand(i: I))
541 continue;
542 if (DiffArgNo)
543 return nullptr;
544 DiffArgNo = I;
545 }
546 if (!DiffArgNo)
547 return nullptr;
548
549 if (!canReplaceOperandWithVariable(I: TII, OpIdx: *DiffArgNo))
550 return nullptr;
551
552 Value *TV = TII->getArgOperand(i: *DiffArgNo);
553 Value *FV = FII->getArgOperand(i: *DiffArgNo);
554 // powi keeps a scalar exponent even in its vector form.
555 if (SI.getCondition()->getType()->isVectorTy() &&
556 !TV->getType()->isVectorTy())
557 return nullptr;
558 // Constant args may enable cheaper lowering, e.g. pow(X, 2.0) -> X * X.
559 // Sinking the select into the argument would hide the constant, so reject
560 // the sink for argument positions where a constant is beneficial.
561 if ((isa<Constant>(Val: TV) || isa<Constant>(Val: FV)) &&
562 isConstantBeneficialInIntrinsicArg(IID: TII->getIntrinsicID(), ArgNo: *DiffArgNo))
563 return nullptr;
564
565 SmallVector<Value *> Args(TII->args());
566 Args[*DiffArgNo] =
567 Builder.CreateSelect(C: SI.getCondition(), True: TV, False: FV, Name: SI.getName() + ".v", MDFrom: &SI);
568 auto *NewCall = CallInst::Create(Func: TII->getCalledFunction(), Args);
569 NewCall->copyIRFlags(V: TII);
570 NewCall->andIRFlags(V: FII);
571 return NewCall;
572}
573
574/// This transforms patterns of the form:
575/// select cond, intrinsic(x, ...), intrinsic(y, ...)
576/// into:
577/// intrinsic(select cond, x, y, ...)
578Instruction *InstCombinerImpl::foldSelectIntrinsic(SelectInst &SI) {
579 auto *LHSIntrinsic = dyn_cast<IntrinsicInst>(Val: SI.getTrueValue());
580 if (!LHSIntrinsic)
581 return nullptr;
582 auto *RHSIntrinsic = dyn_cast<IntrinsicInst>(Val: SI.getFalseValue());
583 if (!RHSIntrinsic ||
584 LHSIntrinsic->getIntrinsicID() != RHSIntrinsic->getIntrinsicID() ||
585 !LHSIntrinsic->hasOneUse() || !RHSIntrinsic->hasOneUse())
586 return nullptr;
587
588 const Intrinsic::ID IID = LHSIntrinsic->getIntrinsicID();
589 switch (IID) {
590 case Intrinsic::abs:
591 case Intrinsic::cttz:
592 case Intrinsic::ctlz: {
593 auto *TZ = cast<ConstantInt>(Val: LHSIntrinsic->getArgOperand(i: 1));
594 auto *FZ = cast<ConstantInt>(Val: RHSIntrinsic->getArgOperand(i: 1));
595
596 Value *TV = LHSIntrinsic->getArgOperand(i: 0);
597 Value *FV = RHSIntrinsic->getArgOperand(i: 0);
598
599 Value *NewSel = Builder.CreateSelect(C: SI.getCondition(), True: TV, False: FV, Name: "", MDFrom: &SI);
600 Value *NewPoisonFlag = Builder.CreateAnd(LHS: TZ, RHS: FZ);
601 Value *NewCall = Builder.CreateBinaryIntrinsic(ID: IID, LHS: NewSel, RHS: NewPoisonFlag);
602
603 return replaceInstUsesWith(I&: SI, V: NewCall);
604 }
605 default:
606 if (isTriviallyVectorizable(ID: IID))
607 return foldSelectIntoIntrinsicArg(SI, TII: LHSIntrinsic, FII: RHSIntrinsic,
608 Builder);
609 return nullptr;
610 }
611}
612
613static bool isSelect01(const APInt &C1I, const APInt &C2I) {
614 if (!C1I.isZero() && !C2I.isZero()) // One side must be zero.
615 return false;
616 return C1I.isOne() || C1I.isAllOnes() || C2I.isOne() || C2I.isAllOnes();
617}
618
619/// Try to fold the select into one of the operands to allow further
620/// optimization.
621Instruction *InstCombinerImpl::foldSelectIntoOp(SelectInst &SI, Value *TrueVal,
622 Value *FalseVal) {
623 // See the comment above getSelectFoldableOperands for a description of the
624 // transformation we are doing here.
625 auto TryFoldSelectIntoOp = [&](SelectInst &SI, Value *TrueVal,
626 Value *FalseVal,
627 bool Swapped) -> Instruction * {
628 auto *TVI = dyn_cast<BinaryOperator>(Val: TrueVal);
629 if (!TVI || !TVI->hasOneUse() || isa<Constant>(Val: FalseVal))
630 return nullptr;
631
632 unsigned SFO = getSelectFoldableOperands(I: TVI);
633 unsigned OpToFold = 0;
634 if ((SFO & 1) && FalseVal == TVI->getOperand(i_nocapture: 0))
635 OpToFold = 1;
636 else if ((SFO & 2) && FalseVal == TVI->getOperand(i_nocapture: 1))
637 OpToFold = 2;
638
639 if (!OpToFold)
640 return nullptr;
641
642 FastMathFlags FMF;
643 if (const auto *FPO = dyn_cast<FPMathOperator>(Val: &SI))
644 FMF = FPO->getFastMathFlags();
645 Constant *C = ConstantExpr::getBinOpIdentity(
646 Opcode: TVI->getOpcode(), Ty: TVI->getType(), AllowRHSConstant: true, NSZ: FMF.noSignedZeros());
647 Value *OOp = TVI->getOperand(i_nocapture: 2 - OpToFold);
648 // Avoid creating select between 2 constants unless it's selecting
649 // between 0, 1 and -1.
650 const APInt *OOpC;
651 bool OOpIsAPInt = match(V: OOp, P: m_APInt(Res&: OOpC));
652 if (isa<Constant>(Val: OOp) &&
653 (!OOpIsAPInt || !isSelect01(C1I: C->getUniqueInteger(), C2I: *OOpC)))
654 return nullptr;
655
656 // If the false value is a NaN then we have that the floating point math
657 // operation in the transformed code may not preserve the exact NaN
658 // bit-pattern -- e.g. `fadd sNaN, 0.0 -> qNaN`.
659 // This makes the transformation incorrect since the original program would
660 // have preserved the exact NaN bit-pattern.
661 // Avoid the folding if the false value might be a NaN.
662 if (isa<FPMathOperator>(Val: &SI) &&
663 !computeKnownFPClass(V: FalseVal, FMF, InterestedClasses: fcNan, SQ: SQ.getWithInstruction(I: &SI))
664 .isKnownNeverNaN())
665 return nullptr;
666
667 Value *NewSel = Builder.CreateSelect(C: SI.getCondition(), True: Swapped ? C : OOp,
668 False: Swapped ? OOp : C, Name: "", MDFrom: &SI);
669 if (isa<FPMathOperator>(Val: &SI)) {
670 FastMathFlags NewSelFMF = FMF;
671 // We cannot propagate ninf from the original select, because OOp may be
672 // inf and the flag only guarantees that FalseVal (op OOp) is never
673 // infinity.
674 // Examples: -inf + +inf = NaN, -inf - -inf = NaN, 0 * inf = NaN
675 // Specifically, if the original select has both ninf and nnan, we can
676 // safely propagate the flag.
677 // Note: This property holds for fadd, fsub, and fmul, but does not
678 // hold for fdiv (e.g. A / Inf == 0.0).
679 bool CanInferFiniteOperandsFromResult =
680 TVI->getOpcode() == Instruction::FAdd ||
681 TVI->getOpcode() == Instruction::FSub ||
682 TVI->getOpcode() == Instruction::FMul;
683 NewSelFMF.setNoInfs(TVI->hasNoInfs() ||
684 (CanInferFiniteOperandsFromResult &&
685 NewSelFMF.noInfs() && NewSelFMF.noNaNs()));
686 cast<Instruction>(Val: NewSel)->setFastMathFlags(NewSelFMF);
687 }
688 NewSel->takeName(V: TVI);
689 BinaryOperator *BO =
690 BinaryOperator::Create(Op: TVI->getOpcode(), S1: FalseVal, S2: NewSel);
691 BO->copyIRFlags(V: TVI);
692 if (isa<FPMathOperator>(Val: &SI)) {
693 // Merge poison generating flags from the select.
694 BO->setHasNoNaNs(BO->hasNoNaNs() && FMF.noNaNs());
695 BO->setHasNoInfs(BO->hasNoInfs() && FMF.noInfs());
696 // Merge no-signed-zeros flag from the select.
697 // Otherwise we may produce zeros with different sign.
698 BO->setHasNoSignedZeros(BO->hasNoSignedZeros() && FMF.noSignedZeros());
699 }
700 return BO;
701 };
702
703 if (Instruction *R = TryFoldSelectIntoOp(SI, TrueVal, FalseVal, false))
704 return R;
705
706 if (Instruction *R = TryFoldSelectIntoOp(SI, FalseVal, TrueVal, true))
707 return R;
708
709 return nullptr;
710}
711
712static Value *canoncalizeSelectICmpMinMax(const ICmpInst *Cmp, Value *TVal,
713 Value *FVal,
714 InstCombiner::BuilderTy &Builder,
715 const SimplifyQuery &SQ) {
716 Value *CmpLHS = Cmp->getOperand(i_nocapture: 0);
717 Value *CmpRHS = Cmp->getOperand(i_nocapture: 1);
718 ICmpInst::Predicate Pred = Cmp->getPredicate();
719 if (match(V: FVal, P: m_Zero())) {
720 std::swap(a&: TVal, b&: FVal);
721 Pred = ICmpInst::getInversePredicate(pred: Pred);
722 }
723 if (!match(V: TVal, P: m_Zero()))
724 return nullptr;
725
726 if (Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE) {
727 std::swap(a&: CmpLHS, b&: CmpRHS);
728 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
729 }
730
731 // Handles:
732 // (X <= Y) ? 0 : (X - Y)
733 // (X <= Y) ? (Y - X) : 0
734 // (X >= Y) ? 0 : (Y - X)
735 // (X >= Y) ? (X - Y) : 0
736 if ((Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE) &&
737 match(V: FVal, P: m_NSWSub(L: m_Specific(V: CmpLHS), R: m_Specific(V: CmpRHS))) &&
738 isGuaranteedNotToBeUndef(V: CmpLHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)) {
739 Value *SMin =
740 Builder.CreateBinaryIntrinsic(ID: Intrinsic::smin, LHS: CmpRHS, RHS: CmpLHS);
741 return Builder.CreateNSWSub(LHS: CmpLHS, RHS: SMin);
742 }
743
744 return nullptr;
745}
746
747/// Try to fold a select to a min/max intrinsic. Many cases are already handled
748/// by matchDecomposedSelectPattern but here we handle the cases where more
749/// extensive modification of the IR is required.
750static Value *foldSelectICmpMinMax(const ICmpInst *Cmp, Value *TVal,
751 Value *FVal,
752 InstCombiner::BuilderTy &Builder,
753 const SimplifyQuery &SQ) {
754 Value *CmpLHS = Cmp->getOperand(i_nocapture: 0);
755 Value *CmpRHS = Cmp->getOperand(i_nocapture: 1);
756 ICmpInst::Predicate Pred = Cmp->getPredicate();
757
758 if (Value *V = canoncalizeSelectICmpMinMax(Cmp, TVal, FVal, Builder, SQ))
759 return V;
760
761 // (X > Y) ? X : (Y - 1) ==> MIN(X, Y - 1)
762 // (X < Y) ? X : (Y + 1) ==> MAX(X, Y + 1)
763 // This transformation is valid when overflow corresponding to the sign of
764 // the comparison is poison and we must drop the non-matching overflow flag.
765 if (CmpRHS == TVal) {
766 std::swap(a&: CmpLHS, b&: CmpRHS);
767 Pred = CmpInst::getSwappedPredicate(pred: Pred);
768 }
769
770 // TODO: consider handling 'or disjoint' as well, though these would need to
771 // be converted to 'add' instructions.
772 if (!(CmpLHS == TVal && isa<Instruction>(Val: FVal)))
773 return nullptr;
774
775 if (Pred == CmpInst::ICMP_SGT &&
776 match(V: FVal, P: m_NSWAdd(L: m_Specific(V: CmpRHS), R: m_One()))) {
777 cast<Instruction>(Val: FVal)->setHasNoUnsignedWrap(false);
778 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::smax, LHS: TVal, RHS: FVal);
779 }
780
781 if (Pred == CmpInst::ICMP_SLT &&
782 match(V: FVal, P: m_NSWAdd(L: m_Specific(V: CmpRHS), R: m_AllOnes()))) {
783 cast<Instruction>(Val: FVal)->setHasNoUnsignedWrap(false);
784 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::smin, LHS: TVal, RHS: FVal);
785 }
786
787 if (Pred == CmpInst::ICMP_UGT &&
788 match(V: FVal, P: m_NUWAdd(L: m_Specific(V: CmpRHS), R: m_One()))) {
789 cast<Instruction>(Val: FVal)->setHasNoSignedWrap(false);
790 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::umax, LHS: TVal, RHS: FVal);
791 }
792
793 // Note: We must use isKnownNonZero here because "sub nuw %x, 1" will be
794 // canonicalized to "add %x, -1" discarding the nuw flag.
795 if (Pred == CmpInst::ICMP_ULT &&
796 match(V: FVal, P: m_Add(L: m_Specific(V: CmpRHS), R: m_AllOnes())) &&
797 isKnownNonZero(V: CmpRHS, Q: SQ)) {
798 cast<Instruction>(Val: FVal)->setHasNoSignedWrap(false);
799 cast<Instruction>(Val: FVal)->setHasNoUnsignedWrap(false);
800 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: TVal, RHS: FVal);
801 }
802
803 return nullptr;
804}
805
806/// We want to turn:
807/// (select (icmp eq (and X, Y), 0), (and (lshr X, Z), 1), 1)
808/// into:
809/// zext (icmp ne i32 (and X, (or Y, (shl 1, Z))), 0)
810/// Note:
811/// Z may be 0 if lshr is missing.
812/// Worst-case scenario is that we will replace 5 instructions with 5 different
813/// instructions, but we got rid of select.
814static Instruction *foldSelectICmpAndAnd(Type *SelType, const Value *Cond,
815 Value *TVal, Value *FVal,
816 InstCombiner::BuilderTy &Builder) {
817 Value *A, *X, *Y, *Z;
818 CmpPredicate Pred;
819 unsigned NumReplaced = 1 + Cond->hasOneUse();
820 if (match(V: Cond, P: m_Trunc(Op: m_Value(V&: X)))) {
821 Y = ConstantInt::get(Ty: X->getType(), V: 1);
822 Pred = ICmpInst::ICMP_NE;
823 } else if (match(V: Cond,
824 P: m_ICmp(Pred, L: m_And(L: m_Value(V&: X), R: m_Value(V&: Y)), R: m_Zero())) &&
825 ICmpInst::isEquality(P: Pred)) {
826 NumReplaced +=
827 Cond->hasOneUse() && cast<ICmpInst>(Val: Cond)->getOperand(i_nocapture: 0)->hasOneUse();
828 } else
829 return nullptr;
830
831 if (Pred == ICmpInst::ICMP_NE)
832 std::swap(a&: TVal, b&: FVal);
833
834 if (!match(V: FVal, P: m_One()))
835 return nullptr;
836
837 // The TrueVal has general form of: and %B, 1
838 if (!match(V: TVal, P: m_And(L: m_Value(V&: A), R: m_One())))
839 return nullptr;
840
841 APInt BitWidth(SelType->getScalarSizeInBits(),
842 SelType->getScalarSizeInBits());
843 auto TValPattern = m_CombineOr(
844 Ps: m_Deferred(V: X),
845 Ps: m_LShr(L: m_Deferred(V: X), R: m_Value(V&: Z, P: m_SpecificInt_ICMP_ForbidPoison(
846 Predicate: CmpInst::ICMP_ULT, Threshold: BitWidth))));
847
848 if (!match(V: A, P: TValPattern)) {
849 std::swap(a&: X, b&: Y);
850 if (!match(V: A, P: TValPattern))
851 return nullptr;
852 }
853
854 bool HasShift = A != X;
855 if (TVal->hasOneUse())
856 NumReplaced += 1 + (HasShift && A->hasOneUse());
857
858 if (NumReplaced < (4u - isa<Constant>(Val: Y)))
859 return nullptr;
860
861 // ((X & Y) == 0) ? ((X >> Z) & 1) : 1 --> (X & (Y | (1 << Z))) != 0
862 // ((X & Y) == 0) ? (X & 1) : 1 --> (X & (Y | 1)) != 0
863 Constant *One = ConstantInt::get(Ty: SelType, V: 1);
864 Value *MaskB = HasShift ? Builder.CreateShl(LHS: One, RHS: Z) : One;
865 Value *FullMask = Builder.CreateOr(LHS: Y, RHS: MaskB);
866 Value *MaskedX = Builder.CreateAnd(LHS: X, RHS: FullMask);
867 Value *ICmpNeZero = Builder.CreateIsNotNull(Arg: MaskedX);
868 return new ZExtInst(ICmpNeZero, SelType);
869}
870
871/// We want to turn:
872/// (select (icmp eq (and X, C1), 0), 0, (shl [nsw/nuw] X, C2));
873/// iff C1 is a mask and the number of its leading zeros is equal to C2
874/// into:
875/// shl X, C2
876static Value *foldSelectICmpAndZeroShl(const ICmpInst *Cmp, Value *TVal,
877 Value *FVal,
878 InstCombiner::BuilderTy &Builder) {
879 CmpPredicate Pred;
880 Value *AndVal;
881 if (!match(V: Cmp, P: m_ICmp(Pred, L: m_Value(V&: AndVal), R: m_Zero())))
882 return nullptr;
883
884 if (Pred == ICmpInst::ICMP_NE) {
885 Pred = ICmpInst::ICMP_EQ;
886 std::swap(a&: TVal, b&: FVal);
887 }
888
889 Value *X;
890 const APInt *C2, *C1;
891 if (Pred != ICmpInst::ICMP_EQ ||
892 !match(V: AndVal, P: m_And(L: m_Value(V&: X), R: m_APInt(Res&: C1))) ||
893 !match(V: TVal, P: m_Zero()) || !match(V: FVal, P: m_Shl(L: m_Specific(V: X), R: m_APInt(Res&: C2))))
894 return nullptr;
895
896 if (!C1->isMask() ||
897 C1->countLeadingZeros() != static_cast<unsigned>(C2->getZExtValue()))
898 return nullptr;
899
900 auto *FI = dyn_cast<Instruction>(Val: FVal);
901 if (!FI)
902 return nullptr;
903
904 FI->setHasNoSignedWrap(false);
905 FI->setHasNoUnsignedWrap(false);
906 return FVal;
907}
908
909/// We want to turn:
910/// (select (icmp sgt x, C), lshr (X, Y), ashr (X, Y)); iff C s>= -1
911/// (select (icmp slt x, C), ashr (X, Y), lshr (X, Y)); iff C s>= 0
912/// into:
913/// ashr (X, Y)
914static Value *foldSelectICmpLshrAshr(const ICmpInst *IC, Value *TrueVal,
915 Value *FalseVal,
916 InstCombiner::BuilderTy &Builder) {
917 ICmpInst::Predicate Pred = IC->getPredicate();
918 Value *CmpLHS = IC->getOperand(i_nocapture: 0);
919 Value *CmpRHS = IC->getOperand(i_nocapture: 1);
920 if (!CmpRHS->getType()->isIntOrIntVectorTy())
921 return nullptr;
922
923 Value *X, *Y;
924 unsigned Bitwidth = CmpRHS->getType()->getScalarSizeInBits();
925 if ((Pred != ICmpInst::ICMP_SGT ||
926 !match(V: CmpRHS, P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_SGE,
927 Threshold: APInt::getAllOnes(numBits: Bitwidth)))) &&
928 (Pred != ICmpInst::ICMP_SLT ||
929 !match(V: CmpRHS, P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_SGE,
930 Threshold: APInt::getZero(numBits: Bitwidth)))))
931 return nullptr;
932
933 // Canonicalize so that ashr is in FalseVal.
934 if (Pred == ICmpInst::ICMP_SLT)
935 std::swap(a&: TrueVal, b&: FalseVal);
936
937 if (match(V: TrueVal, P: m_LShr(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
938 match(V: FalseVal, P: m_AShr(L: m_Specific(V: X), R: m_Specific(V: Y))) &&
939 match(V: CmpLHS, P: m_Specific(V: X))) {
940 const auto *Ashr = cast<Instruction>(Val: FalseVal);
941 // if lshr is not exact and ashr is, this new ashr must not be exact.
942 bool IsExact = Ashr->isExact() && cast<Instruction>(Val: TrueVal)->isExact();
943 return Builder.CreateAShr(LHS: X, RHS: Y, Name: IC->getName(), isExact: IsExact);
944 }
945
946 return nullptr;
947}
948
949/// We want to turn:
950/// (select (icmp eq (and X, C1), 0), Y, (BinOp Y, C2))
951/// into:
952/// IF C2 u>= C1
953/// (BinOp Y, (shl (and X, C1), C3))
954/// ELSE
955/// (BinOp Y, (lshr (and X, C1), C3))
956/// iff:
957/// 0 on the RHS is the identity value (i.e add, xor, shl, etc...)
958/// C1 and C2 are both powers of 2
959/// where:
960/// IF C2 u>= C1
961/// C3 = Log(C2) - Log(C1)
962/// ELSE
963/// C3 = Log(C1) - Log(C2)
964///
965/// This transform handles cases where:
966/// 1. The icmp predicate is inverted
967/// 2. The select operands are reversed
968/// 3. The magnitude of C2 and C1 are flipped
969static Value *foldSelectICmpAndBinOp(Value *CondVal, Value *TrueVal,
970 Value *FalseVal, Value *V,
971 const APInt &AndMask, bool CreateAnd,
972 InstCombiner::BuilderTy &Builder) {
973 // Only handle integer compares.
974 if (!TrueVal->getType()->isIntOrIntVectorTy())
975 return nullptr;
976
977 unsigned C1Log = AndMask.logBase2();
978 Value *Y;
979 BinaryOperator *BinOp;
980 const APInt *C2;
981 bool NeedXor;
982 if (match(V: FalseVal, P: m_BinOp(L: m_Specific(V: TrueVal), R: m_Power2(V&: C2)))) {
983 Y = TrueVal;
984 BinOp = cast<BinaryOperator>(Val: FalseVal);
985 NeedXor = false;
986 } else if (match(V: TrueVal, P: m_BinOp(L: m_Specific(V: FalseVal), R: m_Power2(V&: C2)))) {
987 Y = FalseVal;
988 BinOp = cast<BinaryOperator>(Val: TrueVal);
989 NeedXor = true;
990 } else {
991 return nullptr;
992 }
993
994 // Check that 0 on RHS is identity value for this binop.
995 auto *IdentityC =
996 ConstantExpr::getBinOpIdentity(Opcode: BinOp->getOpcode(), Ty: BinOp->getType(),
997 /*AllowRHSConstant*/ true);
998 if (IdentityC == nullptr || !IdentityC->isNullValue())
999 return nullptr;
1000
1001 unsigned C2Log = C2->logBase2();
1002
1003 bool NeedShift = C1Log != C2Log;
1004 bool NeedZExtTrunc = Y->getType()->getScalarSizeInBits() !=
1005 V->getType()->getScalarSizeInBits();
1006
1007 // the demanded bits for the created shl make the and redundant
1008 if (AndMask.isOne() && C2->isSignBitSet())
1009 CreateAnd = false;
1010
1011 // Make sure we don't create more instructions than we save.
1012 if ((NeedShift + NeedXor + NeedZExtTrunc + CreateAnd) >
1013 (CondVal->hasOneUse() + BinOp->hasOneUse()))
1014 return nullptr;
1015
1016 if (CreateAnd) {
1017 // Insert the AND instruction on the input to the truncate.
1018 V = Builder.CreateAnd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: AndMask));
1019 }
1020
1021 if (C2Log > C1Log) {
1022 V = Builder.CreateZExtOrTrunc(V, DestTy: Y->getType());
1023 V = Builder.CreateShl(LHS: V, RHS: C2Log - C1Log);
1024 } else if (C1Log > C2Log) {
1025 V = Builder.CreateLShr(LHS: V, RHS: C1Log - C2Log);
1026 V = Builder.CreateZExtOrTrunc(V, DestTy: Y->getType());
1027 } else
1028 V = Builder.CreateZExtOrTrunc(V, DestTy: Y->getType());
1029
1030 if (NeedXor)
1031 V = Builder.CreateXor(LHS: V, RHS: *C2);
1032
1033 auto *Res = Builder.CreateBinOp(Opc: BinOp->getOpcode(), LHS: Y, RHS: V);
1034 if (auto *BO = dyn_cast<BinaryOperator>(Val: Res))
1035 BO->copyIRFlags(V: BinOp);
1036 return Res;
1037}
1038
1039/// Canonicalize a set or clear of a masked set of constant bits to
1040/// select-of-constants form.
1041static Instruction *foldSetClearBits(SelectInst &Sel,
1042 InstCombiner::BuilderTy &Builder) {
1043 Value *Cond = Sel.getCondition();
1044 Value *T = Sel.getTrueValue();
1045 Value *F = Sel.getFalseValue();
1046 Type *Ty = Sel.getType();
1047 Value *X;
1048 const APInt *NotC, *C;
1049
1050 // Cond ? (X & ~C) : (X | C) --> (X & ~C) | (Cond ? 0 : C)
1051 if (match(V: T, P: m_And(L: m_Value(V&: X), R: m_APInt(Res&: NotC))) &&
1052 match(V: F, P: m_OneUse(SubPattern: m_Or(L: m_Specific(V: X), R: m_APInt(Res&: C)))) && *NotC == ~(*C)) {
1053 Constant *Zero = ConstantInt::getNullValue(Ty);
1054 Constant *OrC = ConstantInt::get(Ty, V: *C);
1055 Value *NewSel = Builder.CreateSelect(C: Cond, True: Zero, False: OrC, Name: "masksel", MDFrom: &Sel);
1056 return BinaryOperator::CreateOr(V1: T, V2: NewSel);
1057 }
1058
1059 // Cond ? (X | C) : (X & ~C) --> (X & ~C) | (Cond ? C : 0)
1060 if (match(V: F, P: m_And(L: m_Value(V&: X), R: m_APInt(Res&: NotC))) &&
1061 match(V: T, P: m_OneUse(SubPattern: m_Or(L: m_Specific(V: X), R: m_APInt(Res&: C)))) && *NotC == ~(*C)) {
1062 Constant *Zero = ConstantInt::getNullValue(Ty);
1063 Constant *OrC = ConstantInt::get(Ty, V: *C);
1064 Value *NewSel = Builder.CreateSelect(C: Cond, True: OrC, False: Zero, Name: "masksel", MDFrom: &Sel);
1065 return BinaryOperator::CreateOr(V1: F, V2: NewSel);
1066 }
1067
1068 return nullptr;
1069}
1070
1071// select (x == 0), 0, x * y --> freeze(y) * x
1072// select (y == 0), 0, x * y --> freeze(x) * y
1073// select (x == 0), undef, x * y --> freeze(y) * x
1074// select (x == undef), 0, x * y --> freeze(y) * x
1075// Usage of mul instead of 0 will make the result more poisonous,
1076// so the operand that was not checked in the condition should be frozen.
1077// The latter folding is applied only when a constant compared with x is
1078// is a vector consisting of 0 and undefs. If a constant compared with x
1079// is a scalar undefined value or undefined vector then an expression
1080// should be already folded into a constant.
1081//
1082// This also holds all operations such that Op(0) == 0
1083// e.g. Shl, Umin, etc
1084static Instruction *foldSelectZeroOrFixedOp(SelectInst &SI,
1085 InstCombinerImpl &IC) {
1086 auto *CondVal = SI.getCondition();
1087 auto *TrueVal = SI.getTrueValue();
1088 auto *FalseVal = SI.getFalseValue();
1089 Value *X, *Y;
1090 CmpPredicate Predicate;
1091
1092 // Assuming that constant compared with zero is not undef (but it may be
1093 // a vector with some undef elements). Otherwise (when a constant is undef)
1094 // the select expression should be already simplified.
1095 if (!match(V: CondVal, P: m_ICmp(Pred&: Predicate, L: m_Value(V&: X), R: m_Zero())) ||
1096 !ICmpInst::isEquality(P: Predicate))
1097 return nullptr;
1098
1099 if (Predicate == ICmpInst::ICMP_NE)
1100 std::swap(a&: TrueVal, b&: FalseVal);
1101
1102 // Check that TrueVal is a constant instead of matching it with m_Zero()
1103 // to handle the case when it is a scalar undef value or a vector containing
1104 // non-zero elements that are masked by undef elements in the compare
1105 // constant.
1106 auto *TrueValC = dyn_cast<Constant>(Val: TrueVal);
1107 if (TrueValC == nullptr || !isa<Instruction>(Val: FalseVal))
1108 return nullptr;
1109
1110 bool FreezeY;
1111 if (match(V: FalseVal, P: m_c_Mul(L: m_Specific(V: X), R: m_Value(V&: Y))) ||
1112 match(V: FalseVal, P: m_c_And(L: m_Specific(V: X), R: m_Value(V&: Y))) ||
1113 match(V: FalseVal, P: m_FShl(Op0: m_Specific(V: X), Op1: m_Specific(V: X), Op2: m_Value(V&: Y))) ||
1114 match(V: FalseVal, P: m_FShr(Op0: m_Specific(V: X), Op1: m_Specific(V: X), Op2: m_Value(V&: Y))) ||
1115 match(V: FalseVal,
1116 P: m_c_Intrinsic<Intrinsic::umin>(Op0: m_Specific(V: X), Op1: m_Value(V&: Y)))) {
1117 FreezeY = true;
1118 } else if (match(V: FalseVal, P: m_IDiv(L: m_Specific(V: X), R: m_Value(V&: Y))) ||
1119 match(V: FalseVal, P: m_IRem(L: m_Specific(V: X), R: m_Value(V&: Y)))) {
1120 FreezeY = false;
1121 } else {
1122 return nullptr;
1123 }
1124
1125 auto *ZeroC = cast<Constant>(Val: cast<Instruction>(Val: CondVal)->getOperand(i: 1));
1126 auto *MergedC = Constant::mergeUndefsWith(C: TrueValC, Other: ZeroC);
1127 // If X is compared with 0 then TrueVal could be either zero or undef.
1128 // m_Zero match vectors containing some undef elements, but for scalars
1129 // m_Undef should be used explicitly.
1130 if (!match(V: MergedC, P: m_Zero()) && !match(V: MergedC, P: m_Undef()))
1131 return nullptr;
1132
1133 auto *FalseValI = cast<Instruction>(Val: FalseVal);
1134 if (FreezeY) {
1135 auto *FrY = IC.InsertNewInstBefore(New: new FreezeInst(Y, Y->getName() + ".fr"),
1136 Old: FalseValI->getIterator());
1137 IC.replaceOperand(I&: *FalseValI,
1138 OpNum: FalseValI->getOperand(i: 0) == Y
1139 ? 0
1140 : (FalseValI->getOperand(i: 1) == Y ? 1 : 2),
1141 V: FrY);
1142 }
1143 return IC.replaceInstUsesWith(I&: SI, V: FalseValI);
1144}
1145
1146/// Transform patterns such as (a > b) ? a - b : 0 into usub.sat(a, b).
1147/// There are 8 commuted/swapped variants of this pattern.
1148static Value *
1149canonicalizeSaturatedSubtractUnsigned(const ICmpInst *ICI, const Value *TrueVal,
1150 const Value *FalseVal,
1151 InstCombiner::BuilderTy &Builder) {
1152 ICmpInst::Predicate Pred = ICI->getPredicate();
1153 Value *A = ICI->getOperand(i_nocapture: 0);
1154 Value *B = ICI->getOperand(i_nocapture: 1);
1155
1156 // (b > a) ? 0 : a - b -> (b <= a) ? a - b : 0
1157 // (a == 0) ? 0 : a - 1 -> (a != 0) ? a - 1 : 0
1158 if (match(V: TrueVal, P: m_Zero())) {
1159 Pred = ICmpInst::getInversePredicate(pred: Pred);
1160 std::swap(a&: TrueVal, b&: FalseVal);
1161 }
1162
1163 if (!match(V: FalseVal, P: m_Zero()))
1164 return nullptr;
1165
1166 // ugt 0 is canonicalized to ne 0 and requires special handling
1167 // (a != 0) ? a + -1 : 0 -> usub.sat(a, 1)
1168 if (Pred == ICmpInst::ICMP_NE) {
1169 if (match(V: B, P: m_Zero()) && match(V: TrueVal, P: m_Add(L: m_Specific(V: A), R: m_AllOnes())))
1170 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::usub_sat, LHS: A,
1171 RHS: ConstantInt::get(Ty: A->getType(), V: 1));
1172 return nullptr;
1173 }
1174
1175 if (!ICmpInst::isUnsigned(Pred))
1176 return nullptr;
1177
1178 if (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_ULT) {
1179 // (b < a) ? a - b : 0 -> (a > b) ? a - b : 0
1180 std::swap(a&: A, b&: B);
1181 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
1182 }
1183
1184 assert((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_UGT) &&
1185 "Unexpected isUnsigned predicate!");
1186
1187 // Ensure the sub is of the form:
1188 // (a > b) ? a - b : 0 -> usub.sat(a, b)
1189 // (a > b) ? b - a : 0 -> -usub.sat(a, b)
1190 // Checking for both a-b and a+(-b) as a constant.
1191 bool IsNegative = false;
1192 const APInt *C;
1193 if (match(V: TrueVal, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))) ||
1194 (match(V: A, P: m_APInt(Res&: C)) &&
1195 match(V: TrueVal, P: m_Add(L: m_Specific(V: B), R: m_SpecificInt(V: -*C)))))
1196 IsNegative = true;
1197 else if (!match(V: TrueVal, P: m_Sub(L: m_Specific(V: A), R: m_Specific(V: B))) &&
1198 !(match(V: B, P: m_APInt(Res&: C)) &&
1199 match(V: TrueVal, P: m_Add(L: m_Specific(V: A), R: m_SpecificInt(V: -*C)))))
1200 return nullptr;
1201
1202 // If we are adding a negate and the sub and icmp are used anywhere else, we
1203 // would end up with more instructions.
1204 if (IsNegative && !TrueVal->hasOneUse() && !ICI->hasOneUse())
1205 return nullptr;
1206
1207 // (a > b) ? a - b : 0 -> usub.sat(a, b)
1208 // (a > b) ? b - a : 0 -> -usub.sat(a, b)
1209 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::usub_sat, LHS: A, RHS: B);
1210 if (IsNegative)
1211 Result = Builder.CreateNeg(V: Result);
1212 return Result;
1213}
1214
1215static Value *
1216canonicalizeSaturatedSubtractSigned(const ICmpInst *ICI, const Value *TrueVal,
1217 const Value *FalseVal,
1218 InstCombiner::BuilderTy &Builder) {
1219 ICmpInst::Predicate Pred = ICI->getPredicate();
1220 Value *CmpLHS = ICI->getOperand(i_nocapture: 0);
1221 Value *CmpRHS = ICI->getOperand(i_nocapture: 1);
1222
1223 // `A != B ? X : Y` --> `A == B ? Y : X`
1224 // This canonicalization allows us to handle more patterns with fewer checks.
1225 if (Pred == ICmpInst::ICMP_NE) {
1226 Pred = ICmpInst::ICMP_EQ;
1227 std::swap(a&: TrueVal, b&: FalseVal);
1228 }
1229
1230 // `A == MIN_INT ? MAX_INT : 0 - A` --> `ssub_sat 0, A`
1231 if (Pred == ICmpInst::ICMP_EQ && match(V: CmpRHS, P: m_SignMask()) &&
1232 match(V: TrueVal, P: m_MaxSignedValue()) &&
1233 match(V: FalseVal, P: m_Neg(V: m_Specific(V: CmpLHS)))) {
1234 return Builder.CreateBinaryIntrinsic(
1235 ID: Intrinsic::ssub_sat, LHS: ConstantInt::getNullValue(Ty: CmpLHS->getType()),
1236 RHS: CmpLHS);
1237 }
1238
1239 return nullptr;
1240}
1241
1242static Value *canonicalizeSaturatedSubtract(const ICmpInst *ICI,
1243 const Value *TrueVal,
1244 const Value *FalseVal,
1245 InstCombiner::BuilderTy &Builder) {
1246 if (Value *V = canonicalizeSaturatedSubtractUnsigned(ICI, TrueVal, FalseVal,
1247 Builder))
1248 return V;
1249
1250 if (Value *V =
1251 canonicalizeSaturatedSubtractSigned(ICI, TrueVal, FalseVal, Builder))
1252 return V;
1253
1254 return nullptr;
1255}
1256
1257static Value *
1258canonicalizeSaturatedAddUnsigned(ICmpInst *Cmp, Value *TVal, Value *FVal,
1259 InstCombiner::BuilderTy &Builder) {
1260
1261 // Match unsigned saturated add with constant.
1262 Value *Cmp0 = Cmp->getOperand(i_nocapture: 0);
1263 Value *Cmp1 = Cmp->getOperand(i_nocapture: 1);
1264 ICmpInst::Predicate Pred = Cmp->getPredicate();
1265 Value *X;
1266 const APInt *C;
1267
1268 // Match unsigned saturated add of 2 variables with an unnecessary 'not'.
1269 // There are 8 commuted variants.
1270 // Canonicalize -1 (saturated result) to true value of the select.
1271 if (match(V: FVal, P: m_AllOnes())) {
1272 std::swap(a&: TVal, b&: FVal);
1273 Pred = CmpInst::getInversePredicate(pred: Pred);
1274 }
1275 if (!match(V: TVal, P: m_AllOnes()))
1276 return nullptr;
1277
1278 // uge -1 is canonicalized to eq -1 and requires special handling
1279 // (a == -1) ? -1 : a + 1 -> uadd.sat(a, 1)
1280 if (Pred == ICmpInst::ICMP_EQ) {
1281 if (match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_One())) &&
1282 match(V: Cmp1, P: m_AllOnes())) {
1283 return Builder.CreateBinaryIntrinsic(
1284 ID: Intrinsic::uadd_sat, LHS: Cmp0, RHS: ConstantInt::get(Ty: Cmp0->getType(), V: 1));
1285 }
1286 return nullptr;
1287 }
1288
1289 if ((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_UGT) &&
1290 match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_APIntAllowPoison(Res&: C))) &&
1291 match(V: Cmp1, P: m_SpecificIntAllowPoison(V: ~*C))) {
1292 // (X u> ~C) ? -1 : (X + C) --> uadd.sat(X, C)
1293 // (X u>= ~C)? -1 : (X + C) --> uadd.sat(X, C)
1294 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::uadd_sat, LHS: Cmp0,
1295 RHS: ConstantInt::get(Ty: Cmp0->getType(), V: *C));
1296 }
1297
1298 // Negative one does not work here because X u> -1 ? -1, X + -1 is not a
1299 // saturated add.
1300 if (Pred == ICmpInst::ICMP_UGT &&
1301 match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_APIntAllowPoison(Res&: C))) &&
1302 match(V: Cmp1, P: m_SpecificIntAllowPoison(V: ~*C - 1)) && !C->isAllOnes()) {
1303 // (X u> ~C - 1) ? -1 : (X + C) --> uadd.sat(X, C)
1304 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::uadd_sat, LHS: Cmp0,
1305 RHS: ConstantInt::get(Ty: Cmp0->getType(), V: *C));
1306 }
1307
1308 // Zero does not work here because X u>= 0 ? -1 : X -> is always -1, which is
1309 // not a saturated add.
1310 if (Pred == ICmpInst::ICMP_UGE &&
1311 match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_APIntAllowPoison(Res&: C))) &&
1312 match(V: Cmp1, P: m_SpecificIntAllowPoison(V: -*C)) && !C->isZero()) {
1313 // (X u >= -C) ? -1 : (X + C) --> uadd.sat(X, C)
1314 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::uadd_sat, LHS: Cmp0,
1315 RHS: ConstantInt::get(Ty: Cmp0->getType(), V: *C));
1316 }
1317
1318 // Canonicalize predicate to less-than or less-or-equal-than.
1319 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
1320 std::swap(a&: Cmp0, b&: Cmp1);
1321 Pred = CmpInst::getSwappedPredicate(pred: Pred);
1322 }
1323 if (Pred != ICmpInst::ICMP_ULT && Pred != ICmpInst::ICMP_ULE)
1324 return nullptr;
1325
1326 // Match unsigned saturated add of 2 variables with an unnecessary 'not'.
1327 // Strictness of the comparison is irrelevant.
1328 Value *Y;
1329 if (match(V: Cmp0, P: m_Not(V: m_Value(V&: X))) &&
1330 match(V: FVal, P: m_c_Add(L: m_Specific(V: X), R: m_Value(V&: Y))) && Y == Cmp1) {
1331 // (~X u< Y) ? -1 : (X + Y) --> uadd.sat(X, Y)
1332 // (~X u< Y) ? -1 : (Y + X) --> uadd.sat(X, Y)
1333 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::uadd_sat, LHS: X, RHS: Y);
1334 }
1335 // The 'not' op may be included in the sum but not the compare.
1336 // Strictness of the comparison is irrelevant.
1337 X = Cmp0;
1338 Y = Cmp1;
1339 if (match(V: FVal, P: m_c_Add(L: m_NotForbidPoison(V: m_Specific(V: X)), R: m_Specific(V: Y)))) {
1340 // (X u< Y) ? -1 : (~X + Y) --> uadd.sat(~X, Y)
1341 // (X u< Y) ? -1 : (Y + ~X) --> uadd.sat(Y, ~X)
1342 BinaryOperator *BO = cast<BinaryOperator>(Val: FVal);
1343 return Builder.CreateBinaryIntrinsic(
1344 ID: Intrinsic::uadd_sat, LHS: BO->getOperand(i_nocapture: 0), RHS: BO->getOperand(i_nocapture: 1));
1345 }
1346 // The overflow may be detected via the add wrapping round.
1347 // This is only valid for strict comparison!
1348 if (Pred == ICmpInst::ICMP_ULT &&
1349 match(V: Cmp0, P: m_c_Add(L: m_Specific(V: Cmp1), R: m_Value(V&: Y))) &&
1350 match(V: FVal, P: m_c_Add(L: m_Specific(V: Cmp1), R: m_Specific(V: Y)))) {
1351 // ((X + Y) u< X) ? -1 : (X + Y) --> uadd.sat(X, Y)
1352 // ((X + Y) u< Y) ? -1 : (X + Y) --> uadd.sat(X, Y)
1353 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::uadd_sat, LHS: Cmp1, RHS: Y);
1354 }
1355
1356 return nullptr;
1357}
1358
1359static Value *canonicalizeSaturatedAddSigned(ICmpInst *Cmp, Value *TVal,
1360 Value *FVal,
1361 InstCombiner::BuilderTy &Builder) {
1362 // Match saturated add with constant.
1363 Value *Cmp0 = Cmp->getOperand(i_nocapture: 0);
1364 Value *Cmp1 = Cmp->getOperand(i_nocapture: 1);
1365 ICmpInst::Predicate Pred = Cmp->getPredicate();
1366
1367 // Canonicalize TVal to be the saturation constant.
1368 if (match(V: FVal, P: m_MaxSignedValue()) || match(V: FVal, P: m_SignMask())) {
1369 std::swap(a&: TVal, b&: FVal);
1370 Pred = CmpInst::getInversePredicate(pred: Pred);
1371 }
1372
1373 const APInt *SatC;
1374 if (!match(V: TVal, P: m_APInt(Res&: SatC)) ||
1375 !(SatC->isMaxSignedValue() || SatC->isSignMask()))
1376 return nullptr;
1377
1378 bool IsMax = SatC->isMaxSignedValue();
1379
1380 // sge maximum signed value is canonicalized to eq maximum signed value and
1381 // requires special handling. sle minimum signed value is similarly
1382 // canonicalized to eq minimum signed value.
1383 if (Pred == ICmpInst::ICMP_EQ && Cmp1 == TVal) {
1384 // (a == INT_MAX) ? INT_MAX : a + 1 -> sadd.sat(a, 1)
1385 if (IsMax && match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_One()))) {
1386 return Builder.CreateBinaryIntrinsic(
1387 ID: Intrinsic::sadd_sat, LHS: Cmp0, RHS: ConstantInt::get(Ty: Cmp0->getType(), V: 1));
1388 }
1389
1390 // (a == INT_MIN) ? INT_MIN : a + -1 -> sadd.sat(a, -1)
1391 if (!IsMax && match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_AllOnes()))) {
1392 return Builder.CreateBinaryIntrinsic(
1393 ID: Intrinsic::sadd_sat, LHS: Cmp0,
1394 RHS: ConstantInt::getAllOnesValue(Ty: Cmp0->getType()));
1395 }
1396 return nullptr;
1397 }
1398
1399 const APInt *C;
1400
1401 // (X > Y) ? INT_MAX : (X + C) --> sadd.sat(X, C)
1402 // (X >= Y) ? INT_MAX : (X + C) --> sadd.sat(X, C)
1403 // where C > 0 and Y is INT_MAX - C or INT_MAX - C - 1
1404 if (IsMax && (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) &&
1405 isa<Constant>(Val: Cmp1) &&
1406 match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_StrictlyPositive(V&: C)))) {
1407 // Normalize SGE to SGT for threshold comparison.
1408 if (Pred == ICmpInst::ICMP_SGE) {
1409 if (auto Flipped = getFlippedStrictnessPredicateAndConstant(
1410 Pred, C: cast<Constant>(Val: Cmp1))) {
1411 Pred = Flipped->first;
1412 Cmp1 = Flipped->second;
1413 }
1414 }
1415 // Check: X > INT_MAX - C or X > INT_MAX - C - 1
1416 APInt Threshold = *SatC - *C;
1417 if (Pred == ICmpInst::ICMP_SGT &&
1418 (match(V: Cmp1, P: m_SpecificIntAllowPoison(V: Threshold)) ||
1419 match(V: Cmp1, P: m_SpecificIntAllowPoison(V: Threshold - 1))))
1420 return Builder.CreateBinaryIntrinsic(
1421 ID: Intrinsic::sadd_sat, LHS: Cmp0, RHS: ConstantInt::get(Ty: Cmp0->getType(), V: *C));
1422 }
1423
1424 // (X < Y) ? INT_MIN : (X + C) --> sadd.sat(X, C)
1425 // (X <= Y) ? INT_MIN : (X + C) --> sadd.sat(X, C)
1426 // where C < 0 and Y is INT_MIN - C or INT_MIN - C + 1
1427 if (!IsMax && (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) &&
1428 isa<Constant>(Val: Cmp1) &&
1429 match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_Negative(V&: C)))) {
1430 // Normalize SLE to SLT for threshold comparison.
1431 if (Pred == ICmpInst::ICMP_SLE) {
1432 if (auto Flipped = getFlippedStrictnessPredicateAndConstant(
1433 Pred, C: cast<Constant>(Val: Cmp1))) {
1434 Pred = Flipped->first;
1435 Cmp1 = Flipped->second;
1436 }
1437 }
1438 // Check: X < INT_MIN - C or X < INT_MIN - C + 1
1439 // INT_MIN - C for negative C is like INT_MIN + |C|
1440 APInt Threshold = *SatC - *C;
1441 if (Pred == ICmpInst::ICMP_SLT &&
1442 (match(V: Cmp1, P: m_SpecificIntAllowPoison(V: Threshold)) ||
1443 match(V: Cmp1, P: m_SpecificIntAllowPoison(V: Threshold + 1))))
1444 return Builder.CreateBinaryIntrinsic(
1445 ID: Intrinsic::sadd_sat, LHS: Cmp0, RHS: ConstantInt::get(Ty: Cmp0->getType(), V: *C));
1446 }
1447
1448 // Canonicalize predicate to less-than or less-or-equal-than.
1449 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) {
1450 std::swap(a&: Cmp0, b&: Cmp1);
1451 Pred = CmpInst::getSwappedPredicate(pred: Pred);
1452 }
1453
1454 if (Pred != ICmpInst::ICMP_SLT && Pred != ICmpInst::ICMP_SLE)
1455 return nullptr;
1456
1457 Value *X;
1458
1459 // (INT_MAX - X s< Y) ? INT_MAX : (X + Y) --> sadd.sat(X, Y)
1460 // (INT_MAX - X s< Y) ? INT_MAX : (Y + X) --> sadd.sat(X, Y)
1461 if (IsMax && match(V: Cmp0, P: m_NSWSub(L: m_SpecificInt(V: *SatC), R: m_Value(V&: X))) &&
1462 match(V: FVal, P: m_c_Add(L: m_Specific(V: X), R: m_Specific(V: Cmp1)))) {
1463 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::sadd_sat, LHS: X, RHS: Cmp1);
1464 }
1465
1466 // (INT_MIN - X s> Y) ? INT_MIN : (X + Y) --> sadd.sat(X, Y)
1467 // (INT_MIN - X s> Y) ? INT_MIN : (Y + X) --> sadd.sat(X, Y)
1468 // After swapping operands from the SGT/SGE canonicalization above,
1469 // this becomes (Y s< INT_MIN - X).
1470 if (!IsMax && match(V: Cmp1, P: m_NSWSub(L: m_SpecificInt(V: *SatC), R: m_Value(V&: X))) &&
1471 match(V: FVal, P: m_c_Add(L: m_Specific(V: X), R: m_Specific(V: Cmp0)))) {
1472 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::sadd_sat, LHS: X, RHS: Cmp0);
1473 }
1474
1475 return nullptr;
1476}
1477
1478static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
1479 InstCombiner::BuilderTy &Builder) {
1480 if (!Cmp->hasOneUse())
1481 return nullptr;
1482
1483 if (Value *V = canonicalizeSaturatedAddUnsigned(Cmp, TVal, FVal, Builder))
1484 return V;
1485
1486 if (Value *V = canonicalizeSaturatedAddSigned(Cmp, TVal, FVal, Builder))
1487 return V;
1488
1489 return nullptr;
1490}
1491
1492/// Try to match patterns with select and subtract as absolute difference.
1493static Value *foldAbsDiff(ICmpInst *Cmp, Value *TVal, Value *FVal,
1494 InstCombiner::BuilderTy &Builder) {
1495 auto *TI = dyn_cast<Instruction>(Val: TVal);
1496 auto *FI = dyn_cast<Instruction>(Val: FVal);
1497 if (!TI || !FI)
1498 return nullptr;
1499
1500 // Normalize predicate to gt/lt rather than ge/le.
1501 ICmpInst::Predicate Pred = Cmp->getStrictPredicate();
1502 Value *A = Cmp->getOperand(i_nocapture: 0);
1503 Value *B = Cmp->getOperand(i_nocapture: 1);
1504
1505 // Normalize "A - B" as the true value of the select.
1506 if (match(V: FI, P: m_Sub(L: m_Specific(V: A), R: m_Specific(V: B)))) {
1507 std::swap(a&: FI, b&: TI);
1508 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
1509 }
1510
1511 // With any pair of no-wrap subtracts:
1512 // (A > B) ? (A - B) : (B - A) --> abs(A - B)
1513 if (Pred == CmpInst::ICMP_SGT &&
1514 match(V: TI, P: m_Sub(L: m_Specific(V: A), R: m_Specific(V: B))) &&
1515 match(V: FI, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))) &&
1516 (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap()) &&
1517 (FI->hasNoSignedWrap() || FI->hasNoUnsignedWrap())) {
1518 // The remaining subtract is not "nuw" any more.
1519 // If there's one use of the subtract (no other use than the use we are
1520 // about to replace), then we know that the sub is "nsw" in this context
1521 // even if it was only "nuw" before. If there's another use, then we can't
1522 // add "nsw" to the existing instruction because it may not be safe in the
1523 // other user's context.
1524 TI->setHasNoUnsignedWrap(false);
1525 if (!TI->hasNoSignedWrap())
1526 TI->setHasNoSignedWrap(TI->hasOneUse());
1527 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: TI, RHS: Builder.getTrue());
1528 }
1529
1530 // Match: (A > B) ? (A - B) : (0 - (A - B)) --> abs(A - B)
1531 if (Pred == CmpInst::ICMP_SGT &&
1532 match(V: TI, P: m_NSWSub(L: m_Specific(V: A), R: m_Specific(V: B))) &&
1533 match(V: FI, P: m_Neg(V: m_Specific(V: TI)))) {
1534 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: TI,
1535 RHS: Builder.getFalse());
1536 }
1537
1538 // Match: (A < B) ? (0 - (A - B)) : (A - B) --> abs(A - B)
1539 if (Pred == CmpInst::ICMP_SLT &&
1540 match(V: FI, P: m_NSWSub(L: m_Specific(V: A), R: m_Specific(V: B))) &&
1541 match(V: TI, P: m_Neg(V: m_Specific(V: FI)))) {
1542 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: FI,
1543 RHS: Builder.getFalse());
1544 }
1545
1546 // Match: (A > B) ? (0 - (B - A)) : (B - A) --> abs(B - A)
1547 if (Pred == CmpInst::ICMP_SGT &&
1548 match(V: FI, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A))) &&
1549 match(V: TI, P: m_Neg(V: m_Specific(V: FI)))) {
1550 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: FI,
1551 RHS: Builder.getFalse());
1552 }
1553
1554 // Match: (A < B) ? (B - A) : (0 - (B - A)) --> abs(B - A)
1555 if (Pred == CmpInst::ICMP_SLT &&
1556 match(V: TI, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A))) &&
1557 match(V: FI, P: m_Neg(V: m_Specific(V: TI)))) {
1558 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: TI,
1559 RHS: Builder.getFalse());
1560 }
1561
1562 return nullptr;
1563}
1564
1565/// Fold the following code sequence:
1566/// \code
1567/// int a = ctlz(x & -x);
1568// x ? 31 - a : 32;
1569/// \code
1570///
1571/// into:
1572/// cttz(x)
1573static Instruction *foldSelectCtlzToCttz(ICmpInst *ICI, Value *TrueVal,
1574 Value *FalseVal,
1575 InstCombiner::BuilderTy &Builder) {
1576 unsigned BitWidth = TrueVal->getType()->getScalarSizeInBits();
1577 if (!ICI->isEquality() || !match(V: ICI->getOperand(i_nocapture: 1), P: m_Zero()))
1578 return nullptr;
1579
1580 if (ICI->getPredicate() == ICmpInst::ICMP_NE)
1581 std::swap(a&: TrueVal, b&: FalseVal);
1582
1583 Value *Ctlz;
1584 if (match(V: FalseVal,
1585 P: m_Xor(L: m_Value(V&: Ctlz), R: m_SpecificIntAllowPoison(V: BitWidth - 1)))) {
1586 if (!isPowerOf2_32(Value: BitWidth))
1587 return nullptr;
1588 } else if (!match(V: FalseVal, P: m_Sub(L: m_SpecificIntAllowPoison(V: BitWidth - 1),
1589 R: m_Value(V&: Ctlz)))) {
1590 return nullptr;
1591 }
1592
1593 if (!match(V: Ctlz, P: m_Ctlz(Op0: m_Value(), Op1: m_Value())))
1594 return nullptr;
1595
1596 if (!match(V: TrueVal, P: m_SpecificInt(V: BitWidth)))
1597 return nullptr;
1598
1599 Value *X = ICI->getOperand(i_nocapture: 0);
1600 auto *II = cast<IntrinsicInst>(Val: Ctlz);
1601 if (!match(V: II->getOperand(i_nocapture: 0), P: m_c_And(L: m_Specific(V: X), R: m_Neg(V: m_Specific(V: X)))))
1602 return nullptr;
1603
1604 // The original select returns the constant bitwidth when x == 0, so the
1605 // result is defined there; the cttz must use is_zero_poison = false.
1606 Function *F = Intrinsic::getOrInsertDeclaration(
1607 M: II->getModule(), id: Intrinsic::cttz, OverloadTys: II->getType());
1608 return CallInst::Create(Func: F, Args: {X, Builder.getFalse()});
1609}
1610
1611/// Attempt to fold a cttz/ctlz followed by a icmp plus select into a single
1612/// call to cttz/ctlz with flag 'is_zero_poison' cleared.
1613///
1614/// For example, we can fold the following code sequence:
1615/// \code
1616/// %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
1617/// %1 = icmp ne i32 %x, 0
1618/// %2 = select i1 %1, i32 %0, i32 32
1619/// \code
1620///
1621/// into:
1622/// %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 false)
1623static Value *foldSelectCttzCtlz(ICmpInst *ICI, Value *TrueVal, Value *FalseVal,
1624 InstCombinerImpl &IC) {
1625 ICmpInst::Predicate Pred = ICI->getPredicate();
1626 Value *CmpLHS = ICI->getOperand(i_nocapture: 0);
1627 Value *CmpRHS = ICI->getOperand(i_nocapture: 1);
1628
1629 // Check if the select condition compares a value for equality.
1630 if (!ICI->isEquality())
1631 return nullptr;
1632
1633 Value *SelectArg = FalseVal;
1634 Value *ValueOnZero = TrueVal;
1635 if (Pred == ICmpInst::ICMP_NE)
1636 std::swap(a&: SelectArg, b&: ValueOnZero);
1637
1638 // Skip zero extend/truncate.
1639 Value *Count = nullptr;
1640 if (!match(V: SelectArg, P: m_ZExt(Op: m_Value(V&: Count))) &&
1641 !match(V: SelectArg, P: m_Trunc(Op: m_Value(V&: Count))))
1642 Count = SelectArg;
1643
1644 // Check that 'Count' is a call to intrinsic cttz/ctlz. Also check that the
1645 // input to the cttz/ctlz is used as LHS for the compare instruction.
1646 Value *X;
1647 if (!match(V: Count, P: m_Cttz(Op0: m_Value(V&: X), Op1: m_Value())) &&
1648 !match(V: Count, P: m_Ctlz(Op0: m_Value(V&: X), Op1: m_Value())))
1649 return nullptr;
1650
1651 // (X == 0) ? BitWidth : ctz(X)
1652 // (X == -1) ? BitWidth : ctz(~X)
1653 // (X == Y) ? BitWidth : ctz(X ^ Y)
1654 if ((X != CmpLHS || !match(V: CmpRHS, P: m_Zero())) &&
1655 (!match(V: X, P: m_Not(V: m_Specific(V: CmpLHS))) || !match(V: CmpRHS, P: m_AllOnes())) &&
1656 !match(V: X, P: m_c_Xor(L: m_Specific(V: CmpLHS), R: m_Specific(V: CmpRHS))))
1657 return nullptr;
1658
1659 IntrinsicInst *II = cast<IntrinsicInst>(Val: Count);
1660
1661 // Check if the value propagated on zero is a constant number equal to the
1662 // sizeof in bits of 'Count'.
1663 unsigned SizeOfInBits = Count->getType()->getScalarSizeInBits();
1664 if (match(V: ValueOnZero, P: m_SpecificInt(V: SizeOfInBits))) {
1665 // A range annotation on the intrinsic may no longer be valid.
1666 II->dropPoisonGeneratingAnnotations();
1667 IC.addToWorklist(I: II);
1668 return SelectArg;
1669 }
1670
1671 // The ValueOnZero is not the bitwidth. But if the cttz/ctlz (and optional
1672 // zext/trunc) have one use (ending at the select), the cttz/ctlz result will
1673 // not be used if the input is zero. Relax to 'zero is poison' for that case.
1674 if (II->hasOneUse() && SelectArg->hasOneUse() &&
1675 !match(V: II->getArgOperand(i: 1), P: m_One())) {
1676 II->setArgOperand(i: 1, v: ConstantInt::getTrue(Context&: II->getContext()));
1677 // noundef attribute on the intrinsic may no longer be valid.
1678 II->dropUBImplyingAttrsAndMetadata();
1679 IC.addToWorklist(I: II);
1680 }
1681
1682 return nullptr;
1683}
1684
1685/// SPF can match patterns like:
1686/// (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
1687/// However, if the UMIN has poison-generating annotations, the result may
1688/// be more poisonous, because the poison is propagated unconditionally after
1689/// the transform. Drop annotations in this case.
1690/// FIXME: This is something of a hack that relies on details of patterns
1691// supported by SPF.
1692static void dropPoisonGeneratingAnnotationsForSPFCanonicalization(Value *Cond,
1693 Value *LHS,
1694 Value *RHS) {
1695 if (!impliesPoison(ValAssumedPoison: LHS, V: Cond))
1696 if (auto *I = dyn_cast<Instruction>(Val: LHS))
1697 I->dropPoisonGeneratingAnnotations();
1698 if (!impliesPoison(ValAssumedPoison: RHS, V: Cond))
1699 if (auto *I = dyn_cast<Instruction>(Val: RHS))
1700 I->dropPoisonGeneratingAnnotations();
1701}
1702
1703static Value *canonicalizeSPF(ICmpInst &Cmp, Value *TrueVal, Value *FalseVal,
1704 InstCombinerImpl &IC) {
1705 Value *LHS, *RHS;
1706 // TODO: What to do with pointer min/max patterns?
1707 if (!TrueVal->getType()->isIntOrIntVectorTy())
1708 return nullptr;
1709
1710 SelectPatternFlavor SPF =
1711 matchDecomposedSelectPattern(CmpI: &Cmp, TrueVal, FalseVal, LHS, RHS).Flavor;
1712 if (SPF == SelectPatternFlavor::SPF_ABS ||
1713 SPF == SelectPatternFlavor::SPF_NABS) {
1714 if (!Cmp.hasOneUse() && !RHS->hasOneUse())
1715 return nullptr; // TODO: Relax this restriction.
1716
1717 // Note that NSW flag can only be propagated for normal, non-negated abs!
1718 bool IntMinIsPoison = SPF == SelectPatternFlavor::SPF_ABS &&
1719 match(V: RHS, P: m_NSWNeg(V: m_Specific(V: LHS)));
1720 Constant *IntMinIsPoisonC =
1721 ConstantInt::get(Ty: Type::getInt1Ty(C&: Cmp.getContext()), V: IntMinIsPoison);
1722 Value *Abs =
1723 IC.Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS, RHS: IntMinIsPoisonC);
1724
1725 if (SPF == SelectPatternFlavor::SPF_NABS)
1726 return IC.Builder.CreateNeg(V: Abs); // Always without NSW flag!
1727 return Abs;
1728 }
1729
1730 if (SelectPatternResult::isMinOrMax(SPF)) {
1731 Intrinsic::ID IntrinsicID = getMinMaxIntrinsic(SPF);
1732 dropPoisonGeneratingAnnotationsForSPFCanonicalization(Cond: &Cmp, LHS, RHS);
1733 return IC.Builder.CreateBinaryIntrinsic(ID: IntrinsicID, LHS, RHS);
1734 }
1735
1736 return nullptr;
1737}
1738
1739bool InstCombinerImpl::replaceInInstruction(Value *V, Value *Old, Value *New,
1740 unsigned Depth) {
1741 // Conservatively limit replacement to two instructions upwards.
1742 if (Depth == 2)
1743 return false;
1744
1745 assert(!isa<Constant>(Old) && "Only replace non-constant values");
1746
1747 auto *I = dyn_cast<Instruction>(Val: V);
1748 if (!I || !I->hasOneUse() ||
1749 !isSafeToSpeculativelyExecuteWithVariableReplaced(I))
1750 return false;
1751
1752 // Forbid potentially lane-crossing instructions.
1753 if (Old->getType()->isVectorTy() && !isNotCrossLaneOperation(I))
1754 return false;
1755
1756 bool Changed = false;
1757 for (Use &U : I->operands()) {
1758 if (U == Old) {
1759 replaceUse(U, NewValue: New);
1760 Worklist.add(I);
1761 Changed = true;
1762 } else {
1763 Changed |= replaceInInstruction(V: U, Old, New, Depth: Depth + 1);
1764 }
1765 }
1766 return Changed;
1767}
1768
1769/// If we have a select with an equality comparison, then we know the value in
1770/// one of the arms of the select. See if substituting this value into an arm
1771/// and simplifying the result yields the same value as the other arm.
1772///
1773/// To make this transform safe, we must drop poison-generating flags
1774/// (nsw, etc) if we simplified to a binop because the select may be guarding
1775/// that poison from propagating. If the existing binop already had no
1776/// poison-generating flags, then this transform can be done by instsimplify.
1777///
1778/// Consider:
1779/// %cmp = icmp eq i32 %x, 2147483647
1780/// %add = add nsw i32 %x, 1
1781/// %sel = select i1 %cmp, i32 -2147483648, i32 %add
1782///
1783/// We can't replace %sel with %add unless we strip away the flags.
1784/// TODO: Wrapping flags could be preserved in some cases with better analysis.
1785Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
1786 CmpInst &Cmp) {
1787 // Canonicalize the pattern to an equivalence on the predicate by swapping the
1788 // select operands.
1789 Value *TrueVal = Sel.getTrueValue(), *FalseVal = Sel.getFalseValue();
1790 bool Swapped = false;
1791 if (Cmp.isEquivalence(/*Invert=*/true)) {
1792 std::swap(a&: TrueVal, b&: FalseVal);
1793 Swapped = true;
1794 } else if (!Cmp.isEquivalence()) {
1795 return nullptr;
1796 }
1797
1798 Value *CmpLHS = Cmp.getOperand(i_nocapture: 0), *CmpRHS = Cmp.getOperand(i_nocapture: 1);
1799 auto ReplaceOldOpWithNewOp = [&](Value *OldOp,
1800 Value *NewOp) -> Instruction * {
1801 // In X == Y ? f(X) : Z, try to evaluate f(Y) and replace the operand.
1802 // Take care to avoid replacing X == Y ? X : Z with X == Y ? Y : Z, as that
1803 // would lead to an infinite replacement cycle.
1804 // If we will be able to evaluate f(Y) to a constant, we can allow undef,
1805 // otherwise Y cannot be undef as we might pick different values for undef
1806 // in the cmp and in f(Y).
1807 if (TrueVal == OldOp && (isa<Constant>(Val: OldOp) || !isa<Constant>(Val: NewOp)))
1808 return nullptr;
1809
1810 if (Value *V = simplifyWithOpReplaced(V: TrueVal, Op: OldOp, RepOp: NewOp, Q: SQ,
1811 /* AllowRefinement=*/true)) {
1812 // Need some guarantees about the new simplified op to ensure we don't inf
1813 // loop.
1814 // If we simplify to a constant, replace if we aren't creating new undef.
1815 if (match(V, P: m_ImmConstant()) &&
1816 isGuaranteedNotToBeUndef(V, AC: SQ.AC, CtxI: &Sel, DT: &DT))
1817 return replaceOperand(I&: Sel, OpNum: Swapped ? 2 : 1, V);
1818
1819 // If NewOp is a constant and OldOp is not replace iff NewOp doesn't
1820 // contain and undef elements.
1821 // Make sure that V is always simpler than TrueVal, otherwise we might
1822 // end up in an infinite loop.
1823 if (match(V: NewOp, P: m_ImmConstant()) ||
1824 (isa<Instruction>(Val: TrueVal) &&
1825 is_contained(Range: cast<Instruction>(Val: TrueVal)->operands(), Element: V))) {
1826 if (isGuaranteedNotToBeUndef(V: NewOp, AC: SQ.AC, CtxI: &Sel, DT: &DT))
1827 return replaceOperand(I&: Sel, OpNum: Swapped ? 2 : 1, V);
1828 return nullptr;
1829 }
1830 }
1831
1832 // Even if TrueVal does not simplify, we can directly replace a use of
1833 // CmpLHS with CmpRHS, as long as the instruction is not used anywhere
1834 // else and is safe to speculatively execute (we may end up executing it
1835 // with different operands, which should not cause side-effects or trigger
1836 // undefined behavior). Only do this if CmpRHS is a constant, as
1837 // profitability is not clear for other cases.
1838 if (OldOp == CmpLHS && match(V: NewOp, P: m_ImmConstant()) &&
1839 !match(V: OldOp, P: m_Constant()) &&
1840 isGuaranteedNotToBeUndef(V: NewOp, AC: SQ.AC, CtxI: &Sel, DT: &DT))
1841 if (replaceInInstruction(V: TrueVal, Old: OldOp, New: NewOp))
1842 return &Sel;
1843 return nullptr;
1844 };
1845
1846 bool CanReplaceCmpLHSWithRHS = canReplacePointersIfEqual(From: CmpLHS, To: CmpRHS, DL);
1847 if (CanReplaceCmpLHSWithRHS) {
1848 if (Instruction *R = ReplaceOldOpWithNewOp(CmpLHS, CmpRHS))
1849 return R;
1850 }
1851 bool CanReplaceCmpRHSWithLHS = canReplacePointersIfEqual(From: CmpRHS, To: CmpLHS, DL);
1852 if (CanReplaceCmpRHSWithLHS) {
1853 if (Instruction *R = ReplaceOldOpWithNewOp(CmpRHS, CmpLHS))
1854 return R;
1855 }
1856
1857 auto *FalseInst = dyn_cast<Instruction>(Val: FalseVal);
1858 if (!FalseInst)
1859 return nullptr;
1860
1861 // InstSimplify already performed this fold if it was possible subject to
1862 // current poison-generating flags. Check whether dropping poison-generating
1863 // flags enables the transform.
1864
1865 // Try each equivalence substitution possibility.
1866 // We have an 'EQ' comparison, so the select's false value will propagate.
1867 // Example:
1868 // (X == 42) ? 43 : (X + 1) --> (X == 42) ? (X + 1) : (X + 1) --> X + 1
1869 SmallVector<Instruction *> DropFlags;
1870 if ((CanReplaceCmpLHSWithRHS &&
1871 simplifyWithOpReplaced(V: FalseVal, Op: CmpLHS, RepOp: CmpRHS, Q: SQ,
1872 /* AllowRefinement */ false,
1873 DropFlags: &DropFlags) == TrueVal) ||
1874 (CanReplaceCmpRHSWithLHS &&
1875 simplifyWithOpReplaced(V: FalseVal, Op: CmpRHS, RepOp: CmpLHS, Q: SQ,
1876 /* AllowRefinement */ false,
1877 DropFlags: &DropFlags) == TrueVal)) {
1878 for (Instruction *I : DropFlags) {
1879 I->dropPoisonGeneratingAnnotations();
1880 Worklist.add(I);
1881 }
1882
1883 return replaceInstUsesWith(I&: Sel, V: FalseVal);
1884 }
1885
1886 Constant *CmpC;
1887 if (FalseVal->getType()->isIntOrIntVectorTy(BitWidth: 1) &&
1888 match(V: FalseVal, P: m_NUWTrunc(Op: m_Specific(V: CmpLHS))) &&
1889 match(V: CmpRHS, P: m_ImmConstant(C&: CmpC)) &&
1890 ConstantFoldCompareInstOperands(
1891 Predicate: ICmpInst::Predicate::ICMP_NE, LHS: CmpC,
1892 RHS: ConstantInt::getNullValue(Ty: CmpLHS->getType()), DL) == TrueVal) {
1893 return new ICmpInst(CmpInst::Predicate::ICMP_NE, CmpLHS,
1894 ConstantInt::getNullValue(Ty: CmpLHS->getType()));
1895 }
1896
1897 return nullptr;
1898}
1899
1900/// Fold the following code sequence:
1901/// \code
1902/// %XeqZ = icmp eq i64 %X, %Z
1903/// %YeqZ = icmp eq i64 %Y, %Z
1904/// %XeqY = icmp eq i64 %X, %Y
1905/// %not.YeqZ = xor i1 %YeqZ, true
1906/// %and = select i1 %not.YeqZ, i1 %XeqY, i1 false
1907/// %equal = select i1 %XeqZ, i1 %YeqZ, i1 %and
1908/// \code
1909///
1910/// into:
1911/// %equal = icmp eq i64 %X, %Y
1912Instruction *InstCombinerImpl::foldSelectEqualityTest(SelectInst &Sel) {
1913 Value *X, *Y, *Z;
1914 Value *XeqY, *XeqZ = Sel.getCondition(), *YeqZ = Sel.getTrueValue();
1915
1916 if (!match(V: XeqZ, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Value(V&: X), R: m_Value(V&: Z))))
1917 return nullptr;
1918
1919 if (!match(V: YeqZ,
1920 P: m_c_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Value(V&: Y), R: m_Specific(V: Z))))
1921 std::swap(a&: X, b&: Z);
1922
1923 if (!match(V: YeqZ,
1924 P: m_c_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Value(V&: Y), R: m_Specific(V: Z))))
1925 return nullptr;
1926
1927 if (!match(V: Sel.getFalseValue(),
1928 P: m_c_LogicalAnd(L: m_Not(V: m_Specific(V: YeqZ)), R: m_Value(V&: XeqY))))
1929 return nullptr;
1930
1931 if (!match(V: XeqY,
1932 P: m_c_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Specific(V: X), R: m_Specific(V: Y))))
1933 return nullptr;
1934
1935 cast<ICmpInst>(Val: XeqY)->setSameSign(false);
1936 return replaceInstUsesWith(I&: Sel, V: XeqY);
1937}
1938
1939// See if this is a pattern like:
1940// %old_cmp1 = icmp slt i32 %x, C2
1941// %old_replacement = select i1 %old_cmp1, i32 %target_low, i32 %target_high
1942// %old_x_offseted = add i32 %x, C1
1943// %old_cmp0 = icmp ult i32 %old_x_offseted, C0
1944// %r = select i1 %old_cmp0, i32 %x, i32 %old_replacement
1945// This can be rewritten as more canonical pattern:
1946// %new_cmp1 = icmp slt i32 %x, -C1
1947// %new_cmp2 = icmp sge i32 %x, C0-C1
1948// %new_clamped_low = select i1 %new_cmp1, i32 %target_low, i32 %x
1949// %r = select i1 %new_cmp2, i32 %target_high, i32 %new_clamped_low
1950// Iff -C1 s<= C2 s<= C0-C1
1951// Also ULT predicate can also be UGT iff C0 != -1 (+invert result)
1952// SLT predicate can also be SGT iff C2 != INT_MAX (+invert res.)
1953static Value *canonicalizeClampLike(SelectInst &Sel0, ICmpInst &Cmp0,
1954 InstCombiner::BuilderTy &Builder,
1955 InstCombiner &IC) {
1956 Value *X = Sel0.getTrueValue();
1957 Value *Sel1 = Sel0.getFalseValue();
1958
1959 // First match the condition of the outermost select.
1960 // Said condition must be one-use.
1961 if (!Cmp0.hasOneUse())
1962 return nullptr;
1963 ICmpInst::Predicate Pred0 = Cmp0.getPredicate();
1964 Value *Cmp00 = Cmp0.getOperand(i_nocapture: 0);
1965 Constant *C0;
1966 if (!match(V: Cmp0.getOperand(i_nocapture: 1),
1967 P: m_CombineAnd(Ps: m_AnyIntegralConstant(), Ps: m_Constant(C&: C0))))
1968 return nullptr;
1969
1970 if (!match(V: Sel1, P: m_SelectLike(C: m_Value(), TrueC: m_Value(), FalseC: m_Value()))) {
1971 Pred0 = ICmpInst::getInversePredicate(pred: Pred0);
1972 std::swap(a&: X, b&: Sel1);
1973 }
1974
1975 // Canonicalize Cmp0 into ult or uge.
1976 // FIXME: we shouldn't care about lanes that are 'undef' in the end?
1977 switch (Pred0) {
1978 case ICmpInst::Predicate::ICMP_ULT:
1979 case ICmpInst::Predicate::ICMP_UGE:
1980 // Although icmp ult %x, 0 is an unusual thing to try and should generally
1981 // have been simplified, it does not verify with undef inputs so ensure we
1982 // are not in a strange state.
1983 if (!match(V: C0, P: m_SpecificInt_ICMP(
1984 Predicate: ICmpInst::Predicate::ICMP_NE,
1985 Threshold: APInt::getZero(numBits: C0->getType()->getScalarSizeInBits()))))
1986 return nullptr;
1987 break; // Great!
1988 case ICmpInst::Predicate::ICMP_ULE:
1989 case ICmpInst::Predicate::ICMP_UGT:
1990 // We want to canonicalize it to 'ult' or 'uge', so we'll need to increment
1991 // C0, which again means it must not have any all-ones elements.
1992 if (!match(V: C0,
1993 P: m_SpecificInt_ICMP(
1994 Predicate: ICmpInst::Predicate::ICMP_NE,
1995 Threshold: APInt::getAllOnes(numBits: C0->getType()->getScalarSizeInBits()))))
1996 return nullptr; // Can't do, have all-ones element[s].
1997 Pred0 = ICmpInst::getFlippedStrictnessPredicate(pred: Pred0);
1998 C0 = InstCombiner::AddOne(C: C0);
1999 break;
2000 default:
2001 return nullptr; // Unknown predicate.
2002 }
2003
2004 // Now that we've canonicalized the ICmp, we know the X we expect;
2005 // the select in other hand should be one-use.
2006 if (!Sel1->hasOneUse())
2007 return nullptr;
2008
2009 // If the types do not match, look through any truncs to the underlying
2010 // instruction.
2011 if (Cmp00->getType() != X->getType() && X->hasOneUse())
2012 match(V: X, P: m_TruncOrSelf(Op: m_Value(V&: X)));
2013
2014 // We now can finish matching the condition of the outermost select:
2015 // it should either be the X itself, or an addition of some constant to X.
2016 Constant *C1;
2017 if (Cmp00 == X)
2018 C1 = ConstantInt::getNullValue(Ty: X->getType());
2019 else if (!match(V: Cmp00,
2020 P: m_Add(L: m_Specific(V: X),
2021 R: m_CombineAnd(Ps: m_AnyIntegralConstant(), Ps: m_Constant(C&: C1)))))
2022 return nullptr;
2023
2024 Value *Cmp1;
2025 CmpPredicate Pred1;
2026 Constant *C2;
2027 Value *ReplacementLow, *ReplacementHigh;
2028 if (!match(V: Sel1, P: m_SelectLike(C: m_Value(V&: Cmp1), TrueC: m_Value(V&: ReplacementLow),
2029 FalseC: m_Value(V&: ReplacementHigh))) ||
2030 !match(V: Cmp1,
2031 P: m_ICmp(Pred&: Pred1, L: m_Specific(V: X),
2032 R: m_CombineAnd(Ps: m_AnyIntegralConstant(), Ps: m_Constant(C&: C2)))))
2033 return nullptr;
2034
2035 if (!Cmp1->hasOneUse() && (Cmp00 == X || !Cmp00->hasOneUse()))
2036 return nullptr; // Not enough one-use instructions for the fold.
2037 // FIXME: this restriction could be relaxed if Cmp1 can be reused as one of
2038 // two comparisons we'll need to build.
2039
2040 // Canonicalize Cmp1 into the form we expect.
2041 // FIXME: we shouldn't care about lanes that are 'undef' in the end?
2042 switch (Pred1) {
2043 case ICmpInst::Predicate::ICMP_SLT:
2044 break;
2045 case ICmpInst::Predicate::ICMP_SLE:
2046 // We'd have to increment C2 by one, and for that it must not have signed
2047 // max element, but then it would have been canonicalized to 'slt' before
2048 // we get here. So we can't do anything useful with 'sle'.
2049 return nullptr;
2050 case ICmpInst::Predicate::ICMP_SGT:
2051 // We want to canonicalize it to 'slt', so we'll need to increment C2,
2052 // which again means it must not have any signed max elements.
2053 if (!match(V: C2,
2054 P: m_SpecificInt_ICMP(Predicate: ICmpInst::Predicate::ICMP_NE,
2055 Threshold: APInt::getSignedMaxValue(
2056 numBits: C2->getType()->getScalarSizeInBits()))))
2057 return nullptr; // Can't do, have signed max element[s].
2058 C2 = InstCombiner::AddOne(C: C2);
2059 [[fallthrough]];
2060 case ICmpInst::Predicate::ICMP_SGE:
2061 // Also non-canonical, but here we don't need to change C2,
2062 // so we don't have any restrictions on C2, so we can just handle it.
2063 Pred1 = ICmpInst::Predicate::ICMP_SLT;
2064 std::swap(a&: ReplacementLow, b&: ReplacementHigh);
2065 break;
2066 default:
2067 return nullptr; // Unknown predicate.
2068 }
2069 assert(Pred1 == ICmpInst::Predicate::ICMP_SLT &&
2070 "Unexpected predicate type.");
2071
2072 // The thresholds of this clamp-like pattern.
2073 auto *ThresholdLowIncl = ConstantExpr::getNeg(C: C1);
2074 auto *ThresholdHighExcl = ConstantExpr::getSub(C1: C0, C2: C1);
2075
2076 assert((Pred0 == ICmpInst::Predicate::ICMP_ULT ||
2077 Pred0 == ICmpInst::Predicate::ICMP_UGE) &&
2078 "Unexpected predicate type.");
2079 if (Pred0 == ICmpInst::Predicate::ICMP_UGE)
2080 std::swap(a&: ThresholdLowIncl, b&: ThresholdHighExcl);
2081
2082 // The fold has a precondition 1: C2 s>= ThresholdLow
2083 auto *Precond1 = ConstantFoldCompareInstOperands(
2084 Predicate: ICmpInst::Predicate::ICMP_SGE, LHS: C2, RHS: ThresholdLowIncl, DL: IC.getDataLayout());
2085 if (!Precond1 || !match(V: Precond1, P: m_One()))
2086 return nullptr;
2087 // The fold has a precondition 2: C2 s<= ThresholdHigh
2088 auto *Precond2 = ConstantFoldCompareInstOperands(
2089 Predicate: ICmpInst::Predicate::ICMP_SLE, LHS: C2, RHS: ThresholdHighExcl, DL: IC.getDataLayout());
2090 if (!Precond2 || !match(V: Precond2, P: m_One()))
2091 return nullptr;
2092
2093 // If we are matching from a truncated input, we need to sext the
2094 // ReplacementLow and ReplacementHigh values. Only do the transform if they
2095 // are free to extend due to being constants.
2096 if (X->getType() != Sel0.getType()) {
2097 Constant *LowC, *HighC;
2098 if (!match(V: ReplacementLow, P: m_ImmConstant(C&: LowC)) ||
2099 !match(V: ReplacementHigh, P: m_ImmConstant(C&: HighC)))
2100 return nullptr;
2101 const DataLayout &DL = Sel0.getDataLayout();
2102 ReplacementLow =
2103 ConstantFoldCastOperand(Opcode: Instruction::SExt, C: LowC, DestTy: X->getType(), DL);
2104 ReplacementHigh =
2105 ConstantFoldCastOperand(Opcode: Instruction::SExt, C: HighC, DestTy: X->getType(), DL);
2106 assert(ReplacementLow && ReplacementHigh &&
2107 "Constant folding of ImmConstant cannot fail");
2108 }
2109
2110 // We mark the select instructions below as having an unknown profile as it is
2111 // not possible to recover profile information from the original selects in
2112 // the general case. From them we can only know the probability that we clamp
2113 // whereas we need the probabilities for clamping specific to the low end/high
2114 // end.
2115
2116 // All good, finally emit the new pattern.
2117 Value *ShouldReplaceLow = Builder.CreateICmpSLT(LHS: X, RHS: ThresholdLowIncl);
2118 Value *ShouldReplaceHigh = Builder.CreateICmpSGE(LHS: X, RHS: ThresholdHighExcl);
2119 Value *MaybeReplacedLow = Builder.CreateSelectWithUnknownProfile(
2120 C: ShouldReplaceLow, True: ReplacementLow, False: X, DEBUG_TYPE);
2121
2122 // Create the final select. If we looked through a truncate above, we will
2123 // need to retruncate the result.
2124 Value *MaybeReplacedHigh = Builder.CreateSelectWithUnknownProfile(
2125 C: ShouldReplaceHigh, True: ReplacementHigh, False: MaybeReplacedLow, DEBUG_TYPE);
2126 return Builder.CreateTrunc(V: MaybeReplacedHigh, DestTy: Sel0.getType());
2127}
2128
2129// If we have
2130// %cmp = icmp [canonical predicate] i32 %x, C0
2131// %r = select i1 %cmp, i32 %y, i32 C1
2132// Where C0 != C1 and %x may be different from %y, see if the constant that we
2133// will have if we flip the strictness of the predicate (i.e. without changing
2134// the result) is identical to the C1 in select. If it matches we can change
2135// original comparison to one with swapped predicate, reuse the constant,
2136// and swap the hands of select.
2137static Instruction *
2138tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp,
2139 InstCombinerImpl &IC) {
2140 CmpPredicate Pred;
2141 Value *X;
2142 Constant *C0;
2143 if (!match(V: &Cmp, P: m_OneUse(SubPattern: m_ICmp(
2144 Pred, L: m_Value(V&: X),
2145 R: m_CombineAnd(Ps: m_AnyIntegralConstant(), Ps: m_Constant(C&: C0))))))
2146 return nullptr;
2147
2148 // If comparison predicate is non-relational, we won't be able to do anything.
2149 if (ICmpInst::isEquality(P: Pred))
2150 return nullptr;
2151
2152 // If comparison predicate is non-canonical, then we certainly won't be able
2153 // to make it canonical; canonicalizeCmpWithConstant() already tried.
2154 if (!InstCombiner::isCanonicalPredicate(Pred))
2155 return nullptr;
2156
2157 // If the [input] type of comparison and select type are different, lets abort
2158 // for now. We could try to compare constants with trunc/[zs]ext though.
2159 if (C0->getType() != Sel.getType())
2160 return nullptr;
2161
2162 // ULT with 'add' of a constant is canonical. See foldICmpAddConstant().
2163 // FIXME: Are there more magic icmp predicate+constant pairs we must avoid?
2164 // Or should we just abandon this transform entirely?
2165 if (Pred == CmpInst::ICMP_ULT && match(V: X, P: m_Add(L: m_Value(), R: m_Constant())))
2166 return nullptr;
2167
2168
2169 Value *SelVal0, *SelVal1; // We do not care which one is from where.
2170 match(V: &Sel, P: m_Select(C: m_Value(), L: m_Value(V&: SelVal0), R: m_Value(V&: SelVal1)));
2171 // At least one of these values we are selecting between must be a constant
2172 // else we'll never succeed.
2173 if (!match(V: SelVal0, P: m_AnyIntegralConstant()) &&
2174 !match(V: SelVal1, P: m_AnyIntegralConstant()))
2175 return nullptr;
2176
2177 // Does this constant C match any of the `select` values?
2178 auto MatchesSelectValue = [SelVal0, SelVal1](Constant *C) {
2179 return C->isElementWiseEqual(Y: SelVal0) || C->isElementWiseEqual(Y: SelVal1);
2180 };
2181
2182 // If C0 *already* matches true/false value of select, we are done.
2183 if (MatchesSelectValue(C0))
2184 return nullptr;
2185
2186 // Check the constant we'd have with flipped-strictness predicate.
2187 auto FlippedStrictness = getFlippedStrictnessPredicateAndConstant(Pred, C: C0);
2188 if (!FlippedStrictness)
2189 return nullptr;
2190
2191 // If said constant doesn't match either, then there is no hope,
2192 if (!MatchesSelectValue(FlippedStrictness->second))
2193 return nullptr;
2194
2195 // It matched! Lets insert the new comparison just before select.
2196 InstCombiner::BuilderTy::InsertPointGuard Guard(IC.Builder);
2197 IC.Builder.SetInsertPoint(&Sel);
2198
2199 Pred = ICmpInst::getSwappedPredicate(pred: Pred); // Yes, swapped.
2200 Value *NewCmp = IC.Builder.CreateICmp(P: Pred, LHS: X, RHS: FlippedStrictness->second,
2201 Name: Cmp.getName() + ".inv");
2202 IC.replaceOperand(I&: Sel, OpNum: 0, V: NewCmp);
2203 Sel.swapValues();
2204 Sel.swapProfMetadata();
2205
2206 return &Sel;
2207}
2208
2209static Instruction *foldSelectZeroOrOnes(ICmpInst *Cmp, Value *TVal,
2210 Value *FVal,
2211 InstCombiner::BuilderTy &Builder) {
2212 if (!Cmp->hasOneUse())
2213 return nullptr;
2214
2215 const APInt *CmpC;
2216 if (!match(V: Cmp->getOperand(i_nocapture: 1), P: m_APIntAllowPoison(Res&: CmpC)))
2217 return nullptr;
2218
2219 // (X u< 2) ? -X : -1 --> sext (X != 0)
2220 Value *X = Cmp->getOperand(i_nocapture: 0);
2221 if (Cmp->getPredicate() == ICmpInst::ICMP_ULT && *CmpC == 2 &&
2222 match(V: TVal, P: m_Neg(V: m_Specific(V: X))) && match(V: FVal, P: m_AllOnes()))
2223 return new SExtInst(Builder.CreateIsNotNull(Arg: X), TVal->getType());
2224
2225 // (X u> 1) ? -1 : -X --> sext (X != 0)
2226 if (Cmp->getPredicate() == ICmpInst::ICMP_UGT && *CmpC == 1 &&
2227 match(V: FVal, P: m_Neg(V: m_Specific(V: X))) && match(V: TVal, P: m_AllOnes()))
2228 return new SExtInst(Builder.CreateIsNotNull(Arg: X), TVal->getType());
2229
2230 return nullptr;
2231}
2232
2233static Value *foldSelectInstWithICmpConst(SelectInst &SI, ICmpInst *ICI,
2234 InstCombiner::BuilderTy &Builder) {
2235 const APInt *CmpC;
2236 Value *V;
2237 CmpPredicate Pred;
2238 if (!match(V: ICI, P: m_ICmp(Pred, L: m_Value(V), R: m_APInt(Res&: CmpC))))
2239 return nullptr;
2240
2241 // Match clamp away from min/max value as a max/min operation.
2242 Value *TVal = SI.getTrueValue();
2243 Value *FVal = SI.getFalseValue();
2244 if (Pred == ICmpInst::ICMP_EQ && V == FVal) {
2245 // (V == UMIN) ? UMIN+1 : V --> umax(V, UMIN+1)
2246 if (CmpC->isMinValue() && match(V: TVal, P: m_SpecificInt(V: *CmpC + 1)))
2247 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::umax, LHS: V, RHS: TVal);
2248 // (V == UMAX) ? UMAX-1 : V --> umin(V, UMAX-1)
2249 if (CmpC->isMaxValue() && match(V: TVal, P: m_SpecificInt(V: *CmpC - 1)))
2250 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: V, RHS: TVal);
2251 // (V == SMIN) ? SMIN+1 : V --> smax(V, SMIN+1)
2252 if (CmpC->isMinSignedValue() && match(V: TVal, P: m_SpecificInt(V: *CmpC + 1)))
2253 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::smax, LHS: V, RHS: TVal);
2254 // (V == SMAX) ? SMAX-1 : V --> smin(V, SMAX-1)
2255 if (CmpC->isMaxSignedValue() && match(V: TVal, P: m_SpecificInt(V: *CmpC - 1)))
2256 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::smin, LHS: V, RHS: TVal);
2257 }
2258
2259 // Fold icmp(X) ? f(X) : C to f(X) when f(X) is guaranteed to be equal to C
2260 // for all X in the exact range of the inverse predicate.
2261 Instruction *Op;
2262 const APInt *C;
2263 CmpInst::Predicate CPred;
2264 if (match(V: &SI, P: m_Select(C: m_Specific(V: ICI), L: m_APInt(Res&: C), R: m_Instruction(I&: Op))))
2265 CPred = ICI->getPredicate();
2266 else if (match(V: &SI, P: m_Select(C: m_Specific(V: ICI), L: m_Instruction(I&: Op), R: m_APInt(Res&: C))))
2267 CPred = ICI->getInversePredicate();
2268 else
2269 return nullptr;
2270
2271 ConstantRange InvDomCR = ConstantRange::makeExactICmpRegion(Pred: CPred, Other: *CmpC);
2272 const APInt *OpC;
2273 if (match(V: Op, P: m_BinOp(L: m_Specific(V), R: m_APInt(Res&: OpC)))) {
2274 ConstantRange R = InvDomCR.binaryOp(
2275 BinOp: static_cast<Instruction::BinaryOps>(Op->getOpcode()), Other: *OpC);
2276 if (R == *C) {
2277 Op->dropPoisonGeneratingFlags();
2278 return Op;
2279 }
2280 }
2281 if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: Op);
2282 MMI && MMI->getLHS() == V && match(V: MMI->getRHS(), P: m_APInt(Res&: OpC))) {
2283 ConstantRange R = ConstantRange::intrinsic(IntrinsicID: MMI->getIntrinsicID(),
2284 Ops: {InvDomCR, ConstantRange(*OpC)});
2285 if (R == *C) {
2286 MMI->dropPoisonGeneratingAnnotations();
2287 return MMI;
2288 }
2289 }
2290
2291 return nullptr;
2292}
2293
2294/// `A == MIN_INT ? B != MIN_INT : A < B` --> `A < B`
2295/// `A == MAX_INT ? B != MAX_INT : A > B` --> `A > B`
2296static Instruction *foldSelectWithExtremeEqCond(Value *CmpLHS, Value *CmpRHS,
2297 Value *TrueVal,
2298 Value *FalseVal) {
2299 Type *Ty = CmpLHS->getType();
2300
2301 if (Ty->isPtrOrPtrVectorTy())
2302 return nullptr;
2303
2304 CmpPredicate Pred;
2305 Value *B;
2306
2307 if (!match(V: FalseVal, P: m_c_ICmp(Pred, L: m_Specific(V: CmpLHS), R: m_Value(V&: B))))
2308 return nullptr;
2309
2310 Value *TValRHS;
2311 if (!match(V: TrueVal, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_NE, L: m_Specific(V: B),
2312 R: m_Value(V&: TValRHS))))
2313 return nullptr;
2314
2315 APInt C;
2316 unsigned BitWidth = Ty->getScalarSizeInBits();
2317
2318 if (ICmpInst::isLT(P: Pred)) {
2319 C = CmpInst::isSigned(Pred) ? APInt::getSignedMinValue(numBits: BitWidth)
2320 : APInt::getMinValue(numBits: BitWidth);
2321 } else if (ICmpInst::isGT(P: Pred)) {
2322 C = CmpInst::isSigned(Pred) ? APInt::getSignedMaxValue(numBits: BitWidth)
2323 : APInt::getMaxValue(numBits: BitWidth);
2324 } else {
2325 return nullptr;
2326 }
2327
2328 if (!match(V: CmpRHS, P: m_SpecificInt(V: C)) || !match(V: TValRHS, P: m_SpecificInt(V: C)))
2329 return nullptr;
2330
2331 return new ICmpInst(Pred, CmpLHS, B);
2332}
2333
2334static Instruction *foldSelectICmpEq(SelectInst &SI, ICmpInst *ICI,
2335 InstCombinerImpl &IC) {
2336 ICmpInst::Predicate Pred = ICI->getPredicate();
2337 if (!ICmpInst::isEquality(P: Pred))
2338 return nullptr;
2339
2340 Value *TrueVal = SI.getTrueValue();
2341 Value *FalseVal = SI.getFalseValue();
2342 Value *CmpLHS = ICI->getOperand(i_nocapture: 0);
2343 Value *CmpRHS = ICI->getOperand(i_nocapture: 1);
2344
2345 if (Pred == ICmpInst::ICMP_NE)
2346 std::swap(a&: TrueVal, b&: FalseVal);
2347
2348 if (Instruction *Res =
2349 foldSelectWithExtremeEqCond(CmpLHS, CmpRHS, TrueVal, FalseVal))
2350 return Res;
2351
2352 return nullptr;
2353}
2354
2355/// Fold `X Pred C1 ? X BOp C2 : C1 BOp C2` to `min/max(X, C1) BOp C2`.
2356/// This allows for better canonicalization.
2357Value *InstCombinerImpl::foldSelectWithConstOpToBinOp(ICmpInst *Cmp,
2358 Value *TrueVal,
2359 Value *FalseVal) {
2360 Constant *C1, *C2, *C3;
2361 Value *X;
2362 CmpPredicate Predicate;
2363
2364 if (!match(V: Cmp, P: m_ICmp(Pred&: Predicate, L: m_Value(V&: X), R: m_Constant(C&: C1))))
2365 return nullptr;
2366
2367 if (!ICmpInst::isRelational(P: Predicate))
2368 return nullptr;
2369
2370 if (match(V: TrueVal, P: m_Constant())) {
2371 std::swap(a&: FalseVal, b&: TrueVal);
2372 Predicate = ICmpInst::getInversePredicate(pred: Predicate);
2373 }
2374
2375 if (!match(V: FalseVal, P: m_Constant(C&: C3)) || !TrueVal->hasOneUse())
2376 return nullptr;
2377
2378 bool IsIntrinsic;
2379 unsigned Opcode;
2380 if (BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: TrueVal)) {
2381 Opcode = BOp->getOpcode();
2382 IsIntrinsic = false;
2383
2384 // This fold causes some regressions and is primarily intended for
2385 // add and sub. So we early exit for div and rem to minimize the
2386 // regressions.
2387 if (Instruction::isIntDivRem(Opcode))
2388 return nullptr;
2389
2390 if (!match(V: BOp, P: m_BinOp(L: m_Specific(V: X), R: m_Constant(C&: C2))))
2391 return nullptr;
2392
2393 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: TrueVal)) {
2394 if (!match(V: II, P: m_MaxOrMin(Op0: m_Specific(V: X), Op1: m_Constant(C&: C2))))
2395 return nullptr;
2396 Opcode = II->getIntrinsicID();
2397 IsIntrinsic = true;
2398 } else {
2399 return nullptr;
2400 }
2401
2402 Value *RHS;
2403 SelectPatternFlavor SPF;
2404 const DataLayout &DL = Cmp->getDataLayout();
2405 auto Flipped = getFlippedStrictnessPredicateAndConstant(Pred: Predicate, C: C1);
2406
2407 auto FoldBinaryOpOrIntrinsic = [&](Constant *LHS, Constant *RHS) {
2408 return IsIntrinsic
2409 ? ConstantFoldIntrinsic(ID: Opcode, Ops: {LHS, RHS}, Ty: LHS->getType(), DL)
2410 : ConstantFoldBinaryOpOperands(Opcode, LHS, RHS, DL);
2411 };
2412
2413 if (C3 == FoldBinaryOpOrIntrinsic(C1, C2)) {
2414 SPF = getSelectPattern(Pred: Predicate).Flavor;
2415 RHS = C1;
2416 } else if (Flipped && C3 == FoldBinaryOpOrIntrinsic(Flipped->second, C2)) {
2417 SPF = getSelectPattern(Pred: Flipped->first).Flavor;
2418 RHS = Flipped->second;
2419 } else {
2420 return nullptr;
2421 }
2422
2423 Intrinsic::ID MinMaxID = getMinMaxIntrinsic(SPF);
2424 Value *MinMax = Builder.CreateBinaryIntrinsic(ID: MinMaxID, LHS: X, RHS);
2425 if (IsIntrinsic)
2426 return Builder.CreateBinaryIntrinsic(ID: Opcode, LHS: MinMax, RHS: C2);
2427
2428 const auto BinOpc = Instruction::BinaryOps(Opcode);
2429 Value *BinOp = Builder.CreateBinOp(Opc: BinOpc, LHS: MinMax, RHS: C2);
2430
2431 // If we can attach no-wrap flags to the new instruction, do so if the
2432 // old instruction had them and C1 BinOp C2 does not overflow.
2433 if (Instruction *BinOpInst = dyn_cast<Instruction>(Val: BinOp)) {
2434 if (BinOpc == Instruction::Add || BinOpc == Instruction::Sub ||
2435 BinOpc == Instruction::Mul) {
2436 Instruction *OldBinOp = cast<BinaryOperator>(Val: TrueVal);
2437 if (OldBinOp->hasNoSignedWrap() &&
2438 willNotOverflow(Opcode: BinOpc, LHS: RHS, RHS: C2, CtxI: *BinOpInst, /*IsSigned=*/true))
2439 BinOpInst->setHasNoSignedWrap();
2440 if (OldBinOp->hasNoUnsignedWrap() &&
2441 willNotOverflow(Opcode: BinOpc, LHS: RHS, RHS: C2, CtxI: *BinOpInst, /*IsSigned=*/false))
2442 BinOpInst->setHasNoUnsignedWrap();
2443 }
2444 }
2445 return BinOp;
2446}
2447
2448/// Folds:
2449/// %a_sub = call @llvm.usub.sat(x, IntConst1)
2450/// %b_sub = call @llvm.usub.sat(y, IntConst2)
2451/// %or = or %a_sub, %b_sub
2452/// %cmp = icmp eq %or, 0
2453/// %sel = select %cmp, 0, MostSignificantBit
2454/// into:
2455/// %a_sub' = usub.sat(x, IntConst1 - MostSignificantBit)
2456/// %b_sub' = usub.sat(y, IntConst2 - MostSignificantBit)
2457/// %or = or %a_sub', %b_sub'
2458/// %and = and %or, MostSignificantBit
2459/// Likewise, for vector arguments as well.
2460static Instruction *foldICmpUSubSatWithAndForMostSignificantBitCmp(
2461 SelectInst &SI, ICmpInst *ICI, InstCombiner::BuilderTy &Builder) {
2462 if (!SI.hasOneUse() || !ICI->hasOneUse())
2463 return nullptr;
2464 CmpPredicate Pred;
2465 Value *A, *B;
2466 const APInt *Constant1, *Constant2;
2467 if (!match(V: SI.getCondition(),
2468 P: m_ICmp(Pred,
2469 L: m_OneUse(SubPattern: m_Or(L: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::usub_sat>(
2470 Ops: m_Value(V&: A), Ops: m_APInt(Res&: Constant1))),
2471 R: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::usub_sat>(
2472 Ops: m_Value(V&: B), Ops: m_APInt(Res&: Constant2))))),
2473 R: m_Zero())))
2474 return nullptr;
2475
2476 // The new instructions have the type of the usub.sat operands, so they can
2477 // only replace a select of the same type.
2478 if (SI.getType() != A->getType())
2479 return nullptr;
2480
2481 Value *TrueVal = SI.getTrueValue();
2482 Value *FalseVal = SI.getFalseValue();
2483 if (!((Pred == ICmpInst::ICMP_EQ && match(V: TrueVal, P: m_Zero()) &&
2484 match(V: FalseVal, P: m_SignMask())) ||
2485 (Pred == ICmpInst::ICMP_NE && match(V: TrueVal, P: m_SignMask()) &&
2486 match(V: FalseVal, P: m_Zero()))))
2487 return nullptr;
2488
2489 auto *Ty = A->getType();
2490 unsigned BW = Constant1->getBitWidth();
2491 APInt MostSignificantBit = APInt::getSignMask(BitWidth: BW);
2492
2493 // Anything over MSB is negative
2494 if (Constant1->isNonNegative() || Constant2->isNonNegative())
2495 return nullptr;
2496
2497 APInt AdjAP1 = *Constant1 - MostSignificantBit + 1;
2498 APInt AdjAP2 = *Constant2 - MostSignificantBit + 1;
2499
2500 auto *Adj1 = ConstantInt::get(Ty, V: AdjAP1);
2501 auto *Adj2 = ConstantInt::get(Ty, V: AdjAP2);
2502
2503 Value *NewA = Builder.CreateBinaryIntrinsic(ID: Intrinsic::usub_sat, LHS: A, RHS: Adj1);
2504 Value *NewB = Builder.CreateBinaryIntrinsic(ID: Intrinsic::usub_sat, LHS: B, RHS: Adj2);
2505 Value *Or = Builder.CreateOr(LHS: NewA, RHS: NewB);
2506 Constant *MSBConst = ConstantInt::get(Ty, V: MostSignificantBit);
2507 return BinaryOperator::CreateAnd(V1: Or, V2: MSBConst);
2508}
2509
2510/// Visit a SelectInst that has an ICmpInst as its first operand.
2511Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
2512 ICmpInst *ICI) {
2513 if (Value *V =
2514 canonicalizeSPF(Cmp&: *ICI, TrueVal: SI.getTrueValue(), FalseVal: SI.getFalseValue(), IC&: *this))
2515 return replaceInstUsesWith(I&: SI, V);
2516
2517 if (Value *V = foldSelectInstWithICmpConst(SI, ICI, Builder))
2518 return replaceInstUsesWith(I&: SI, V);
2519
2520 if (Value *V = canonicalizeClampLike(Sel0&: SI, Cmp0&: *ICI, Builder, IC&: *this))
2521 return replaceInstUsesWith(I&: SI, V);
2522
2523 if (Instruction *NewSel =
2524 tryToReuseConstantFromSelectInComparison(Sel&: SI, Cmp&: *ICI, IC&: *this))
2525 return NewSel;
2526 if (Instruction *Folded =
2527 foldICmpUSubSatWithAndForMostSignificantBitCmp(SI, ICI, Builder))
2528 return Folded;
2529
2530 // NOTE: if we wanted to, this is where to detect integer MIN/MAX
2531 bool Changed = false;
2532 Value *TrueVal = SI.getTrueValue();
2533 Value *FalseVal = SI.getFalseValue();
2534 ICmpInst::Predicate Pred = ICI->getPredicate();
2535 Value *CmpLHS = ICI->getOperand(i_nocapture: 0);
2536 Value *CmpRHS = ICI->getOperand(i_nocapture: 1);
2537
2538 if (Instruction *NewSel = foldSelectICmpEq(SI, ICI, IC&: *this))
2539 return NewSel;
2540
2541 // Canonicalize a signbit condition to use zero constant by swapping:
2542 // (CmpLHS > -1) ? TV : FV --> (CmpLHS < 0) ? FV : TV
2543 // To avoid conflicts (infinite loops) with other canonicalizations, this is
2544 // not applied with any constant select arm.
2545 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: m_AllOnes()) &&
2546 !match(V: TrueVal, P: m_Constant()) && !match(V: FalseVal, P: m_Constant()) &&
2547 ICI->hasOneUse()) {
2548 InstCombiner::BuilderTy::InsertPointGuard Guard(Builder);
2549 Builder.SetInsertPoint(&SI);
2550 Value *IsNeg = Builder.CreateIsNeg(Arg: CmpLHS, Name: ICI->getName());
2551 replaceOperand(I&: SI, OpNum: 0, V: IsNeg);
2552 SI.swapValues();
2553 SI.swapProfMetadata();
2554 return &SI;
2555 }
2556
2557 if (Value *V = foldSelectICmpMinMax(Cmp: ICI, TVal: TrueVal, FVal: FalseVal, Builder,
2558 SQ: SQ.getWithInstruction(I: &SI)))
2559 return replaceInstUsesWith(I&: SI, V);
2560
2561 if (Value *V = foldSelectICmpAndZeroShl(Cmp: ICI, TVal: TrueVal, FVal: FalseVal, Builder))
2562 return replaceInstUsesWith(I&: SI, V);
2563
2564 if (Instruction *V = foldSelectCtlzToCttz(ICI, TrueVal, FalseVal, Builder))
2565 return V;
2566
2567 if (Instruction *V = foldSelectZeroOrOnes(Cmp: ICI, TVal: TrueVal, FVal: FalseVal, Builder))
2568 return V;
2569
2570 if (Value *V = foldSelectICmpLshrAshr(IC: ICI, TrueVal, FalseVal, Builder))
2571 return replaceInstUsesWith(I&: SI, V);
2572
2573 if (Value *V = foldSelectCttzCtlz(ICI, TrueVal, FalseVal, IC&: *this))
2574 return replaceInstUsesWith(I&: SI, V);
2575
2576 if (Value *V = canonicalizeSaturatedSubtract(ICI, TrueVal, FalseVal, Builder))
2577 return replaceInstUsesWith(I&: SI, V);
2578
2579 if (Value *V = canonicalizeSaturatedAdd(Cmp: ICI, TVal: TrueVal, FVal: FalseVal, Builder))
2580 return replaceInstUsesWith(I&: SI, V);
2581
2582 if (Value *V = foldAbsDiff(Cmp: ICI, TVal: TrueVal, FVal: FalseVal, Builder))
2583 return replaceInstUsesWith(I&: SI, V);
2584
2585 if (Value *V = foldSelectWithConstOpToBinOp(Cmp: ICI, TrueVal, FalseVal))
2586 return replaceInstUsesWith(I&: SI, V);
2587
2588 return Changed ? &SI : nullptr;
2589}
2590
2591/// We have an SPF (e.g. a min or max) of an SPF of the form:
2592/// SPF2(SPF1(A, B), C)
2593Instruction *InstCombinerImpl::foldSPFofSPF(Instruction *Inner,
2594 SelectPatternFlavor SPF1, Value *A,
2595 Value *B, Instruction &Outer,
2596 SelectPatternFlavor SPF2,
2597 Value *C) {
2598 if (Outer.getType() != Inner->getType())
2599 return nullptr;
2600
2601 if (C == A || C == B) {
2602 // MAX(MAX(A, B), B) -> MAX(A, B)
2603 // MIN(MIN(a, b), a) -> MIN(a, b)
2604 // TODO: This could be done in instsimplify.
2605 if (SPF1 == SPF2 && SelectPatternResult::isMinOrMax(SPF: SPF1))
2606 return replaceInstUsesWith(I&: Outer, V: Inner);
2607 }
2608
2609 return nullptr;
2610}
2611
2612/// Turn select C, (X + Y), (X - Y) --> (X + (select C, Y, (-Y))).
2613/// This is even legal for FP.
2614static Instruction *foldAddSubSelect(SelectInst &SI,
2615 InstCombiner::BuilderTy &Builder) {
2616 Value *CondVal = SI.getCondition();
2617 Value *TrueVal = SI.getTrueValue();
2618 Value *FalseVal = SI.getFalseValue();
2619 auto *TI = dyn_cast<Instruction>(Val: TrueVal);
2620 auto *FI = dyn_cast<Instruction>(Val: FalseVal);
2621 if (!TI || !FI || !TI->hasOneUse() || !FI->hasOneUse())
2622 return nullptr;
2623
2624 Instruction *AddOp = nullptr, *SubOp = nullptr;
2625 if ((TI->getOpcode() == Instruction::Sub &&
2626 FI->getOpcode() == Instruction::Add) ||
2627 (TI->getOpcode() == Instruction::FSub &&
2628 FI->getOpcode() == Instruction::FAdd)) {
2629 AddOp = FI;
2630 SubOp = TI;
2631 } else if ((FI->getOpcode() == Instruction::Sub &&
2632 TI->getOpcode() == Instruction::Add) ||
2633 (FI->getOpcode() == Instruction::FSub &&
2634 TI->getOpcode() == Instruction::FAdd)) {
2635 AddOp = TI;
2636 SubOp = FI;
2637 }
2638
2639 if (AddOp) {
2640 Value *OtherAddOp = nullptr;
2641 if (SubOp->getOperand(i: 0) == AddOp->getOperand(i: 0)) {
2642 OtherAddOp = AddOp->getOperand(i: 1);
2643 } else if (SubOp->getOperand(i: 0) == AddOp->getOperand(i: 1)) {
2644 OtherAddOp = AddOp->getOperand(i: 0);
2645 }
2646
2647 if (OtherAddOp) {
2648 // So at this point we know we have (Y -> OtherAddOp):
2649 // select C, (add X, Y), (sub X, Z)
2650 Value *NegVal; // Compute -Z
2651 if (SI.getType()->isFPOrFPVectorTy()) {
2652 NegVal = Builder.CreateFNeg(V: SubOp->getOperand(i: 1));
2653 if (Instruction *NegInst = dyn_cast<Instruction>(Val: NegVal)) {
2654 FastMathFlags Flags = AddOp->getFastMathFlags();
2655 Flags &= SubOp->getFastMathFlags();
2656 NegInst->setFastMathFlags(Flags);
2657 }
2658 } else {
2659 NegVal = Builder.CreateNeg(V: SubOp->getOperand(i: 1));
2660 }
2661
2662 Value *NewTrueOp = OtherAddOp;
2663 Value *NewFalseOp = NegVal;
2664 if (AddOp != TI)
2665 std::swap(a&: NewTrueOp, b&: NewFalseOp);
2666 Value *NewSel = Builder.CreateSelect(C: CondVal, True: NewTrueOp, False: NewFalseOp,
2667 Name: SI.getName() + ".p", MDFrom: &SI);
2668
2669 if (SI.getType()->isFPOrFPVectorTy()) {
2670 Instruction *RI =
2671 BinaryOperator::CreateFAdd(V1: SubOp->getOperand(i: 0), V2: NewSel);
2672
2673 FastMathFlags Flags = AddOp->getFastMathFlags();
2674 Flags &= SubOp->getFastMathFlags();
2675 RI->setFastMathFlags(Flags);
2676 return RI;
2677 } else
2678 return BinaryOperator::CreateAdd(V1: SubOp->getOperand(i: 0), V2: NewSel);
2679 }
2680 }
2681 return nullptr;
2682}
2683
2684/// Turn X + Y overflows ? -1 : X + Y -> uadd_sat X, Y
2685/// And X - Y overflows ? 0 : X - Y -> usub_sat X, Y
2686/// Along with a number of patterns similar to:
2687/// X + Y overflows ? (X < 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2688/// X - Y overflows ? (X > 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2689static Instruction *
2690foldOverflowingAddSubSelect(SelectInst &SI, InstCombiner::BuilderTy &Builder) {
2691 Value *CondVal = SI.getCondition();
2692 Value *TrueVal = SI.getTrueValue();
2693 Value *FalseVal = SI.getFalseValue();
2694
2695 WithOverflowInst *II;
2696 if (!match(V: CondVal, P: m_ExtractValue<1>(V: m_WithOverflowInst(I&: II))) ||
2697 !match(V: FalseVal, P: m_ExtractValue<0>(V: m_Specific(V: II))))
2698 return nullptr;
2699
2700 Value *X = II->getLHS();
2701 Value *Y = II->getRHS();
2702
2703 auto IsSignedSaturateLimit = [&](Value *Limit, bool IsAdd) {
2704 Type *Ty = Limit->getType();
2705
2706 CmpPredicate Pred;
2707 Value *TrueVal, *FalseVal, *Op;
2708 const APInt *C;
2709 if (!match(V: Limit, P: m_Select(C: m_ICmp(Pred, L: m_Value(V&: Op), R: m_APInt(Res&: C)),
2710 L: m_Value(V&: TrueVal), R: m_Value(V&: FalseVal))))
2711 return false;
2712
2713 auto IsZeroOrOne = [](const APInt &C) { return C.isZero() || C.isOne(); };
2714 auto IsMinMax = [&](Value *Min, Value *Max) {
2715 APInt MinVal = APInt::getSignedMinValue(numBits: Ty->getScalarSizeInBits());
2716 APInt MaxVal = APInt::getSignedMaxValue(numBits: Ty->getScalarSizeInBits());
2717 return match(V: Min, P: m_SpecificInt(V: MinVal)) &&
2718 match(V: Max, P: m_SpecificInt(V: MaxVal));
2719 };
2720
2721 if (Op != X && Op != Y)
2722 return false;
2723
2724 if (IsAdd) {
2725 // X + Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2726 // X + Y overflows ? (X <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2727 // X + Y overflows ? (Y <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2728 // X + Y overflows ? (Y <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2729 if (Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C) &&
2730 IsMinMax(TrueVal, FalseVal))
2731 return true;
2732 // X + Y overflows ? (X >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2733 // X + Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2734 // X + Y overflows ? (Y >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2735 // X + Y overflows ? (Y >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2736 if (Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 1) &&
2737 IsMinMax(FalseVal, TrueVal))
2738 return true;
2739 } else {
2740 // X - Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2741 // X - Y overflows ? (X <s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2742 if (Op == X && Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C + 1) &&
2743 IsMinMax(TrueVal, FalseVal))
2744 return true;
2745 // X - Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2746 // X - Y overflows ? (X >s -2 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2747 if (Op == X && Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 2) &&
2748 IsMinMax(FalseVal, TrueVal))
2749 return true;
2750 // X - Y overflows ? (Y <s 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2751 // X - Y overflows ? (Y <s 1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2752 if (Op == Y && Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C) &&
2753 IsMinMax(FalseVal, TrueVal))
2754 return true;
2755 // X - Y overflows ? (Y >s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2756 // X - Y overflows ? (Y >s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2757 if (Op == Y && Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 1) &&
2758 IsMinMax(TrueVal, FalseVal))
2759 return true;
2760 }
2761
2762 return false;
2763 };
2764
2765 Intrinsic::ID NewIntrinsicID;
2766 if (II->getIntrinsicID() == Intrinsic::uadd_with_overflow &&
2767 match(V: TrueVal, P: m_AllOnes()))
2768 // X + Y overflows ? -1 : X + Y -> uadd_sat X, Y
2769 NewIntrinsicID = Intrinsic::uadd_sat;
2770 else if (II->getIntrinsicID() == Intrinsic::usub_with_overflow &&
2771 match(V: TrueVal, P: m_Zero()))
2772 // X - Y overflows ? 0 : X - Y -> usub_sat X, Y
2773 NewIntrinsicID = Intrinsic::usub_sat;
2774 else if (II->getIntrinsicID() == Intrinsic::sadd_with_overflow &&
2775 IsSignedSaturateLimit(TrueVal, /*IsAdd=*/true))
2776 // X + Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2777 // X + Y overflows ? (X <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2778 // X + Y overflows ? (X >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2779 // X + Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2780 // X + Y overflows ? (Y <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2781 // X + Y overflows ? (Y <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2782 // X + Y overflows ? (Y >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2783 // X + Y overflows ? (Y >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2784 NewIntrinsicID = Intrinsic::sadd_sat;
2785 else if (II->getIntrinsicID() == Intrinsic::ssub_with_overflow &&
2786 IsSignedSaturateLimit(TrueVal, /*IsAdd=*/false))
2787 // X - Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2788 // X - Y overflows ? (X <s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2789 // X - Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2790 // X - Y overflows ? (X >s -2 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2791 // X - Y overflows ? (Y <s 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2792 // X - Y overflows ? (Y <s 1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2793 // X - Y overflows ? (Y >s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2794 // X - Y overflows ? (Y >s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2795 NewIntrinsicID = Intrinsic::ssub_sat;
2796 else
2797 return nullptr;
2798
2799 Function *F = Intrinsic::getOrInsertDeclaration(M: SI.getModule(),
2800 id: NewIntrinsicID, OverloadTys: SI.getType());
2801 return CallInst::Create(Func: F, Args: {X, Y});
2802}
2803
2804Instruction *InstCombinerImpl::foldSelectExtConst(SelectInst &Sel) {
2805 Constant *C;
2806 if (!match(V: Sel.getTrueValue(), P: m_Constant(C)) &&
2807 !match(V: Sel.getFalseValue(), P: m_Constant(C)))
2808 return nullptr;
2809
2810 Instruction *ExtInst;
2811 if (!match(V: Sel.getTrueValue(), P: m_Instruction(I&: ExtInst)) &&
2812 !match(V: Sel.getFalseValue(), P: m_Instruction(I&: ExtInst)))
2813 return nullptr;
2814
2815 auto ExtOpcode = ExtInst->getOpcode();
2816 if (ExtOpcode != Instruction::ZExt && ExtOpcode != Instruction::SExt)
2817 return nullptr;
2818
2819 // If we are extending from a boolean type or if we can create a select that
2820 // has the same size operands as its condition, try to narrow the select.
2821 Value *X = ExtInst->getOperand(i: 0);
2822 Type *SmallType = X->getType();
2823 Value *Cond = Sel.getCondition();
2824 if (!SmallType->isIntOrIntVectorTy(BitWidth: 1) &&
2825 (!isa<CmpInst, TruncInst>(Val: Cond) ||
2826 cast<Instruction>(Val: Cond)->getOperand(i: 0)->getType() != SmallType))
2827 return nullptr;
2828
2829 // If the constant is the same after truncation to the smaller type and
2830 // extension to the original type, we can narrow the select.
2831 Type *SelType = Sel.getType();
2832 Constant *TruncC = getLosslessInvCast(C, InvCastTo: SmallType, CastOp: ExtOpcode, DL);
2833 if (TruncC && ExtInst->hasOneUse()) {
2834 Value *TruncCVal = cast<Value>(Val: TruncC);
2835 if (ExtInst == Sel.getFalseValue())
2836 std::swap(a&: X, b&: TruncCVal);
2837
2838 // select Cond, (ext X), C --> ext(select Cond, X, C')
2839 // select Cond, C, (ext X) --> ext(select Cond, C', X)
2840 Value *NewSel = Builder.CreateSelect(C: Cond, True: X, False: TruncCVal, Name: "narrow", MDFrom: &Sel);
2841 return CastInst::Create(Instruction::CastOps(ExtOpcode), S: NewSel, Ty: SelType);
2842 }
2843
2844 return nullptr;
2845}
2846
2847/// Try to transform a vector select with a constant condition vector into a
2848/// shuffle for easier combining with other shuffles and insert/extract.
2849static Instruction *canonicalizeSelectToShuffle(SelectInst &SI) {
2850 Value *CondVal = SI.getCondition();
2851 Constant *CondC;
2852 auto *CondValTy = dyn_cast<FixedVectorType>(Val: CondVal->getType());
2853 if (!CondValTy || !match(V: CondVal, P: m_Constant(C&: CondC)))
2854 return nullptr;
2855
2856 unsigned NumElts = CondValTy->getNumElements();
2857 SmallVector<int, 16> Mask;
2858 Mask.reserve(N: NumElts);
2859 for (unsigned i = 0; i != NumElts; ++i) {
2860 Constant *Elt = CondC->getAggregateElement(Elt: i);
2861 if (!Elt)
2862 return nullptr;
2863
2864 if (Elt->isOneValue()) {
2865 // If the select condition element is true, choose from the 1st vector.
2866 Mask.push_back(Elt: i);
2867 } else if (Elt->isNullValue()) {
2868 // If the select condition element is false, choose from the 2nd vector.
2869 Mask.push_back(Elt: i + NumElts);
2870 } else if (isa<UndefValue>(Val: Elt)) {
2871 // Undef in a select condition (choose one of the operands) does not mean
2872 // the same thing as undef in a shuffle mask (any value is acceptable), so
2873 // give up.
2874 return nullptr;
2875 } else {
2876 // Bail out on a constant expression.
2877 return nullptr;
2878 }
2879 }
2880
2881 return new ShuffleVectorInst(SI.getTrueValue(), SI.getFalseValue(), Mask);
2882}
2883
2884/// If we have a select of vectors with a scalar condition, try to convert that
2885/// to a vector select by splatting the condition. A splat may get folded with
2886/// other operations in IR and having all operands of a select be vector types
2887/// is likely better for vector codegen.
2888static Instruction *canonicalizeScalarSelectOfVecs(SelectInst &Sel,
2889 InstCombinerImpl &IC) {
2890 auto *Ty = dyn_cast<VectorType>(Val: Sel.getType());
2891 if (!Ty)
2892 return nullptr;
2893
2894 // We can replace a single-use extract with constant index.
2895 Value *Cond = Sel.getCondition();
2896 if (!match(V: Cond, P: m_OneUse(SubPattern: m_ExtractElt(Val: m_Value(), Idx: m_ConstantInt()))))
2897 return nullptr;
2898
2899 // select (extelt V, Index), T, F --> select (splat V, Index), T, F
2900 // Splatting the extracted condition reduces code (we could directly create a
2901 // splat shuffle of the source vector to eliminate the intermediate step).
2902 return IC.replaceOperand(
2903 I&: Sel, OpNum: 0, V: IC.Builder.CreateVectorSplat(EC: Ty->getElementCount(), V: Cond));
2904}
2905
2906/// Reuse bitcasted operands between a compare and select:
2907/// select (cmp (bitcast C), (bitcast D)), (bitcast' C), (bitcast' D) -->
2908/// bitcast (select (cmp (bitcast C), (bitcast D)), (bitcast C), (bitcast D))
2909static Instruction *foldSelectCmpBitcasts(SelectInst &Sel,
2910 InstCombiner::BuilderTy &Builder) {
2911 Value *Cond = Sel.getCondition();
2912 Value *TVal = Sel.getTrueValue();
2913 Value *FVal = Sel.getFalseValue();
2914
2915 CmpPredicate Pred;
2916 Value *A, *B;
2917 if (!match(V: Cond, P: m_Cmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B))))
2918 return nullptr;
2919
2920 // The select condition is a compare instruction. If the select's true/false
2921 // values are already the same as the compare operands, there's nothing to do.
2922 if (TVal == A || TVal == B || FVal == A || FVal == B)
2923 return nullptr;
2924
2925 Value *C, *D;
2926 if (!match(V: A, P: m_BitCast(Op: m_Value(V&: C))) || !match(V: B, P: m_BitCast(Op: m_Value(V&: D))))
2927 return nullptr;
2928
2929 // select (cmp (bitcast C), (bitcast D)), (bitcast TSrc), (bitcast FSrc)
2930 Value *TSrc, *FSrc;
2931 if (!match(V: TVal, P: m_BitCast(Op: m_Value(V&: TSrc))) ||
2932 !match(V: FVal, P: m_BitCast(Op: m_Value(V&: FSrc))))
2933 return nullptr;
2934
2935 // If the select true/false values are *different bitcasts* of the same source
2936 // operands, make the select operands the same as the compare operands and
2937 // cast the result. This is the canonical select form for min/max.
2938 Value *NewSel;
2939 if (TSrc == C && FSrc == D) {
2940 // select (cmp (bitcast C), (bitcast D)), (bitcast' C), (bitcast' D) -->
2941 // bitcast (select (cmp A, B), A, B)
2942 NewSel = Builder.CreateSelect(C: Cond, True: A, False: B, Name: "", MDFrom: &Sel);
2943 } else if (TSrc == D && FSrc == C) {
2944 // select (cmp (bitcast C), (bitcast D)), (bitcast' D), (bitcast' C) -->
2945 // bitcast (select (cmp A, B), B, A)
2946 NewSel = Builder.CreateSelect(C: Cond, True: B, False: A, Name: "", MDFrom: &Sel);
2947 } else {
2948 return nullptr;
2949 }
2950 return new BitCastInst(NewSel, Sel.getType());
2951}
2952
2953/// Try to eliminate select instructions that test the returned flag of cmpxchg
2954/// instructions.
2955///
2956/// If a select instruction tests the returned flag of a cmpxchg instruction and
2957/// selects between the returned value of the cmpxchg instruction its compare
2958/// operand, the result of the select will always be equal to its false value.
2959/// For example:
2960///
2961/// %cmpxchg = cmpxchg ptr %ptr, i64 %compare, i64 %new_value seq_cst seq_cst
2962/// %val = extractvalue { i64, i1 } %cmpxchg, 0
2963/// %success = extractvalue { i64, i1 } %cmpxchg, 1
2964/// %sel = select i1 %success, i64 %compare, i64 %val
2965/// ret i64 %sel
2966///
2967/// The returned value of the cmpxchg instruction (%val) is the original value
2968/// located at %ptr prior to any update. If the cmpxchg operation succeeds, %val
2969/// must have been equal to %compare. Thus, the result of the select is always
2970/// equal to %val, and the code can be simplified to:
2971///
2972/// %cmpxchg = cmpxchg ptr %ptr, i64 %compare, i64 %new_value seq_cst seq_cst
2973/// %val = extractvalue { i64, i1 } %cmpxchg, 0
2974/// ret i64 %val
2975///
2976static Value *foldSelectCmpXchg(SelectInst &SI) {
2977 // A helper that determines if V is an extractvalue instruction whose
2978 // aggregate operand is a cmpxchg instruction and whose single index is equal
2979 // to I. If such conditions are true, the helper returns the cmpxchg
2980 // instruction; otherwise, a nullptr is returned.
2981 auto isExtractFromCmpXchg = [](Value *V, unsigned I) -> AtomicCmpXchgInst * {
2982 // When extracting the value loaded by a cmpxchg, allow peeking through a
2983 // bitcast. These are inserted for floating-point cmpxchg, for example:
2984 // %bc = bitcast float %compare to i32
2985 // %cmpxchg = cmpxchg ptr %ptr, i32 %bc, i32 %new_value seq_cst seq_cst
2986 // %val = extractvalue { i32, i1 } %cmpxchg, 0
2987 // %success = extractvalue { i32, i1 } %cmpxchg, 1
2988 // %val.bc = bitcast i32 %val to float
2989 // %sel = select i1 %success, float %compare, float %val.bc
2990 if (auto *BI = dyn_cast<BitCastInst>(Val: V); BI && I == 0)
2991 V = BI->getOperand(i_nocapture: 0);
2992 auto *Extract = dyn_cast<ExtractValueInst>(Val: V);
2993 if (!Extract)
2994 return nullptr;
2995 if (Extract->getIndices()[0] != I)
2996 return nullptr;
2997 return dyn_cast<AtomicCmpXchgInst>(Val: Extract->getAggregateOperand());
2998 };
2999
3000 // Check if the compare value of a cmpxchg matches another value.
3001 auto isCompareSameAsValue = [](Value *CmpVal, Value *SelVal) {
3002 // The values match if they are the same or %CmpVal = bitcast %SelVal (see
3003 // above).
3004 if (CmpVal == SelVal || match(V: CmpVal, P: m_BitCast(Op: m_Specific(V: SelVal))))
3005 return true;
3006 // For FP constants, the value may have been bitcast to Int directly.
3007 auto *IntC = dyn_cast<ConstantInt>(Val: CmpVal);
3008 auto *FpC = dyn_cast<ConstantFP>(Val: SelVal);
3009 return IntC && FpC && IntC->getValue() == FpC->getValue().bitcastToAPInt();
3010 };
3011
3012 // If the select has a single user, and this user is a select instruction that
3013 // we can simplify, skip the cmpxchg simplification for now.
3014 if (SI.hasOneUse())
3015 if (auto *Select = dyn_cast<SelectInst>(Val: SI.user_back()))
3016 if (Select->getCondition() == SI.getCondition())
3017 if (Select->getFalseValue() == SI.getTrueValue() ||
3018 Select->getTrueValue() == SI.getFalseValue())
3019 return nullptr;
3020
3021 // Ensure the select condition is the returned flag of a cmpxchg instruction.
3022 auto *CmpXchg = isExtractFromCmpXchg(SI.getCondition(), 1);
3023 if (!CmpXchg)
3024 return nullptr;
3025
3026 // Check the true value case: The true value of the select is the returned
3027 // value of the same cmpxchg used by the condition, and the false value is the
3028 // cmpxchg instruction's compare operand.
3029 if (auto *X = isExtractFromCmpXchg(SI.getTrueValue(), 0))
3030 if (X == CmpXchg &&
3031 isCompareSameAsValue(X->getCompareOperand(), SI.getFalseValue()))
3032 return SI.getFalseValue();
3033
3034 // Check the false value case: The false value of the select is the returned
3035 // value of the same cmpxchg used by the condition, and the true value is the
3036 // cmpxchg instruction's compare operand.
3037 if (auto *X = isExtractFromCmpXchg(SI.getFalseValue(), 0))
3038 if (X == CmpXchg &&
3039 isCompareSameAsValue(X->getCompareOperand(), SI.getTrueValue()))
3040 return SI.getFalseValue();
3041
3042 return nullptr;
3043}
3044
3045/// Try to reduce a funnel/rotate pattern that includes a compare and select
3046/// into a funnel shift intrinsic. Example:
3047/// rotl32(a, b) --> (b == 0 ? a : ((a >> (32 - b)) | (a << b)))
3048/// --> call llvm.fshl.i32(a, a, b)
3049/// fshl32(a, b, c) --> (c == 0 ? a : ((b >> (32 - c)) | (a << c)))
3050/// --> call llvm.fshl.i32(a, b, c)
3051/// fshr32(a, b, c) --> (c == 0 ? b : ((a >> (32 - c)) | (b << c)))
3052/// --> call llvm.fshr.i32(a, b, c)
3053static Instruction *foldSelectFunnelShift(SelectInst &Sel,
3054 InstCombiner::BuilderTy &Builder) {
3055 // This must be a power-of-2 type for a bitmasking transform to be valid.
3056 unsigned Width = Sel.getType()->getScalarSizeInBits();
3057 if (!isPowerOf2_32(Value: Width))
3058 return nullptr;
3059
3060 BinaryOperator *Or0, *Or1;
3061 if (!match(V: Sel.getFalseValue(), P: m_OneUse(SubPattern: m_Or(L: m_BinOp(I&: Or0), R: m_BinOp(I&: Or1)))))
3062 return nullptr;
3063
3064 Value *SV0, *SV1, *SA0, *SA1;
3065 if (!match(V: Or0, P: m_OneUse(SubPattern: m_LogicalShift(L: m_Value(V&: SV0),
3066 R: m_ZExtOrSelf(Op: m_Value(V&: SA0))))) ||
3067 !match(V: Or1, P: m_OneUse(SubPattern: m_LogicalShift(L: m_Value(V&: SV1),
3068 R: m_ZExtOrSelf(Op: m_Value(V&: SA1))))) ||
3069 Or0->getOpcode() == Or1->getOpcode())
3070 return nullptr;
3071
3072 // Canonicalize to or(shl(SV0, SA0), lshr(SV1, SA1)).
3073 if (Or0->getOpcode() == BinaryOperator::LShr) {
3074 std::swap(a&: Or0, b&: Or1);
3075 std::swap(a&: SV0, b&: SV1);
3076 std::swap(a&: SA0, b&: SA1);
3077 }
3078 assert(Or0->getOpcode() == BinaryOperator::Shl &&
3079 Or1->getOpcode() == BinaryOperator::LShr &&
3080 "Illegal or(shift,shift) pair");
3081
3082 // Check the shift amounts to see if they are an opposite pair.
3083 Value *ShAmt;
3084 if (match(V: SA1, P: m_OneUse(SubPattern: m_Sub(L: m_SpecificInt(V: Width), R: m_Specific(V: SA0)))))
3085 ShAmt = SA0;
3086 else if (match(V: SA0, P: m_OneUse(SubPattern: m_Sub(L: m_SpecificInt(V: Width), R: m_Specific(V: SA1)))))
3087 ShAmt = SA1;
3088 else
3089 return nullptr;
3090
3091 // We should now have this pattern:
3092 // select ?, TVal, (or (shl SV0, SA0), (lshr SV1, SA1))
3093 // The false value of the select must be a funnel-shift of the true value:
3094 // IsFShl -> TVal must be SV0 else TVal must be SV1.
3095 bool IsFshl = (ShAmt == SA0);
3096 Value *TVal = Sel.getTrueValue();
3097 if ((IsFshl && TVal != SV0) || (!IsFshl && TVal != SV1))
3098 return nullptr;
3099
3100 // Finally, see if the select is filtering out a shift-by-zero.
3101 Value *Cond = Sel.getCondition();
3102 if (!match(V: Cond, P: m_OneUse(SubPattern: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Specific(V: ShAmt),
3103 R: m_ZeroInt()))))
3104 return nullptr;
3105
3106 // If this is not a rotate then the select was blocking poison from the
3107 // 'shift-by-zero' non-TVal, but a funnel shift won't - so freeze it.
3108 if (SV0 != SV1) {
3109 if (IsFshl && !llvm::isGuaranteedNotToBePoison(V: SV1))
3110 SV1 = Builder.CreateFreeze(V: SV1);
3111 else if (!IsFshl && !llvm::isGuaranteedNotToBePoison(V: SV0))
3112 SV0 = Builder.CreateFreeze(V: SV0);
3113 }
3114
3115 // This is a funnel/rotate that avoids shift-by-bitwidth UB in a suboptimal way.
3116 // Convert to funnel shift intrinsic.
3117 Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
3118 Function *F =
3119 Intrinsic::getOrInsertDeclaration(M: Sel.getModule(), id: IID, OverloadTys: Sel.getType());
3120 ShAmt = Builder.CreateZExt(V: ShAmt, DestTy: Sel.getType());
3121 return CallInst::Create(Func: F, Args: { SV0, SV1, ShAmt });
3122}
3123
3124static Instruction *foldSelectToCopysign(SelectInst &Sel,
3125 InstCombiner::BuilderTy &Builder) {
3126 Value *Cond = Sel.getCondition();
3127 Value *TVal = Sel.getTrueValue();
3128 Value *FVal = Sel.getFalseValue();
3129 Type *SelType = Sel.getType();
3130
3131 // Match select ?, TC, FC where the constants are equal but negated.
3132 // TODO: Generalize to handle a negated variable operand?
3133 const APFloat *TC, *FC;
3134 if (!match(V: TVal, P: m_APFloatAllowPoison(Res&: TC)) ||
3135 !match(V: FVal, P: m_APFloatAllowPoison(Res&: FC)) ||
3136 !abs(X: *TC).bitwiseIsEqual(RHS: abs(X: *FC)))
3137 return nullptr;
3138
3139 assert(TC != FC && "Expected equal select arms to simplify");
3140
3141 Value *X;
3142 const APInt *C;
3143 bool IsTrueIfSignSet;
3144 CmpPredicate Pred;
3145 if (!match(V: Cond, P: m_OneUse(SubPattern: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Value(V&: X)),
3146 R: m_APInt(Res&: C)))) ||
3147 !isSignBitCheck(Pred, RHS: *C, TrueIfSigned&: IsTrueIfSignSet) || X->getType() != SelType)
3148 return nullptr;
3149
3150 // If needed, negate the value that will be the sign argument of the copysign:
3151 // (bitcast X) < 0 ? -TC : TC --> copysign(TC, X)
3152 // (bitcast X) < 0 ? TC : -TC --> copysign(TC, -X)
3153 // (bitcast X) >= 0 ? -TC : TC --> copysign(TC, -X)
3154 // (bitcast X) >= 0 ? TC : -TC --> copysign(TC, X)
3155 // Note: FMF from the select can not be propagated to the new instructions.
3156 if (IsTrueIfSignSet ^ TC->isNegative())
3157 X = Builder.CreateFNeg(V: X);
3158
3159 // Canonicalize the magnitude argument as the positive constant since we do
3160 // not care about its sign.
3161 Value *MagArg = ConstantFP::get(Ty: SelType, V: abs(X: *TC));
3162 Function *F = Intrinsic::getOrInsertDeclaration(
3163 M: Sel.getModule(), id: Intrinsic::copysign, OverloadTys: Sel.getType());
3164 return CallInst::Create(Func: F, Args: { MagArg, X });
3165}
3166
3167Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) {
3168 if (!isa<VectorType>(Val: Sel.getType()))
3169 return nullptr;
3170
3171 Value *Cond = Sel.getCondition();
3172 Value *TVal = Sel.getTrueValue();
3173 Value *FVal = Sel.getFalseValue();
3174 Value *C, *X, *Y;
3175
3176 if (match(V: Cond, P: m_VecReverse(Op0: m_Value(V&: C)))) {
3177 auto createSelReverse = [&](Value *C, Value *X, Value *Y) {
3178 Value *V = Builder.CreateSelect(C, True: X, False: Y, Name: Sel.getName(), MDFrom: &Sel);
3179 if (auto *I = dyn_cast<Instruction>(Val: V))
3180 I->copyIRFlags(V: &Sel);
3181 Module *M = Sel.getModule();
3182 Function *F = Intrinsic::getOrInsertDeclaration(
3183 M, id: Intrinsic::vector_reverse, OverloadTys: V->getType());
3184 return CallInst::Create(Func: F, Args: V);
3185 };
3186
3187 if (match(V: TVal, P: m_VecReverse(Op0: m_Value(V&: X)))) {
3188 // select rev(C), rev(X), rev(Y) --> rev(select C, X, Y)
3189 if (match(V: FVal, P: m_VecReverse(Op0: m_Value(V&: Y))) &&
3190 (Cond->hasOneUse() || TVal->hasOneUse() || FVal->hasOneUse()))
3191 return createSelReverse(C, X, Y);
3192
3193 // select rev(C), rev(X), FValSplat --> rev(select C, X, FValSplat)
3194 if ((Cond->hasOneUse() || TVal->hasOneUse()) && isSplatValue(V: FVal))
3195 return createSelReverse(C, X, FVal);
3196 }
3197 // select rev(C), TValSplat, rev(Y) --> rev(select C, TValSplat, Y)
3198 else if (isSplatValue(V: TVal) && match(V: FVal, P: m_VecReverse(Op0: m_Value(V&: Y))) &&
3199 (Cond->hasOneUse() || FVal->hasOneUse()))
3200 return createSelReverse(C, TVal, Y);
3201 }
3202
3203 auto *VecTy = dyn_cast<FixedVectorType>(Val: Sel.getType());
3204 if (!VecTy)
3205 return nullptr;
3206
3207 unsigned NumElts = VecTy->getNumElements();
3208 APInt PoisonElts(NumElts, 0);
3209 APInt AllOnesEltMask(APInt::getAllOnes(numBits: NumElts));
3210 if (Value *V = SimplifyDemandedVectorElts(V: &Sel, DemandedElts: AllOnesEltMask, PoisonElts)) {
3211 if (V != &Sel)
3212 return replaceInstUsesWith(I&: Sel, V);
3213 return &Sel;
3214 }
3215
3216 // A select of a "select shuffle" with a common operand can be rearranged
3217 // to select followed by "select shuffle". Because of poison, this only works
3218 // in the case of a shuffle with no undefined mask elements.
3219 ArrayRef<int> Mask;
3220 if (match(V: TVal, P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: X), v2: m_Value(V&: Y), mask: m_Mask(Mask)))) &&
3221 !is_contained(Range&: Mask, Element: PoisonMaskElem) &&
3222 cast<ShuffleVectorInst>(Val: TVal)->isSelect()) {
3223 if (X == FVal) {
3224 // select Cond, (shuf_sel X, Y), X --> shuf_sel X, (select Cond, Y, X)
3225 Value *NewSel = Builder.CreateSelect(C: Cond, True: Y, False: X, Name: "sel", MDFrom: &Sel);
3226 return new ShuffleVectorInst(X, NewSel, Mask);
3227 }
3228 if (Y == FVal) {
3229 // select Cond, (shuf_sel X, Y), Y --> shuf_sel (select Cond, X, Y), Y
3230 Value *NewSel = Builder.CreateSelect(C: Cond, True: X, False: Y, Name: "sel", MDFrom: &Sel);
3231 return new ShuffleVectorInst(NewSel, Y, Mask);
3232 }
3233 }
3234 if (match(V: FVal, P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: X), v2: m_Value(V&: Y), mask: m_Mask(Mask)))) &&
3235 !is_contained(Range&: Mask, Element: PoisonMaskElem) &&
3236 cast<ShuffleVectorInst>(Val: FVal)->isSelect()) {
3237 if (X == TVal) {
3238 // select Cond, X, (shuf_sel X, Y) --> shuf_sel X, (select Cond, X, Y)
3239 Value *NewSel = Builder.CreateSelect(C: Cond, True: X, False: Y, Name: "sel", MDFrom: &Sel);
3240 return new ShuffleVectorInst(X, NewSel, Mask);
3241 }
3242 if (Y == TVal) {
3243 // select Cond, Y, (shuf_sel X, Y) --> shuf_sel (select Cond, Y, X), Y
3244 Value *NewSel = Builder.CreateSelect(C: Cond, True: Y, False: X, Name: "sel", MDFrom: &Sel);
3245 return new ShuffleVectorInst(NewSel, Y, Mask);
3246 }
3247 }
3248
3249 return nullptr;
3250}
3251
3252static Instruction *foldSelectToPhiImpl(SelectInst &Sel, BasicBlock *BB,
3253 const DominatorTree &DT,
3254 InstCombiner::BuilderTy &Builder) {
3255 // Find the block's immediate dominator that ends with a conditional branch
3256 // that matches select's condition (maybe inverted).
3257 auto *IDomNode = DT[BB]->getIDom();
3258 if (!IDomNode)
3259 return nullptr;
3260 BasicBlock *IDom = IDomNode->getBlock();
3261
3262 Value *Cond = Sel.getCondition();
3263 Value *IfTrue, *IfFalse;
3264 BasicBlock *TrueSucc, *FalseSucc;
3265 if (match(V: IDom->getTerminator(),
3266 P: m_Br(C: m_Specific(V: Cond), T: m_BasicBlock(V&: TrueSucc),
3267 F: m_BasicBlock(V&: FalseSucc)))) {
3268 IfTrue = Sel.getTrueValue();
3269 IfFalse = Sel.getFalseValue();
3270 } else if (match(V: IDom->getTerminator(),
3271 P: m_Br(C: m_Not(V: m_Specific(V: Cond)), T: m_BasicBlock(V&: TrueSucc),
3272 F: m_BasicBlock(V&: FalseSucc)))) {
3273 IfTrue = Sel.getFalseValue();
3274 IfFalse = Sel.getTrueValue();
3275 } else
3276 return nullptr;
3277
3278 // Make sure the branches are actually different.
3279 if (TrueSucc == FalseSucc)
3280 return nullptr;
3281
3282 // We want to replace select %cond, %a, %b with a phi that takes value %a
3283 // for all incoming edges that are dominated by condition `%cond == true`,
3284 // and value %b for edges dominated by condition `%cond == false`. If %a
3285 // or %b are also phis from the same basic block, we can go further and take
3286 // their incoming values from the corresponding blocks.
3287 BasicBlockEdge TrueEdge(IDom, TrueSucc);
3288 BasicBlockEdge FalseEdge(IDom, FalseSucc);
3289 DenseMap<BasicBlock *, Value *> Inputs;
3290 for (auto *Pred : predecessors(BB)) {
3291 // Check implication.
3292 BasicBlockEdge Incoming(Pred, BB);
3293 if (DT.dominates(BBE1: TrueEdge, BBE2: Incoming))
3294 Inputs[Pred] = IfTrue->DoPHITranslation(CurBB: BB, PredBB: Pred);
3295 else if (DT.dominates(BBE1: FalseEdge, BBE2: Incoming))
3296 Inputs[Pred] = IfFalse->DoPHITranslation(CurBB: BB, PredBB: Pred);
3297 else
3298 return nullptr;
3299 // Check availability.
3300 if (auto *Insn = dyn_cast<Instruction>(Val: Inputs[Pred]))
3301 if (!DT.dominates(Def: Insn, User: Pred->getTerminator()))
3302 return nullptr;
3303 }
3304
3305 Builder.SetInsertPoint(BB->begin());
3306 auto *PN = Builder.CreatePHI(Ty: Sel.getType(), NumReservedValues: Inputs.size());
3307 for (auto *Pred : predecessors(BB))
3308 PN->addIncoming(V: Inputs[Pred], BB: Pred);
3309 PN->takeName(V: &Sel);
3310 return PN;
3311}
3312
3313static Instruction *foldSelectToPhi(SelectInst &Sel, const DominatorTree &DT,
3314 InstCombiner::BuilderTy &Builder) {
3315 // Try to replace this select with Phi in one of these blocks.
3316 SmallSetVector<BasicBlock *, 4> CandidateBlocks;
3317 CandidateBlocks.insert(X: Sel.getParent());
3318 for (Value *V : Sel.operands())
3319 if (auto *I = dyn_cast<Instruction>(Val: V))
3320 CandidateBlocks.insert(X: I->getParent());
3321
3322 for (BasicBlock *BB : CandidateBlocks)
3323 if (auto *PN = foldSelectToPhiImpl(Sel, BB, DT, Builder))
3324 return PN;
3325 return nullptr;
3326}
3327
3328/// Tries to reduce a pattern that arises when calculating the remainder of the
3329/// Euclidean division. When the divisor is a power of two and is guaranteed not
3330/// to be negative, a signed remainder can be folded with a bitwise and.
3331///
3332/// (x % n) < 0 ? (x % n) + n : (x % n)
3333/// -> x & (n - 1)
3334static Instruction *foldSelectWithSRem(SelectInst &SI, InstCombinerImpl &IC,
3335 IRBuilderBase &Builder) {
3336 Value *CondVal = SI.getCondition();
3337 Value *TrueVal = SI.getTrueValue();
3338 Value *FalseVal = SI.getFalseValue();
3339
3340 CmpPredicate Pred;
3341 Value *Op, *RemRes, *Remainder;
3342 const APInt *C;
3343 bool TrueIfSigned = false;
3344
3345 if (!(match(V: CondVal, P: m_ICmp(Pred, L: m_Value(V&: RemRes), R: m_APInt(Res&: C))) &&
3346 isSignBitCheck(Pred, RHS: *C, TrueIfSigned)))
3347 return nullptr;
3348
3349 // If the sign bit is not set, we have a SGE/SGT comparison, and the operands
3350 // of the select are inverted.
3351 if (!TrueIfSigned)
3352 std::swap(a&: TrueVal, b&: FalseVal);
3353
3354 auto FoldToBitwiseAnd = [&](Value *Remainder) -> Instruction * {
3355 Value *Add = Builder.CreateAdd(
3356 LHS: Remainder, RHS: Constant::getAllOnesValue(Ty: RemRes->getType()));
3357 return BinaryOperator::CreateAnd(V1: Op, V2: Add);
3358 };
3359
3360 // Match the general case:
3361 // %rem = srem i32 %x, %n
3362 // %cnd = icmp slt i32 %rem, 0
3363 // %add = add i32 %rem, %n
3364 // %sel = select i1 %cnd, i32 %add, i32 %rem
3365 if (match(V: TrueVal, P: m_c_Add(L: m_Specific(V: RemRes), R: m_Value(V&: Remainder))) &&
3366 match(V: RemRes, P: m_SRem(L: m_Value(V&: Op), R: m_Specific(V: Remainder))) &&
3367 IC.isKnownToBeAPowerOfTwo(V: Remainder, /*OrZero=*/true) &&
3368 FalseVal == RemRes)
3369 return FoldToBitwiseAnd(Remainder);
3370
3371 // Match the case where the one arm has been replaced by constant 1:
3372 // %rem = srem i32 %n, 2
3373 // %cnd = icmp slt i32 %rem, 0
3374 // %sel = select i1 %cnd, i32 1, i32 %rem
3375 if (match(V: TrueVal, P: m_One()) &&
3376 match(V: RemRes, P: m_SRem(L: m_Value(V&: Op), R: m_SpecificInt(V: 2))) &&
3377 FalseVal == RemRes)
3378 return FoldToBitwiseAnd(ConstantInt::get(Ty: RemRes->getType(), V: 2));
3379
3380 return nullptr;
3381}
3382
3383Value *InstCombinerImpl::simplifyNestedSelectsUsingImpliedCond(
3384 SelectInst &SI, Value *CondVal, bool CondIsTrue) {
3385 Value *InnerCondVal = SI.getCondition();
3386 Value *InnerTrueVal = SI.getTrueValue();
3387 Value *InnerFalseVal = SI.getFalseValue();
3388 assert(CondVal->getType() == InnerCondVal->getType() &&
3389 "The type of inner condition must match with the outer.");
3390 if (auto Implied = isImpliedCondition(LHS: CondVal, RHS: InnerCondVal, DL, LHSIsTrue: CondIsTrue))
3391 return *Implied ? InnerTrueVal : InnerFalseVal;
3392 return nullptr;
3393}
3394
3395Instruction *InstCombinerImpl::foldAndOrOfSelectUsingImpliedCond(Value *Op,
3396 SelectInst &SI,
3397 bool IsAnd) {
3398 assert(Op->getType()->isIntOrIntVectorTy(1) &&
3399 "Op must be either i1 or vector of i1.");
3400 if (SI.getCondition()->getType() != Op->getType())
3401 return nullptr;
3402 if (Value *V = simplifyNestedSelectsUsingImpliedCond(SI, CondVal: Op, CondIsTrue: IsAnd))
3403 return createSelectInstWithUnknownProfile(
3404 C: Op, S1: IsAnd ? V : ConstantInt::getTrue(Ty: Op->getType()),
3405 S2: IsAnd ? ConstantInt::getFalse(Ty: Op->getType()) : V);
3406 return nullptr;
3407}
3408
3409// Canonicalize select with fcmp to fabs(). -0.0 makes this tricky. We need
3410// fast-math-flags (nsz) or fsub with +0.0 (not fneg) for this to work.
3411static Instruction *foldSelectWithFCmpToFabs(SelectInst &SI,
3412 InstCombinerImpl &IC) {
3413 Value *CondVal = SI.getCondition();
3414
3415 bool ChangedFMF = false;
3416 for (bool Swap : {false, true}) {
3417 Value *TrueVal = SI.getTrueValue();
3418 Value *X = SI.getFalseValue();
3419 CmpPredicate Pred;
3420
3421 if (Swap)
3422 std::swap(a&: TrueVal, b&: X);
3423
3424 if (!match(V: CondVal, P: m_FCmp(Pred, L: m_Specific(V: X), R: m_AnyZeroFP())))
3425 continue;
3426
3427 // fold (X <= +/-0.0) ? (0.0 - X) : X to fabs(X), when 'Swap' is false
3428 // fold (X > +/-0.0) ? X : (0.0 - X) to fabs(X), when 'Swap' is true
3429 // Note: We require "nnan" for this fold because fcmp ignores the signbit
3430 // of NAN, but IEEE-754 specifies the signbit of NAN values with
3431 // fneg/fabs operations.
3432 if (match(V: TrueVal, P: m_FSub(L: m_PosZeroFP(), R: m_Specific(V: X))) &&
3433 (cast<FPMathOperator>(Val: CondVal)->hasNoNaNs() || SI.hasNoNaNs() ||
3434 (SI.hasOneUse() && canIgnoreSignBitOfNaN(U: *SI.use_begin())) ||
3435 isKnownNeverNaN(V: X, SQ: IC.getSimplifyQuery().getWithInstruction(
3436 I: cast<Instruction>(Val: CondVal))))) {
3437 if (!Swap && (Pred == FCmpInst::FCMP_OLE || Pred == FCmpInst::FCMP_ULE)) {
3438 Value *Fabs = IC.Builder.CreateFAbs(V: X, FMFSource: &SI);
3439 return IC.replaceInstUsesWith(I&: SI, V: Fabs);
3440 }
3441 if (Swap && (Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_UGT)) {
3442 Value *Fabs = IC.Builder.CreateFAbs(V: X, FMFSource: &SI);
3443 return IC.replaceInstUsesWith(I&: SI, V: Fabs);
3444 }
3445 }
3446
3447 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
3448 if (!match(V: TrueVal, P: m_FNegOrFSubNegZero(X: m_Specific(V: X))))
3449 return nullptr;
3450
3451 // Forward-propagate nnan and ninf from the fcmp to the select.
3452 // If all inputs are not those values, then the select is not either.
3453 // Note: nsz is defined differently, so it may not be correct to propagate.
3454 FastMathFlags FMF = cast<FPMathOperator>(Val: CondVal)->getFastMathFlags();
3455 if (FMF.noNaNs() && !SI.hasNoNaNs()) {
3456 SI.setHasNoNaNs(true);
3457 ChangedFMF = true;
3458 }
3459 if (FMF.noInfs() && !SI.hasNoInfs()) {
3460 SI.setHasNoInfs(true);
3461 ChangedFMF = true;
3462 }
3463 // Forward-propagate nnan from the fneg to the select.
3464 // The nnan flag can be propagated iff fneg is selected when X is NaN.
3465 if (!SI.hasNoNaNs() && cast<FPMathOperator>(Val: TrueVal)->hasNoNaNs() &&
3466 (Swap ? FCmpInst::isOrdered(predicate: Pred) : FCmpInst::isUnordered(predicate: Pred))) {
3467 SI.setHasNoNaNs(true);
3468 ChangedFMF = true;
3469 }
3470
3471 // With nsz, when 'Swap' is false:
3472 // fold (X < +/-0.0) ? -X : X or (X <= +/-0.0) ? -X : X to fabs(X)
3473 // fold (X > +/-0.0) ? -X : X or (X >= +/-0.0) ? -X : X to -fabs(x)
3474 // when 'Swap' is true:
3475 // fold (X > +/-0.0) ? X : -X or (X >= +/-0.0) ? X : -X to fabs(X)
3476 // fold (X < +/-0.0) ? X : -X or (X <= +/-0.0) ? X : -X to -fabs(X)
3477 //
3478 // Note: We require "nnan" for this fold because fcmp ignores the signbit
3479 // of NAN, but IEEE-754 specifies the signbit of NAN values with
3480 // fneg/fabs operations.
3481 if (!SI.hasNoSignedZeros() &&
3482 (!SI.hasOneUse() || !canIgnoreSignBitOfZero(U: *SI.use_begin())))
3483 return nullptr;
3484 if (!SI.hasNoNaNs() &&
3485 (!SI.hasOneUse() || !canIgnoreSignBitOfNaN(U: *SI.use_begin())))
3486 return nullptr;
3487
3488 if (Swap)
3489 Pred = FCmpInst::getSwappedPredicate(pred: Pred);
3490
3491 bool IsLTOrLE = Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE ||
3492 Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE;
3493 bool IsGTOrGE = Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_OGE ||
3494 Pred == FCmpInst::FCMP_UGT || Pred == FCmpInst::FCMP_UGE;
3495
3496 if (IsLTOrLE) {
3497 Value *Fabs = IC.Builder.CreateFAbs(V: X, FMFSource: &SI);
3498 return IC.replaceInstUsesWith(I&: SI, V: Fabs);
3499 }
3500 if (IsGTOrGE) {
3501 Value *Fabs = IC.Builder.CreateFAbs(V: X, FMFSource: &SI);
3502 Instruction *NewFNeg = UnaryOperator::CreateFNeg(V: Fabs);
3503 NewFNeg->setFastMathFlags(SI.getFastMathFlags());
3504 return NewFNeg;
3505 }
3506 }
3507
3508 // Match select with (icmp slt (bitcast X to int), 0)
3509 // or (icmp sgt (bitcast X to int), -1)
3510
3511 for (bool Swap : {false, true}) {
3512 Value *TrueVal = SI.getTrueValue();
3513 Value *X = SI.getFalseValue();
3514
3515 if (Swap)
3516 std::swap(a&: TrueVal, b&: X);
3517
3518 CmpPredicate Pred;
3519 const APInt *C;
3520 bool TrueIfSigned;
3521 if (!match(V: CondVal,
3522 P: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Specific(V: X)), R: m_APInt(Res&: C))) ||
3523 !isSignBitCheck(Pred, RHS: *C, TrueIfSigned))
3524 continue;
3525 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
3526 if (!match(V: TrueVal, P: m_FNegOrFSubNegZero(X: m_Specific(V: X))))
3527 return nullptr;
3528 if (Swap == TrueIfSigned && !CondVal->hasOneUse() && !TrueVal->hasOneUse())
3529 return nullptr;
3530
3531 // Fold (IsNeg ? -X : X) or (!IsNeg ? X : -X) to fabs(X)
3532 // Fold (IsNeg ? X : -X) or (!IsNeg ? -X : X) to -fabs(X)
3533 Value *Fabs = IC.Builder.CreateFAbs(V: X, FMFSource: &SI);
3534 if (Swap != TrueIfSigned)
3535 return IC.replaceInstUsesWith(I&: SI, V: Fabs);
3536 return UnaryOperator::CreateFNegFMF(Op: Fabs, FMFSource: &SI);
3537 }
3538
3539 return ChangedFMF ? &SI : nullptr;
3540}
3541
3542// Fold a select of an ordered fcmp using fabs of a NaN-scrubbed value:
3543// %s = select i1 (isnotnan T %x), T %x, T %y
3544// %a = call T @llvm.fabs.T(T %s)
3545// %c = fcmp <ordered-pred> T %a, %k
3546// %r = select i1 %c, T %s, T %y
3547// =>
3548// %a2 = call T @llvm.fabs.T(T %x)
3549// %c2 = fcmp <ordered-pred> T %a2, %k
3550// %r2 = select i1 %c2, T %x, T %y
3551static Instruction *
3552foldSelectOfOrderedFAbsCmpOfNaNScrubbedValue(SelectInst &SI,
3553 InstCombinerImpl &IC) {
3554 Instruction *OuterCmpI;
3555 Value *Cmp0, *Cmp1;
3556 if (!match(V: SI.getCondition(),
3557 P: m_OneUse(SubPattern: m_Instruction(I&: OuterCmpI,
3558 P: m_FCmp(L: m_Value(V&: Cmp0), R: m_Value(V&: Cmp1))))))
3559 return nullptr;
3560
3561 auto *OuterCmp = cast<FCmpInst>(Val: OuterCmpI);
3562 CmpInst::Predicate Pred = OuterCmp->getPredicate();
3563 if (!FCmpInst::isOrdered(predicate: Pred))
3564 return nullptr;
3565
3566 Value *Y = SI.getFalseValue();
3567 Value *InnerSel = SI.getTrueValue();
3568
3569 // Match a select that returns X when X is not NaN, and Y otherwise:
3570 // select (fcmp ord X, 0.0), X, Y
3571 Value *X;
3572 if (!match(V: InnerSel,
3573 P: m_Select(C: m_OneUse(SubPattern: m_SpecificFCmp(MatchPred: FCmpInst::FCMP_ORD, L: m_Value(V&: X),
3574 R: m_AnyZeroFP())),
3575 L: m_Deferred(V: X), R: m_Specific(V: Y))))
3576 return nullptr;
3577
3578 Instruction *FAbsI;
3579 auto MatchFAbsOfInnerSel = [&](Value *V) {
3580 return match(V,
3581 P: m_OneUse(SubPattern: m_Instruction(I&: FAbsI, P: m_FAbs(Op0: m_Specific(V: InnerSel)))));
3582 };
3583
3584 if (!MatchFAbsOfInnerSel(Cmp0)) {
3585 if (!MatchFAbsOfInnerSel(Cmp1))
3586 return nullptr;
3587
3588 std::swap(a&: Cmp0, b&: Cmp1);
3589 Pred = CmpInst::getSwappedPredicate(pred: Pred);
3590 }
3591
3592 FastMathFlags FAbsFMF = FAbsI->getFastMathFlags();
3593 FastMathFlags CmpFMF = OuterCmp->getFastMathFlags();
3594
3595 FastMathFlags CommonRewriteFMF =
3596 FastMathFlags::intersectRewrite(LHS: FAbsFMF, RHS: CmpFMF);
3597
3598 // unionValue with FastMathFlags() drops all rewriter based flags
3599 FastMathFlags NewFAbsFMF =
3600 CommonRewriteFMF | FastMathFlags::unionValue(LHS: FAbsFMF, RHS: FastMathFlags());
3601 FastMathFlags NewCmpFMF =
3602 CommonRewriteFMF | FastMathFlags::unionValue(LHS: CmpFMF, RHS: FastMathFlags());
3603
3604 // When X is NaN, the old code evaluated fabs(Y), while the new code evaluates
3605 // fabs(X). Do not preserve nnan on either newly-created instruction.
3606 NewFAbsFMF.setNoNaNs(false);
3607 NewCmpFMF.setNoNaNs(false);
3608
3609 Value *NewAbs = IC.Builder.CreateFAbs(V: X, FMFSource: FMFSource(NewFAbsFMF));
3610 Value *NewCmp =
3611 IC.Builder.CreateFCmpFMF(P: Pred, LHS: NewAbs, RHS: Cmp1, FMFSource: FMFSource(NewCmpFMF));
3612 Value *NewSel = IC.Builder.CreateSelectFMF(C: NewCmp, True: X, False: Y, FMFSource: &SI);
3613
3614 Instruction *NewSelUsesReplaced = IC.replaceInstUsesWith(I&: SI, V: NewSel);
3615
3616 uint64_t WeightNotNaN, WeightNaN, WeightComparisonTrue,
3617 WeightComparisonFalse = 0;
3618 bool HasProfile = extractBranchWeights(I: *cast<SelectInst>(Val: InnerSel),
3619 TrueVal&: WeightNotNaN, FalseVal&: WeightNaN);
3620 HasProfile &=
3621 extractBranchWeights(I: SI, TrueVal&: WeightComparisonTrue, FalseVal&: WeightComparisonFalse);
3622 if (!HasProfile || !isa<SelectInst>(Val: NewSel))
3623 return NewSelUsesReplaced;
3624 // The branch weights for the new select will be the same as before, except
3625 // they will additionally account for the probability of NaN values which was
3626 // previously handled with the inner select. For the true arm the new
3627 // probability is P(not Nan) * P(fcmp true). For the false arm, the new
3628 // probability is P(NaN) + (P(not NaN) * P(fcmp false)). We can assume the
3629 // probabilities are independent given the first select only checks for NaNs
3630 // and the second select's condition will never see NaNs because of the first
3631 // select. The code below uses some algebraic simplifications on top of those
3632 // formulas.
3633 uint64_t WeightNewSelTrue = WeightNotNaN * WeightComparisonTrue;
3634 uint64_t WeightNewSelFalse =
3635 WeightNaN * (WeightComparisonTrue + WeightComparisonFalse) +
3636 WeightNotNaN * WeightComparisonFalse;
3637 if (!ProfcheckDisableMetadataFixes)
3638 setFittedBranchWeights(I&: *cast<SelectInst>(Val: NewSel),
3639 Weights: {WeightNewSelTrue, WeightNewSelFalse},
3640 /*IsExpected*/ false);
3641 return NewSelUsesReplaced;
3642}
3643
3644// Match the following IR pattern:
3645// %x.lowbits = and i8 %x, %lowbitmask
3646// %x.lowbits.are.zero = icmp eq i8 %x.lowbits, 0
3647// %x.biased = add i8 %x, %bias
3648// %x.biased.highbits = and i8 %x.biased, %highbitmask
3649// %x.roundedup = select i1 %x.lowbits.are.zero, i8 %x, i8 %x.biased.highbits
3650// Define:
3651// %alignment = add i8 %lowbitmask, 1
3652// Iff 1. an %alignment is a power-of-two (aka, %lowbitmask is a low bit mask)
3653// and 2. %bias is equal to either %lowbitmask or %alignment,
3654// and 3. %highbitmask is equal to ~%lowbitmask (aka, to -%alignment)
3655// then this pattern can be transformed into:
3656// %x.offset = add i8 %x, %lowbitmask
3657// %x.roundedup = and i8 %x.offset, %highbitmask
3658static Value *
3659foldRoundUpIntegerWithPow2Alignment(SelectInst &SI,
3660 InstCombiner::BuilderTy &Builder) {
3661 Value *Cond = SI.getCondition();
3662 Value *X = SI.getTrueValue();
3663 Value *XBiasedHighBits = SI.getFalseValue();
3664
3665 CmpPredicate Pred;
3666 Value *XLowBits;
3667 if (!match(V: Cond, P: m_ICmp(Pred, L: m_Value(V&: XLowBits), R: m_ZeroInt())) ||
3668 !ICmpInst::isEquality(P: Pred))
3669 return nullptr;
3670
3671 if (Pred == ICmpInst::Predicate::ICMP_NE)
3672 std::swap(a&: X, b&: XBiasedHighBits);
3673
3674 // FIXME: we could support non non-splats here.
3675
3676 const APInt *LowBitMaskCst;
3677 if (!match(V: XLowBits, P: m_And(L: m_Specific(V: X), R: m_APIntAllowPoison(Res&: LowBitMaskCst))))
3678 return nullptr;
3679
3680 // Match even if the AND and ADD are swapped.
3681 const APInt *BiasCst, *HighBitMaskCst;
3682 if (!match(V: XBiasedHighBits,
3683 P: m_And(L: m_Add(L: m_Specific(V: X), R: m_APIntAllowPoison(Res&: BiasCst)),
3684 R: m_APIntAllowPoison(Res&: HighBitMaskCst))) &&
3685 !match(V: XBiasedHighBits,
3686 P: m_Add(L: m_And(L: m_Specific(V: X), R: m_APIntAllowPoison(Res&: HighBitMaskCst)),
3687 R: m_APIntAllowPoison(Res&: BiasCst))))
3688 return nullptr;
3689
3690 if (!LowBitMaskCst->isMask())
3691 return nullptr;
3692
3693 APInt InvertedLowBitMaskCst = ~*LowBitMaskCst;
3694 if (InvertedLowBitMaskCst != *HighBitMaskCst)
3695 return nullptr;
3696
3697 APInt AlignmentCst = *LowBitMaskCst + 1;
3698
3699 if (*BiasCst != AlignmentCst && *BiasCst != *LowBitMaskCst)
3700 return nullptr;
3701
3702 if (!XBiasedHighBits->hasOneUse()) {
3703 // We can't directly return XBiasedHighBits if it is more poisonous.
3704 if (*BiasCst == *LowBitMaskCst && impliesPoison(ValAssumedPoison: XBiasedHighBits, V: X))
3705 return XBiasedHighBits;
3706 return nullptr;
3707 }
3708
3709 // FIXME: could we preserve undef's here?
3710 Type *Ty = X->getType();
3711 Value *XOffset = Builder.CreateAdd(LHS: X, RHS: ConstantInt::get(Ty, V: *LowBitMaskCst),
3712 Name: X->getName() + ".biased");
3713 Value *R = Builder.CreateAnd(LHS: XOffset, RHS: ConstantInt::get(Ty, V: *HighBitMaskCst));
3714 R->takeName(V: &SI);
3715 return R;
3716}
3717
3718namespace {
3719struct DecomposedSelect {
3720 Value *Cond = nullptr;
3721 Value *TrueVal = nullptr;
3722 Value *FalseVal = nullptr;
3723};
3724} // namespace
3725
3726/// Folds patterns like:
3727/// select c2 (select c1 a b) (select c1 b a)
3728/// into:
3729/// select (xor c1 c2) b a
3730static Instruction *
3731foldSelectOfSymmetricSelect(SelectInst &OuterSelVal,
3732 InstCombiner::BuilderTy &Builder) {
3733
3734 Value *OuterCond, *InnerCond, *InnerTrueVal, *InnerFalseVal;
3735 if (!match(
3736 V: &OuterSelVal,
3737 P: m_Select(C: m_Value(V&: OuterCond),
3738 L: m_OneUse(SubPattern: m_Select(C: m_Value(V&: InnerCond), L: m_Value(V&: InnerTrueVal),
3739 R: m_Value(V&: InnerFalseVal))),
3740 R: m_OneUse(SubPattern: m_Select(C: m_Deferred(V: InnerCond),
3741 L: m_Deferred(V: InnerFalseVal),
3742 R: m_Deferred(V: InnerTrueVal))))))
3743 return nullptr;
3744
3745 if (OuterCond->getType() != InnerCond->getType())
3746 return nullptr;
3747
3748 Value *Xor = Builder.CreateXor(LHS: InnerCond, RHS: OuterCond);
3749 SelectInst *SI = SelectInst::Create(C: Xor, S1: InnerFalseVal, S2: InnerTrueVal);
3750 // We cannot construct branch weights without assuming that the distributions
3751 // of the two conditions are independent, so mark the branch weights as
3752 // unknown.
3753 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, DEBUG_TYPE,
3754 F: OuterSelVal.getFunction());
3755 return SI;
3756}
3757
3758/// Look for patterns like
3759/// %outer.cond = select i1 %inner.cond, i1 %alt.cond, i1 false
3760/// %inner.sel = select i1 %inner.cond, i8 %inner.sel.t, i8 %inner.sel.f
3761/// %outer.sel = select i1 %outer.cond, i8 %outer.sel.t, i8 %inner.sel
3762/// and rewrite it as
3763/// %inner.sel = select i1 %cond.alternative, i8 %sel.outer.t, i8 %sel.inner.t
3764/// %sel.outer = select i1 %cond.inner, i8 %inner.sel, i8 %sel.inner.f
3765static Instruction *foldNestedSelects(SelectInst &OuterSelVal,
3766 InstCombiner::BuilderTy &Builder) {
3767 // We must start with a `select`.
3768 DecomposedSelect OuterSel;
3769 match(V: &OuterSelVal,
3770 P: m_Select(C: m_Value(V&: OuterSel.Cond), L: m_Value(V&: OuterSel.TrueVal),
3771 R: m_Value(V&: OuterSel.FalseVal)));
3772
3773 // Canonicalize inversion of the outermost `select`'s condition.
3774 if (match(V: OuterSel.Cond, P: m_Not(V: m_Value(V&: OuterSel.Cond))))
3775 std::swap(a&: OuterSel.TrueVal, b&: OuterSel.FalseVal);
3776
3777 // The condition of the outermost select must be an `and`/`or`.
3778 if (!match(V: OuterSel.Cond, P: m_c_LogicalOp(L: m_Value(), R: m_Value())))
3779 return nullptr;
3780
3781 // Depending on the logical op, inner select might be in different hand.
3782 bool IsAndVariant = match(V: OuterSel.Cond, P: m_LogicalAnd());
3783 Value *InnerSelVal = IsAndVariant ? OuterSel.FalseVal : OuterSel.TrueVal;
3784
3785 // Profitability check - avoid increasing instruction count.
3786 if (none_of(Range: ArrayRef<Value *>({OuterSelVal.getCondition(), InnerSelVal}),
3787 P: match_fn(P: m_OneUse(SubPattern: m_Value()))))
3788 return nullptr;
3789
3790 // The appropriate hand of the outermost `select` must be a select itself.
3791 DecomposedSelect InnerSel;
3792 if (!match(V: InnerSelVal,
3793 P: m_Select(C: m_Value(V&: InnerSel.Cond), L: m_Value(V&: InnerSel.TrueVal),
3794 R: m_Value(V&: InnerSel.FalseVal))))
3795 return nullptr;
3796
3797 // Canonicalize inversion of the innermost `select`'s condition.
3798 bool SwapInnerSelCond = false;
3799 if (match(V: InnerSel.Cond, P: m_Not(V: m_Value(V&: InnerSel.Cond)))) {
3800 std::swap(a&: InnerSel.TrueVal, b&: InnerSel.FalseVal);
3801 SwapInnerSelCond = !SwapInnerSelCond;
3802 }
3803
3804 Value *AltCond = nullptr;
3805 auto matchOuterCond = [OuterSel, IsAndVariant, &AltCond](auto m_InnerCond) {
3806 // An unsimplified select condition can match both LogicalAnd and LogicalOr
3807 // (select true, true, false). Since below we assume that LogicalAnd implies
3808 // InnerSel match the FVal and vice versa for LogicalOr, we can't match the
3809 // alternative pattern here.
3810 return IsAndVariant ? match(OuterSel.Cond,
3811 m_c_LogicalAnd(m_InnerCond, m_Value(V&: AltCond)))
3812 : match(OuterSel.Cond,
3813 m_c_LogicalOr(m_InnerCond, m_Value(V&: AltCond)));
3814 };
3815
3816 // Finally, match the condition that was driving the outermost `select`,
3817 // it should be a logical operation between the condition that was driving
3818 // the innermost `select` (after accounting for the possible inversions
3819 // of the condition), and some other condition.
3820 if (matchOuterCond(m_Specific(V: InnerSel.Cond))) {
3821 // Done!
3822 } else if (Value * NotInnerCond; matchOuterCond(m_CombineAnd(
3823 Ps: m_Not(V: m_Specific(V: InnerSel.Cond)), Ps: m_Value(V&: NotInnerCond)))) {
3824 // Done!
3825 std::swap(a&: InnerSel.TrueVal, b&: InnerSel.FalseVal);
3826 InnerSel.Cond = NotInnerCond;
3827 SwapInnerSelCond = !SwapInnerSelCond;
3828 } else // Not the pattern we were looking for.
3829 return nullptr;
3830
3831 // We mark the select with AltCond as having an unknown profile given the
3832 // condition is derived from an and/or. We might have profile information on
3833 // the operands of the and/or, but there is no guarantee that they are
3834 // independent.
3835 Value *SelInner = Builder.CreateSelectWithUnknownProfile(
3836 C: AltCond, True: IsAndVariant ? OuterSel.TrueVal : InnerSel.FalseVal,
3837 False: IsAndVariant ? InnerSel.TrueVal : OuterSel.FalseVal, DEBUG_TYPE);
3838 SelInner->takeName(V: InnerSelVal);
3839 SelectInst *SI = SelectInst::Create(
3840 C: InnerSel.Cond, S1: IsAndVariant ? SelInner : InnerSel.TrueVal,
3841 S2: !IsAndVariant ? SelInner : InnerSel.FalseVal, NameStr: "", InsertBefore: nullptr,
3842 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : cast<Instruction>(Val: InnerSelVal));
3843 if (SwapInnerSelCond)
3844 SI->swapProfMetadata();
3845 return SI;
3846}
3847
3848/// Return true if V is poison or \p Expected given that ValAssumedPoison is
3849/// already poison. For example, if ValAssumedPoison is `icmp samesign X, 10`
3850/// and V is `icmp ne X, 5`, impliesPoisonOrCond returns true.
3851static bool impliesPoisonOrCond(const Value *ValAssumedPoison, const Value *V,
3852 bool Expected, const SimplifyQuery &SQ) {
3853 if (impliesPoison(ValAssumedPoison, V))
3854 return true;
3855
3856 // Handle the case that ValAssumedPoison is `icmp samesign pred X, C1` and V
3857 // is `icmp pred X, C2`, where C1 is well-defined.
3858 if (auto *ICmp = dyn_cast<ICmpInst>(Val: ValAssumedPoison)) {
3859 Value *LHS = ICmp->getOperand(i_nocapture: 0);
3860 const APInt *RHSC1;
3861 const APInt *RHSC2;
3862 CmpPredicate Pred;
3863 if (ICmp->hasSameSign() &&
3864 match(V: ICmp->getOperand(i_nocapture: 1), P: m_APIntForbidPoison(Res&: RHSC1)) &&
3865 match(V, P: m_ICmp(Pred, L: m_Specific(V: LHS), R: m_APIntAllowPoison(Res&: RHSC2)))) {
3866 unsigned BitWidth = RHSC1->getBitWidth();
3867 ConstantRange CRX =
3868 RHSC1->isNonNegative()
3869 ? ConstantRange(APInt::getSignedMinValue(numBits: BitWidth),
3870 APInt::getZero(numBits: BitWidth))
3871 : ConstantRange(APInt::getZero(numBits: BitWidth),
3872 APInt::getSignedMinValue(numBits: BitWidth));
3873 return CRX.icmp(Pred: Expected ? Pred : ICmpInst::getInverseCmpPredicate(Pred),
3874 Other: *RHSC2);
3875 }
3876 }
3877 // For non-poison X in [0, 1], `trunc nuw X to i1` is not poison, but an
3878 // additional `nsw` flag makes it poison for X == 1.
3879 Value *A;
3880 if (match(V: ValAssumedPoison, P: m_NUWTrunc(Op: m_Value(V&: A))) &&
3881 !cast<TruncInst>(Val: ValAssumedPoison)->hasNoSignedWrap() &&
3882 isGuaranteedNotToBePoison(V: A)) {
3883 assert(ValAssumedPoison->getType()->isIntOrIntVectorTy(1));
3884 return computeKnownBits(
3885 V: A, Q: SQ.getWithInstruction(I: cast<Instruction>(Val: ValAssumedPoison)))
3886 .getMaxValue() == 1;
3887 }
3888
3889 return false;
3890}
3891
3892Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) {
3893 Value *CondVal = SI.getCondition();
3894 Value *TrueVal = SI.getTrueValue();
3895 Value *FalseVal = SI.getFalseValue();
3896 Type *SelType = SI.getType();
3897
3898 // Avoid potential infinite loops by checking for non-constant condition.
3899 // TODO: Can we assert instead by improving canonicalizeSelectToShuffle()?
3900 // Scalar select must have simplified?
3901 if (!SelType->isIntOrIntVectorTy(BitWidth: 1) || isa<Constant>(Val: CondVal) ||
3902 TrueVal->getType() != CondVal->getType())
3903 return nullptr;
3904
3905 auto *One = ConstantInt::getTrue(Ty: SelType);
3906 auto *Zero = ConstantInt::getFalse(Ty: SelType);
3907 Value *A, *B, *C, *D;
3908
3909 // Folding select to and/or i1 isn't poison safe in general. impliesPoison
3910 // checks whether folding it does not convert a well-defined value into
3911 // poison.
3912 if (match(V: TrueVal, P: m_One())) {
3913 if (impliesPoisonOrCond(ValAssumedPoison: FalseVal, V: CondVal, /*Expected=*/false, SQ)) {
3914 // Change: A = select B, true, C --> A = or B, C
3915 return BinaryOperator::CreateOr(V1: CondVal, V2: FalseVal);
3916 }
3917
3918 if (match(V: CondVal, P: m_OneUse(SubPattern: m_Select(C: m_Value(V&: A), L: m_One(), R: m_Value(V&: B)))) &&
3919 impliesPoisonOrCond(ValAssumedPoison: FalseVal, V: B, /*Expected=*/false, SQ)) {
3920 // (A || B) || C --> A || (B | C)
3921 // CondVal is select A, true, B, so its weights describe A.
3922 Value *LOr = Builder.CreateLogicalOr(
3923 Cond1: A, Cond2: Builder.CreateOr(LHS: B, RHS: FalseVal), Name: "",
3924 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : cast<Instruction>(Val: CondVal));
3925 return replaceInstUsesWith(I&: SI, V: LOr);
3926 }
3927
3928 // (A && B) || (C && B) --> (A || C) && B
3929 if (match(V: CondVal, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3930 match(V: FalseVal, P: m_LogicalAnd(L: m_Value(V&: C), R: m_Value(V&: D))) &&
3931 (CondVal->hasOneUse() || FalseVal->hasOneUse())) {
3932 bool CondLogicAnd = isa<SelectInst>(Val: CondVal);
3933 bool FalseLogicAnd = isa<SelectInst>(Val: FalseVal);
3934 auto AndFactorization = [&](Value *Common, Value *InnerCond,
3935 Value *InnerVal,
3936 bool SelFirst = false) -> Instruction * {
3937 Value *InnerSel = Builder.CreateSelectWithUnknownProfile(
3938 C: InnerCond, True: One, False: InnerVal, DEBUG_TYPE);
3939 if (SelFirst)
3940 std::swap(a&: Common, b&: InnerSel);
3941 if (FalseLogicAnd || (CondLogicAnd && Common == A))
3942 return createSelectInstWithUnknownProfile(C: Common, S1: InnerSel, S2: Zero);
3943 else
3944 return BinaryOperator::CreateAnd(V1: Common, V2: InnerSel);
3945 };
3946
3947 if (A == C)
3948 return AndFactorization(A, B, D);
3949 if (A == D)
3950 return AndFactorization(A, B, C);
3951 if (B == C)
3952 return AndFactorization(B, A, D);
3953 if (B == D)
3954 return AndFactorization(B, A, C, CondLogicAnd && FalseLogicAnd);
3955 }
3956 }
3957
3958 if (match(V: FalseVal, P: m_Zero())) {
3959 if (impliesPoisonOrCond(ValAssumedPoison: TrueVal, V: CondVal, /*Expected=*/true, SQ)) {
3960 // Change: A = select B, C, false --> A = and B, C
3961 return BinaryOperator::CreateAnd(V1: CondVal, V2: TrueVal);
3962 }
3963
3964 if (match(V: CondVal, P: m_OneUse(SubPattern: m_Select(C: m_Value(V&: A), L: m_Value(V&: B), R: m_Zero()))) &&
3965 impliesPoisonOrCond(ValAssumedPoison: TrueVal, V: B, /*Expected=*/true, SQ)) {
3966 // (A && B) && C --> A && (B & C)
3967 // CondVal is select A, B, false, so its weights describe A.
3968 Value *LAnd = Builder.CreateLogicalAnd(
3969 Cond1: A, Cond2: Builder.CreateAnd(LHS: B, RHS: TrueVal), Name: "",
3970 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : cast<Instruction>(Val: CondVal));
3971 return replaceInstUsesWith(I&: SI, V: LAnd);
3972 }
3973
3974 // (A || B) && (C || B) --> (A && C) || B
3975 if (match(V: CondVal, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3976 match(V: TrueVal, P: m_LogicalOr(L: m_Value(V&: C), R: m_Value(V&: D))) &&
3977 (CondVal->hasOneUse() || TrueVal->hasOneUse())) {
3978 bool CondLogicOr = isa<SelectInst>(Val: CondVal);
3979 bool TrueLogicOr = isa<SelectInst>(Val: TrueVal);
3980 auto OrFactorization = [&](Value *Common, Value *InnerCond,
3981 Value *InnerVal,
3982 bool SelFirst = false) -> Instruction * {
3983 Value *InnerSel = Builder.CreateSelectWithUnknownProfile(
3984 C: InnerCond, True: InnerVal, False: Zero, DEBUG_TYPE);
3985 if (SelFirst)
3986 std::swap(a&: Common, b&: InnerSel);
3987 if (TrueLogicOr || (CondLogicOr && Common == A))
3988 return createSelectInstWithUnknownProfile(C: Common, S1: One, S2: InnerSel);
3989 else
3990 return BinaryOperator::CreateOr(V1: Common, V2: InnerSel);
3991 };
3992
3993 if (A == C)
3994 return OrFactorization(A, B, D);
3995 if (A == D)
3996 return OrFactorization(A, B, C);
3997 if (B == C)
3998 return OrFactorization(B, A, D);
3999 if (B == D)
4000 return OrFactorization(B, A, C, CondLogicOr && TrueLogicOr);
4001 }
4002 }
4003
4004 // We match the "full" 0 or 1 constant here to avoid a potential infinite
4005 // loop with vectors that may have undefined/poison elements.
4006 // select a, false, b -> select !a, b, false
4007 if (match(V: TrueVal, P: m_Specific(V: Zero))) {
4008 Value *NotCond = Builder.CreateNot(V: CondVal, Name: "not." + CondVal->getName());
4009 SelectInst *NewSI = SelectInst::Create(C: NotCond, S1: FalseVal, S2: Zero, NameStr: "", InsertBefore: nullptr,
4010 /*MDFrom=*/&SI);
4011 NewSI->swapProfMetadata();
4012 return NewSI;
4013 }
4014 // select a, b, true -> select !a, true, b
4015 if (match(V: FalseVal, P: m_Specific(V: One))) {
4016 Value *NotCond = Builder.CreateNot(V: CondVal, Name: "not." + CondVal->getName());
4017 SelectInst *NewSI =
4018 SelectInst::Create(C: NotCond, S1: One, S2: TrueVal, NameStr: "", InsertBefore: nullptr, /*MDFrom=*/&SI);
4019 NewSI->swapProfMetadata();
4020 return NewSI;
4021 }
4022
4023 // DeMorgan in select form: !a && !b --> !(a || b)
4024 // select !a, !b, false --> not (select a, true, b)
4025 if (match(V: &SI, P: m_LogicalAnd(L: m_Not(V: m_Value(V&: A)), R: m_Not(V: m_Value(V&: B)))) &&
4026 (CondVal->hasOneUse() || TrueVal->hasOneUse()) &&
4027 !match(V: A, P: m_ConstantExpr()) && !match(V: B, P: m_ConstantExpr())) {
4028 SelectInst *NewSI =
4029 cast<SelectInst>(Val: Builder.CreateSelect(C: A, True: One, False: B, Name: "", /*MDFrom=*/&SI));
4030 NewSI->swapProfMetadata();
4031 return BinaryOperator::CreateNot(Op: NewSI);
4032 }
4033
4034 // DeMorgan in select form: !a || !b --> !(a && b)
4035 // select !a, true, !b --> not (select a, b, false)
4036 if (match(V: &SI, P: m_LogicalOr(L: m_Not(V: m_Value(V&: A)), R: m_Not(V: m_Value(V&: B)))) &&
4037 (CondVal->hasOneUse() || FalseVal->hasOneUse()) &&
4038 !match(V: A, P: m_ConstantExpr()) && !match(V: B, P: m_ConstantExpr())) {
4039 SelectInst *NewSI =
4040 cast<SelectInst>(Val: Builder.CreateSelect(C: A, True: B, False: Zero, Name: "", /*MDFrom=*/&SI));
4041 NewSI->swapProfMetadata();
4042 return BinaryOperator::CreateNot(Op: NewSI);
4043 }
4044
4045 // select (select a, true, b), true, b -> select a, true, b
4046 if (match(V: CondVal, P: m_Select(C: m_Value(V&: A), L: m_One(), R: m_Value(V&: B))) &&
4047 match(V: TrueVal, P: m_One()) && match(V: FalseVal, P: m_Specific(V: B)))
4048 return replaceOperand(I&: SI, OpNum: 0, V: A);
4049 // select (select a, b, false), b, false -> select a, b, false
4050 if (match(V: CondVal, P: m_Select(C: m_Value(V&: A), L: m_Value(V&: B), R: m_Zero())) &&
4051 match(V: TrueVal, P: m_Specific(V: B)) && match(V: FalseVal, P: m_Zero()))
4052 return replaceOperand(I&: SI, OpNum: 0, V: A);
4053
4054 // ~(A & B) & (A | B) --> A ^ B
4055 if (match(V: &SI, P: m_c_LogicalAnd(L: m_Not(V: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B))),
4056 R: m_c_LogicalOr(L: m_Deferred(V: A), R: m_Deferred(V: B)))))
4057 return BinaryOperator::CreateXor(V1: A, V2: B);
4058
4059 // select (~a | c), a, b -> select a, (select c, true, b), false
4060 if (match(V: CondVal,
4061 P: m_OneUse(SubPattern: m_c_Or(L: m_Not(V: m_Specific(V: TrueVal)), R: m_Value(V&: C))))) {
4062 // TODO(#183864): We could improve the profile if P(~a | c) < 0.5, which
4063 // implies strong bounds on both operands (P(a) is high, P(c) is low).
4064 Value *OrV =
4065 Builder.CreateSelectWithUnknownProfile(C, True: One, False: FalseVal, DEBUG_TYPE);
4066 return createSelectInstWithUnknownProfile(C: TrueVal, S1: OrV, S2: Zero);
4067 }
4068 // select (c & b), a, b -> select b, (select ~c, true, a), false
4069 if (match(V: CondVal, P: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: C), R: m_Specific(V: FalseVal))))) {
4070 if (Value *NotC = getFreelyInverted(V: C, WillInvertAllUses: C->hasOneUse(), Builder: &Builder)) {
4071 Value *OrV = Builder.CreateSelectWithUnknownProfile(C: NotC, True: One, False: TrueVal,
4072 DEBUG_TYPE);
4073 return createSelectInstWithUnknownProfile(C: FalseVal, S1: OrV, S2: Zero);
4074 }
4075 }
4076 // select (a | c), a, b -> select a, true, (select ~c, b, false)
4077 if (match(V: CondVal, P: m_OneUse(SubPattern: m_c_Or(L: m_Specific(V: TrueVal), R: m_Value(V&: C))))) {
4078 if (Value *NotC = getFreelyInverted(V: C, WillInvertAllUses: C->hasOneUse(), Builder: &Builder)) {
4079 // TODO(#183864): We could improve the profile if P(a | c) < 0.5, which
4080 // implies strong bounds on both operands (both P(a) and P(c) are low).
4081 Value *AndV = Builder.CreateSelectWithUnknownProfile(C: NotC, True: FalseVal, False: Zero,
4082 DEBUG_TYPE);
4083 return createSelectInstWithUnknownProfile(C: TrueVal, S1: One, S2: AndV);
4084 }
4085 }
4086 // select (c & ~b), a, b -> select b, true, (select c, a, false)
4087 if (match(V: CondVal,
4088 P: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: C), R: m_Not(V: m_Specific(V: FalseVal)))))) {
4089 Value *AndV =
4090 Builder.CreateSelectWithUnknownProfile(C, True: TrueVal, False: Zero, DEBUG_TYPE);
4091 return createSelectInstWithUnknownProfile(C: FalseVal, S1: One, S2: AndV);
4092 }
4093
4094 if (match(V: FalseVal, P: m_Zero()) || match(V: TrueVal, P: m_One())) {
4095 Use *Y = nullptr;
4096 bool IsAnd = match(V: FalseVal, P: m_Zero()) ? true : false;
4097 Value *Op1 = IsAnd ? TrueVal : FalseVal;
4098 if (isCheckForZeroAndMulWithOverflow(Op0: CondVal, Op1, IsAnd, Y)) {
4099 auto *FI = new FreezeInst(*Y, (*Y)->getName() + ".fr");
4100 InsertNewInstBefore(New: FI, Old: cast<Instruction>(Val: Y->getUser())->getIterator());
4101 replaceUse(U&: *Y, NewValue: FI);
4102 return replaceInstUsesWith(I&: SI, V: Op1);
4103 }
4104
4105 if (auto *V = foldBooleanAndOr(LHS: CondVal, RHS: Op1, I&: SI, IsAnd,
4106 /*IsLogical=*/true))
4107 return replaceInstUsesWith(I&: SI, V);
4108 }
4109
4110 // select (a || b), c, false -> select a, c, false
4111 // select c, (a || b), false -> select c, a, false
4112 // if c implies that b is false.
4113 if (match(V: CondVal, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4114 match(V: FalseVal, P: m_Zero())) {
4115 std::optional<bool> Res = isImpliedCondition(LHS: TrueVal, RHS: B, DL);
4116 if (Res && *Res == false)
4117 return replaceOperand(I&: SI, OpNum: 0, V: A);
4118 }
4119 if (match(V: TrueVal, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4120 match(V: FalseVal, P: m_Zero())) {
4121 std::optional<bool> Res = isImpliedCondition(LHS: CondVal, RHS: B, DL);
4122 if (Res && *Res == false)
4123 return replaceOperand(I&: SI, OpNum: 1, V: A);
4124 }
4125 // select c, true, (a && b) -> select c, true, a
4126 // select (a && b), true, c -> select a, true, c
4127 // if c = false implies that b = true
4128 if (match(V: TrueVal, P: m_One()) &&
4129 match(V: FalseVal, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B)))) {
4130 std::optional<bool> Res = isImpliedCondition(LHS: CondVal, RHS: B, DL, LHSIsTrue: false);
4131 if (Res && *Res == true)
4132 return replaceOperand(I&: SI, OpNum: 2, V: A);
4133 }
4134 if (match(V: CondVal, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4135 match(V: TrueVal, P: m_One())) {
4136 std::optional<bool> Res = isImpliedCondition(LHS: FalseVal, RHS: B, DL, LHSIsTrue: false);
4137 if (Res && *Res == true)
4138 return replaceOperand(I&: SI, OpNum: 0, V: A);
4139 }
4140
4141 if (match(V: TrueVal, P: m_One())) {
4142 // (C && A) || (!C && B) --> select C, A, B (and similar cases)
4143 if (auto *V = FoldOrOfLogicalAnds(Op0: CondVal, Op1: FalseVal)) {
4144 return V;
4145 }
4146 }
4147
4148 return nullptr;
4149}
4150
4151// Return true if we can safely remove the select instruction for std::bit_ceil
4152// pattern.
4153static bool isSafeToRemoveBitCeilSelect(ICmpInst::Predicate Pred, Value *Cond0,
4154 const APInt *Cond1, Value *CtlzOp,
4155 unsigned BitWidth,
4156 bool &ShouldDropNoWrap) {
4157 // The challenge in recognizing std::bit_ceil(X) is that the operand is used
4158 // for the CTLZ proper and select condition, each possibly with some
4159 // operation like add and sub.
4160 //
4161 // Our aim is to make sure that -ctlz & (BitWidth - 1) == 0 even when the
4162 // select instruction would select 1, which allows us to get rid of the select
4163 // instruction.
4164 //
4165 // To see if we can do so, we do some symbolic execution with ConstantRange.
4166 // Specifically, we compute the range of values that Cond0 could take when
4167 // Cond == false. Then we successively transform the range until we obtain
4168 // the range of values that CtlzOp could take.
4169 //
4170 // Conceptually, we follow the def-use chain backward from Cond0 while
4171 // transforming the range for Cond0 until we meet the common ancestor of Cond0
4172 // and CtlzOp. Then we follow the def-use chain forward until we obtain the
4173 // range for CtlzOp. That said, we only follow at most one ancestor from
4174 // Cond0. Likewise, we only follow at most one ancestor from CtrlOp.
4175
4176 ConstantRange CR = ConstantRange::makeExactICmpRegion(
4177 Pred: CmpInst::getInversePredicate(pred: Pred), Other: *Cond1);
4178
4179 ShouldDropNoWrap = false;
4180
4181 // Match the operation that's used to compute CtlzOp from CommonAncestor. If
4182 // CtlzOp == CommonAncestor, return true as no operation is needed. If a
4183 // match is found, execute the operation on CR, update CR, and return true.
4184 // Otherwise, return false.
4185 auto MatchForward = [&](Value *CommonAncestor) {
4186 const APInt *C = nullptr;
4187 if (CtlzOp == CommonAncestor)
4188 return true;
4189 if (match(V: CtlzOp, P: m_Add(L: m_Specific(V: CommonAncestor), R: m_APInt(Res&: C)))) {
4190 ShouldDropNoWrap = true;
4191 CR = CR.add(Other: *C);
4192 return true;
4193 }
4194 if (match(V: CtlzOp, P: m_Sub(L: m_APInt(Res&: C), R: m_Specific(V: CommonAncestor)))) {
4195 ShouldDropNoWrap = true;
4196 CR = ConstantRange(*C).sub(Other: CR);
4197 return true;
4198 }
4199 if (match(V: CtlzOp, P: m_Not(V: m_Specific(V: CommonAncestor)))) {
4200 CR = CR.binaryNot();
4201 return true;
4202 }
4203 return false;
4204 };
4205
4206 const APInt *C = nullptr;
4207 Value *CommonAncestor;
4208 if (MatchForward(Cond0)) {
4209 // Cond0 is either CtlzOp or CtlzOp's parent. CR has been updated.
4210 } else if (match(V: Cond0, P: m_Add(L: m_Value(V&: CommonAncestor), R: m_APInt(Res&: C)))) {
4211 CR = CR.sub(Other: *C);
4212 if (!MatchForward(CommonAncestor))
4213 return false;
4214 // Cond0's parent is either CtlzOp or CtlzOp's parent. CR has been updated.
4215 } else {
4216 return false;
4217 }
4218
4219 // Return true if all the values in the range are either 0 or negative (if
4220 // treated as signed). We do so by evaluating:
4221 //
4222 // CR - 1 u>= (1 << BitWidth) - 1.
4223 APInt IntMax = APInt::getSignMask(BitWidth) - 1;
4224 CR = CR.sub(Other: APInt(BitWidth, 1));
4225 return CR.icmp(Pred: ICmpInst::ICMP_UGE, Other: IntMax);
4226}
4227
4228// Transform the std::bit_ceil(X) pattern like:
4229//
4230// %dec = add i32 %x, -1
4231// %ctlz = tail call i32 @llvm.ctlz.i32(i32 %dec, i1 false)
4232// %sub = sub i32 32, %ctlz
4233// %shl = shl i32 1, %sub
4234// %ugt = icmp ugt i32 %x, 1
4235// %sel = select i1 %ugt, i32 %shl, i32 1
4236//
4237// into:
4238//
4239// %dec = add i32 %x, -1
4240// %ctlz = tail call i32 @llvm.ctlz.i32(i32 %dec, i1 false)
4241// %neg = sub i32 0, %ctlz
4242// %masked = and i32 %ctlz, 31
4243// %shl = shl i32 1, %sub
4244//
4245// Note that the select is optimized away while the shift count is masked with
4246// 31. We handle some variations of the input operand like std::bit_ceil(X +
4247// 1).
4248static Instruction *foldBitCeil(SelectInst &SI, IRBuilderBase &Builder,
4249 InstCombinerImpl &IC) {
4250 Type *SelType = SI.getType();
4251 unsigned BitWidth = SelType->getScalarSizeInBits();
4252 if (!isPowerOf2_32(Value: BitWidth))
4253 return nullptr;
4254
4255 Value *FalseVal = SI.getFalseValue();
4256 Value *TrueVal = SI.getTrueValue();
4257 CmpPredicate Pred;
4258 const APInt *Cond1;
4259 Value *Cond0, *Ctlz, *CtlzOp;
4260 if (!match(V: SI.getCondition(), P: m_ICmp(Pred, L: m_Value(V&: Cond0), R: m_APInt(Res&: Cond1))))
4261 return nullptr;
4262
4263 if (match(V: TrueVal, P: m_One())) {
4264 std::swap(a&: FalseVal, b&: TrueVal);
4265 Pred = CmpInst::getInversePredicate(pred: Pred);
4266 }
4267
4268 bool ShouldDropNoWrap;
4269
4270 if (!match(V: FalseVal, P: m_One()) ||
4271 !match(V: TrueVal,
4272 P: m_OneUse(SubPattern: m_Shl(L: m_One(), R: m_OneUse(SubPattern: m_Sub(L: m_SpecificInt(V: BitWidth),
4273 R: m_Value(V&: Ctlz)))))) ||
4274 !match(V: Ctlz, P: m_Ctlz(Op0: m_Value(V&: CtlzOp), Op1: m_Value())) ||
4275 !isSafeToRemoveBitCeilSelect(Pred, Cond0, Cond1, CtlzOp, BitWidth,
4276 ShouldDropNoWrap))
4277 return nullptr;
4278
4279 if (ShouldDropNoWrap) {
4280 cast<Instruction>(Val: CtlzOp)->setHasNoUnsignedWrap(false);
4281 cast<Instruction>(Val: CtlzOp)->setHasNoSignedWrap(false);
4282 }
4283
4284 // Build 1 << (-CTLZ & (BitWidth-1)). The negation likely corresponds to a
4285 // single hardware instruction as opposed to BitWidth - CTLZ, where BitWidth
4286 // is an integer constant. Masking with BitWidth-1 comes free on some
4287 // hardware as part of the shift instruction.
4288
4289 // Drop range attributes and re-infer them in the next iteration.
4290 cast<Instruction>(Val: Ctlz)->dropPoisonGeneratingAnnotations();
4291 IC.addToWorklist(I: cast<Instruction>(Val: Ctlz));
4292 Value *Neg = Builder.CreateNeg(V: Ctlz);
4293 Value *Masked =
4294 Builder.CreateAnd(LHS: Neg, RHS: ConstantInt::get(Ty: SelType, V: BitWidth - 1));
4295 return BinaryOperator::Create(Op: Instruction::Shl, S1: ConstantInt::get(Ty: SelType, V: 1),
4296 S2: Masked);
4297}
4298
4299// This function tries to fold the following operations:
4300// (x < y) ? -1 : zext(x != y)
4301// (x < y) ? -1 : zext(x > y)
4302// (x > y) ? 1 : sext(x != y)
4303// (x > y) ? 1 : sext(x < y)
4304// (x == y) ? 0 : (x > y ? 1 : -1)
4305// (x == y) ? 0 : (x < y ? -1 : 1)
4306// Special case: x == C ? 0 : (x > C - 1 ? 1 : -1)
4307// Special case: x == C ? 0 : (x < C + 1 ? -1 : 1)
4308// Into ucmp/scmp(x, y), where signedness is determined by the signedness
4309// of the comparison in the original sequence.
4310Instruction *InstCombinerImpl::foldSelectToCmp(SelectInst &SI) {
4311 Value *TV = SI.getTrueValue();
4312 Value *FV = SI.getFalseValue();
4313
4314 CmpPredicate Pred;
4315 Value *LHS, *RHS;
4316 if (!match(V: SI.getCondition(), P: m_ICmp(Pred, L: m_Value(V&: LHS), R: m_Value(V&: RHS))))
4317 return nullptr;
4318
4319 if (!LHS->getType()->isIntOrIntVectorTy())
4320 return nullptr;
4321
4322 // If there is no -1, 0 or 1 at TV, then invert the select statement and try
4323 // to canonicalize to one of the forms above
4324 if (!isa<Constant>(Val: TV)) {
4325 if (!isa<Constant>(Val: FV))
4326 return nullptr;
4327 Pred = ICmpInst::getInverseCmpPredicate(Pred);
4328 std::swap(a&: TV, b&: FV);
4329 }
4330
4331 if (ICmpInst::isNonStrictPredicate(predicate: Pred)) {
4332 if (Constant *C = dyn_cast<Constant>(Val: RHS)) {
4333 auto FlippedPredAndConst =
4334 getFlippedStrictnessPredicateAndConstant(Pred, C);
4335 if (!FlippedPredAndConst)
4336 return nullptr;
4337 Pred = FlippedPredAndConst->first;
4338 RHS = FlippedPredAndConst->second;
4339 } else {
4340 return nullptr;
4341 }
4342 }
4343
4344 // Try to swap operands and the predicate. We need to be careful when doing
4345 // so because two of the patterns have opposite predicates, so use the
4346 // constant inside select to determine if swapping operands would be
4347 // beneficial to us.
4348 if ((ICmpInst::isGT(P: Pred) && match(V: TV, P: m_AllOnes())) ||
4349 (ICmpInst::isLT(P: Pred) && match(V: TV, P: m_One()))) {
4350 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
4351 std::swap(a&: LHS, b&: RHS);
4352 }
4353 bool IsSigned = ICmpInst::isSigned(Pred);
4354
4355 bool Replace = false;
4356 CmpPredicate ExtendedCmpPredicate;
4357 // (x < y) ? -1 : zext(x != y)
4358 // (x < y) ? -1 : zext(x > y)
4359 if (ICmpInst::isLT(P: Pred) && match(V: TV, P: m_AllOnes()) &&
4360 match(V: FV, P: m_ZExt(Op: m_c_ICmp(Pred&: ExtendedCmpPredicate, L: m_Specific(V: LHS),
4361 R: m_Specific(V: RHS)))) &&
4362 (ExtendedCmpPredicate == ICmpInst::ICMP_NE ||
4363 ICmpInst::getSwappedPredicate(pred: ExtendedCmpPredicate) == Pred))
4364 Replace = true;
4365
4366 // (x > y) ? 1 : sext(x != y)
4367 // (x > y) ? 1 : sext(x < y)
4368 if (ICmpInst::isGT(P: Pred) && match(V: TV, P: m_One()) &&
4369 match(V: FV, P: m_SExt(Op: m_c_ICmp(Pred&: ExtendedCmpPredicate, L: m_Specific(V: LHS),
4370 R: m_Specific(V: RHS)))) &&
4371 (ExtendedCmpPredicate == ICmpInst::ICMP_NE ||
4372 ICmpInst::getSwappedPredicate(pred: ExtendedCmpPredicate) == Pred))
4373 Replace = true;
4374
4375 // (x == y) ? 0 : (x > y ? 1 : -1)
4376 CmpPredicate FalseBranchSelectPredicate;
4377 const APInt *InnerTV, *InnerFV;
4378 if (Pred == ICmpInst::ICMP_EQ && match(V: TV, P: m_Zero()) &&
4379 match(V: FV, P: m_Select(C: m_c_ICmp(Pred&: FalseBranchSelectPredicate, L: m_Specific(V: LHS),
4380 R: m_Specific(V: RHS)),
4381 L: m_APInt(Res&: InnerTV), R: m_APInt(Res&: InnerFV)))) {
4382 if (!ICmpInst::isGT(P: FalseBranchSelectPredicate)) {
4383 FalseBranchSelectPredicate =
4384 ICmpInst::getSwappedPredicate(pred: FalseBranchSelectPredicate);
4385 std::swap(a&: LHS, b&: RHS);
4386 }
4387
4388 if (!InnerTV->isOne()) {
4389 std::swap(a&: InnerTV, b&: InnerFV);
4390 std::swap(a&: LHS, b&: RHS);
4391 }
4392
4393 if (ICmpInst::isGT(P: FalseBranchSelectPredicate) && InnerTV->isOne() &&
4394 InnerFV->isAllOnes()) {
4395 IsSigned = ICmpInst::isSigned(Pred: FalseBranchSelectPredicate);
4396 Replace = true;
4397 }
4398 }
4399
4400 // Special cases with constants: x == C ? 0 : (x > C-1 ? 1 : -1)
4401 if (Pred == ICmpInst::ICMP_EQ && match(V: TV, P: m_Zero())) {
4402 const APInt *C;
4403 if (match(V: RHS, P: m_APInt(Res&: C))) {
4404 CmpPredicate InnerPred;
4405 Value *InnerRHS;
4406 const APInt *InnerTV, *InnerFV;
4407 if (match(V: FV,
4408 P: m_Select(C: m_ICmp(Pred&: InnerPred, L: m_Specific(V: LHS), R: m_Value(V&: InnerRHS)),
4409 L: m_APInt(Res&: InnerTV), R: m_APInt(Res&: InnerFV)))) {
4410
4411 // x == C ? 0 : (x > C-1 ? 1 : -1)
4412 if (ICmpInst::isGT(P: InnerPred) && InnerTV->isOne() &&
4413 InnerFV->isAllOnes()) {
4414 IsSigned = ICmpInst::isSigned(Pred: InnerPred);
4415 bool CanSubOne = IsSigned ? !C->isMinSignedValue() : !C->isMinValue();
4416 if (CanSubOne) {
4417 APInt Cminus1 = *C - 1;
4418 if (match(V: InnerRHS, P: m_SpecificInt(V: Cminus1)))
4419 Replace = true;
4420 }
4421 }
4422
4423 // x == C ? 0 : (x < C+1 ? -1 : 1)
4424 if (ICmpInst::isLT(P: InnerPred) && InnerTV->isAllOnes() &&
4425 InnerFV->isOne()) {
4426 IsSigned = ICmpInst::isSigned(Pred: InnerPred);
4427 bool CanAddOne = IsSigned ? !C->isMaxSignedValue() : !C->isMaxValue();
4428 if (CanAddOne) {
4429 APInt Cplus1 = *C + 1;
4430 if (match(V: InnerRHS, P: m_SpecificInt(V: Cplus1)))
4431 Replace = true;
4432 }
4433 }
4434 }
4435 }
4436 }
4437
4438 Intrinsic::ID IID = IsSigned ? Intrinsic::scmp : Intrinsic::ucmp;
4439 if (Replace)
4440 return replaceInstUsesWith(
4441 I&: SI, V: Builder.CreateIntrinsic(RetTy: SI.getType(), ID: IID, Args: {LHS, RHS}));
4442 return nullptr;
4443}
4444
4445bool InstCombinerImpl::fmulByZeroIsZero(Value *MulVal, FastMathFlags FMF,
4446 const Instruction *CtxI) const {
4447 KnownFPClass Known =
4448 computeKnownFPClass(V: MulVal, FMF, InterestedClasses: fcNegative, SQ: SQ.getWithInstruction(I: CtxI));
4449
4450 return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity() &&
4451 (FMF.noSignedZeros() || Known.signBitIsZeroOrNaN());
4452}
4453
4454static bool matchFMulByZeroIfResultEqZero(InstCombinerImpl &IC, Value *Cmp0,
4455 Value *Cmp1, Value *TrueVal,
4456 Value *FalseVal, Instruction &CtxI,
4457 bool SelectIsNSZ) {
4458 Value *MulRHS;
4459 if (match(V: Cmp1, P: m_PosZeroFP()) &&
4460 match(V: TrueVal, P: m_c_FMul(L: m_Specific(V: Cmp0), R: m_Value(V&: MulRHS)))) {
4461 FastMathFlags FMF = cast<FPMathOperator>(Val: TrueVal)->getFastMathFlags();
4462 // nsz must be on the select, it must be ignored on the multiply. We
4463 // need nnan and ninf on the multiply for the other value.
4464 FMF.setNoSignedZeros(SelectIsNSZ);
4465 return IC.fmulByZeroIsZero(MulVal: MulRHS, FMF, CtxI: &CtxI);
4466 }
4467
4468 return false;
4469}
4470
4471/// Check whether the KnownBits of a select arm may be affected by the
4472/// select condition.
4473static bool hasAffectedValue(Value *V, SmallPtrSetImpl<Value *> &Affected,
4474 unsigned Depth) {
4475 if (Depth == MaxAnalysisRecursionDepth)
4476 return false;
4477
4478 // Ignore the case where the select arm itself is affected. These cases
4479 // are handled more efficiently by other optimizations.
4480 if (Depth != 0 && Affected.contains(Ptr: V))
4481 return true;
4482
4483 if (auto *I = dyn_cast<Instruction>(Val: V)) {
4484 if (isa<PHINode>(Val: I)) {
4485 if (Depth == MaxAnalysisRecursionDepth - 1)
4486 return false;
4487 Depth = MaxAnalysisRecursionDepth - 2;
4488 }
4489 return any_of(Range: I->operands(), P: [&](Value *Op) {
4490 return Op->getType()->isIntOrIntVectorTy() &&
4491 hasAffectedValue(V: Op, Affected, Depth: Depth + 1);
4492 });
4493 }
4494
4495 return false;
4496}
4497
4498// This transformation enables the possibility of transforming fcmp + sel into
4499// a fmaxnum/fminnum intrinsic.
4500static Value *foldSelectIntoAddConstant(SelectInst &SI,
4501 InstCombiner::BuilderTy &Builder) {
4502 // Do this transformation only when select instruction gives NaN and NSZ
4503 // guarantee.
4504 auto *SIFOp = dyn_cast<FPMathOperator>(Val: &SI);
4505 if (!SIFOp || !SIFOp->hasNoSignedZeros() || !SIFOp->hasNoNaNs())
4506 return nullptr;
4507
4508 auto TryFoldIntoAddConstant =
4509 [&Builder, &SI](CmpInst::Predicate Pred, Value *X, Value *Z,
4510 Instruction *FAdd, Constant *C, bool Swapped) -> Value * {
4511 // Only these relational predicates can be transformed into maxnum/minnum
4512 // intrinsic.
4513 if (!CmpInst::isRelational(P: Pred) || !match(V: Z, P: m_AnyZeroFP()))
4514 return nullptr;
4515
4516 if (!match(V: FAdd, P: m_FAdd(L: m_Specific(V: X), R: m_Specific(V: C))))
4517 return nullptr;
4518
4519 Value *NewSelect = Builder.CreateSelect(C: SI.getCondition(), True: Swapped ? Z : X,
4520 False: Swapped ? X : Z, Name: "", MDFrom: &SI);
4521 NewSelect->takeName(V: &SI);
4522
4523 Value *NewFAdd = Builder.CreateFAdd(L: NewSelect, R: C);
4524 NewFAdd->takeName(V: FAdd);
4525
4526 // Propagate FastMath flags
4527 FastMathFlags SelectFMF = SI.getFastMathFlags();
4528 FastMathFlags FAddFMF = FAdd->getFastMathFlags();
4529 FastMathFlags NewFMF = FastMathFlags::intersectRewrite(LHS: SelectFMF, RHS: FAddFMF) |
4530 FastMathFlags::unionValue(LHS: SelectFMF, RHS: FAddFMF);
4531 cast<Instruction>(Val: NewFAdd)->setFastMathFlags(NewFMF);
4532 cast<Instruction>(Val: NewSelect)->setFastMathFlags(NewFMF);
4533
4534 return NewFAdd;
4535 };
4536
4537 // select((fcmp Pred, X, 0), (fadd X, C), C)
4538 // => fadd((select (fcmp Pred, X, 0), X, 0), C)
4539 //
4540 // Pred := OGT, OGE, OLT, OLE, UGT, UGE, ULT, and ULE
4541 Instruction *FAdd;
4542 Constant *C;
4543 Value *X, *Z;
4544 CmpPredicate Pred;
4545
4546 // Note: OneUse check for `Cmp` is necessary because it makes sure that other
4547 // InstCombine folds don't undo this transformation and cause an infinite
4548 // loop. Furthermore, it could also increase the operation count.
4549 if (match(V: &SI, P: m_Select(C: m_OneUse(SubPattern: m_FCmp(Pred, L: m_Value(V&: X), R: m_Value(V&: Z))),
4550 L: m_OneUse(SubPattern: m_Instruction(I&: FAdd)), R: m_Constant(C))))
4551 return TryFoldIntoAddConstant(Pred, X, Z, FAdd, C, /*Swapped=*/false);
4552
4553 if (match(V: &SI, P: m_Select(C: m_OneUse(SubPattern: m_FCmp(Pred, L: m_Value(V&: X), R: m_Value(V&: Z))),
4554 L: m_Constant(C), R: m_OneUse(SubPattern: m_Instruction(I&: FAdd)))))
4555 return TryFoldIntoAddConstant(Pred, X, Z, FAdd, C, /*Swapped=*/true);
4556
4557 return nullptr;
4558}
4559
4560static Value *foldSelectBitTest(SelectInst &Sel, Value *CondVal, Value *TrueVal,
4561 Value *FalseVal,
4562 InstCombiner::BuilderTy &Builder,
4563 const SimplifyQuery &SQ) {
4564 // If this is a vector select, we need a vector compare.
4565 Type *SelType = Sel.getType();
4566 if (SelType->isVectorTy() != CondVal->getType()->isVectorTy())
4567 return nullptr;
4568
4569 Value *V;
4570 APInt AndMask;
4571 bool CreateAnd = false;
4572 CmpPredicate Pred;
4573 Value *CmpLHS, *CmpRHS;
4574
4575 if (match(V: CondVal, P: m_ICmp(Pred, L: m_Value(V&: CmpLHS), R: m_Value(V&: CmpRHS)))) {
4576 if (ICmpInst::isEquality(P: Pred)) {
4577 if (!match(V: CmpRHS, P: m_Zero()))
4578 return nullptr;
4579
4580 V = CmpLHS;
4581 const APInt *AndRHS;
4582 if (!match(V: CmpLHS, P: m_And(L: m_Value(), R: m_Power2(V&: AndRHS))))
4583 return nullptr;
4584
4585 AndMask = *AndRHS;
4586 } else if (auto Res = decomposeBitTestICmp(LHS: CmpLHS, RHS: CmpRHS, Pred)) {
4587 assert(ICmpInst::isEquality(Res->Pred) && "Not equality test?");
4588 AndMask = Res->Mask;
4589 V = Res->X;
4590 KnownBits Known = computeKnownBits(V, Q: SQ.getWithInstruction(I: &Sel));
4591 AndMask &= Known.getMaxValue();
4592 if (!AndMask.isPowerOf2())
4593 return nullptr;
4594
4595 Pred = Res->Pred;
4596 CreateAnd = true;
4597 } else {
4598 return nullptr;
4599 }
4600 } else if (auto *Trunc = dyn_cast<TruncInst>(Val: CondVal)) {
4601 V = Trunc->getOperand(i_nocapture: 0);
4602 AndMask = APInt(V->getType()->getScalarSizeInBits(), 1);
4603 Pred = ICmpInst::ICMP_NE;
4604 CreateAnd = !Trunc->hasNoUnsignedWrap();
4605 } else {
4606 return nullptr;
4607 }
4608
4609 if (Pred == ICmpInst::ICMP_NE)
4610 std::swap(a&: TrueVal, b&: FalseVal);
4611
4612 if (Value *X = foldSelectICmpAnd(Sel, CondVal, TrueVal, FalseVal, V, AndMask,
4613 CreateAnd, Builder))
4614 return X;
4615
4616 if (Value *X = foldSelectICmpAndBinOp(CondVal, TrueVal, FalseVal, V, AndMask,
4617 CreateAnd, Builder))
4618 return X;
4619
4620 return nullptr;
4621}
4622
4623/// This function makes the following folds:
4624/// select C, (sub 0, X), (xor X, -1)
4625/// -> sub (sext !C), X
4626/// select C, (xor X, -1), (sub 0, X)
4627/// -> sub (sext C), X
4628static Instruction *foldSelectNegNot(SelectInst &SI,
4629 InstCombiner::BuilderTy &Builder) {
4630 auto *CondVal = SI.getCondition();
4631 auto *TrueVal = SI.getTrueValue();
4632 auto *FalseVal = SI.getFalseValue();
4633 auto *SelTy = SI.getType();
4634
4635 if (!SelTy->isIntOrIntVectorTy() || SelTy->isIntOrIntVectorTy(BitWidth: 1))
4636 return nullptr;
4637
4638 if (CondVal->getType()->isVectorTy() != SelTy->isVectorTy())
4639 return nullptr;
4640
4641 auto matchNegNot = [&](Value *Neg, Value *Not, Value *&X) -> bool {
4642 return match(V: Neg, P: m_OneUse(SubPattern: m_Neg(V: m_Value(V&: X)))) &&
4643 match(V: Not, P: m_OneUse(SubPattern: m_Not(V: m_Specific(V: X))));
4644 };
4645
4646 Value *X;
4647 Value *Mask;
4648
4649 // select C, (sub 0, X), (xor X, -1) -> sub (sext !C), X
4650 if (matchNegNot(TrueVal, FalseVal, X)) {
4651 Value *NotCond = Builder.CreateNot(V: CondVal, Name: "not." + CondVal->getName());
4652 Mask = Builder.CreateSExt(V: NotCond, DestTy: SelTy);
4653 return BinaryOperator::CreateSub(V1: Mask, V2: X);
4654 }
4655
4656 // select C, (xor X, -1), (sub 0, X) -> sub (sext C), X
4657 if (matchNegNot(FalseVal, TrueVal, X)) {
4658 Mask = Builder.CreateSExt(V: CondVal, DestTy: SelTy);
4659 return BinaryOperator::CreateSub(V1: Mask, V2: X);
4660 }
4661
4662 return nullptr;
4663}
4664
4665/// Fold select (A & Shift == 0 | B & Shift == 0), 0, Shift -> Shift & A & B
4666/// where Shift is known to be a power of two.
4667static Instruction *foldSelectAndOrPowerOfTwo(SelectInst &SI,
4668 InstCombiner::BuilderTy &Builder,
4669 const SimplifyQuery &SQ) {
4670 Value *Cond = SI.getCondition();
4671
4672 if (!Cond->hasOneUse())
4673 return nullptr;
4674
4675 Value *TrueVal = SI.getTrueValue();
4676 Value *FalseVal = SI.getFalseValue();
4677
4678 Value *A, *B, *Shift;
4679
4680 bool Case1 =
4681 match(V: TrueVal, P: m_Zero()) && match(V: FalseVal, P: m_Value(V&: Shift)) &&
4682 match(V: Cond, P: m_Or(L: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
4683 L: m_c_And(L: m_Specific(V: Shift), R: m_Value(V&: A)),
4684 R: m_Zero()),
4685 R: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
4686 L: m_c_And(L: m_Specific(V: Shift), R: m_Value(V&: B)),
4687 R: m_Zero())));
4688
4689 bool Case2 =
4690 match(V: FalseVal, P: m_Zero()) && match(V: TrueVal, P: m_Value(V&: Shift)) &&
4691 match(V: Cond, P: m_And(L: m_SpecificICmp(MatchPred: ICmpInst::ICMP_NE,
4692 L: m_c_And(L: m_Specific(V: Shift), R: m_Value(V&: A)),
4693 R: m_Zero()),
4694 R: m_SpecificICmp(MatchPred: ICmpInst::ICMP_NE,
4695 L: m_c_And(L: m_Specific(V: Shift), R: m_Value(V&: B)),
4696 R: m_Zero())));
4697
4698 if ((Case1 || Case2) && isKnownToBeAPowerOfTwo(V: Shift, /*OrZero=*/true,
4699 Q: SQ.getWithInstruction(I: &SI))) {
4700 Value *And1 = Builder.CreateAnd(LHS: Shift, RHS: A);
4701 return BinaryOperator::CreateAnd(V1: And1, V2: B);
4702 }
4703
4704 return nullptr;
4705}
4706
4707// Return true if no use can observe the sign of zero of the select result,
4708// looking through phis, selects and the loop back edge to the select itself.
4709static bool isSelectZeroSignInsignificant(SelectInst &SI) {
4710 // Bound the number of uses to look through to keep the compile time in
4711 // check.
4712 constexpr unsigned MaxUsesToLookThrough = 16;
4713 unsigned NumUses = 0;
4714 SmallPtrSet<Instruction *, 4> Visited;
4715 SmallVector<Instruction *> Worklist(1, &SI);
4716 while (!Worklist.empty()) {
4717 for (Use &U : Worklist.pop_back_val()->uses()) {
4718 if (++NumUses > MaxUsesToLookThrough)
4719 return false;
4720 auto *User = cast<Instruction>(Val: U.getUser());
4721 if (User == &SI)
4722 continue;
4723 if (canIgnoreSignBitOfZero(U))
4724 continue;
4725 if (isa<PHINode, SelectInst>(Val: User)) {
4726 if (Visited.insert(Ptr: User).second)
4727 Worklist.push_back(Elt: User);
4728 continue;
4729 }
4730 return false;
4731 }
4732 }
4733 return true;
4734}
4735
4736Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
4737 Value *CondVal = SI.getCondition();
4738 Value *TrueVal = SI.getTrueValue();
4739 Value *FalseVal = SI.getFalseValue();
4740 Type *SelType = SI.getType();
4741
4742 FastMathFlags FMF;
4743 if (auto *FPMO = dyn_cast_if_present<FPMathOperator>(Val: &SI))
4744 FMF = FPMO->getFastMathFlags();
4745
4746 if (Value *V = simplifySelectInst(Cond: CondVal, TrueVal, FalseVal, FMF,
4747 Q: SQ.getWithInstruction(I: &SI)))
4748 return replaceInstUsesWith(I&: SI, V);
4749
4750 if (Instruction *I = canonicalizeSelectToShuffle(SI))
4751 return I;
4752
4753 if (Instruction *I = canonicalizeScalarSelectOfVecs(Sel&: SI, IC&: *this))
4754 return I;
4755
4756 // Fold: select (icmp ult X, 2), X, ctpop(X) --> ctpop(X)
4757 // ctpop(0)==0 and ctpop(1)==1, so the guard is always redundant.
4758 if (match(V: FalseVal, P: m_Ctpop(Op0: m_Specific(V: TrueVal))) &&
4759 match(V: CondVal, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_ULT, L: m_Specific(V: TrueVal),
4760 R: m_SpecificInt(V: 2)))) {
4761 cast<Instruction>(Val: FalseVal)->dropPoisonGeneratingAnnotations();
4762 addToWorklist(I: cast<Instruction>(Val: FalseVal));
4763 return replaceInstUsesWith(I&: SI, V: FalseVal);
4764 }
4765
4766 // If the condition is a constant or if the condition and the selection
4767 // type are not both scalar nor both vector types, there is no point in
4768 // attempting to match these patterns.
4769 Type *CondType = CondVal->getType();
4770 if (!isa<Constant>(Val: CondVal) &&
4771 CondType->isVectorTy() == SelType->isVectorTy()) {
4772 if (Value *S = simplifyWithOpReplaced(V: TrueVal, Op: CondVal,
4773 RepOp: ConstantInt::getTrue(Ty: CondType), Q: SQ,
4774 /* AllowRefinement */ true))
4775 return replaceOperand(I&: SI, OpNum: 1, V: S);
4776
4777 if (Value *S = simplifyWithOpReplaced(V: FalseVal, Op: CondVal,
4778 RepOp: ConstantInt::getFalse(Ty: CondType), Q: SQ,
4779 /* AllowRefinement */ true))
4780 return replaceOperand(I&: SI, OpNum: 2, V: S);
4781
4782 if (replaceInInstruction(V: TrueVal, Old: CondVal,
4783 New: ConstantInt::getTrue(Ty: CondType)) ||
4784 replaceInInstruction(V: FalseVal, Old: CondVal,
4785 New: ConstantInt::getFalse(Ty: CondType)))
4786 return &SI;
4787 }
4788
4789 if (Instruction *R = foldSelectOfBools(SI))
4790 return R;
4791
4792 // Selecting between two integer or vector splat integer constants?
4793 //
4794 // Note that we don't handle a scalar select of vectors:
4795 // select i1 %c, <2 x i8> <1, 1>, <2 x i8> <0, 0>
4796 // because that may need 3 instructions to splat the condition value:
4797 // extend, insertelement, shufflevector.
4798 //
4799 // Do not handle i1 TrueVal and FalseVal otherwise would result in
4800 // zext/sext i1 to i1.
4801 if (SelType->isIntOrIntVectorTy() && !SelType->isIntOrIntVectorTy(BitWidth: 1) &&
4802 CondVal->getType()->isVectorTy() == SelType->isVectorTy()) {
4803 // select C, 1, 0 -> zext C to int
4804 if (match(V: TrueVal, P: m_One()) && match(V: FalseVal, P: m_Zero()))
4805 return new ZExtInst(CondVal, SelType);
4806
4807 // select C, -1, 0 -> sext C to int
4808 if (match(V: TrueVal, P: m_AllOnes()) && match(V: FalseVal, P: m_Zero()))
4809 return new SExtInst(CondVal, SelType);
4810
4811 // select C, 0, 1 -> zext !C to int
4812 if (match(V: TrueVal, P: m_Zero()) && match(V: FalseVal, P: m_One())) {
4813 Value *NotCond = Builder.CreateNot(V: CondVal, Name: "not." + CondVal->getName());
4814 return new ZExtInst(NotCond, SelType);
4815 }
4816
4817 // select C, 0, -1 -> sext !C to int
4818 if (match(V: TrueVal, P: m_Zero()) && match(V: FalseVal, P: m_AllOnes())) {
4819 Value *NotCond = Builder.CreateNot(V: CondVal, Name: "not." + CondVal->getName());
4820 return new SExtInst(NotCond, SelType);
4821 }
4822 }
4823
4824 if (Instruction *I = foldSelectNegNot(SI, Builder))
4825 return I;
4826
4827 if (Instruction *I = foldSelectAndOrPowerOfTwo(SI, Builder, SQ))
4828 return I;
4829
4830 auto *SIFPOp = dyn_cast<FPMathOperator>(Val: &SI);
4831
4832 if (auto *FCmp = dyn_cast<FCmpInst>(Val: CondVal)) {
4833 FCmpInst::Predicate Pred = FCmp->getPredicate();
4834 Value *Cmp0 = FCmp->getOperand(i_nocapture: 0), *Cmp1 = FCmp->getOperand(i_nocapture: 1);
4835 // Are we selecting a value based on a comparison of the two values?
4836 if ((Cmp0 == TrueVal && Cmp1 == FalseVal) ||
4837 (Cmp0 == FalseVal && Cmp1 == TrueVal)) {
4838 // Canonicalize to use ordered comparisons by swapping the select
4839 // operands.
4840 //
4841 // e.g.
4842 // (X ugt Y) ? X : Y -> (X ole Y) ? Y : X
4843 if (FCmp->hasOneUse() && FCmpInst::isUnordered(predicate: Pred)) {
4844 FCmpInst::Predicate InvPred = FCmp->getInversePredicate();
4845 Value *NewCond = Builder.CreateFCmpFMF(P: InvPred, LHS: Cmp0, RHS: Cmp1, FMFSource: FCmp,
4846 Name: FCmp->getName() + ".inv");
4847 // Propagate ninf/nnan from fcmp to select.
4848 FastMathFlags FMF = SI.getFastMathFlags();
4849 if (FCmp->hasNoNaNs())
4850 FMF.setNoNaNs(true);
4851 if (FCmp->hasNoInfs())
4852 FMF.setNoInfs(true);
4853 Value *NewSel = Builder.CreateSelectFMF(
4854 C: NewCond, True: FalseVal, False: TrueVal, FMFSource: FMF, Name: "",
4855 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &SI);
4856 if (auto *NewSI = dyn_cast<SelectInst>(Val: NewSel))
4857 NewSI->swapProfMetadata();
4858 return replaceInstUsesWith(I&: SI, V: NewSel);
4859 }
4860 }
4861
4862 if (SIFPOp) {
4863 // Fold out scale-if-equals-zero pattern.
4864 //
4865 // This pattern appears in code with denormal range checks after it's
4866 // assumed denormals are treated as zero. This drops a canonicalization.
4867
4868 // TODO: Could relax the signed zero logic. We just need to know the sign
4869 // of the result matches (fmul x, y has the same sign as x).
4870 //
4871 // TODO: Handle always-canonicalizing variant that selects some value or 1
4872 // scaling factor in the fmul visitor.
4873
4874 // TODO: Handle ldexp too
4875
4876 Value *MatchCmp0 = nullptr;
4877 Value *MatchCmp1 = nullptr;
4878
4879 // (select (fcmp [ou]eq x, 0.0), (fmul x, K), x => x
4880 // (select (fcmp [ou]ne x, 0.0), x, (fmul x, K) => x
4881 if (Pred == CmpInst::FCMP_OEQ || Pred == CmpInst::FCMP_UEQ) {
4882 MatchCmp0 = FalseVal;
4883 MatchCmp1 = TrueVal;
4884 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UNE) {
4885 MatchCmp0 = TrueVal;
4886 MatchCmp1 = FalseVal;
4887 }
4888
4889 if (Cmp0 == MatchCmp0 &&
4890 matchFMulByZeroIfResultEqZero(IC&: *this, Cmp0, Cmp1, TrueVal: MatchCmp1, FalseVal: MatchCmp0,
4891 CtxI&: SI, SelectIsNSZ: SIFPOp->hasNoSignedZeros()))
4892 return replaceInstUsesWith(I&: SI, V: Cmp0);
4893
4894 Type *EltTy = SelType->getScalarType();
4895
4896 // TODO: Generalize to any ordered / unordered compare.
4897 if ((Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) &&
4898 match(V: Cmp1, P: m_PosZeroFP()) && EltTy->isIEEELikeFPTy()) {
4899 // Fold out only-canonicalize-non-nans pattern. This implements a
4900 // wrapper around llvm.canonicalize which is not required to quiet
4901 // signaling nans or preserve nan payload bits.
4902 //
4903 // %hard.canonical = call @llvm.canonicalize(%x)
4904 // %soft.canonical = fdiv 1.0, %x
4905 // %ord = fcmp ord %x, 0.0
4906 // %x.canon = select i1 %ord, %hard.canonical, %soft.canonical
4907 //
4908 // With known IEEE handling:
4909 // => %x
4910 //
4911 // With other denormal behaviors:
4912 // => llvm.canonicalize(%x)
4913 //
4914 // Note the fdiv could be any value preserving, potentially
4915 // canonicalizing floating-point operation such as fmul by 1.0. However,
4916 // since in the llvm model canonicalization is not mandatory, the fmul
4917 // would have been dropped by the time we reached here. The trick here
4918 // is to use a reciprocal fdiv. It's not a droppable no-op, as it could
4919 // return an infinity if %x were sufficiently small, but in this pattern
4920 // we're only using the output for nan values.
4921
4922 if (Pred == CmpInst::FCMP_ORD) {
4923 MatchCmp0 = TrueVal;
4924 MatchCmp1 = FalseVal;
4925 } else {
4926 MatchCmp0 = FalseVal;
4927 MatchCmp1 = TrueVal;
4928 }
4929
4930 bool RcpIfNan = match(V: MatchCmp1, P: m_FDiv(L: m_FPOne(), R: m_Specific(V: Cmp0)));
4931 bool CanonicalizeIfNotNan =
4932 match(V: MatchCmp0, P: m_FCanonicalize(Op0: m_Specific(V: Cmp0)));
4933
4934 if (RcpIfNan || CanonicalizeIfNotNan) {
4935 const fltSemantics &FPSem = EltTy->getFltSemantics();
4936 DenormalMode Mode = F.getDenormalMode(FPType: FPSem);
4937
4938 if (RcpIfNan) {
4939 if (Mode == DenormalMode::getIEEE()) {
4940 // Special case for the other select operand. Otherwise, we may
4941 // need to insert freeze on Cmp0 in the compare and select.
4942 if (CanonicalizeIfNotNan)
4943 return replaceInstUsesWith(I&: SI, V: Cmp0);
4944
4945 if (isGuaranteedNotToBeUndef(V: Cmp0, AC: &AC, CtxI: &SI, DT: &DT)) {
4946 // select (fcmp ord x, 0), y, (fdiv 1, x)
4947 // => select (fcmp ord x, 0), y, x
4948 //
4949 // select (fcmp uno x, 0), (fdiv 1, x), y
4950 // => select (fcmp uno x, 0), x, y
4951 replaceOperand(I&: SI, OpNum: Pred == CmpInst::FCMP_ORD ? 2 : 1, V: Cmp0);
4952 return &SI;
4953 }
4954
4955 auto *FrCmp0 = InsertNewInstBefore(
4956 New: new FreezeInst(Cmp0, Cmp0->getName() + ".fr"),
4957 Old: FCmp->getIterator());
4958
4959 replaceOperand(I&: *FCmp, OpNum: 0, V: FrCmp0);
4960 return replaceOperand(I&: SI, OpNum: Pred == CmpInst::FCMP_ORD ? 2 : 1,
4961 V: FrCmp0);
4962 }
4963 }
4964
4965 if (CanonicalizeIfNotNan) {
4966 // IEEE handling does not have non-canonical values, so the
4967 // canonicalize can be dropped for direct replacement without
4968 // looking for the intermediate maybe-canonicalizing operation.
4969 if (Mode == DenormalMode::getIEEE()) {
4970 // select (fcmp ord x, 0), canonicalize(x), y
4971 // => select (fcmp ord x, 0), x, y
4972
4973 replaceOperand(I&: SI, OpNum: Pred == CmpInst::FCMP_ORD ? 1 : 2, V: Cmp0);
4974 return &SI;
4975 }
4976
4977 // If denormals may be flushed, we need to retain the canonicalize
4978 // call. This introduces a canonicalization on the nan path, which
4979 // we are not free to do as that could change the sign bit or
4980 // payload bits. We can only do this if there were a no-op like
4981 // floating-point instruction which may have changed the nan bits
4982 // anyway.
4983
4984 // Leave the dynamic mode case alone. This would introduce new
4985 // constraints if the mode may be refined later.
4986 if (RcpIfNan && (Mode.inputsAreZero() || Mode.outputsAreZero()))
4987 return replaceInstUsesWith(I&: SI, V: MatchCmp0);
4988 assert(RcpIfNan || Mode != DenormalMode::getIEEE());
4989 }
4990 }
4991 }
4992 }
4993 }
4994
4995 if (SIFPOp) {
4996 // TODO: Try to forward-propagate FMF from select arms to the select.
4997
4998 auto *FCmp = dyn_cast<FCmpInst>(Val: CondVal);
4999
5000 // Canonicalize select of FP values where NaN and -0.0 are not valid as
5001 // minnum/maxnum intrinsics.
5002 //
5003 // Note that the `nnan` flag is propagated from the comparison, not from the
5004 // select. While it's technically possible to transform a `fcmp` + `select
5005 // nnan` to a `minnum`/`maxnum` call *without* an `nnan`, that would be a
5006 // pessimization in practice. Many targets can't map `minnum`/`maxnum` to a
5007 // single instruction, and if they cannot prove the absence of NaN, must
5008 // lower it to a routine or a libcall. There are additional reasons besides
5009 // performance to avoid introducing libcalls where none existed before
5010 // (https://github.com/llvm/llvm-project/issues/54554).
5011 //
5012 // As such, we want to ensure that the generated `minnum`/`maxnum` intrinsic
5013 // has the `nnan nsz` flags, which allow it to be lowered *back* to a
5014 // fcmp+select if that's the best way to express it on the target.
5015 if (FCmp && FCmp->hasNoNaNs() &&
5016 (SIFPOp->hasNoSignedZeros() || isSelectZeroSignInsignificant(SI))) {
5017 Value *X, *Y;
5018 if (match(V: &SI, P: m_OrdOrUnordFMax(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
5019 Value *BinIntr =
5020 Builder.CreateBinaryIntrinsic(ID: Intrinsic::maxnum, LHS: X, RHS: Y, FMFSource: &SI);
5021 if (auto *BinIntrInst = dyn_cast<Instruction>(Val: BinIntr)) {
5022 // `ninf` must be propagated from the comparison too, rather than the
5023 // select: https://github.com/llvm/llvm-project/pull/136433
5024 BinIntrInst->setHasNoInfs(FCmp->hasNoInfs());
5025 // The `nsz` flag is a precondition, so let's ensure it's always added
5026 // to the min/max operation, even if it wasn't on the select. This
5027 // could happen if the select doesn't have `nsz`, but no use of the
5028 // result can observe the sign of zero.
5029 BinIntrInst->setHasNoSignedZeros(true);
5030 // As mentioned above, `nnan` is also a precondition, so we always set
5031 // the flag.
5032 BinIntrInst->setHasNoNaNs(true);
5033 }
5034 return replaceInstUsesWith(I&: SI, V: BinIntr);
5035 }
5036
5037 if (match(V: &SI, P: m_OrdOrUnordFMin(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
5038 Value *BinIntr =
5039 Builder.CreateBinaryIntrinsic(ID: Intrinsic::minnum, LHS: X, RHS: Y, FMFSource: &SI);
5040 if (auto *BinIntrInst = dyn_cast<Instruction>(Val: BinIntr)) {
5041 BinIntrInst->setHasNoInfs(FCmp->hasNoInfs());
5042 BinIntrInst->setHasNoSignedZeros(true);
5043 BinIntrInst->setHasNoNaNs(true);
5044 }
5045 return replaceInstUsesWith(I&: SI, V: BinIntr);
5046 }
5047 }
5048 }
5049
5050 // Fold selecting to fabs.
5051 if (Instruction *Fabs = foldSelectWithFCmpToFabs(SI, IC&: *this))
5052 return Fabs;
5053
5054 if (Instruction *I = foldSelectOfOrderedFAbsCmpOfNaNScrubbedValue(SI, IC&: *this))
5055 return I;
5056
5057 // See if we are selecting two values based on a comparison of the two values.
5058 if (CmpInst *CI = dyn_cast<CmpInst>(Val: CondVal))
5059 if (Instruction *NewSel = foldSelectValueEquivalence(Sel&: SI, Cmp&: *CI))
5060 return NewSel;
5061
5062 if (ICmpInst *ICI = dyn_cast<ICmpInst>(Val: CondVal))
5063 if (Instruction *Result = foldSelectInstWithICmp(SI, ICI))
5064 return Result;
5065
5066 if (Instruction *V =
5067 foldSelectICmpAndAnd(SelType, Cond: CondVal, TVal: TrueVal, FVal: FalseVal, Builder))
5068 return V;
5069
5070 if (Value *V = foldSelectBitTest(Sel&: SI, CondVal, TrueVal, FalseVal, Builder, SQ))
5071 return replaceInstUsesWith(I&: SI, V);
5072
5073 if (Instruction *Add = foldAddSubSelect(SI, Builder))
5074 return Add;
5075 if (Instruction *Add = foldOverflowingAddSubSelect(SI, Builder))
5076 return Add;
5077 if (Instruction *Or = foldSetClearBits(Sel&: SI, Builder))
5078 return Or;
5079 if (Instruction *Mul = foldSelectZeroOrFixedOp(SI, IC&: *this))
5080 return Mul;
5081
5082 // Turn (select C, (op X, Y), (op X, Z)) -> (op X, (select C, Y, Z))
5083 auto *TI = dyn_cast<Instruction>(Val: TrueVal);
5084 auto *FI = dyn_cast<Instruction>(Val: FalseVal);
5085 if (TI && FI && TI->getOpcode() == FI->getOpcode())
5086 if (Instruction *IV = foldSelectOpOp(SI, TI, FI))
5087 return IV;
5088
5089 if (Instruction *I = foldSelectIntrinsic(SI))
5090 return I;
5091
5092 if (Instruction *I = foldSelectExtConst(Sel&: SI))
5093 return I;
5094
5095 if (Instruction *I = foldSelectWithSRem(SI, IC&: *this, Builder))
5096 return I;
5097
5098 // Fold (select C, (gep Ptr, Idx), Ptr) -> (gep Ptr, (select C, Idx, 0))
5099 // Fold (select C, Ptr, (gep Ptr, Idx)) -> (gep Ptr, (select C, 0, Idx))
5100 auto SelectGepWithBase = [&](GetElementPtrInst *Gep, Value *Base,
5101 bool Swap) -> GetElementPtrInst * {
5102 Value *Ptr = Gep->getPointerOperand();
5103 if (Gep->getNumOperands() != 2 || Gep->getPointerOperand() != Base ||
5104 !Gep->hasOneUse())
5105 return nullptr;
5106 Value *Idx = Gep->getOperand(i_nocapture: 1);
5107 if (isa<VectorType>(Val: CondVal->getType()) && !isa<VectorType>(Val: Idx->getType()))
5108 return nullptr;
5109 Type *ElementType = Gep->getSourceElementType();
5110 Value *NewT = Idx;
5111 Value *NewF = Constant::getNullValue(Ty: Idx->getType());
5112 if (Swap)
5113 std::swap(a&: NewT, b&: NewF);
5114 Value *NewSI =
5115 Builder.CreateSelect(C: CondVal, True: NewT, False: NewF, Name: SI.getName() + ".idx", MDFrom: &SI);
5116 return GetElementPtrInst::Create(PointeeType: ElementType, Ptr, IdxList: NewSI,
5117 NW: Gep->getNoWrapFlags());
5118 };
5119 if (auto *TrueGep = dyn_cast<GetElementPtrInst>(Val: TrueVal))
5120 if (auto *NewGep = SelectGepWithBase(TrueGep, FalseVal, false))
5121 return NewGep;
5122 if (auto *FalseGep = dyn_cast<GetElementPtrInst>(Val: FalseVal))
5123 if (auto *NewGep = SelectGepWithBase(FalseGep, TrueVal, true))
5124 return NewGep;
5125
5126 // See if we can fold the select into one of our operands.
5127 if (SelType->isIntOrIntVectorTy() || SelType->isFPOrFPVectorTy()) {
5128 if (Instruction *FoldI = foldSelectIntoOp(SI, TrueVal, FalseVal))
5129 return FoldI;
5130
5131 Value *LHS, *RHS;
5132 Instruction::CastOps CastOp;
5133 SelectPatternResult SPR = matchSelectPattern(V: &SI, LHS, RHS, CastOp: &CastOp);
5134 auto SPF = SPR.Flavor;
5135 if (SPF) {
5136 Value *LHS2, *RHS2;
5137 if (SelectPatternFlavor SPF2 = matchSelectPattern(V: LHS, LHS&: LHS2, RHS&: RHS2).Flavor)
5138 if (Instruction *R = foldSPFofSPF(Inner: cast<Instruction>(Val: LHS), SPF1: SPF2, A: LHS2,
5139 B: RHS2, Outer&: SI, SPF2: SPF, C: RHS))
5140 return R;
5141 if (SelectPatternFlavor SPF2 = matchSelectPattern(V: RHS, LHS&: LHS2, RHS&: RHS2).Flavor)
5142 if (Instruction *R = foldSPFofSPF(Inner: cast<Instruction>(Val: RHS), SPF1: SPF2, A: LHS2,
5143 B: RHS2, Outer&: SI, SPF2: SPF, C: LHS))
5144 return R;
5145 }
5146
5147 if (SelectPatternResult::isMinOrMax(SPF)) {
5148 // Canonicalize so that
5149 // - type casts are outside select patterns.
5150 // - float clamp is transformed to min/max pattern
5151
5152 bool IsCastNeeded = LHS->getType() != SelType;
5153 Value *CmpLHS = cast<CmpInst>(Val: CondVal)->getOperand(i_nocapture: 0);
5154 Value *CmpRHS = cast<CmpInst>(Val: CondVal)->getOperand(i_nocapture: 1);
5155 if (IsCastNeeded ||
5156 (LHS->getType()->isFPOrFPVectorTy() &&
5157 ((CmpLHS != LHS && CmpLHS != RHS) ||
5158 (CmpRHS != LHS && CmpRHS != RHS)))) {
5159 CmpInst::Predicate MinMaxPred = getMinMaxPred(SPF, Ordered: SPR.Ordered);
5160
5161 Value *Cmp;
5162 if (CmpInst::isIntPredicate(P: MinMaxPred)) {
5163 dropPoisonGeneratingAnnotationsForSPFCanonicalization(Cond: CondVal, LHS,
5164 RHS);
5165 Cmp = Builder.CreateICmp(P: MinMaxPred, LHS, RHS);
5166 } else
5167 Cmp = Builder.CreateFCmpFMF(P: MinMaxPred, LHS, RHS,
5168 FMFSource: cast<Instruction>(Val: SI.getCondition()));
5169
5170 Value *NewSI = Builder.CreateSelect(C: Cmp, True: LHS, False: RHS, Name: SI.getName(), MDFrom: &SI);
5171 if (!IsCastNeeded)
5172 return replaceInstUsesWith(I&: SI, V: NewSI);
5173
5174 Value *NewCast = Builder.CreateCast(Op: CastOp, V: NewSI, DestTy: SelType);
5175 return replaceInstUsesWith(I&: SI, V: NewCast);
5176 }
5177 }
5178 }
5179
5180 // See if we can fold the select into a phi node if the condition is a select.
5181 if (auto *PN = dyn_cast<PHINode>(Val: SI.getCondition()))
5182 if (Instruction *NV = foldOpIntoPhi(I&: SI, PN))
5183 return NV;
5184
5185 if (SelectInst *TrueSI = dyn_cast<SelectInst>(Val: TrueVal)) {
5186 if (TrueSI->getCondition()->getType() == CondVal->getType()) {
5187 // Fold nested selects if the inner condition can be implied by the outer
5188 // condition.
5189 if (Value *V = simplifyNestedSelectsUsingImpliedCond(SI&: *TrueSI, CondVal,
5190 /*CondIsTrue=*/true))
5191 return replaceOperand(I&: SI, OpNum: 1, V);
5192
5193 // We choose this as normal form to enable folding on the And and
5194 // shortening paths for the values (this helps getUnderlyingObjects() for
5195 // example).
5196 if (TrueSI->hasOneUse()) {
5197 Value *And = nullptr, *OtherVal = nullptr;
5198 // select(C0, select(C1, a, b), b) -> select(C0&&C1, a, b)
5199 if (TrueSI->getFalseValue() == FalseVal) {
5200 And = Builder.CreateLogicalAnd(Cond1: CondVal, Cond2: TrueSI->getCondition(), Name: "",
5201 MDFrom: &SI);
5202 OtherVal = TrueSI->getTrueValue();
5203 }
5204 // select(C0, select(C1, b, a), b) -> select(C0&&!C1, a, b)
5205 else if (TrueSI->getTrueValue() == FalseVal) {
5206 Value *InvertedCond = Builder.CreateNot(V: TrueSI->getCondition());
5207 And = Builder.CreateLogicalAnd(Cond1: CondVal, Cond2: InvertedCond, Name: "", MDFrom: &SI);
5208 OtherVal = TrueSI->getFalseValue();
5209 }
5210 if (And && OtherVal) {
5211 replaceOperand(I&: SI, OpNum: 0, V: And);
5212 replaceOperand(I&: SI, OpNum: 1, V: OtherVal);
5213 setExplicitlyUnknownBranchWeightsIfProfiled(I&: SI, DEBUG_TYPE);
5214 return &SI;
5215 }
5216 }
5217 }
5218 }
5219 if (SelectInst *FalseSI = dyn_cast<SelectInst>(Val: FalseVal)) {
5220 if (FalseSI->getCondition()->getType() == CondVal->getType()) {
5221 // Fold nested selects if the inner condition can be implied by the outer
5222 // condition.
5223 if (Value *V = simplifyNestedSelectsUsingImpliedCond(
5224 SI&: *FalseSI, CondVal, /*CondIsTrue=*/false))
5225 return replaceOperand(I&: SI, OpNum: 2, V);
5226
5227 if (FalseSI->hasOneUse()) {
5228 Value *Or = nullptr, *OtherVal = nullptr;
5229 // select(C0, a, select(C1, a, b)) -> select(C0||C1, a, b)
5230 if (FalseSI->getTrueValue() == TrueVal) {
5231 Or = Builder.CreateLogicalOr(Cond1: CondVal, Cond2: FalseSI->getCondition(), Name: "",
5232 MDFrom: &SI);
5233 OtherVal = FalseSI->getFalseValue();
5234 }
5235 // select(C0, a, select(C1, b, a)) -> select(C0||!C1, a, b)
5236 else if (FalseSI->getFalseValue() == TrueVal) {
5237 Value *InvertedCond = Builder.CreateNot(V: FalseSI->getCondition());
5238 Or = Builder.CreateLogicalOr(Cond1: CondVal, Cond2: InvertedCond, Name: "", MDFrom: &SI);
5239 OtherVal = FalseSI->getTrueValue();
5240 }
5241 if (Or && OtherVal) {
5242 replaceOperand(I&: SI, OpNum: 0, V: Or);
5243 replaceOperand(I&: SI, OpNum: 2, V: OtherVal);
5244 setExplicitlyUnknownBranchWeightsIfProfiled(I&: SI, DEBUG_TYPE);
5245 return &SI;
5246 }
5247 }
5248 }
5249 }
5250
5251 // Try to simplify a binop sandwiched between 2 selects with the same
5252 // condition. This is not valid for div/rem because the select might be
5253 // preventing a division-by-zero.
5254 // TODO: A div/rem restriction is conservative; use something like
5255 // isSafeToSpeculativelyExecute().
5256 // select(C, binop(select(C, X, Y), W), Z) -> select(C, binop(X, W), Z)
5257 BinaryOperator *TrueBO;
5258 if (match(V: TrueVal, P: m_OneUse(SubPattern: m_BinOp(I&: TrueBO))) && !TrueBO->isIntDivRem()) {
5259 if (auto *TrueBOSI = dyn_cast<SelectInst>(Val: TrueBO->getOperand(i_nocapture: 0))) {
5260 if (TrueBOSI->getCondition() == CondVal) {
5261 replaceOperand(I&: *TrueBO, OpNum: 0, V: TrueBOSI->getTrueValue());
5262 Worklist.push(I: TrueBO);
5263 return &SI;
5264 }
5265 }
5266 if (auto *TrueBOSI = dyn_cast<SelectInst>(Val: TrueBO->getOperand(i_nocapture: 1))) {
5267 if (TrueBOSI->getCondition() == CondVal) {
5268 replaceOperand(I&: *TrueBO, OpNum: 1, V: TrueBOSI->getTrueValue());
5269 Worklist.push(I: TrueBO);
5270 return &SI;
5271 }
5272 }
5273 }
5274
5275 // select(C, Z, binop(select(C, X, Y), W)) -> select(C, Z, binop(Y, W))
5276 BinaryOperator *FalseBO;
5277 if (match(V: FalseVal, P: m_OneUse(SubPattern: m_BinOp(I&: FalseBO))) && !FalseBO->isIntDivRem()) {
5278 if (auto *FalseBOSI = dyn_cast<SelectInst>(Val: FalseBO->getOperand(i_nocapture: 0))) {
5279 if (FalseBOSI->getCondition() == CondVal) {
5280 replaceOperand(I&: *FalseBO, OpNum: 0, V: FalseBOSI->getFalseValue());
5281 Worklist.push(I: FalseBO);
5282 return &SI;
5283 }
5284 }
5285 if (auto *FalseBOSI = dyn_cast<SelectInst>(Val: FalseBO->getOperand(i_nocapture: 1))) {
5286 if (FalseBOSI->getCondition() == CondVal) {
5287 replaceOperand(I&: *FalseBO, OpNum: 1, V: FalseBOSI->getFalseValue());
5288 Worklist.push(I: FalseBO);
5289 return &SI;
5290 }
5291 }
5292 }
5293
5294 Value *NotCond;
5295 if (match(V: CondVal, P: m_Not(V: m_Value(V&: NotCond))) &&
5296 !InstCombiner::shouldAvoidAbsorbingNotIntoSelect(SI)) {
5297 replaceOperand(I&: SI, OpNum: 0, V: NotCond);
5298 SI.swapValues();
5299 SI.swapProfMetadata();
5300 return &SI;
5301 }
5302
5303 if (Instruction *I = foldVectorSelect(Sel&: SI))
5304 return I;
5305
5306 // If we can compute the condition, there's no need for a select.
5307 // Like the above fold, we are attempting to reduce compile-time cost by
5308 // putting this fold here with limitations rather than in InstSimplify.
5309 // The motivation for this call into value tracking is to take advantage of
5310 // the assumption cache, so make sure that is populated.
5311 if (!CondVal->getType()->isVectorTy() && !AC.assumptions().empty()) {
5312 KnownBits Known(1);
5313 computeKnownBits(V: CondVal, Known, CtxI: &SI);
5314 if (Known.One.isOne())
5315 return replaceInstUsesWith(I&: SI, V: TrueVal);
5316 if (Known.Zero.isOne())
5317 return replaceInstUsesWith(I&: SI, V: FalseVal);
5318 }
5319
5320 if (Instruction *BitCastSel = foldSelectCmpBitcasts(Sel&: SI, Builder))
5321 return BitCastSel;
5322
5323 // Simplify selects that test the returned flag of cmpxchg instructions.
5324 if (Value *V = foldSelectCmpXchg(SI))
5325 return replaceInstUsesWith(I&: SI, V);
5326
5327 if (Instruction *Select = foldSelectBinOpIdentity(Sel&: SI, TLI, IC&: *this))
5328 return Select;
5329
5330 if (Instruction *Funnel = foldSelectFunnelShift(Sel&: SI, Builder))
5331 return Funnel;
5332
5333 if (Instruction *Copysign = foldSelectToCopysign(Sel&: SI, Builder))
5334 return Copysign;
5335
5336 if (Instruction *PN = foldSelectToPhi(Sel&: SI, DT, Builder))
5337 return replaceInstUsesWith(I&: SI, V: PN);
5338
5339 if (Value *V = foldRoundUpIntegerWithPow2Alignment(SI, Builder))
5340 return replaceInstUsesWith(I&: SI, V);
5341
5342 if (Value *V = foldSelectIntoAddConstant(SI, Builder))
5343 return replaceInstUsesWith(I&: SI, V);
5344
5345 // select(mask, mload(ptr,mask,0), 0) -> mload(ptr,mask,0)
5346 // Load inst is intentionally not checked for hasOneUse()
5347 if (match(V: FalseVal, P: m_Zero()) &&
5348 (match(V: TrueVal, P: m_MaskedLoad(Op0: m_Value(), Op1: m_Specific(V: CondVal),
5349 Op2: m_CombineOr(Ps: m_Undef(), Ps: m_Zero()))) ||
5350 match(V: TrueVal, P: m_MaskedGather(Op0: m_Value(), Op1: m_Specific(V: CondVal),
5351 Op2: m_CombineOr(Ps: m_Undef(), Ps: m_Zero()))))) {
5352 auto *MaskedInst = cast<IntrinsicInst>(Val: TrueVal);
5353 if (isa<UndefValue>(Val: MaskedInst->getArgOperand(i: 2)))
5354 MaskedInst->setArgOperand(i: 2, v: FalseVal /* Zero */);
5355 return replaceInstUsesWith(I&: SI, V: MaskedInst);
5356 }
5357
5358 Value *Mask;
5359 if (match(V: TrueVal, P: m_Zero()) &&
5360 (match(V: FalseVal, P: m_MaskedLoad(Op0: m_Value(), Op1: m_Value(V&: Mask),
5361 Op2: m_CombineOr(Ps: m_Undef(), Ps: m_Zero()))) ||
5362 match(V: FalseVal, P: m_MaskedGather(Op0: m_Value(), Op1: m_Value(V&: Mask),
5363 Op2: m_CombineOr(Ps: m_Undef(), Ps: m_Zero())))) &&
5364 (CondVal->getType() == Mask->getType())) {
5365 // We can remove the select by ensuring the load zeros all lanes the
5366 // select would have. We determine this by proving there is no overlap
5367 // between the load and select masks.
5368 // (i.e (load_mask & select_mask) == 0 == no overlap)
5369 bool CanMergeSelectIntoLoad = false;
5370 if (Value *V = simplifyAndInst(LHS: CondVal, RHS: Mask, Q: SQ.getWithInstruction(I: &SI)))
5371 CanMergeSelectIntoLoad = match(V, P: m_Zero());
5372
5373 if (CanMergeSelectIntoLoad) {
5374 auto *MaskedInst = cast<IntrinsicInst>(Val: FalseVal);
5375 if (isa<UndefValue>(Val: MaskedInst->getArgOperand(i: 2)))
5376 MaskedInst->setArgOperand(i: 2, v: TrueVal /* Zero */);
5377 return replaceInstUsesWith(I&: SI, V: MaskedInst);
5378 }
5379 }
5380
5381 if (Instruction *I = foldSelectOfSymmetricSelect(OuterSelVal&: SI, Builder))
5382 return I;
5383
5384 if (Instruction *I = foldNestedSelects(OuterSelVal&: SI, Builder))
5385 return I;
5386
5387 // Match logical variants of the pattern,
5388 // and transform them iff that gets rid of inversions.
5389 // (~x) | y --> ~(x & (~y))
5390 // (~x) & y --> ~(x | (~y))
5391 if (sinkNotIntoOtherHandOfLogicalOp(I&: SI))
5392 return &SI;
5393
5394 if (Instruction *I = foldBitCeil(SI, Builder, IC&: *this))
5395 return I;
5396
5397 if (Instruction *I = foldSelectToCmp(SI))
5398 return I;
5399
5400 if (Instruction *I = foldSelectEqualityTest(Sel&: SI))
5401 return I;
5402
5403 // Fold:
5404 // (select A && B, T, F) -> (select A, (select B, T, F), F)
5405 // (select A || B, T, F) -> (select A, T, (select B, T, F))
5406 // if (select B, T, F) is foldable.
5407 // TODO: preserve FMF flags
5408 auto FoldSelectWithAndOrCond = [&](bool IsAnd, Value *A,
5409 Value *B) -> Instruction * {
5410 Instruction *MDFrom = nullptr;
5411 // If the and/or is logical, we can propagate branch weights from that
5412 // select. Otherwise, we cannot infer the branch weights in the
5413 // general case.
5414 if (auto *NestedSI = dyn_cast<SelectInst>(Val: SI.getCondition()))
5415 if (NestedSI->getCondition() == A)
5416 MDFrom = NestedSI;
5417
5418 if (Value *V = simplifySelectInst(Cond: B, TrueVal, FalseVal, FMF,
5419 Q: SQ.getWithInstruction(I: &SI))) {
5420 SelectInst *NewSI = SelectInst::Create(
5421 C: A, S1: IsAnd ? V : TrueVal, S2: IsAnd ? FalseVal : V, NameStr: "", InsertBefore: nullptr,
5422 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : MDFrom);
5423 if (!MDFrom)
5424 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *NewSI, DEBUG_TYPE, F: &F);
5425 return NewSI;
5426 }
5427
5428 // Is (select B, T, F) a SPF?
5429 if (CondVal->hasOneUse() && SelType->isIntOrIntVectorTy()) {
5430 if (ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: B))
5431 if (Value *V = canonicalizeSPF(Cmp&: *Cmp, TrueVal, FalseVal, IC&: *this)) {
5432 SelectInst *NewSI = SelectInst::Create(
5433 C: A, S1: IsAnd ? V : TrueVal, S2: IsAnd ? FalseVal : V, NameStr: "", InsertBefore: nullptr,
5434 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : MDFrom);
5435 if (!MDFrom)
5436 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *NewSI, DEBUG_TYPE, F: &F);
5437 return NewSI;
5438 }
5439 }
5440
5441 return nullptr;
5442 };
5443
5444 Value *LHS, *RHS;
5445 if (match(V: CondVal, P: m_And(L: m_Value(V&: LHS), R: m_Value(V&: RHS)))) {
5446 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ true, LHS, RHS))
5447 return I;
5448 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ true, RHS, LHS))
5449 return I;
5450 } else if (match(V: CondVal, P: m_Or(L: m_Value(V&: LHS), R: m_Value(V&: RHS)))) {
5451 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ false, LHS, RHS))
5452 return I;
5453 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ false, RHS, LHS))
5454 return I;
5455 } else {
5456 // We cannot swap the operands of logical and/or.
5457 // TODO: Can we swap the operands by inserting a freeze?
5458 if (match(V: CondVal, P: m_LogicalAnd(L: m_Value(V&: LHS), R: m_Value(V&: RHS)))) {
5459 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ true, LHS, RHS))
5460 return I;
5461 } else if (match(V: CondVal, P: m_LogicalOr(L: m_Value(V&: LHS), R: m_Value(V&: RHS)))) {
5462 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ false, LHS, RHS))
5463 return I;
5464 }
5465 }
5466
5467 // select Cond, !X, X -> xor Cond, X
5468 if (CondVal->getType() == SI.getType() && isKnownInversion(X: FalseVal, Y: TrueVal))
5469 return BinaryOperator::CreateXor(V1: CondVal, V2: FalseVal);
5470
5471 // For vectors, this transform is only safe if the simplification does not
5472 // look through any lane-crossing operations. For now, limit to scalars only.
5473 if (SelType->isIntegerTy() &&
5474 (!isa<Constant>(Val: TrueVal) || !isa<Constant>(Val: FalseVal))) {
5475 // Try to simplify select arms based on KnownBits implied by the condition.
5476 CondContext CC(CondVal);
5477 findValuesAffectedByCondition(Cond: CondVal, /*IsAssume=*/false, InsertAffected: [&](Value *V) {
5478 CC.AffectedValues.insert(Ptr: V);
5479 });
5480 SimplifyQuery Q = SQ.getWithInstruction(I: &SI).getWithCondContext(CC);
5481 if (!CC.AffectedValues.empty()) {
5482 if (!isa<Constant>(Val: TrueVal) &&
5483 hasAffectedValue(V: TrueVal, Affected&: CC.AffectedValues, /*Depth=*/0)) {
5484 KnownBits Known = llvm::computeKnownBits(V: TrueVal, Q);
5485 if (Known.isConstant())
5486 return replaceOperand(I&: SI, OpNum: 1,
5487 V: ConstantInt::get(Ty: SelType, V: Known.getConstant()));
5488 }
5489
5490 CC.Invert = true;
5491 if (!isa<Constant>(Val: FalseVal) &&
5492 hasAffectedValue(V: FalseVal, Affected&: CC.AffectedValues, /*Depth=*/0)) {
5493 KnownBits Known = llvm::computeKnownBits(V: FalseVal, Q);
5494 if (Known.isConstant())
5495 return replaceOperand(I&: SI, OpNum: 2,
5496 V: ConstantInt::get(Ty: SelType, V: Known.getConstant()));
5497 }
5498 }
5499 }
5500
5501 // select (trunc nuw X to i1), X, Y --> select (trunc nuw X to i1), 1, Y
5502 // select (trunc nuw X to i1), Y, X --> select (trunc nuw X to i1), Y, 0
5503 // select (trunc nsw X to i1), X, Y --> select (trunc nsw X to i1), -1, Y
5504 // select (trunc nsw X to i1), Y, X --> select (trunc nsw X to i1), Y, 0
5505 Value *Trunc;
5506 if (match(V: CondVal, P: m_NUWTrunc(Op: m_Value(V&: Trunc))) && !isa<Constant>(Val: Trunc)) {
5507 if (TrueVal == Trunc)
5508 return replaceOperand(I&: SI, OpNum: 1, V: ConstantInt::get(Ty: TrueVal->getType(), V: 1));
5509 if (FalseVal == Trunc)
5510 return replaceOperand(I&: SI, OpNum: 2, V: ConstantInt::get(Ty: FalseVal->getType(), V: 0));
5511 }
5512 if (match(V: CondVal, P: m_NSWTrunc(Op: m_Value(V&: Trunc))) && !isa<Constant>(Val: Trunc)) {
5513 if (TrueVal == Trunc)
5514 return replaceOperand(I&: SI, OpNum: 1,
5515 V: Constant::getAllOnesValue(Ty: TrueVal->getType()));
5516 if (FalseVal == Trunc)
5517 return replaceOperand(I&: SI, OpNum: 2, V: ConstantInt::get(Ty: FalseVal->getType(), V: 0));
5518 }
5519
5520 if (match(V: CondVal, P: m_Trunc(Op: m_Value(V&: Trunc))) && Trunc->getType() == SelType) {
5521 if (match(V: FalseVal, P: m_Zero()) && impliesPoison(ValAssumedPoison: TrueVal, V: CondVal) &&
5522 llvm::computeKnownBits(V: TrueVal, Q: SQ.getWithInstruction(I: &SI))
5523 .countMaxActiveBits() == 1)
5524 return BinaryOperator::CreateAnd(V1: Trunc, V2: TrueVal);
5525
5526 if (cast<TruncInst>(Val: CondVal)->hasNoUnsignedWrap() &&
5527 match(V: TrueVal, P: m_One()) && impliesPoison(ValAssumedPoison: FalseVal, V: CondVal) &&
5528 llvm::computeKnownBits(V: FalseVal, Q: SQ.getWithInstruction(I: &SI))
5529 .countMaxActiveBits() == 1) {
5530 return BinaryOperator::CreateOr(V1: Trunc, V2: FalseVal);
5531 }
5532 }
5533
5534 Value *MaskedLoadPtr;
5535 if (match(V: TrueVal, P: m_OneUse(SubPattern: m_MaskedLoad(Op0: m_Value(V&: MaskedLoadPtr),
5536 Op1: m_Specific(V: CondVal), Op2: m_Value())))) {
5537 auto *LoadInst = cast<IntrinsicInst>(Val: TrueVal);
5538 // Keep the load at its original position to avoid crossing writes. The new
5539 // passthrough must therefore be available there.
5540 if (DT.dominates(Def: FalseVal, User: LoadInst)) {
5541 Builder.SetInsertPoint(LoadInst);
5542 Instruction *In = Builder.CreateMaskedLoad(
5543 Ty: TrueVal->getType(), Ptr: MaskedLoadPtr,
5544 Alignment: LoadInst->getParamAlign(ArgNo: 0).valueOrOne(), Mask: CondVal, PassThru: FalseVal);
5545 In->setAAMetadata(LoadInst->getAAMetadata());
5546 return replaceInstUsesWith(I&: SI, V: In);
5547 }
5548 }
5549
5550 // Canonicalize sign function ashr pattern: select (icmp slt X, 1), ashr X,
5551 // bitwidth-1, 1 -> scmp(X, 0)
5552 // Also handles: select (icmp sgt X, 0), 1, ashr X, bitwidth-1 -> scmp(X, 0)
5553 unsigned BitWidth = SI.getType()->getScalarSizeInBits();
5554 CmpPredicate Pred;
5555 Value *CmpLHS, *CmpRHS;
5556
5557 // Canonicalize sign function ashr patterns:
5558 // select (icmp slt X, 1), ashr X, bitwidth-1, 1 -> scmp(X, 0)
5559 // select (icmp sgt X, 0), 1, ashr X, bitwidth-1 -> scmp(X, 0)
5560 if (match(V: &SI, P: m_Select(C: m_ICmp(Pred, L: m_Value(V&: CmpLHS), R: m_Value(V&: CmpRHS)),
5561 L: m_Value(V&: TrueVal), R: m_Value(V&: FalseVal))) &&
5562 ((Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: m_One()) &&
5563 match(V: TrueVal,
5564 P: m_AShr(L: m_Specific(V: CmpLHS), R: m_SpecificInt(V: BitWidth - 1))) &&
5565 match(V: FalseVal, P: m_One())) ||
5566 (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: m_Zero()) &&
5567 match(V: TrueVal, P: m_One()) &&
5568 match(V: FalseVal,
5569 P: m_AShr(L: m_Specific(V: CmpLHS), R: m_SpecificInt(V: BitWidth - 1)))))) {
5570
5571 Function *Scmp = Intrinsic::getOrInsertDeclaration(
5572 M: SI.getModule(), id: Intrinsic::scmp, OverloadTys: {SI.getType(), SI.getType()});
5573 return CallInst::Create(Func: Scmp, Args: {CmpLHS, ConstantInt::get(Ty: SI.getType(), V: 0)});
5574 }
5575
5576 return nullptr;
5577}
5578