1//===- InstCombineSelect.cpp ----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitSelect function.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/CmpInstAnalysis.h"
19#include "llvm/Analysis/InstructionSimplify.h"
20#include "llvm/Analysis/Loads.h"
21#include "llvm/Analysis/OverflowInstAnalysis.h"
22#include "llvm/Analysis/ValueTracking.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/IR/BasicBlock.h"
25#include "llvm/IR/Constant.h"
26#include "llvm/IR/ConstantRange.h"
27#include "llvm/IR/Constants.h"
28#include "llvm/IR/DerivedTypes.h"
29#include "llvm/IR/FMF.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstrTypes.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/Operator.h"
37#include "llvm/IR/PatternMatch.h"
38#include "llvm/IR/ProfDataUtils.h"
39#include "llvm/IR/Type.h"
40#include "llvm/IR/User.h"
41#include "llvm/IR/Value.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/KnownBits.h"
45#include "llvm/Support/MathExtras.h"
46#include "llvm/Transforms/InstCombine/InstCombiner.h"
47#include <cassert>
48#include <optional>
49#include <utility>
50
51#define DEBUG_TYPE "instcombine"
52#include "llvm/Transforms/Utils/InstructionWorklist.h"
53
54using namespace llvm;
55using namespace PatternMatch;
56
57namespace llvm {
58extern cl::opt<bool> ProfcheckDisableMetadataFixes;
59}
60
61/// Replace a select operand based on an equality comparison with the identity
62/// constant of a binop.
63static Instruction *foldSelectBinOpIdentity(SelectInst &Sel,
64 const TargetLibraryInfo &TLI,
65 InstCombinerImpl &IC) {
66 // The select condition must be an equality compare with a constant operand.
67 Value *X;
68 Constant *C;
69 CmpPredicate Pred;
70 if (!match(V: Sel.getCondition(), P: m_Cmp(Pred, L: m_Value(V&: X), R: m_Constant(C))))
71 return nullptr;
72
73 bool IsEq;
74 if (ICmpInst::isEquality(P: Pred))
75 IsEq = Pred == ICmpInst::ICMP_EQ;
76 else if (Pred == FCmpInst::FCMP_OEQ)
77 IsEq = true;
78 else if (Pred == FCmpInst::FCMP_UNE)
79 IsEq = false;
80 else
81 return nullptr;
82
83 // A select operand must be a binop.
84 BinaryOperator *BO;
85 if (!match(V: Sel.getOperand(i_nocapture: IsEq ? 1 : 2), P: m_BinOp(I&: BO)))
86 return nullptr;
87
88 // For absorbing values, we can fold to the compared value.
89 bool IsAbsorbingValue = false;
90
91 // Last, match the compare variable operand with a binop operand.
92 Value *Y;
93 if (BO->isCommutative()) {
94 // Recognized 0 as an absorbing value for fmul, but we need to be careful
95 // about the sign. This could be more aggressive, by handling arbitrary sign
96 // bit operations as long as we know the fmul sign matches (and handling
97 // arbitrary opcodes).
98 if (match(V: BO, P: m_c_FMul(L: m_FAbs(Op0: m_Specific(V: X)), R: m_Value(V&: Y))) &&
99 match(V: C, P: m_AnyZeroFP()) &&
100 IC.fmulByZeroIsZero(MulVal: Y, FMF: BO->getFastMathFlags(), CtxI: &Sel))
101 IsAbsorbingValue = true;
102 else if (!match(V: BO, P: m_c_BinOp(L: m_Value(V&: Y), R: m_Specific(V: X))))
103 return nullptr;
104 } else {
105 if (!match(V: BO, P: m_BinOp(L: m_Value(V&: Y), R: m_Specific(V: X))))
106 return nullptr;
107 }
108
109 // The compare constant must be the identity constant for that binop.
110 // If this a floating-point compare with 0.0, any zero constant will do.
111 Type *Ty = BO->getType();
112
113 Value *FoldedVal;
114 if (IsAbsorbingValue) {
115 FoldedVal = C;
116 } else {
117 Constant *IdC = ConstantExpr::getBinOpIdentity(Opcode: BO->getOpcode(), Ty, AllowRHSConstant: true);
118 if (IdC != C) {
119 if (!IdC || !CmpInst::isFPPredicate(P: Pred))
120 return nullptr;
121
122 if (!match(V: IdC, P: m_AnyZeroFP()) || !match(V: C, P: m_AnyZeroFP()))
123 return nullptr;
124 }
125
126 // +0.0 compares equal to -0.0, and so it does not behave as required for
127 // this transform. Bail out if we can not exclude that possibility.
128 if (const auto *FPO = dyn_cast<FPMathOperator>(Val: BO))
129 if (!FPO->hasNoSignedZeros() &&
130 !cannotBeNegativeZero(V: Y,
131 SQ: IC.getSimplifyQuery().getWithInstruction(I: &Sel)))
132 return nullptr;
133
134 FoldedVal = Y;
135 }
136
137 // BO = binop Y, X
138 // S = { select (cmp eq X, C), BO, ? } or { select (cmp ne X, C), ?, BO }
139 // =>
140 // S = { select (cmp eq X, C), Y, ? } or { select (cmp ne X, C), ?, Y }
141 return IC.replaceOperand(I&: Sel, OpNum: IsEq ? 1 : 2, V: FoldedVal);
142}
143
144/// This folds:
145/// select (icmp eq (and X, C1)), TC, FC
146/// iff C1 is a power 2 and the difference between TC and FC is a power-of-2.
147/// To something like:
148/// (shr (and (X, C1)), (log2(C1) - log2(TC-FC))) + FC
149/// Or:
150/// (shl (and (X, C1)), (log2(TC-FC) - log2(C1))) + FC
151/// With some variations depending if FC is larger than TC, or the shift
152/// isn't needed, or the bit widths don't match.
153static Value *foldSelectICmpAnd(SelectInst &Sel, Value *CondVal, Value *TrueVal,
154 Value *FalseVal, Value *V, const APInt &AndMask,
155 bool CreateAnd,
156 InstCombiner::BuilderTy &Builder) {
157 const APInt *SelTC, *SelFC;
158 if (!match(V: TrueVal, P: m_APInt(Res&: SelTC)) || !match(V: FalseVal, P: m_APInt(Res&: SelFC)))
159 return nullptr;
160
161 Type *SelType = Sel.getType();
162 // In general, when both constants are non-zero, we would need an offset to
163 // replace the select. This would require more instructions than we started
164 // with. But there's one special-case that we handle here because it can
165 // simplify/reduce the instructions.
166 const APInt &TC = *SelTC;
167 const APInt &FC = *SelFC;
168 if (!TC.isZero() && !FC.isZero()) {
169 if (TC.getBitWidth() != AndMask.getBitWidth())
170 return nullptr;
171 // If we have to create an 'and', then we must kill the cmp to not
172 // increase the instruction count.
173 if (CreateAnd && !CondVal->hasOneUse())
174 return nullptr;
175
176 // (V & AndMaskC) == 0 ? TC : FC --> TC | (V & AndMaskC)
177 // (V & AndMaskC) == 0 ? TC : FC --> TC ^ (V & AndMaskC)
178 // (V & AndMaskC) == 0 ? TC : FC --> TC + (V & AndMaskC)
179 // (V & AndMaskC) == 0 ? TC : FC --> TC - (V & AndMaskC)
180 Constant *TCC = ConstantInt::get(Ty: SelType, V: TC);
181 Constant *FCC = ConstantInt::get(Ty: SelType, V: FC);
182 Constant *MaskC = ConstantInt::get(Ty: SelType, V: AndMask);
183 for (auto Opc : {Instruction::Or, Instruction::Xor, Instruction::Add,
184 Instruction::Sub}) {
185 if (ConstantFoldBinaryOpOperands(Opcode: Opc, LHS: TCC, RHS: MaskC, DL: Sel.getDataLayout()) ==
186 FCC) {
187 if (CreateAnd)
188 V = Builder.CreateAnd(LHS: V, RHS: MaskC);
189 return Builder.CreateBinOp(Opc, LHS: TCC, RHS: V);
190 }
191 }
192
193 return nullptr;
194 }
195
196 // Make sure one of the select arms is a power-of-2.
197 if (!TC.isPowerOf2() && !FC.isPowerOf2())
198 return nullptr;
199
200 // Determine which shift is needed to transform result of the 'and' into the
201 // desired result.
202 const APInt &ValC = !TC.isZero() ? TC : FC;
203 unsigned ValZeros = ValC.logBase2();
204 unsigned AndZeros = AndMask.logBase2();
205 bool ShouldNotVal = !TC.isZero();
206 bool NeedShift = ValZeros != AndZeros;
207 bool NeedZExtTrunc =
208 SelType->getScalarSizeInBits() != V->getType()->getScalarSizeInBits();
209
210 // If we would need to create an 'and' + 'shift' + 'xor' + cast to replace
211 // a 'select' + 'icmp', then this transformation would result in more
212 // instructions and potentially interfere with other folding.
213 if (CreateAnd + ShouldNotVal + NeedShift + NeedZExtTrunc >
214 1 + CondVal->hasOneUse())
215 return nullptr;
216
217 // Insert the 'and' instruction on the input to the truncate.
218 if (CreateAnd)
219 V = Builder.CreateAnd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: AndMask));
220
221 // If types don't match, we can still convert the select by introducing a zext
222 // or a trunc of the 'and'.
223 if (ValZeros > AndZeros) {
224 V = Builder.CreateZExtOrTrunc(V, DestTy: SelType);
225 V = Builder.CreateShl(LHS: V, RHS: ValZeros - AndZeros);
226 } else if (ValZeros < AndZeros) {
227 V = Builder.CreateLShr(LHS: V, RHS: AndZeros - ValZeros);
228 V = Builder.CreateZExtOrTrunc(V, DestTy: SelType);
229 } else {
230 V = Builder.CreateZExtOrTrunc(V, DestTy: SelType);
231 }
232
233 // Okay, now we know that everything is set up, we just don't know whether we
234 // have a icmp_ne or icmp_eq and whether the true or false val is the zero.
235 if (ShouldNotVal)
236 V = Builder.CreateXor(LHS: V, RHS: ValC);
237
238 return V;
239}
240
241/// We want to turn code that looks like this:
242/// %C = or %A, %B
243/// %D = select %cond, %C, %A
244/// into:
245/// %C = select %cond, %B, 0
246/// %D = or %A, %C
247///
248/// Assuming that the specified instruction is an operand to the select, return
249/// a bitmask indicating which operands of this instruction are foldable if they
250/// equal the other incoming value of the select.
251static unsigned getSelectFoldableOperands(BinaryOperator *I) {
252 switch (I->getOpcode()) {
253 case Instruction::Add:
254 case Instruction::FAdd:
255 case Instruction::Mul:
256 case Instruction::FMul:
257 case Instruction::And:
258 case Instruction::Or:
259 case Instruction::Xor:
260 return 3; // Can fold through either operand.
261 case Instruction::Sub: // Can only fold on the amount subtracted.
262 case Instruction::FSub:
263 case Instruction::FDiv: // Can only fold on the divisor amount.
264 case Instruction::Shl: // Can only fold on the shift amount.
265 case Instruction::LShr:
266 case Instruction::AShr:
267 return 1;
268 default:
269 return 0; // Cannot fold
270 }
271}
272
273/// We have (select c, TI, FI), and we know that TI and FI have the same opcode.
274Instruction *InstCombinerImpl::foldSelectOpOp(SelectInst &SI, Instruction *TI,
275 Instruction *FI) {
276 // If this is a cast from the same type, merge.
277 Value *Cond = SI.getCondition();
278 Type *CondTy = Cond->getType();
279 if (TI->getNumOperands() == 1 && TI->isCast()) {
280 Type *FIOpndTy = FI->getOperand(i: 0)->getType();
281 if (TI->getOperand(i: 0)->getType() != FIOpndTy)
282 return nullptr;
283
284 // The select condition may be a vector. We may only change the operand
285 // type if the vector width remains the same (and matches the condition).
286 if (auto *CondVTy = dyn_cast<VectorType>(Val: CondTy)) {
287 if (!FIOpndTy->isVectorTy() ||
288 CondVTy->getElementCount() !=
289 cast<VectorType>(Val: FIOpndTy)->getElementCount())
290 return nullptr;
291
292 // TODO: If the backend knew how to deal with casts better, we could
293 // remove this limitation. For now, there's too much potential to create
294 // worse codegen by promoting the select ahead of size-altering casts
295 // (PR28160).
296 //
297 // Note that ValueTracking's matchSelectPattern() looks through casts
298 // without checking 'hasOneUse' when it matches min/max patterns, so this
299 // transform may end up happening anyway.
300 if (TI->getOpcode() != Instruction::BitCast &&
301 (!TI->hasOneUse() || !FI->hasOneUse()))
302 return nullptr;
303 } else if (!TI->hasOneUse() || !FI->hasOneUse()) {
304 // TODO: The one-use restrictions for a scalar select could be eased if
305 // the fold of a select in visitLoadInst() was enhanced to match a pattern
306 // that includes a cast.
307 return nullptr;
308 }
309
310 // Fold this by inserting a select from the input values.
311 Value *NewSI =
312 Builder.CreateSelect(C: Cond, True: TI->getOperand(i: 0), False: FI->getOperand(i: 0),
313 Name: SI.getName() + ".v", MDFrom: &SI);
314 return CastInst::Create(Instruction::CastOps(TI->getOpcode()), S: NewSI,
315 Ty: TI->getType());
316 }
317
318 Value *OtherOpT, *OtherOpF;
319 bool MatchIsOpZero;
320 auto getCommonOp = [&](Instruction *TI, Instruction *FI, bool Commute,
321 bool Swapped = false) -> Value * {
322 assert(!(Commute && Swapped) &&
323 "Commute and Swapped can't set at the same time");
324 if (!Swapped) {
325 if (TI->getOperand(i: 0) == FI->getOperand(i: 0)) {
326 OtherOpT = TI->getOperand(i: 1);
327 OtherOpF = FI->getOperand(i: 1);
328 MatchIsOpZero = true;
329 return TI->getOperand(i: 0);
330 } else if (TI->getOperand(i: 1) == FI->getOperand(i: 1)) {
331 OtherOpT = TI->getOperand(i: 0);
332 OtherOpF = FI->getOperand(i: 0);
333 MatchIsOpZero = false;
334 return TI->getOperand(i: 1);
335 }
336 }
337
338 if (!Commute && !Swapped)
339 return nullptr;
340
341 // If we are allowing commute or swap of operands, then
342 // allow a cross-operand match. In that case, MatchIsOpZero
343 // means that TI's operand 0 (FI's operand 1) is the common op.
344 if (TI->getOperand(i: 0) == FI->getOperand(i: 1)) {
345 OtherOpT = TI->getOperand(i: 1);
346 OtherOpF = FI->getOperand(i: 0);
347 MatchIsOpZero = true;
348 return TI->getOperand(i: 0);
349 } else if (TI->getOperand(i: 1) == FI->getOperand(i: 0)) {
350 OtherOpT = TI->getOperand(i: 0);
351 OtherOpF = FI->getOperand(i: 1);
352 MatchIsOpZero = false;
353 return TI->getOperand(i: 1);
354 }
355 return nullptr;
356 };
357
358 if (TI->hasOneUse() || FI->hasOneUse()) {
359 // Cond ? -X : -Y --> -(Cond ? X : Y)
360 Value *X, *Y;
361 if (match(V: TI, P: m_FNeg(X: m_Value(V&: X))) && match(V: FI, P: m_FNeg(X: m_Value(V&: Y)))) {
362 // Intersect FMF from the fneg instructions and union those with the
363 // select.
364 FastMathFlags FMF = TI->getFastMathFlags();
365 FMF &= FI->getFastMathFlags();
366 FMF |= SI.getFastMathFlags();
367 Value *NewSel =
368 Builder.CreateSelect(C: Cond, True: X, False: Y, Name: SI.getName() + ".v", MDFrom: &SI);
369 if (auto *NewSelI = dyn_cast<Instruction>(Val: NewSel))
370 NewSelI->setFastMathFlags(FMF);
371 Instruction *NewFNeg = UnaryOperator::CreateFNeg(V: NewSel);
372 NewFNeg->setFastMathFlags(FMF);
373 return NewFNeg;
374 }
375
376 // Min/max intrinsic with a common operand can have the common operand
377 // pulled after the select. This is the same transform as below for binops,
378 // but specialized for intrinsic matching and without the restrictive uses
379 // clause.
380 auto *TII = dyn_cast<IntrinsicInst>(Val: TI);
381 auto *FII = dyn_cast<IntrinsicInst>(Val: FI);
382 if (TII && FII && TII->getIntrinsicID() == FII->getIntrinsicID()) {
383 if (match(V: TII, P: m_MaxOrMin(Op0: m_Value(), Op1: m_Value()))) {
384 if (Value *MatchOp = getCommonOp(TI, FI, true)) {
385 Value *NewSel =
386 Builder.CreateSelect(C: Cond, True: OtherOpT, False: OtherOpF, Name: "minmaxop", MDFrom: &SI);
387 return CallInst::Create(Func: TII->getCalledFunction(), Args: {NewSel, MatchOp});
388 }
389 }
390
391 // select c, (ldexp v, e0), (ldexp v, e1) -> ldexp v, (select c, e0, e1)
392 // select c, (ldexp v0, e), (ldexp v1, e) -> ldexp (select c, v0, v1), e
393 //
394 // select c, (ldexp v0, e0), (ldexp v1, e1) ->
395 // ldexp (select c, v0, v1), (select c, e0, e1)
396 if (TII->getIntrinsicID() == Intrinsic::ldexp) {
397 Value *LdexpVal0 = TII->getArgOperand(i: 0);
398 Value *LdexpExp0 = TII->getArgOperand(i: 1);
399 Value *LdexpVal1 = FII->getArgOperand(i: 0);
400 Value *LdexpExp1 = FII->getArgOperand(i: 1);
401 if (LdexpExp0->getType() == LdexpExp1->getType()) {
402 FPMathOperator *SelectFPOp = cast<FPMathOperator>(Val: &SI);
403 FastMathFlags FMF = cast<FPMathOperator>(Val: TII)->getFastMathFlags();
404 FMF &= cast<FPMathOperator>(Val: FII)->getFastMathFlags();
405 FMF |= SelectFPOp->getFastMathFlags();
406
407 Value *SelectVal = Builder.CreateSelect(
408 C: Cond, True: LdexpVal0, False: LdexpVal1, Name: "",
409 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &SI);
410 Value *SelectExp = Builder.CreateSelect(
411 C: Cond, True: LdexpExp0, False: LdexpExp1, Name: "",
412 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &SI);
413
414 Value *NewLdexp = Builder.CreateIntrinsic(
415 RetTy: TII->getType(), ID: Intrinsic::ldexp, Args: {SelectVal, SelectExp}, FMFSource: FMF);
416 return replaceInstUsesWith(I&: SI, V: NewLdexp);
417 }
418 }
419 }
420
421 auto CreateCmpSel = [&](std::optional<CmpPredicate> P,
422 bool Swapped) -> CmpInst * {
423 if (!P)
424 return nullptr;
425 auto *MatchOp = getCommonOp(TI, FI, ICmpInst::isEquality(P: *P),
426 ICmpInst::isRelational(P: *P) && Swapped);
427 if (!MatchOp)
428 return nullptr;
429 Value *NewSel = Builder.CreateSelect(C: Cond, True: OtherOpT, False: OtherOpF,
430 Name: SI.getName() + ".v", MDFrom: &SI);
431 return new ICmpInst(MatchIsOpZero ? *P
432 : ICmpInst::getSwappedCmpPredicate(Pred: *P),
433 MatchOp, NewSel);
434 };
435
436 // icmp with a common operand also can have the common operand
437 // pulled after the select.
438 CmpPredicate TPred, FPred;
439 if (match(V: TI, P: m_ICmp(Pred&: TPred, L: m_Value(), R: m_Value())) &&
440 match(V: FI, P: m_ICmp(Pred&: FPred, L: m_Value(), R: m_Value()))) {
441 if (auto *R =
442 CreateCmpSel(CmpPredicate::getMatching(A: TPred, B: FPred), false))
443 return R;
444 if (auto *R =
445 CreateCmpSel(CmpPredicate::getMatching(
446 A: TPred, B: ICmpInst::getSwappedCmpPredicate(Pred: FPred)),
447 true))
448 return R;
449 }
450 }
451
452 // Only handle binary operators (including two-operand getelementptr) with
453 // one-use here. As with the cast case above, it may be possible to relax the
454 // one-use constraint, but that needs be examined carefully since it may not
455 // reduce the total number of instructions.
456 if (TI->getNumOperands() != 2 || FI->getNumOperands() != 2 ||
457 !TI->isSameOperationAs(I: FI) ||
458 (!isa<BinaryOperator>(Val: TI) && !isa<GetElementPtrInst>(Val: TI)) ||
459 !TI->hasOneUse() || !FI->hasOneUse())
460 return nullptr;
461
462 // Figure out if the operations have any operands in common.
463 Value *MatchOp = getCommonOp(TI, FI, TI->isCommutative());
464 if (!MatchOp)
465 return nullptr;
466
467 // If the select condition is a vector, the operands of the original select's
468 // operands also must be vectors. This may not be the case for getelementptr
469 // for example.
470 if (CondTy->isVectorTy() && (!OtherOpT->getType()->isVectorTy() ||
471 !OtherOpF->getType()->isVectorTy()))
472 return nullptr;
473
474 // If we are sinking div/rem after a select, we may need to freeze the
475 // condition because div/rem may induce immediate UB with a poison operand.
476 // For example, the following transform is not safe if Cond can ever be poison
477 // because we can replace poison with zero and then we have div-by-zero that
478 // didn't exist in the original code:
479 // Cond ? x/y : x/z --> x / (Cond ? y : z)
480 auto *BO = dyn_cast<BinaryOperator>(Val: TI);
481 if (BO && BO->isIntDivRem() && !isGuaranteedNotToBePoison(V: Cond)) {
482 // A udiv/urem with a common divisor is safe because UB can only occur with
483 // div-by-zero, and that would be present in the original code.
484 if (BO->getOpcode() == Instruction::SDiv ||
485 BO->getOpcode() == Instruction::SRem || MatchIsOpZero)
486 Cond = Builder.CreateFreeze(V: Cond);
487 }
488
489 // If we reach here, they do have operations in common.
490 Value *NewSI = Builder.CreateSelect(C: Cond, True: OtherOpT, False: OtherOpF,
491 Name: SI.getName() + ".v", MDFrom: &SI);
492 Value *Op0 = MatchIsOpZero ? MatchOp : NewSI;
493 Value *Op1 = MatchIsOpZero ? NewSI : MatchOp;
494 if (auto *BO = dyn_cast<BinaryOperator>(Val: TI)) {
495 BinaryOperator *NewBO = BinaryOperator::Create(Op: BO->getOpcode(), S1: Op0, S2: Op1);
496 NewBO->copyIRFlags(V: TI);
497 NewBO->andIRFlags(V: FI);
498 return NewBO;
499 }
500 if (auto *TGEP = dyn_cast<GetElementPtrInst>(Val: TI)) {
501 auto *FGEP = cast<GetElementPtrInst>(Val: FI);
502 Type *ElementType = TGEP->getSourceElementType();
503 return GetElementPtrInst::Create(
504 PointeeType: ElementType, Ptr: Op0, IdxList: Op1, NW: TGEP->getNoWrapFlags() & FGEP->getNoWrapFlags());
505 }
506 llvm_unreachable("Expected BinaryOperator or GEP");
507 return nullptr;
508}
509
510/// This transforms patterns of the form:
511/// select cond, intrinsic(x, ...), intrinsic(y, ...)
512/// into:
513/// intrinsic(select cond, x, y, ...)
514Instruction *InstCombinerImpl::foldSelectIntrinsic(SelectInst &SI) {
515 auto *LHSIntrinsic = dyn_cast<IntrinsicInst>(Val: SI.getTrueValue());
516 if (!LHSIntrinsic)
517 return nullptr;
518 auto *RHSIntrinsic = dyn_cast<IntrinsicInst>(Val: SI.getFalseValue());
519 if (!RHSIntrinsic ||
520 LHSIntrinsic->getIntrinsicID() != RHSIntrinsic->getIntrinsicID() ||
521 !LHSIntrinsic->hasOneUse() || !RHSIntrinsic->hasOneUse())
522 return nullptr;
523
524 const Intrinsic::ID IID = LHSIntrinsic->getIntrinsicID();
525 switch (IID) {
526 case Intrinsic::abs:
527 case Intrinsic::cttz:
528 case Intrinsic::ctlz: {
529 auto *TZ = cast<ConstantInt>(Val: LHSIntrinsic->getArgOperand(i: 1));
530 auto *FZ = cast<ConstantInt>(Val: RHSIntrinsic->getArgOperand(i: 1));
531
532 Value *TV = LHSIntrinsic->getArgOperand(i: 0);
533 Value *FV = RHSIntrinsic->getArgOperand(i: 0);
534
535 Value *NewSel = Builder.CreateSelect(C: SI.getCondition(), True: TV, False: FV, Name: "", MDFrom: &SI);
536 Value *NewPoisonFlag = Builder.CreateAnd(LHS: TZ, RHS: FZ);
537 Value *NewCall = Builder.CreateBinaryIntrinsic(ID: IID, LHS: NewSel, RHS: NewPoisonFlag);
538
539 return replaceInstUsesWith(I&: SI, V: NewCall);
540 }
541 case Intrinsic::ctpop: {
542 Value *TV = LHSIntrinsic->getArgOperand(i: 0);
543 Value *FV = RHSIntrinsic->getArgOperand(i: 0);
544
545 Value *NewSel = Builder.CreateSelect(C: SI.getCondition(), True: TV, False: FV, Name: "", MDFrom: &SI);
546 Value *NewCall = Builder.CreateUnaryIntrinsic(ID: IID, Op: NewSel);
547
548 return replaceInstUsesWith(I&: SI, V: NewCall);
549 }
550 default:
551 return nullptr;
552 }
553}
554
555static bool isSelect01(const APInt &C1I, const APInt &C2I) {
556 if (!C1I.isZero() && !C2I.isZero()) // One side must be zero.
557 return false;
558 return C1I.isOne() || C1I.isAllOnes() || C2I.isOne() || C2I.isAllOnes();
559}
560
561/// Try to fold the select into one of the operands to allow further
562/// optimization.
563Instruction *InstCombinerImpl::foldSelectIntoOp(SelectInst &SI, Value *TrueVal,
564 Value *FalseVal) {
565 // See the comment above getSelectFoldableOperands for a description of the
566 // transformation we are doing here.
567 auto TryFoldSelectIntoOp = [&](SelectInst &SI, Value *TrueVal,
568 Value *FalseVal,
569 bool Swapped) -> Instruction * {
570 auto *TVI = dyn_cast<BinaryOperator>(Val: TrueVal);
571 if (!TVI || !TVI->hasOneUse() || isa<Constant>(Val: FalseVal))
572 return nullptr;
573
574 unsigned SFO = getSelectFoldableOperands(I: TVI);
575 unsigned OpToFold = 0;
576 if ((SFO & 1) && FalseVal == TVI->getOperand(i_nocapture: 0))
577 OpToFold = 1;
578 else if ((SFO & 2) && FalseVal == TVI->getOperand(i_nocapture: 1))
579 OpToFold = 2;
580
581 if (!OpToFold)
582 return nullptr;
583
584 FastMathFlags FMF;
585 if (const auto *FPO = dyn_cast<FPMathOperator>(Val: &SI))
586 FMF = FPO->getFastMathFlags();
587 Constant *C = ConstantExpr::getBinOpIdentity(
588 Opcode: TVI->getOpcode(), Ty: TVI->getType(), AllowRHSConstant: true, NSZ: FMF.noSignedZeros());
589 Value *OOp = TVI->getOperand(i_nocapture: 2 - OpToFold);
590 // Avoid creating select between 2 constants unless it's selecting
591 // between 0, 1 and -1.
592 const APInt *OOpC;
593 bool OOpIsAPInt = match(V: OOp, P: m_APInt(Res&: OOpC));
594 if (isa<Constant>(Val: OOp) &&
595 (!OOpIsAPInt || !isSelect01(C1I: C->getUniqueInteger(), C2I: *OOpC)))
596 return nullptr;
597
598 // If the false value is a NaN then we have that the floating point math
599 // operation in the transformed code may not preserve the exact NaN
600 // bit-pattern -- e.g. `fadd sNaN, 0.0 -> qNaN`.
601 // This makes the transformation incorrect since the original program would
602 // have preserved the exact NaN bit-pattern.
603 // Avoid the folding if the false value might be a NaN.
604 if (isa<FPMathOperator>(Val: &SI) &&
605 !computeKnownFPClass(V: FalseVal, FMF, InterestedClasses: fcNan, SQ: SQ.getWithInstruction(I: &SI))
606 .isKnownNeverNaN())
607 return nullptr;
608
609 Value *NewSel = Builder.CreateSelect(C: SI.getCondition(), True: Swapped ? C : OOp,
610 False: Swapped ? OOp : C, Name: "", MDFrom: &SI);
611 if (isa<FPMathOperator>(Val: &SI)) {
612 FastMathFlags NewSelFMF = FMF;
613 // We cannot propagate ninf from the original select, because OOp may be
614 // inf and the flag only guarantees that FalseVal (op OOp) is never
615 // infinity.
616 // Examples: -inf + +inf = NaN, -inf - -inf = NaN, 0 * inf = NaN
617 // Specifically, if the original select has both ninf and nnan, we can
618 // safely propagate the flag.
619 // Note: This property holds for fadd, fsub, and fmul, but does not
620 // hold for fdiv (e.g. A / Inf == 0.0).
621 bool CanInferFiniteOperandsFromResult =
622 TVI->getOpcode() == Instruction::FAdd ||
623 TVI->getOpcode() == Instruction::FSub ||
624 TVI->getOpcode() == Instruction::FMul;
625 NewSelFMF.setNoInfs(TVI->hasNoInfs() ||
626 (CanInferFiniteOperandsFromResult &&
627 NewSelFMF.noInfs() && NewSelFMF.noNaNs()));
628 cast<Instruction>(Val: NewSel)->setFastMathFlags(NewSelFMF);
629 }
630 NewSel->takeName(V: TVI);
631 BinaryOperator *BO =
632 BinaryOperator::Create(Op: TVI->getOpcode(), S1: FalseVal, S2: NewSel);
633 BO->copyIRFlags(V: TVI);
634 if (isa<FPMathOperator>(Val: &SI)) {
635 // Merge poison generating flags from the select.
636 BO->setHasNoNaNs(BO->hasNoNaNs() && FMF.noNaNs());
637 BO->setHasNoInfs(BO->hasNoInfs() && FMF.noInfs());
638 // Merge no-signed-zeros flag from the select.
639 // Otherwise we may produce zeros with different sign.
640 BO->setHasNoSignedZeros(BO->hasNoSignedZeros() && FMF.noSignedZeros());
641 }
642 return BO;
643 };
644
645 if (Instruction *R = TryFoldSelectIntoOp(SI, TrueVal, FalseVal, false))
646 return R;
647
648 if (Instruction *R = TryFoldSelectIntoOp(SI, FalseVal, TrueVal, true))
649 return R;
650
651 return nullptr;
652}
653
654static Value *canoncalizeSelectICmpMinMax(const ICmpInst *Cmp, Value *TVal,
655 Value *FVal,
656 InstCombiner::BuilderTy &Builder,
657 const SimplifyQuery &SQ) {
658 Value *CmpLHS = Cmp->getOperand(i_nocapture: 0);
659 Value *CmpRHS = Cmp->getOperand(i_nocapture: 1);
660 ICmpInst::Predicate Pred = Cmp->getPredicate();
661 if (match(V: FVal, P: m_Zero())) {
662 std::swap(a&: TVal, b&: FVal);
663 Pred = ICmpInst::getInversePredicate(pred: Pred);
664 }
665 if (!match(V: TVal, P: m_Zero()))
666 return nullptr;
667
668 if (Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE) {
669 std::swap(a&: CmpLHS, b&: CmpRHS);
670 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
671 }
672
673 // Handles:
674 // (X <= Y) ? 0 : (X - Y)
675 // (X <= Y) ? (Y - X) : 0
676 // (X >= Y) ? 0 : (Y - X)
677 // (X >= Y) ? (X - Y) : 0
678 if ((Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE) &&
679 match(V: FVal, P: m_NSWSub(L: m_Specific(V: CmpLHS), R: m_Specific(V: CmpRHS))) &&
680 isGuaranteedNotToBeUndef(V: CmpLHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)) {
681 Value *SMin =
682 Builder.CreateBinaryIntrinsic(ID: Intrinsic::smin, LHS: CmpRHS, RHS: CmpLHS);
683 return Builder.CreateNSWSub(LHS: CmpLHS, RHS: SMin);
684 }
685
686 return nullptr;
687}
688
689/// Try to fold a select to a min/max intrinsic. Many cases are already handled
690/// by matchDecomposedSelectPattern but here we handle the cases where more
691/// extensive modification of the IR is required.
692static Value *foldSelectICmpMinMax(const ICmpInst *Cmp, Value *TVal,
693 Value *FVal,
694 InstCombiner::BuilderTy &Builder,
695 const SimplifyQuery &SQ) {
696 Value *CmpLHS = Cmp->getOperand(i_nocapture: 0);
697 Value *CmpRHS = Cmp->getOperand(i_nocapture: 1);
698 ICmpInst::Predicate Pred = Cmp->getPredicate();
699
700 if (Value *V = canoncalizeSelectICmpMinMax(Cmp, TVal, FVal, Builder, SQ))
701 return V;
702
703 // (X > Y) ? X : (Y - 1) ==> MIN(X, Y - 1)
704 // (X < Y) ? X : (Y + 1) ==> MAX(X, Y + 1)
705 // This transformation is valid when overflow corresponding to the sign of
706 // the comparison is poison and we must drop the non-matching overflow flag.
707 if (CmpRHS == TVal) {
708 std::swap(a&: CmpLHS, b&: CmpRHS);
709 Pred = CmpInst::getSwappedPredicate(pred: Pred);
710 }
711
712 // TODO: consider handling 'or disjoint' as well, though these would need to
713 // be converted to 'add' instructions.
714 if (!(CmpLHS == TVal && isa<Instruction>(Val: FVal)))
715 return nullptr;
716
717 if (Pred == CmpInst::ICMP_SGT &&
718 match(V: FVal, P: m_NSWAdd(L: m_Specific(V: CmpRHS), R: m_One()))) {
719 cast<Instruction>(Val: FVal)->setHasNoUnsignedWrap(false);
720 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::smax, LHS: TVal, RHS: FVal);
721 }
722
723 if (Pred == CmpInst::ICMP_SLT &&
724 match(V: FVal, P: m_NSWAdd(L: m_Specific(V: CmpRHS), R: m_AllOnes()))) {
725 cast<Instruction>(Val: FVal)->setHasNoUnsignedWrap(false);
726 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::smin, LHS: TVal, RHS: FVal);
727 }
728
729 if (Pred == CmpInst::ICMP_UGT &&
730 match(V: FVal, P: m_NUWAdd(L: m_Specific(V: CmpRHS), R: m_One()))) {
731 cast<Instruction>(Val: FVal)->setHasNoSignedWrap(false);
732 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::umax, LHS: TVal, RHS: FVal);
733 }
734
735 // Note: We must use isKnownNonZero here because "sub nuw %x, 1" will be
736 // canonicalized to "add %x, -1" discarding the nuw flag.
737 if (Pred == CmpInst::ICMP_ULT &&
738 match(V: FVal, P: m_Add(L: m_Specific(V: CmpRHS), R: m_AllOnes())) &&
739 isKnownNonZero(V: CmpRHS, Q: SQ)) {
740 cast<Instruction>(Val: FVal)->setHasNoSignedWrap(false);
741 cast<Instruction>(Val: FVal)->setHasNoUnsignedWrap(false);
742 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: TVal, RHS: FVal);
743 }
744
745 return nullptr;
746}
747
748/// We want to turn:
749/// (select (icmp eq (and X, Y), 0), (and (lshr X, Z), 1), 1)
750/// into:
751/// zext (icmp ne i32 (and X, (or Y, (shl 1, Z))), 0)
752/// Note:
753/// Z may be 0 if lshr is missing.
754/// Worst-case scenario is that we will replace 5 instructions with 5 different
755/// instructions, but we got rid of select.
756static Instruction *foldSelectICmpAndAnd(Type *SelType, const Value *Cond,
757 Value *TVal, Value *FVal,
758 InstCombiner::BuilderTy &Builder) {
759 Value *A, *X, *Y, *Z;
760 CmpPredicate Pred;
761 unsigned NumReplaced = 1 + Cond->hasOneUse();
762 if (match(V: Cond, P: m_Trunc(Op: m_Value(V&: X)))) {
763 Y = ConstantInt::get(Ty: X->getType(), V: 1);
764 Pred = ICmpInst::ICMP_NE;
765 } else if (match(V: Cond,
766 P: m_ICmp(Pred, L: m_And(L: m_Value(V&: X), R: m_Value(V&: Y)), R: m_Zero())) &&
767 ICmpInst::isEquality(P: Pred)) {
768 NumReplaced +=
769 Cond->hasOneUse() && cast<ICmpInst>(Val: Cond)->getOperand(i_nocapture: 0)->hasOneUse();
770 } else
771 return nullptr;
772
773 if (Pred == ICmpInst::ICMP_NE)
774 std::swap(a&: TVal, b&: FVal);
775
776 if (!match(V: FVal, P: m_One()))
777 return nullptr;
778
779 // The TrueVal has general form of: and %B, 1
780 if (!match(V: TVal, P: m_And(L: m_Value(V&: A), R: m_One())))
781 return nullptr;
782
783 APInt BitWidth(SelType->getScalarSizeInBits(),
784 SelType->getScalarSizeInBits());
785 auto TValPattern = m_CombineOr(
786 Ps: m_Deferred(V: X),
787 Ps: m_LShr(L: m_Deferred(V: X), R: m_Value(V&: Z, P: m_SpecificInt_ICMP_ForbidPoison(
788 Predicate: CmpInst::ICMP_ULT, Threshold: BitWidth))));
789
790 if (!match(V: A, P: TValPattern)) {
791 std::swap(a&: X, b&: Y);
792 if (!match(V: A, P: TValPattern))
793 return nullptr;
794 }
795
796 bool HasShift = A != X;
797 if (TVal->hasOneUse())
798 NumReplaced += 1 + (HasShift && A->hasOneUse());
799
800 if (NumReplaced < (4u - isa<Constant>(Val: Y)))
801 return nullptr;
802
803 // ((X & Y) == 0) ? ((X >> Z) & 1) : 1 --> (X & (Y | (1 << Z))) != 0
804 // ((X & Y) == 0) ? (X & 1) : 1 --> (X & (Y | 1)) != 0
805 Constant *One = ConstantInt::get(Ty: SelType, V: 1);
806 Value *MaskB = HasShift ? Builder.CreateShl(LHS: One, RHS: Z) : One;
807 Value *FullMask = Builder.CreateOr(LHS: Y, RHS: MaskB);
808 Value *MaskedX = Builder.CreateAnd(LHS: X, RHS: FullMask);
809 Value *ICmpNeZero = Builder.CreateIsNotNull(Arg: MaskedX);
810 return new ZExtInst(ICmpNeZero, SelType);
811}
812
813/// We want to turn:
814/// (select (icmp eq (and X, C1), 0), 0, (shl [nsw/nuw] X, C2));
815/// iff C1 is a mask and the number of its leading zeros is equal to C2
816/// into:
817/// shl X, C2
818static Value *foldSelectICmpAndZeroShl(const ICmpInst *Cmp, Value *TVal,
819 Value *FVal,
820 InstCombiner::BuilderTy &Builder) {
821 CmpPredicate Pred;
822 Value *AndVal;
823 if (!match(V: Cmp, P: m_ICmp(Pred, L: m_Value(V&: AndVal), R: m_Zero())))
824 return nullptr;
825
826 if (Pred == ICmpInst::ICMP_NE) {
827 Pred = ICmpInst::ICMP_EQ;
828 std::swap(a&: TVal, b&: FVal);
829 }
830
831 Value *X;
832 const APInt *C2, *C1;
833 if (Pred != ICmpInst::ICMP_EQ ||
834 !match(V: AndVal, P: m_And(L: m_Value(V&: X), R: m_APInt(Res&: C1))) ||
835 !match(V: TVal, P: m_Zero()) || !match(V: FVal, P: m_Shl(L: m_Specific(V: X), R: m_APInt(Res&: C2))))
836 return nullptr;
837
838 if (!C1->isMask() ||
839 C1->countLeadingZeros() != static_cast<unsigned>(C2->getZExtValue()))
840 return nullptr;
841
842 auto *FI = dyn_cast<Instruction>(Val: FVal);
843 if (!FI)
844 return nullptr;
845
846 FI->setHasNoSignedWrap(false);
847 FI->setHasNoUnsignedWrap(false);
848 return FVal;
849}
850
851/// We want to turn:
852/// (select (icmp sgt x, C), lshr (X, Y), ashr (X, Y)); iff C s>= -1
853/// (select (icmp slt x, C), ashr (X, Y), lshr (X, Y)); iff C s>= 0
854/// into:
855/// ashr (X, Y)
856static Value *foldSelectICmpLshrAshr(const ICmpInst *IC, Value *TrueVal,
857 Value *FalseVal,
858 InstCombiner::BuilderTy &Builder) {
859 ICmpInst::Predicate Pred = IC->getPredicate();
860 Value *CmpLHS = IC->getOperand(i_nocapture: 0);
861 Value *CmpRHS = IC->getOperand(i_nocapture: 1);
862 if (!CmpRHS->getType()->isIntOrIntVectorTy())
863 return nullptr;
864
865 Value *X, *Y;
866 unsigned Bitwidth = CmpRHS->getType()->getScalarSizeInBits();
867 if ((Pred != ICmpInst::ICMP_SGT ||
868 !match(V: CmpRHS, P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_SGE,
869 Threshold: APInt::getAllOnes(numBits: Bitwidth)))) &&
870 (Pred != ICmpInst::ICMP_SLT ||
871 !match(V: CmpRHS, P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_SGE,
872 Threshold: APInt::getZero(numBits: Bitwidth)))))
873 return nullptr;
874
875 // Canonicalize so that ashr is in FalseVal.
876 if (Pred == ICmpInst::ICMP_SLT)
877 std::swap(a&: TrueVal, b&: FalseVal);
878
879 if (match(V: TrueVal, P: m_LShr(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
880 match(V: FalseVal, P: m_AShr(L: m_Specific(V: X), R: m_Specific(V: Y))) &&
881 match(V: CmpLHS, P: m_Specific(V: X))) {
882 const auto *Ashr = cast<Instruction>(Val: FalseVal);
883 // if lshr is not exact and ashr is, this new ashr must not be exact.
884 bool IsExact = Ashr->isExact() && cast<Instruction>(Val: TrueVal)->isExact();
885 return Builder.CreateAShr(LHS: X, RHS: Y, Name: IC->getName(), isExact: IsExact);
886 }
887
888 return nullptr;
889}
890
891/// We want to turn:
892/// (select (icmp eq (and X, C1), 0), Y, (BinOp Y, C2))
893/// into:
894/// IF C2 u>= C1
895/// (BinOp Y, (shl (and X, C1), C3))
896/// ELSE
897/// (BinOp Y, (lshr (and X, C1), C3))
898/// iff:
899/// 0 on the RHS is the identity value (i.e add, xor, shl, etc...)
900/// C1 and C2 are both powers of 2
901/// where:
902/// IF C2 u>= C1
903/// C3 = Log(C2) - Log(C1)
904/// ELSE
905/// C3 = Log(C1) - Log(C2)
906///
907/// This transform handles cases where:
908/// 1. The icmp predicate is inverted
909/// 2. The select operands are reversed
910/// 3. The magnitude of C2 and C1 are flipped
911static Value *foldSelectICmpAndBinOp(Value *CondVal, Value *TrueVal,
912 Value *FalseVal, Value *V,
913 const APInt &AndMask, bool CreateAnd,
914 InstCombiner::BuilderTy &Builder) {
915 // Only handle integer compares.
916 if (!TrueVal->getType()->isIntOrIntVectorTy())
917 return nullptr;
918
919 unsigned C1Log = AndMask.logBase2();
920 Value *Y;
921 BinaryOperator *BinOp;
922 const APInt *C2;
923 bool NeedXor;
924 if (match(V: FalseVal, P: m_BinOp(L: m_Specific(V: TrueVal), R: m_Power2(V&: C2)))) {
925 Y = TrueVal;
926 BinOp = cast<BinaryOperator>(Val: FalseVal);
927 NeedXor = false;
928 } else if (match(V: TrueVal, P: m_BinOp(L: m_Specific(V: FalseVal), R: m_Power2(V&: C2)))) {
929 Y = FalseVal;
930 BinOp = cast<BinaryOperator>(Val: TrueVal);
931 NeedXor = true;
932 } else {
933 return nullptr;
934 }
935
936 // Check that 0 on RHS is identity value for this binop.
937 auto *IdentityC =
938 ConstantExpr::getBinOpIdentity(Opcode: BinOp->getOpcode(), Ty: BinOp->getType(),
939 /*AllowRHSConstant*/ true);
940 if (IdentityC == nullptr || !IdentityC->isNullValue())
941 return nullptr;
942
943 unsigned C2Log = C2->logBase2();
944
945 bool NeedShift = C1Log != C2Log;
946 bool NeedZExtTrunc = Y->getType()->getScalarSizeInBits() !=
947 V->getType()->getScalarSizeInBits();
948
949 // the demanded bits for the created shl make the and redundant
950 if (AndMask.isOne() && C2->isSignBitSet())
951 CreateAnd = false;
952
953 // Make sure we don't create more instructions than we save.
954 if ((NeedShift + NeedXor + NeedZExtTrunc + CreateAnd) >
955 (CondVal->hasOneUse() + BinOp->hasOneUse()))
956 return nullptr;
957
958 if (CreateAnd) {
959 // Insert the AND instruction on the input to the truncate.
960 V = Builder.CreateAnd(LHS: V, RHS: ConstantInt::get(Ty: V->getType(), V: AndMask));
961 }
962
963 if (C2Log > C1Log) {
964 V = Builder.CreateZExtOrTrunc(V, DestTy: Y->getType());
965 V = Builder.CreateShl(LHS: V, RHS: C2Log - C1Log);
966 } else if (C1Log > C2Log) {
967 V = Builder.CreateLShr(LHS: V, RHS: C1Log - C2Log);
968 V = Builder.CreateZExtOrTrunc(V, DestTy: Y->getType());
969 } else
970 V = Builder.CreateZExtOrTrunc(V, DestTy: Y->getType());
971
972 if (NeedXor)
973 V = Builder.CreateXor(LHS: V, RHS: *C2);
974
975 auto *Res = Builder.CreateBinOp(Opc: BinOp->getOpcode(), LHS: Y, RHS: V);
976 if (auto *BO = dyn_cast<BinaryOperator>(Val: Res))
977 BO->copyIRFlags(V: BinOp);
978 return Res;
979}
980
981/// Canonicalize a set or clear of a masked set of constant bits to
982/// select-of-constants form.
983static Instruction *foldSetClearBits(SelectInst &Sel,
984 InstCombiner::BuilderTy &Builder) {
985 Value *Cond = Sel.getCondition();
986 Value *T = Sel.getTrueValue();
987 Value *F = Sel.getFalseValue();
988 Type *Ty = Sel.getType();
989 Value *X;
990 const APInt *NotC, *C;
991
992 // Cond ? (X & ~C) : (X | C) --> (X & ~C) | (Cond ? 0 : C)
993 if (match(V: T, P: m_And(L: m_Value(V&: X), R: m_APInt(Res&: NotC))) &&
994 match(V: F, P: m_OneUse(SubPattern: m_Or(L: m_Specific(V: X), R: m_APInt(Res&: C)))) && *NotC == ~(*C)) {
995 Constant *Zero = ConstantInt::getNullValue(Ty);
996 Constant *OrC = ConstantInt::get(Ty, V: *C);
997 Value *NewSel = Builder.CreateSelect(C: Cond, True: Zero, False: OrC, Name: "masksel", MDFrom: &Sel);
998 return BinaryOperator::CreateOr(V1: T, V2: NewSel);
999 }
1000
1001 // Cond ? (X | C) : (X & ~C) --> (X & ~C) | (Cond ? C : 0)
1002 if (match(V: F, P: m_And(L: m_Value(V&: X), R: m_APInt(Res&: NotC))) &&
1003 match(V: T, P: m_OneUse(SubPattern: m_Or(L: m_Specific(V: X), R: m_APInt(Res&: C)))) && *NotC == ~(*C)) {
1004 Constant *Zero = ConstantInt::getNullValue(Ty);
1005 Constant *OrC = ConstantInt::get(Ty, V: *C);
1006 Value *NewSel = Builder.CreateSelect(C: Cond, True: OrC, False: Zero, Name: "masksel", MDFrom: &Sel);
1007 return BinaryOperator::CreateOr(V1: F, V2: NewSel);
1008 }
1009
1010 return nullptr;
1011}
1012
1013// select (x == 0), 0, x * y --> freeze(y) * x
1014// select (y == 0), 0, x * y --> freeze(x) * y
1015// select (x == 0), undef, x * y --> freeze(y) * x
1016// select (x == undef), 0, x * y --> freeze(y) * x
1017// Usage of mul instead of 0 will make the result more poisonous,
1018// so the operand that was not checked in the condition should be frozen.
1019// The latter folding is applied only when a constant compared with x is
1020// is a vector consisting of 0 and undefs. If a constant compared with x
1021// is a scalar undefined value or undefined vector then an expression
1022// should be already folded into a constant.
1023//
1024// This also holds all operations such that Op(0) == 0
1025// e.g. Shl, Umin, etc
1026static Instruction *foldSelectZeroOrFixedOp(SelectInst &SI,
1027 InstCombinerImpl &IC) {
1028 auto *CondVal = SI.getCondition();
1029 auto *TrueVal = SI.getTrueValue();
1030 auto *FalseVal = SI.getFalseValue();
1031 Value *X, *Y;
1032 CmpPredicate Predicate;
1033
1034 // Assuming that constant compared with zero is not undef (but it may be
1035 // a vector with some undef elements). Otherwise (when a constant is undef)
1036 // the select expression should be already simplified.
1037 if (!match(V: CondVal, P: m_ICmp(Pred&: Predicate, L: m_Value(V&: X), R: m_Zero())) ||
1038 !ICmpInst::isEquality(P: Predicate))
1039 return nullptr;
1040
1041 if (Predicate == ICmpInst::ICMP_NE)
1042 std::swap(a&: TrueVal, b&: FalseVal);
1043
1044 // Check that TrueVal is a constant instead of matching it with m_Zero()
1045 // to handle the case when it is a scalar undef value or a vector containing
1046 // non-zero elements that are masked by undef elements in the compare
1047 // constant.
1048 auto *TrueValC = dyn_cast<Constant>(Val: TrueVal);
1049 if (TrueValC == nullptr || !isa<Instruction>(Val: FalseVal))
1050 return nullptr;
1051
1052 bool FreezeY;
1053 if (match(V: FalseVal, P: m_c_Mul(L: m_Specific(V: X), R: m_Value(V&: Y))) ||
1054 match(V: FalseVal, P: m_c_And(L: m_Specific(V: X), R: m_Value(V&: Y))) ||
1055 match(V: FalseVal, P: m_FShl(Op0: m_Specific(V: X), Op1: m_Specific(V: X), Op2: m_Value(V&: Y))) ||
1056 match(V: FalseVal, P: m_FShr(Op0: m_Specific(V: X), Op1: m_Specific(V: X), Op2: m_Value(V&: Y))) ||
1057 match(V: FalseVal,
1058 P: m_c_Intrinsic<Intrinsic::umin>(Op0: m_Specific(V: X), Op1: m_Value(V&: Y)))) {
1059 FreezeY = true;
1060 } else if (match(V: FalseVal, P: m_IDiv(L: m_Specific(V: X), R: m_Value(V&: Y))) ||
1061 match(V: FalseVal, P: m_IRem(L: m_Specific(V: X), R: m_Value(V&: Y)))) {
1062 FreezeY = false;
1063 } else {
1064 return nullptr;
1065 }
1066
1067 auto *ZeroC = cast<Constant>(Val: cast<Instruction>(Val: CondVal)->getOperand(i: 1));
1068 auto *MergedC = Constant::mergeUndefsWith(C: TrueValC, Other: ZeroC);
1069 // If X is compared with 0 then TrueVal could be either zero or undef.
1070 // m_Zero match vectors containing some undef elements, but for scalars
1071 // m_Undef should be used explicitly.
1072 if (!match(V: MergedC, P: m_Zero()) && !match(V: MergedC, P: m_Undef()))
1073 return nullptr;
1074
1075 auto *FalseValI = cast<Instruction>(Val: FalseVal);
1076 if (FreezeY) {
1077 auto *FrY = IC.InsertNewInstBefore(New: new FreezeInst(Y, Y->getName() + ".fr"),
1078 Old: FalseValI->getIterator());
1079 IC.replaceOperand(I&: *FalseValI,
1080 OpNum: FalseValI->getOperand(i: 0) == Y
1081 ? 0
1082 : (FalseValI->getOperand(i: 1) == Y ? 1 : 2),
1083 V: FrY);
1084 }
1085 return IC.replaceInstUsesWith(I&: SI, V: FalseValI);
1086}
1087
1088/// Transform patterns such as (a > b) ? a - b : 0 into usub.sat(a, b).
1089/// There are 8 commuted/swapped variants of this pattern.
1090static Value *
1091canonicalizeSaturatedSubtractUnsigned(const ICmpInst *ICI, const Value *TrueVal,
1092 const Value *FalseVal,
1093 InstCombiner::BuilderTy &Builder) {
1094 ICmpInst::Predicate Pred = ICI->getPredicate();
1095 Value *A = ICI->getOperand(i_nocapture: 0);
1096 Value *B = ICI->getOperand(i_nocapture: 1);
1097
1098 // (b > a) ? 0 : a - b -> (b <= a) ? a - b : 0
1099 // (a == 0) ? 0 : a - 1 -> (a != 0) ? a - 1 : 0
1100 if (match(V: TrueVal, P: m_Zero())) {
1101 Pred = ICmpInst::getInversePredicate(pred: Pred);
1102 std::swap(a&: TrueVal, b&: FalseVal);
1103 }
1104
1105 if (!match(V: FalseVal, P: m_Zero()))
1106 return nullptr;
1107
1108 // ugt 0 is canonicalized to ne 0 and requires special handling
1109 // (a != 0) ? a + -1 : 0 -> usub.sat(a, 1)
1110 if (Pred == ICmpInst::ICMP_NE) {
1111 if (match(V: B, P: m_Zero()) && match(V: TrueVal, P: m_Add(L: m_Specific(V: A), R: m_AllOnes())))
1112 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::usub_sat, LHS: A,
1113 RHS: ConstantInt::get(Ty: A->getType(), V: 1));
1114 return nullptr;
1115 }
1116
1117 if (!ICmpInst::isUnsigned(Pred))
1118 return nullptr;
1119
1120 if (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_ULT) {
1121 // (b < a) ? a - b : 0 -> (a > b) ? a - b : 0
1122 std::swap(a&: A, b&: B);
1123 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
1124 }
1125
1126 assert((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_UGT) &&
1127 "Unexpected isUnsigned predicate!");
1128
1129 // Ensure the sub is of the form:
1130 // (a > b) ? a - b : 0 -> usub.sat(a, b)
1131 // (a > b) ? b - a : 0 -> -usub.sat(a, b)
1132 // Checking for both a-b and a+(-b) as a constant.
1133 bool IsNegative = false;
1134 const APInt *C;
1135 if (match(V: TrueVal, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))) ||
1136 (match(V: A, P: m_APInt(Res&: C)) &&
1137 match(V: TrueVal, P: m_Add(L: m_Specific(V: B), R: m_SpecificInt(V: -*C)))))
1138 IsNegative = true;
1139 else if (!match(V: TrueVal, P: m_Sub(L: m_Specific(V: A), R: m_Specific(V: B))) &&
1140 !(match(V: B, P: m_APInt(Res&: C)) &&
1141 match(V: TrueVal, P: m_Add(L: m_Specific(V: A), R: m_SpecificInt(V: -*C)))))
1142 return nullptr;
1143
1144 // If we are adding a negate and the sub and icmp are used anywhere else, we
1145 // would end up with more instructions.
1146 if (IsNegative && !TrueVal->hasOneUse() && !ICI->hasOneUse())
1147 return nullptr;
1148
1149 // (a > b) ? a - b : 0 -> usub.sat(a, b)
1150 // (a > b) ? b - a : 0 -> -usub.sat(a, b)
1151 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::usub_sat, LHS: A, RHS: B);
1152 if (IsNegative)
1153 Result = Builder.CreateNeg(V: Result);
1154 return Result;
1155}
1156
1157static Value *
1158canonicalizeSaturatedSubtractSigned(const ICmpInst *ICI, const Value *TrueVal,
1159 const Value *FalseVal,
1160 InstCombiner::BuilderTy &Builder) {
1161 ICmpInst::Predicate Pred = ICI->getPredicate();
1162 Value *CmpLHS = ICI->getOperand(i_nocapture: 0);
1163 Value *CmpRHS = ICI->getOperand(i_nocapture: 1);
1164
1165 // `A != B ? X : Y` --> `A == B ? Y : X`
1166 // This canonicalization allows us to handle more patterns with fewer checks.
1167 if (Pred == ICmpInst::ICMP_NE) {
1168 Pred = ICmpInst::ICMP_EQ;
1169 std::swap(a&: TrueVal, b&: FalseVal);
1170 }
1171
1172 // `A == MIN_INT ? MAX_INT : 0 - A` --> `ssub_sat 0, A`
1173 if (Pred == ICmpInst::ICMP_EQ && match(V: CmpRHS, P: m_SignMask()) &&
1174 match(V: TrueVal, P: m_MaxSignedValue()) &&
1175 match(V: FalseVal, P: m_Neg(V: m_Specific(V: CmpLHS)))) {
1176 return Builder.CreateBinaryIntrinsic(
1177 ID: Intrinsic::ssub_sat, LHS: ConstantInt::getNullValue(Ty: CmpLHS->getType()),
1178 RHS: CmpLHS);
1179 }
1180
1181 return nullptr;
1182}
1183
1184static Value *canonicalizeSaturatedSubtract(const ICmpInst *ICI,
1185 const Value *TrueVal,
1186 const Value *FalseVal,
1187 InstCombiner::BuilderTy &Builder) {
1188 if (Value *V = canonicalizeSaturatedSubtractUnsigned(ICI, TrueVal, FalseVal,
1189 Builder))
1190 return V;
1191
1192 if (Value *V =
1193 canonicalizeSaturatedSubtractSigned(ICI, TrueVal, FalseVal, Builder))
1194 return V;
1195
1196 return nullptr;
1197}
1198
1199static Value *
1200canonicalizeSaturatedAddUnsigned(ICmpInst *Cmp, Value *TVal, Value *FVal,
1201 InstCombiner::BuilderTy &Builder) {
1202
1203 // Match unsigned saturated add with constant.
1204 Value *Cmp0 = Cmp->getOperand(i_nocapture: 0);
1205 Value *Cmp1 = Cmp->getOperand(i_nocapture: 1);
1206 ICmpInst::Predicate Pred = Cmp->getPredicate();
1207 Value *X;
1208 const APInt *C;
1209
1210 // Match unsigned saturated add of 2 variables with an unnecessary 'not'.
1211 // There are 8 commuted variants.
1212 // Canonicalize -1 (saturated result) to true value of the select.
1213 if (match(V: FVal, P: m_AllOnes())) {
1214 std::swap(a&: TVal, b&: FVal);
1215 Pred = CmpInst::getInversePredicate(pred: Pred);
1216 }
1217 if (!match(V: TVal, P: m_AllOnes()))
1218 return nullptr;
1219
1220 // uge -1 is canonicalized to eq -1 and requires special handling
1221 // (a == -1) ? -1 : a + 1 -> uadd.sat(a, 1)
1222 if (Pred == ICmpInst::ICMP_EQ) {
1223 if (match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_One())) &&
1224 match(V: Cmp1, P: m_AllOnes())) {
1225 return Builder.CreateBinaryIntrinsic(
1226 ID: Intrinsic::uadd_sat, LHS: Cmp0, RHS: ConstantInt::get(Ty: Cmp0->getType(), V: 1));
1227 }
1228 return nullptr;
1229 }
1230
1231 if ((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_UGT) &&
1232 match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_APIntAllowPoison(Res&: C))) &&
1233 match(V: Cmp1, P: m_SpecificIntAllowPoison(V: ~*C))) {
1234 // (X u> ~C) ? -1 : (X + C) --> uadd.sat(X, C)
1235 // (X u>= ~C)? -1 : (X + C) --> uadd.sat(X, C)
1236 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::uadd_sat, LHS: Cmp0,
1237 RHS: ConstantInt::get(Ty: Cmp0->getType(), V: *C));
1238 }
1239
1240 // Negative one does not work here because X u> -1 ? -1, X + -1 is not a
1241 // saturated add.
1242 if (Pred == ICmpInst::ICMP_UGT &&
1243 match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_APIntAllowPoison(Res&: C))) &&
1244 match(V: Cmp1, P: m_SpecificIntAllowPoison(V: ~*C - 1)) && !C->isAllOnes()) {
1245 // (X u> ~C - 1) ? -1 : (X + C) --> uadd.sat(X, C)
1246 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::uadd_sat, LHS: Cmp0,
1247 RHS: ConstantInt::get(Ty: Cmp0->getType(), V: *C));
1248 }
1249
1250 // Zero does not work here because X u>= 0 ? -1 : X -> is always -1, which is
1251 // not a saturated add.
1252 if (Pred == ICmpInst::ICMP_UGE &&
1253 match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_APIntAllowPoison(Res&: C))) &&
1254 match(V: Cmp1, P: m_SpecificIntAllowPoison(V: -*C)) && !C->isZero()) {
1255 // (X u >= -C) ? -1 : (X + C) --> uadd.sat(X, C)
1256 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::uadd_sat, LHS: Cmp0,
1257 RHS: ConstantInt::get(Ty: Cmp0->getType(), V: *C));
1258 }
1259
1260 // Canonicalize predicate to less-than or less-or-equal-than.
1261 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
1262 std::swap(a&: Cmp0, b&: Cmp1);
1263 Pred = CmpInst::getSwappedPredicate(pred: Pred);
1264 }
1265 if (Pred != ICmpInst::ICMP_ULT && Pred != ICmpInst::ICMP_ULE)
1266 return nullptr;
1267
1268 // Match unsigned saturated add of 2 variables with an unnecessary 'not'.
1269 // Strictness of the comparison is irrelevant.
1270 Value *Y;
1271 if (match(V: Cmp0, P: m_Not(V: m_Value(V&: X))) &&
1272 match(V: FVal, P: m_c_Add(L: m_Specific(V: X), R: m_Value(V&: Y))) && Y == Cmp1) {
1273 // (~X u< Y) ? -1 : (X + Y) --> uadd.sat(X, Y)
1274 // (~X u< Y) ? -1 : (Y + X) --> uadd.sat(X, Y)
1275 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::uadd_sat, LHS: X, RHS: Y);
1276 }
1277 // The 'not' op may be included in the sum but not the compare.
1278 // Strictness of the comparison is irrelevant.
1279 X = Cmp0;
1280 Y = Cmp1;
1281 if (match(V: FVal, P: m_c_Add(L: m_NotForbidPoison(V: m_Specific(V: X)), R: m_Specific(V: Y)))) {
1282 // (X u< Y) ? -1 : (~X + Y) --> uadd.sat(~X, Y)
1283 // (X u< Y) ? -1 : (Y + ~X) --> uadd.sat(Y, ~X)
1284 BinaryOperator *BO = cast<BinaryOperator>(Val: FVal);
1285 return Builder.CreateBinaryIntrinsic(
1286 ID: Intrinsic::uadd_sat, LHS: BO->getOperand(i_nocapture: 0), RHS: BO->getOperand(i_nocapture: 1));
1287 }
1288 // The overflow may be detected via the add wrapping round.
1289 // This is only valid for strict comparison!
1290 if (Pred == ICmpInst::ICMP_ULT &&
1291 match(V: Cmp0, P: m_c_Add(L: m_Specific(V: Cmp1), R: m_Value(V&: Y))) &&
1292 match(V: FVal, P: m_c_Add(L: m_Specific(V: Cmp1), R: m_Specific(V: Y)))) {
1293 // ((X + Y) u< X) ? -1 : (X + Y) --> uadd.sat(X, Y)
1294 // ((X + Y) u< Y) ? -1 : (X + Y) --> uadd.sat(X, Y)
1295 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::uadd_sat, LHS: Cmp1, RHS: Y);
1296 }
1297
1298 return nullptr;
1299}
1300
1301static Value *canonicalizeSaturatedAddSigned(ICmpInst *Cmp, Value *TVal,
1302 Value *FVal,
1303 InstCombiner::BuilderTy &Builder) {
1304 // Match saturated add with constant.
1305 Value *Cmp0 = Cmp->getOperand(i_nocapture: 0);
1306 Value *Cmp1 = Cmp->getOperand(i_nocapture: 1);
1307 ICmpInst::Predicate Pred = Cmp->getPredicate();
1308
1309 // Canonicalize TVal to be the saturation constant.
1310 if (match(V: FVal, P: m_MaxSignedValue()) || match(V: FVal, P: m_SignMask())) {
1311 std::swap(a&: TVal, b&: FVal);
1312 Pred = CmpInst::getInversePredicate(pred: Pred);
1313 }
1314
1315 const APInt *SatC;
1316 if (!match(V: TVal, P: m_APInt(Res&: SatC)) ||
1317 !(SatC->isMaxSignedValue() || SatC->isSignMask()))
1318 return nullptr;
1319
1320 bool IsMax = SatC->isMaxSignedValue();
1321
1322 // sge maximum signed value is canonicalized to eq maximum signed value and
1323 // requires special handling. sle minimum signed value is similarly
1324 // canonicalized to eq minimum signed value.
1325 if (Pred == ICmpInst::ICMP_EQ && Cmp1 == TVal) {
1326 // (a == INT_MAX) ? INT_MAX : a + 1 -> sadd.sat(a, 1)
1327 if (IsMax && match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_One()))) {
1328 return Builder.CreateBinaryIntrinsic(
1329 ID: Intrinsic::sadd_sat, LHS: Cmp0, RHS: ConstantInt::get(Ty: Cmp0->getType(), V: 1));
1330 }
1331
1332 // (a == INT_MIN) ? INT_MIN : a + -1 -> sadd.sat(a, -1)
1333 if (!IsMax && match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_AllOnes()))) {
1334 return Builder.CreateBinaryIntrinsic(
1335 ID: Intrinsic::sadd_sat, LHS: Cmp0,
1336 RHS: ConstantInt::getAllOnesValue(Ty: Cmp0->getType()));
1337 }
1338 return nullptr;
1339 }
1340
1341 const APInt *C;
1342
1343 // (X > Y) ? INT_MAX : (X + C) --> sadd.sat(X, C)
1344 // (X >= Y) ? INT_MAX : (X + C) --> sadd.sat(X, C)
1345 // where C > 0 and Y is INT_MAX - C or INT_MAX - C - 1
1346 if (IsMax && (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) &&
1347 isa<Constant>(Val: Cmp1) &&
1348 match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_StrictlyPositive(V&: C)))) {
1349 // Normalize SGE to SGT for threshold comparison.
1350 if (Pred == ICmpInst::ICMP_SGE) {
1351 if (auto Flipped = getFlippedStrictnessPredicateAndConstant(
1352 Pred, C: cast<Constant>(Val: Cmp1))) {
1353 Pred = Flipped->first;
1354 Cmp1 = Flipped->second;
1355 }
1356 }
1357 // Check: X > INT_MAX - C or X > INT_MAX - C - 1
1358 APInt Threshold = *SatC - *C;
1359 if (Pred == ICmpInst::ICMP_SGT &&
1360 (match(V: Cmp1, P: m_SpecificIntAllowPoison(V: Threshold)) ||
1361 match(V: Cmp1, P: m_SpecificIntAllowPoison(V: Threshold - 1))))
1362 return Builder.CreateBinaryIntrinsic(
1363 ID: Intrinsic::sadd_sat, LHS: Cmp0, RHS: ConstantInt::get(Ty: Cmp0->getType(), V: *C));
1364 }
1365
1366 // (X < Y) ? INT_MIN : (X + C) --> sadd.sat(X, C)
1367 // (X <= Y) ? INT_MIN : (X + C) --> sadd.sat(X, C)
1368 // where C < 0 and Y is INT_MIN - C or INT_MIN - C + 1
1369 if (!IsMax && (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) &&
1370 isa<Constant>(Val: Cmp1) &&
1371 match(V: FVal, P: m_Add(L: m_Specific(V: Cmp0), R: m_Negative(V&: C)))) {
1372 // Normalize SLE to SLT for threshold comparison.
1373 if (Pred == ICmpInst::ICMP_SLE) {
1374 if (auto Flipped = getFlippedStrictnessPredicateAndConstant(
1375 Pred, C: cast<Constant>(Val: Cmp1))) {
1376 Pred = Flipped->first;
1377 Cmp1 = Flipped->second;
1378 }
1379 }
1380 // Check: X < INT_MIN - C or X < INT_MIN - C + 1
1381 // INT_MIN - C for negative C is like INT_MIN + |C|
1382 APInt Threshold = *SatC - *C;
1383 if (Pred == ICmpInst::ICMP_SLT &&
1384 (match(V: Cmp1, P: m_SpecificIntAllowPoison(V: Threshold)) ||
1385 match(V: Cmp1, P: m_SpecificIntAllowPoison(V: Threshold + 1))))
1386 return Builder.CreateBinaryIntrinsic(
1387 ID: Intrinsic::sadd_sat, LHS: Cmp0, RHS: ConstantInt::get(Ty: Cmp0->getType(), V: *C));
1388 }
1389
1390 // Canonicalize predicate to less-than or less-or-equal-than.
1391 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) {
1392 std::swap(a&: Cmp0, b&: Cmp1);
1393 Pred = CmpInst::getSwappedPredicate(pred: Pred);
1394 }
1395
1396 if (Pred != ICmpInst::ICMP_SLT && Pred != ICmpInst::ICMP_SLE)
1397 return nullptr;
1398
1399 Value *X;
1400
1401 // (INT_MAX - X s< Y) ? INT_MAX : (X + Y) --> sadd.sat(X, Y)
1402 // (INT_MAX - X s< Y) ? INT_MAX : (Y + X) --> sadd.sat(X, Y)
1403 if (IsMax && match(V: Cmp0, P: m_NSWSub(L: m_SpecificInt(V: *SatC), R: m_Value(V&: X))) &&
1404 match(V: FVal, P: m_c_Add(L: m_Specific(V: X), R: m_Specific(V: Cmp1)))) {
1405 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::sadd_sat, LHS: X, RHS: Cmp1);
1406 }
1407
1408 // (INT_MIN - X s> Y) ? INT_MIN : (X + Y) --> sadd.sat(X, Y)
1409 // (INT_MIN - X s> Y) ? INT_MIN : (Y + X) --> sadd.sat(X, Y)
1410 // After swapping operands from the SGT/SGE canonicalization above,
1411 // this becomes (Y s< INT_MIN - X).
1412 if (!IsMax && match(V: Cmp1, P: m_NSWSub(L: m_SpecificInt(V: *SatC), R: m_Value(V&: X))) &&
1413 match(V: FVal, P: m_c_Add(L: m_Specific(V: X), R: m_Specific(V: Cmp0)))) {
1414 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::sadd_sat, LHS: X, RHS: Cmp0);
1415 }
1416
1417 return nullptr;
1418}
1419
1420static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
1421 InstCombiner::BuilderTy &Builder) {
1422 if (!Cmp->hasOneUse())
1423 return nullptr;
1424
1425 if (Value *V = canonicalizeSaturatedAddUnsigned(Cmp, TVal, FVal, Builder))
1426 return V;
1427
1428 if (Value *V = canonicalizeSaturatedAddSigned(Cmp, TVal, FVal, Builder))
1429 return V;
1430
1431 return nullptr;
1432}
1433
1434/// Try to match patterns with select and subtract as absolute difference.
1435static Value *foldAbsDiff(ICmpInst *Cmp, Value *TVal, Value *FVal,
1436 InstCombiner::BuilderTy &Builder) {
1437 auto *TI = dyn_cast<Instruction>(Val: TVal);
1438 auto *FI = dyn_cast<Instruction>(Val: FVal);
1439 if (!TI || !FI)
1440 return nullptr;
1441
1442 // Normalize predicate to gt/lt rather than ge/le.
1443 ICmpInst::Predicate Pred = Cmp->getStrictPredicate();
1444 Value *A = Cmp->getOperand(i_nocapture: 0);
1445 Value *B = Cmp->getOperand(i_nocapture: 1);
1446
1447 // Normalize "A - B" as the true value of the select.
1448 if (match(V: FI, P: m_Sub(L: m_Specific(V: A), R: m_Specific(V: B)))) {
1449 std::swap(a&: FI, b&: TI);
1450 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
1451 }
1452
1453 // With any pair of no-wrap subtracts:
1454 // (A > B) ? (A - B) : (B - A) --> abs(A - B)
1455 if (Pred == CmpInst::ICMP_SGT &&
1456 match(V: TI, P: m_Sub(L: m_Specific(V: A), R: m_Specific(V: B))) &&
1457 match(V: FI, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))) &&
1458 (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap()) &&
1459 (FI->hasNoSignedWrap() || FI->hasNoUnsignedWrap())) {
1460 // The remaining subtract is not "nuw" any more.
1461 // If there's one use of the subtract (no other use than the use we are
1462 // about to replace), then we know that the sub is "nsw" in this context
1463 // even if it was only "nuw" before. If there's another use, then we can't
1464 // add "nsw" to the existing instruction because it may not be safe in the
1465 // other user's context.
1466 TI->setHasNoUnsignedWrap(false);
1467 if (!TI->hasNoSignedWrap())
1468 TI->setHasNoSignedWrap(TI->hasOneUse());
1469 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: TI, RHS: Builder.getTrue());
1470 }
1471
1472 // Match: (A > B) ? (A - B) : (0 - (A - B)) --> abs(A - B)
1473 if (Pred == CmpInst::ICMP_SGT &&
1474 match(V: TI, P: m_NSWSub(L: m_Specific(V: A), R: m_Specific(V: B))) &&
1475 match(V: FI, P: m_Neg(V: m_Specific(V: TI)))) {
1476 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: TI,
1477 RHS: Builder.getFalse());
1478 }
1479
1480 // Match: (A < B) ? (0 - (A - B)) : (A - B) --> abs(A - B)
1481 if (Pred == CmpInst::ICMP_SLT &&
1482 match(V: FI, P: m_NSWSub(L: m_Specific(V: A), R: m_Specific(V: B))) &&
1483 match(V: TI, P: m_Neg(V: m_Specific(V: FI)))) {
1484 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: FI,
1485 RHS: Builder.getFalse());
1486 }
1487
1488 // Match: (A > B) ? (0 - (B - A)) : (B - A) --> abs(B - A)
1489 if (Pred == CmpInst::ICMP_SGT &&
1490 match(V: FI, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A))) &&
1491 match(V: TI, P: m_Neg(V: m_Specific(V: FI)))) {
1492 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: FI,
1493 RHS: Builder.getFalse());
1494 }
1495
1496 // Match: (A < B) ? (B - A) : (0 - (B - A)) --> abs(B - A)
1497 if (Pred == CmpInst::ICMP_SLT &&
1498 match(V: TI, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A))) &&
1499 match(V: FI, P: m_Neg(V: m_Specific(V: TI)))) {
1500 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: TI,
1501 RHS: Builder.getFalse());
1502 }
1503
1504 return nullptr;
1505}
1506
1507/// Fold the following code sequence:
1508/// \code
1509/// int a = ctlz(x & -x);
1510// x ? 31 - a : 32;
1511/// \code
1512///
1513/// into:
1514/// cttz(x)
1515static Instruction *foldSelectCtlzToCttz(ICmpInst *ICI, Value *TrueVal,
1516 Value *FalseVal,
1517 InstCombiner::BuilderTy &Builder) {
1518 unsigned BitWidth = TrueVal->getType()->getScalarSizeInBits();
1519 if (!ICI->isEquality() || !match(V: ICI->getOperand(i_nocapture: 1), P: m_Zero()))
1520 return nullptr;
1521
1522 if (ICI->getPredicate() == ICmpInst::ICMP_NE)
1523 std::swap(a&: TrueVal, b&: FalseVal);
1524
1525 Value *Ctlz;
1526 if (match(V: FalseVal,
1527 P: m_Xor(L: m_Value(V&: Ctlz), R: m_SpecificIntAllowPoison(V: BitWidth - 1)))) {
1528 if (!isPowerOf2_32(Value: BitWidth))
1529 return nullptr;
1530 } else if (!match(V: FalseVal, P: m_Sub(L: m_SpecificIntAllowPoison(V: BitWidth - 1),
1531 R: m_Value(V&: Ctlz)))) {
1532 return nullptr;
1533 }
1534
1535 if (!match(V: Ctlz, P: m_Ctlz(Op0: m_Value(), Op1: m_Value())))
1536 return nullptr;
1537
1538 if (!match(V: TrueVal, P: m_SpecificInt(V: BitWidth)))
1539 return nullptr;
1540
1541 Value *X = ICI->getOperand(i_nocapture: 0);
1542 auto *II = cast<IntrinsicInst>(Val: Ctlz);
1543 if (!match(V: II->getOperand(i_nocapture: 0), P: m_c_And(L: m_Specific(V: X), R: m_Neg(V: m_Specific(V: X)))))
1544 return nullptr;
1545
1546 // The original select returns the constant bitwidth when x == 0, so the
1547 // result is defined there; the cttz must use is_zero_poison = false.
1548 Function *F = Intrinsic::getOrInsertDeclaration(
1549 M: II->getModule(), id: Intrinsic::cttz, OverloadTys: II->getType());
1550 return CallInst::Create(Func: F, Args: {X, Builder.getFalse()});
1551}
1552
1553/// Attempt to fold a cttz/ctlz followed by a icmp plus select into a single
1554/// call to cttz/ctlz with flag 'is_zero_poison' cleared.
1555///
1556/// For example, we can fold the following code sequence:
1557/// \code
1558/// %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
1559/// %1 = icmp ne i32 %x, 0
1560/// %2 = select i1 %1, i32 %0, i32 32
1561/// \code
1562///
1563/// into:
1564/// %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 false)
1565static Value *foldSelectCttzCtlz(ICmpInst *ICI, Value *TrueVal, Value *FalseVal,
1566 InstCombinerImpl &IC) {
1567 ICmpInst::Predicate Pred = ICI->getPredicate();
1568 Value *CmpLHS = ICI->getOperand(i_nocapture: 0);
1569 Value *CmpRHS = ICI->getOperand(i_nocapture: 1);
1570
1571 // Check if the select condition compares a value for equality.
1572 if (!ICI->isEquality())
1573 return nullptr;
1574
1575 Value *SelectArg = FalseVal;
1576 Value *ValueOnZero = TrueVal;
1577 if (Pred == ICmpInst::ICMP_NE)
1578 std::swap(a&: SelectArg, b&: ValueOnZero);
1579
1580 // Skip zero extend/truncate.
1581 Value *Count = nullptr;
1582 if (!match(V: SelectArg, P: m_ZExt(Op: m_Value(V&: Count))) &&
1583 !match(V: SelectArg, P: m_Trunc(Op: m_Value(V&: Count))))
1584 Count = SelectArg;
1585
1586 // Check that 'Count' is a call to intrinsic cttz/ctlz. Also check that the
1587 // input to the cttz/ctlz is used as LHS for the compare instruction.
1588 Value *X;
1589 if (!match(V: Count, P: m_Cttz(Op0: m_Value(V&: X), Op1: m_Value())) &&
1590 !match(V: Count, P: m_Ctlz(Op0: m_Value(V&: X), Op1: m_Value())))
1591 return nullptr;
1592
1593 // (X == 0) ? BitWidth : ctz(X)
1594 // (X == -1) ? BitWidth : ctz(~X)
1595 // (X == Y) ? BitWidth : ctz(X ^ Y)
1596 if ((X != CmpLHS || !match(V: CmpRHS, P: m_Zero())) &&
1597 (!match(V: X, P: m_Not(V: m_Specific(V: CmpLHS))) || !match(V: CmpRHS, P: m_AllOnes())) &&
1598 !match(V: X, P: m_c_Xor(L: m_Specific(V: CmpLHS), R: m_Specific(V: CmpRHS))))
1599 return nullptr;
1600
1601 IntrinsicInst *II = cast<IntrinsicInst>(Val: Count);
1602
1603 // Check if the value propagated on zero is a constant number equal to the
1604 // sizeof in bits of 'Count'.
1605 unsigned SizeOfInBits = Count->getType()->getScalarSizeInBits();
1606 if (match(V: ValueOnZero, P: m_SpecificInt(V: SizeOfInBits))) {
1607 // A range annotation on the intrinsic may no longer be valid.
1608 II->dropPoisonGeneratingAnnotations();
1609 IC.addToWorklist(I: II);
1610 return SelectArg;
1611 }
1612
1613 // The ValueOnZero is not the bitwidth. But if the cttz/ctlz (and optional
1614 // zext/trunc) have one use (ending at the select), the cttz/ctlz result will
1615 // not be used if the input is zero. Relax to 'zero is poison' for that case.
1616 if (II->hasOneUse() && SelectArg->hasOneUse() &&
1617 !match(V: II->getArgOperand(i: 1), P: m_One())) {
1618 II->setArgOperand(i: 1, v: ConstantInt::getTrue(Context&: II->getContext()));
1619 // noundef attribute on the intrinsic may no longer be valid.
1620 II->dropUBImplyingAttrsAndMetadata();
1621 IC.addToWorklist(I: II);
1622 }
1623
1624 return nullptr;
1625}
1626
1627static Value *canonicalizeSPF(ICmpInst &Cmp, Value *TrueVal, Value *FalseVal,
1628 InstCombinerImpl &IC) {
1629 Value *LHS, *RHS;
1630 // TODO: What to do with pointer min/max patterns?
1631 if (!TrueVal->getType()->isIntOrIntVectorTy())
1632 return nullptr;
1633
1634 SelectPatternFlavor SPF =
1635 matchDecomposedSelectPattern(CmpI: &Cmp, TrueVal, FalseVal, LHS, RHS).Flavor;
1636 if (SPF == SelectPatternFlavor::SPF_ABS ||
1637 SPF == SelectPatternFlavor::SPF_NABS) {
1638 if (!Cmp.hasOneUse() && !RHS->hasOneUse())
1639 return nullptr; // TODO: Relax this restriction.
1640
1641 // Note that NSW flag can only be propagated for normal, non-negated abs!
1642 bool IntMinIsPoison = SPF == SelectPatternFlavor::SPF_ABS &&
1643 match(V: RHS, P: m_NSWNeg(V: m_Specific(V: LHS)));
1644 Constant *IntMinIsPoisonC =
1645 ConstantInt::get(Ty: Type::getInt1Ty(C&: Cmp.getContext()), V: IntMinIsPoison);
1646 Value *Abs =
1647 IC.Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS, RHS: IntMinIsPoisonC);
1648
1649 if (SPF == SelectPatternFlavor::SPF_NABS)
1650 return IC.Builder.CreateNeg(V: Abs); // Always without NSW flag!
1651 return Abs;
1652 }
1653
1654 if (SelectPatternResult::isMinOrMax(SPF)) {
1655 Intrinsic::ID IntrinsicID = getMinMaxIntrinsic(SPF);
1656 return IC.Builder.CreateBinaryIntrinsic(ID: IntrinsicID, LHS, RHS);
1657 }
1658
1659 return nullptr;
1660}
1661
1662bool InstCombinerImpl::replaceInInstruction(Value *V, Value *Old, Value *New,
1663 unsigned Depth) {
1664 // Conservatively limit replacement to two instructions upwards.
1665 if (Depth == 2)
1666 return false;
1667
1668 assert(!isa<Constant>(Old) && "Only replace non-constant values");
1669
1670 auto *I = dyn_cast<Instruction>(Val: V);
1671 if (!I || !I->hasOneUse() ||
1672 !isSafeToSpeculativelyExecuteWithVariableReplaced(I))
1673 return false;
1674
1675 // Forbid potentially lane-crossing instructions.
1676 if (Old->getType()->isVectorTy() && !isNotCrossLaneOperation(I))
1677 return false;
1678
1679 bool Changed = false;
1680 for (Use &U : I->operands()) {
1681 if (U == Old) {
1682 replaceUse(U, NewValue: New);
1683 Worklist.add(I);
1684 Changed = true;
1685 } else {
1686 Changed |= replaceInInstruction(V: U, Old, New, Depth: Depth + 1);
1687 }
1688 }
1689 return Changed;
1690}
1691
1692/// If we have a select with an equality comparison, then we know the value in
1693/// one of the arms of the select. See if substituting this value into an arm
1694/// and simplifying the result yields the same value as the other arm.
1695///
1696/// To make this transform safe, we must drop poison-generating flags
1697/// (nsw, etc) if we simplified to a binop because the select may be guarding
1698/// that poison from propagating. If the existing binop already had no
1699/// poison-generating flags, then this transform can be done by instsimplify.
1700///
1701/// Consider:
1702/// %cmp = icmp eq i32 %x, 2147483647
1703/// %add = add nsw i32 %x, 1
1704/// %sel = select i1 %cmp, i32 -2147483648, i32 %add
1705///
1706/// We can't replace %sel with %add unless we strip away the flags.
1707/// TODO: Wrapping flags could be preserved in some cases with better analysis.
1708Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
1709 CmpInst &Cmp) {
1710 // Canonicalize the pattern to an equivalence on the predicate by swapping the
1711 // select operands.
1712 Value *TrueVal = Sel.getTrueValue(), *FalseVal = Sel.getFalseValue();
1713 bool Swapped = false;
1714 if (Cmp.isEquivalence(/*Invert=*/true)) {
1715 std::swap(a&: TrueVal, b&: FalseVal);
1716 Swapped = true;
1717 } else if (!Cmp.isEquivalence()) {
1718 return nullptr;
1719 }
1720
1721 Value *CmpLHS = Cmp.getOperand(i_nocapture: 0), *CmpRHS = Cmp.getOperand(i_nocapture: 1);
1722 auto ReplaceOldOpWithNewOp = [&](Value *OldOp,
1723 Value *NewOp) -> Instruction * {
1724 // In X == Y ? f(X) : Z, try to evaluate f(Y) and replace the operand.
1725 // Take care to avoid replacing X == Y ? X : Z with X == Y ? Y : Z, as that
1726 // would lead to an infinite replacement cycle.
1727 // If we will be able to evaluate f(Y) to a constant, we can allow undef,
1728 // otherwise Y cannot be undef as we might pick different values for undef
1729 // in the cmp and in f(Y).
1730 if (TrueVal == OldOp && (isa<Constant>(Val: OldOp) || !isa<Constant>(Val: NewOp)))
1731 return nullptr;
1732
1733 if (Value *V = simplifyWithOpReplaced(V: TrueVal, Op: OldOp, RepOp: NewOp, Q: SQ,
1734 /* AllowRefinement=*/true)) {
1735 // Need some guarantees about the new simplified op to ensure we don't inf
1736 // loop.
1737 // If we simplify to a constant, replace if we aren't creating new undef.
1738 if (match(V, P: m_ImmConstant()) &&
1739 isGuaranteedNotToBeUndef(V, AC: SQ.AC, CtxI: &Sel, DT: &DT))
1740 return replaceOperand(I&: Sel, OpNum: Swapped ? 2 : 1, V);
1741
1742 // If NewOp is a constant and OldOp is not replace iff NewOp doesn't
1743 // contain and undef elements.
1744 // Make sure that V is always simpler than TrueVal, otherwise we might
1745 // end up in an infinite loop.
1746 if (match(V: NewOp, P: m_ImmConstant()) ||
1747 (isa<Instruction>(Val: TrueVal) &&
1748 is_contained(Range: cast<Instruction>(Val: TrueVal)->operands(), Element: V))) {
1749 if (isGuaranteedNotToBeUndef(V: NewOp, AC: SQ.AC, CtxI: &Sel, DT: &DT))
1750 return replaceOperand(I&: Sel, OpNum: Swapped ? 2 : 1, V);
1751 return nullptr;
1752 }
1753 }
1754
1755 // Even if TrueVal does not simplify, we can directly replace a use of
1756 // CmpLHS with CmpRHS, as long as the instruction is not used anywhere
1757 // else and is safe to speculatively execute (we may end up executing it
1758 // with different operands, which should not cause side-effects or trigger
1759 // undefined behavior). Only do this if CmpRHS is a constant, as
1760 // profitability is not clear for other cases.
1761 if (OldOp == CmpLHS && match(V: NewOp, P: m_ImmConstant()) &&
1762 !match(V: OldOp, P: m_Constant()) &&
1763 isGuaranteedNotToBeUndef(V: NewOp, AC: SQ.AC, CtxI: &Sel, DT: &DT))
1764 if (replaceInInstruction(V: TrueVal, Old: OldOp, New: NewOp))
1765 return &Sel;
1766 return nullptr;
1767 };
1768
1769 bool CanReplaceCmpLHSWithRHS = canReplacePointersIfEqual(From: CmpLHS, To: CmpRHS, DL);
1770 if (CanReplaceCmpLHSWithRHS) {
1771 if (Instruction *R = ReplaceOldOpWithNewOp(CmpLHS, CmpRHS))
1772 return R;
1773 }
1774 bool CanReplaceCmpRHSWithLHS = canReplacePointersIfEqual(From: CmpRHS, To: CmpLHS, DL);
1775 if (CanReplaceCmpRHSWithLHS) {
1776 if (Instruction *R = ReplaceOldOpWithNewOp(CmpRHS, CmpLHS))
1777 return R;
1778 }
1779
1780 auto *FalseInst = dyn_cast<Instruction>(Val: FalseVal);
1781 if (!FalseInst)
1782 return nullptr;
1783
1784 // InstSimplify already performed this fold if it was possible subject to
1785 // current poison-generating flags. Check whether dropping poison-generating
1786 // flags enables the transform.
1787
1788 // Try each equivalence substitution possibility.
1789 // We have an 'EQ' comparison, so the select's false value will propagate.
1790 // Example:
1791 // (X == 42) ? 43 : (X + 1) --> (X == 42) ? (X + 1) : (X + 1) --> X + 1
1792 SmallVector<Instruction *> DropFlags;
1793 if ((CanReplaceCmpLHSWithRHS &&
1794 simplifyWithOpReplaced(V: FalseVal, Op: CmpLHS, RepOp: CmpRHS, Q: SQ,
1795 /* AllowRefinement */ false,
1796 DropFlags: &DropFlags) == TrueVal) ||
1797 (CanReplaceCmpRHSWithLHS &&
1798 simplifyWithOpReplaced(V: FalseVal, Op: CmpRHS, RepOp: CmpLHS, Q: SQ,
1799 /* AllowRefinement */ false,
1800 DropFlags: &DropFlags) == TrueVal)) {
1801 for (Instruction *I : DropFlags) {
1802 I->dropPoisonGeneratingAnnotations();
1803 Worklist.add(I);
1804 }
1805
1806 return replaceInstUsesWith(I&: Sel, V: FalseVal);
1807 }
1808
1809 Constant *CmpC;
1810 if (FalseVal->getType()->isIntOrIntVectorTy(BitWidth: 1) &&
1811 match(V: FalseVal, P: m_NUWTrunc(Op: m_Specific(V: CmpLHS))) &&
1812 match(V: CmpRHS, P: m_ImmConstant(C&: CmpC)) &&
1813 ConstantFoldCompareInstOperands(
1814 Predicate: ICmpInst::Predicate::ICMP_NE, LHS: CmpC,
1815 RHS: ConstantInt::getNullValue(Ty: CmpLHS->getType()), DL) == TrueVal) {
1816 return new ICmpInst(CmpInst::Predicate::ICMP_NE, CmpLHS,
1817 ConstantInt::getNullValue(Ty: CmpLHS->getType()));
1818 }
1819
1820 return nullptr;
1821}
1822
1823/// Fold the following code sequence:
1824/// \code
1825/// %XeqZ = icmp eq i64 %X, %Z
1826/// %YeqZ = icmp eq i64 %Y, %Z
1827/// %XeqY = icmp eq i64 %X, %Y
1828/// %not.YeqZ = xor i1 %YeqZ, true
1829/// %and = select i1 %not.YeqZ, i1 %XeqY, i1 false
1830/// %equal = select i1 %XeqZ, i1 %YeqZ, i1 %and
1831/// \code
1832///
1833/// into:
1834/// %equal = icmp eq i64 %X, %Y
1835Instruction *InstCombinerImpl::foldSelectEqualityTest(SelectInst &Sel) {
1836 Value *X, *Y, *Z;
1837 Value *XeqY, *XeqZ = Sel.getCondition(), *YeqZ = Sel.getTrueValue();
1838
1839 if (!match(V: XeqZ, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Value(V&: X), R: m_Value(V&: Z))))
1840 return nullptr;
1841
1842 if (!match(V: YeqZ,
1843 P: m_c_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Value(V&: Y), R: m_Specific(V: Z))))
1844 std::swap(a&: X, b&: Z);
1845
1846 if (!match(V: YeqZ,
1847 P: m_c_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Value(V&: Y), R: m_Specific(V: Z))))
1848 return nullptr;
1849
1850 if (!match(V: Sel.getFalseValue(),
1851 P: m_c_LogicalAnd(L: m_Not(V: m_Specific(V: YeqZ)), R: m_Value(V&: XeqY))))
1852 return nullptr;
1853
1854 if (!match(V: XeqY,
1855 P: m_c_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Specific(V: X), R: m_Specific(V: Y))))
1856 return nullptr;
1857
1858 cast<ICmpInst>(Val: XeqY)->setSameSign(false);
1859 return replaceInstUsesWith(I&: Sel, V: XeqY);
1860}
1861
1862// See if this is a pattern like:
1863// %old_cmp1 = icmp slt i32 %x, C2
1864// %old_replacement = select i1 %old_cmp1, i32 %target_low, i32 %target_high
1865// %old_x_offseted = add i32 %x, C1
1866// %old_cmp0 = icmp ult i32 %old_x_offseted, C0
1867// %r = select i1 %old_cmp0, i32 %x, i32 %old_replacement
1868// This can be rewritten as more canonical pattern:
1869// %new_cmp1 = icmp slt i32 %x, -C1
1870// %new_cmp2 = icmp sge i32 %x, C0-C1
1871// %new_clamped_low = select i1 %new_cmp1, i32 %target_low, i32 %x
1872// %r = select i1 %new_cmp2, i32 %target_high, i32 %new_clamped_low
1873// Iff -C1 s<= C2 s<= C0-C1
1874// Also ULT predicate can also be UGT iff C0 != -1 (+invert result)
1875// SLT predicate can also be SGT iff C2 != INT_MAX (+invert res.)
1876static Value *canonicalizeClampLike(SelectInst &Sel0, ICmpInst &Cmp0,
1877 InstCombiner::BuilderTy &Builder,
1878 InstCombiner &IC) {
1879 Value *X = Sel0.getTrueValue();
1880 Value *Sel1 = Sel0.getFalseValue();
1881
1882 // First match the condition of the outermost select.
1883 // Said condition must be one-use.
1884 if (!Cmp0.hasOneUse())
1885 return nullptr;
1886 ICmpInst::Predicate Pred0 = Cmp0.getPredicate();
1887 Value *Cmp00 = Cmp0.getOperand(i_nocapture: 0);
1888 Constant *C0;
1889 if (!match(V: Cmp0.getOperand(i_nocapture: 1),
1890 P: m_CombineAnd(Ps: m_AnyIntegralConstant(), Ps: m_Constant(C&: C0))))
1891 return nullptr;
1892
1893 if (!match(V: Sel1, P: m_SelectLike(C: m_Value(), TrueC: m_Value(), FalseC: m_Value()))) {
1894 Pred0 = ICmpInst::getInversePredicate(pred: Pred0);
1895 std::swap(a&: X, b&: Sel1);
1896 }
1897
1898 // Canonicalize Cmp0 into ult or uge.
1899 // FIXME: we shouldn't care about lanes that are 'undef' in the end?
1900 switch (Pred0) {
1901 case ICmpInst::Predicate::ICMP_ULT:
1902 case ICmpInst::Predicate::ICMP_UGE:
1903 // Although icmp ult %x, 0 is an unusual thing to try and should generally
1904 // have been simplified, it does not verify with undef inputs so ensure we
1905 // are not in a strange state.
1906 if (!match(V: C0, P: m_SpecificInt_ICMP(
1907 Predicate: ICmpInst::Predicate::ICMP_NE,
1908 Threshold: APInt::getZero(numBits: C0->getType()->getScalarSizeInBits()))))
1909 return nullptr;
1910 break; // Great!
1911 case ICmpInst::Predicate::ICMP_ULE:
1912 case ICmpInst::Predicate::ICMP_UGT:
1913 // We want to canonicalize it to 'ult' or 'uge', so we'll need to increment
1914 // C0, which again means it must not have any all-ones elements.
1915 if (!match(V: C0,
1916 P: m_SpecificInt_ICMP(
1917 Predicate: ICmpInst::Predicate::ICMP_NE,
1918 Threshold: APInt::getAllOnes(numBits: C0->getType()->getScalarSizeInBits()))))
1919 return nullptr; // Can't do, have all-ones element[s].
1920 Pred0 = ICmpInst::getFlippedStrictnessPredicate(pred: Pred0);
1921 C0 = InstCombiner::AddOne(C: C0);
1922 break;
1923 default:
1924 return nullptr; // Unknown predicate.
1925 }
1926
1927 // Now that we've canonicalized the ICmp, we know the X we expect;
1928 // the select in other hand should be one-use.
1929 if (!Sel1->hasOneUse())
1930 return nullptr;
1931
1932 // If the types do not match, look through any truncs to the underlying
1933 // instruction.
1934 if (Cmp00->getType() != X->getType() && X->hasOneUse())
1935 match(V: X, P: m_TruncOrSelf(Op: m_Value(V&: X)));
1936
1937 // We now can finish matching the condition of the outermost select:
1938 // it should either be the X itself, or an addition of some constant to X.
1939 Constant *C1;
1940 if (Cmp00 == X)
1941 C1 = ConstantInt::getNullValue(Ty: X->getType());
1942 else if (!match(V: Cmp00,
1943 P: m_Add(L: m_Specific(V: X),
1944 R: m_CombineAnd(Ps: m_AnyIntegralConstant(), Ps: m_Constant(C&: C1)))))
1945 return nullptr;
1946
1947 Value *Cmp1;
1948 CmpPredicate Pred1;
1949 Constant *C2;
1950 Value *ReplacementLow, *ReplacementHigh;
1951 if (!match(V: Sel1, P: m_SelectLike(C: m_Value(V&: Cmp1), TrueC: m_Value(V&: ReplacementLow),
1952 FalseC: m_Value(V&: ReplacementHigh))) ||
1953 !match(V: Cmp1,
1954 P: m_ICmp(Pred&: Pred1, L: m_Specific(V: X),
1955 R: m_CombineAnd(Ps: m_AnyIntegralConstant(), Ps: m_Constant(C&: C2)))))
1956 return nullptr;
1957
1958 if (!Cmp1->hasOneUse() && (Cmp00 == X || !Cmp00->hasOneUse()))
1959 return nullptr; // Not enough one-use instructions for the fold.
1960 // FIXME: this restriction could be relaxed if Cmp1 can be reused as one of
1961 // two comparisons we'll need to build.
1962
1963 // Canonicalize Cmp1 into the form we expect.
1964 // FIXME: we shouldn't care about lanes that are 'undef' in the end?
1965 switch (Pred1) {
1966 case ICmpInst::Predicate::ICMP_SLT:
1967 break;
1968 case ICmpInst::Predicate::ICMP_SLE:
1969 // We'd have to increment C2 by one, and for that it must not have signed
1970 // max element, but then it would have been canonicalized to 'slt' before
1971 // we get here. So we can't do anything useful with 'sle'.
1972 return nullptr;
1973 case ICmpInst::Predicate::ICMP_SGT:
1974 // We want to canonicalize it to 'slt', so we'll need to increment C2,
1975 // which again means it must not have any signed max elements.
1976 if (!match(V: C2,
1977 P: m_SpecificInt_ICMP(Predicate: ICmpInst::Predicate::ICMP_NE,
1978 Threshold: APInt::getSignedMaxValue(
1979 numBits: C2->getType()->getScalarSizeInBits()))))
1980 return nullptr; // Can't do, have signed max element[s].
1981 C2 = InstCombiner::AddOne(C: C2);
1982 [[fallthrough]];
1983 case ICmpInst::Predicate::ICMP_SGE:
1984 // Also non-canonical, but here we don't need to change C2,
1985 // so we don't have any restrictions on C2, so we can just handle it.
1986 Pred1 = ICmpInst::Predicate::ICMP_SLT;
1987 std::swap(a&: ReplacementLow, b&: ReplacementHigh);
1988 break;
1989 default:
1990 return nullptr; // Unknown predicate.
1991 }
1992 assert(Pred1 == ICmpInst::Predicate::ICMP_SLT &&
1993 "Unexpected predicate type.");
1994
1995 // The thresholds of this clamp-like pattern.
1996 auto *ThresholdLowIncl = ConstantExpr::getNeg(C: C1);
1997 auto *ThresholdHighExcl = ConstantExpr::getSub(C1: C0, C2: C1);
1998
1999 assert((Pred0 == ICmpInst::Predicate::ICMP_ULT ||
2000 Pred0 == ICmpInst::Predicate::ICMP_UGE) &&
2001 "Unexpected predicate type.");
2002 if (Pred0 == ICmpInst::Predicate::ICMP_UGE)
2003 std::swap(a&: ThresholdLowIncl, b&: ThresholdHighExcl);
2004
2005 // The fold has a precondition 1: C2 s>= ThresholdLow
2006 auto *Precond1 = ConstantFoldCompareInstOperands(
2007 Predicate: ICmpInst::Predicate::ICMP_SGE, LHS: C2, RHS: ThresholdLowIncl, DL: IC.getDataLayout());
2008 if (!Precond1 || !match(V: Precond1, P: m_One()))
2009 return nullptr;
2010 // The fold has a precondition 2: C2 s<= ThresholdHigh
2011 auto *Precond2 = ConstantFoldCompareInstOperands(
2012 Predicate: ICmpInst::Predicate::ICMP_SLE, LHS: C2, RHS: ThresholdHighExcl, DL: IC.getDataLayout());
2013 if (!Precond2 || !match(V: Precond2, P: m_One()))
2014 return nullptr;
2015
2016 // If we are matching from a truncated input, we need to sext the
2017 // ReplacementLow and ReplacementHigh values. Only do the transform if they
2018 // are free to extend due to being constants.
2019 if (X->getType() != Sel0.getType()) {
2020 Constant *LowC, *HighC;
2021 if (!match(V: ReplacementLow, P: m_ImmConstant(C&: LowC)) ||
2022 !match(V: ReplacementHigh, P: m_ImmConstant(C&: HighC)))
2023 return nullptr;
2024 const DataLayout &DL = Sel0.getDataLayout();
2025 ReplacementLow =
2026 ConstantFoldCastOperand(Opcode: Instruction::SExt, C: LowC, DestTy: X->getType(), DL);
2027 ReplacementHigh =
2028 ConstantFoldCastOperand(Opcode: Instruction::SExt, C: HighC, DestTy: X->getType(), DL);
2029 assert(ReplacementLow && ReplacementHigh &&
2030 "Constant folding of ImmConstant cannot fail");
2031 }
2032
2033 // We mark the select instructions below as having an unknown profile as it is
2034 // not possible to recover profile information from the original selects in
2035 // the general case. From them we can only know the probability that we clamp
2036 // whereas we need the probabilities for clamping specific to the low end/high
2037 // end.
2038
2039 // All good, finally emit the new pattern.
2040 Value *ShouldReplaceLow = Builder.CreateICmpSLT(LHS: X, RHS: ThresholdLowIncl);
2041 Value *ShouldReplaceHigh = Builder.CreateICmpSGE(LHS: X, RHS: ThresholdHighExcl);
2042 Value *MaybeReplacedLow = Builder.CreateSelectWithUnknownProfile(
2043 C: ShouldReplaceLow, True: ReplacementLow, False: X, DEBUG_TYPE);
2044
2045 // Create the final select. If we looked through a truncate above, we will
2046 // need to retruncate the result.
2047 Value *MaybeReplacedHigh = Builder.CreateSelectWithUnknownProfile(
2048 C: ShouldReplaceHigh, True: ReplacementHigh, False: MaybeReplacedLow, DEBUG_TYPE);
2049 return Builder.CreateTrunc(V: MaybeReplacedHigh, DestTy: Sel0.getType());
2050}
2051
2052// If we have
2053// %cmp = icmp [canonical predicate] i32 %x, C0
2054// %r = select i1 %cmp, i32 %y, i32 C1
2055// Where C0 != C1 and %x may be different from %y, see if the constant that we
2056// will have if we flip the strictness of the predicate (i.e. without changing
2057// the result) is identical to the C1 in select. If it matches we can change
2058// original comparison to one with swapped predicate, reuse the constant,
2059// and swap the hands of select.
2060static Instruction *
2061tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp,
2062 InstCombinerImpl &IC) {
2063 CmpPredicate Pred;
2064 Value *X;
2065 Constant *C0;
2066 if (!match(V: &Cmp, P: m_OneUse(SubPattern: m_ICmp(
2067 Pred, L: m_Value(V&: X),
2068 R: m_CombineAnd(Ps: m_AnyIntegralConstant(), Ps: m_Constant(C&: C0))))))
2069 return nullptr;
2070
2071 // If comparison predicate is non-relational, we won't be able to do anything.
2072 if (ICmpInst::isEquality(P: Pred))
2073 return nullptr;
2074
2075 // If comparison predicate is non-canonical, then we certainly won't be able
2076 // to make it canonical; canonicalizeCmpWithConstant() already tried.
2077 if (!InstCombiner::isCanonicalPredicate(Pred))
2078 return nullptr;
2079
2080 // If the [input] type of comparison and select type are different, lets abort
2081 // for now. We could try to compare constants with trunc/[zs]ext though.
2082 if (C0->getType() != Sel.getType())
2083 return nullptr;
2084
2085 // ULT with 'add' of a constant is canonical. See foldICmpAddConstant().
2086 // FIXME: Are there more magic icmp predicate+constant pairs we must avoid?
2087 // Or should we just abandon this transform entirely?
2088 if (Pred == CmpInst::ICMP_ULT && match(V: X, P: m_Add(L: m_Value(), R: m_Constant())))
2089 return nullptr;
2090
2091
2092 Value *SelVal0, *SelVal1; // We do not care which one is from where.
2093 match(V: &Sel, P: m_Select(C: m_Value(), L: m_Value(V&: SelVal0), R: m_Value(V&: SelVal1)));
2094 // At least one of these values we are selecting between must be a constant
2095 // else we'll never succeed.
2096 if (!match(V: SelVal0, P: m_AnyIntegralConstant()) &&
2097 !match(V: SelVal1, P: m_AnyIntegralConstant()))
2098 return nullptr;
2099
2100 // Does this constant C match any of the `select` values?
2101 auto MatchesSelectValue = [SelVal0, SelVal1](Constant *C) {
2102 return C->isElementWiseEqual(Y: SelVal0) || C->isElementWiseEqual(Y: SelVal1);
2103 };
2104
2105 // If C0 *already* matches true/false value of select, we are done.
2106 if (MatchesSelectValue(C0))
2107 return nullptr;
2108
2109 // Check the constant we'd have with flipped-strictness predicate.
2110 auto FlippedStrictness = getFlippedStrictnessPredicateAndConstant(Pred, C: C0);
2111 if (!FlippedStrictness)
2112 return nullptr;
2113
2114 // If said constant doesn't match either, then there is no hope,
2115 if (!MatchesSelectValue(FlippedStrictness->second))
2116 return nullptr;
2117
2118 // It matched! Lets insert the new comparison just before select.
2119 InstCombiner::BuilderTy::InsertPointGuard Guard(IC.Builder);
2120 IC.Builder.SetInsertPoint(&Sel);
2121
2122 Pred = ICmpInst::getSwappedPredicate(pred: Pred); // Yes, swapped.
2123 Value *NewCmp = IC.Builder.CreateICmp(P: Pred, LHS: X, RHS: FlippedStrictness->second,
2124 Name: Cmp.getName() + ".inv");
2125 IC.replaceOperand(I&: Sel, OpNum: 0, V: NewCmp);
2126 Sel.swapValues();
2127 Sel.swapProfMetadata();
2128
2129 return &Sel;
2130}
2131
2132static Instruction *foldSelectZeroOrOnes(ICmpInst *Cmp, Value *TVal,
2133 Value *FVal,
2134 InstCombiner::BuilderTy &Builder) {
2135 if (!Cmp->hasOneUse())
2136 return nullptr;
2137
2138 const APInt *CmpC;
2139 if (!match(V: Cmp->getOperand(i_nocapture: 1), P: m_APIntAllowPoison(Res&: CmpC)))
2140 return nullptr;
2141
2142 // (X u< 2) ? -X : -1 --> sext (X != 0)
2143 Value *X = Cmp->getOperand(i_nocapture: 0);
2144 if (Cmp->getPredicate() == ICmpInst::ICMP_ULT && *CmpC == 2 &&
2145 match(V: TVal, P: m_Neg(V: m_Specific(V: X))) && match(V: FVal, P: m_AllOnes()))
2146 return new SExtInst(Builder.CreateIsNotNull(Arg: X), TVal->getType());
2147
2148 // (X u> 1) ? -1 : -X --> sext (X != 0)
2149 if (Cmp->getPredicate() == ICmpInst::ICMP_UGT && *CmpC == 1 &&
2150 match(V: FVal, P: m_Neg(V: m_Specific(V: X))) && match(V: TVal, P: m_AllOnes()))
2151 return new SExtInst(Builder.CreateIsNotNull(Arg: X), TVal->getType());
2152
2153 return nullptr;
2154}
2155
2156static Value *foldSelectInstWithICmpConst(SelectInst &SI, ICmpInst *ICI,
2157 InstCombiner::BuilderTy &Builder) {
2158 const APInt *CmpC;
2159 Value *V;
2160 CmpPredicate Pred;
2161 if (!match(V: ICI, P: m_ICmp(Pred, L: m_Value(V), R: m_APInt(Res&: CmpC))))
2162 return nullptr;
2163
2164 // Match clamp away from min/max value as a max/min operation.
2165 Value *TVal = SI.getTrueValue();
2166 Value *FVal = SI.getFalseValue();
2167 if (Pred == ICmpInst::ICMP_EQ && V == FVal) {
2168 // (V == UMIN) ? UMIN+1 : V --> umax(V, UMIN+1)
2169 if (CmpC->isMinValue() && match(V: TVal, P: m_SpecificInt(V: *CmpC + 1)))
2170 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::umax, LHS: V, RHS: TVal);
2171 // (V == UMAX) ? UMAX-1 : V --> umin(V, UMAX-1)
2172 if (CmpC->isMaxValue() && match(V: TVal, P: m_SpecificInt(V: *CmpC - 1)))
2173 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: V, RHS: TVal);
2174 // (V == SMIN) ? SMIN+1 : V --> smax(V, SMIN+1)
2175 if (CmpC->isMinSignedValue() && match(V: TVal, P: m_SpecificInt(V: *CmpC + 1)))
2176 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::smax, LHS: V, RHS: TVal);
2177 // (V == SMAX) ? SMAX-1 : V --> smin(V, SMAX-1)
2178 if (CmpC->isMaxSignedValue() && match(V: TVal, P: m_SpecificInt(V: *CmpC - 1)))
2179 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::smin, LHS: V, RHS: TVal);
2180 }
2181
2182 // Fold icmp(X) ? f(X) : C to f(X) when f(X) is guaranteed to be equal to C
2183 // for all X in the exact range of the inverse predicate.
2184 Instruction *Op;
2185 const APInt *C;
2186 CmpInst::Predicate CPred;
2187 if (match(V: &SI, P: m_Select(C: m_Specific(V: ICI), L: m_APInt(Res&: C), R: m_Instruction(I&: Op))))
2188 CPred = ICI->getPredicate();
2189 else if (match(V: &SI, P: m_Select(C: m_Specific(V: ICI), L: m_Instruction(I&: Op), R: m_APInt(Res&: C))))
2190 CPred = ICI->getInversePredicate();
2191 else
2192 return nullptr;
2193
2194 ConstantRange InvDomCR = ConstantRange::makeExactICmpRegion(Pred: CPred, Other: *CmpC);
2195 const APInt *OpC;
2196 if (match(V: Op, P: m_BinOp(L: m_Specific(V), R: m_APInt(Res&: OpC)))) {
2197 ConstantRange R = InvDomCR.binaryOp(
2198 BinOp: static_cast<Instruction::BinaryOps>(Op->getOpcode()), Other: *OpC);
2199 if (R == *C) {
2200 Op->dropPoisonGeneratingFlags();
2201 return Op;
2202 }
2203 }
2204 if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: Op);
2205 MMI && MMI->getLHS() == V && match(V: MMI->getRHS(), P: m_APInt(Res&: OpC))) {
2206 ConstantRange R = ConstantRange::intrinsic(IntrinsicID: MMI->getIntrinsicID(),
2207 Ops: {InvDomCR, ConstantRange(*OpC)});
2208 if (R == *C) {
2209 MMI->dropPoisonGeneratingAnnotations();
2210 return MMI;
2211 }
2212 }
2213
2214 return nullptr;
2215}
2216
2217/// `A == MIN_INT ? B != MIN_INT : A < B` --> `A < B`
2218/// `A == MAX_INT ? B != MAX_INT : A > B` --> `A > B`
2219static Instruction *foldSelectWithExtremeEqCond(Value *CmpLHS, Value *CmpRHS,
2220 Value *TrueVal,
2221 Value *FalseVal) {
2222 Type *Ty = CmpLHS->getType();
2223
2224 if (Ty->isPtrOrPtrVectorTy())
2225 return nullptr;
2226
2227 CmpPredicate Pred;
2228 Value *B;
2229
2230 if (!match(V: FalseVal, P: m_c_ICmp(Pred, L: m_Specific(V: CmpLHS), R: m_Value(V&: B))))
2231 return nullptr;
2232
2233 Value *TValRHS;
2234 if (!match(V: TrueVal, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_NE, L: m_Specific(V: B),
2235 R: m_Value(V&: TValRHS))))
2236 return nullptr;
2237
2238 APInt C;
2239 unsigned BitWidth = Ty->getScalarSizeInBits();
2240
2241 if (ICmpInst::isLT(P: Pred)) {
2242 C = CmpInst::isSigned(Pred) ? APInt::getSignedMinValue(numBits: BitWidth)
2243 : APInt::getMinValue(numBits: BitWidth);
2244 } else if (ICmpInst::isGT(P: Pred)) {
2245 C = CmpInst::isSigned(Pred) ? APInt::getSignedMaxValue(numBits: BitWidth)
2246 : APInt::getMaxValue(numBits: BitWidth);
2247 } else {
2248 return nullptr;
2249 }
2250
2251 if (!match(V: CmpRHS, P: m_SpecificInt(V: C)) || !match(V: TValRHS, P: m_SpecificInt(V: C)))
2252 return nullptr;
2253
2254 return new ICmpInst(Pred, CmpLHS, B);
2255}
2256
2257static Instruction *foldSelectICmpEq(SelectInst &SI, ICmpInst *ICI,
2258 InstCombinerImpl &IC) {
2259 ICmpInst::Predicate Pred = ICI->getPredicate();
2260 if (!ICmpInst::isEquality(P: Pred))
2261 return nullptr;
2262
2263 Value *TrueVal = SI.getTrueValue();
2264 Value *FalseVal = SI.getFalseValue();
2265 Value *CmpLHS = ICI->getOperand(i_nocapture: 0);
2266 Value *CmpRHS = ICI->getOperand(i_nocapture: 1);
2267
2268 if (Pred == ICmpInst::ICMP_NE)
2269 std::swap(a&: TrueVal, b&: FalseVal);
2270
2271 if (Instruction *Res =
2272 foldSelectWithExtremeEqCond(CmpLHS, CmpRHS, TrueVal, FalseVal))
2273 return Res;
2274
2275 return nullptr;
2276}
2277
2278/// Fold `X Pred C1 ? X BOp C2 : C1 BOp C2` to `min/max(X, C1) BOp C2`.
2279/// This allows for better canonicalization.
2280Value *InstCombinerImpl::foldSelectWithConstOpToBinOp(ICmpInst *Cmp,
2281 Value *TrueVal,
2282 Value *FalseVal) {
2283 Constant *C1, *C2, *C3;
2284 Value *X;
2285 CmpPredicate Predicate;
2286
2287 if (!match(V: Cmp, P: m_ICmp(Pred&: Predicate, L: m_Value(V&: X), R: m_Constant(C&: C1))))
2288 return nullptr;
2289
2290 if (!ICmpInst::isRelational(P: Predicate))
2291 return nullptr;
2292
2293 if (match(V: TrueVal, P: m_Constant())) {
2294 std::swap(a&: FalseVal, b&: TrueVal);
2295 Predicate = ICmpInst::getInversePredicate(pred: Predicate);
2296 }
2297
2298 if (!match(V: FalseVal, P: m_Constant(C&: C3)) || !TrueVal->hasOneUse())
2299 return nullptr;
2300
2301 bool IsIntrinsic;
2302 unsigned Opcode;
2303 if (BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: TrueVal)) {
2304 Opcode = BOp->getOpcode();
2305 IsIntrinsic = false;
2306
2307 // This fold causes some regressions and is primarily intended for
2308 // add and sub. So we early exit for div and rem to minimize the
2309 // regressions.
2310 if (Instruction::isIntDivRem(Opcode))
2311 return nullptr;
2312
2313 if (!match(V: BOp, P: m_BinOp(L: m_Specific(V: X), R: m_Constant(C&: C2))))
2314 return nullptr;
2315
2316 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: TrueVal)) {
2317 if (!match(V: II, P: m_MaxOrMin(Op0: m_Specific(V: X), Op1: m_Constant(C&: C2))))
2318 return nullptr;
2319 Opcode = II->getIntrinsicID();
2320 IsIntrinsic = true;
2321 } else {
2322 return nullptr;
2323 }
2324
2325 Value *RHS;
2326 SelectPatternFlavor SPF;
2327 const DataLayout &DL = Cmp->getDataLayout();
2328 auto Flipped = getFlippedStrictnessPredicateAndConstant(Pred: Predicate, C: C1);
2329
2330 auto FoldBinaryOpOrIntrinsic = [&](Constant *LHS, Constant *RHS) {
2331 return IsIntrinsic
2332 ? ConstantFoldIntrinsic(ID: Opcode, Ops: {LHS, RHS}, Ty: LHS->getType(), DL)
2333 : ConstantFoldBinaryOpOperands(Opcode, LHS, RHS, DL);
2334 };
2335
2336 if (C3 == FoldBinaryOpOrIntrinsic(C1, C2)) {
2337 SPF = getSelectPattern(Pred: Predicate).Flavor;
2338 RHS = C1;
2339 } else if (Flipped && C3 == FoldBinaryOpOrIntrinsic(Flipped->second, C2)) {
2340 SPF = getSelectPattern(Pred: Flipped->first).Flavor;
2341 RHS = Flipped->second;
2342 } else {
2343 return nullptr;
2344 }
2345
2346 Intrinsic::ID MinMaxID = getMinMaxIntrinsic(SPF);
2347 Value *MinMax = Builder.CreateBinaryIntrinsic(ID: MinMaxID, LHS: X, RHS);
2348 if (IsIntrinsic)
2349 return Builder.CreateBinaryIntrinsic(ID: Opcode, LHS: MinMax, RHS: C2);
2350
2351 const auto BinOpc = Instruction::BinaryOps(Opcode);
2352 Value *BinOp = Builder.CreateBinOp(Opc: BinOpc, LHS: MinMax, RHS: C2);
2353
2354 // If we can attach no-wrap flags to the new instruction, do so if the
2355 // old instruction had them and C1 BinOp C2 does not overflow.
2356 if (Instruction *BinOpInst = dyn_cast<Instruction>(Val: BinOp)) {
2357 if (BinOpc == Instruction::Add || BinOpc == Instruction::Sub ||
2358 BinOpc == Instruction::Mul) {
2359 Instruction *OldBinOp = cast<BinaryOperator>(Val: TrueVal);
2360 if (OldBinOp->hasNoSignedWrap() &&
2361 willNotOverflow(Opcode: BinOpc, LHS: RHS, RHS: C2, CtxI: *BinOpInst, /*IsSigned=*/true))
2362 BinOpInst->setHasNoSignedWrap();
2363 if (OldBinOp->hasNoUnsignedWrap() &&
2364 willNotOverflow(Opcode: BinOpc, LHS: RHS, RHS: C2, CtxI: *BinOpInst, /*IsSigned=*/false))
2365 BinOpInst->setHasNoUnsignedWrap();
2366 }
2367 }
2368 return BinOp;
2369}
2370
2371/// Folds:
2372/// %a_sub = call @llvm.usub.sat(x, IntConst1)
2373/// %b_sub = call @llvm.usub.sat(y, IntConst2)
2374/// %or = or %a_sub, %b_sub
2375/// %cmp = icmp eq %or, 0
2376/// %sel = select %cmp, 0, MostSignificantBit
2377/// into:
2378/// %a_sub' = usub.sat(x, IntConst1 - MostSignificantBit)
2379/// %b_sub' = usub.sat(y, IntConst2 - MostSignificantBit)
2380/// %or = or %a_sub', %b_sub'
2381/// %and = and %or, MostSignificantBit
2382/// Likewise, for vector arguments as well.
2383static Instruction *foldICmpUSubSatWithAndForMostSignificantBitCmp(
2384 SelectInst &SI, ICmpInst *ICI, InstCombiner::BuilderTy &Builder) {
2385 if (!SI.hasOneUse() || !ICI->hasOneUse())
2386 return nullptr;
2387 CmpPredicate Pred;
2388 Value *A, *B;
2389 const APInt *Constant1, *Constant2;
2390 if (!match(V: SI.getCondition(),
2391 P: m_ICmp(Pred,
2392 L: m_OneUse(SubPattern: m_Or(L: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::usub_sat>(
2393 Ops: m_Value(V&: A), Ops: m_APInt(Res&: Constant1))),
2394 R: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::usub_sat>(
2395 Ops: m_Value(V&: B), Ops: m_APInt(Res&: Constant2))))),
2396 R: m_Zero())))
2397 return nullptr;
2398
2399 Value *TrueVal = SI.getTrueValue();
2400 Value *FalseVal = SI.getFalseValue();
2401 if (!((Pred == ICmpInst::ICMP_EQ && match(V: TrueVal, P: m_Zero()) &&
2402 match(V: FalseVal, P: m_SignMask())) ||
2403 (Pred == ICmpInst::ICMP_NE && match(V: TrueVal, P: m_SignMask()) &&
2404 match(V: FalseVal, P: m_Zero()))))
2405 return nullptr;
2406
2407 auto *Ty = A->getType();
2408 unsigned BW = Constant1->getBitWidth();
2409 APInt MostSignificantBit = APInt::getSignMask(BitWidth: BW);
2410
2411 // Anything over MSB is negative
2412 if (Constant1->isNonNegative() || Constant2->isNonNegative())
2413 return nullptr;
2414
2415 APInt AdjAP1 = *Constant1 - MostSignificantBit + 1;
2416 APInt AdjAP2 = *Constant2 - MostSignificantBit + 1;
2417
2418 auto *Adj1 = ConstantInt::get(Ty, V: AdjAP1);
2419 auto *Adj2 = ConstantInt::get(Ty, V: AdjAP2);
2420
2421 Value *NewA = Builder.CreateBinaryIntrinsic(ID: Intrinsic::usub_sat, LHS: A, RHS: Adj1);
2422 Value *NewB = Builder.CreateBinaryIntrinsic(ID: Intrinsic::usub_sat, LHS: B, RHS: Adj2);
2423 Value *Or = Builder.CreateOr(LHS: NewA, RHS: NewB);
2424 Constant *MSBConst = ConstantInt::get(Ty, V: MostSignificantBit);
2425 return BinaryOperator::CreateAnd(V1: Or, V2: MSBConst);
2426}
2427
2428/// Visit a SelectInst that has an ICmpInst as its first operand.
2429Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
2430 ICmpInst *ICI) {
2431 if (Value *V =
2432 canonicalizeSPF(Cmp&: *ICI, TrueVal: SI.getTrueValue(), FalseVal: SI.getFalseValue(), IC&: *this))
2433 return replaceInstUsesWith(I&: SI, V);
2434
2435 if (Value *V = foldSelectInstWithICmpConst(SI, ICI, Builder))
2436 return replaceInstUsesWith(I&: SI, V);
2437
2438 if (Value *V = canonicalizeClampLike(Sel0&: SI, Cmp0&: *ICI, Builder, IC&: *this))
2439 return replaceInstUsesWith(I&: SI, V);
2440
2441 if (Instruction *NewSel =
2442 tryToReuseConstantFromSelectInComparison(Sel&: SI, Cmp&: *ICI, IC&: *this))
2443 return NewSel;
2444 if (Instruction *Folded =
2445 foldICmpUSubSatWithAndForMostSignificantBitCmp(SI, ICI, Builder))
2446 return Folded;
2447
2448 // NOTE: if we wanted to, this is where to detect integer MIN/MAX
2449 bool Changed = false;
2450 Value *TrueVal = SI.getTrueValue();
2451 Value *FalseVal = SI.getFalseValue();
2452 ICmpInst::Predicate Pred = ICI->getPredicate();
2453 Value *CmpLHS = ICI->getOperand(i_nocapture: 0);
2454 Value *CmpRHS = ICI->getOperand(i_nocapture: 1);
2455
2456 if (Instruction *NewSel = foldSelectICmpEq(SI, ICI, IC&: *this))
2457 return NewSel;
2458
2459 // Canonicalize a signbit condition to use zero constant by swapping:
2460 // (CmpLHS > -1) ? TV : FV --> (CmpLHS < 0) ? FV : TV
2461 // To avoid conflicts (infinite loops) with other canonicalizations, this is
2462 // not applied with any constant select arm.
2463 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: m_AllOnes()) &&
2464 !match(V: TrueVal, P: m_Constant()) && !match(V: FalseVal, P: m_Constant()) &&
2465 ICI->hasOneUse()) {
2466 InstCombiner::BuilderTy::InsertPointGuard Guard(Builder);
2467 Builder.SetInsertPoint(&SI);
2468 Value *IsNeg = Builder.CreateIsNeg(Arg: CmpLHS, Name: ICI->getName());
2469 replaceOperand(I&: SI, OpNum: 0, V: IsNeg);
2470 SI.swapValues();
2471 SI.swapProfMetadata();
2472 return &SI;
2473 }
2474
2475 if (Value *V = foldSelectICmpMinMax(Cmp: ICI, TVal: TrueVal, FVal: FalseVal, Builder,
2476 SQ: SQ.getWithInstruction(I: &SI)))
2477 return replaceInstUsesWith(I&: SI, V);
2478
2479 if (Value *V = foldSelectICmpAndZeroShl(Cmp: ICI, TVal: TrueVal, FVal: FalseVal, Builder))
2480 return replaceInstUsesWith(I&: SI, V);
2481
2482 if (Instruction *V = foldSelectCtlzToCttz(ICI, TrueVal, FalseVal, Builder))
2483 return V;
2484
2485 if (Instruction *V = foldSelectZeroOrOnes(Cmp: ICI, TVal: TrueVal, FVal: FalseVal, Builder))
2486 return V;
2487
2488 if (Value *V = foldSelectICmpLshrAshr(IC: ICI, TrueVal, FalseVal, Builder))
2489 return replaceInstUsesWith(I&: SI, V);
2490
2491 if (Value *V = foldSelectCttzCtlz(ICI, TrueVal, FalseVal, IC&: *this))
2492 return replaceInstUsesWith(I&: SI, V);
2493
2494 if (Value *V = canonicalizeSaturatedSubtract(ICI, TrueVal, FalseVal, Builder))
2495 return replaceInstUsesWith(I&: SI, V);
2496
2497 if (Value *V = canonicalizeSaturatedAdd(Cmp: ICI, TVal: TrueVal, FVal: FalseVal, Builder))
2498 return replaceInstUsesWith(I&: SI, V);
2499
2500 if (Value *V = foldAbsDiff(Cmp: ICI, TVal: TrueVal, FVal: FalseVal, Builder))
2501 return replaceInstUsesWith(I&: SI, V);
2502
2503 if (Value *V = foldSelectWithConstOpToBinOp(Cmp: ICI, TrueVal, FalseVal))
2504 return replaceInstUsesWith(I&: SI, V);
2505
2506 return Changed ? &SI : nullptr;
2507}
2508
2509/// We have an SPF (e.g. a min or max) of an SPF of the form:
2510/// SPF2(SPF1(A, B), C)
2511Instruction *InstCombinerImpl::foldSPFofSPF(Instruction *Inner,
2512 SelectPatternFlavor SPF1, Value *A,
2513 Value *B, Instruction &Outer,
2514 SelectPatternFlavor SPF2,
2515 Value *C) {
2516 if (Outer.getType() != Inner->getType())
2517 return nullptr;
2518
2519 if (C == A || C == B) {
2520 // MAX(MAX(A, B), B) -> MAX(A, B)
2521 // MIN(MIN(a, b), a) -> MIN(a, b)
2522 // TODO: This could be done in instsimplify.
2523 if (SPF1 == SPF2 && SelectPatternResult::isMinOrMax(SPF: SPF1))
2524 return replaceInstUsesWith(I&: Outer, V: Inner);
2525 }
2526
2527 return nullptr;
2528}
2529
2530/// Turn select C, (X + Y), (X - Y) --> (X + (select C, Y, (-Y))).
2531/// This is even legal for FP.
2532static Instruction *foldAddSubSelect(SelectInst &SI,
2533 InstCombiner::BuilderTy &Builder) {
2534 Value *CondVal = SI.getCondition();
2535 Value *TrueVal = SI.getTrueValue();
2536 Value *FalseVal = SI.getFalseValue();
2537 auto *TI = dyn_cast<Instruction>(Val: TrueVal);
2538 auto *FI = dyn_cast<Instruction>(Val: FalseVal);
2539 if (!TI || !FI || !TI->hasOneUse() || !FI->hasOneUse())
2540 return nullptr;
2541
2542 Instruction *AddOp = nullptr, *SubOp = nullptr;
2543 if ((TI->getOpcode() == Instruction::Sub &&
2544 FI->getOpcode() == Instruction::Add) ||
2545 (TI->getOpcode() == Instruction::FSub &&
2546 FI->getOpcode() == Instruction::FAdd)) {
2547 AddOp = FI;
2548 SubOp = TI;
2549 } else if ((FI->getOpcode() == Instruction::Sub &&
2550 TI->getOpcode() == Instruction::Add) ||
2551 (FI->getOpcode() == Instruction::FSub &&
2552 TI->getOpcode() == Instruction::FAdd)) {
2553 AddOp = TI;
2554 SubOp = FI;
2555 }
2556
2557 if (AddOp) {
2558 Value *OtherAddOp = nullptr;
2559 if (SubOp->getOperand(i: 0) == AddOp->getOperand(i: 0)) {
2560 OtherAddOp = AddOp->getOperand(i: 1);
2561 } else if (SubOp->getOperand(i: 0) == AddOp->getOperand(i: 1)) {
2562 OtherAddOp = AddOp->getOperand(i: 0);
2563 }
2564
2565 if (OtherAddOp) {
2566 // So at this point we know we have (Y -> OtherAddOp):
2567 // select C, (add X, Y), (sub X, Z)
2568 Value *NegVal; // Compute -Z
2569 if (SI.getType()->isFPOrFPVectorTy()) {
2570 NegVal = Builder.CreateFNeg(V: SubOp->getOperand(i: 1));
2571 if (Instruction *NegInst = dyn_cast<Instruction>(Val: NegVal)) {
2572 FastMathFlags Flags = AddOp->getFastMathFlags();
2573 Flags &= SubOp->getFastMathFlags();
2574 NegInst->setFastMathFlags(Flags);
2575 }
2576 } else {
2577 NegVal = Builder.CreateNeg(V: SubOp->getOperand(i: 1));
2578 }
2579
2580 Value *NewTrueOp = OtherAddOp;
2581 Value *NewFalseOp = NegVal;
2582 if (AddOp != TI)
2583 std::swap(a&: NewTrueOp, b&: NewFalseOp);
2584 Value *NewSel = Builder.CreateSelect(C: CondVal, True: NewTrueOp, False: NewFalseOp,
2585 Name: SI.getName() + ".p", MDFrom: &SI);
2586
2587 if (SI.getType()->isFPOrFPVectorTy()) {
2588 Instruction *RI =
2589 BinaryOperator::CreateFAdd(V1: SubOp->getOperand(i: 0), V2: NewSel);
2590
2591 FastMathFlags Flags = AddOp->getFastMathFlags();
2592 Flags &= SubOp->getFastMathFlags();
2593 RI->setFastMathFlags(Flags);
2594 return RI;
2595 } else
2596 return BinaryOperator::CreateAdd(V1: SubOp->getOperand(i: 0), V2: NewSel);
2597 }
2598 }
2599 return nullptr;
2600}
2601
2602/// Turn X + Y overflows ? -1 : X + Y -> uadd_sat X, Y
2603/// And X - Y overflows ? 0 : X - Y -> usub_sat X, Y
2604/// Along with a number of patterns similar to:
2605/// X + Y overflows ? (X < 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2606/// X - Y overflows ? (X > 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2607static Instruction *
2608foldOverflowingAddSubSelect(SelectInst &SI, InstCombiner::BuilderTy &Builder) {
2609 Value *CondVal = SI.getCondition();
2610 Value *TrueVal = SI.getTrueValue();
2611 Value *FalseVal = SI.getFalseValue();
2612
2613 WithOverflowInst *II;
2614 if (!match(V: CondVal, P: m_ExtractValue<1>(V: m_WithOverflowInst(I&: II))) ||
2615 !match(V: FalseVal, P: m_ExtractValue<0>(V: m_Specific(V: II))))
2616 return nullptr;
2617
2618 Value *X = II->getLHS();
2619 Value *Y = II->getRHS();
2620
2621 auto IsSignedSaturateLimit = [&](Value *Limit, bool IsAdd) {
2622 Type *Ty = Limit->getType();
2623
2624 CmpPredicate Pred;
2625 Value *TrueVal, *FalseVal, *Op;
2626 const APInt *C;
2627 if (!match(V: Limit, P: m_Select(C: m_ICmp(Pred, L: m_Value(V&: Op), R: m_APInt(Res&: C)),
2628 L: m_Value(V&: TrueVal), R: m_Value(V&: FalseVal))))
2629 return false;
2630
2631 auto IsZeroOrOne = [](const APInt &C) { return C.isZero() || C.isOne(); };
2632 auto IsMinMax = [&](Value *Min, Value *Max) {
2633 APInt MinVal = APInt::getSignedMinValue(numBits: Ty->getScalarSizeInBits());
2634 APInt MaxVal = APInt::getSignedMaxValue(numBits: Ty->getScalarSizeInBits());
2635 return match(V: Min, P: m_SpecificInt(V: MinVal)) &&
2636 match(V: Max, P: m_SpecificInt(V: MaxVal));
2637 };
2638
2639 if (Op != X && Op != Y)
2640 return false;
2641
2642 if (IsAdd) {
2643 // X + Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2644 // X + Y overflows ? (X <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2645 // X + Y overflows ? (Y <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2646 // X + Y overflows ? (Y <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2647 if (Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C) &&
2648 IsMinMax(TrueVal, FalseVal))
2649 return true;
2650 // X + Y overflows ? (X >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2651 // X + Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2652 // X + Y overflows ? (Y >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2653 // X + Y overflows ? (Y >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2654 if (Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 1) &&
2655 IsMinMax(FalseVal, TrueVal))
2656 return true;
2657 } else {
2658 // X - Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2659 // X - Y overflows ? (X <s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2660 if (Op == X && Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C + 1) &&
2661 IsMinMax(TrueVal, FalseVal))
2662 return true;
2663 // X - Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2664 // X - Y overflows ? (X >s -2 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2665 if (Op == X && Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 2) &&
2666 IsMinMax(FalseVal, TrueVal))
2667 return true;
2668 // X - Y overflows ? (Y <s 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2669 // X - Y overflows ? (Y <s 1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2670 if (Op == Y && Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C) &&
2671 IsMinMax(FalseVal, TrueVal))
2672 return true;
2673 // X - Y overflows ? (Y >s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2674 // X - Y overflows ? (Y >s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2675 if (Op == Y && Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 1) &&
2676 IsMinMax(TrueVal, FalseVal))
2677 return true;
2678 }
2679
2680 return false;
2681 };
2682
2683 Intrinsic::ID NewIntrinsicID;
2684 if (II->getIntrinsicID() == Intrinsic::uadd_with_overflow &&
2685 match(V: TrueVal, P: m_AllOnes()))
2686 // X + Y overflows ? -1 : X + Y -> uadd_sat X, Y
2687 NewIntrinsicID = Intrinsic::uadd_sat;
2688 else if (II->getIntrinsicID() == Intrinsic::usub_with_overflow &&
2689 match(V: TrueVal, P: m_Zero()))
2690 // X - Y overflows ? 0 : X - Y -> usub_sat X, Y
2691 NewIntrinsicID = Intrinsic::usub_sat;
2692 else if (II->getIntrinsicID() == Intrinsic::sadd_with_overflow &&
2693 IsSignedSaturateLimit(TrueVal, /*IsAdd=*/true))
2694 // X + Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2695 // X + Y overflows ? (X <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2696 // X + Y overflows ? (X >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2697 // X + Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2698 // X + Y overflows ? (Y <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2699 // X + Y overflows ? (Y <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
2700 // X + Y overflows ? (Y >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2701 // X + Y overflows ? (Y >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
2702 NewIntrinsicID = Intrinsic::sadd_sat;
2703 else if (II->getIntrinsicID() == Intrinsic::ssub_with_overflow &&
2704 IsSignedSaturateLimit(TrueVal, /*IsAdd=*/false))
2705 // X - Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2706 // X - Y overflows ? (X <s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2707 // X - Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2708 // X - Y overflows ? (X >s -2 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2709 // X - Y overflows ? (Y <s 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2710 // X - Y overflows ? (Y <s 1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
2711 // X - Y overflows ? (Y >s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2712 // X - Y overflows ? (Y >s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
2713 NewIntrinsicID = Intrinsic::ssub_sat;
2714 else
2715 return nullptr;
2716
2717 Function *F = Intrinsic::getOrInsertDeclaration(M: SI.getModule(),
2718 id: NewIntrinsicID, OverloadTys: SI.getType());
2719 return CallInst::Create(Func: F, Args: {X, Y});
2720}
2721
2722Instruction *InstCombinerImpl::foldSelectExtConst(SelectInst &Sel) {
2723 Constant *C;
2724 if (!match(V: Sel.getTrueValue(), P: m_Constant(C)) &&
2725 !match(V: Sel.getFalseValue(), P: m_Constant(C)))
2726 return nullptr;
2727
2728 Instruction *ExtInst;
2729 if (!match(V: Sel.getTrueValue(), P: m_Instruction(I&: ExtInst)) &&
2730 !match(V: Sel.getFalseValue(), P: m_Instruction(I&: ExtInst)))
2731 return nullptr;
2732
2733 auto ExtOpcode = ExtInst->getOpcode();
2734 if (ExtOpcode != Instruction::ZExt && ExtOpcode != Instruction::SExt)
2735 return nullptr;
2736
2737 // If we are extending from a boolean type or if we can create a select that
2738 // has the same size operands as its condition, try to narrow the select.
2739 Value *X = ExtInst->getOperand(i: 0);
2740 Type *SmallType = X->getType();
2741 Value *Cond = Sel.getCondition();
2742 if (!SmallType->isIntOrIntVectorTy(BitWidth: 1) &&
2743 (!isa<CmpInst, TruncInst>(Val: Cond) ||
2744 cast<Instruction>(Val: Cond)->getOperand(i: 0)->getType() != SmallType))
2745 return nullptr;
2746
2747 // If the constant is the same after truncation to the smaller type and
2748 // extension to the original type, we can narrow the select.
2749 Type *SelType = Sel.getType();
2750 Constant *TruncC = getLosslessInvCast(C, InvCastTo: SmallType, CastOp: ExtOpcode, DL);
2751 if (TruncC && ExtInst->hasOneUse()) {
2752 Value *TruncCVal = cast<Value>(Val: TruncC);
2753 if (ExtInst == Sel.getFalseValue())
2754 std::swap(a&: X, b&: TruncCVal);
2755
2756 // select Cond, (ext X), C --> ext(select Cond, X, C')
2757 // select Cond, C, (ext X) --> ext(select Cond, C', X)
2758 Value *NewSel = Builder.CreateSelect(C: Cond, True: X, False: TruncCVal, Name: "narrow", MDFrom: &Sel);
2759 return CastInst::Create(Instruction::CastOps(ExtOpcode), S: NewSel, Ty: SelType);
2760 }
2761
2762 return nullptr;
2763}
2764
2765/// Try to transform a vector select with a constant condition vector into a
2766/// shuffle for easier combining with other shuffles and insert/extract.
2767static Instruction *canonicalizeSelectToShuffle(SelectInst &SI) {
2768 Value *CondVal = SI.getCondition();
2769 Constant *CondC;
2770 auto *CondValTy = dyn_cast<FixedVectorType>(Val: CondVal->getType());
2771 if (!CondValTy || !match(V: CondVal, P: m_Constant(C&: CondC)))
2772 return nullptr;
2773
2774 unsigned NumElts = CondValTy->getNumElements();
2775 SmallVector<int, 16> Mask;
2776 Mask.reserve(N: NumElts);
2777 for (unsigned i = 0; i != NumElts; ++i) {
2778 Constant *Elt = CondC->getAggregateElement(Elt: i);
2779 if (!Elt)
2780 return nullptr;
2781
2782 if (Elt->isOneValue()) {
2783 // If the select condition element is true, choose from the 1st vector.
2784 Mask.push_back(Elt: i);
2785 } else if (Elt->isNullValue()) {
2786 // If the select condition element is false, choose from the 2nd vector.
2787 Mask.push_back(Elt: i + NumElts);
2788 } else if (isa<UndefValue>(Val: Elt)) {
2789 // Undef in a select condition (choose one of the operands) does not mean
2790 // the same thing as undef in a shuffle mask (any value is acceptable), so
2791 // give up.
2792 return nullptr;
2793 } else {
2794 // Bail out on a constant expression.
2795 return nullptr;
2796 }
2797 }
2798
2799 return new ShuffleVectorInst(SI.getTrueValue(), SI.getFalseValue(), Mask);
2800}
2801
2802/// If we have a select of vectors with a scalar condition, try to convert that
2803/// to a vector select by splatting the condition. A splat may get folded with
2804/// other operations in IR and having all operands of a select be vector types
2805/// is likely better for vector codegen.
2806static Instruction *canonicalizeScalarSelectOfVecs(SelectInst &Sel,
2807 InstCombinerImpl &IC) {
2808 auto *Ty = dyn_cast<VectorType>(Val: Sel.getType());
2809 if (!Ty)
2810 return nullptr;
2811
2812 // We can replace a single-use extract with constant index.
2813 Value *Cond = Sel.getCondition();
2814 if (!match(V: Cond, P: m_OneUse(SubPattern: m_ExtractElt(Val: m_Value(), Idx: m_ConstantInt()))))
2815 return nullptr;
2816
2817 // select (extelt V, Index), T, F --> select (splat V, Index), T, F
2818 // Splatting the extracted condition reduces code (we could directly create a
2819 // splat shuffle of the source vector to eliminate the intermediate step).
2820 return IC.replaceOperand(
2821 I&: Sel, OpNum: 0, V: IC.Builder.CreateVectorSplat(EC: Ty->getElementCount(), V: Cond));
2822}
2823
2824/// Reuse bitcasted operands between a compare and select:
2825/// select (cmp (bitcast C), (bitcast D)), (bitcast' C), (bitcast' D) -->
2826/// bitcast (select (cmp (bitcast C), (bitcast D)), (bitcast C), (bitcast D))
2827static Instruction *foldSelectCmpBitcasts(SelectInst &Sel,
2828 InstCombiner::BuilderTy &Builder) {
2829 Value *Cond = Sel.getCondition();
2830 Value *TVal = Sel.getTrueValue();
2831 Value *FVal = Sel.getFalseValue();
2832
2833 CmpPredicate Pred;
2834 Value *A, *B;
2835 if (!match(V: Cond, P: m_Cmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B))))
2836 return nullptr;
2837
2838 // The select condition is a compare instruction. If the select's true/false
2839 // values are already the same as the compare operands, there's nothing to do.
2840 if (TVal == A || TVal == B || FVal == A || FVal == B)
2841 return nullptr;
2842
2843 Value *C, *D;
2844 if (!match(V: A, P: m_BitCast(Op: m_Value(V&: C))) || !match(V: B, P: m_BitCast(Op: m_Value(V&: D))))
2845 return nullptr;
2846
2847 // select (cmp (bitcast C), (bitcast D)), (bitcast TSrc), (bitcast FSrc)
2848 Value *TSrc, *FSrc;
2849 if (!match(V: TVal, P: m_BitCast(Op: m_Value(V&: TSrc))) ||
2850 !match(V: FVal, P: m_BitCast(Op: m_Value(V&: FSrc))))
2851 return nullptr;
2852
2853 // If the select true/false values are *different bitcasts* of the same source
2854 // operands, make the select operands the same as the compare operands and
2855 // cast the result. This is the canonical select form for min/max.
2856 Value *NewSel;
2857 if (TSrc == C && FSrc == D) {
2858 // select (cmp (bitcast C), (bitcast D)), (bitcast' C), (bitcast' D) -->
2859 // bitcast (select (cmp A, B), A, B)
2860 NewSel = Builder.CreateSelect(C: Cond, True: A, False: B, Name: "", MDFrom: &Sel);
2861 } else if (TSrc == D && FSrc == C) {
2862 // select (cmp (bitcast C), (bitcast D)), (bitcast' D), (bitcast' C) -->
2863 // bitcast (select (cmp A, B), B, A)
2864 NewSel = Builder.CreateSelect(C: Cond, True: B, False: A, Name: "", MDFrom: &Sel);
2865 } else {
2866 return nullptr;
2867 }
2868 return new BitCastInst(NewSel, Sel.getType());
2869}
2870
2871/// Try to eliminate select instructions that test the returned flag of cmpxchg
2872/// instructions.
2873///
2874/// If a select instruction tests the returned flag of a cmpxchg instruction and
2875/// selects between the returned value of the cmpxchg instruction its compare
2876/// operand, the result of the select will always be equal to its false value.
2877/// For example:
2878///
2879/// %cmpxchg = cmpxchg ptr %ptr, i64 %compare, i64 %new_value seq_cst seq_cst
2880/// %val = extractvalue { i64, i1 } %cmpxchg, 0
2881/// %success = extractvalue { i64, i1 } %cmpxchg, 1
2882/// %sel = select i1 %success, i64 %compare, i64 %val
2883/// ret i64 %sel
2884///
2885/// The returned value of the cmpxchg instruction (%val) is the original value
2886/// located at %ptr prior to any update. If the cmpxchg operation succeeds, %val
2887/// must have been equal to %compare. Thus, the result of the select is always
2888/// equal to %val, and the code can be simplified to:
2889///
2890/// %cmpxchg = cmpxchg ptr %ptr, i64 %compare, i64 %new_value seq_cst seq_cst
2891/// %val = extractvalue { i64, i1 } %cmpxchg, 0
2892/// ret i64 %val
2893///
2894static Value *foldSelectCmpXchg(SelectInst &SI) {
2895 // A helper that determines if V is an extractvalue instruction whose
2896 // aggregate operand is a cmpxchg instruction and whose single index is equal
2897 // to I. If such conditions are true, the helper returns the cmpxchg
2898 // instruction; otherwise, a nullptr is returned.
2899 auto isExtractFromCmpXchg = [](Value *V, unsigned I) -> AtomicCmpXchgInst * {
2900 // When extracting the value loaded by a cmpxchg, allow peeking through a
2901 // bitcast. These are inserted for floating-point cmpxchg, for example:
2902 // %bc = bitcast float %compare to i32
2903 // %cmpxchg = cmpxchg ptr %ptr, i32 %bc, i32 %new_value seq_cst seq_cst
2904 // %val = extractvalue { i32, i1 } %cmpxchg, 0
2905 // %success = extractvalue { i32, i1 } %cmpxchg, 1
2906 // %val.bc = bitcast i32 %val to float
2907 // %sel = select i1 %success, float %compare, float %val.bc
2908 if (auto *BI = dyn_cast<BitCastInst>(Val: V); BI && I == 0)
2909 V = BI->getOperand(i_nocapture: 0);
2910 auto *Extract = dyn_cast<ExtractValueInst>(Val: V);
2911 if (!Extract)
2912 return nullptr;
2913 if (Extract->getIndices()[0] != I)
2914 return nullptr;
2915 return dyn_cast<AtomicCmpXchgInst>(Val: Extract->getAggregateOperand());
2916 };
2917
2918 // Check if the compare value of a cmpxchg matches another value.
2919 auto isCompareSameAsValue = [](Value *CmpVal, Value *SelVal) {
2920 // The values match if they are the same or %CmpVal = bitcast %SelVal (see
2921 // above).
2922 if (CmpVal == SelVal || match(V: CmpVal, P: m_BitCast(Op: m_Specific(V: SelVal))))
2923 return true;
2924 // For FP constants, the value may have been bitcast to Int directly.
2925 auto *IntC = dyn_cast<ConstantInt>(Val: CmpVal);
2926 auto *FpC = dyn_cast<ConstantFP>(Val: SelVal);
2927 return IntC && FpC && IntC->getValue() == FpC->getValue().bitcastToAPInt();
2928 };
2929
2930 // If the select has a single user, and this user is a select instruction that
2931 // we can simplify, skip the cmpxchg simplification for now.
2932 if (SI.hasOneUse())
2933 if (auto *Select = dyn_cast<SelectInst>(Val: SI.user_back()))
2934 if (Select->getCondition() == SI.getCondition())
2935 if (Select->getFalseValue() == SI.getTrueValue() ||
2936 Select->getTrueValue() == SI.getFalseValue())
2937 return nullptr;
2938
2939 // Ensure the select condition is the returned flag of a cmpxchg instruction.
2940 auto *CmpXchg = isExtractFromCmpXchg(SI.getCondition(), 1);
2941 if (!CmpXchg)
2942 return nullptr;
2943
2944 // Check the true value case: The true value of the select is the returned
2945 // value of the same cmpxchg used by the condition, and the false value is the
2946 // cmpxchg instruction's compare operand.
2947 if (auto *X = isExtractFromCmpXchg(SI.getTrueValue(), 0))
2948 if (X == CmpXchg &&
2949 isCompareSameAsValue(X->getCompareOperand(), SI.getFalseValue()))
2950 return SI.getFalseValue();
2951
2952 // Check the false value case: The false value of the select is the returned
2953 // value of the same cmpxchg used by the condition, and the true value is the
2954 // cmpxchg instruction's compare operand.
2955 if (auto *X = isExtractFromCmpXchg(SI.getFalseValue(), 0))
2956 if (X == CmpXchg &&
2957 isCompareSameAsValue(X->getCompareOperand(), SI.getTrueValue()))
2958 return SI.getFalseValue();
2959
2960 return nullptr;
2961}
2962
2963/// Try to reduce a funnel/rotate pattern that includes a compare and select
2964/// into a funnel shift intrinsic. Example:
2965/// rotl32(a, b) --> (b == 0 ? a : ((a >> (32 - b)) | (a << b)))
2966/// --> call llvm.fshl.i32(a, a, b)
2967/// fshl32(a, b, c) --> (c == 0 ? a : ((b >> (32 - c)) | (a << c)))
2968/// --> call llvm.fshl.i32(a, b, c)
2969/// fshr32(a, b, c) --> (c == 0 ? b : ((a >> (32 - c)) | (b << c)))
2970/// --> call llvm.fshr.i32(a, b, c)
2971static Instruction *foldSelectFunnelShift(SelectInst &Sel,
2972 InstCombiner::BuilderTy &Builder) {
2973 // This must be a power-of-2 type for a bitmasking transform to be valid.
2974 unsigned Width = Sel.getType()->getScalarSizeInBits();
2975 if (!isPowerOf2_32(Value: Width))
2976 return nullptr;
2977
2978 BinaryOperator *Or0, *Or1;
2979 if (!match(V: Sel.getFalseValue(), P: m_OneUse(SubPattern: m_Or(L: m_BinOp(I&: Or0), R: m_BinOp(I&: Or1)))))
2980 return nullptr;
2981
2982 Value *SV0, *SV1, *SA0, *SA1;
2983 if (!match(V: Or0, P: m_OneUse(SubPattern: m_LogicalShift(L: m_Value(V&: SV0),
2984 R: m_ZExtOrSelf(Op: m_Value(V&: SA0))))) ||
2985 !match(V: Or1, P: m_OneUse(SubPattern: m_LogicalShift(L: m_Value(V&: SV1),
2986 R: m_ZExtOrSelf(Op: m_Value(V&: SA1))))) ||
2987 Or0->getOpcode() == Or1->getOpcode())
2988 return nullptr;
2989
2990 // Canonicalize to or(shl(SV0, SA0), lshr(SV1, SA1)).
2991 if (Or0->getOpcode() == BinaryOperator::LShr) {
2992 std::swap(a&: Or0, b&: Or1);
2993 std::swap(a&: SV0, b&: SV1);
2994 std::swap(a&: SA0, b&: SA1);
2995 }
2996 assert(Or0->getOpcode() == BinaryOperator::Shl &&
2997 Or1->getOpcode() == BinaryOperator::LShr &&
2998 "Illegal or(shift,shift) pair");
2999
3000 // Check the shift amounts to see if they are an opposite pair.
3001 Value *ShAmt;
3002 if (match(V: SA1, P: m_OneUse(SubPattern: m_Sub(L: m_SpecificInt(V: Width), R: m_Specific(V: SA0)))))
3003 ShAmt = SA0;
3004 else if (match(V: SA0, P: m_OneUse(SubPattern: m_Sub(L: m_SpecificInt(V: Width), R: m_Specific(V: SA1)))))
3005 ShAmt = SA1;
3006 else
3007 return nullptr;
3008
3009 // We should now have this pattern:
3010 // select ?, TVal, (or (shl SV0, SA0), (lshr SV1, SA1))
3011 // The false value of the select must be a funnel-shift of the true value:
3012 // IsFShl -> TVal must be SV0 else TVal must be SV1.
3013 bool IsFshl = (ShAmt == SA0);
3014 Value *TVal = Sel.getTrueValue();
3015 if ((IsFshl && TVal != SV0) || (!IsFshl && TVal != SV1))
3016 return nullptr;
3017
3018 // Finally, see if the select is filtering out a shift-by-zero.
3019 Value *Cond = Sel.getCondition();
3020 if (!match(V: Cond, P: m_OneUse(SubPattern: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Specific(V: ShAmt),
3021 R: m_ZeroInt()))))
3022 return nullptr;
3023
3024 // If this is not a rotate then the select was blocking poison from the
3025 // 'shift-by-zero' non-TVal, but a funnel shift won't - so freeze it.
3026 if (SV0 != SV1) {
3027 if (IsFshl && !llvm::isGuaranteedNotToBePoison(V: SV1))
3028 SV1 = Builder.CreateFreeze(V: SV1);
3029 else if (!IsFshl && !llvm::isGuaranteedNotToBePoison(V: SV0))
3030 SV0 = Builder.CreateFreeze(V: SV0);
3031 }
3032
3033 // This is a funnel/rotate that avoids shift-by-bitwidth UB in a suboptimal way.
3034 // Convert to funnel shift intrinsic.
3035 Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
3036 Function *F =
3037 Intrinsic::getOrInsertDeclaration(M: Sel.getModule(), id: IID, OverloadTys: Sel.getType());
3038 ShAmt = Builder.CreateZExt(V: ShAmt, DestTy: Sel.getType());
3039 return CallInst::Create(Func: F, Args: { SV0, SV1, ShAmt });
3040}
3041
3042static Instruction *foldSelectToCopysign(SelectInst &Sel,
3043 InstCombiner::BuilderTy &Builder) {
3044 Value *Cond = Sel.getCondition();
3045 Value *TVal = Sel.getTrueValue();
3046 Value *FVal = Sel.getFalseValue();
3047 Type *SelType = Sel.getType();
3048
3049 // Match select ?, TC, FC where the constants are equal but negated.
3050 // TODO: Generalize to handle a negated variable operand?
3051 const APFloat *TC, *FC;
3052 if (!match(V: TVal, P: m_APFloatAllowPoison(Res&: TC)) ||
3053 !match(V: FVal, P: m_APFloatAllowPoison(Res&: FC)) ||
3054 !abs(X: *TC).bitwiseIsEqual(RHS: abs(X: *FC)))
3055 return nullptr;
3056
3057 assert(TC != FC && "Expected equal select arms to simplify");
3058
3059 Value *X;
3060 const APInt *C;
3061 bool IsTrueIfSignSet;
3062 CmpPredicate Pred;
3063 if (!match(V: Cond, P: m_OneUse(SubPattern: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Value(V&: X)),
3064 R: m_APInt(Res&: C)))) ||
3065 !isSignBitCheck(Pred, RHS: *C, TrueIfSigned&: IsTrueIfSignSet) || X->getType() != SelType)
3066 return nullptr;
3067
3068 // If needed, negate the value that will be the sign argument of the copysign:
3069 // (bitcast X) < 0 ? -TC : TC --> copysign(TC, X)
3070 // (bitcast X) < 0 ? TC : -TC --> copysign(TC, -X)
3071 // (bitcast X) >= 0 ? -TC : TC --> copysign(TC, -X)
3072 // (bitcast X) >= 0 ? TC : -TC --> copysign(TC, X)
3073 // Note: FMF from the select can not be propagated to the new instructions.
3074 if (IsTrueIfSignSet ^ TC->isNegative())
3075 X = Builder.CreateFNeg(V: X);
3076
3077 // Canonicalize the magnitude argument as the positive constant since we do
3078 // not care about its sign.
3079 Value *MagArg = ConstantFP::get(Ty: SelType, V: abs(X: *TC));
3080 Function *F = Intrinsic::getOrInsertDeclaration(
3081 M: Sel.getModule(), id: Intrinsic::copysign, OverloadTys: Sel.getType());
3082 return CallInst::Create(Func: F, Args: { MagArg, X });
3083}
3084
3085Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) {
3086 if (!isa<VectorType>(Val: Sel.getType()))
3087 return nullptr;
3088
3089 Value *Cond = Sel.getCondition();
3090 Value *TVal = Sel.getTrueValue();
3091 Value *FVal = Sel.getFalseValue();
3092 Value *C, *X, *Y;
3093
3094 if (match(V: Cond, P: m_VecReverse(Op0: m_Value(V&: C)))) {
3095 auto createSelReverse = [&](Value *C, Value *X, Value *Y) {
3096 Value *V = Builder.CreateSelect(C, True: X, False: Y, Name: Sel.getName(), MDFrom: &Sel);
3097 if (auto *I = dyn_cast<Instruction>(Val: V))
3098 I->copyIRFlags(V: &Sel);
3099 Module *M = Sel.getModule();
3100 Function *F = Intrinsic::getOrInsertDeclaration(
3101 M, id: Intrinsic::vector_reverse, OverloadTys: V->getType());
3102 return CallInst::Create(Func: F, Args: V);
3103 };
3104
3105 if (match(V: TVal, P: m_VecReverse(Op0: m_Value(V&: X)))) {
3106 // select rev(C), rev(X), rev(Y) --> rev(select C, X, Y)
3107 if (match(V: FVal, P: m_VecReverse(Op0: m_Value(V&: Y))) &&
3108 (Cond->hasOneUse() || TVal->hasOneUse() || FVal->hasOneUse()))
3109 return createSelReverse(C, X, Y);
3110
3111 // select rev(C), rev(X), FValSplat --> rev(select C, X, FValSplat)
3112 if ((Cond->hasOneUse() || TVal->hasOneUse()) && isSplatValue(V: FVal))
3113 return createSelReverse(C, X, FVal);
3114 }
3115 // select rev(C), TValSplat, rev(Y) --> rev(select C, TValSplat, Y)
3116 else if (isSplatValue(V: TVal) && match(V: FVal, P: m_VecReverse(Op0: m_Value(V&: Y))) &&
3117 (Cond->hasOneUse() || FVal->hasOneUse()))
3118 return createSelReverse(C, TVal, Y);
3119 }
3120
3121 auto *VecTy = dyn_cast<FixedVectorType>(Val: Sel.getType());
3122 if (!VecTy)
3123 return nullptr;
3124
3125 unsigned NumElts = VecTy->getNumElements();
3126 APInt PoisonElts(NumElts, 0);
3127 APInt AllOnesEltMask(APInt::getAllOnes(numBits: NumElts));
3128 if (Value *V = SimplifyDemandedVectorElts(V: &Sel, DemandedElts: AllOnesEltMask, PoisonElts)) {
3129 if (V != &Sel)
3130 return replaceInstUsesWith(I&: Sel, V);
3131 return &Sel;
3132 }
3133
3134 // A select of a "select shuffle" with a common operand can be rearranged
3135 // to select followed by "select shuffle". Because of poison, this only works
3136 // in the case of a shuffle with no undefined mask elements.
3137 ArrayRef<int> Mask;
3138 if (match(V: TVal, P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: X), v2: m_Value(V&: Y), mask: m_Mask(Mask)))) &&
3139 !is_contained(Range&: Mask, Element: PoisonMaskElem) &&
3140 cast<ShuffleVectorInst>(Val: TVal)->isSelect()) {
3141 if (X == FVal) {
3142 // select Cond, (shuf_sel X, Y), X --> shuf_sel X, (select Cond, Y, X)
3143 Value *NewSel = Builder.CreateSelect(C: Cond, True: Y, False: X, Name: "sel", MDFrom: &Sel);
3144 return new ShuffleVectorInst(X, NewSel, Mask);
3145 }
3146 if (Y == FVal) {
3147 // select Cond, (shuf_sel X, Y), Y --> shuf_sel (select Cond, X, Y), Y
3148 Value *NewSel = Builder.CreateSelect(C: Cond, True: X, False: Y, Name: "sel", MDFrom: &Sel);
3149 return new ShuffleVectorInst(NewSel, Y, Mask);
3150 }
3151 }
3152 if (match(V: FVal, P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: X), v2: m_Value(V&: Y), mask: m_Mask(Mask)))) &&
3153 !is_contained(Range&: Mask, Element: PoisonMaskElem) &&
3154 cast<ShuffleVectorInst>(Val: FVal)->isSelect()) {
3155 if (X == TVal) {
3156 // select Cond, X, (shuf_sel X, Y) --> shuf_sel X, (select Cond, X, Y)
3157 Value *NewSel = Builder.CreateSelect(C: Cond, True: X, False: Y, Name: "sel", MDFrom: &Sel);
3158 return new ShuffleVectorInst(X, NewSel, Mask);
3159 }
3160 if (Y == TVal) {
3161 // select Cond, Y, (shuf_sel X, Y) --> shuf_sel (select Cond, Y, X), Y
3162 Value *NewSel = Builder.CreateSelect(C: Cond, True: Y, False: X, Name: "sel", MDFrom: &Sel);
3163 return new ShuffleVectorInst(NewSel, Y, Mask);
3164 }
3165 }
3166
3167 return nullptr;
3168}
3169
3170static Instruction *foldSelectToPhiImpl(SelectInst &Sel, BasicBlock *BB,
3171 const DominatorTree &DT,
3172 InstCombiner::BuilderTy &Builder) {
3173 // Find the block's immediate dominator that ends with a conditional branch
3174 // that matches select's condition (maybe inverted).
3175 auto *IDomNode = DT[BB]->getIDom();
3176 if (!IDomNode)
3177 return nullptr;
3178 BasicBlock *IDom = IDomNode->getBlock();
3179
3180 Value *Cond = Sel.getCondition();
3181 Value *IfTrue, *IfFalse;
3182 BasicBlock *TrueSucc, *FalseSucc;
3183 if (match(V: IDom->getTerminator(),
3184 P: m_Br(C: m_Specific(V: Cond), T: m_BasicBlock(V&: TrueSucc),
3185 F: m_BasicBlock(V&: FalseSucc)))) {
3186 IfTrue = Sel.getTrueValue();
3187 IfFalse = Sel.getFalseValue();
3188 } else if (match(V: IDom->getTerminator(),
3189 P: m_Br(C: m_Not(V: m_Specific(V: Cond)), T: m_BasicBlock(V&: TrueSucc),
3190 F: m_BasicBlock(V&: FalseSucc)))) {
3191 IfTrue = Sel.getFalseValue();
3192 IfFalse = Sel.getTrueValue();
3193 } else
3194 return nullptr;
3195
3196 // Make sure the branches are actually different.
3197 if (TrueSucc == FalseSucc)
3198 return nullptr;
3199
3200 // We want to replace select %cond, %a, %b with a phi that takes value %a
3201 // for all incoming edges that are dominated by condition `%cond == true`,
3202 // and value %b for edges dominated by condition `%cond == false`. If %a
3203 // or %b are also phis from the same basic block, we can go further and take
3204 // their incoming values from the corresponding blocks.
3205 BasicBlockEdge TrueEdge(IDom, TrueSucc);
3206 BasicBlockEdge FalseEdge(IDom, FalseSucc);
3207 DenseMap<BasicBlock *, Value *> Inputs;
3208 for (auto *Pred : predecessors(BB)) {
3209 // Check implication.
3210 BasicBlockEdge Incoming(Pred, BB);
3211 if (DT.dominates(BBE1: TrueEdge, BBE2: Incoming))
3212 Inputs[Pred] = IfTrue->DoPHITranslation(CurBB: BB, PredBB: Pred);
3213 else if (DT.dominates(BBE1: FalseEdge, BBE2: Incoming))
3214 Inputs[Pred] = IfFalse->DoPHITranslation(CurBB: BB, PredBB: Pred);
3215 else
3216 return nullptr;
3217 // Check availability.
3218 if (auto *Insn = dyn_cast<Instruction>(Val: Inputs[Pred]))
3219 if (!DT.dominates(Def: Insn, User: Pred->getTerminator()))
3220 return nullptr;
3221 }
3222
3223 Builder.SetInsertPoint(BB->begin());
3224 auto *PN = Builder.CreatePHI(Ty: Sel.getType(), NumReservedValues: Inputs.size());
3225 for (auto *Pred : predecessors(BB))
3226 PN->addIncoming(V: Inputs[Pred], BB: Pred);
3227 PN->takeName(V: &Sel);
3228 return PN;
3229}
3230
3231static Instruction *foldSelectToPhi(SelectInst &Sel, const DominatorTree &DT,
3232 InstCombiner::BuilderTy &Builder) {
3233 // Try to replace this select with Phi in one of these blocks.
3234 SmallSetVector<BasicBlock *, 4> CandidateBlocks;
3235 CandidateBlocks.insert(X: Sel.getParent());
3236 for (Value *V : Sel.operands())
3237 if (auto *I = dyn_cast<Instruction>(Val: V))
3238 CandidateBlocks.insert(X: I->getParent());
3239
3240 for (BasicBlock *BB : CandidateBlocks)
3241 if (auto *PN = foldSelectToPhiImpl(Sel, BB, DT, Builder))
3242 return PN;
3243 return nullptr;
3244}
3245
3246/// Tries to reduce a pattern that arises when calculating the remainder of the
3247/// Euclidean division. When the divisor is a power of two and is guaranteed not
3248/// to be negative, a signed remainder can be folded with a bitwise and.
3249///
3250/// (x % n) < 0 ? (x % n) + n : (x % n)
3251/// -> x & (n - 1)
3252static Instruction *foldSelectWithSRem(SelectInst &SI, InstCombinerImpl &IC,
3253 IRBuilderBase &Builder) {
3254 Value *CondVal = SI.getCondition();
3255 Value *TrueVal = SI.getTrueValue();
3256 Value *FalseVal = SI.getFalseValue();
3257
3258 CmpPredicate Pred;
3259 Value *Op, *RemRes, *Remainder;
3260 const APInt *C;
3261 bool TrueIfSigned = false;
3262
3263 if (!(match(V: CondVal, P: m_ICmp(Pred, L: m_Value(V&: RemRes), R: m_APInt(Res&: C))) &&
3264 isSignBitCheck(Pred, RHS: *C, TrueIfSigned)))
3265 return nullptr;
3266
3267 // If the sign bit is not set, we have a SGE/SGT comparison, and the operands
3268 // of the select are inverted.
3269 if (!TrueIfSigned)
3270 std::swap(a&: TrueVal, b&: FalseVal);
3271
3272 auto FoldToBitwiseAnd = [&](Value *Remainder) -> Instruction * {
3273 Value *Add = Builder.CreateAdd(
3274 LHS: Remainder, RHS: Constant::getAllOnesValue(Ty: RemRes->getType()));
3275 return BinaryOperator::CreateAnd(V1: Op, V2: Add);
3276 };
3277
3278 // Match the general case:
3279 // %rem = srem i32 %x, %n
3280 // %cnd = icmp slt i32 %rem, 0
3281 // %add = add i32 %rem, %n
3282 // %sel = select i1 %cnd, i32 %add, i32 %rem
3283 if (match(V: TrueVal, P: m_c_Add(L: m_Specific(V: RemRes), R: m_Value(V&: Remainder))) &&
3284 match(V: RemRes, P: m_SRem(L: m_Value(V&: Op), R: m_Specific(V: Remainder))) &&
3285 IC.isKnownToBeAPowerOfTwo(V: Remainder, /*OrZero=*/true) &&
3286 FalseVal == RemRes)
3287 return FoldToBitwiseAnd(Remainder);
3288
3289 // Match the case where the one arm has been replaced by constant 1:
3290 // %rem = srem i32 %n, 2
3291 // %cnd = icmp slt i32 %rem, 0
3292 // %sel = select i1 %cnd, i32 1, i32 %rem
3293 if (match(V: TrueVal, P: m_One()) &&
3294 match(V: RemRes, P: m_SRem(L: m_Value(V&: Op), R: m_SpecificInt(V: 2))) &&
3295 FalseVal == RemRes)
3296 return FoldToBitwiseAnd(ConstantInt::get(Ty: RemRes->getType(), V: 2));
3297
3298 return nullptr;
3299}
3300
3301/// Given that \p CondVal is known to be \p CondIsTrue, try to simplify \p SI.
3302static Value *simplifyNestedSelectsUsingImpliedCond(SelectInst &SI,
3303 Value *CondVal,
3304 bool CondIsTrue,
3305 const DataLayout &DL) {
3306 Value *InnerCondVal = SI.getCondition();
3307 Value *InnerTrueVal = SI.getTrueValue();
3308 Value *InnerFalseVal = SI.getFalseValue();
3309 assert(CondVal->getType() == InnerCondVal->getType() &&
3310 "The type of inner condition must match with the outer.");
3311 if (auto Implied = isImpliedCondition(LHS: CondVal, RHS: InnerCondVal, DL, LHSIsTrue: CondIsTrue))
3312 return *Implied ? InnerTrueVal : InnerFalseVal;
3313 return nullptr;
3314}
3315
3316Instruction *InstCombinerImpl::foldAndOrOfSelectUsingImpliedCond(Value *Op,
3317 SelectInst &SI,
3318 bool IsAnd) {
3319 assert(Op->getType()->isIntOrIntVectorTy(1) &&
3320 "Op must be either i1 or vector of i1.");
3321 if (SI.getCondition()->getType() != Op->getType())
3322 return nullptr;
3323 if (Value *V = simplifyNestedSelectsUsingImpliedCond(SI, CondVal: Op, CondIsTrue: IsAnd, DL))
3324 return createSelectInstWithUnknownProfile(
3325 C: Op, S1: IsAnd ? V : ConstantInt::getTrue(Ty: Op->getType()),
3326 S2: IsAnd ? ConstantInt::getFalse(Ty: Op->getType()) : V);
3327 return nullptr;
3328}
3329
3330// Canonicalize select with fcmp to fabs(). -0.0 makes this tricky. We need
3331// fast-math-flags (nsz) or fsub with +0.0 (not fneg) for this to work.
3332static Instruction *foldSelectWithFCmpToFabs(SelectInst &SI,
3333 InstCombinerImpl &IC) {
3334 Value *CondVal = SI.getCondition();
3335
3336 bool ChangedFMF = false;
3337 for (bool Swap : {false, true}) {
3338 Value *TrueVal = SI.getTrueValue();
3339 Value *X = SI.getFalseValue();
3340 CmpPredicate Pred;
3341
3342 if (Swap)
3343 std::swap(a&: TrueVal, b&: X);
3344
3345 if (!match(V: CondVal, P: m_FCmp(Pred, L: m_Specific(V: X), R: m_AnyZeroFP())))
3346 continue;
3347
3348 // fold (X <= +/-0.0) ? (0.0 - X) : X to fabs(X), when 'Swap' is false
3349 // fold (X > +/-0.0) ? X : (0.0 - X) to fabs(X), when 'Swap' is true
3350 // Note: We require "nnan" for this fold because fcmp ignores the signbit
3351 // of NAN, but IEEE-754 specifies the signbit of NAN values with
3352 // fneg/fabs operations.
3353 if (match(V: TrueVal, P: m_FSub(L: m_PosZeroFP(), R: m_Specific(V: X))) &&
3354 (cast<FPMathOperator>(Val: CondVal)->hasNoNaNs() || SI.hasNoNaNs() ||
3355 (SI.hasOneUse() && canIgnoreSignBitOfNaN(U: *SI.use_begin())) ||
3356 isKnownNeverNaN(V: X, SQ: IC.getSimplifyQuery().getWithInstruction(
3357 I: cast<Instruction>(Val: CondVal))))) {
3358 if (!Swap && (Pred == FCmpInst::FCMP_OLE || Pred == FCmpInst::FCMP_ULE)) {
3359 Value *Fabs = IC.Builder.CreateFAbs(V: X, FMFSource: &SI);
3360 return IC.replaceInstUsesWith(I&: SI, V: Fabs);
3361 }
3362 if (Swap && (Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_UGT)) {
3363 Value *Fabs = IC.Builder.CreateFAbs(V: X, FMFSource: &SI);
3364 return IC.replaceInstUsesWith(I&: SI, V: Fabs);
3365 }
3366 }
3367
3368 if (!match(V: TrueVal, P: m_FNeg(X: m_Specific(V: X))))
3369 return nullptr;
3370
3371 // Forward-propagate nnan and ninf from the fcmp to the select.
3372 // If all inputs are not those values, then the select is not either.
3373 // Note: nsz is defined differently, so it may not be correct to propagate.
3374 FastMathFlags FMF = cast<FPMathOperator>(Val: CondVal)->getFastMathFlags();
3375 if (FMF.noNaNs() && !SI.hasNoNaNs()) {
3376 SI.setHasNoNaNs(true);
3377 ChangedFMF = true;
3378 }
3379 if (FMF.noInfs() && !SI.hasNoInfs()) {
3380 SI.setHasNoInfs(true);
3381 ChangedFMF = true;
3382 }
3383 // Forward-propagate nnan from the fneg to the select.
3384 // The nnan flag can be propagated iff fneg is selected when X is NaN.
3385 if (!SI.hasNoNaNs() && cast<FPMathOperator>(Val: TrueVal)->hasNoNaNs() &&
3386 (Swap ? FCmpInst::isOrdered(predicate: Pred) : FCmpInst::isUnordered(predicate: Pred))) {
3387 SI.setHasNoNaNs(true);
3388 ChangedFMF = true;
3389 }
3390
3391 // With nsz, when 'Swap' is false:
3392 // fold (X < +/-0.0) ? -X : X or (X <= +/-0.0) ? -X : X to fabs(X)
3393 // fold (X > +/-0.0) ? -X : X or (X >= +/-0.0) ? -X : X to -fabs(x)
3394 // when 'Swap' is true:
3395 // fold (X > +/-0.0) ? X : -X or (X >= +/-0.0) ? X : -X to fabs(X)
3396 // fold (X < +/-0.0) ? X : -X or (X <= +/-0.0) ? X : -X to -fabs(X)
3397 //
3398 // Note: We require "nnan" for this fold because fcmp ignores the signbit
3399 // of NAN, but IEEE-754 specifies the signbit of NAN values with
3400 // fneg/fabs operations.
3401 if (!SI.hasNoSignedZeros() &&
3402 (!SI.hasOneUse() || !canIgnoreSignBitOfZero(U: *SI.use_begin())))
3403 return nullptr;
3404 if (!SI.hasNoNaNs() &&
3405 (!SI.hasOneUse() || !canIgnoreSignBitOfNaN(U: *SI.use_begin())))
3406 return nullptr;
3407
3408 if (Swap)
3409 Pred = FCmpInst::getSwappedPredicate(pred: Pred);
3410
3411 bool IsLTOrLE = Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE ||
3412 Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE;
3413 bool IsGTOrGE = Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_OGE ||
3414 Pred == FCmpInst::FCMP_UGT || Pred == FCmpInst::FCMP_UGE;
3415
3416 if (IsLTOrLE) {
3417 Value *Fabs = IC.Builder.CreateFAbs(V: X, FMFSource: &SI);
3418 return IC.replaceInstUsesWith(I&: SI, V: Fabs);
3419 }
3420 if (IsGTOrGE) {
3421 Value *Fabs = IC.Builder.CreateFAbs(V: X, FMFSource: &SI);
3422 Instruction *NewFNeg = UnaryOperator::CreateFNeg(V: Fabs);
3423 NewFNeg->setFastMathFlags(SI.getFastMathFlags());
3424 return NewFNeg;
3425 }
3426 }
3427
3428 // Match select with (icmp slt (bitcast X to int), 0)
3429 // or (icmp sgt (bitcast X to int), -1)
3430
3431 for (bool Swap : {false, true}) {
3432 Value *TrueVal = SI.getTrueValue();
3433 Value *X = SI.getFalseValue();
3434
3435 if (Swap)
3436 std::swap(a&: TrueVal, b&: X);
3437
3438 CmpPredicate Pred;
3439 const APInt *C;
3440 bool TrueIfSigned;
3441 if (!match(V: CondVal,
3442 P: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Specific(V: X)), R: m_APInt(Res&: C))) ||
3443 !isSignBitCheck(Pred, RHS: *C, TrueIfSigned))
3444 continue;
3445 if (!match(V: TrueVal, P: m_FNeg(X: m_Specific(V: X))))
3446 return nullptr;
3447 if (Swap == TrueIfSigned && !CondVal->hasOneUse() && !TrueVal->hasOneUse())
3448 return nullptr;
3449
3450 // Fold (IsNeg ? -X : X) or (!IsNeg ? X : -X) to fabs(X)
3451 // Fold (IsNeg ? X : -X) or (!IsNeg ? -X : X) to -fabs(X)
3452 Value *Fabs = IC.Builder.CreateFAbs(V: X, FMFSource: &SI);
3453 if (Swap != TrueIfSigned)
3454 return IC.replaceInstUsesWith(I&: SI, V: Fabs);
3455 return UnaryOperator::CreateFNegFMF(Op: Fabs, FMFSource: &SI);
3456 }
3457
3458 return ChangedFMF ? &SI : nullptr;
3459}
3460
3461// Fold a select of an ordered fcmp using fabs of a NaN-scrubbed value:
3462// %s = select i1 (isnotnan T %x), T %x, T %y
3463// %a = call T @llvm.fabs.T(T %s)
3464// %c = fcmp <ordered-pred> T %a, %k
3465// %r = select i1 %c, T %s, T %y
3466// =>
3467// %a2 = call T @llvm.fabs.T(T %x)
3468// %c2 = fcmp <ordered-pred> T %a2, %k
3469// %r2 = select i1 %c2, T %x, T %y
3470static Instruction *
3471foldSelectOfOrderedFAbsCmpOfNaNScrubbedValue(SelectInst &SI,
3472 InstCombinerImpl &IC) {
3473 Instruction *OuterCmpI;
3474 Value *Cmp0, *Cmp1;
3475 if (!match(V: SI.getCondition(),
3476 P: m_OneUse(SubPattern: m_Instruction(I&: OuterCmpI,
3477 P: m_FCmp(L: m_Value(V&: Cmp0), R: m_Value(V&: Cmp1))))))
3478 return nullptr;
3479
3480 auto *OuterCmp = cast<FCmpInst>(Val: OuterCmpI);
3481 CmpInst::Predicate Pred = OuterCmp->getPredicate();
3482 if (!FCmpInst::isOrdered(predicate: Pred))
3483 return nullptr;
3484
3485 Value *Y = SI.getFalseValue();
3486 Value *InnerSel = SI.getTrueValue();
3487
3488 // Match a select that returns X when X is not NaN, and Y otherwise:
3489 // select (fcmp ord X, 0.0), X, Y
3490 Value *X;
3491 if (!match(V: InnerSel,
3492 P: m_Select(C: m_OneUse(SubPattern: m_SpecificFCmp(MatchPred: FCmpInst::FCMP_ORD, L: m_Value(V&: X),
3493 R: m_AnyZeroFP())),
3494 L: m_Deferred(V: X), R: m_Specific(V: Y))))
3495 return nullptr;
3496
3497 Instruction *FAbsI;
3498 auto MatchFAbsOfInnerSel = [&](Value *V) {
3499 return match(V,
3500 P: m_OneUse(SubPattern: m_Instruction(I&: FAbsI, P: m_FAbs(Op0: m_Specific(V: InnerSel)))));
3501 };
3502
3503 if (!MatchFAbsOfInnerSel(Cmp0)) {
3504 if (!MatchFAbsOfInnerSel(Cmp1))
3505 return nullptr;
3506
3507 std::swap(a&: Cmp0, b&: Cmp1);
3508 Pred = CmpInst::getSwappedPredicate(pred: Pred);
3509 }
3510
3511 FastMathFlags FAbsFMF = FAbsI->getFastMathFlags();
3512 FastMathFlags CmpFMF = OuterCmp->getFastMathFlags();
3513
3514 FastMathFlags CommonRewriteFMF =
3515 FastMathFlags::intersectRewrite(LHS: FAbsFMF, RHS: CmpFMF);
3516
3517 // unionValue with FastMathFlags() drops all rewriter based flags
3518 FastMathFlags NewFAbsFMF =
3519 CommonRewriteFMF | FastMathFlags::unionValue(LHS: FAbsFMF, RHS: FastMathFlags());
3520 FastMathFlags NewCmpFMF =
3521 CommonRewriteFMF | FastMathFlags::unionValue(LHS: CmpFMF, RHS: FastMathFlags());
3522
3523 // When X is NaN, the old code evaluated fabs(Y), while the new code evaluates
3524 // fabs(X). Do not preserve nnan on either newly-created instruction.
3525 NewFAbsFMF.setNoNaNs(false);
3526 NewCmpFMF.setNoNaNs(false);
3527
3528 Value *NewAbs = IC.Builder.CreateFAbs(V: X, FMFSource: FMFSource(NewFAbsFMF));
3529 Value *NewCmp =
3530 IC.Builder.CreateFCmpFMF(P: Pred, LHS: NewAbs, RHS: Cmp1, FMFSource: FMFSource(NewCmpFMF));
3531 Value *NewSel = IC.Builder.CreateSelectFMF(C: NewCmp, True: X, False: Y, FMFSource: &SI);
3532
3533 Instruction *NewSelUsesReplaced = IC.replaceInstUsesWith(I&: SI, V: NewSel);
3534
3535 uint64_t WeightNotNaN, WeightNaN, WeightComparisonTrue,
3536 WeightComparisonFalse = 0;
3537 bool HasProfile = extractBranchWeights(I: *cast<SelectInst>(Val: InnerSel),
3538 TrueVal&: WeightNotNaN, FalseVal&: WeightNaN);
3539 HasProfile &=
3540 extractBranchWeights(I: SI, TrueVal&: WeightComparisonTrue, FalseVal&: WeightComparisonFalse);
3541 if (!HasProfile || !isa<SelectInst>(Val: NewSel))
3542 return NewSelUsesReplaced;
3543 // The branch weights for the new select will be the same as before, except
3544 // they will additionally account for the probability of NaN values which was
3545 // previously handled with the inner select. For the true arm the new
3546 // probability is P(not Nan) * P(fcmp true). For the false arm, the new
3547 // probability is P(NaN) + (P(not NaN) * P(fcmp false)). We can assume the
3548 // probabilities are independent given the first select only checks for NaNs
3549 // and the second select's condition will never see NaNs because of the first
3550 // select. The code below uses some algebraic simplifications on top of those
3551 // formulas.
3552 uint64_t WeightNewSelTrue = WeightNotNaN * WeightComparisonTrue;
3553 uint64_t WeightNewSelFalse =
3554 WeightNaN * (WeightComparisonTrue + WeightComparisonFalse) +
3555 WeightNotNaN * WeightComparisonFalse;
3556 if (!ProfcheckDisableMetadataFixes)
3557 setFittedBranchWeights(I&: *cast<SelectInst>(Val: NewSel),
3558 Weights: {WeightNewSelTrue, WeightNewSelFalse},
3559 /*IsExpected*/ false);
3560 return NewSelUsesReplaced;
3561}
3562
3563// Match the following IR pattern:
3564// %x.lowbits = and i8 %x, %lowbitmask
3565// %x.lowbits.are.zero = icmp eq i8 %x.lowbits, 0
3566// %x.biased = add i8 %x, %bias
3567// %x.biased.highbits = and i8 %x.biased, %highbitmask
3568// %x.roundedup = select i1 %x.lowbits.are.zero, i8 %x, i8 %x.biased.highbits
3569// Define:
3570// %alignment = add i8 %lowbitmask, 1
3571// Iff 1. an %alignment is a power-of-two (aka, %lowbitmask is a low bit mask)
3572// and 2. %bias is equal to either %lowbitmask or %alignment,
3573// and 3. %highbitmask is equal to ~%lowbitmask (aka, to -%alignment)
3574// then this pattern can be transformed into:
3575// %x.offset = add i8 %x, %lowbitmask
3576// %x.roundedup = and i8 %x.offset, %highbitmask
3577static Value *
3578foldRoundUpIntegerWithPow2Alignment(SelectInst &SI,
3579 InstCombiner::BuilderTy &Builder) {
3580 Value *Cond = SI.getCondition();
3581 Value *X = SI.getTrueValue();
3582 Value *XBiasedHighBits = SI.getFalseValue();
3583
3584 CmpPredicate Pred;
3585 Value *XLowBits;
3586 if (!match(V: Cond, P: m_ICmp(Pred, L: m_Value(V&: XLowBits), R: m_ZeroInt())) ||
3587 !ICmpInst::isEquality(P: Pred))
3588 return nullptr;
3589
3590 if (Pred == ICmpInst::Predicate::ICMP_NE)
3591 std::swap(a&: X, b&: XBiasedHighBits);
3592
3593 // FIXME: we could support non non-splats here.
3594
3595 const APInt *LowBitMaskCst;
3596 if (!match(V: XLowBits, P: m_And(L: m_Specific(V: X), R: m_APIntAllowPoison(Res&: LowBitMaskCst))))
3597 return nullptr;
3598
3599 // Match even if the AND and ADD are swapped.
3600 const APInt *BiasCst, *HighBitMaskCst;
3601 if (!match(V: XBiasedHighBits,
3602 P: m_And(L: m_Add(L: m_Specific(V: X), R: m_APIntAllowPoison(Res&: BiasCst)),
3603 R: m_APIntAllowPoison(Res&: HighBitMaskCst))) &&
3604 !match(V: XBiasedHighBits,
3605 P: m_Add(L: m_And(L: m_Specific(V: X), R: m_APIntAllowPoison(Res&: HighBitMaskCst)),
3606 R: m_APIntAllowPoison(Res&: BiasCst))))
3607 return nullptr;
3608
3609 if (!LowBitMaskCst->isMask())
3610 return nullptr;
3611
3612 APInt InvertedLowBitMaskCst = ~*LowBitMaskCst;
3613 if (InvertedLowBitMaskCst != *HighBitMaskCst)
3614 return nullptr;
3615
3616 APInt AlignmentCst = *LowBitMaskCst + 1;
3617
3618 if (*BiasCst != AlignmentCst && *BiasCst != *LowBitMaskCst)
3619 return nullptr;
3620
3621 if (!XBiasedHighBits->hasOneUse()) {
3622 // We can't directly return XBiasedHighBits if it is more poisonous.
3623 if (*BiasCst == *LowBitMaskCst && impliesPoison(ValAssumedPoison: XBiasedHighBits, V: X))
3624 return XBiasedHighBits;
3625 return nullptr;
3626 }
3627
3628 // FIXME: could we preserve undef's here?
3629 Type *Ty = X->getType();
3630 Value *XOffset = Builder.CreateAdd(LHS: X, RHS: ConstantInt::get(Ty, V: *LowBitMaskCst),
3631 Name: X->getName() + ".biased");
3632 Value *R = Builder.CreateAnd(LHS: XOffset, RHS: ConstantInt::get(Ty, V: *HighBitMaskCst));
3633 R->takeName(V: &SI);
3634 return R;
3635}
3636
3637namespace {
3638struct DecomposedSelect {
3639 Value *Cond = nullptr;
3640 Value *TrueVal = nullptr;
3641 Value *FalseVal = nullptr;
3642};
3643} // namespace
3644
3645/// Folds patterns like:
3646/// select c2 (select c1 a b) (select c1 b a)
3647/// into:
3648/// select (xor c1 c2) b a
3649static Instruction *
3650foldSelectOfSymmetricSelect(SelectInst &OuterSelVal,
3651 InstCombiner::BuilderTy &Builder) {
3652
3653 Value *OuterCond, *InnerCond, *InnerTrueVal, *InnerFalseVal;
3654 if (!match(
3655 V: &OuterSelVal,
3656 P: m_Select(C: m_Value(V&: OuterCond),
3657 L: m_OneUse(SubPattern: m_Select(C: m_Value(V&: InnerCond), L: m_Value(V&: InnerTrueVal),
3658 R: m_Value(V&: InnerFalseVal))),
3659 R: m_OneUse(SubPattern: m_Select(C: m_Deferred(V: InnerCond),
3660 L: m_Deferred(V: InnerFalseVal),
3661 R: m_Deferred(V: InnerTrueVal))))))
3662 return nullptr;
3663
3664 if (OuterCond->getType() != InnerCond->getType())
3665 return nullptr;
3666
3667 Value *Xor = Builder.CreateXor(LHS: InnerCond, RHS: OuterCond);
3668 return SelectInst::Create(C: Xor, S1: InnerFalseVal, S2: InnerTrueVal);
3669}
3670
3671/// Look for patterns like
3672/// %outer.cond = select i1 %inner.cond, i1 %alt.cond, i1 false
3673/// %inner.sel = select i1 %inner.cond, i8 %inner.sel.t, i8 %inner.sel.f
3674/// %outer.sel = select i1 %outer.cond, i8 %outer.sel.t, i8 %inner.sel
3675/// and rewrite it as
3676/// %inner.sel = select i1 %cond.alternative, i8 %sel.outer.t, i8 %sel.inner.t
3677/// %sel.outer = select i1 %cond.inner, i8 %inner.sel, i8 %sel.inner.f
3678static Instruction *foldNestedSelects(SelectInst &OuterSelVal,
3679 InstCombiner::BuilderTy &Builder) {
3680 // We must start with a `select`.
3681 DecomposedSelect OuterSel;
3682 match(V: &OuterSelVal,
3683 P: m_Select(C: m_Value(V&: OuterSel.Cond), L: m_Value(V&: OuterSel.TrueVal),
3684 R: m_Value(V&: OuterSel.FalseVal)));
3685
3686 // Canonicalize inversion of the outermost `select`'s condition.
3687 if (match(V: OuterSel.Cond, P: m_Not(V: m_Value(V&: OuterSel.Cond))))
3688 std::swap(a&: OuterSel.TrueVal, b&: OuterSel.FalseVal);
3689
3690 // The condition of the outermost select must be an `and`/`or`.
3691 if (!match(V: OuterSel.Cond, P: m_c_LogicalOp(L: m_Value(), R: m_Value())))
3692 return nullptr;
3693
3694 // Depending on the logical op, inner select might be in different hand.
3695 bool IsAndVariant = match(V: OuterSel.Cond, P: m_LogicalAnd());
3696 Value *InnerSelVal = IsAndVariant ? OuterSel.FalseVal : OuterSel.TrueVal;
3697
3698 // Profitability check - avoid increasing instruction count.
3699 if (none_of(Range: ArrayRef<Value *>({OuterSelVal.getCondition(), InnerSelVal}),
3700 P: match_fn(P: m_OneUse(SubPattern: m_Value()))))
3701 return nullptr;
3702
3703 // The appropriate hand of the outermost `select` must be a select itself.
3704 DecomposedSelect InnerSel;
3705 if (!match(V: InnerSelVal,
3706 P: m_Select(C: m_Value(V&: InnerSel.Cond), L: m_Value(V&: InnerSel.TrueVal),
3707 R: m_Value(V&: InnerSel.FalseVal))))
3708 return nullptr;
3709
3710 // Canonicalize inversion of the innermost `select`'s condition.
3711 bool SwapInnerSelCond = false;
3712 if (match(V: InnerSel.Cond, P: m_Not(V: m_Value(V&: InnerSel.Cond)))) {
3713 std::swap(a&: InnerSel.TrueVal, b&: InnerSel.FalseVal);
3714 SwapInnerSelCond = !SwapInnerSelCond;
3715 }
3716
3717 Value *AltCond = nullptr;
3718 auto matchOuterCond = [OuterSel, IsAndVariant, &AltCond](auto m_InnerCond) {
3719 // An unsimplified select condition can match both LogicalAnd and LogicalOr
3720 // (select true, true, false). Since below we assume that LogicalAnd implies
3721 // InnerSel match the FVal and vice versa for LogicalOr, we can't match the
3722 // alternative pattern here.
3723 return IsAndVariant ? match(OuterSel.Cond,
3724 m_c_LogicalAnd(m_InnerCond, m_Value(V&: AltCond)))
3725 : match(OuterSel.Cond,
3726 m_c_LogicalOr(m_InnerCond, m_Value(V&: AltCond)));
3727 };
3728
3729 // Finally, match the condition that was driving the outermost `select`,
3730 // it should be a logical operation between the condition that was driving
3731 // the innermost `select` (after accounting for the possible inversions
3732 // of the condition), and some other condition.
3733 if (matchOuterCond(m_Specific(V: InnerSel.Cond))) {
3734 // Done!
3735 } else if (Value * NotInnerCond; matchOuterCond(m_CombineAnd(
3736 Ps: m_Not(V: m_Specific(V: InnerSel.Cond)), Ps: m_Value(V&: NotInnerCond)))) {
3737 // Done!
3738 std::swap(a&: InnerSel.TrueVal, b&: InnerSel.FalseVal);
3739 InnerSel.Cond = NotInnerCond;
3740 SwapInnerSelCond = !SwapInnerSelCond;
3741 } else // Not the pattern we were looking for.
3742 return nullptr;
3743
3744 // We mark the select with AltCond as having an unknown profile given the
3745 // condition is derived from an and/or. We might have profile information on
3746 // the operands of the and/or, but there is no guarantee that they are
3747 // independent.
3748 Value *SelInner = Builder.CreateSelectWithUnknownProfile(
3749 C: AltCond, True: IsAndVariant ? OuterSel.TrueVal : InnerSel.FalseVal,
3750 False: IsAndVariant ? InnerSel.TrueVal : OuterSel.FalseVal, DEBUG_TYPE);
3751 SelInner->takeName(V: InnerSelVal);
3752 SelectInst *SI = SelectInst::Create(
3753 C: InnerSel.Cond, S1: IsAndVariant ? SelInner : InnerSel.TrueVal,
3754 S2: !IsAndVariant ? SelInner : InnerSel.FalseVal, NameStr: "", InsertBefore: nullptr,
3755 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : cast<Instruction>(Val: InnerSelVal));
3756 if (SwapInnerSelCond)
3757 SI->swapProfMetadata();
3758 return SI;
3759}
3760
3761/// Return true if V is poison or \p Expected given that ValAssumedPoison is
3762/// already poison. For example, if ValAssumedPoison is `icmp samesign X, 10`
3763/// and V is `icmp ne X, 5`, impliesPoisonOrCond returns true.
3764static bool impliesPoisonOrCond(const Value *ValAssumedPoison, const Value *V,
3765 bool Expected, const SimplifyQuery &SQ) {
3766 if (impliesPoison(ValAssumedPoison, V))
3767 return true;
3768
3769 // Handle the case that ValAssumedPoison is `icmp samesign pred X, C1` and V
3770 // is `icmp pred X, C2`, where C1 is well-defined.
3771 if (auto *ICmp = dyn_cast<ICmpInst>(Val: ValAssumedPoison)) {
3772 Value *LHS = ICmp->getOperand(i_nocapture: 0);
3773 const APInt *RHSC1;
3774 const APInt *RHSC2;
3775 CmpPredicate Pred;
3776 if (ICmp->hasSameSign() &&
3777 match(V: ICmp->getOperand(i_nocapture: 1), P: m_APIntForbidPoison(Res&: RHSC1)) &&
3778 match(V, P: m_ICmp(Pred, L: m_Specific(V: LHS), R: m_APIntAllowPoison(Res&: RHSC2)))) {
3779 unsigned BitWidth = RHSC1->getBitWidth();
3780 ConstantRange CRX =
3781 RHSC1->isNonNegative()
3782 ? ConstantRange(APInt::getSignedMinValue(numBits: BitWidth),
3783 APInt::getZero(numBits: BitWidth))
3784 : ConstantRange(APInt::getZero(numBits: BitWidth),
3785 APInt::getSignedMinValue(numBits: BitWidth));
3786 return CRX.icmp(Pred: Expected ? Pred : ICmpInst::getInverseCmpPredicate(Pred),
3787 Other: *RHSC2);
3788 }
3789 }
3790 // For non-poison X in [0, 1], `trunc nuw X to i1` is not poison, but an
3791 // additional `nsw` flag makes it poison for X == 1.
3792 Value *A;
3793 if (match(V: ValAssumedPoison, P: m_NUWTrunc(Op: m_Value(V&: A))) &&
3794 !cast<TruncInst>(Val: ValAssumedPoison)->hasNoSignedWrap() &&
3795 isGuaranteedNotToBePoison(V: A)) {
3796 assert(ValAssumedPoison->getType()->isIntOrIntVectorTy(1));
3797 return computeKnownBits(
3798 V: A, Q: SQ.getWithInstruction(I: cast<Instruction>(Val: ValAssumedPoison)))
3799 .getMaxValue() == 1;
3800 }
3801
3802 return false;
3803}
3804
3805Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) {
3806 Value *CondVal = SI.getCondition();
3807 Value *TrueVal = SI.getTrueValue();
3808 Value *FalseVal = SI.getFalseValue();
3809 Type *SelType = SI.getType();
3810
3811 // Avoid potential infinite loops by checking for non-constant condition.
3812 // TODO: Can we assert instead by improving canonicalizeSelectToShuffle()?
3813 // Scalar select must have simplified?
3814 if (!SelType->isIntOrIntVectorTy(BitWidth: 1) || isa<Constant>(Val: CondVal) ||
3815 TrueVal->getType() != CondVal->getType())
3816 return nullptr;
3817
3818 auto *One = ConstantInt::getTrue(Ty: SelType);
3819 auto *Zero = ConstantInt::getFalse(Ty: SelType);
3820 Value *A, *B, *C, *D;
3821
3822 // Folding select to and/or i1 isn't poison safe in general. impliesPoison
3823 // checks whether folding it does not convert a well-defined value into
3824 // poison.
3825 if (match(V: TrueVal, P: m_One())) {
3826 if (impliesPoisonOrCond(ValAssumedPoison: FalseVal, V: CondVal, /*Expected=*/false, SQ)) {
3827 // Change: A = select B, true, C --> A = or B, C
3828 return BinaryOperator::CreateOr(V1: CondVal, V2: FalseVal);
3829 }
3830
3831 if (match(V: CondVal, P: m_OneUse(SubPattern: m_Select(C: m_Value(V&: A), L: m_One(), R: m_Value(V&: B)))) &&
3832 impliesPoisonOrCond(ValAssumedPoison: FalseVal, V: B, /*Expected=*/false, SQ)) {
3833 // (A || B) || C --> A || (B | C)
3834 Value *LOr = Builder.CreateLogicalOr(Cond1: A, Cond2: Builder.CreateOr(LHS: B, RHS: FalseVal));
3835 if (auto *I = dyn_cast<Instruction>(Val: LOr)) {
3836 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *I, DEBUG_TYPE);
3837 }
3838 return replaceInstUsesWith(I&: SI, V: LOr);
3839 }
3840
3841 // (A && B) || (C && B) --> (A || C) && B
3842 if (match(V: CondVal, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3843 match(V: FalseVal, P: m_LogicalAnd(L: m_Value(V&: C), R: m_Value(V&: D))) &&
3844 (CondVal->hasOneUse() || FalseVal->hasOneUse())) {
3845 bool CondLogicAnd = isa<SelectInst>(Val: CondVal);
3846 bool FalseLogicAnd = isa<SelectInst>(Val: FalseVal);
3847 auto AndFactorization = [&](Value *Common, Value *InnerCond,
3848 Value *InnerVal,
3849 bool SelFirst = false) -> Instruction * {
3850 Value *InnerSel = Builder.CreateSelectWithUnknownProfile(
3851 C: InnerCond, True: One, False: InnerVal, DEBUG_TYPE);
3852 if (SelFirst)
3853 std::swap(a&: Common, b&: InnerSel);
3854 if (FalseLogicAnd || (CondLogicAnd && Common == A))
3855 return createSelectInstWithUnknownProfile(C: Common, S1: InnerSel, S2: Zero);
3856 else
3857 return BinaryOperator::CreateAnd(V1: Common, V2: InnerSel);
3858 };
3859
3860 if (A == C)
3861 return AndFactorization(A, B, D);
3862 if (A == D)
3863 return AndFactorization(A, B, C);
3864 if (B == C)
3865 return AndFactorization(B, A, D);
3866 if (B == D)
3867 return AndFactorization(B, A, C, CondLogicAnd && FalseLogicAnd);
3868 }
3869 }
3870
3871 if (match(V: FalseVal, P: m_Zero())) {
3872 if (impliesPoisonOrCond(ValAssumedPoison: TrueVal, V: CondVal, /*Expected=*/true, SQ)) {
3873 // Change: A = select B, C, false --> A = and B, C
3874 return BinaryOperator::CreateAnd(V1: CondVal, V2: TrueVal);
3875 }
3876
3877 if (match(V: CondVal, P: m_OneUse(SubPattern: m_Select(C: m_Value(V&: A), L: m_Value(V&: B), R: m_Zero()))) &&
3878 impliesPoisonOrCond(ValAssumedPoison: TrueVal, V: B, /*Expected=*/true, SQ)) {
3879 // (A && B) && C --> A && (B & C)
3880 Value *LAnd = Builder.CreateLogicalAnd(Cond1: A, Cond2: Builder.CreateAnd(LHS: B, RHS: TrueVal));
3881 if (auto *I = dyn_cast<Instruction>(Val: LAnd)) {
3882 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *I, DEBUG_TYPE);
3883 }
3884 return replaceInstUsesWith(I&: SI, V: LAnd);
3885 }
3886
3887 // (A || B) && (C || B) --> (A && C) || B
3888 if (match(V: CondVal, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3889 match(V: TrueVal, P: m_LogicalOr(L: m_Value(V&: C), R: m_Value(V&: D))) &&
3890 (CondVal->hasOneUse() || TrueVal->hasOneUse())) {
3891 bool CondLogicOr = isa<SelectInst>(Val: CondVal);
3892 bool TrueLogicOr = isa<SelectInst>(Val: TrueVal);
3893 auto OrFactorization = [&](Value *Common, Value *InnerCond,
3894 Value *InnerVal,
3895 bool SelFirst = false) -> Instruction * {
3896 Value *InnerSel = Builder.CreateSelectWithUnknownProfile(
3897 C: InnerCond, True: InnerVal, False: Zero, DEBUG_TYPE);
3898 if (SelFirst)
3899 std::swap(a&: Common, b&: InnerSel);
3900 if (TrueLogicOr || (CondLogicOr && Common == A))
3901 return createSelectInstWithUnknownProfile(C: Common, S1: One, S2: InnerSel);
3902 else
3903 return BinaryOperator::CreateOr(V1: Common, V2: InnerSel);
3904 };
3905
3906 if (A == C)
3907 return OrFactorization(A, B, D);
3908 if (A == D)
3909 return OrFactorization(A, B, C);
3910 if (B == C)
3911 return OrFactorization(B, A, D);
3912 if (B == D)
3913 return OrFactorization(B, A, C, CondLogicOr && TrueLogicOr);
3914 }
3915 }
3916
3917 // We match the "full" 0 or 1 constant here to avoid a potential infinite
3918 // loop with vectors that may have undefined/poison elements.
3919 // select a, false, b -> select !a, b, false
3920 if (match(V: TrueVal, P: m_Specific(V: Zero))) {
3921 Value *NotCond = Builder.CreateNot(V: CondVal, Name: "not." + CondVal->getName());
3922 SelectInst *NewSI = SelectInst::Create(C: NotCond, S1: FalseVal, S2: Zero, NameStr: "", InsertBefore: nullptr,
3923 /*MDFrom=*/&SI);
3924 NewSI->swapProfMetadata();
3925 return NewSI;
3926 }
3927 // select a, b, true -> select !a, true, b
3928 if (match(V: FalseVal, P: m_Specific(V: One))) {
3929 Value *NotCond = Builder.CreateNot(V: CondVal, Name: "not." + CondVal->getName());
3930 SelectInst *NewSI =
3931 SelectInst::Create(C: NotCond, S1: One, S2: TrueVal, NameStr: "", InsertBefore: nullptr, /*MDFrom=*/&SI);
3932 NewSI->swapProfMetadata();
3933 return NewSI;
3934 }
3935
3936 // DeMorgan in select form: !a && !b --> !(a || b)
3937 // select !a, !b, false --> not (select a, true, b)
3938 if (match(V: &SI, P: m_LogicalAnd(L: m_Not(V: m_Value(V&: A)), R: m_Not(V: m_Value(V&: B)))) &&
3939 (CondVal->hasOneUse() || TrueVal->hasOneUse()) &&
3940 !match(V: A, P: m_ConstantExpr()) && !match(V: B, P: m_ConstantExpr())) {
3941 SelectInst *NewSI =
3942 cast<SelectInst>(Val: Builder.CreateSelect(C: A, True: One, False: B, Name: "", /*MDFrom=*/&SI));
3943 NewSI->swapProfMetadata();
3944 return BinaryOperator::CreateNot(Op: NewSI);
3945 }
3946
3947 // DeMorgan in select form: !a || !b --> !(a && b)
3948 // select !a, true, !b --> not (select a, b, false)
3949 if (match(V: &SI, P: m_LogicalOr(L: m_Not(V: m_Value(V&: A)), R: m_Not(V: m_Value(V&: B)))) &&
3950 (CondVal->hasOneUse() || FalseVal->hasOneUse()) &&
3951 !match(V: A, P: m_ConstantExpr()) && !match(V: B, P: m_ConstantExpr())) {
3952 SelectInst *NewSI =
3953 cast<SelectInst>(Val: Builder.CreateSelect(C: A, True: B, False: Zero, Name: "", /*MDFrom=*/&SI));
3954 NewSI->swapProfMetadata();
3955 return BinaryOperator::CreateNot(Op: NewSI);
3956 }
3957
3958 // select (select a, true, b), true, b -> select a, true, b
3959 if (match(V: CondVal, P: m_Select(C: m_Value(V&: A), L: m_One(), R: m_Value(V&: B))) &&
3960 match(V: TrueVal, P: m_One()) && match(V: FalseVal, P: m_Specific(V: B)))
3961 return replaceOperand(I&: SI, OpNum: 0, V: A);
3962 // select (select a, b, false), b, false -> select a, b, false
3963 if (match(V: CondVal, P: m_Select(C: m_Value(V&: A), L: m_Value(V&: B), R: m_Zero())) &&
3964 match(V: TrueVal, P: m_Specific(V: B)) && match(V: FalseVal, P: m_Zero()))
3965 return replaceOperand(I&: SI, OpNum: 0, V: A);
3966
3967 // ~(A & B) & (A | B) --> A ^ B
3968 if (match(V: &SI, P: m_c_LogicalAnd(L: m_Not(V: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B))),
3969 R: m_c_LogicalOr(L: m_Deferred(V: A), R: m_Deferred(V: B)))))
3970 return BinaryOperator::CreateXor(V1: A, V2: B);
3971
3972 // select (~a | c), a, b -> select a, (select c, true, b), false
3973 if (match(V: CondVal,
3974 P: m_OneUse(SubPattern: m_c_Or(L: m_Not(V: m_Specific(V: TrueVal)), R: m_Value(V&: C))))) {
3975 // TODO(#183864): We could improve the profile if P(~a | c) < 0.5, which
3976 // implies strong bounds on both operands (P(a) is high, P(c) is low).
3977 Value *OrV =
3978 Builder.CreateSelectWithUnknownProfile(C, True: One, False: FalseVal, DEBUG_TYPE);
3979 return createSelectInstWithUnknownProfile(C: TrueVal, S1: OrV, S2: Zero);
3980 }
3981 // select (c & b), a, b -> select b, (select ~c, true, a), false
3982 if (match(V: CondVal, P: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: C), R: m_Specific(V: FalseVal))))) {
3983 if (Value *NotC = getFreelyInverted(V: C, WillInvertAllUses: C->hasOneUse(), Builder: &Builder)) {
3984 Value *OrV = Builder.CreateSelectWithUnknownProfile(C: NotC, True: One, False: TrueVal,
3985 DEBUG_TYPE);
3986 return createSelectInstWithUnknownProfile(C: FalseVal, S1: OrV, S2: Zero);
3987 }
3988 }
3989 // select (a | c), a, b -> select a, true, (select ~c, b, false)
3990 if (match(V: CondVal, P: m_OneUse(SubPattern: m_c_Or(L: m_Specific(V: TrueVal), R: m_Value(V&: C))))) {
3991 if (Value *NotC = getFreelyInverted(V: C, WillInvertAllUses: C->hasOneUse(), Builder: &Builder)) {
3992 // TODO(#183864): We could improve the profile if P(a | c) < 0.5, which
3993 // implies strong bounds on both operands (both P(a) and P(c) are low).
3994 Value *AndV = Builder.CreateSelectWithUnknownProfile(C: NotC, True: FalseVal, False: Zero,
3995 DEBUG_TYPE);
3996 return createSelectInstWithUnknownProfile(C: TrueVal, S1: One, S2: AndV);
3997 }
3998 }
3999 // select (c & ~b), a, b -> select b, true, (select c, a, false)
4000 if (match(V: CondVal,
4001 P: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: C), R: m_Not(V: m_Specific(V: FalseVal)))))) {
4002 Value *AndV =
4003 Builder.CreateSelectWithUnknownProfile(C, True: TrueVal, False: Zero, DEBUG_TYPE);
4004 return createSelectInstWithUnknownProfile(C: FalseVal, S1: One, S2: AndV);
4005 }
4006
4007 if (match(V: FalseVal, P: m_Zero()) || match(V: TrueVal, P: m_One())) {
4008 Use *Y = nullptr;
4009 bool IsAnd = match(V: FalseVal, P: m_Zero()) ? true : false;
4010 Value *Op1 = IsAnd ? TrueVal : FalseVal;
4011 if (isCheckForZeroAndMulWithOverflow(Op0: CondVal, Op1, IsAnd, Y)) {
4012 auto *FI = new FreezeInst(*Y, (*Y)->getName() + ".fr");
4013 InsertNewInstBefore(New: FI, Old: cast<Instruction>(Val: Y->getUser())->getIterator());
4014 replaceUse(U&: *Y, NewValue: FI);
4015 return replaceInstUsesWith(I&: SI, V: Op1);
4016 }
4017
4018 if (auto *V = foldBooleanAndOr(LHS: CondVal, RHS: Op1, I&: SI, IsAnd,
4019 /*IsLogical=*/true))
4020 return replaceInstUsesWith(I&: SI, V);
4021 }
4022
4023 // select (a || b), c, false -> select a, c, false
4024 // select c, (a || b), false -> select c, a, false
4025 // if c implies that b is false.
4026 if (match(V: CondVal, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4027 match(V: FalseVal, P: m_Zero())) {
4028 std::optional<bool> Res = isImpliedCondition(LHS: TrueVal, RHS: B, DL);
4029 if (Res && *Res == false)
4030 return replaceOperand(I&: SI, OpNum: 0, V: A);
4031 }
4032 if (match(V: TrueVal, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4033 match(V: FalseVal, P: m_Zero())) {
4034 std::optional<bool> Res = isImpliedCondition(LHS: CondVal, RHS: B, DL);
4035 if (Res && *Res == false)
4036 return replaceOperand(I&: SI, OpNum: 1, V: A);
4037 }
4038 // select c, true, (a && b) -> select c, true, a
4039 // select (a && b), true, c -> select a, true, c
4040 // if c = false implies that b = true
4041 if (match(V: TrueVal, P: m_One()) &&
4042 match(V: FalseVal, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B)))) {
4043 std::optional<bool> Res = isImpliedCondition(LHS: CondVal, RHS: B, DL, LHSIsTrue: false);
4044 if (Res && *Res == true)
4045 return replaceOperand(I&: SI, OpNum: 2, V: A);
4046 }
4047 if (match(V: CondVal, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4048 match(V: TrueVal, P: m_One())) {
4049 std::optional<bool> Res = isImpliedCondition(LHS: FalseVal, RHS: B, DL, LHSIsTrue: false);
4050 if (Res && *Res == true)
4051 return replaceOperand(I&: SI, OpNum: 0, V: A);
4052 }
4053
4054 if (match(V: TrueVal, P: m_One())) {
4055 // (C && A) || (!C && B) --> select C, A, B (and similar cases)
4056 if (auto *V = FoldOrOfLogicalAnds(Op0: CondVal, Op1: FalseVal)) {
4057 return V;
4058 }
4059 }
4060
4061 return nullptr;
4062}
4063
4064// Return true if we can safely remove the select instruction for std::bit_ceil
4065// pattern.
4066static bool isSafeToRemoveBitCeilSelect(ICmpInst::Predicate Pred, Value *Cond0,
4067 const APInt *Cond1, Value *CtlzOp,
4068 unsigned BitWidth,
4069 bool &ShouldDropNoWrap) {
4070 // The challenge in recognizing std::bit_ceil(X) is that the operand is used
4071 // for the CTLZ proper and select condition, each possibly with some
4072 // operation like add and sub.
4073 //
4074 // Our aim is to make sure that -ctlz & (BitWidth - 1) == 0 even when the
4075 // select instruction would select 1, which allows us to get rid of the select
4076 // instruction.
4077 //
4078 // To see if we can do so, we do some symbolic execution with ConstantRange.
4079 // Specifically, we compute the range of values that Cond0 could take when
4080 // Cond == false. Then we successively transform the range until we obtain
4081 // the range of values that CtlzOp could take.
4082 //
4083 // Conceptually, we follow the def-use chain backward from Cond0 while
4084 // transforming the range for Cond0 until we meet the common ancestor of Cond0
4085 // and CtlzOp. Then we follow the def-use chain forward until we obtain the
4086 // range for CtlzOp. That said, we only follow at most one ancestor from
4087 // Cond0. Likewise, we only follow at most one ancestor from CtrlOp.
4088
4089 ConstantRange CR = ConstantRange::makeExactICmpRegion(
4090 Pred: CmpInst::getInversePredicate(pred: Pred), Other: *Cond1);
4091
4092 ShouldDropNoWrap = false;
4093
4094 // Match the operation that's used to compute CtlzOp from CommonAncestor. If
4095 // CtlzOp == CommonAncestor, return true as no operation is needed. If a
4096 // match is found, execute the operation on CR, update CR, and return true.
4097 // Otherwise, return false.
4098 auto MatchForward = [&](Value *CommonAncestor) {
4099 const APInt *C = nullptr;
4100 if (CtlzOp == CommonAncestor)
4101 return true;
4102 if (match(V: CtlzOp, P: m_Add(L: m_Specific(V: CommonAncestor), R: m_APInt(Res&: C)))) {
4103 ShouldDropNoWrap = true;
4104 CR = CR.add(Other: *C);
4105 return true;
4106 }
4107 if (match(V: CtlzOp, P: m_Sub(L: m_APInt(Res&: C), R: m_Specific(V: CommonAncestor)))) {
4108 ShouldDropNoWrap = true;
4109 CR = ConstantRange(*C).sub(Other: CR);
4110 return true;
4111 }
4112 if (match(V: CtlzOp, P: m_Not(V: m_Specific(V: CommonAncestor)))) {
4113 CR = CR.binaryNot();
4114 return true;
4115 }
4116 return false;
4117 };
4118
4119 const APInt *C = nullptr;
4120 Value *CommonAncestor;
4121 if (MatchForward(Cond0)) {
4122 // Cond0 is either CtlzOp or CtlzOp's parent. CR has been updated.
4123 } else if (match(V: Cond0, P: m_Add(L: m_Value(V&: CommonAncestor), R: m_APInt(Res&: C)))) {
4124 CR = CR.sub(Other: *C);
4125 if (!MatchForward(CommonAncestor))
4126 return false;
4127 // Cond0's parent is either CtlzOp or CtlzOp's parent. CR has been updated.
4128 } else {
4129 return false;
4130 }
4131
4132 // Return true if all the values in the range are either 0 or negative (if
4133 // treated as signed). We do so by evaluating:
4134 //
4135 // CR - 1 u>= (1 << BitWidth) - 1.
4136 APInt IntMax = APInt::getSignMask(BitWidth) - 1;
4137 CR = CR.sub(Other: APInt(BitWidth, 1));
4138 return CR.icmp(Pred: ICmpInst::ICMP_UGE, Other: IntMax);
4139}
4140
4141// Transform the std::bit_ceil(X) pattern like:
4142//
4143// %dec = add i32 %x, -1
4144// %ctlz = tail call i32 @llvm.ctlz.i32(i32 %dec, i1 false)
4145// %sub = sub i32 32, %ctlz
4146// %shl = shl i32 1, %sub
4147// %ugt = icmp ugt i32 %x, 1
4148// %sel = select i1 %ugt, i32 %shl, i32 1
4149//
4150// into:
4151//
4152// %dec = add i32 %x, -1
4153// %ctlz = tail call i32 @llvm.ctlz.i32(i32 %dec, i1 false)
4154// %neg = sub i32 0, %ctlz
4155// %masked = and i32 %ctlz, 31
4156// %shl = shl i32 1, %sub
4157//
4158// Note that the select is optimized away while the shift count is masked with
4159// 31. We handle some variations of the input operand like std::bit_ceil(X +
4160// 1).
4161static Instruction *foldBitCeil(SelectInst &SI, IRBuilderBase &Builder,
4162 InstCombinerImpl &IC) {
4163 Type *SelType = SI.getType();
4164 unsigned BitWidth = SelType->getScalarSizeInBits();
4165 if (!isPowerOf2_32(Value: BitWidth))
4166 return nullptr;
4167
4168 Value *FalseVal = SI.getFalseValue();
4169 Value *TrueVal = SI.getTrueValue();
4170 CmpPredicate Pred;
4171 const APInt *Cond1;
4172 Value *Cond0, *Ctlz, *CtlzOp;
4173 if (!match(V: SI.getCondition(), P: m_ICmp(Pred, L: m_Value(V&: Cond0), R: m_APInt(Res&: Cond1))))
4174 return nullptr;
4175
4176 if (match(V: TrueVal, P: m_One())) {
4177 std::swap(a&: FalseVal, b&: TrueVal);
4178 Pred = CmpInst::getInversePredicate(pred: Pred);
4179 }
4180
4181 bool ShouldDropNoWrap;
4182
4183 if (!match(V: FalseVal, P: m_One()) ||
4184 !match(V: TrueVal,
4185 P: m_OneUse(SubPattern: m_Shl(L: m_One(), R: m_OneUse(SubPattern: m_Sub(L: m_SpecificInt(V: BitWidth),
4186 R: m_Value(V&: Ctlz)))))) ||
4187 !match(V: Ctlz, P: m_Ctlz(Op0: m_Value(V&: CtlzOp), Op1: m_Value())) ||
4188 !isSafeToRemoveBitCeilSelect(Pred, Cond0, Cond1, CtlzOp, BitWidth,
4189 ShouldDropNoWrap))
4190 return nullptr;
4191
4192 if (ShouldDropNoWrap) {
4193 cast<Instruction>(Val: CtlzOp)->setHasNoUnsignedWrap(false);
4194 cast<Instruction>(Val: CtlzOp)->setHasNoSignedWrap(false);
4195 }
4196
4197 // Build 1 << (-CTLZ & (BitWidth-1)). The negation likely corresponds to a
4198 // single hardware instruction as opposed to BitWidth - CTLZ, where BitWidth
4199 // is an integer constant. Masking with BitWidth-1 comes free on some
4200 // hardware as part of the shift instruction.
4201
4202 // Drop range attributes and re-infer them in the next iteration.
4203 cast<Instruction>(Val: Ctlz)->dropPoisonGeneratingAnnotations();
4204 IC.addToWorklist(I: cast<Instruction>(Val: Ctlz));
4205 Value *Neg = Builder.CreateNeg(V: Ctlz);
4206 Value *Masked =
4207 Builder.CreateAnd(LHS: Neg, RHS: ConstantInt::get(Ty: SelType, V: BitWidth - 1));
4208 return BinaryOperator::Create(Op: Instruction::Shl, S1: ConstantInt::get(Ty: SelType, V: 1),
4209 S2: Masked);
4210}
4211
4212// This function tries to fold the following operations:
4213// (x < y) ? -1 : zext(x != y)
4214// (x < y) ? -1 : zext(x > y)
4215// (x > y) ? 1 : sext(x != y)
4216// (x > y) ? 1 : sext(x < y)
4217// (x == y) ? 0 : (x > y ? 1 : -1)
4218// (x == y) ? 0 : (x < y ? -1 : 1)
4219// Special case: x == C ? 0 : (x > C - 1 ? 1 : -1)
4220// Special case: x == C ? 0 : (x < C + 1 ? -1 : 1)
4221// Into ucmp/scmp(x, y), where signedness is determined by the signedness
4222// of the comparison in the original sequence.
4223Instruction *InstCombinerImpl::foldSelectToCmp(SelectInst &SI) {
4224 Value *TV = SI.getTrueValue();
4225 Value *FV = SI.getFalseValue();
4226
4227 CmpPredicate Pred;
4228 Value *LHS, *RHS;
4229 if (!match(V: SI.getCondition(), P: m_ICmp(Pred, L: m_Value(V&: LHS), R: m_Value(V&: RHS))))
4230 return nullptr;
4231
4232 if (!LHS->getType()->isIntOrIntVectorTy())
4233 return nullptr;
4234
4235 // If there is no -1, 0 or 1 at TV, then invert the select statement and try
4236 // to canonicalize to one of the forms above
4237 if (!isa<Constant>(Val: TV)) {
4238 if (!isa<Constant>(Val: FV))
4239 return nullptr;
4240 Pred = ICmpInst::getInverseCmpPredicate(Pred);
4241 std::swap(a&: TV, b&: FV);
4242 }
4243
4244 if (ICmpInst::isNonStrictPredicate(predicate: Pred)) {
4245 if (Constant *C = dyn_cast<Constant>(Val: RHS)) {
4246 auto FlippedPredAndConst =
4247 getFlippedStrictnessPredicateAndConstant(Pred, C);
4248 if (!FlippedPredAndConst)
4249 return nullptr;
4250 Pred = FlippedPredAndConst->first;
4251 RHS = FlippedPredAndConst->second;
4252 } else {
4253 return nullptr;
4254 }
4255 }
4256
4257 // Try to swap operands and the predicate. We need to be careful when doing
4258 // so because two of the patterns have opposite predicates, so use the
4259 // constant inside select to determine if swapping operands would be
4260 // beneficial to us.
4261 if ((ICmpInst::isGT(P: Pred) && match(V: TV, P: m_AllOnes())) ||
4262 (ICmpInst::isLT(P: Pred) && match(V: TV, P: m_One()))) {
4263 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
4264 std::swap(a&: LHS, b&: RHS);
4265 }
4266 bool IsSigned = ICmpInst::isSigned(Pred);
4267
4268 bool Replace = false;
4269 CmpPredicate ExtendedCmpPredicate;
4270 // (x < y) ? -1 : zext(x != y)
4271 // (x < y) ? -1 : zext(x > y)
4272 if (ICmpInst::isLT(P: Pred) && match(V: TV, P: m_AllOnes()) &&
4273 match(V: FV, P: m_ZExt(Op: m_c_ICmp(Pred&: ExtendedCmpPredicate, L: m_Specific(V: LHS),
4274 R: m_Specific(V: RHS)))) &&
4275 (ExtendedCmpPredicate == ICmpInst::ICMP_NE ||
4276 ICmpInst::getSwappedPredicate(pred: ExtendedCmpPredicate) == Pred))
4277 Replace = true;
4278
4279 // (x > y) ? 1 : sext(x != y)
4280 // (x > y) ? 1 : sext(x < y)
4281 if (ICmpInst::isGT(P: Pred) && match(V: TV, P: m_One()) &&
4282 match(V: FV, P: m_SExt(Op: m_c_ICmp(Pred&: ExtendedCmpPredicate, L: m_Specific(V: LHS),
4283 R: m_Specific(V: RHS)))) &&
4284 (ExtendedCmpPredicate == ICmpInst::ICMP_NE ||
4285 ICmpInst::getSwappedPredicate(pred: ExtendedCmpPredicate) == Pred))
4286 Replace = true;
4287
4288 // (x == y) ? 0 : (x > y ? 1 : -1)
4289 CmpPredicate FalseBranchSelectPredicate;
4290 const APInt *InnerTV, *InnerFV;
4291 if (Pred == ICmpInst::ICMP_EQ && match(V: TV, P: m_Zero()) &&
4292 match(V: FV, P: m_Select(C: m_c_ICmp(Pred&: FalseBranchSelectPredicate, L: m_Specific(V: LHS),
4293 R: m_Specific(V: RHS)),
4294 L: m_APInt(Res&: InnerTV), R: m_APInt(Res&: InnerFV)))) {
4295 if (!ICmpInst::isGT(P: FalseBranchSelectPredicate)) {
4296 FalseBranchSelectPredicate =
4297 ICmpInst::getSwappedPredicate(pred: FalseBranchSelectPredicate);
4298 std::swap(a&: LHS, b&: RHS);
4299 }
4300
4301 if (!InnerTV->isOne()) {
4302 std::swap(a&: InnerTV, b&: InnerFV);
4303 std::swap(a&: LHS, b&: RHS);
4304 }
4305
4306 if (ICmpInst::isGT(P: FalseBranchSelectPredicate) && InnerTV->isOne() &&
4307 InnerFV->isAllOnes()) {
4308 IsSigned = ICmpInst::isSigned(Pred: FalseBranchSelectPredicate);
4309 Replace = true;
4310 }
4311 }
4312
4313 // Special cases with constants: x == C ? 0 : (x > C-1 ? 1 : -1)
4314 if (Pred == ICmpInst::ICMP_EQ && match(V: TV, P: m_Zero())) {
4315 const APInt *C;
4316 if (match(V: RHS, P: m_APInt(Res&: C))) {
4317 CmpPredicate InnerPred;
4318 Value *InnerRHS;
4319 const APInt *InnerTV, *InnerFV;
4320 if (match(V: FV,
4321 P: m_Select(C: m_ICmp(Pred&: InnerPred, L: m_Specific(V: LHS), R: m_Value(V&: InnerRHS)),
4322 L: m_APInt(Res&: InnerTV), R: m_APInt(Res&: InnerFV)))) {
4323
4324 // x == C ? 0 : (x > C-1 ? 1 : -1)
4325 if (ICmpInst::isGT(P: InnerPred) && InnerTV->isOne() &&
4326 InnerFV->isAllOnes()) {
4327 IsSigned = ICmpInst::isSigned(Pred: InnerPred);
4328 bool CanSubOne = IsSigned ? !C->isMinSignedValue() : !C->isMinValue();
4329 if (CanSubOne) {
4330 APInt Cminus1 = *C - 1;
4331 if (match(V: InnerRHS, P: m_SpecificInt(V: Cminus1)))
4332 Replace = true;
4333 }
4334 }
4335
4336 // x == C ? 0 : (x < C+1 ? -1 : 1)
4337 if (ICmpInst::isLT(P: InnerPred) && InnerTV->isAllOnes() &&
4338 InnerFV->isOne()) {
4339 IsSigned = ICmpInst::isSigned(Pred: InnerPred);
4340 bool CanAddOne = IsSigned ? !C->isMaxSignedValue() : !C->isMaxValue();
4341 if (CanAddOne) {
4342 APInt Cplus1 = *C + 1;
4343 if (match(V: InnerRHS, P: m_SpecificInt(V: Cplus1)))
4344 Replace = true;
4345 }
4346 }
4347 }
4348 }
4349 }
4350
4351 Intrinsic::ID IID = IsSigned ? Intrinsic::scmp : Intrinsic::ucmp;
4352 if (Replace)
4353 return replaceInstUsesWith(
4354 I&: SI, V: Builder.CreateIntrinsic(RetTy: SI.getType(), ID: IID, Args: {LHS, RHS}));
4355 return nullptr;
4356}
4357
4358bool InstCombinerImpl::fmulByZeroIsZero(Value *MulVal, FastMathFlags FMF,
4359 const Instruction *CtxI) const {
4360 KnownFPClass Known =
4361 computeKnownFPClass(V: MulVal, FMF, InterestedClasses: fcNegative, SQ: SQ.getWithInstruction(I: CtxI));
4362
4363 return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity() &&
4364 (FMF.noSignedZeros() || Known.signBitIsZeroOrNaN());
4365}
4366
4367static bool matchFMulByZeroIfResultEqZero(InstCombinerImpl &IC, Value *Cmp0,
4368 Value *Cmp1, Value *TrueVal,
4369 Value *FalseVal, Instruction &CtxI,
4370 bool SelectIsNSZ) {
4371 Value *MulRHS;
4372 if (match(V: Cmp1, P: m_PosZeroFP()) &&
4373 match(V: TrueVal, P: m_c_FMul(L: m_Specific(V: Cmp0), R: m_Value(V&: MulRHS)))) {
4374 FastMathFlags FMF = cast<FPMathOperator>(Val: TrueVal)->getFastMathFlags();
4375 // nsz must be on the select, it must be ignored on the multiply. We
4376 // need nnan and ninf on the multiply for the other value.
4377 FMF.setNoSignedZeros(SelectIsNSZ);
4378 return IC.fmulByZeroIsZero(MulVal: MulRHS, FMF, CtxI: &CtxI);
4379 }
4380
4381 return false;
4382}
4383
4384/// Check whether the KnownBits of a select arm may be affected by the
4385/// select condition.
4386static bool hasAffectedValue(Value *V, SmallPtrSetImpl<Value *> &Affected,
4387 unsigned Depth) {
4388 if (Depth == MaxAnalysisRecursionDepth)
4389 return false;
4390
4391 // Ignore the case where the select arm itself is affected. These cases
4392 // are handled more efficiently by other optimizations.
4393 if (Depth != 0 && Affected.contains(Ptr: V))
4394 return true;
4395
4396 if (auto *I = dyn_cast<Instruction>(Val: V)) {
4397 if (isa<PHINode>(Val: I)) {
4398 if (Depth == MaxAnalysisRecursionDepth - 1)
4399 return false;
4400 Depth = MaxAnalysisRecursionDepth - 2;
4401 }
4402 return any_of(Range: I->operands(), P: [&](Value *Op) {
4403 return Op->getType()->isIntOrIntVectorTy() &&
4404 hasAffectedValue(V: Op, Affected, Depth: Depth + 1);
4405 });
4406 }
4407
4408 return false;
4409}
4410
4411// This transformation enables the possibility of transforming fcmp + sel into
4412// a fmaxnum/fminnum intrinsic.
4413static Value *foldSelectIntoAddConstant(SelectInst &SI,
4414 InstCombiner::BuilderTy &Builder) {
4415 // Do this transformation only when select instruction gives NaN and NSZ
4416 // guarantee.
4417 auto *SIFOp = dyn_cast<FPMathOperator>(Val: &SI);
4418 if (!SIFOp || !SIFOp->hasNoSignedZeros() || !SIFOp->hasNoNaNs())
4419 return nullptr;
4420
4421 auto TryFoldIntoAddConstant =
4422 [&Builder, &SI](CmpInst::Predicate Pred, Value *X, Value *Z,
4423 Instruction *FAdd, Constant *C, bool Swapped) -> Value * {
4424 // Only these relational predicates can be transformed into maxnum/minnum
4425 // intrinsic.
4426 if (!CmpInst::isRelational(P: Pred) || !match(V: Z, P: m_AnyZeroFP()))
4427 return nullptr;
4428
4429 if (!match(V: FAdd, P: m_FAdd(L: m_Specific(V: X), R: m_Specific(V: C))))
4430 return nullptr;
4431
4432 Value *NewSelect = Builder.CreateSelect(C: SI.getCondition(), True: Swapped ? Z : X,
4433 False: Swapped ? X : Z, Name: "", MDFrom: &SI);
4434 NewSelect->takeName(V: &SI);
4435
4436 Value *NewFAdd = Builder.CreateFAdd(L: NewSelect, R: C);
4437 NewFAdd->takeName(V: FAdd);
4438
4439 // Propagate FastMath flags
4440 FastMathFlags SelectFMF = SI.getFastMathFlags();
4441 FastMathFlags FAddFMF = FAdd->getFastMathFlags();
4442 FastMathFlags NewFMF = FastMathFlags::intersectRewrite(LHS: SelectFMF, RHS: FAddFMF) |
4443 FastMathFlags::unionValue(LHS: SelectFMF, RHS: FAddFMF);
4444 cast<Instruction>(Val: NewFAdd)->setFastMathFlags(NewFMF);
4445 cast<Instruction>(Val: NewSelect)->setFastMathFlags(NewFMF);
4446
4447 return NewFAdd;
4448 };
4449
4450 // select((fcmp Pred, X, 0), (fadd X, C), C)
4451 // => fadd((select (fcmp Pred, X, 0), X, 0), C)
4452 //
4453 // Pred := OGT, OGE, OLT, OLE, UGT, UGE, ULT, and ULE
4454 Instruction *FAdd;
4455 Constant *C;
4456 Value *X, *Z;
4457 CmpPredicate Pred;
4458
4459 // Note: OneUse check for `Cmp` is necessary because it makes sure that other
4460 // InstCombine folds don't undo this transformation and cause an infinite
4461 // loop. Furthermore, it could also increase the operation count.
4462 if (match(V: &SI, P: m_Select(C: m_OneUse(SubPattern: m_FCmp(Pred, L: m_Value(V&: X), R: m_Value(V&: Z))),
4463 L: m_OneUse(SubPattern: m_Instruction(I&: FAdd)), R: m_Constant(C))))
4464 return TryFoldIntoAddConstant(Pred, X, Z, FAdd, C, /*Swapped=*/false);
4465
4466 if (match(V: &SI, P: m_Select(C: m_OneUse(SubPattern: m_FCmp(Pred, L: m_Value(V&: X), R: m_Value(V&: Z))),
4467 L: m_Constant(C), R: m_OneUse(SubPattern: m_Instruction(I&: FAdd)))))
4468 return TryFoldIntoAddConstant(Pred, X, Z, FAdd, C, /*Swapped=*/true);
4469
4470 return nullptr;
4471}
4472
4473static Value *foldSelectBitTest(SelectInst &Sel, Value *CondVal, Value *TrueVal,
4474 Value *FalseVal,
4475 InstCombiner::BuilderTy &Builder,
4476 const SimplifyQuery &SQ) {
4477 // If this is a vector select, we need a vector compare.
4478 Type *SelType = Sel.getType();
4479 if (SelType->isVectorTy() != CondVal->getType()->isVectorTy())
4480 return nullptr;
4481
4482 Value *V;
4483 APInt AndMask;
4484 bool CreateAnd = false;
4485 CmpPredicate Pred;
4486 Value *CmpLHS, *CmpRHS;
4487
4488 if (match(V: CondVal, P: m_ICmp(Pred, L: m_Value(V&: CmpLHS), R: m_Value(V&: CmpRHS)))) {
4489 if (ICmpInst::isEquality(P: Pred)) {
4490 if (!match(V: CmpRHS, P: m_Zero()))
4491 return nullptr;
4492
4493 V = CmpLHS;
4494 const APInt *AndRHS;
4495 if (!match(V: CmpLHS, P: m_And(L: m_Value(), R: m_Power2(V&: AndRHS))))
4496 return nullptr;
4497
4498 AndMask = *AndRHS;
4499 } else if (auto Res = decomposeBitTestICmp(LHS: CmpLHS, RHS: CmpRHS, Pred)) {
4500 assert(ICmpInst::isEquality(Res->Pred) && "Not equality test?");
4501 AndMask = Res->Mask;
4502 V = Res->X;
4503 KnownBits Known = computeKnownBits(V, Q: SQ.getWithInstruction(I: &Sel));
4504 AndMask &= Known.getMaxValue();
4505 if (!AndMask.isPowerOf2())
4506 return nullptr;
4507
4508 Pred = Res->Pred;
4509 CreateAnd = true;
4510 } else {
4511 return nullptr;
4512 }
4513 } else if (auto *Trunc = dyn_cast<TruncInst>(Val: CondVal)) {
4514 V = Trunc->getOperand(i_nocapture: 0);
4515 AndMask = APInt(V->getType()->getScalarSizeInBits(), 1);
4516 Pred = ICmpInst::ICMP_NE;
4517 CreateAnd = !Trunc->hasNoUnsignedWrap();
4518 } else {
4519 return nullptr;
4520 }
4521
4522 if (Pred == ICmpInst::ICMP_NE)
4523 std::swap(a&: TrueVal, b&: FalseVal);
4524
4525 if (Value *X = foldSelectICmpAnd(Sel, CondVal, TrueVal, FalseVal, V, AndMask,
4526 CreateAnd, Builder))
4527 return X;
4528
4529 if (Value *X = foldSelectICmpAndBinOp(CondVal, TrueVal, FalseVal, V, AndMask,
4530 CreateAnd, Builder))
4531 return X;
4532
4533 return nullptr;
4534}
4535
4536/// This function makes the following folds:
4537/// select C, (sub 0, X), (xor X, -1)
4538/// -> sub (sext !C), X
4539/// select C, (xor X, -1), (sub 0, X)
4540/// -> sub (sext C), X
4541static Instruction *foldSelectNegNot(SelectInst &SI,
4542 InstCombiner::BuilderTy &Builder) {
4543 auto *CondVal = SI.getCondition();
4544 auto *TrueVal = SI.getTrueValue();
4545 auto *FalseVal = SI.getFalseValue();
4546 auto *SelTy = SI.getType();
4547
4548 if (!SelTy->isIntOrIntVectorTy() || SelTy->isIntOrIntVectorTy(BitWidth: 1))
4549 return nullptr;
4550
4551 if (CondVal->getType()->isVectorTy() != SelTy->isVectorTy())
4552 return nullptr;
4553
4554 auto matchNegNot = [&](Value *Neg, Value *Not, Value *&X) -> bool {
4555 return match(V: Neg, P: m_OneUse(SubPattern: m_Neg(V: m_Value(V&: X)))) &&
4556 match(V: Not, P: m_OneUse(SubPattern: m_Not(V: m_Specific(V: X))));
4557 };
4558
4559 Value *X;
4560 Value *Mask;
4561
4562 // select C, (sub 0, X), (xor X, -1) -> sub (sext !C), X
4563 if (matchNegNot(TrueVal, FalseVal, X)) {
4564 Value *NotCond = Builder.CreateNot(V: CondVal, Name: "not." + CondVal->getName());
4565 Mask = Builder.CreateSExt(V: NotCond, DestTy: SelTy);
4566 return BinaryOperator::CreateSub(V1: Mask, V2: X);
4567 }
4568
4569 // select C, (xor X, -1), (sub 0, X) -> sub (sext C), X
4570 if (matchNegNot(FalseVal, TrueVal, X)) {
4571 Mask = Builder.CreateSExt(V: CondVal, DestTy: SelTy);
4572 return BinaryOperator::CreateSub(V1: Mask, V2: X);
4573 }
4574
4575 return nullptr;
4576}
4577
4578/// Fold select (A & Shift == 0 | B & Shift == 0), 0, Shift -> Shift & A & B
4579/// where Shift is known to be a power of two.
4580static Instruction *foldSelectAndOrPowerOfTwo(SelectInst &SI,
4581 InstCombiner::BuilderTy &Builder,
4582 const SimplifyQuery &SQ) {
4583 Value *Cond = SI.getCondition();
4584
4585 if (!Cond->hasOneUse())
4586 return nullptr;
4587
4588 Value *TrueVal = SI.getTrueValue();
4589 Value *FalseVal = SI.getFalseValue();
4590
4591 Value *A, *B, *Shift;
4592
4593 bool Case1 =
4594 match(V: TrueVal, P: m_Zero()) && match(V: FalseVal, P: m_Value(V&: Shift)) &&
4595 match(V: Cond, P: m_Or(L: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
4596 L: m_c_And(L: m_Specific(V: Shift), R: m_Value(V&: A)),
4597 R: m_Zero()),
4598 R: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
4599 L: m_c_And(L: m_Specific(V: Shift), R: m_Value(V&: B)),
4600 R: m_Zero())));
4601
4602 bool Case2 =
4603 match(V: FalseVal, P: m_Zero()) && match(V: TrueVal, P: m_Value(V&: Shift)) &&
4604 match(V: Cond, P: m_And(L: m_SpecificICmp(MatchPred: ICmpInst::ICMP_NE,
4605 L: m_c_And(L: m_Specific(V: Shift), R: m_Value(V&: A)),
4606 R: m_Zero()),
4607 R: m_SpecificICmp(MatchPred: ICmpInst::ICMP_NE,
4608 L: m_c_And(L: m_Specific(V: Shift), R: m_Value(V&: B)),
4609 R: m_Zero())));
4610
4611 if ((Case1 || Case2) && isKnownToBeAPowerOfTwo(V: Shift, /*OrZero=*/true,
4612 Q: SQ.getWithInstruction(I: &SI))) {
4613 Value *And1 = Builder.CreateAnd(LHS: Shift, RHS: A);
4614 return BinaryOperator::CreateAnd(V1: And1, V2: B);
4615 }
4616
4617 return nullptr;
4618}
4619
4620// Return true if no use can observe the sign of zero of the select result,
4621// looking through phis, selects and the loop back edge to the select itself.
4622static bool isSelectZeroSignInsignificant(SelectInst &SI) {
4623 // Bound the number of uses to look through to keep the compile time in
4624 // check.
4625 constexpr unsigned MaxUsesToLookThrough = 16;
4626 unsigned NumUses = 0;
4627 SmallPtrSet<Instruction *, 4> Visited;
4628 SmallVector<Instruction *> Worklist(1, &SI);
4629 while (!Worklist.empty()) {
4630 for (Use &U : Worklist.pop_back_val()->uses()) {
4631 if (++NumUses > MaxUsesToLookThrough)
4632 return false;
4633 auto *User = cast<Instruction>(Val: U.getUser());
4634 if (User == &SI)
4635 continue;
4636 if (canIgnoreSignBitOfZero(U))
4637 continue;
4638 if (isa<PHINode, SelectInst>(Val: User)) {
4639 if (Visited.insert(Ptr: User).second)
4640 Worklist.push_back(Elt: User);
4641 continue;
4642 }
4643 return false;
4644 }
4645 }
4646 return true;
4647}
4648
4649Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
4650 Value *CondVal = SI.getCondition();
4651 Value *TrueVal = SI.getTrueValue();
4652 Value *FalseVal = SI.getFalseValue();
4653 Type *SelType = SI.getType();
4654
4655 FastMathFlags FMF;
4656 if (auto *FPMO = dyn_cast_if_present<FPMathOperator>(Val: &SI))
4657 FMF = FPMO->getFastMathFlags();
4658
4659 if (Value *V = simplifySelectInst(Cond: CondVal, TrueVal, FalseVal, FMF,
4660 Q: SQ.getWithInstruction(I: &SI)))
4661 return replaceInstUsesWith(I&: SI, V);
4662
4663 if (Instruction *I = canonicalizeSelectToShuffle(SI))
4664 return I;
4665
4666 if (Instruction *I = canonicalizeScalarSelectOfVecs(Sel&: SI, IC&: *this))
4667 return I;
4668
4669 // Fold: select (icmp ult X, 2), X, ctpop(X) --> ctpop(X)
4670 // ctpop(0)==0 and ctpop(1)==1, so the guard is always redundant.
4671 if (match(V: FalseVal, P: m_Ctpop(Op0: m_Specific(V: TrueVal))) &&
4672 match(V: CondVal, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_ULT, L: m_Specific(V: TrueVal),
4673 R: m_SpecificInt(V: 2)))) {
4674 cast<Instruction>(Val: FalseVal)->dropPoisonGeneratingAnnotations();
4675 addToWorklist(I: cast<Instruction>(Val: FalseVal));
4676 return replaceInstUsesWith(I&: SI, V: FalseVal);
4677 }
4678
4679 // If the type of select is not an integer type or if the condition and
4680 // the selection type are not both scalar nor both vector types, there is no
4681 // point in attempting to match these patterns.
4682 Type *CondType = CondVal->getType();
4683 if (!isa<Constant>(Val: CondVal) && SelType->isIntOrIntVectorTy() &&
4684 CondType->isVectorTy() == SelType->isVectorTy()) {
4685 if (Value *S = simplifyWithOpReplaced(V: TrueVal, Op: CondVal,
4686 RepOp: ConstantInt::getTrue(Ty: CondType), Q: SQ,
4687 /* AllowRefinement */ true))
4688 return replaceOperand(I&: SI, OpNum: 1, V: S);
4689
4690 if (Value *S = simplifyWithOpReplaced(V: FalseVal, Op: CondVal,
4691 RepOp: ConstantInt::getFalse(Ty: CondType), Q: SQ,
4692 /* AllowRefinement */ true))
4693 return replaceOperand(I&: SI, OpNum: 2, V: S);
4694
4695 if (replaceInInstruction(V: TrueVal, Old: CondVal,
4696 New: ConstantInt::getTrue(Ty: CondType)) ||
4697 replaceInInstruction(V: FalseVal, Old: CondVal,
4698 New: ConstantInt::getFalse(Ty: CondType)))
4699 return &SI;
4700 }
4701
4702 if (Instruction *R = foldSelectOfBools(SI))
4703 return R;
4704
4705 // Selecting between two integer or vector splat integer constants?
4706 //
4707 // Note that we don't handle a scalar select of vectors:
4708 // select i1 %c, <2 x i8> <1, 1>, <2 x i8> <0, 0>
4709 // because that may need 3 instructions to splat the condition value:
4710 // extend, insertelement, shufflevector.
4711 //
4712 // Do not handle i1 TrueVal and FalseVal otherwise would result in
4713 // zext/sext i1 to i1.
4714 if (SelType->isIntOrIntVectorTy() && !SelType->isIntOrIntVectorTy(BitWidth: 1) &&
4715 CondVal->getType()->isVectorTy() == SelType->isVectorTy()) {
4716 // select C, 1, 0 -> zext C to int
4717 if (match(V: TrueVal, P: m_One()) && match(V: FalseVal, P: m_Zero()))
4718 return new ZExtInst(CondVal, SelType);
4719
4720 // select C, -1, 0 -> sext C to int
4721 if (match(V: TrueVal, P: m_AllOnes()) && match(V: FalseVal, P: m_Zero()))
4722 return new SExtInst(CondVal, SelType);
4723
4724 // select C, 0, 1 -> zext !C to int
4725 if (match(V: TrueVal, P: m_Zero()) && match(V: FalseVal, P: m_One())) {
4726 Value *NotCond = Builder.CreateNot(V: CondVal, Name: "not." + CondVal->getName());
4727 return new ZExtInst(NotCond, SelType);
4728 }
4729
4730 // select C, 0, -1 -> sext !C to int
4731 if (match(V: TrueVal, P: m_Zero()) && match(V: FalseVal, P: m_AllOnes())) {
4732 Value *NotCond = Builder.CreateNot(V: CondVal, Name: "not." + CondVal->getName());
4733 return new SExtInst(NotCond, SelType);
4734 }
4735 }
4736
4737 if (Instruction *I = foldSelectNegNot(SI, Builder))
4738 return I;
4739
4740 if (Instruction *I = foldSelectAndOrPowerOfTwo(SI, Builder, SQ))
4741 return I;
4742
4743 auto *SIFPOp = dyn_cast<FPMathOperator>(Val: &SI);
4744
4745 if (auto *FCmp = dyn_cast<FCmpInst>(Val: CondVal)) {
4746 FCmpInst::Predicate Pred = FCmp->getPredicate();
4747 Value *Cmp0 = FCmp->getOperand(i_nocapture: 0), *Cmp1 = FCmp->getOperand(i_nocapture: 1);
4748 // Are we selecting a value based on a comparison of the two values?
4749 if ((Cmp0 == TrueVal && Cmp1 == FalseVal) ||
4750 (Cmp0 == FalseVal && Cmp1 == TrueVal)) {
4751 // Canonicalize to use ordered comparisons by swapping the select
4752 // operands.
4753 //
4754 // e.g.
4755 // (X ugt Y) ? X : Y -> (X ole Y) ? Y : X
4756 if (FCmp->hasOneUse() && FCmpInst::isUnordered(predicate: Pred)) {
4757 FCmpInst::Predicate InvPred = FCmp->getInversePredicate();
4758 Value *NewCond = Builder.CreateFCmpFMF(P: InvPred, LHS: Cmp0, RHS: Cmp1, FMFSource: FCmp,
4759 Name: FCmp->getName() + ".inv");
4760 // Propagate ninf/nnan from fcmp to select.
4761 FastMathFlags FMF = SI.getFastMathFlags();
4762 if (FCmp->hasNoNaNs())
4763 FMF.setNoNaNs(true);
4764 if (FCmp->hasNoInfs())
4765 FMF.setNoInfs(true);
4766 Value *NewSel = Builder.CreateSelectFMF(
4767 C: NewCond, True: FalseVal, False: TrueVal, FMFSource: FMF, Name: "",
4768 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &SI);
4769 if (auto *NewSI = dyn_cast<SelectInst>(Val: NewSel))
4770 NewSI->swapProfMetadata();
4771 return replaceInstUsesWith(I&: SI, V: NewSel);
4772 }
4773 }
4774
4775 if (SIFPOp) {
4776 // Fold out scale-if-equals-zero pattern.
4777 //
4778 // This pattern appears in code with denormal range checks after it's
4779 // assumed denormals are treated as zero. This drops a canonicalization.
4780
4781 // TODO: Could relax the signed zero logic. We just need to know the sign
4782 // of the result matches (fmul x, y has the same sign as x).
4783 //
4784 // TODO: Handle always-canonicalizing variant that selects some value or 1
4785 // scaling factor in the fmul visitor.
4786
4787 // TODO: Handle ldexp too
4788
4789 Value *MatchCmp0 = nullptr;
4790 Value *MatchCmp1 = nullptr;
4791
4792 // (select (fcmp [ou]eq x, 0.0), (fmul x, K), x => x
4793 // (select (fcmp [ou]ne x, 0.0), x, (fmul x, K) => x
4794 if (Pred == CmpInst::FCMP_OEQ || Pred == CmpInst::FCMP_UEQ) {
4795 MatchCmp0 = FalseVal;
4796 MatchCmp1 = TrueVal;
4797 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UNE) {
4798 MatchCmp0 = TrueVal;
4799 MatchCmp1 = FalseVal;
4800 }
4801
4802 if (Cmp0 == MatchCmp0 &&
4803 matchFMulByZeroIfResultEqZero(IC&: *this, Cmp0, Cmp1, TrueVal: MatchCmp1, FalseVal: MatchCmp0,
4804 CtxI&: SI, SelectIsNSZ: SIFPOp->hasNoSignedZeros()))
4805 return replaceInstUsesWith(I&: SI, V: Cmp0);
4806
4807 Type *EltTy = SelType->getScalarType();
4808
4809 // TODO: Generalize to any ordered / unordered compare.
4810 if ((Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) &&
4811 match(V: Cmp1, P: m_PosZeroFP()) && EltTy->isIEEELikeFPTy()) {
4812 // Fold out only-canonicalize-non-nans pattern. This implements a
4813 // wrapper around llvm.canonicalize which is not required to quiet
4814 // signaling nans or preserve nan payload bits.
4815 //
4816 // %hard.canonical = call @llvm.canonicalize(%x)
4817 // %soft.canonical = fdiv 1.0, %x
4818 // %ord = fcmp ord %x, 0.0
4819 // %x.canon = select i1 %ord, %hard.canonical, %soft.canonical
4820 //
4821 // With known IEEE handling:
4822 // => %x
4823 //
4824 // With other denormal behaviors:
4825 // => llvm.canonicalize(%x)
4826 //
4827 // Note the fdiv could be any value preserving, potentially
4828 // canonicalizing floating-point operation such as fmul by 1.0. However,
4829 // since in the llvm model canonicalization is not mandatory, the fmul
4830 // would have been dropped by the time we reached here. The trick here
4831 // is to use a reciprocal fdiv. It's not a droppable no-op, as it could
4832 // return an infinity if %x were sufficiently small, but in this pattern
4833 // we're only using the output for nan values.
4834
4835 if (Pred == CmpInst::FCMP_ORD) {
4836 MatchCmp0 = TrueVal;
4837 MatchCmp1 = FalseVal;
4838 } else {
4839 MatchCmp0 = FalseVal;
4840 MatchCmp1 = TrueVal;
4841 }
4842
4843 bool RcpIfNan = match(V: MatchCmp1, P: m_FDiv(L: m_FPOne(), R: m_Specific(V: Cmp0)));
4844 bool CanonicalizeIfNotNan =
4845 match(V: MatchCmp0, P: m_FCanonicalize(Op0: m_Specific(V: Cmp0)));
4846
4847 if (RcpIfNan || CanonicalizeIfNotNan) {
4848 const fltSemantics &FPSem = EltTy->getFltSemantics();
4849 DenormalMode Mode = F.getDenormalMode(FPType: FPSem);
4850
4851 if (RcpIfNan) {
4852 if (Mode == DenormalMode::getIEEE()) {
4853 // Special case for the other select operand. Otherwise, we may
4854 // need to insert freeze on Cmp0 in the compare and select.
4855 if (CanonicalizeIfNotNan)
4856 return replaceInstUsesWith(I&: SI, V: Cmp0);
4857
4858 if (isGuaranteedNotToBeUndef(V: Cmp0, AC: &AC, CtxI: &SI, DT: &DT)) {
4859 // select (fcmp ord x, 0), y, (fdiv 1, x)
4860 // => select (fcmp ord x, 0), y, x
4861 //
4862 // select (fcmp uno x, 0), (fdiv 1, x), y
4863 // => select (fcmp uno x, 0), x, y
4864 replaceOperand(I&: SI, OpNum: Pred == CmpInst::FCMP_ORD ? 2 : 1, V: Cmp0);
4865 return &SI;
4866 }
4867
4868 auto *FrCmp0 = InsertNewInstBefore(
4869 New: new FreezeInst(Cmp0, Cmp0->getName() + ".fr"),
4870 Old: FCmp->getIterator());
4871
4872 replaceOperand(I&: *FCmp, OpNum: 0, V: FrCmp0);
4873 return replaceOperand(I&: SI, OpNum: Pred == CmpInst::FCMP_ORD ? 2 : 1,
4874 V: FrCmp0);
4875 }
4876 }
4877
4878 if (CanonicalizeIfNotNan) {
4879 // IEEE handling does not have non-canonical values, so the
4880 // canonicalize can be dropped for direct replacement without
4881 // looking for the intermediate maybe-canonicalizing operation.
4882 if (Mode == DenormalMode::getIEEE()) {
4883 // select (fcmp ord x, 0), canonicalize(x), y
4884 // => select (fcmp ord x, 0), x, y
4885
4886 replaceOperand(I&: SI, OpNum: Pred == CmpInst::FCMP_ORD ? 1 : 2, V: Cmp0);
4887 return &SI;
4888 }
4889
4890 // If denormals may be flushed, we need to retain the canonicalize
4891 // call. This introduces a canonicalization on the nan path, which
4892 // we are not free to do as that could change the sign bit or
4893 // payload bits. We can only do this if there were a no-op like
4894 // floating-point instruction which may have changed the nan bits
4895 // anyway.
4896
4897 // Leave the dynamic mode case alone. This would introduce new
4898 // constraints if the mode may be refined later.
4899 if (RcpIfNan && (Mode.inputsAreZero() || Mode.outputsAreZero()))
4900 return replaceInstUsesWith(I&: SI, V: MatchCmp0);
4901 assert(RcpIfNan || Mode != DenormalMode::getIEEE());
4902 }
4903 }
4904 }
4905 }
4906 }
4907
4908 if (SIFPOp) {
4909 // TODO: Try to forward-propagate FMF from select arms to the select.
4910
4911 auto *FCmp = dyn_cast<FCmpInst>(Val: CondVal);
4912
4913 // Canonicalize select of FP values where NaN and -0.0 are not valid as
4914 // minnum/maxnum intrinsics.
4915 //
4916 // Note that the `nnan` flag is propagated from the comparison, not from the
4917 // select. While it's technically possible to transform a `fcmp` + `select
4918 // nnan` to a `minnum`/`maxnum` call *without* an `nnan`, that would be a
4919 // pessimization in practice. Many targets can't map `minnum`/`maxnum` to a
4920 // single instruction, and if they cannot prove the absence of NaN, must
4921 // lower it to a routine or a libcall. There are additional reasons besides
4922 // performance to avoid introducing libcalls where none existed before
4923 // (https://github.com/llvm/llvm-project/issues/54554).
4924 //
4925 // As such, we want to ensure that the generated `minnum`/`maxnum` intrinsic
4926 // has the `nnan nsz` flags, which allow it to be lowered *back* to a
4927 // fcmp+select if that's the best way to express it on the target.
4928 if (FCmp && FCmp->hasNoNaNs() &&
4929 (SIFPOp->hasNoSignedZeros() || isSelectZeroSignInsignificant(SI))) {
4930 Value *X, *Y;
4931 if (match(V: &SI, P: m_OrdOrUnordFMax(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
4932 Value *BinIntr =
4933 Builder.CreateBinaryIntrinsic(ID: Intrinsic::maxnum, LHS: X, RHS: Y, FMFSource: &SI);
4934 if (auto *BinIntrInst = dyn_cast<Instruction>(Val: BinIntr)) {
4935 // `ninf` must be propagated from the comparison too, rather than the
4936 // select: https://github.com/llvm/llvm-project/pull/136433
4937 BinIntrInst->setHasNoInfs(FCmp->hasNoInfs());
4938 // The `nsz` flag is a precondition, so let's ensure it's always added
4939 // to the min/max operation, even if it wasn't on the select. This
4940 // could happen if the select doesn't have `nsz`, but no use of the
4941 // result can observe the sign of zero.
4942 BinIntrInst->setHasNoSignedZeros(true);
4943 // As mentioned above, `nnan` is also a precondition, so we always set
4944 // the flag.
4945 BinIntrInst->setHasNoNaNs(true);
4946 }
4947 return replaceInstUsesWith(I&: SI, V: BinIntr);
4948 }
4949
4950 if (match(V: &SI, P: m_OrdOrUnordFMin(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
4951 Value *BinIntr =
4952 Builder.CreateBinaryIntrinsic(ID: Intrinsic::minnum, LHS: X, RHS: Y, FMFSource: &SI);
4953 if (auto *BinIntrInst = dyn_cast<Instruction>(Val: BinIntr)) {
4954 BinIntrInst->setHasNoInfs(FCmp->hasNoInfs());
4955 BinIntrInst->setHasNoSignedZeros(true);
4956 BinIntrInst->setHasNoNaNs(true);
4957 }
4958 return replaceInstUsesWith(I&: SI, V: BinIntr);
4959 }
4960 }
4961 }
4962
4963 // Fold selecting to fabs.
4964 if (Instruction *Fabs = foldSelectWithFCmpToFabs(SI, IC&: *this))
4965 return Fabs;
4966
4967 if (Instruction *I = foldSelectOfOrderedFAbsCmpOfNaNScrubbedValue(SI, IC&: *this))
4968 return I;
4969
4970 // See if we are selecting two values based on a comparison of the two values.
4971 if (CmpInst *CI = dyn_cast<CmpInst>(Val: CondVal))
4972 if (Instruction *NewSel = foldSelectValueEquivalence(Sel&: SI, Cmp&: *CI))
4973 return NewSel;
4974
4975 if (ICmpInst *ICI = dyn_cast<ICmpInst>(Val: CondVal))
4976 if (Instruction *Result = foldSelectInstWithICmp(SI, ICI))
4977 return Result;
4978
4979 if (Instruction *V =
4980 foldSelectICmpAndAnd(SelType, Cond: CondVal, TVal: TrueVal, FVal: FalseVal, Builder))
4981 return V;
4982
4983 if (Value *V = foldSelectBitTest(Sel&: SI, CondVal, TrueVal, FalseVal, Builder, SQ))
4984 return replaceInstUsesWith(I&: SI, V);
4985
4986 if (Instruction *Add = foldAddSubSelect(SI, Builder))
4987 return Add;
4988 if (Instruction *Add = foldOverflowingAddSubSelect(SI, Builder))
4989 return Add;
4990 if (Instruction *Or = foldSetClearBits(Sel&: SI, Builder))
4991 return Or;
4992 if (Instruction *Mul = foldSelectZeroOrFixedOp(SI, IC&: *this))
4993 return Mul;
4994
4995 // Turn (select C, (op X, Y), (op X, Z)) -> (op X, (select C, Y, Z))
4996 auto *TI = dyn_cast<Instruction>(Val: TrueVal);
4997 auto *FI = dyn_cast<Instruction>(Val: FalseVal);
4998 if (TI && FI && TI->getOpcode() == FI->getOpcode())
4999 if (Instruction *IV = foldSelectOpOp(SI, TI, FI))
5000 return IV;
5001
5002 if (Instruction *I = foldSelectIntrinsic(SI))
5003 return I;
5004
5005 if (Instruction *I = foldSelectExtConst(Sel&: SI))
5006 return I;
5007
5008 if (Instruction *I = foldSelectWithSRem(SI, IC&: *this, Builder))
5009 return I;
5010
5011 // Fold (select C, (gep Ptr, Idx), Ptr) -> (gep Ptr, (select C, Idx, 0))
5012 // Fold (select C, Ptr, (gep Ptr, Idx)) -> (gep Ptr, (select C, 0, Idx))
5013 auto SelectGepWithBase = [&](GetElementPtrInst *Gep, Value *Base,
5014 bool Swap) -> GetElementPtrInst * {
5015 Value *Ptr = Gep->getPointerOperand();
5016 if (Gep->getNumOperands() != 2 || Gep->getPointerOperand() != Base ||
5017 !Gep->hasOneUse())
5018 return nullptr;
5019 Value *Idx = Gep->getOperand(i_nocapture: 1);
5020 if (isa<VectorType>(Val: CondVal->getType()) && !isa<VectorType>(Val: Idx->getType()))
5021 return nullptr;
5022 Type *ElementType = Gep->getSourceElementType();
5023 Value *NewT = Idx;
5024 Value *NewF = Constant::getNullValue(Ty: Idx->getType());
5025 if (Swap)
5026 std::swap(a&: NewT, b&: NewF);
5027 Value *NewSI =
5028 Builder.CreateSelect(C: CondVal, True: NewT, False: NewF, Name: SI.getName() + ".idx", MDFrom: &SI);
5029 return GetElementPtrInst::Create(PointeeType: ElementType, Ptr, IdxList: NewSI,
5030 NW: Gep->getNoWrapFlags());
5031 };
5032 if (auto *TrueGep = dyn_cast<GetElementPtrInst>(Val: TrueVal))
5033 if (auto *NewGep = SelectGepWithBase(TrueGep, FalseVal, false))
5034 return NewGep;
5035 if (auto *FalseGep = dyn_cast<GetElementPtrInst>(Val: FalseVal))
5036 if (auto *NewGep = SelectGepWithBase(FalseGep, TrueVal, true))
5037 return NewGep;
5038
5039 // See if we can fold the select into one of our operands.
5040 if (SelType->isIntOrIntVectorTy() || SelType->isFPOrFPVectorTy()) {
5041 if (Instruction *FoldI = foldSelectIntoOp(SI, TrueVal, FalseVal))
5042 return FoldI;
5043
5044 Value *LHS, *RHS;
5045 Instruction::CastOps CastOp;
5046 SelectPatternResult SPR = matchSelectPattern(V: &SI, LHS, RHS, CastOp: &CastOp);
5047 auto SPF = SPR.Flavor;
5048 if (SPF) {
5049 Value *LHS2, *RHS2;
5050 if (SelectPatternFlavor SPF2 = matchSelectPattern(V: LHS, LHS&: LHS2, RHS&: RHS2).Flavor)
5051 if (Instruction *R = foldSPFofSPF(Inner: cast<Instruction>(Val: LHS), SPF1: SPF2, A: LHS2,
5052 B: RHS2, Outer&: SI, SPF2: SPF, C: RHS))
5053 return R;
5054 if (SelectPatternFlavor SPF2 = matchSelectPattern(V: RHS, LHS&: LHS2, RHS&: RHS2).Flavor)
5055 if (Instruction *R = foldSPFofSPF(Inner: cast<Instruction>(Val: RHS), SPF1: SPF2, A: LHS2,
5056 B: RHS2, Outer&: SI, SPF2: SPF, C: LHS))
5057 return R;
5058 }
5059
5060 if (SelectPatternResult::isMinOrMax(SPF)) {
5061 // Canonicalize so that
5062 // - type casts are outside select patterns.
5063 // - float clamp is transformed to min/max pattern
5064
5065 bool IsCastNeeded = LHS->getType() != SelType;
5066 Value *CmpLHS = cast<CmpInst>(Val: CondVal)->getOperand(i_nocapture: 0);
5067 Value *CmpRHS = cast<CmpInst>(Val: CondVal)->getOperand(i_nocapture: 1);
5068 if (IsCastNeeded ||
5069 (LHS->getType()->isFPOrFPVectorTy() &&
5070 ((CmpLHS != LHS && CmpLHS != RHS) ||
5071 (CmpRHS != LHS && CmpRHS != RHS)))) {
5072 CmpInst::Predicate MinMaxPred = getMinMaxPred(SPF, Ordered: SPR.Ordered);
5073
5074 Value *Cmp;
5075 if (CmpInst::isIntPredicate(P: MinMaxPred))
5076 Cmp = Builder.CreateICmp(P: MinMaxPred, LHS, RHS);
5077 else
5078 Cmp = Builder.CreateFCmpFMF(P: MinMaxPred, LHS, RHS,
5079 FMFSource: cast<Instruction>(Val: SI.getCondition()));
5080
5081 Value *NewSI = Builder.CreateSelect(C: Cmp, True: LHS, False: RHS, Name: SI.getName(), MDFrom: &SI);
5082 if (!IsCastNeeded)
5083 return replaceInstUsesWith(I&: SI, V: NewSI);
5084
5085 Value *NewCast = Builder.CreateCast(Op: CastOp, V: NewSI, DestTy: SelType);
5086 return replaceInstUsesWith(I&: SI, V: NewCast);
5087 }
5088 }
5089 }
5090
5091 // See if we can fold the select into a phi node if the condition is a select.
5092 if (auto *PN = dyn_cast<PHINode>(Val: SI.getCondition()))
5093 if (Instruction *NV = foldOpIntoPhi(I&: SI, PN))
5094 return NV;
5095
5096 if (SelectInst *TrueSI = dyn_cast<SelectInst>(Val: TrueVal)) {
5097 if (TrueSI->getCondition()->getType() == CondVal->getType()) {
5098 // Fold nested selects if the inner condition can be implied by the outer
5099 // condition.
5100 if (Value *V = simplifyNestedSelectsUsingImpliedCond(
5101 SI&: *TrueSI, CondVal, /*CondIsTrue=*/true, DL))
5102 return replaceOperand(I&: SI, OpNum: 1, V);
5103
5104 // We choose this as normal form to enable folding on the And and
5105 // shortening paths for the values (this helps getUnderlyingObjects() for
5106 // example).
5107 if (TrueSI->hasOneUse()) {
5108 Value *And = nullptr, *OtherVal = nullptr;
5109 // select(C0, select(C1, a, b), b) -> select(C0&&C1, a, b)
5110 if (TrueSI->getFalseValue() == FalseVal) {
5111 And = Builder.CreateLogicalAnd(Cond1: CondVal, Cond2: TrueSI->getCondition(), Name: "",
5112 MDFrom: &SI);
5113 OtherVal = TrueSI->getTrueValue();
5114 }
5115 // select(C0, select(C1, b, a), b) -> select(C0&&!C1, a, b)
5116 else if (TrueSI->getTrueValue() == FalseVal) {
5117 Value *InvertedCond = Builder.CreateNot(V: TrueSI->getCondition());
5118 And = Builder.CreateLogicalAnd(Cond1: CondVal, Cond2: InvertedCond, Name: "", MDFrom: &SI);
5119 OtherVal = TrueSI->getFalseValue();
5120 }
5121 if (And && OtherVal) {
5122 replaceOperand(I&: SI, OpNum: 0, V: And);
5123 replaceOperand(I&: SI, OpNum: 1, V: OtherVal);
5124 setExplicitlyUnknownBranchWeightsIfProfiled(I&: SI, DEBUG_TYPE);
5125 return &SI;
5126 }
5127 }
5128 }
5129 }
5130 if (SelectInst *FalseSI = dyn_cast<SelectInst>(Val: FalseVal)) {
5131 if (FalseSI->getCondition()->getType() == CondVal->getType()) {
5132 // Fold nested selects if the inner condition can be implied by the outer
5133 // condition.
5134 if (Value *V = simplifyNestedSelectsUsingImpliedCond(
5135 SI&: *FalseSI, CondVal, /*CondIsTrue=*/false, DL))
5136 return replaceOperand(I&: SI, OpNum: 2, V);
5137
5138 if (FalseSI->hasOneUse()) {
5139 Value *Or = nullptr, *OtherVal = nullptr;
5140 // select(C0, a, select(C1, a, b)) -> select(C0||C1, a, b)
5141 if (FalseSI->getTrueValue() == TrueVal) {
5142 Or = Builder.CreateLogicalOr(Cond1: CondVal, Cond2: FalseSI->getCondition(), Name: "",
5143 MDFrom: &SI);
5144 OtherVal = FalseSI->getFalseValue();
5145 }
5146 // select(C0, a, select(C1, b, a)) -> select(C0||!C1, a, b)
5147 else if (FalseSI->getFalseValue() == TrueVal) {
5148 Value *InvertedCond = Builder.CreateNot(V: FalseSI->getCondition());
5149 Or = Builder.CreateLogicalOr(Cond1: CondVal, Cond2: InvertedCond, Name: "", MDFrom: &SI);
5150 OtherVal = FalseSI->getTrueValue();
5151 }
5152 if (Or && OtherVal) {
5153 replaceOperand(I&: SI, OpNum: 0, V: Or);
5154 replaceOperand(I&: SI, OpNum: 2, V: OtherVal);
5155 setExplicitlyUnknownBranchWeightsIfProfiled(I&: SI, DEBUG_TYPE);
5156 return &SI;
5157 }
5158 }
5159 }
5160 }
5161
5162 // Try to simplify a binop sandwiched between 2 selects with the same
5163 // condition. This is not valid for div/rem because the select might be
5164 // preventing a division-by-zero.
5165 // TODO: A div/rem restriction is conservative; use something like
5166 // isSafeToSpeculativelyExecute().
5167 // select(C, binop(select(C, X, Y), W), Z) -> select(C, binop(X, W), Z)
5168 BinaryOperator *TrueBO;
5169 if (match(V: TrueVal, P: m_OneUse(SubPattern: m_BinOp(I&: TrueBO))) && !TrueBO->isIntDivRem()) {
5170 if (auto *TrueBOSI = dyn_cast<SelectInst>(Val: TrueBO->getOperand(i_nocapture: 0))) {
5171 if (TrueBOSI->getCondition() == CondVal) {
5172 replaceOperand(I&: *TrueBO, OpNum: 0, V: TrueBOSI->getTrueValue());
5173 Worklist.push(I: TrueBO);
5174 return &SI;
5175 }
5176 }
5177 if (auto *TrueBOSI = dyn_cast<SelectInst>(Val: TrueBO->getOperand(i_nocapture: 1))) {
5178 if (TrueBOSI->getCondition() == CondVal) {
5179 replaceOperand(I&: *TrueBO, OpNum: 1, V: TrueBOSI->getTrueValue());
5180 Worklist.push(I: TrueBO);
5181 return &SI;
5182 }
5183 }
5184 }
5185
5186 // select(C, Z, binop(select(C, X, Y), W)) -> select(C, Z, binop(Y, W))
5187 BinaryOperator *FalseBO;
5188 if (match(V: FalseVal, P: m_OneUse(SubPattern: m_BinOp(I&: FalseBO))) && !FalseBO->isIntDivRem()) {
5189 if (auto *FalseBOSI = dyn_cast<SelectInst>(Val: FalseBO->getOperand(i_nocapture: 0))) {
5190 if (FalseBOSI->getCondition() == CondVal) {
5191 replaceOperand(I&: *FalseBO, OpNum: 0, V: FalseBOSI->getFalseValue());
5192 Worklist.push(I: FalseBO);
5193 return &SI;
5194 }
5195 }
5196 if (auto *FalseBOSI = dyn_cast<SelectInst>(Val: FalseBO->getOperand(i_nocapture: 1))) {
5197 if (FalseBOSI->getCondition() == CondVal) {
5198 replaceOperand(I&: *FalseBO, OpNum: 1, V: FalseBOSI->getFalseValue());
5199 Worklist.push(I: FalseBO);
5200 return &SI;
5201 }
5202 }
5203 }
5204
5205 Value *NotCond;
5206 if (match(V: CondVal, P: m_Not(V: m_Value(V&: NotCond))) &&
5207 !InstCombiner::shouldAvoidAbsorbingNotIntoSelect(SI)) {
5208 replaceOperand(I&: SI, OpNum: 0, V: NotCond);
5209 SI.swapValues();
5210 SI.swapProfMetadata();
5211 return &SI;
5212 }
5213
5214 if (Instruction *I = foldVectorSelect(Sel&: SI))
5215 return I;
5216
5217 // If we can compute the condition, there's no need for a select.
5218 // Like the above fold, we are attempting to reduce compile-time cost by
5219 // putting this fold here with limitations rather than in InstSimplify.
5220 // The motivation for this call into value tracking is to take advantage of
5221 // the assumption cache, so make sure that is populated.
5222 if (!CondVal->getType()->isVectorTy() && !AC.assumptions().empty()) {
5223 KnownBits Known(1);
5224 computeKnownBits(V: CondVal, Known, CtxI: &SI);
5225 if (Known.One.isOne())
5226 return replaceInstUsesWith(I&: SI, V: TrueVal);
5227 if (Known.Zero.isOne())
5228 return replaceInstUsesWith(I&: SI, V: FalseVal);
5229 }
5230
5231 if (Instruction *BitCastSel = foldSelectCmpBitcasts(Sel&: SI, Builder))
5232 return BitCastSel;
5233
5234 // Simplify selects that test the returned flag of cmpxchg instructions.
5235 if (Value *V = foldSelectCmpXchg(SI))
5236 return replaceInstUsesWith(I&: SI, V);
5237
5238 if (Instruction *Select = foldSelectBinOpIdentity(Sel&: SI, TLI, IC&: *this))
5239 return Select;
5240
5241 if (Instruction *Funnel = foldSelectFunnelShift(Sel&: SI, Builder))
5242 return Funnel;
5243
5244 if (Instruction *Copysign = foldSelectToCopysign(Sel&: SI, Builder))
5245 return Copysign;
5246
5247 if (Instruction *PN = foldSelectToPhi(Sel&: SI, DT, Builder))
5248 return replaceInstUsesWith(I&: SI, V: PN);
5249
5250 if (Value *V = foldRoundUpIntegerWithPow2Alignment(SI, Builder))
5251 return replaceInstUsesWith(I&: SI, V);
5252
5253 if (Value *V = foldSelectIntoAddConstant(SI, Builder))
5254 return replaceInstUsesWith(I&: SI, V);
5255
5256 // select(mask, mload(ptr,mask,0), 0) -> mload(ptr,mask,0)
5257 // Load inst is intentionally not checked for hasOneUse()
5258 if (match(V: FalseVal, P: m_Zero()) &&
5259 (match(V: TrueVal, P: m_MaskedLoad(Op0: m_Value(), Op1: m_Specific(V: CondVal),
5260 Op2: m_CombineOr(Ps: m_Undef(), Ps: m_Zero()))) ||
5261 match(V: TrueVal, P: m_MaskedGather(Op0: m_Value(), Op1: m_Specific(V: CondVal),
5262 Op2: m_CombineOr(Ps: m_Undef(), Ps: m_Zero()))))) {
5263 auto *MaskedInst = cast<IntrinsicInst>(Val: TrueVal);
5264 if (isa<UndefValue>(Val: MaskedInst->getArgOperand(i: 2)))
5265 MaskedInst->setArgOperand(i: 2, v: FalseVal /* Zero */);
5266 return replaceInstUsesWith(I&: SI, V: MaskedInst);
5267 }
5268
5269 Value *Mask;
5270 if (match(V: TrueVal, P: m_Zero()) &&
5271 (match(V: FalseVal, P: m_MaskedLoad(Op0: m_Value(), Op1: m_Value(V&: Mask),
5272 Op2: m_CombineOr(Ps: m_Undef(), Ps: m_Zero()))) ||
5273 match(V: FalseVal, P: m_MaskedGather(Op0: m_Value(), Op1: m_Value(V&: Mask),
5274 Op2: m_CombineOr(Ps: m_Undef(), Ps: m_Zero())))) &&
5275 (CondVal->getType() == Mask->getType())) {
5276 // We can remove the select by ensuring the load zeros all lanes the
5277 // select would have. We determine this by proving there is no overlap
5278 // between the load and select masks.
5279 // (i.e (load_mask & select_mask) == 0 == no overlap)
5280 bool CanMergeSelectIntoLoad = false;
5281 if (Value *V = simplifyAndInst(LHS: CondVal, RHS: Mask, Q: SQ.getWithInstruction(I: &SI)))
5282 CanMergeSelectIntoLoad = match(V, P: m_Zero());
5283
5284 if (CanMergeSelectIntoLoad) {
5285 auto *MaskedInst = cast<IntrinsicInst>(Val: FalseVal);
5286 if (isa<UndefValue>(Val: MaskedInst->getArgOperand(i: 2)))
5287 MaskedInst->setArgOperand(i: 2, v: TrueVal /* Zero */);
5288 return replaceInstUsesWith(I&: SI, V: MaskedInst);
5289 }
5290 }
5291
5292 if (Instruction *I = foldSelectOfSymmetricSelect(OuterSelVal&: SI, Builder))
5293 return I;
5294
5295 if (Instruction *I = foldNestedSelects(OuterSelVal&: SI, Builder))
5296 return I;
5297
5298 // Match logical variants of the pattern,
5299 // and transform them iff that gets rid of inversions.
5300 // (~x) | y --> ~(x & (~y))
5301 // (~x) & y --> ~(x | (~y))
5302 if (sinkNotIntoOtherHandOfLogicalOp(I&: SI))
5303 return &SI;
5304
5305 if (Instruction *I = foldBitCeil(SI, Builder, IC&: *this))
5306 return I;
5307
5308 if (Instruction *I = foldSelectToCmp(SI))
5309 return I;
5310
5311 if (Instruction *I = foldSelectEqualityTest(Sel&: SI))
5312 return I;
5313
5314 // Fold:
5315 // (select A && B, T, F) -> (select A, (select B, T, F), F)
5316 // (select A || B, T, F) -> (select A, T, (select B, T, F))
5317 // if (select B, T, F) is foldable.
5318 // TODO: preserve FMF flags
5319 auto FoldSelectWithAndOrCond = [&](bool IsAnd, Value *A,
5320 Value *B) -> Instruction * {
5321 if (Value *V = simplifySelectInst(Cond: B, TrueVal, FalseVal, FMF,
5322 Q: SQ.getWithInstruction(I: &SI))) {
5323 Value *NewTrueVal = IsAnd ? V : TrueVal;
5324 Value *NewFalseVal = IsAnd ? FalseVal : V;
5325
5326 // If the True and False values don't change, then preserve the branch
5327 // metadata of the original select as the net effect of this change is to
5328 // simplify the conditional.
5329 Instruction *MDFrom = nullptr;
5330 if (NewTrueVal == TrueVal && NewFalseVal == FalseVal) {
5331 MDFrom = &SI;
5332 }
5333 return SelectInst::Create(C: A, S1: NewTrueVal, S2: NewFalseVal, NameStr: "", InsertBefore: nullptr,
5334 MDFrom);
5335 }
5336
5337 // Is (select B, T, F) a SPF?
5338 if (CondVal->hasOneUse() && SelType->isIntOrIntVectorTy()) {
5339 if (ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: B))
5340 if (Value *V = canonicalizeSPF(Cmp&: *Cmp, TrueVal, FalseVal, IC&: *this)) {
5341 return SelectInst::Create(
5342 C: A, S1: IsAnd ? V : TrueVal, S2: IsAnd ? FalseVal : V, NameStr: "", InsertBefore: nullptr,
5343 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &SI);
5344 }
5345 }
5346
5347 return nullptr;
5348 };
5349
5350 Value *LHS, *RHS;
5351 if (match(V: CondVal, P: m_And(L: m_Value(V&: LHS), R: m_Value(V&: RHS)))) {
5352 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ true, LHS, RHS))
5353 return I;
5354 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ true, RHS, LHS))
5355 return I;
5356 } else if (match(V: CondVal, P: m_Or(L: m_Value(V&: LHS), R: m_Value(V&: RHS)))) {
5357 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ false, LHS, RHS))
5358 return I;
5359 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ false, RHS, LHS))
5360 return I;
5361 } else {
5362 // We cannot swap the operands of logical and/or.
5363 // TODO: Can we swap the operands by inserting a freeze?
5364 if (match(V: CondVal, P: m_LogicalAnd(L: m_Value(V&: LHS), R: m_Value(V&: RHS)))) {
5365 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ true, LHS, RHS))
5366 return I;
5367 } else if (match(V: CondVal, P: m_LogicalOr(L: m_Value(V&: LHS), R: m_Value(V&: RHS)))) {
5368 if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ false, LHS, RHS))
5369 return I;
5370 }
5371 }
5372
5373 // select Cond, !X, X -> xor Cond, X
5374 if (CondVal->getType() == SI.getType() && isKnownInversion(X: FalseVal, Y: TrueVal))
5375 return BinaryOperator::CreateXor(V1: CondVal, V2: FalseVal);
5376
5377 // For vectors, this transform is only safe if the simplification does not
5378 // look through any lane-crossing operations. For now, limit to scalars only.
5379 if (SelType->isIntegerTy() &&
5380 (!isa<Constant>(Val: TrueVal) || !isa<Constant>(Val: FalseVal))) {
5381 // Try to simplify select arms based on KnownBits implied by the condition.
5382 CondContext CC(CondVal);
5383 findValuesAffectedByCondition(Cond: CondVal, /*IsAssume=*/false, InsertAffected: [&](Value *V) {
5384 CC.AffectedValues.insert(Ptr: V);
5385 });
5386 SimplifyQuery Q = SQ.getWithInstruction(I: &SI).getWithCondContext(CC);
5387 if (!CC.AffectedValues.empty()) {
5388 if (!isa<Constant>(Val: TrueVal) &&
5389 hasAffectedValue(V: TrueVal, Affected&: CC.AffectedValues, /*Depth=*/0)) {
5390 KnownBits Known = llvm::computeKnownBits(V: TrueVal, Q);
5391 if (Known.isConstant())
5392 return replaceOperand(I&: SI, OpNum: 1,
5393 V: ConstantInt::get(Ty: SelType, V: Known.getConstant()));
5394 }
5395
5396 CC.Invert = true;
5397 if (!isa<Constant>(Val: FalseVal) &&
5398 hasAffectedValue(V: FalseVal, Affected&: CC.AffectedValues, /*Depth=*/0)) {
5399 KnownBits Known = llvm::computeKnownBits(V: FalseVal, Q);
5400 if (Known.isConstant())
5401 return replaceOperand(I&: SI, OpNum: 2,
5402 V: ConstantInt::get(Ty: SelType, V: Known.getConstant()));
5403 }
5404 }
5405 }
5406
5407 // select (trunc nuw X to i1), X, Y --> select (trunc nuw X to i1), 1, Y
5408 // select (trunc nuw X to i1), Y, X --> select (trunc nuw X to i1), Y, 0
5409 // select (trunc nsw X to i1), X, Y --> select (trunc nsw X to i1), -1, Y
5410 // select (trunc nsw X to i1), Y, X --> select (trunc nsw X to i1), Y, 0
5411 Value *Trunc;
5412 if (match(V: CondVal, P: m_NUWTrunc(Op: m_Value(V&: Trunc))) && !isa<Constant>(Val: Trunc)) {
5413 if (TrueVal == Trunc)
5414 return replaceOperand(I&: SI, OpNum: 1, V: ConstantInt::get(Ty: TrueVal->getType(), V: 1));
5415 if (FalseVal == Trunc)
5416 return replaceOperand(I&: SI, OpNum: 2, V: ConstantInt::get(Ty: FalseVal->getType(), V: 0));
5417 }
5418 if (match(V: CondVal, P: m_NSWTrunc(Op: m_Value(V&: Trunc))) && !isa<Constant>(Val: Trunc)) {
5419 if (TrueVal == Trunc)
5420 return replaceOperand(I&: SI, OpNum: 1,
5421 V: Constant::getAllOnesValue(Ty: TrueVal->getType()));
5422 if (FalseVal == Trunc)
5423 return replaceOperand(I&: SI, OpNum: 2, V: ConstantInt::get(Ty: FalseVal->getType(), V: 0));
5424 }
5425
5426 if (match(V: CondVal, P: m_Trunc(Op: m_Value(V&: Trunc))) && Trunc->getType() == SelType) {
5427 if (match(V: FalseVal, P: m_Zero()) && impliesPoison(ValAssumedPoison: TrueVal, V: CondVal) &&
5428 llvm::computeKnownBits(V: TrueVal, Q: SQ.getWithInstruction(I: &SI))
5429 .countMaxActiveBits() == 1)
5430 return BinaryOperator::CreateAnd(V1: Trunc, V2: TrueVal);
5431
5432 if (cast<TruncInst>(Val: CondVal)->hasNoUnsignedWrap() &&
5433 match(V: TrueVal, P: m_One()) && impliesPoison(ValAssumedPoison: FalseVal, V: CondVal) &&
5434 llvm::computeKnownBits(V: FalseVal, Q: SQ.getWithInstruction(I: &SI))
5435 .countMaxActiveBits() == 1) {
5436 return BinaryOperator::CreateOr(V1: Trunc, V2: FalseVal);
5437 }
5438 }
5439
5440 Value *MaskedLoadPtr;
5441 if (match(V: TrueVal, P: m_OneUse(SubPattern: m_MaskedLoad(Op0: m_Value(V&: MaskedLoadPtr),
5442 Op1: m_Specific(V: CondVal), Op2: m_Value())))) {
5443 auto *LoadInst = cast<IntrinsicInst>(Val: TrueVal);
5444 // Keep the load at its original position to avoid crossing writes. The new
5445 // passthrough must therefore be available there.
5446 if (DT.dominates(Def: FalseVal, User: LoadInst)) {
5447 Builder.SetInsertPoint(LoadInst);
5448 Instruction *In = Builder.CreateMaskedLoad(
5449 Ty: TrueVal->getType(), Ptr: MaskedLoadPtr,
5450 Alignment: LoadInst->getParamAlign(ArgNo: 0).valueOrOne(), Mask: CondVal, PassThru: FalseVal);
5451 In->setAAMetadata(LoadInst->getAAMetadata());
5452 return replaceInstUsesWith(I&: SI, V: In);
5453 }
5454 }
5455
5456 // Canonicalize sign function ashr pattern: select (icmp slt X, 1), ashr X,
5457 // bitwidth-1, 1 -> scmp(X, 0)
5458 // Also handles: select (icmp sgt X, 0), 1, ashr X, bitwidth-1 -> scmp(X, 0)
5459 unsigned BitWidth = SI.getType()->getScalarSizeInBits();
5460 CmpPredicate Pred;
5461 Value *CmpLHS, *CmpRHS;
5462
5463 // Canonicalize sign function ashr patterns:
5464 // select (icmp slt X, 1), ashr X, bitwidth-1, 1 -> scmp(X, 0)
5465 // select (icmp sgt X, 0), 1, ashr X, bitwidth-1 -> scmp(X, 0)
5466 if (match(V: &SI, P: m_Select(C: m_ICmp(Pred, L: m_Value(V&: CmpLHS), R: m_Value(V&: CmpRHS)),
5467 L: m_Value(V&: TrueVal), R: m_Value(V&: FalseVal))) &&
5468 ((Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: m_One()) &&
5469 match(V: TrueVal,
5470 P: m_AShr(L: m_Specific(V: CmpLHS), R: m_SpecificInt(V: BitWidth - 1))) &&
5471 match(V: FalseVal, P: m_One())) ||
5472 (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: m_Zero()) &&
5473 match(V: TrueVal, P: m_One()) &&
5474 match(V: FalseVal,
5475 P: m_AShr(L: m_Specific(V: CmpLHS), R: m_SpecificInt(V: BitWidth - 1)))))) {
5476
5477 Function *Scmp = Intrinsic::getOrInsertDeclaration(
5478 M: SI.getModule(), id: Intrinsic::scmp, OverloadTys: {SI.getType(), SI.getType()});
5479 return CallInst::Create(Func: Scmp, Args: {CmpLHS, ConstantInt::get(Ty: SI.getType(), V: 0)});
5480 }
5481
5482 return nullptr;
5483}
5484