1//===- InstCombineCompares.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitICmp and visitFCmp functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/APSInt.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/Statistic.h"
18#include "llvm/Analysis/CaptureTracking.h"
19#include "llvm/Analysis/CmpInstAnalysis.h"
20#include "llvm/Analysis/ConstantFolding.h"
21#include "llvm/Analysis/InstructionSimplify.h"
22#include "llvm/Analysis/Utils/Local.h"
23#include "llvm/Analysis/VectorUtils.h"
24#include "llvm/IR/ConstantRange.h"
25#include "llvm/IR/DataLayout.h"
26#include "llvm/IR/InstrTypes.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/PatternMatch.h"
29#include "llvm/Support/KnownBits.h"
30#include "llvm/Transforms/InstCombine/InstCombiner.h"
31#include <bitset>
32
33using namespace llvm;
34using namespace PatternMatch;
35
36#define DEBUG_TYPE "instcombine"
37
38// How many times is a select replaced by one of its operands?
39STATISTIC(NumSel, "Number of select opts");
40
41
42/// Compute Result = In1+In2, returning true if the result overflowed for this
43/// type.
44static bool addWithOverflow(APInt &Result, const APInt &In1,
45 const APInt &In2, bool IsSigned = false) {
46 bool Overflow;
47 if (IsSigned)
48 Result = In1.sadd_ov(RHS: In2, Overflow);
49 else
50 Result = In1.uadd_ov(RHS: In2, Overflow);
51
52 return Overflow;
53}
54
55/// Compute Result = In1-In2, returning true if the result overflowed for this
56/// type.
57static bool subWithOverflow(APInt &Result, const APInt &In1,
58 const APInt &In2, bool IsSigned = false) {
59 bool Overflow;
60 if (IsSigned)
61 Result = In1.ssub_ov(RHS: In2, Overflow);
62 else
63 Result = In1.usub_ov(RHS: In2, Overflow);
64
65 return Overflow;
66}
67
68/// Given an icmp instruction, return true if any use of this comparison is a
69/// branch on sign bit comparison.
70static bool hasBranchUse(ICmpInst &I) {
71 for (auto *U : I.users())
72 if (isa<BranchInst>(Val: U))
73 return true;
74 return false;
75}
76
77/// Returns true if the exploded icmp can be expressed as a signed comparison
78/// to zero and updates the predicate accordingly.
79/// The signedness of the comparison is preserved.
80/// TODO: Refactor with decomposeBitTestICmp()?
81static bool isSignTest(ICmpInst::Predicate &Pred, const APInt &C) {
82 if (!ICmpInst::isSigned(predicate: Pred))
83 return false;
84
85 if (C.isZero())
86 return ICmpInst::isRelational(P: Pred);
87
88 if (C.isOne()) {
89 if (Pred == ICmpInst::ICMP_SLT) {
90 Pred = ICmpInst::ICMP_SLE;
91 return true;
92 }
93 } else if (C.isAllOnes()) {
94 if (Pred == ICmpInst::ICMP_SGT) {
95 Pred = ICmpInst::ICMP_SGE;
96 return true;
97 }
98 }
99
100 return false;
101}
102
103/// This is called when we see this pattern:
104/// cmp pred (load (gep GV, ...)), cmpcst
105/// where GV is a global variable with a constant initializer. Try to simplify
106/// this into some simple computation that does not need the load. For example
107/// we can optimize "icmp eq (load (gep "foo", 0, i)), 0" into "icmp eq i, 3".
108///
109/// If AndCst is non-null, then the loaded value is masked with that constant
110/// before doing the comparison. This handles cases like "A[i]&4 == 0".
111Instruction *InstCombinerImpl::foldCmpLoadFromIndexedGlobal(
112 LoadInst *LI, GetElementPtrInst *GEP, GlobalVariable *GV, CmpInst &ICI,
113 ConstantInt *AndCst) {
114 if (LI->isVolatile() || LI->getType() != GEP->getResultElementType() ||
115 GV->getValueType() != GEP->getSourceElementType() || !GV->isConstant() ||
116 !GV->hasDefinitiveInitializer())
117 return nullptr;
118
119 Constant *Init = GV->getInitializer();
120 if (!isa<ConstantArray>(Val: Init) && !isa<ConstantDataArray>(Val: Init))
121 return nullptr;
122
123 uint64_t ArrayElementCount = Init->getType()->getArrayNumElements();
124 // Don't blow up on huge arrays.
125 if (ArrayElementCount > MaxArraySizeForCombine)
126 return nullptr;
127
128 // There are many forms of this optimization we can handle, for now, just do
129 // the simple index into a single-dimensional array.
130 //
131 // Require: GEP GV, 0, i {{, constant indices}}
132 if (GEP->getNumOperands() < 3 || !isa<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1)) ||
133 !cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1))->isZero() ||
134 isa<Constant>(Val: GEP->getOperand(i_nocapture: 2)))
135 return nullptr;
136
137 // Check that indices after the variable are constants and in-range for the
138 // type they index. Collect the indices. This is typically for arrays of
139 // structs.
140 SmallVector<unsigned, 4> LaterIndices;
141
142 Type *EltTy = Init->getType()->getArrayElementType();
143 for (unsigned i = 3, e = GEP->getNumOperands(); i != e; ++i) {
144 ConstantInt *Idx = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: i));
145 if (!Idx)
146 return nullptr; // Variable index.
147
148 uint64_t IdxVal = Idx->getZExtValue();
149 if ((unsigned)IdxVal != IdxVal)
150 return nullptr; // Too large array index.
151
152 if (StructType *STy = dyn_cast<StructType>(Val: EltTy))
153 EltTy = STy->getElementType(N: IdxVal);
154 else if (ArrayType *ATy = dyn_cast<ArrayType>(Val: EltTy)) {
155 if (IdxVal >= ATy->getNumElements())
156 return nullptr;
157 EltTy = ATy->getElementType();
158 } else {
159 return nullptr; // Unknown type.
160 }
161
162 LaterIndices.push_back(Elt: IdxVal);
163 }
164
165 enum { Overdefined = -3, Undefined = -2 };
166
167 // Variables for our state machines.
168
169 // FirstTrueElement/SecondTrueElement - Used to emit a comparison of the form
170 // "i == 47 | i == 87", where 47 is the first index the condition is true for,
171 // and 87 is the second (and last) index. FirstTrueElement is -2 when
172 // undefined, otherwise set to the first true element. SecondTrueElement is
173 // -2 when undefined, -3 when overdefined and >= 0 when that index is true.
174 int FirstTrueElement = Undefined, SecondTrueElement = Undefined;
175
176 // FirstFalseElement/SecondFalseElement - Used to emit a comparison of the
177 // form "i != 47 & i != 87". Same state transitions as for true elements.
178 int FirstFalseElement = Undefined, SecondFalseElement = Undefined;
179
180 /// TrueRangeEnd/FalseRangeEnd - In conjunction with First*Element, these
181 /// define a state machine that triggers for ranges of values that the index
182 /// is true or false for. This triggers on things like "abbbbc"[i] == 'b'.
183 /// This is -2 when undefined, -3 when overdefined, and otherwise the last
184 /// index in the range (inclusive). We use -2 for undefined here because we
185 /// use relative comparisons and don't want 0-1 to match -1.
186 int TrueRangeEnd = Undefined, FalseRangeEnd = Undefined;
187
188 // MagicBitvector - This is a magic bitvector where we set a bit if the
189 // comparison is true for element 'i'. If there are 64 elements or less in
190 // the array, this will fully represent all the comparison results.
191 uint64_t MagicBitvector = 0;
192
193 // Scan the array and see if one of our patterns matches.
194 Constant *CompareRHS = cast<Constant>(Val: ICI.getOperand(i_nocapture: 1));
195 for (unsigned i = 0, e = ArrayElementCount; i != e; ++i) {
196 Constant *Elt = Init->getAggregateElement(Elt: i);
197 if (!Elt)
198 return nullptr;
199
200 // If this is indexing an array of structures, get the structure element.
201 if (!LaterIndices.empty()) {
202 Elt = ConstantFoldExtractValueInstruction(Agg: Elt, Idxs: LaterIndices);
203 if (!Elt)
204 return nullptr;
205 }
206
207 // If the element is masked, handle it.
208 if (AndCst) {
209 Elt = ConstantFoldBinaryOpOperands(Opcode: Instruction::And, LHS: Elt, RHS: AndCst, DL);
210 if (!Elt)
211 return nullptr;
212 }
213
214 // Find out if the comparison would be true or false for the i'th element.
215 Constant *C = ConstantFoldCompareInstOperands(Predicate: ICI.getPredicate(), LHS: Elt,
216 RHS: CompareRHS, DL, TLI: &TLI);
217 if (!C)
218 return nullptr;
219
220 // If the result is undef for this element, ignore it.
221 if (isa<UndefValue>(Val: C)) {
222 // Extend range state machines to cover this element in case there is an
223 // undef in the middle of the range.
224 if (TrueRangeEnd == (int)i - 1)
225 TrueRangeEnd = i;
226 if (FalseRangeEnd == (int)i - 1)
227 FalseRangeEnd = i;
228 continue;
229 }
230
231 // If we can't compute the result for any of the elements, we have to give
232 // up evaluating the entire conditional.
233 if (!isa<ConstantInt>(Val: C))
234 return nullptr;
235
236 // Otherwise, we know if the comparison is true or false for this element,
237 // update our state machines.
238 bool IsTrueForElt = !cast<ConstantInt>(Val: C)->isZero();
239
240 // State machine for single/double/range index comparison.
241 if (IsTrueForElt) {
242 // Update the TrueElement state machine.
243 if (FirstTrueElement == Undefined)
244 FirstTrueElement = TrueRangeEnd = i; // First true element.
245 else {
246 // Update double-compare state machine.
247 if (SecondTrueElement == Undefined)
248 SecondTrueElement = i;
249 else
250 SecondTrueElement = Overdefined;
251
252 // Update range state machine.
253 if (TrueRangeEnd == (int)i - 1)
254 TrueRangeEnd = i;
255 else
256 TrueRangeEnd = Overdefined;
257 }
258 } else {
259 // Update the FalseElement state machine.
260 if (FirstFalseElement == Undefined)
261 FirstFalseElement = FalseRangeEnd = i; // First false element.
262 else {
263 // Update double-compare state machine.
264 if (SecondFalseElement == Undefined)
265 SecondFalseElement = i;
266 else
267 SecondFalseElement = Overdefined;
268
269 // Update range state machine.
270 if (FalseRangeEnd == (int)i - 1)
271 FalseRangeEnd = i;
272 else
273 FalseRangeEnd = Overdefined;
274 }
275 }
276
277 // If this element is in range, update our magic bitvector.
278 if (i < 64 && IsTrueForElt)
279 MagicBitvector |= 1ULL << i;
280
281 // If all of our states become overdefined, bail out early. Since the
282 // predicate is expensive, only check it every 8 elements. This is only
283 // really useful for really huge arrays.
284 if ((i & 8) == 0 && i >= 64 && SecondTrueElement == Overdefined &&
285 SecondFalseElement == Overdefined && TrueRangeEnd == Overdefined &&
286 FalseRangeEnd == Overdefined)
287 return nullptr;
288 }
289
290 // Now that we've scanned the entire array, emit our new comparison(s). We
291 // order the state machines in complexity of the generated code.
292 Value *Idx = GEP->getOperand(i_nocapture: 2);
293
294 // If the index is larger than the pointer offset size of the target, truncate
295 // the index down like the GEP would do implicitly. We don't have to do this
296 // for an inbounds GEP because the index can't be out of range.
297 if (!GEP->isInBounds()) {
298 Type *PtrIdxTy = DL.getIndexType(PtrTy: GEP->getType());
299 unsigned OffsetSize = PtrIdxTy->getIntegerBitWidth();
300 if (Idx->getType()->getPrimitiveSizeInBits().getFixedValue() > OffsetSize)
301 Idx = Builder.CreateTrunc(V: Idx, DestTy: PtrIdxTy);
302 }
303
304 // If inbounds keyword is not present, Idx * ElementSize can overflow.
305 // Let's assume that ElementSize is 2 and the wanted value is at offset 0.
306 // Then, there are two possible values for Idx to match offset 0:
307 // 0x00..00, 0x80..00.
308 // Emitting 'icmp eq Idx, 0' isn't correct in this case because the
309 // comparison is false if Idx was 0x80..00.
310 // We need to erase the highest countTrailingZeros(ElementSize) bits of Idx.
311 unsigned ElementSize =
312 DL.getTypeAllocSize(Ty: Init->getType()->getArrayElementType());
313 auto MaskIdx = [&](Value *Idx) {
314 if (!GEP->isInBounds() && llvm::countr_zero(Val: ElementSize) != 0) {
315 Value *Mask = ConstantInt::get(Ty: Idx->getType(), V: -1);
316 Mask = Builder.CreateLShr(LHS: Mask, RHS: llvm::countr_zero(Val: ElementSize));
317 Idx = Builder.CreateAnd(LHS: Idx, RHS: Mask);
318 }
319 return Idx;
320 };
321
322 // If the comparison is only true for one or two elements, emit direct
323 // comparisons.
324 if (SecondTrueElement != Overdefined) {
325 Idx = MaskIdx(Idx);
326 // None true -> false.
327 if (FirstTrueElement == Undefined)
328 return replaceInstUsesWith(I&: ICI, V: Builder.getFalse());
329
330 Value *FirstTrueIdx = ConstantInt::get(Ty: Idx->getType(), V: FirstTrueElement);
331
332 // True for one element -> 'i == 47'.
333 if (SecondTrueElement == Undefined)
334 return new ICmpInst(ICmpInst::ICMP_EQ, Idx, FirstTrueIdx);
335
336 // True for two elements -> 'i == 47 | i == 72'.
337 Value *C1 = Builder.CreateICmpEQ(LHS: Idx, RHS: FirstTrueIdx);
338 Value *SecondTrueIdx = ConstantInt::get(Ty: Idx->getType(), V: SecondTrueElement);
339 Value *C2 = Builder.CreateICmpEQ(LHS: Idx, RHS: SecondTrueIdx);
340 return BinaryOperator::CreateOr(V1: C1, V2: C2);
341 }
342
343 // If the comparison is only false for one or two elements, emit direct
344 // comparisons.
345 if (SecondFalseElement != Overdefined) {
346 Idx = MaskIdx(Idx);
347 // None false -> true.
348 if (FirstFalseElement == Undefined)
349 return replaceInstUsesWith(I&: ICI, V: Builder.getTrue());
350
351 Value *FirstFalseIdx = ConstantInt::get(Ty: Idx->getType(), V: FirstFalseElement);
352
353 // False for one element -> 'i != 47'.
354 if (SecondFalseElement == Undefined)
355 return new ICmpInst(ICmpInst::ICMP_NE, Idx, FirstFalseIdx);
356
357 // False for two elements -> 'i != 47 & i != 72'.
358 Value *C1 = Builder.CreateICmpNE(LHS: Idx, RHS: FirstFalseIdx);
359 Value *SecondFalseIdx =
360 ConstantInt::get(Ty: Idx->getType(), V: SecondFalseElement);
361 Value *C2 = Builder.CreateICmpNE(LHS: Idx, RHS: SecondFalseIdx);
362 return BinaryOperator::CreateAnd(V1: C1, V2: C2);
363 }
364
365 // If the comparison can be replaced with a range comparison for the elements
366 // where it is true, emit the range check.
367 if (TrueRangeEnd != Overdefined) {
368 assert(TrueRangeEnd != FirstTrueElement && "Should emit single compare");
369 Idx = MaskIdx(Idx);
370
371 // Generate (i-FirstTrue) <u (TrueRangeEnd-FirstTrue+1).
372 if (FirstTrueElement) {
373 Value *Offs = ConstantInt::get(Ty: Idx->getType(), V: -FirstTrueElement);
374 Idx = Builder.CreateAdd(LHS: Idx, RHS: Offs);
375 }
376
377 Value *End =
378 ConstantInt::get(Ty: Idx->getType(), V: TrueRangeEnd - FirstTrueElement + 1);
379 return new ICmpInst(ICmpInst::ICMP_ULT, Idx, End);
380 }
381
382 // False range check.
383 if (FalseRangeEnd != Overdefined) {
384 assert(FalseRangeEnd != FirstFalseElement && "Should emit single compare");
385 Idx = MaskIdx(Idx);
386 // Generate (i-FirstFalse) >u (FalseRangeEnd-FirstFalse).
387 if (FirstFalseElement) {
388 Value *Offs = ConstantInt::get(Ty: Idx->getType(), V: -FirstFalseElement);
389 Idx = Builder.CreateAdd(LHS: Idx, RHS: Offs);
390 }
391
392 Value *End =
393 ConstantInt::get(Ty: Idx->getType(), V: FalseRangeEnd - FirstFalseElement);
394 return new ICmpInst(ICmpInst::ICMP_UGT, Idx, End);
395 }
396
397 // If a magic bitvector captures the entire comparison state
398 // of this load, replace it with computation that does:
399 // ((magic_cst >> i) & 1) != 0
400 {
401 Type *Ty = nullptr;
402
403 // Look for an appropriate type:
404 // - The type of Idx if the magic fits
405 // - The smallest fitting legal type
406 if (ArrayElementCount <= Idx->getType()->getIntegerBitWidth())
407 Ty = Idx->getType();
408 else
409 Ty = DL.getSmallestLegalIntType(C&: Init->getContext(), Width: ArrayElementCount);
410
411 if (Ty) {
412 Idx = MaskIdx(Idx);
413 Value *V = Builder.CreateIntCast(V: Idx, DestTy: Ty, isSigned: false);
414 V = Builder.CreateLShr(LHS: ConstantInt::get(Ty, V: MagicBitvector), RHS: V);
415 V = Builder.CreateAnd(LHS: ConstantInt::get(Ty, V: 1), RHS: V);
416 return new ICmpInst(ICmpInst::ICMP_NE, V, ConstantInt::get(Ty, V: 0));
417 }
418 }
419
420 return nullptr;
421}
422
423/// Returns true if we can rewrite Start as a GEP with pointer Base
424/// and some integer offset. The nodes that need to be re-written
425/// for this transformation will be added to Explored.
426static bool canRewriteGEPAsOffset(Value *Start, Value *Base,
427 const DataLayout &DL,
428 SetVector<Value *> &Explored) {
429 SmallVector<Value *, 16> WorkList(1, Start);
430 Explored.insert(X: Base);
431
432 // The following traversal gives us an order which can be used
433 // when doing the final transformation. Since in the final
434 // transformation we create the PHI replacement instructions first,
435 // we don't have to get them in any particular order.
436 //
437 // However, for other instructions we will have to traverse the
438 // operands of an instruction first, which means that we have to
439 // do a post-order traversal.
440 while (!WorkList.empty()) {
441 SetVector<PHINode *> PHIs;
442
443 while (!WorkList.empty()) {
444 if (Explored.size() >= 100)
445 return false;
446
447 Value *V = WorkList.back();
448
449 if (Explored.contains(key: V)) {
450 WorkList.pop_back();
451 continue;
452 }
453
454 if (!isa<GetElementPtrInst>(Val: V) && !isa<PHINode>(Val: V))
455 // We've found some value that we can't explore which is different from
456 // the base. Therefore we can't do this transformation.
457 return false;
458
459 if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
460 // Only allow inbounds GEPs with at most one variable offset.
461 auto IsNonConst = [](Value *V) { return !isa<ConstantInt>(Val: V); };
462 if (!GEP->isInBounds() || count_if(Range: GEP->indices(), P: IsNonConst) > 1)
463 return false;
464
465 if (!Explored.contains(key: GEP->getOperand(i_nocapture: 0)))
466 WorkList.push_back(Elt: GEP->getOperand(i_nocapture: 0));
467 }
468
469 if (WorkList.back() == V) {
470 WorkList.pop_back();
471 // We've finished visiting this node, mark it as such.
472 Explored.insert(X: V);
473 }
474
475 if (auto *PN = dyn_cast<PHINode>(Val: V)) {
476 // We cannot transform PHIs on unsplittable basic blocks.
477 if (isa<CatchSwitchInst>(Val: PN->getParent()->getTerminator()))
478 return false;
479 Explored.insert(X: PN);
480 PHIs.insert(X: PN);
481 }
482 }
483
484 // Explore the PHI nodes further.
485 for (auto *PN : PHIs)
486 for (Value *Op : PN->incoming_values())
487 if (!Explored.contains(key: Op))
488 WorkList.push_back(Elt: Op);
489 }
490
491 // Make sure that we can do this. Since we can't insert GEPs in a basic
492 // block before a PHI node, we can't easily do this transformation if
493 // we have PHI node users of transformed instructions.
494 for (Value *Val : Explored) {
495 for (Value *Use : Val->uses()) {
496
497 auto *PHI = dyn_cast<PHINode>(Val: Use);
498 auto *Inst = dyn_cast<Instruction>(Val);
499
500 if (Inst == Base || Inst == PHI || !Inst || !PHI ||
501 !Explored.contains(key: PHI))
502 continue;
503
504 if (PHI->getParent() == Inst->getParent())
505 return false;
506 }
507 }
508 return true;
509}
510
511// Sets the appropriate insert point on Builder where we can add
512// a replacement Instruction for V (if that is possible).
513static void setInsertionPoint(IRBuilder<> &Builder, Value *V,
514 bool Before = true) {
515 if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
516 BasicBlock *Parent = PHI->getParent();
517 Builder.SetInsertPoint(TheBB: Parent, IP: Parent->getFirstInsertionPt());
518 return;
519 }
520 if (auto *I = dyn_cast<Instruction>(Val: V)) {
521 if (!Before)
522 I = &*std::next(x: I->getIterator());
523 Builder.SetInsertPoint(I);
524 return;
525 }
526 if (auto *A = dyn_cast<Argument>(Val: V)) {
527 // Set the insertion point in the entry block.
528 BasicBlock &Entry = A->getParent()->getEntryBlock();
529 Builder.SetInsertPoint(TheBB: &Entry, IP: Entry.getFirstInsertionPt());
530 return;
531 }
532 // Otherwise, this is a constant and we don't need to set a new
533 // insertion point.
534 assert(isa<Constant>(V) && "Setting insertion point for unknown value!");
535}
536
537/// Returns a re-written value of Start as an indexed GEP using Base as a
538/// pointer.
539static Value *rewriteGEPAsOffset(Value *Start, Value *Base,
540 const DataLayout &DL,
541 SetVector<Value *> &Explored,
542 InstCombiner &IC) {
543 // Perform all the substitutions. This is a bit tricky because we can
544 // have cycles in our use-def chains.
545 // 1. Create the PHI nodes without any incoming values.
546 // 2. Create all the other values.
547 // 3. Add the edges for the PHI nodes.
548 // 4. Emit GEPs to get the original pointers.
549 // 5. Remove the original instructions.
550 Type *IndexType = IntegerType::get(
551 C&: Base->getContext(), NumBits: DL.getIndexTypeSizeInBits(Ty: Start->getType()));
552
553 DenseMap<Value *, Value *> NewInsts;
554 NewInsts[Base] = ConstantInt::getNullValue(Ty: IndexType);
555
556 // Create the new PHI nodes, without adding any incoming values.
557 for (Value *Val : Explored) {
558 if (Val == Base)
559 continue;
560 // Create empty phi nodes. This avoids cyclic dependencies when creating
561 // the remaining instructions.
562 if (auto *PHI = dyn_cast<PHINode>(Val))
563 NewInsts[PHI] =
564 PHINode::Create(Ty: IndexType, NumReservedValues: PHI->getNumIncomingValues(),
565 NameStr: PHI->getName() + ".idx", InsertBefore: PHI->getIterator());
566 }
567 IRBuilder<> Builder(Base->getContext());
568
569 // Create all the other instructions.
570 for (Value *Val : Explored) {
571 if (NewInsts.contains(Val))
572 continue;
573
574 if (auto *GEP = dyn_cast<GEPOperator>(Val)) {
575 setInsertionPoint(Builder, V: GEP);
576 Value *Op = NewInsts[GEP->getOperand(i_nocapture: 0)];
577 Value *OffsetV = emitGEPOffset(Builder: &Builder, DL, GEP);
578 if (isa<ConstantInt>(Val: Op) && cast<ConstantInt>(Val: Op)->isZero())
579 NewInsts[GEP] = OffsetV;
580 else
581 NewInsts[GEP] = Builder.CreateNSWAdd(
582 LHS: Op, RHS: OffsetV, Name: GEP->getOperand(i_nocapture: 0)->getName() + ".add");
583 continue;
584 }
585 if (isa<PHINode>(Val))
586 continue;
587
588 llvm_unreachable("Unexpected instruction type");
589 }
590
591 // Add the incoming values to the PHI nodes.
592 for (Value *Val : Explored) {
593 if (Val == Base)
594 continue;
595 // All the instructions have been created, we can now add edges to the
596 // phi nodes.
597 if (auto *PHI = dyn_cast<PHINode>(Val)) {
598 PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]);
599 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
600 Value *NewIncoming = PHI->getIncomingValue(i: I);
601
602 if (NewInsts.contains(Val: NewIncoming))
603 NewIncoming = NewInsts[NewIncoming];
604
605 NewPhi->addIncoming(V: NewIncoming, BB: PHI->getIncomingBlock(i: I));
606 }
607 }
608 }
609
610 for (Value *Val : Explored) {
611 if (Val == Base)
612 continue;
613
614 setInsertionPoint(Builder, V: Val, Before: false);
615 // Create GEP for external users.
616 Value *NewVal = Builder.CreateInBoundsGEP(
617 Ty: Builder.getInt8Ty(), Ptr: Base, IdxList: NewInsts[Val], Name: Val->getName() + ".ptr");
618 IC.replaceInstUsesWith(I&: *cast<Instruction>(Val), V: NewVal);
619 // Add old instruction to worklist for DCE. We don't directly remove it
620 // here because the original compare is one of the users.
621 IC.addToWorklist(I: cast<Instruction>(Val));
622 }
623
624 return NewInsts[Start];
625}
626
627/// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant.
628/// We can look through PHIs, GEPs and casts in order to determine a common base
629/// between GEPLHS and RHS.
630static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS,
631 ICmpInst::Predicate Cond,
632 const DataLayout &DL,
633 InstCombiner &IC) {
634 // FIXME: Support vector of pointers.
635 if (GEPLHS->getType()->isVectorTy())
636 return nullptr;
637
638 if (!GEPLHS->hasAllConstantIndices())
639 return nullptr;
640
641 APInt Offset(DL.getIndexTypeSizeInBits(Ty: GEPLHS->getType()), 0);
642 Value *PtrBase =
643 GEPLHS->stripAndAccumulateConstantOffsets(DL, Offset,
644 /*AllowNonInbounds*/ false);
645
646 // Bail if we looked through addrspacecast.
647 if (PtrBase->getType() != GEPLHS->getType())
648 return nullptr;
649
650 // The set of nodes that will take part in this transformation.
651 SetVector<Value *> Nodes;
652
653 if (!canRewriteGEPAsOffset(Start: RHS, Base: PtrBase, DL, Explored&: Nodes))
654 return nullptr;
655
656 // We know we can re-write this as
657 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2)
658 // Since we've only looked through inbouds GEPs we know that we
659 // can't have overflow on either side. We can therefore re-write
660 // this as:
661 // OFFSET1 cmp OFFSET2
662 Value *NewRHS = rewriteGEPAsOffset(Start: RHS, Base: PtrBase, DL, Explored&: Nodes, IC);
663
664 // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written
665 // GEP having PtrBase as the pointer base, and has returned in NewRHS the
666 // offset. Since Index is the offset of LHS to the base pointer, we will now
667 // compare the offsets instead of comparing the pointers.
668 return new ICmpInst(ICmpInst::getSignedPredicate(pred: Cond),
669 IC.Builder.getInt(AI: Offset), NewRHS);
670}
671
672/// Fold comparisons between a GEP instruction and something else. At this point
673/// we know that the GEP is on the LHS of the comparison.
674Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
675 ICmpInst::Predicate Cond,
676 Instruction &I) {
677 // Don't transform signed compares of GEPs into index compares. Even if the
678 // GEP is inbounds, the final add of the base pointer can have signed overflow
679 // and would change the result of the icmp.
680 // e.g. "&foo[0] <s &foo[1]" can't be folded to "true" because "foo" could be
681 // the maximum signed value for the pointer type.
682 if (ICmpInst::isSigned(predicate: Cond))
683 return nullptr;
684
685 // Look through bitcasts and addrspacecasts. We do not however want to remove
686 // 0 GEPs.
687 if (!isa<GetElementPtrInst>(Val: RHS))
688 RHS = RHS->stripPointerCasts();
689
690 Value *PtrBase = GEPLHS->getOperand(i_nocapture: 0);
691 if (PtrBase == RHS && (GEPLHS->isInBounds() || ICmpInst::isEquality(P: Cond))) {
692 // ((gep Ptr, OFFSET) cmp Ptr) ---> (OFFSET cmp 0).
693 Value *Offset = EmitGEPOffset(GEP: GEPLHS);
694 return new ICmpInst(ICmpInst::getSignedPredicate(pred: Cond), Offset,
695 Constant::getNullValue(Ty: Offset->getType()));
696 }
697
698 if (GEPLHS->isInBounds() && ICmpInst::isEquality(P: Cond) &&
699 isa<Constant>(Val: RHS) && cast<Constant>(Val: RHS)->isNullValue() &&
700 !NullPointerIsDefined(F: I.getFunction(),
701 AS: RHS->getType()->getPointerAddressSpace())) {
702 // For most address spaces, an allocation can't be placed at null, but null
703 // itself is treated as a 0 size allocation in the in bounds rules. Thus,
704 // the only valid inbounds address derived from null, is null itself.
705 // Thus, we have four cases to consider:
706 // 1) Base == nullptr, Offset == 0 -> inbounds, null
707 // 2) Base == nullptr, Offset != 0 -> poison as the result is out of bounds
708 // 3) Base != nullptr, Offset == (-base) -> poison (crossing allocations)
709 // 4) Base != nullptr, Offset != (-base) -> nonnull (and possibly poison)
710 //
711 // (Note if we're indexing a type of size 0, that simply collapses into one
712 // of the buckets above.)
713 //
714 // In general, we're allowed to make values less poison (i.e. remove
715 // sources of full UB), so in this case, we just select between the two
716 // non-poison cases (1 and 4 above).
717 //
718 // For vectors, we apply the same reasoning on a per-lane basis.
719 auto *Base = GEPLHS->getPointerOperand();
720 if (GEPLHS->getType()->isVectorTy() && Base->getType()->isPointerTy()) {
721 auto EC = cast<VectorType>(Val: GEPLHS->getType())->getElementCount();
722 Base = Builder.CreateVectorSplat(EC, V: Base);
723 }
724 return new ICmpInst(Cond, Base,
725 ConstantExpr::getPointerBitCastOrAddrSpaceCast(
726 C: cast<Constant>(Val: RHS), Ty: Base->getType()));
727 } else if (GEPOperator *GEPRHS = dyn_cast<GEPOperator>(Val: RHS)) {
728 // If the base pointers are different, but the indices are the same, just
729 // compare the base pointer.
730 if (PtrBase != GEPRHS->getOperand(i_nocapture: 0)) {
731 bool IndicesTheSame =
732 GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
733 GEPLHS->getPointerOperand()->getType() ==
734 GEPRHS->getPointerOperand()->getType() &&
735 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType();
736 if (IndicesTheSame)
737 for (unsigned i = 1, e = GEPLHS->getNumOperands(); i != e; ++i)
738 if (GEPLHS->getOperand(i_nocapture: i) != GEPRHS->getOperand(i_nocapture: i)) {
739 IndicesTheSame = false;
740 break;
741 }
742
743 // If all indices are the same, just compare the base pointers.
744 Type *BaseType = GEPLHS->getOperand(i_nocapture: 0)->getType();
745 if (IndicesTheSame && CmpInst::makeCmpResultType(opnd_type: BaseType) == I.getType())
746 return new ICmpInst(Cond, GEPLHS->getOperand(i_nocapture: 0), GEPRHS->getOperand(i_nocapture: 0));
747
748 // If we're comparing GEPs with two base pointers that only differ in type
749 // and both GEPs have only constant indices or just one use, then fold
750 // the compare with the adjusted indices.
751 // FIXME: Support vector of pointers.
752 if (GEPLHS->isInBounds() && GEPRHS->isInBounds() &&
753 (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) &&
754 (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse()) &&
755 PtrBase->stripPointerCasts() ==
756 GEPRHS->getOperand(i_nocapture: 0)->stripPointerCasts() &&
757 !GEPLHS->getType()->isVectorTy()) {
758 Value *LOffset = EmitGEPOffset(GEP: GEPLHS);
759 Value *ROffset = EmitGEPOffset(GEP: GEPRHS);
760
761 // If we looked through an addrspacecast between different sized address
762 // spaces, the LHS and RHS pointers are different sized
763 // integers. Truncate to the smaller one.
764 Type *LHSIndexTy = LOffset->getType();
765 Type *RHSIndexTy = ROffset->getType();
766 if (LHSIndexTy != RHSIndexTy) {
767 if (LHSIndexTy->getPrimitiveSizeInBits().getFixedValue() <
768 RHSIndexTy->getPrimitiveSizeInBits().getFixedValue()) {
769 ROffset = Builder.CreateTrunc(V: ROffset, DestTy: LHSIndexTy);
770 } else
771 LOffset = Builder.CreateTrunc(V: LOffset, DestTy: RHSIndexTy);
772 }
773
774 Value *Cmp = Builder.CreateICmp(P: ICmpInst::getSignedPredicate(pred: Cond),
775 LHS: LOffset, RHS: ROffset);
776 return replaceInstUsesWith(I, V: Cmp);
777 }
778
779 // Otherwise, the base pointers are different and the indices are
780 // different. Try convert this to an indexed compare by looking through
781 // PHIs/casts.
782 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, IC&: *this);
783 }
784
785 bool GEPsInBounds = GEPLHS->isInBounds() && GEPRHS->isInBounds();
786 if (GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
787 GEPLHS->getSourceElementType() == GEPRHS->getSourceElementType()) {
788 // If the GEPs only differ by one index, compare it.
789 unsigned NumDifferences = 0; // Keep track of # differences.
790 unsigned DiffOperand = 0; // The operand that differs.
791 for (unsigned i = 1, e = GEPRHS->getNumOperands(); i != e; ++i)
792 if (GEPLHS->getOperand(i_nocapture: i) != GEPRHS->getOperand(i_nocapture: i)) {
793 Type *LHSType = GEPLHS->getOperand(i_nocapture: i)->getType();
794 Type *RHSType = GEPRHS->getOperand(i_nocapture: i)->getType();
795 // FIXME: Better support for vector of pointers.
796 if (LHSType->getPrimitiveSizeInBits() !=
797 RHSType->getPrimitiveSizeInBits() ||
798 (GEPLHS->getType()->isVectorTy() &&
799 (!LHSType->isVectorTy() || !RHSType->isVectorTy()))) {
800 // Irreconcilable differences.
801 NumDifferences = 2;
802 break;
803 }
804
805 if (NumDifferences++) break;
806 DiffOperand = i;
807 }
808
809 if (NumDifferences == 0) // SAME GEP?
810 return replaceInstUsesWith(I, // No comparison is needed here.
811 V: ConstantInt::get(Ty: I.getType(), V: ICmpInst::isTrueWhenEqual(predicate: Cond)));
812
813 else if (NumDifferences == 1 && GEPsInBounds) {
814 Value *LHSV = GEPLHS->getOperand(i_nocapture: DiffOperand);
815 Value *RHSV = GEPRHS->getOperand(i_nocapture: DiffOperand);
816 // Make sure we do a signed comparison here.
817 return new ICmpInst(ICmpInst::getSignedPredicate(pred: Cond), LHSV, RHSV);
818 }
819 }
820
821 if (GEPsInBounds || CmpInst::isEquality(pred: Cond)) {
822 // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2)
823 Value *L = EmitGEPOffset(GEP: GEPLHS, /*RewriteGEP=*/true);
824 Value *R = EmitGEPOffset(GEP: GEPRHS, /*RewriteGEP=*/true);
825 return new ICmpInst(ICmpInst::getSignedPredicate(pred: Cond), L, R);
826 }
827 }
828
829 // Try convert this to an indexed compare by looking through PHIs/casts as a
830 // last resort.
831 return transformToIndexedCompare(GEPLHS, RHS, Cond, DL, IC&: *this);
832}
833
834bool InstCombinerImpl::foldAllocaCmp(AllocaInst *Alloca) {
835 // It would be tempting to fold away comparisons between allocas and any
836 // pointer not based on that alloca (e.g. an argument). However, even
837 // though such pointers cannot alias, they can still compare equal.
838 //
839 // But LLVM doesn't specify where allocas get their memory, so if the alloca
840 // doesn't escape we can argue that it's impossible to guess its value, and we
841 // can therefore act as if any such guesses are wrong.
842 //
843 // However, we need to ensure that this folding is consistent: We can't fold
844 // one comparison to false, and then leave a different comparison against the
845 // same value alone (as it might evaluate to true at runtime, leading to a
846 // contradiction). As such, this code ensures that all comparisons are folded
847 // at the same time, and there are no other escapes.
848
849 struct CmpCaptureTracker : public CaptureTracker {
850 AllocaInst *Alloca;
851 bool Captured = false;
852 /// The value of the map is a bit mask of which icmp operands the alloca is
853 /// used in.
854 SmallMapVector<ICmpInst *, unsigned, 4> ICmps;
855
856 CmpCaptureTracker(AllocaInst *Alloca) : Alloca(Alloca) {}
857
858 void tooManyUses() override { Captured = true; }
859
860 bool captured(const Use *U) override {
861 auto *ICmp = dyn_cast<ICmpInst>(Val: U->getUser());
862 // We need to check that U is based *only* on the alloca, and doesn't
863 // have other contributions from a select/phi operand.
864 // TODO: We could check whether getUnderlyingObjects() reduces to one
865 // object, which would allow looking through phi nodes.
866 if (ICmp && ICmp->isEquality() && getUnderlyingObject(V: *U) == Alloca) {
867 // Collect equality icmps of the alloca, and don't treat them as
868 // captures.
869 auto Res = ICmps.insert(KV: {ICmp, 0});
870 Res.first->second |= 1u << U->getOperandNo();
871 return false;
872 }
873
874 Captured = true;
875 return true;
876 }
877 };
878
879 CmpCaptureTracker Tracker(Alloca);
880 PointerMayBeCaptured(V: Alloca, Tracker: &Tracker);
881 if (Tracker.Captured)
882 return false;
883
884 bool Changed = false;
885 for (auto [ICmp, Operands] : Tracker.ICmps) {
886 switch (Operands) {
887 case 1:
888 case 2: {
889 // The alloca is only used in one icmp operand. Assume that the
890 // equality is false.
891 auto *Res = ConstantInt::get(
892 Ty: ICmp->getType(), V: ICmp->getPredicate() == ICmpInst::ICMP_NE);
893 replaceInstUsesWith(I&: *ICmp, V: Res);
894 eraseInstFromFunction(I&: *ICmp);
895 Changed = true;
896 break;
897 }
898 case 3:
899 // Both icmp operands are based on the alloca, so this is comparing
900 // pointer offsets, without leaking any information about the address
901 // of the alloca. Ignore such comparisons.
902 break;
903 default:
904 llvm_unreachable("Cannot happen");
905 }
906 }
907
908 return Changed;
909}
910
911/// Fold "icmp pred (X+C), X".
912Instruction *InstCombinerImpl::foldICmpAddOpConst(Value *X, const APInt &C,
913 ICmpInst::Predicate Pred) {
914 // From this point on, we know that (X+C <= X) --> (X+C < X) because C != 0,
915 // so the values can never be equal. Similarly for all other "or equals"
916 // operators.
917 assert(!!C && "C should not be zero!");
918
919 // (X+1) <u X --> X >u (MAXUINT-1) --> X == 255
920 // (X+2) <u X --> X >u (MAXUINT-2) --> X > 253
921 // (X+MAXUINT) <u X --> X >u (MAXUINT-MAXUINT) --> X != 0
922 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
923 Constant *R = ConstantInt::get(Ty: X->getType(),
924 V: APInt::getMaxValue(numBits: C.getBitWidth()) - C);
925 return new ICmpInst(ICmpInst::ICMP_UGT, X, R);
926 }
927
928 // (X+1) >u X --> X <u (0-1) --> X != 255
929 // (X+2) >u X --> X <u (0-2) --> X <u 254
930 // (X+MAXUINT) >u X --> X <u (0-MAXUINT) --> X <u 1 --> X == 0
931 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
932 return new ICmpInst(ICmpInst::ICMP_ULT, X,
933 ConstantInt::get(Ty: X->getType(), V: -C));
934
935 APInt SMax = APInt::getSignedMaxValue(numBits: C.getBitWidth());
936
937 // (X+ 1) <s X --> X >s (MAXSINT-1) --> X == 127
938 // (X+ 2) <s X --> X >s (MAXSINT-2) --> X >s 125
939 // (X+MAXSINT) <s X --> X >s (MAXSINT-MAXSINT) --> X >s 0
940 // (X+MINSINT) <s X --> X >s (MAXSINT-MINSINT) --> X >s -1
941 // (X+ -2) <s X --> X >s (MAXSINT- -2) --> X >s 126
942 // (X+ -1) <s X --> X >s (MAXSINT- -1) --> X != 127
943 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
944 return new ICmpInst(ICmpInst::ICMP_SGT, X,
945 ConstantInt::get(Ty: X->getType(), V: SMax - C));
946
947 // (X+ 1) >s X --> X <s (MAXSINT-(1-1)) --> X != 127
948 // (X+ 2) >s X --> X <s (MAXSINT-(2-1)) --> X <s 126
949 // (X+MAXSINT) >s X --> X <s (MAXSINT-(MAXSINT-1)) --> X <s 1
950 // (X+MINSINT) >s X --> X <s (MAXSINT-(MINSINT-1)) --> X <s -2
951 // (X+ -2) >s X --> X <s (MAXSINT-(-2-1)) --> X <s -126
952 // (X+ -1) >s X --> X <s (MAXSINT-(-1-1)) --> X == -128
953
954 assert(Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE);
955 return new ICmpInst(ICmpInst::ICMP_SLT, X,
956 ConstantInt::get(Ty: X->getType(), V: SMax - (C - 1)));
957}
958
959/// Handle "(icmp eq/ne (ashr/lshr AP2, A), AP1)" ->
960/// (icmp eq/ne A, Log2(AP2/AP1)) ->
961/// (icmp eq/ne A, Log2(AP2) - Log2(AP1)).
962Instruction *InstCombinerImpl::foldICmpShrConstConst(ICmpInst &I, Value *A,
963 const APInt &AP1,
964 const APInt &AP2) {
965 assert(I.isEquality() && "Cannot fold icmp gt/lt");
966
967 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
968 if (I.getPredicate() == I.ICMP_NE)
969 Pred = CmpInst::getInversePredicate(pred: Pred);
970 return new ICmpInst(Pred, LHS, RHS);
971 };
972
973 // Don't bother doing any work for cases which InstSimplify handles.
974 if (AP2.isZero())
975 return nullptr;
976
977 bool IsAShr = isa<AShrOperator>(Val: I.getOperand(i_nocapture: 0));
978 if (IsAShr) {
979 if (AP2.isAllOnes())
980 return nullptr;
981 if (AP2.isNegative() != AP1.isNegative())
982 return nullptr;
983 if (AP2.sgt(RHS: AP1))
984 return nullptr;
985 }
986
987 if (!AP1)
988 // 'A' must be large enough to shift out the highest set bit.
989 return getICmp(I.ICMP_UGT, A,
990 ConstantInt::get(Ty: A->getType(), V: AP2.logBase2()));
991
992 if (AP1 == AP2)
993 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(Ty: A->getType()));
994
995 int Shift;
996 if (IsAShr && AP1.isNegative())
997 Shift = AP1.countl_one() - AP2.countl_one();
998 else
999 Shift = AP1.countl_zero() - AP2.countl_zero();
1000
1001 if (Shift > 0) {
1002 if (IsAShr && AP1 == AP2.ashr(ShiftAmt: Shift)) {
1003 // There are multiple solutions if we are comparing against -1 and the LHS
1004 // of the ashr is not a power of two.
1005 if (AP1.isAllOnes() && !AP2.isPowerOf2())
1006 return getICmp(I.ICMP_UGE, A, ConstantInt::get(Ty: A->getType(), V: Shift));
1007 return getICmp(I.ICMP_EQ, A, ConstantInt::get(Ty: A->getType(), V: Shift));
1008 } else if (AP1 == AP2.lshr(shiftAmt: Shift)) {
1009 return getICmp(I.ICMP_EQ, A, ConstantInt::get(Ty: A->getType(), V: Shift));
1010 }
1011 }
1012
1013 // Shifting const2 will never be equal to const1.
1014 // FIXME: This should always be handled by InstSimplify?
1015 auto *TorF = ConstantInt::get(Ty: I.getType(), V: I.getPredicate() == I.ICMP_NE);
1016 return replaceInstUsesWith(I, V: TorF);
1017}
1018
1019/// Handle "(icmp eq/ne (shl AP2, A), AP1)" ->
1020/// (icmp eq/ne A, TrailingZeros(AP1) - TrailingZeros(AP2)).
1021Instruction *InstCombinerImpl::foldICmpShlConstConst(ICmpInst &I, Value *A,
1022 const APInt &AP1,
1023 const APInt &AP2) {
1024 assert(I.isEquality() && "Cannot fold icmp gt/lt");
1025
1026 auto getICmp = [&I](CmpInst::Predicate Pred, Value *LHS, Value *RHS) {
1027 if (I.getPredicate() == I.ICMP_NE)
1028 Pred = CmpInst::getInversePredicate(pred: Pred);
1029 return new ICmpInst(Pred, LHS, RHS);
1030 };
1031
1032 // Don't bother doing any work for cases which InstSimplify handles.
1033 if (AP2.isZero())
1034 return nullptr;
1035
1036 unsigned AP2TrailingZeros = AP2.countr_zero();
1037
1038 if (!AP1 && AP2TrailingZeros != 0)
1039 return getICmp(
1040 I.ICMP_UGE, A,
1041 ConstantInt::get(Ty: A->getType(), V: AP2.getBitWidth() - AP2TrailingZeros));
1042
1043 if (AP1 == AP2)
1044 return getICmp(I.ICMP_EQ, A, ConstantInt::getNullValue(Ty: A->getType()));
1045
1046 // Get the distance between the lowest bits that are set.
1047 int Shift = AP1.countr_zero() - AP2TrailingZeros;
1048
1049 if (Shift > 0 && AP2.shl(shiftAmt: Shift) == AP1)
1050 return getICmp(I.ICMP_EQ, A, ConstantInt::get(Ty: A->getType(), V: Shift));
1051
1052 // Shifting const2 will never be equal to const1.
1053 // FIXME: This should always be handled by InstSimplify?
1054 auto *TorF = ConstantInt::get(Ty: I.getType(), V: I.getPredicate() == I.ICMP_NE);
1055 return replaceInstUsesWith(I, V: TorF);
1056}
1057
1058/// The caller has matched a pattern of the form:
1059/// I = icmp ugt (add (add A, B), CI2), CI1
1060/// If this is of the form:
1061/// sum = a + b
1062/// if (sum+128 >u 255)
1063/// Then replace it with llvm.sadd.with.overflow.i8.
1064///
1065static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
1066 ConstantInt *CI2, ConstantInt *CI1,
1067 InstCombinerImpl &IC) {
1068 // The transformation we're trying to do here is to transform this into an
1069 // llvm.sadd.with.overflow. To do this, we have to replace the original add
1070 // with a narrower add, and discard the add-with-constant that is part of the
1071 // range check (if we can't eliminate it, this isn't profitable).
1072
1073 // In order to eliminate the add-with-constant, the compare can be its only
1074 // use.
1075 Instruction *AddWithCst = cast<Instruction>(Val: I.getOperand(i_nocapture: 0));
1076 if (!AddWithCst->hasOneUse())
1077 return nullptr;
1078
1079 // If CI2 is 2^7, 2^15, 2^31, then it might be an sadd.with.overflow.
1080 if (!CI2->getValue().isPowerOf2())
1081 return nullptr;
1082 unsigned NewWidth = CI2->getValue().countr_zero();
1083 if (NewWidth != 7 && NewWidth != 15 && NewWidth != 31)
1084 return nullptr;
1085
1086 // The width of the new add formed is 1 more than the bias.
1087 ++NewWidth;
1088
1089 // Check to see that CI1 is an all-ones value with NewWidth bits.
1090 if (CI1->getBitWidth() == NewWidth ||
1091 CI1->getValue() != APInt::getLowBitsSet(numBits: CI1->getBitWidth(), loBitsSet: NewWidth))
1092 return nullptr;
1093
1094 // This is only really a signed overflow check if the inputs have been
1095 // sign-extended; check for that condition. For example, if CI2 is 2^31 and
1096 // the operands of the add are 64 bits wide, we need at least 33 sign bits.
1097 if (IC.ComputeMaxSignificantBits(Op: A, Depth: 0, CxtI: &I) > NewWidth ||
1098 IC.ComputeMaxSignificantBits(Op: B, Depth: 0, CxtI: &I) > NewWidth)
1099 return nullptr;
1100
1101 // In order to replace the original add with a narrower
1102 // llvm.sadd.with.overflow, the only uses allowed are the add-with-constant
1103 // and truncates that discard the high bits of the add. Verify that this is
1104 // the case.
1105 Instruction *OrigAdd = cast<Instruction>(Val: AddWithCst->getOperand(i: 0));
1106 for (User *U : OrigAdd->users()) {
1107 if (U == AddWithCst)
1108 continue;
1109
1110 // Only accept truncates for now. We would really like a nice recursive
1111 // predicate like SimplifyDemandedBits, but which goes downwards the use-def
1112 // chain to see which bits of a value are actually demanded. If the
1113 // original add had another add which was then immediately truncated, we
1114 // could still do the transformation.
1115 TruncInst *TI = dyn_cast<TruncInst>(Val: U);
1116 if (!TI || TI->getType()->getPrimitiveSizeInBits() > NewWidth)
1117 return nullptr;
1118 }
1119
1120 // If the pattern matches, truncate the inputs to the narrower type and
1121 // use the sadd_with_overflow intrinsic to efficiently compute both the
1122 // result and the overflow bit.
1123 Type *NewType = IntegerType::get(C&: OrigAdd->getContext(), NumBits: NewWidth);
1124 Function *F = Intrinsic::getDeclaration(
1125 M: I.getModule(), id: Intrinsic::sadd_with_overflow, Tys: NewType);
1126
1127 InstCombiner::BuilderTy &Builder = IC.Builder;
1128
1129 // Put the new code above the original add, in case there are any uses of the
1130 // add between the add and the compare.
1131 Builder.SetInsertPoint(OrigAdd);
1132
1133 Value *TruncA = Builder.CreateTrunc(V: A, DestTy: NewType, Name: A->getName() + ".trunc");
1134 Value *TruncB = Builder.CreateTrunc(V: B, DestTy: NewType, Name: B->getName() + ".trunc");
1135 CallInst *Call = Builder.CreateCall(Callee: F, Args: {TruncA, TruncB}, Name: "sadd");
1136 Value *Add = Builder.CreateExtractValue(Agg: Call, Idxs: 0, Name: "sadd.result");
1137 Value *ZExt = Builder.CreateZExt(V: Add, DestTy: OrigAdd->getType());
1138
1139 // The inner add was the result of the narrow add, zero extended to the
1140 // wider type. Replace it with the result computed by the intrinsic.
1141 IC.replaceInstUsesWith(I&: *OrigAdd, V: ZExt);
1142 IC.eraseInstFromFunction(I&: *OrigAdd);
1143
1144 // The original icmp gets replaced with the overflow value.
1145 return ExtractValueInst::Create(Agg: Call, Idxs: 1, NameStr: "sadd.overflow");
1146}
1147
1148/// If we have:
1149/// icmp eq/ne (urem/srem %x, %y), 0
1150/// iff %y is a power-of-two, we can replace this with a bit test:
1151/// icmp eq/ne (and %x, (add %y, -1)), 0
1152Instruction *InstCombinerImpl::foldIRemByPowerOfTwoToBitTest(ICmpInst &I) {
1153 // This fold is only valid for equality predicates.
1154 if (!I.isEquality())
1155 return nullptr;
1156 ICmpInst::Predicate Pred;
1157 Value *X, *Y, *Zero;
1158 if (!match(V: &I, P: m_ICmp(Pred, L: m_OneUse(SubPattern: m_IRem(L: m_Value(V&: X), R: m_Value(V&: Y))),
1159 R: m_CombineAnd(L: m_Zero(), R: m_Value(V&: Zero)))))
1160 return nullptr;
1161 if (!isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ true, Depth: 0, CxtI: &I))
1162 return nullptr;
1163 // This may increase instruction count, we don't enforce that Y is a constant.
1164 Value *Mask = Builder.CreateAdd(LHS: Y, RHS: Constant::getAllOnesValue(Ty: Y->getType()));
1165 Value *Masked = Builder.CreateAnd(LHS: X, RHS: Mask);
1166 return ICmpInst::Create(Op: Instruction::ICmp, Pred, S1: Masked, S2: Zero);
1167}
1168
1169/// Fold equality-comparison between zero and any (maybe truncated) right-shift
1170/// by one-less-than-bitwidth into a sign test on the original value.
1171Instruction *InstCombinerImpl::foldSignBitTest(ICmpInst &I) {
1172 Instruction *Val;
1173 ICmpInst::Predicate Pred;
1174 if (!I.isEquality() || !match(V: &I, P: m_ICmp(Pred, L: m_Instruction(I&: Val), R: m_Zero())))
1175 return nullptr;
1176
1177 Value *X;
1178 Type *XTy;
1179
1180 Constant *C;
1181 if (match(V: Val, P: m_TruncOrSelf(Op: m_Shr(L: m_Value(V&: X), R: m_Constant(C))))) {
1182 XTy = X->getType();
1183 unsigned XBitWidth = XTy->getScalarSizeInBits();
1184 if (!match(V: C, P: m_SpecificInt_ICMP(Predicate: ICmpInst::Predicate::ICMP_EQ,
1185 Threshold: APInt(XBitWidth, XBitWidth - 1))))
1186 return nullptr;
1187 } else if (isa<BinaryOperator>(Val) &&
1188 (X = reassociateShiftAmtsOfTwoSameDirectionShifts(
1189 Sh0: cast<BinaryOperator>(Val), SQ: SQ.getWithInstruction(I: Val),
1190 /*AnalyzeForSignBitExtraction=*/true))) {
1191 XTy = X->getType();
1192 } else
1193 return nullptr;
1194
1195 return ICmpInst::Create(Op: Instruction::ICmp,
1196 Pred: Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_SGE
1197 : ICmpInst::ICMP_SLT,
1198 S1: X, S2: ConstantInt::getNullValue(Ty: XTy));
1199}
1200
1201// Handle icmp pred X, 0
1202Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) {
1203 CmpInst::Predicate Pred = Cmp.getPredicate();
1204 if (!match(V: Cmp.getOperand(i_nocapture: 1), P: m_Zero()))
1205 return nullptr;
1206
1207 // (icmp sgt smin(PosA, B) 0) -> (icmp sgt B 0)
1208 if (Pred == ICmpInst::ICMP_SGT) {
1209 Value *A, *B;
1210 if (match(V: Cmp.getOperand(i_nocapture: 0), P: m_SMin(L: m_Value(V&: A), R: m_Value(V&: B)))) {
1211 if (isKnownPositive(V: A, SQ: SQ.getWithInstruction(I: &Cmp)))
1212 return new ICmpInst(Pred, B, Cmp.getOperand(i_nocapture: 1));
1213 if (isKnownPositive(V: B, SQ: SQ.getWithInstruction(I: &Cmp)))
1214 return new ICmpInst(Pred, A, Cmp.getOperand(i_nocapture: 1));
1215 }
1216 }
1217
1218 if (Instruction *New = foldIRemByPowerOfTwoToBitTest(I&: Cmp))
1219 return New;
1220
1221 // Given:
1222 // icmp eq/ne (urem %x, %y), 0
1223 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
1224 // icmp eq/ne %x, 0
1225 Value *X, *Y;
1226 if (match(V: Cmp.getOperand(i_nocapture: 0), P: m_URem(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
1227 ICmpInst::isEquality(P: Pred)) {
1228 KnownBits XKnown = computeKnownBits(V: X, Depth: 0, CxtI: &Cmp);
1229 KnownBits YKnown = computeKnownBits(V: Y, Depth: 0, CxtI: &Cmp);
1230 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
1231 return new ICmpInst(Pred, X, Cmp.getOperand(i_nocapture: 1));
1232 }
1233
1234 // (icmp eq/ne (mul X Y)) -> (icmp eq/ne X/Y) if we know about whether X/Y are
1235 // odd/non-zero/there is no overflow.
1236 if (match(V: Cmp.getOperand(i_nocapture: 0), P: m_Mul(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
1237 ICmpInst::isEquality(P: Pred)) {
1238
1239 KnownBits XKnown = computeKnownBits(V: X, Depth: 0, CxtI: &Cmp);
1240 // if X % 2 != 0
1241 // (icmp eq/ne Y)
1242 if (XKnown.countMaxTrailingZeros() == 0)
1243 return new ICmpInst(Pred, Y, Cmp.getOperand(i_nocapture: 1));
1244
1245 KnownBits YKnown = computeKnownBits(V: Y, Depth: 0, CxtI: &Cmp);
1246 // if Y % 2 != 0
1247 // (icmp eq/ne X)
1248 if (YKnown.countMaxTrailingZeros() == 0)
1249 return new ICmpInst(Pred, X, Cmp.getOperand(i_nocapture: 1));
1250
1251 auto *BO0 = cast<OverflowingBinaryOperator>(Val: Cmp.getOperand(i_nocapture: 0));
1252 if (BO0->hasNoUnsignedWrap() || BO0->hasNoSignedWrap()) {
1253 const SimplifyQuery Q = SQ.getWithInstruction(I: &Cmp);
1254 // `isKnownNonZero` does more analysis than just `!KnownBits.One.isZero()`
1255 // but to avoid unnecessary work, first just if this is an obvious case.
1256
1257 // if X non-zero and NoOverflow(X * Y)
1258 // (icmp eq/ne Y)
1259 if (!XKnown.One.isZero() || isKnownNonZero(V: X, Q))
1260 return new ICmpInst(Pred, Y, Cmp.getOperand(i_nocapture: 1));
1261
1262 // if Y non-zero and NoOverflow(X * Y)
1263 // (icmp eq/ne X)
1264 if (!YKnown.One.isZero() || isKnownNonZero(V: Y, Q))
1265 return new ICmpInst(Pred, X, Cmp.getOperand(i_nocapture: 1));
1266 }
1267 // Note, we are skipping cases:
1268 // if Y % 2 != 0 AND X % 2 != 0
1269 // (false/true)
1270 // if X non-zero and Y non-zero and NoOverflow(X * Y)
1271 // (false/true)
1272 // Those can be simplified later as we would have already replaced the (icmp
1273 // eq/ne (mul X, Y)) with (icmp eq/ne X/Y) and if X/Y is known non-zero that
1274 // will fold to a constant elsewhere.
1275 }
1276 return nullptr;
1277}
1278
1279/// Fold icmp Pred X, C.
1280/// TODO: This code structure does not make sense. The saturating add fold
1281/// should be moved to some other helper and extended as noted below (it is also
1282/// possible that code has been made unnecessary - do we canonicalize IR to
1283/// overflow/saturating intrinsics or not?).
1284Instruction *InstCombinerImpl::foldICmpWithConstant(ICmpInst &Cmp) {
1285 // Match the following pattern, which is a common idiom when writing
1286 // overflow-safe integer arithmetic functions. The source performs an addition
1287 // in wider type and explicitly checks for overflow using comparisons against
1288 // INT_MIN and INT_MAX. Simplify by using the sadd_with_overflow intrinsic.
1289 //
1290 // TODO: This could probably be generalized to handle other overflow-safe
1291 // operations if we worked out the formulas to compute the appropriate magic
1292 // constants.
1293 //
1294 // sum = a + b
1295 // if (sum+128 >u 255) ... -> llvm.sadd.with.overflow.i8
1296 CmpInst::Predicate Pred = Cmp.getPredicate();
1297 Value *Op0 = Cmp.getOperand(i_nocapture: 0), *Op1 = Cmp.getOperand(i_nocapture: 1);
1298 Value *A, *B;
1299 ConstantInt *CI, *CI2; // I = icmp ugt (add (add A, B), CI2), CI
1300 if (Pred == ICmpInst::ICMP_UGT && match(V: Op1, P: m_ConstantInt(CI)) &&
1301 match(V: Op0, P: m_Add(L: m_Add(L: m_Value(V&: A), R: m_Value(V&: B)), R: m_ConstantInt(CI&: CI2))))
1302 if (Instruction *Res = processUGT_ADDCST_ADD(I&: Cmp, A, B, CI2, CI1: CI, IC&: *this))
1303 return Res;
1304
1305 // icmp(phi(C1, C2, ...), C) -> phi(icmp(C1, C), icmp(C2, C), ...).
1306 Constant *C = dyn_cast<Constant>(Val: Op1);
1307 if (!C)
1308 return nullptr;
1309
1310 if (auto *Phi = dyn_cast<PHINode>(Val: Op0))
1311 if (all_of(Range: Phi->operands(), P: [](Value *V) { return isa<Constant>(Val: V); })) {
1312 SmallVector<Constant *> Ops;
1313 for (Value *V : Phi->incoming_values()) {
1314 Constant *Res =
1315 ConstantFoldCompareInstOperands(Predicate: Pred, LHS: cast<Constant>(Val: V), RHS: C, DL);
1316 if (!Res)
1317 return nullptr;
1318 Ops.push_back(Elt: Res);
1319 }
1320 Builder.SetInsertPoint(Phi);
1321 PHINode *NewPhi = Builder.CreatePHI(Ty: Cmp.getType(), NumReservedValues: Phi->getNumOperands());
1322 for (auto [V, Pred] : zip(t&: Ops, u: Phi->blocks()))
1323 NewPhi->addIncoming(V, BB: Pred);
1324 return replaceInstUsesWith(I&: Cmp, V: NewPhi);
1325 }
1326
1327 if (Instruction *R = tryFoldInstWithCtpopWithNot(I: &Cmp))
1328 return R;
1329
1330 return nullptr;
1331}
1332
1333/// Canonicalize icmp instructions based on dominating conditions.
1334Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) {
1335 // We already checked simple implication in InstSimplify, only handle complex
1336 // cases here.
1337 Value *X = Cmp.getOperand(i_nocapture: 0), *Y = Cmp.getOperand(i_nocapture: 1);
1338 const APInt *C;
1339 if (!match(V: Y, P: m_APInt(Res&: C)))
1340 return nullptr;
1341
1342 CmpInst::Predicate Pred = Cmp.getPredicate();
1343 ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
1344
1345 auto handleDomCond = [&](ICmpInst::Predicate DomPred,
1346 const APInt *DomC) -> Instruction * {
1347 // We have 2 compares of a variable with constants. Calculate the constant
1348 // ranges of those compares to see if we can transform the 2nd compare:
1349 // DomBB:
1350 // DomCond = icmp DomPred X, DomC
1351 // br DomCond, CmpBB, FalseBB
1352 // CmpBB:
1353 // Cmp = icmp Pred X, C
1354 ConstantRange DominatingCR =
1355 ConstantRange::makeExactICmpRegion(Pred: DomPred, Other: *DomC);
1356 ConstantRange Intersection = DominatingCR.intersectWith(CR);
1357 ConstantRange Difference = DominatingCR.difference(CR);
1358 if (Intersection.isEmptySet())
1359 return replaceInstUsesWith(I&: Cmp, V: Builder.getFalse());
1360 if (Difference.isEmptySet())
1361 return replaceInstUsesWith(I&: Cmp, V: Builder.getTrue());
1362
1363 // Canonicalizing a sign bit comparison that gets used in a branch,
1364 // pessimizes codegen by generating branch on zero instruction instead
1365 // of a test and branch. So we avoid canonicalizing in such situations
1366 // because test and branch instruction has better branch displacement
1367 // than compare and branch instruction.
1368 bool UnusedBit;
1369 bool IsSignBit = isSignBitCheck(Pred, RHS: *C, TrueIfSigned&: UnusedBit);
1370 if (Cmp.isEquality() || (IsSignBit && hasBranchUse(I&: Cmp)))
1371 return nullptr;
1372
1373 // Avoid an infinite loop with min/max canonicalization.
1374 // TODO: This will be unnecessary if we canonicalize to min/max intrinsics.
1375 if (Cmp.hasOneUse() &&
1376 match(V: Cmp.user_back(), P: m_MaxOrMin(L: m_Value(), R: m_Value())))
1377 return nullptr;
1378
1379 if (const APInt *EqC = Intersection.getSingleElement())
1380 return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(AI: *EqC));
1381 if (const APInt *NeC = Difference.getSingleElement())
1382 return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(AI: *NeC));
1383 return nullptr;
1384 };
1385
1386 for (BranchInst *BI : DC.conditionsFor(V: X)) {
1387 ICmpInst::Predicate DomPred;
1388 const APInt *DomC;
1389 if (!match(V: BI->getCondition(),
1390 P: m_ICmp(Pred&: DomPred, L: m_Specific(V: X), R: m_APInt(Res&: DomC))))
1391 continue;
1392
1393 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
1394 if (DT.dominates(BBE: Edge0, BB: Cmp.getParent())) {
1395 if (auto *V = handleDomCond(DomPred, DomC))
1396 return V;
1397 } else {
1398 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
1399 if (DT.dominates(BBE: Edge1, BB: Cmp.getParent()))
1400 if (auto *V =
1401 handleDomCond(CmpInst::getInversePredicate(pred: DomPred), DomC))
1402 return V;
1403 }
1404 }
1405
1406 return nullptr;
1407}
1408
1409/// Fold icmp (trunc X), C.
1410Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp,
1411 TruncInst *Trunc,
1412 const APInt &C) {
1413 ICmpInst::Predicate Pred = Cmp.getPredicate();
1414 Value *X = Trunc->getOperand(i_nocapture: 0);
1415 Type *SrcTy = X->getType();
1416 unsigned DstBits = Trunc->getType()->getScalarSizeInBits(),
1417 SrcBits = SrcTy->getScalarSizeInBits();
1418
1419 // Match (icmp pred (trunc nuw/nsw X), C)
1420 // Which we can convert to (icmp pred X, (sext/zext C))
1421 if (shouldChangeType(From: Trunc->getType(), To: SrcTy)) {
1422 if (Trunc->hasNoSignedWrap())
1423 return new ICmpInst(Pred, X, ConstantInt::get(Ty: SrcTy, V: C.sext(width: SrcBits)));
1424 if (!Cmp.isSigned() && Trunc->hasNoUnsignedWrap())
1425 return new ICmpInst(Pred, X, ConstantInt::get(Ty: SrcTy, V: C.zext(width: SrcBits)));
1426 }
1427
1428 if (C.isOne() && C.getBitWidth() > 1) {
1429 // icmp slt trunc(signum(V)) 1 --> icmp slt V, 1
1430 Value *V = nullptr;
1431 if (Pred == ICmpInst::ICMP_SLT && match(V: X, P: m_Signum(V: m_Value(V))))
1432 return new ICmpInst(ICmpInst::ICMP_SLT, V,
1433 ConstantInt::get(Ty: V->getType(), V: 1));
1434 }
1435
1436 // TODO: Handle any shifted constant by subtracting trailing zeros.
1437 // TODO: Handle non-equality predicates.
1438 Value *Y;
1439 if (Cmp.isEquality() && match(V: X, P: m_Shl(L: m_One(), R: m_Value(V&: Y)))) {
1440 // (trunc (1 << Y) to iN) == 0 --> Y u>= N
1441 // (trunc (1 << Y) to iN) != 0 --> Y u< N
1442 if (C.isZero()) {
1443 auto NewPred = (Pred == Cmp.ICMP_EQ) ? Cmp.ICMP_UGE : Cmp.ICMP_ULT;
1444 return new ICmpInst(NewPred, Y, ConstantInt::get(Ty: SrcTy, V: DstBits));
1445 }
1446 // (trunc (1 << Y) to iN) == 2**C --> Y == C
1447 // (trunc (1 << Y) to iN) != 2**C --> Y != C
1448 if (C.isPowerOf2())
1449 return new ICmpInst(Pred, Y, ConstantInt::get(Ty: SrcTy, V: C.logBase2()));
1450 }
1451
1452 if (Cmp.isEquality() && Trunc->hasOneUse()) {
1453 // Canonicalize to a mask and wider compare if the wide type is suitable:
1454 // (trunc X to i8) == C --> (X & 0xff) == (zext C)
1455 if (!SrcTy->isVectorTy() && shouldChangeType(FromBitWidth: DstBits, ToBitWidth: SrcBits)) {
1456 Constant *Mask =
1457 ConstantInt::get(Ty: SrcTy, V: APInt::getLowBitsSet(numBits: SrcBits, loBitsSet: DstBits));
1458 Value *And = Builder.CreateAnd(LHS: X, RHS: Mask);
1459 Constant *WideC = ConstantInt::get(Ty: SrcTy, V: C.zext(width: SrcBits));
1460 return new ICmpInst(Pred, And, WideC);
1461 }
1462
1463 // Simplify icmp eq (trunc x to i8), 42 -> icmp eq x, 42|highbits if all
1464 // of the high bits truncated out of x are known.
1465 KnownBits Known = computeKnownBits(V: X, Depth: 0, CxtI: &Cmp);
1466
1467 // If all the high bits are known, we can do this xform.
1468 if ((Known.Zero | Known.One).countl_one() >= SrcBits - DstBits) {
1469 // Pull in the high bits from known-ones set.
1470 APInt NewRHS = C.zext(width: SrcBits);
1471 NewRHS |= Known.One & APInt::getHighBitsSet(numBits: SrcBits, hiBitsSet: SrcBits - DstBits);
1472 return new ICmpInst(Pred, X, ConstantInt::get(Ty: SrcTy, V: NewRHS));
1473 }
1474 }
1475
1476 // Look through truncated right-shift of the sign-bit for a sign-bit check:
1477 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0 --> ShOp < 0
1478 // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1
1479 Value *ShOp;
1480 const APInt *ShAmtC;
1481 bool TrueIfSigned;
1482 if (isSignBitCheck(Pred, RHS: C, TrueIfSigned) &&
1483 match(V: X, P: m_Shr(L: m_Value(V&: ShOp), R: m_APInt(Res&: ShAmtC))) &&
1484 DstBits == SrcBits - ShAmtC->getZExtValue()) {
1485 return TrueIfSigned ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp,
1486 ConstantInt::getNullValue(Ty: SrcTy))
1487 : new ICmpInst(ICmpInst::ICMP_SGT, ShOp,
1488 ConstantInt::getAllOnesValue(Ty: SrcTy));
1489 }
1490
1491 return nullptr;
1492}
1493
1494/// Fold icmp (trunc nuw/nsw X), (trunc nuw/nsw Y).
1495/// Fold icmp (trunc nuw/nsw X), (zext/sext Y).
1496Instruction *
1497InstCombinerImpl::foldICmpTruncWithTruncOrExt(ICmpInst &Cmp,
1498 const SimplifyQuery &Q) {
1499 Value *X, *Y;
1500 ICmpInst::Predicate Pred;
1501 bool YIsSExt = false;
1502 // Try to match icmp (trunc X), (trunc Y)
1503 if (match(V: &Cmp, P: m_ICmp(Pred, L: m_Trunc(Op: m_Value(V&: X)), R: m_Trunc(Op: m_Value(V&: Y))))) {
1504 unsigned NoWrapFlags = cast<TruncInst>(Val: Cmp.getOperand(i_nocapture: 0))->getNoWrapKind() &
1505 cast<TruncInst>(Val: Cmp.getOperand(i_nocapture: 1))->getNoWrapKind();
1506 if (Cmp.isSigned()) {
1507 // For signed comparisons, both truncs must be nsw.
1508 if (!(NoWrapFlags & TruncInst::NoSignedWrap))
1509 return nullptr;
1510 } else {
1511 // For unsigned and equality comparisons, either both must be nuw or
1512 // both must be nsw, we don't care which.
1513 if (!NoWrapFlags)
1514 return nullptr;
1515 }
1516
1517 if (X->getType() != Y->getType() &&
1518 (!Cmp.getOperand(i_nocapture: 0)->hasOneUse() || !Cmp.getOperand(i_nocapture: 1)->hasOneUse()))
1519 return nullptr;
1520 if (!isDesirableIntType(BitWidth: X->getType()->getScalarSizeInBits()) &&
1521 isDesirableIntType(BitWidth: Y->getType()->getScalarSizeInBits())) {
1522 std::swap(a&: X, b&: Y);
1523 Pred = Cmp.getSwappedPredicate(pred: Pred);
1524 }
1525 YIsSExt = !(NoWrapFlags & TruncInst::NoUnsignedWrap);
1526 }
1527 // Try to match icmp (trunc nuw X), (zext Y)
1528 else if (!Cmp.isSigned() &&
1529 match(V: &Cmp, P: m_c_ICmp(Pred, L: m_NUWTrunc(Op: m_Value(V&: X)),
1530 R: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: Y)))))) {
1531 // Can fold trunc nuw + zext for unsigned and equality predicates.
1532 }
1533 // Try to match icmp (trunc nsw X), (sext Y)
1534 else if (match(V: &Cmp, P: m_c_ICmp(Pred, L: m_NSWTrunc(Op: m_Value(V&: X)),
1535 R: m_OneUse(SubPattern: m_ZExtOrSExt(Op: m_Value(V&: Y)))))) {
1536 // Can fold trunc nsw + zext/sext for all predicates.
1537 YIsSExt =
1538 isa<SExtInst>(Val: Cmp.getOperand(i_nocapture: 0)) || isa<SExtInst>(Val: Cmp.getOperand(i_nocapture: 1));
1539 } else
1540 return nullptr;
1541
1542 Type *TruncTy = Cmp.getOperand(i_nocapture: 0)->getType();
1543 unsigned TruncBits = TruncTy->getScalarSizeInBits();
1544
1545 // If this transform will end up changing from desirable types -> undesirable
1546 // types skip it.
1547 if (isDesirableIntType(BitWidth: TruncBits) &&
1548 !isDesirableIntType(BitWidth: X->getType()->getScalarSizeInBits()))
1549 return nullptr;
1550
1551 Value *NewY = Builder.CreateIntCast(V: Y, DestTy: X->getType(), isSigned: YIsSExt);
1552 return new ICmpInst(Pred, X, NewY);
1553}
1554
1555/// Fold icmp (xor X, Y), C.
1556Instruction *InstCombinerImpl::foldICmpXorConstant(ICmpInst &Cmp,
1557 BinaryOperator *Xor,
1558 const APInt &C) {
1559 if (Instruction *I = foldICmpXorShiftConst(Cmp, Xor, C))
1560 return I;
1561
1562 Value *X = Xor->getOperand(i_nocapture: 0);
1563 Value *Y = Xor->getOperand(i_nocapture: 1);
1564 const APInt *XorC;
1565 if (!match(V: Y, P: m_APInt(Res&: XorC)))
1566 return nullptr;
1567
1568 // If this is a comparison that tests the signbit (X < 0) or (x > -1),
1569 // fold the xor.
1570 ICmpInst::Predicate Pred = Cmp.getPredicate();
1571 bool TrueIfSigned = false;
1572 if (isSignBitCheck(Pred: Cmp.getPredicate(), RHS: C, TrueIfSigned)) {
1573
1574 // If the sign bit of the XorCst is not set, there is no change to
1575 // the operation, just stop using the Xor.
1576 if (!XorC->isNegative())
1577 return replaceOperand(I&: Cmp, OpNum: 0, V: X);
1578
1579 // Emit the opposite comparison.
1580 if (TrueIfSigned)
1581 return new ICmpInst(ICmpInst::ICMP_SGT, X,
1582 ConstantInt::getAllOnesValue(Ty: X->getType()));
1583 else
1584 return new ICmpInst(ICmpInst::ICMP_SLT, X,
1585 ConstantInt::getNullValue(Ty: X->getType()));
1586 }
1587
1588 if (Xor->hasOneUse()) {
1589 // (icmp u/s (xor X SignMask), C) -> (icmp s/u X, (xor C SignMask))
1590 if (!Cmp.isEquality() && XorC->isSignMask()) {
1591 Pred = Cmp.getFlippedSignednessPredicate();
1592 return new ICmpInst(Pred, X, ConstantInt::get(Ty: X->getType(), V: C ^ *XorC));
1593 }
1594
1595 // (icmp u/s (xor X ~SignMask), C) -> (icmp s/u X, (xor C ~SignMask))
1596 if (!Cmp.isEquality() && XorC->isMaxSignedValue()) {
1597 Pred = Cmp.getFlippedSignednessPredicate();
1598 Pred = Cmp.getSwappedPredicate(pred: Pred);
1599 return new ICmpInst(Pred, X, ConstantInt::get(Ty: X->getType(), V: C ^ *XorC));
1600 }
1601 }
1602
1603 // Mask constant magic can eliminate an 'xor' with unsigned compares.
1604 if (Pred == ICmpInst::ICMP_UGT) {
1605 // (xor X, ~C) >u C --> X <u ~C (when C+1 is a power of 2)
1606 if (*XorC == ~C && (C + 1).isPowerOf2())
1607 return new ICmpInst(ICmpInst::ICMP_ULT, X, Y);
1608 // (xor X, C) >u C --> X >u C (when C+1 is a power of 2)
1609 if (*XorC == C && (C + 1).isPowerOf2())
1610 return new ICmpInst(ICmpInst::ICMP_UGT, X, Y);
1611 }
1612 if (Pred == ICmpInst::ICMP_ULT) {
1613 // (xor X, -C) <u C --> X >u ~C (when C is a power of 2)
1614 if (*XorC == -C && C.isPowerOf2())
1615 return new ICmpInst(ICmpInst::ICMP_UGT, X,
1616 ConstantInt::get(Ty: X->getType(), V: ~C));
1617 // (xor X, C) <u C --> X >u ~C (when -C is a power of 2)
1618 if (*XorC == C && (-C).isPowerOf2())
1619 return new ICmpInst(ICmpInst::ICMP_UGT, X,
1620 ConstantInt::get(Ty: X->getType(), V: ~C));
1621 }
1622 return nullptr;
1623}
1624
1625/// For power-of-2 C:
1626/// ((X s>> ShiftC) ^ X) u< C --> (X + C) u< (C << 1)
1627/// ((X s>> ShiftC) ^ X) u> (C - 1) --> (X + C) u> ((C << 1) - 1)
1628Instruction *InstCombinerImpl::foldICmpXorShiftConst(ICmpInst &Cmp,
1629 BinaryOperator *Xor,
1630 const APInt &C) {
1631 CmpInst::Predicate Pred = Cmp.getPredicate();
1632 APInt PowerOf2;
1633 if (Pred == ICmpInst::ICMP_ULT)
1634 PowerOf2 = C;
1635 else if (Pred == ICmpInst::ICMP_UGT && !C.isMaxValue())
1636 PowerOf2 = C + 1;
1637 else
1638 return nullptr;
1639 if (!PowerOf2.isPowerOf2())
1640 return nullptr;
1641 Value *X;
1642 const APInt *ShiftC;
1643 if (!match(V: Xor, P: m_OneUse(SubPattern: m_c_Xor(L: m_Value(V&: X),
1644 R: m_AShr(L: m_Deferred(V: X), R: m_APInt(Res&: ShiftC))))))
1645 return nullptr;
1646 uint64_t Shift = ShiftC->getLimitedValue();
1647 Type *XType = X->getType();
1648 if (Shift == 0 || PowerOf2.isMinSignedValue())
1649 return nullptr;
1650 Value *Add = Builder.CreateAdd(LHS: X, RHS: ConstantInt::get(Ty: XType, V: PowerOf2));
1651 APInt Bound =
1652 Pred == ICmpInst::ICMP_ULT ? PowerOf2 << 1 : ((PowerOf2 << 1) - 1);
1653 return new ICmpInst(Pred, Add, ConstantInt::get(Ty: XType, V: Bound));
1654}
1655
1656/// Fold icmp (and (sh X, Y), C2), C1.
1657Instruction *InstCombinerImpl::foldICmpAndShift(ICmpInst &Cmp,
1658 BinaryOperator *And,
1659 const APInt &C1,
1660 const APInt &C2) {
1661 BinaryOperator *Shift = dyn_cast<BinaryOperator>(Val: And->getOperand(i_nocapture: 0));
1662 if (!Shift || !Shift->isShift())
1663 return nullptr;
1664
1665 // If this is: (X >> C3) & C2 != C1 (where any shift and any compare could
1666 // exist), turn it into (X & (C2 << C3)) != (C1 << C3). This happens a LOT in
1667 // code produced by the clang front-end, for bitfield access.
1668 // This seemingly simple opportunity to fold away a shift turns out to be
1669 // rather complicated. See PR17827 for details.
1670 unsigned ShiftOpcode = Shift->getOpcode();
1671 bool IsShl = ShiftOpcode == Instruction::Shl;
1672 const APInt *C3;
1673 if (match(V: Shift->getOperand(i_nocapture: 1), P: m_APInt(Res&: C3))) {
1674 APInt NewAndCst, NewCmpCst;
1675 bool AnyCmpCstBitsShiftedOut;
1676 if (ShiftOpcode == Instruction::Shl) {
1677 // For a left shift, we can fold if the comparison is not signed. We can
1678 // also fold a signed comparison if the mask value and comparison value
1679 // are not negative. These constraints may not be obvious, but we can
1680 // prove that they are correct using an SMT solver.
1681 if (Cmp.isSigned() && (C2.isNegative() || C1.isNegative()))
1682 return nullptr;
1683
1684 NewCmpCst = C1.lshr(ShiftAmt: *C3);
1685 NewAndCst = C2.lshr(ShiftAmt: *C3);
1686 AnyCmpCstBitsShiftedOut = NewCmpCst.shl(ShiftAmt: *C3) != C1;
1687 } else if (ShiftOpcode == Instruction::LShr) {
1688 // For a logical right shift, we can fold if the comparison is not signed.
1689 // We can also fold a signed comparison if the shifted mask value and the
1690 // shifted comparison value are not negative. These constraints may not be
1691 // obvious, but we can prove that they are correct using an SMT solver.
1692 NewCmpCst = C1.shl(ShiftAmt: *C3);
1693 NewAndCst = C2.shl(ShiftAmt: *C3);
1694 AnyCmpCstBitsShiftedOut = NewCmpCst.lshr(ShiftAmt: *C3) != C1;
1695 if (Cmp.isSigned() && (NewAndCst.isNegative() || NewCmpCst.isNegative()))
1696 return nullptr;
1697 } else {
1698 // For an arithmetic shift, check that both constants don't use (in a
1699 // signed sense) the top bits being shifted out.
1700 assert(ShiftOpcode == Instruction::AShr && "Unknown shift opcode");
1701 NewCmpCst = C1.shl(ShiftAmt: *C3);
1702 NewAndCst = C2.shl(ShiftAmt: *C3);
1703 AnyCmpCstBitsShiftedOut = NewCmpCst.ashr(ShiftAmt: *C3) != C1;
1704 if (NewAndCst.ashr(ShiftAmt: *C3) != C2)
1705 return nullptr;
1706 }
1707
1708 if (AnyCmpCstBitsShiftedOut) {
1709 // If we shifted bits out, the fold is not going to work out. As a
1710 // special case, check to see if this means that the result is always
1711 // true or false now.
1712 if (Cmp.getPredicate() == ICmpInst::ICMP_EQ)
1713 return replaceInstUsesWith(I&: Cmp, V: ConstantInt::getFalse(Ty: Cmp.getType()));
1714 if (Cmp.getPredicate() == ICmpInst::ICMP_NE)
1715 return replaceInstUsesWith(I&: Cmp, V: ConstantInt::getTrue(Ty: Cmp.getType()));
1716 } else {
1717 Value *NewAnd = Builder.CreateAnd(
1718 LHS: Shift->getOperand(i_nocapture: 0), RHS: ConstantInt::get(Ty: And->getType(), V: NewAndCst));
1719 return new ICmpInst(Cmp.getPredicate(),
1720 NewAnd, ConstantInt::get(Ty: And->getType(), V: NewCmpCst));
1721 }
1722 }
1723
1724 // Turn ((X >> Y) & C2) == 0 into (X & (C2 << Y)) == 0. The latter is
1725 // preferable because it allows the C2 << Y expression to be hoisted out of a
1726 // loop if Y is invariant and X is not.
1727 if (Shift->hasOneUse() && C1.isZero() && Cmp.isEquality() &&
1728 !Shift->isArithmeticShift() && !isa<Constant>(Val: Shift->getOperand(i_nocapture: 0))) {
1729 // Compute C2 << Y.
1730 Value *NewShift =
1731 IsShl ? Builder.CreateLShr(LHS: And->getOperand(i_nocapture: 1), RHS: Shift->getOperand(i_nocapture: 1))
1732 : Builder.CreateShl(LHS: And->getOperand(i_nocapture: 1), RHS: Shift->getOperand(i_nocapture: 1));
1733
1734 // Compute X & (C2 << Y).
1735 Value *NewAnd = Builder.CreateAnd(LHS: Shift->getOperand(i_nocapture: 0), RHS: NewShift);
1736 return replaceOperand(I&: Cmp, OpNum: 0, V: NewAnd);
1737 }
1738
1739 return nullptr;
1740}
1741
1742/// Fold icmp (and X, C2), C1.
1743Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp,
1744 BinaryOperator *And,
1745 const APInt &C1) {
1746 bool isICMP_NE = Cmp.getPredicate() == ICmpInst::ICMP_NE;
1747
1748 // For vectors: icmp ne (and X, 1), 0 --> trunc X to N x i1
1749 // TODO: We canonicalize to the longer form for scalars because we have
1750 // better analysis/folds for icmp, and codegen may be better with icmp.
1751 if (isICMP_NE && Cmp.getType()->isVectorTy() && C1.isZero() &&
1752 match(V: And->getOperand(i_nocapture: 1), P: m_One()))
1753 return new TruncInst(And->getOperand(i_nocapture: 0), Cmp.getType());
1754
1755 const APInt *C2;
1756 Value *X;
1757 if (!match(V: And, P: m_And(L: m_Value(V&: X), R: m_APInt(Res&: C2))))
1758 return nullptr;
1759
1760 // Don't perform the following transforms if the AND has multiple uses
1761 if (!And->hasOneUse())
1762 return nullptr;
1763
1764 if (Cmp.isEquality() && C1.isZero()) {
1765 // Restrict this fold to single-use 'and' (PR10267).
1766 // Replace (and X, (1 << size(X)-1) != 0) with X s< 0
1767 if (C2->isSignMask()) {
1768 Constant *Zero = Constant::getNullValue(Ty: X->getType());
1769 auto NewPred = isICMP_NE ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGE;
1770 return new ICmpInst(NewPred, X, Zero);
1771 }
1772
1773 APInt NewC2 = *C2;
1774 KnownBits Know = computeKnownBits(V: And->getOperand(i_nocapture: 0), Depth: 0, CxtI: And);
1775 // Set high zeros of C2 to allow matching negated power-of-2.
1776 NewC2 = *C2 | APInt::getHighBitsSet(numBits: C2->getBitWidth(),
1777 hiBitsSet: Know.countMinLeadingZeros());
1778
1779 // Restrict this fold only for single-use 'and' (PR10267).
1780 // ((%x & C) == 0) --> %x u< (-C) iff (-C) is power of two.
1781 if (NewC2.isNegatedPowerOf2()) {
1782 Constant *NegBOC = ConstantInt::get(Ty: And->getType(), V: -NewC2);
1783 auto NewPred = isICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT;
1784 return new ICmpInst(NewPred, X, NegBOC);
1785 }
1786 }
1787
1788 // If the LHS is an 'and' of a truncate and we can widen the and/compare to
1789 // the input width without changing the value produced, eliminate the cast:
1790 //
1791 // icmp (and (trunc W), C2), C1 -> icmp (and W, C2'), C1'
1792 //
1793 // We can do this transformation if the constants do not have their sign bits
1794 // set or if it is an equality comparison. Extending a relational comparison
1795 // when we're checking the sign bit would not work.
1796 Value *W;
1797 if (match(V: And->getOperand(i_nocapture: 0), P: m_OneUse(SubPattern: m_Trunc(Op: m_Value(V&: W)))) &&
1798 (Cmp.isEquality() || (!C1.isNegative() && !C2->isNegative()))) {
1799 // TODO: Is this a good transform for vectors? Wider types may reduce
1800 // throughput. Should this transform be limited (even for scalars) by using
1801 // shouldChangeType()?
1802 if (!Cmp.getType()->isVectorTy()) {
1803 Type *WideType = W->getType();
1804 unsigned WideScalarBits = WideType->getScalarSizeInBits();
1805 Constant *ZextC1 = ConstantInt::get(Ty: WideType, V: C1.zext(width: WideScalarBits));
1806 Constant *ZextC2 = ConstantInt::get(Ty: WideType, V: C2->zext(width: WideScalarBits));
1807 Value *NewAnd = Builder.CreateAnd(LHS: W, RHS: ZextC2, Name: And->getName());
1808 return new ICmpInst(Cmp.getPredicate(), NewAnd, ZextC1);
1809 }
1810 }
1811
1812 if (Instruction *I = foldICmpAndShift(Cmp, And, C1, C2: *C2))
1813 return I;
1814
1815 // (icmp pred (and (or (lshr A, B), A), 1), 0) -->
1816 // (icmp pred (and A, (or (shl 1, B), 1), 0))
1817 //
1818 // iff pred isn't signed
1819 if (!Cmp.isSigned() && C1.isZero() && And->getOperand(i_nocapture: 0)->hasOneUse() &&
1820 match(V: And->getOperand(i_nocapture: 1), P: m_One())) {
1821 Constant *One = cast<Constant>(Val: And->getOperand(i_nocapture: 1));
1822 Value *Or = And->getOperand(i_nocapture: 0);
1823 Value *A, *B, *LShr;
1824 if (match(V: Or, P: m_Or(L: m_Value(V&: LShr), R: m_Value(V&: A))) &&
1825 match(V: LShr, P: m_LShr(L: m_Specific(V: A), R: m_Value(V&: B)))) {
1826 unsigned UsesRemoved = 0;
1827 if (And->hasOneUse())
1828 ++UsesRemoved;
1829 if (Or->hasOneUse())
1830 ++UsesRemoved;
1831 if (LShr->hasOneUse())
1832 ++UsesRemoved;
1833
1834 // Compute A & ((1 << B) | 1)
1835 unsigned RequireUsesRemoved = match(V: B, P: m_ImmConstant()) ? 1 : 3;
1836 if (UsesRemoved >= RequireUsesRemoved) {
1837 Value *NewOr =
1838 Builder.CreateOr(LHS: Builder.CreateShl(LHS: One, RHS: B, Name: LShr->getName(),
1839 /*HasNUW=*/true),
1840 RHS: One, Name: Or->getName());
1841 Value *NewAnd = Builder.CreateAnd(LHS: A, RHS: NewOr, Name: And->getName());
1842 return replaceOperand(I&: Cmp, OpNum: 0, V: NewAnd);
1843 }
1844 }
1845 }
1846
1847 // (icmp eq (and (bitcast X to int), ExponentMask), ExponentMask) -->
1848 // llvm.is.fpclass(X, fcInf|fcNan)
1849 // (icmp ne (and (bitcast X to int), ExponentMask), ExponentMask) -->
1850 // llvm.is.fpclass(X, ~(fcInf|fcNan))
1851 Value *V;
1852 if (!Cmp.getParent()->getParent()->hasFnAttribute(
1853 Kind: Attribute::NoImplicitFloat) &&
1854 Cmp.isEquality() &&
1855 match(V: X, P: m_OneUse(SubPattern: m_ElementWiseBitCast(Op: m_Value(V))))) {
1856 Type *FPType = V->getType()->getScalarType();
1857 if (FPType->isIEEELikeFPTy() && C1 == *C2) {
1858 APInt ExponentMask =
1859 APFloat::getInf(Sem: FPType->getFltSemantics()).bitcastToAPInt();
1860 if (C1 == ExponentMask) {
1861 unsigned Mask = FPClassTest::fcNan | FPClassTest::fcInf;
1862 if (isICMP_NE)
1863 Mask = ~Mask & fcAllFlags;
1864 return replaceInstUsesWith(I&: Cmp, V: Builder.createIsFPClass(FPNum: V, Test: Mask));
1865 }
1866 }
1867 }
1868
1869 return nullptr;
1870}
1871
1872/// Fold icmp (and X, Y), C.
1873Instruction *InstCombinerImpl::foldICmpAndConstant(ICmpInst &Cmp,
1874 BinaryOperator *And,
1875 const APInt &C) {
1876 if (Instruction *I = foldICmpAndConstConst(Cmp, And, C1: C))
1877 return I;
1878
1879 const ICmpInst::Predicate Pred = Cmp.getPredicate();
1880 bool TrueIfNeg;
1881 if (isSignBitCheck(Pred, RHS: C, TrueIfSigned&: TrueIfNeg)) {
1882 // ((X - 1) & ~X) < 0 --> X == 0
1883 // ((X - 1) & ~X) >= 0 --> X != 0
1884 Value *X;
1885 if (match(V: And->getOperand(i_nocapture: 0), P: m_Add(L: m_Value(V&: X), R: m_AllOnes())) &&
1886 match(V: And->getOperand(i_nocapture: 1), P: m_Not(V: m_Specific(V: X)))) {
1887 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1888 return new ICmpInst(NewPred, X, ConstantInt::getNullValue(Ty: X->getType()));
1889 }
1890 // (X & -X) < 0 --> X == MinSignedC
1891 // (X & -X) > -1 --> X != MinSignedC
1892 if (match(V: And, P: m_c_And(L: m_Neg(V: m_Value(V&: X)), R: m_Deferred(V: X)))) {
1893 Constant *MinSignedC = ConstantInt::get(
1894 Ty: X->getType(),
1895 V: APInt::getSignedMinValue(numBits: X->getType()->getScalarSizeInBits()));
1896 auto NewPred = TrueIfNeg ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1897 return new ICmpInst(NewPred, X, MinSignedC);
1898 }
1899 }
1900
1901 // TODO: These all require that Y is constant too, so refactor with the above.
1902
1903 // Try to optimize things like "A[i] & 42 == 0" to index computations.
1904 Value *X = And->getOperand(i_nocapture: 0);
1905 Value *Y = And->getOperand(i_nocapture: 1);
1906 if (auto *C2 = dyn_cast<ConstantInt>(Val: Y))
1907 if (auto *LI = dyn_cast<LoadInst>(Val: X))
1908 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: LI->getOperand(i_nocapture: 0)))
1909 if (auto *GV = dyn_cast<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0)))
1910 if (Instruction *Res =
1911 foldCmpLoadFromIndexedGlobal(LI, GEP, GV, ICI&: Cmp, AndCst: C2))
1912 return Res;
1913
1914 if (!Cmp.isEquality())
1915 return nullptr;
1916
1917 // X & -C == -C -> X > u ~C
1918 // X & -C != -C -> X <= u ~C
1919 // iff C is a power of 2
1920 if (Cmp.getOperand(i_nocapture: 1) == Y && C.isNegatedPowerOf2()) {
1921 auto NewPred =
1922 Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT : CmpInst::ICMP_ULE;
1923 return new ICmpInst(NewPred, X, SubOne(C: cast<Constant>(Val: Cmp.getOperand(i_nocapture: 1))));
1924 }
1925
1926 // If we are testing the intersection of 2 select-of-nonzero-constants with no
1927 // common bits set, it's the same as checking if exactly one select condition
1928 // is set:
1929 // ((A ? TC : FC) & (B ? TC : FC)) == 0 --> xor A, B
1930 // ((A ? TC : FC) & (B ? TC : FC)) != 0 --> not(xor A, B)
1931 // TODO: Generalize for non-constant values.
1932 // TODO: Handle signed/unsigned predicates.
1933 // TODO: Handle other bitwise logic connectors.
1934 // TODO: Extend to handle a non-zero compare constant.
1935 if (C.isZero() && (Pred == CmpInst::ICMP_EQ || And->hasOneUse())) {
1936 assert(Cmp.isEquality() && "Not expecting non-equality predicates");
1937 Value *A, *B;
1938 const APInt *TC, *FC;
1939 if (match(V: X, P: m_Select(C: m_Value(V&: A), L: m_APInt(Res&: TC), R: m_APInt(Res&: FC))) &&
1940 match(V: Y,
1941 P: m_Select(C: m_Value(V&: B), L: m_SpecificInt(V: *TC), R: m_SpecificInt(V: *FC))) &&
1942 !TC->isZero() && !FC->isZero() && !TC->intersects(RHS: *FC)) {
1943 Value *R = Builder.CreateXor(LHS: A, RHS: B);
1944 if (Pred == CmpInst::ICMP_NE)
1945 R = Builder.CreateNot(V: R);
1946 return replaceInstUsesWith(I&: Cmp, V: R);
1947 }
1948 }
1949
1950 // ((zext i1 X) & Y) == 0 --> !((trunc Y) & X)
1951 // ((zext i1 X) & Y) != 0 --> ((trunc Y) & X)
1952 // ((zext i1 X) & Y) == 1 --> ((trunc Y) & X)
1953 // ((zext i1 X) & Y) != 1 --> !((trunc Y) & X)
1954 if (match(V: And, P: m_OneUse(SubPattern: m_c_And(L: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X))), R: m_Value(V&: Y)))) &&
1955 X->getType()->isIntOrIntVectorTy(BitWidth: 1) && (C.isZero() || C.isOne())) {
1956 Value *TruncY = Builder.CreateTrunc(V: Y, DestTy: X->getType());
1957 if (C.isZero() ^ (Pred == CmpInst::ICMP_NE)) {
1958 Value *And = Builder.CreateAnd(LHS: TruncY, RHS: X);
1959 return BinaryOperator::CreateNot(Op: And);
1960 }
1961 return BinaryOperator::CreateAnd(V1: TruncY, V2: X);
1962 }
1963
1964 // (icmp eq/ne (and (shl -1, X), Y), 0)
1965 // -> (icmp eq/ne (lshr Y, X), 0)
1966 // We could technically handle any C == 0 or (C < 0 && isOdd(C)) but it seems
1967 // highly unlikely the non-zero case will ever show up in code.
1968 if (C.isZero() &&
1969 match(V: And, P: m_OneUse(SubPattern: m_c_And(L: m_OneUse(SubPattern: m_Shl(L: m_AllOnes(), R: m_Value(V&: X))),
1970 R: m_Value(V&: Y))))) {
1971 Value *LShr = Builder.CreateLShr(LHS: Y, RHS: X);
1972 return new ICmpInst(Pred, LShr, Constant::getNullValue(Ty: LShr->getType()));
1973 }
1974
1975 return nullptr;
1976}
1977
1978/// Fold icmp eq/ne (or (xor/sub (X1, X2), xor/sub (X3, X4))), 0.
1979static Value *foldICmpOrXorSubChain(ICmpInst &Cmp, BinaryOperator *Or,
1980 InstCombiner::BuilderTy &Builder) {
1981 // Are we using xors or subs to bitwise check for a pair or pairs of
1982 // (in)equalities? Convert to a shorter form that has more potential to be
1983 // folded even further.
1984 // ((X1 ^/- X2) || (X3 ^/- X4)) == 0 --> (X1 == X2) && (X3 == X4)
1985 // ((X1 ^/- X2) || (X3 ^/- X4)) != 0 --> (X1 != X2) || (X3 != X4)
1986 // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) == 0 -->
1987 // (X1 == X2) && (X3 == X4) && (X5 == X6)
1988 // ((X1 ^/- X2) || (X3 ^/- X4) || (X5 ^/- X6)) != 0 -->
1989 // (X1 != X2) || (X3 != X4) || (X5 != X6)
1990 SmallVector<std::pair<Value *, Value *>, 2> CmpValues;
1991 SmallVector<Value *, 16> WorkList(1, Or);
1992
1993 while (!WorkList.empty()) {
1994 auto MatchOrOperatorArgument = [&](Value *OrOperatorArgument) {
1995 Value *Lhs, *Rhs;
1996
1997 if (match(V: OrOperatorArgument,
1998 P: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: Lhs), R: m_Value(V&: Rhs))))) {
1999 CmpValues.emplace_back(Args&: Lhs, Args&: Rhs);
2000 return;
2001 }
2002
2003 if (match(V: OrOperatorArgument,
2004 P: m_OneUse(SubPattern: m_Sub(L: m_Value(V&: Lhs), R: m_Value(V&: Rhs))))) {
2005 CmpValues.emplace_back(Args&: Lhs, Args&: Rhs);
2006 return;
2007 }
2008
2009 WorkList.push_back(Elt: OrOperatorArgument);
2010 };
2011
2012 Value *CurrentValue = WorkList.pop_back_val();
2013 Value *OrOperatorLhs, *OrOperatorRhs;
2014
2015 if (!match(V: CurrentValue,
2016 P: m_Or(L: m_Value(V&: OrOperatorLhs), R: m_Value(V&: OrOperatorRhs)))) {
2017 return nullptr;
2018 }
2019
2020 MatchOrOperatorArgument(OrOperatorRhs);
2021 MatchOrOperatorArgument(OrOperatorLhs);
2022 }
2023
2024 ICmpInst::Predicate Pred = Cmp.getPredicate();
2025 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2026 Value *LhsCmp = Builder.CreateICmp(P: Pred, LHS: CmpValues.rbegin()->first,
2027 RHS: CmpValues.rbegin()->second);
2028
2029 for (auto It = CmpValues.rbegin() + 1; It != CmpValues.rend(); ++It) {
2030 Value *RhsCmp = Builder.CreateICmp(P: Pred, LHS: It->first, RHS: It->second);
2031 LhsCmp = Builder.CreateBinOp(Opc: BOpc, LHS: LhsCmp, RHS: RhsCmp);
2032 }
2033
2034 return LhsCmp;
2035}
2036
2037/// Fold icmp (or X, Y), C.
2038Instruction *InstCombinerImpl::foldICmpOrConstant(ICmpInst &Cmp,
2039 BinaryOperator *Or,
2040 const APInt &C) {
2041 ICmpInst::Predicate Pred = Cmp.getPredicate();
2042 if (C.isOne()) {
2043 // icmp slt signum(V) 1 --> icmp slt V, 1
2044 Value *V = nullptr;
2045 if (Pred == ICmpInst::ICMP_SLT && match(V: Or, P: m_Signum(V: m_Value(V))))
2046 return new ICmpInst(ICmpInst::ICMP_SLT, V,
2047 ConstantInt::get(Ty: V->getType(), V: 1));
2048 }
2049
2050 Value *OrOp0 = Or->getOperand(i_nocapture: 0), *OrOp1 = Or->getOperand(i_nocapture: 1);
2051
2052 // (icmp eq/ne (or disjoint x, C0), C1)
2053 // -> (icmp eq/ne x, C0^C1)
2054 if (Cmp.isEquality() && match(V: OrOp1, P: m_ImmConstant()) &&
2055 cast<PossiblyDisjointInst>(Val: Or)->isDisjoint()) {
2056 Value *NewC =
2057 Builder.CreateXor(LHS: OrOp1, RHS: ConstantInt::get(Ty: OrOp1->getType(), V: C));
2058 return new ICmpInst(Pred, OrOp0, NewC);
2059 }
2060
2061 const APInt *MaskC;
2062 if (match(V: OrOp1, P: m_APInt(Res&: MaskC)) && Cmp.isEquality()) {
2063 if (*MaskC == C && (C + 1).isPowerOf2()) {
2064 // X | C == C --> X <=u C
2065 // X | C != C --> X >u C
2066 // iff C+1 is a power of 2 (C is a bitmask of the low bits)
2067 Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT;
2068 return new ICmpInst(Pred, OrOp0, OrOp1);
2069 }
2070
2071 // More general: canonicalize 'equality with set bits mask' to
2072 // 'equality with clear bits mask'.
2073 // (X | MaskC) == C --> (X & ~MaskC) == C ^ MaskC
2074 // (X | MaskC) != C --> (X & ~MaskC) != C ^ MaskC
2075 if (Or->hasOneUse()) {
2076 Value *And = Builder.CreateAnd(LHS: OrOp0, RHS: ~(*MaskC));
2077 Constant *NewC = ConstantInt::get(Ty: Or->getType(), V: C ^ (*MaskC));
2078 return new ICmpInst(Pred, And, NewC);
2079 }
2080 }
2081
2082 // (X | (X-1)) s< 0 --> X s< 1
2083 // (X | (X-1)) s> -1 --> X s> 0
2084 Value *X;
2085 bool TrueIfSigned;
2086 if (isSignBitCheck(Pred, RHS: C, TrueIfSigned) &&
2087 match(V: Or, P: m_c_Or(L: m_Add(L: m_Value(V&: X), R: m_AllOnes()), R: m_Deferred(V: X)))) {
2088 auto NewPred = TrueIfSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_SGT;
2089 Constant *NewC = ConstantInt::get(Ty: X->getType(), V: TrueIfSigned ? 1 : 0);
2090 return new ICmpInst(NewPred, X, NewC);
2091 }
2092
2093 const APInt *OrC;
2094 // icmp(X | OrC, C) --> icmp(X, 0)
2095 if (C.isNonNegative() && match(V: Or, P: m_Or(L: m_Value(V&: X), R: m_APInt(Res&: OrC)))) {
2096 switch (Pred) {
2097 // X | OrC s< C --> X s< 0 iff OrC s>= C s>= 0
2098 case ICmpInst::ICMP_SLT:
2099 // X | OrC s>= C --> X s>= 0 iff OrC s>= C s>= 0
2100 case ICmpInst::ICMP_SGE:
2101 if (OrC->sge(RHS: C))
2102 return new ICmpInst(Pred, X, ConstantInt::getNullValue(Ty: X->getType()));
2103 break;
2104 // X | OrC s<= C --> X s< 0 iff OrC s> C s>= 0
2105 case ICmpInst::ICMP_SLE:
2106 // X | OrC s> C --> X s>= 0 iff OrC s> C s>= 0
2107 case ICmpInst::ICMP_SGT:
2108 if (OrC->sgt(RHS: C))
2109 return new ICmpInst(ICmpInst::getFlippedStrictnessPredicate(pred: Pred), X,
2110 ConstantInt::getNullValue(Ty: X->getType()));
2111 break;
2112 default:
2113 break;
2114 }
2115 }
2116
2117 if (!Cmp.isEquality() || !C.isZero() || !Or->hasOneUse())
2118 return nullptr;
2119
2120 Value *P, *Q;
2121 if (match(V: Or, P: m_Or(L: m_PtrToInt(Op: m_Value(V&: P)), R: m_PtrToInt(Op: m_Value(V&: Q))))) {
2122 // Simplify icmp eq (or (ptrtoint P), (ptrtoint Q)), 0
2123 // -> and (icmp eq P, null), (icmp eq Q, null).
2124 Value *CmpP =
2125 Builder.CreateICmp(P: Pred, LHS: P, RHS: ConstantInt::getNullValue(Ty: P->getType()));
2126 Value *CmpQ =
2127 Builder.CreateICmp(P: Pred, LHS: Q, RHS: ConstantInt::getNullValue(Ty: Q->getType()));
2128 auto BOpc = Pred == CmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2129 return BinaryOperator::Create(Op: BOpc, S1: CmpP, S2: CmpQ);
2130 }
2131
2132 if (Value *V = foldICmpOrXorSubChain(Cmp, Or, Builder))
2133 return replaceInstUsesWith(I&: Cmp, V);
2134
2135 return nullptr;
2136}
2137
2138/// Fold icmp (mul X, Y), C.
2139Instruction *InstCombinerImpl::foldICmpMulConstant(ICmpInst &Cmp,
2140 BinaryOperator *Mul,
2141 const APInt &C) {
2142 ICmpInst::Predicate Pred = Cmp.getPredicate();
2143 Type *MulTy = Mul->getType();
2144 Value *X = Mul->getOperand(i_nocapture: 0);
2145
2146 // If there's no overflow:
2147 // X * X == 0 --> X == 0
2148 // X * X != 0 --> X != 0
2149 if (Cmp.isEquality() && C.isZero() && X == Mul->getOperand(i_nocapture: 1) &&
2150 (Mul->hasNoUnsignedWrap() || Mul->hasNoSignedWrap()))
2151 return new ICmpInst(Pred, X, ConstantInt::getNullValue(Ty: MulTy));
2152
2153 const APInt *MulC;
2154 if (!match(V: Mul->getOperand(i_nocapture: 1), P: m_APInt(Res&: MulC)))
2155 return nullptr;
2156
2157 // If this is a test of the sign bit and the multiply is sign-preserving with
2158 // a constant operand, use the multiply LHS operand instead:
2159 // (X * +MulC) < 0 --> X < 0
2160 // (X * -MulC) < 0 --> X > 0
2161 if (isSignTest(Pred, C) && Mul->hasNoSignedWrap()) {
2162 if (MulC->isNegative())
2163 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
2164 return new ICmpInst(Pred, X, ConstantInt::getNullValue(Ty: MulTy));
2165 }
2166
2167 if (MulC->isZero())
2168 return nullptr;
2169
2170 // If the multiply does not wrap or the constant is odd, try to divide the
2171 // compare constant by the multiplication factor.
2172 if (Cmp.isEquality()) {
2173 // (mul nsw X, MulC) eq/ne C --> X eq/ne C /s MulC
2174 if (Mul->hasNoSignedWrap() && C.srem(RHS: *MulC).isZero()) {
2175 Constant *NewC = ConstantInt::get(Ty: MulTy, V: C.sdiv(RHS: *MulC));
2176 return new ICmpInst(Pred, X, NewC);
2177 }
2178
2179 // C % MulC == 0 is weaker than we could use if MulC is odd because it
2180 // correct to transform if MulC * N == C including overflow. I.e with i8
2181 // (icmp eq (mul X, 5), 101) -> (icmp eq X, 225) but since 101 % 5 != 0, we
2182 // miss that case.
2183 if (C.urem(RHS: *MulC).isZero()) {
2184 // (mul nuw X, MulC) eq/ne C --> X eq/ne C /u MulC
2185 // (mul X, OddC) eq/ne N * C --> X eq/ne N
2186 if ((*MulC & 1).isOne() || Mul->hasNoUnsignedWrap()) {
2187 Constant *NewC = ConstantInt::get(Ty: MulTy, V: C.udiv(RHS: *MulC));
2188 return new ICmpInst(Pred, X, NewC);
2189 }
2190 }
2191 }
2192
2193 // With a matching no-overflow guarantee, fold the constants:
2194 // (X * MulC) < C --> X < (C / MulC)
2195 // (X * MulC) > C --> X > (C / MulC)
2196 // TODO: Assert that Pred is not equal to SGE, SLE, UGE, ULE?
2197 Constant *NewC = nullptr;
2198 if (Mul->hasNoSignedWrap() && ICmpInst::isSigned(predicate: Pred)) {
2199 // MININT / -1 --> overflow.
2200 if (C.isMinSignedValue() && MulC->isAllOnes())
2201 return nullptr;
2202 if (MulC->isNegative())
2203 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
2204
2205 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) {
2206 NewC = ConstantInt::get(
2207 Ty: MulTy, V: APIntOps::RoundingSDiv(A: C, B: *MulC, RM: APInt::Rounding::UP));
2208 } else {
2209 assert((Pred == ICmpInst::ICMP_SLE || Pred == ICmpInst::ICMP_SGT) &&
2210 "Unexpected predicate");
2211 NewC = ConstantInt::get(
2212 Ty: MulTy, V: APIntOps::RoundingSDiv(A: C, B: *MulC, RM: APInt::Rounding::DOWN));
2213 }
2214 } else if (Mul->hasNoUnsignedWrap() && ICmpInst::isUnsigned(predicate: Pred)) {
2215 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) {
2216 NewC = ConstantInt::get(
2217 Ty: MulTy, V: APIntOps::RoundingUDiv(A: C, B: *MulC, RM: APInt::Rounding::UP));
2218 } else {
2219 assert((Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
2220 "Unexpected predicate");
2221 NewC = ConstantInt::get(
2222 Ty: MulTy, V: APIntOps::RoundingUDiv(A: C, B: *MulC, RM: APInt::Rounding::DOWN));
2223 }
2224 }
2225
2226 return NewC ? new ICmpInst(Pred, X, NewC) : nullptr;
2227}
2228
2229/// Fold icmp (shl 1, Y), C.
2230static Instruction *foldICmpShlOne(ICmpInst &Cmp, Instruction *Shl,
2231 const APInt &C) {
2232 Value *Y;
2233 if (!match(V: Shl, P: m_Shl(L: m_One(), R: m_Value(V&: Y))))
2234 return nullptr;
2235
2236 Type *ShiftType = Shl->getType();
2237 unsigned TypeBits = C.getBitWidth();
2238 bool CIsPowerOf2 = C.isPowerOf2();
2239 ICmpInst::Predicate Pred = Cmp.getPredicate();
2240 if (Cmp.isUnsigned()) {
2241 // (1 << Y) pred C -> Y pred Log2(C)
2242 if (!CIsPowerOf2) {
2243 // (1 << Y) < 30 -> Y <= 4
2244 // (1 << Y) <= 30 -> Y <= 4
2245 // (1 << Y) >= 30 -> Y > 4
2246 // (1 << Y) > 30 -> Y > 4
2247 if (Pred == ICmpInst::ICMP_ULT)
2248 Pred = ICmpInst::ICMP_ULE;
2249 else if (Pred == ICmpInst::ICMP_UGE)
2250 Pred = ICmpInst::ICMP_UGT;
2251 }
2252
2253 unsigned CLog2 = C.logBase2();
2254 return new ICmpInst(Pred, Y, ConstantInt::get(Ty: ShiftType, V: CLog2));
2255 } else if (Cmp.isSigned()) {
2256 Constant *BitWidthMinusOne = ConstantInt::get(Ty: ShiftType, V: TypeBits - 1);
2257 // (1 << Y) > 0 -> Y != 31
2258 // (1 << Y) > C -> Y != 31 if C is negative.
2259 if (Pred == ICmpInst::ICMP_SGT && C.sle(RHS: 0))
2260 return new ICmpInst(ICmpInst::ICMP_NE, Y, BitWidthMinusOne);
2261
2262 // (1 << Y) < 0 -> Y == 31
2263 // (1 << Y) < 1 -> Y == 31
2264 // (1 << Y) < C -> Y == 31 if C is negative and not signed min.
2265 // Exclude signed min by subtracting 1 and lower the upper bound to 0.
2266 if (Pred == ICmpInst::ICMP_SLT && (C-1).sle(RHS: 0))
2267 return new ICmpInst(ICmpInst::ICMP_EQ, Y, BitWidthMinusOne);
2268 }
2269
2270 return nullptr;
2271}
2272
2273/// Fold icmp (shl X, Y), C.
2274Instruction *InstCombinerImpl::foldICmpShlConstant(ICmpInst &Cmp,
2275 BinaryOperator *Shl,
2276 const APInt &C) {
2277 const APInt *ShiftVal;
2278 if (Cmp.isEquality() && match(V: Shl->getOperand(i_nocapture: 0), P: m_APInt(Res&: ShiftVal)))
2279 return foldICmpShlConstConst(I&: Cmp, A: Shl->getOperand(i_nocapture: 1), AP1: C, AP2: *ShiftVal);
2280
2281 ICmpInst::Predicate Pred = Cmp.getPredicate();
2282 // (icmp pred (shl nuw&nsw X, Y), Csle0)
2283 // -> (icmp pred X, Csle0)
2284 //
2285 // The idea is the nuw/nsw essentially freeze the sign bit for the shift op
2286 // so X's must be what is used.
2287 if (C.sle(RHS: 0) && Shl->hasNoUnsignedWrap() && Shl->hasNoSignedWrap())
2288 return new ICmpInst(Pred, Shl->getOperand(i_nocapture: 0), Cmp.getOperand(i_nocapture: 1));
2289
2290 // (icmp eq/ne (shl nuw|nsw X, Y), 0)
2291 // -> (icmp eq/ne X, 0)
2292 if (ICmpInst::isEquality(P: Pred) && C.isZero() &&
2293 (Shl->hasNoUnsignedWrap() || Shl->hasNoSignedWrap()))
2294 return new ICmpInst(Pred, Shl->getOperand(i_nocapture: 0), Cmp.getOperand(i_nocapture: 1));
2295
2296 // (icmp slt (shl nsw X, Y), 0/1)
2297 // -> (icmp slt X, 0/1)
2298 // (icmp sgt (shl nsw X, Y), 0/-1)
2299 // -> (icmp sgt X, 0/-1)
2300 //
2301 // NB: sge/sle with a constant will canonicalize to sgt/slt.
2302 if (Shl->hasNoSignedWrap() &&
2303 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT))
2304 if (C.isZero() || (Pred == ICmpInst::ICMP_SGT ? C.isAllOnes() : C.isOne()))
2305 return new ICmpInst(Pred, Shl->getOperand(i_nocapture: 0), Cmp.getOperand(i_nocapture: 1));
2306
2307 const APInt *ShiftAmt;
2308 if (!match(V: Shl->getOperand(i_nocapture: 1), P: m_APInt(Res&: ShiftAmt)))
2309 return foldICmpShlOne(Cmp, Shl, C);
2310
2311 // Check that the shift amount is in range. If not, don't perform undefined
2312 // shifts. When the shift is visited, it will be simplified.
2313 unsigned TypeBits = C.getBitWidth();
2314 if (ShiftAmt->uge(RHS: TypeBits))
2315 return nullptr;
2316
2317 Value *X = Shl->getOperand(i_nocapture: 0);
2318 Type *ShType = Shl->getType();
2319
2320 // NSW guarantees that we are only shifting out sign bits from the high bits,
2321 // so we can ASHR the compare constant without needing a mask and eliminate
2322 // the shift.
2323 if (Shl->hasNoSignedWrap()) {
2324 if (Pred == ICmpInst::ICMP_SGT) {
2325 // icmp Pred (shl nsw X, ShiftAmt), C --> icmp Pred X, (C >>s ShiftAmt)
2326 APInt ShiftedC = C.ashr(ShiftAmt: *ShiftAmt);
2327 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC));
2328 }
2329 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2330 C.ashr(ShiftAmt: *ShiftAmt).shl(ShiftAmt: *ShiftAmt) == C) {
2331 APInt ShiftedC = C.ashr(ShiftAmt: *ShiftAmt);
2332 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC));
2333 }
2334 if (Pred == ICmpInst::ICMP_SLT) {
2335 // SLE is the same as above, but SLE is canonicalized to SLT, so convert:
2336 // (X << S) <=s C is equiv to X <=s (C >> S) for all C
2337 // (X << S) <s (C + 1) is equiv to X <s (C >> S) + 1 if C <s SMAX
2338 // (X << S) <s C is equiv to X <s ((C - 1) >> S) + 1 if C >s SMIN
2339 assert(!C.isMinSignedValue() && "Unexpected icmp slt");
2340 APInt ShiftedC = (C - 1).ashr(ShiftAmt: *ShiftAmt) + 1;
2341 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC));
2342 }
2343 }
2344
2345 // NUW guarantees that we are only shifting out zero bits from the high bits,
2346 // so we can LSHR the compare constant without needing a mask and eliminate
2347 // the shift.
2348 if (Shl->hasNoUnsignedWrap()) {
2349 if (Pred == ICmpInst::ICMP_UGT) {
2350 // icmp Pred (shl nuw X, ShiftAmt), C --> icmp Pred X, (C >>u ShiftAmt)
2351 APInt ShiftedC = C.lshr(ShiftAmt: *ShiftAmt);
2352 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC));
2353 }
2354 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2355 C.lshr(ShiftAmt: *ShiftAmt).shl(ShiftAmt: *ShiftAmt) == C) {
2356 APInt ShiftedC = C.lshr(ShiftAmt: *ShiftAmt);
2357 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC));
2358 }
2359 if (Pred == ICmpInst::ICMP_ULT) {
2360 // ULE is the same as above, but ULE is canonicalized to ULT, so convert:
2361 // (X << S) <=u C is equiv to X <=u (C >> S) for all C
2362 // (X << S) <u (C + 1) is equiv to X <u (C >> S) + 1 if C <u ~0u
2363 // (X << S) <u C is equiv to X <u ((C - 1) >> S) + 1 if C >u 0
2364 assert(C.ugt(0) && "ult 0 should have been eliminated");
2365 APInt ShiftedC = (C - 1).lshr(ShiftAmt: *ShiftAmt) + 1;
2366 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShType, V: ShiftedC));
2367 }
2368 }
2369
2370 if (Cmp.isEquality() && Shl->hasOneUse()) {
2371 // Strength-reduce the shift into an 'and'.
2372 Constant *Mask = ConstantInt::get(
2373 Ty: ShType,
2374 V: APInt::getLowBitsSet(numBits: TypeBits, loBitsSet: TypeBits - ShiftAmt->getZExtValue()));
2375 Value *And = Builder.CreateAnd(LHS: X, RHS: Mask, Name: Shl->getName() + ".mask");
2376 Constant *LShrC = ConstantInt::get(Ty: ShType, V: C.lshr(ShiftAmt: *ShiftAmt));
2377 return new ICmpInst(Pred, And, LShrC);
2378 }
2379
2380 // Otherwise, if this is a comparison of the sign bit, simplify to and/test.
2381 bool TrueIfSigned = false;
2382 if (Shl->hasOneUse() && isSignBitCheck(Pred, RHS: C, TrueIfSigned)) {
2383 // (X << 31) <s 0 --> (X & 1) != 0
2384 Constant *Mask = ConstantInt::get(
2385 Ty: ShType,
2386 V: APInt::getOneBitSet(numBits: TypeBits, BitNo: TypeBits - ShiftAmt->getZExtValue() - 1));
2387 Value *And = Builder.CreateAnd(LHS: X, RHS: Mask, Name: Shl->getName() + ".mask");
2388 return new ICmpInst(TrueIfSigned ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ,
2389 And, Constant::getNullValue(Ty: ShType));
2390 }
2391
2392 // Simplify 'shl' inequality test into 'and' equality test.
2393 if (Cmp.isUnsigned() && Shl->hasOneUse()) {
2394 // (X l<< C2) u<=/u> C1 iff C1+1 is power of two -> X & (~C1 l>> C2) ==/!= 0
2395 if ((C + 1).isPowerOf2() &&
2396 (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT)) {
2397 Value *And = Builder.CreateAnd(LHS: X, RHS: (~C).lshr(shiftAmt: ShiftAmt->getZExtValue()));
2398 return new ICmpInst(Pred == ICmpInst::ICMP_ULE ? ICmpInst::ICMP_EQ
2399 : ICmpInst::ICMP_NE,
2400 And, Constant::getNullValue(Ty: ShType));
2401 }
2402 // (X l<< C2) u</u>= C1 iff C1 is power of two -> X & (-C1 l>> C2) ==/!= 0
2403 if (C.isPowerOf2() &&
2404 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
2405 Value *And =
2406 Builder.CreateAnd(LHS: X, RHS: (~(C - 1)).lshr(shiftAmt: ShiftAmt->getZExtValue()));
2407 return new ICmpInst(Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_EQ
2408 : ICmpInst::ICMP_NE,
2409 And, Constant::getNullValue(Ty: ShType));
2410 }
2411 }
2412
2413 // Transform (icmp pred iM (shl iM %v, N), C)
2414 // -> (icmp pred i(M-N) (trunc %v iM to i(M-N)), (trunc (C>>N))
2415 // Transform the shl to a trunc if (trunc (C>>N)) has no loss and M-N.
2416 // This enables us to get rid of the shift in favor of a trunc that may be
2417 // free on the target. It has the additional benefit of comparing to a
2418 // smaller constant that may be more target-friendly.
2419 unsigned Amt = ShiftAmt->getLimitedValue(Limit: TypeBits - 1);
2420 if (Shl->hasOneUse() && Amt != 0 &&
2421 shouldChangeType(FromBitWidth: ShType->getScalarSizeInBits(), ToBitWidth: TypeBits - Amt)) {
2422 ICmpInst::Predicate CmpPred = Pred;
2423 APInt RHSC = C;
2424
2425 if (RHSC.countr_zero() < Amt && ICmpInst::isStrictPredicate(predicate: CmpPred)) {
2426 // Try the flipped strictness predicate.
2427 // e.g.:
2428 // icmp ult i64 (shl X, 32), 8589934593 ->
2429 // icmp ule i64 (shl X, 32), 8589934592 ->
2430 // icmp ule i32 (trunc X, i32), 2 ->
2431 // icmp ult i32 (trunc X, i32), 3
2432 if (auto FlippedStrictness =
2433 InstCombiner::getFlippedStrictnessPredicateAndConstant(
2434 Pred, C: ConstantInt::get(Context&: ShType->getContext(), V: C))) {
2435 CmpPred = FlippedStrictness->first;
2436 RHSC = cast<ConstantInt>(Val: FlippedStrictness->second)->getValue();
2437 }
2438 }
2439
2440 if (RHSC.countr_zero() >= Amt) {
2441 Type *TruncTy = ShType->getWithNewBitWidth(NewBitWidth: TypeBits - Amt);
2442 Constant *NewC =
2443 ConstantInt::get(Ty: TruncTy, V: RHSC.ashr(ShiftAmt: *ShiftAmt).trunc(width: TypeBits - Amt));
2444 return new ICmpInst(CmpPred,
2445 Builder.CreateTrunc(V: X, DestTy: TruncTy, Name: "", /*IsNUW=*/false,
2446 IsNSW: Shl->hasNoSignedWrap()),
2447 NewC);
2448 }
2449 }
2450
2451 return nullptr;
2452}
2453
2454/// Fold icmp ({al}shr X, Y), C.
2455Instruction *InstCombinerImpl::foldICmpShrConstant(ICmpInst &Cmp,
2456 BinaryOperator *Shr,
2457 const APInt &C) {
2458 // An exact shr only shifts out zero bits, so:
2459 // icmp eq/ne (shr X, Y), 0 --> icmp eq/ne X, 0
2460 Value *X = Shr->getOperand(i_nocapture: 0);
2461 CmpInst::Predicate Pred = Cmp.getPredicate();
2462 if (Cmp.isEquality() && Shr->isExact() && C.isZero())
2463 return new ICmpInst(Pred, X, Cmp.getOperand(i_nocapture: 1));
2464
2465 bool IsAShr = Shr->getOpcode() == Instruction::AShr;
2466 const APInt *ShiftValC;
2467 if (match(V: X, P: m_APInt(Res&: ShiftValC))) {
2468 if (Cmp.isEquality())
2469 return foldICmpShrConstConst(I&: Cmp, A: Shr->getOperand(i_nocapture: 1), AP1: C, AP2: *ShiftValC);
2470
2471 // (ShiftValC >> Y) >s -1 --> Y != 0 with ShiftValC < 0
2472 // (ShiftValC >> Y) <s 0 --> Y == 0 with ShiftValC < 0
2473 bool TrueIfSigned;
2474 if (!IsAShr && ShiftValC->isNegative() &&
2475 isSignBitCheck(Pred, RHS: C, TrueIfSigned))
2476 return new ICmpInst(TrueIfSigned ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE,
2477 Shr->getOperand(i_nocapture: 1),
2478 ConstantInt::getNullValue(Ty: X->getType()));
2479
2480 // If the shifted constant is a power-of-2, test the shift amount directly:
2481 // (ShiftValC >> Y) >u C --> X <u (LZ(C) - LZ(ShiftValC))
2482 // (ShiftValC >> Y) <u C --> X >=u (LZ(C-1) - LZ(ShiftValC))
2483 if (!IsAShr && ShiftValC->isPowerOf2() &&
2484 (Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_ULT)) {
2485 bool IsUGT = Pred == CmpInst::ICMP_UGT;
2486 assert(ShiftValC->uge(C) && "Expected simplify of compare");
2487 assert((IsUGT || !C.isZero()) && "Expected X u< 0 to simplify");
2488
2489 unsigned CmpLZ = IsUGT ? C.countl_zero() : (C - 1).countl_zero();
2490 unsigned ShiftLZ = ShiftValC->countl_zero();
2491 Constant *NewC = ConstantInt::get(Ty: Shr->getType(), V: CmpLZ - ShiftLZ);
2492 auto NewPred = IsUGT ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;
2493 return new ICmpInst(NewPred, Shr->getOperand(i_nocapture: 1), NewC);
2494 }
2495 }
2496
2497 const APInt *ShiftAmtC;
2498 if (!match(V: Shr->getOperand(i_nocapture: 1), P: m_APInt(Res&: ShiftAmtC)))
2499 return nullptr;
2500
2501 // Check that the shift amount is in range. If not, don't perform undefined
2502 // shifts. When the shift is visited it will be simplified.
2503 unsigned TypeBits = C.getBitWidth();
2504 unsigned ShAmtVal = ShiftAmtC->getLimitedValue(Limit: TypeBits);
2505 if (ShAmtVal >= TypeBits || ShAmtVal == 0)
2506 return nullptr;
2507
2508 bool IsExact = Shr->isExact();
2509 Type *ShrTy = Shr->getType();
2510 // TODO: If we could guarantee that InstSimplify would handle all of the
2511 // constant-value-based preconditions in the folds below, then we could assert
2512 // those conditions rather than checking them. This is difficult because of
2513 // undef/poison (PR34838).
2514 if (IsAShr && Shr->hasOneUse()) {
2515 if (IsExact && (Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) &&
2516 (C - 1).isPowerOf2() && C.countLeadingZeros() > ShAmtVal) {
2517 // When C - 1 is a power of two and the transform can be legally
2518 // performed, prefer this form so the produced constant is close to a
2519 // power of two.
2520 // icmp slt/ult (ashr exact X, ShAmtC), C
2521 // --> icmp slt/ult X, (C - 1) << ShAmtC) + 1
2522 APInt ShiftedC = (C - 1).shl(shiftAmt: ShAmtVal) + 1;
2523 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC));
2524 }
2525 if (IsExact || Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_ULT) {
2526 // When ShAmtC can be shifted losslessly:
2527 // icmp PRED (ashr exact X, ShAmtC), C --> icmp PRED X, (C << ShAmtC)
2528 // icmp slt/ult (ashr X, ShAmtC), C --> icmp slt/ult X, (C << ShAmtC)
2529 APInt ShiftedC = C.shl(shiftAmt: ShAmtVal);
2530 if (ShiftedC.ashr(ShiftAmt: ShAmtVal) == C)
2531 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC));
2532 }
2533 if (Pred == CmpInst::ICMP_SGT) {
2534 // icmp sgt (ashr X, ShAmtC), C --> icmp sgt X, ((C + 1) << ShAmtC) - 1
2535 APInt ShiftedC = (C + 1).shl(shiftAmt: ShAmtVal) - 1;
2536 if (!C.isMaxSignedValue() && !(C + 1).shl(shiftAmt: ShAmtVal).isMinSignedValue() &&
2537 (ShiftedC + 1).ashr(ShiftAmt: ShAmtVal) == (C + 1))
2538 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC));
2539 }
2540 if (Pred == CmpInst::ICMP_UGT) {
2541 // icmp ugt (ashr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2542 // 'C + 1 << ShAmtC' can overflow as a signed number, so the 2nd
2543 // clause accounts for that pattern.
2544 APInt ShiftedC = (C + 1).shl(shiftAmt: ShAmtVal) - 1;
2545 if ((ShiftedC + 1).ashr(ShiftAmt: ShAmtVal) == (C + 1) ||
2546 (C + 1).shl(shiftAmt: ShAmtVal).isMinSignedValue())
2547 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC));
2548 }
2549
2550 // If the compare constant has significant bits above the lowest sign-bit,
2551 // then convert an unsigned cmp to a test of the sign-bit:
2552 // (ashr X, ShiftC) u> C --> X s< 0
2553 // (ashr X, ShiftC) u< C --> X s> -1
2554 if (C.getBitWidth() > 2 && C.getNumSignBits() <= ShAmtVal) {
2555 if (Pred == CmpInst::ICMP_UGT) {
2556 return new ICmpInst(CmpInst::ICMP_SLT, X,
2557 ConstantInt::getNullValue(Ty: ShrTy));
2558 }
2559 if (Pred == CmpInst::ICMP_ULT) {
2560 return new ICmpInst(CmpInst::ICMP_SGT, X,
2561 ConstantInt::getAllOnesValue(Ty: ShrTy));
2562 }
2563 }
2564 } else if (!IsAShr) {
2565 if (Pred == CmpInst::ICMP_ULT || (Pred == CmpInst::ICMP_UGT && IsExact)) {
2566 // icmp ult (lshr X, ShAmtC), C --> icmp ult X, (C << ShAmtC)
2567 // icmp ugt (lshr exact X, ShAmtC), C --> icmp ugt X, (C << ShAmtC)
2568 APInt ShiftedC = C.shl(shiftAmt: ShAmtVal);
2569 if (ShiftedC.lshr(shiftAmt: ShAmtVal) == C)
2570 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC));
2571 }
2572 if (Pred == CmpInst::ICMP_UGT) {
2573 // icmp ugt (lshr X, ShAmtC), C --> icmp ugt X, ((C + 1) << ShAmtC) - 1
2574 APInt ShiftedC = (C + 1).shl(shiftAmt: ShAmtVal) - 1;
2575 if ((ShiftedC + 1).lshr(shiftAmt: ShAmtVal) == (C + 1))
2576 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: ShiftedC));
2577 }
2578 }
2579
2580 if (!Cmp.isEquality())
2581 return nullptr;
2582
2583 // Handle equality comparisons of shift-by-constant.
2584
2585 // If the comparison constant changes with the shift, the comparison cannot
2586 // succeed (bits of the comparison constant cannot match the shifted value).
2587 // This should be known by InstSimplify and already be folded to true/false.
2588 assert(((IsAShr && C.shl(ShAmtVal).ashr(ShAmtVal) == C) ||
2589 (!IsAShr && C.shl(ShAmtVal).lshr(ShAmtVal) == C)) &&
2590 "Expected icmp+shr simplify did not occur.");
2591
2592 // If the bits shifted out are known zero, compare the unshifted value:
2593 // (X & 4) >> 1 == 2 --> (X & 4) == 4.
2594 if (Shr->isExact())
2595 return new ICmpInst(Pred, X, ConstantInt::get(Ty: ShrTy, V: C << ShAmtVal));
2596
2597 if (C.isZero()) {
2598 // == 0 is u< 1.
2599 if (Pred == CmpInst::ICMP_EQ)
2600 return new ICmpInst(CmpInst::ICMP_ULT, X,
2601 ConstantInt::get(Ty: ShrTy, V: (C + 1).shl(shiftAmt: ShAmtVal)));
2602 else
2603 return new ICmpInst(CmpInst::ICMP_UGT, X,
2604 ConstantInt::get(Ty: ShrTy, V: (C + 1).shl(shiftAmt: ShAmtVal) - 1));
2605 }
2606
2607 if (Shr->hasOneUse()) {
2608 // Canonicalize the shift into an 'and':
2609 // icmp eq/ne (shr X, ShAmt), C --> icmp eq/ne (and X, HiMask), (C << ShAmt)
2610 APInt Val(APInt::getHighBitsSet(numBits: TypeBits, hiBitsSet: TypeBits - ShAmtVal));
2611 Constant *Mask = ConstantInt::get(Ty: ShrTy, V: Val);
2612 Value *And = Builder.CreateAnd(LHS: X, RHS: Mask, Name: Shr->getName() + ".mask");
2613 return new ICmpInst(Pred, And, ConstantInt::get(Ty: ShrTy, V: C << ShAmtVal));
2614 }
2615
2616 return nullptr;
2617}
2618
2619Instruction *InstCombinerImpl::foldICmpSRemConstant(ICmpInst &Cmp,
2620 BinaryOperator *SRem,
2621 const APInt &C) {
2622 // Match an 'is positive' or 'is negative' comparison of remainder by a
2623 // constant power-of-2 value:
2624 // (X % pow2C) sgt/slt 0
2625 const ICmpInst::Predicate Pred = Cmp.getPredicate();
2626 if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_SLT &&
2627 Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
2628 return nullptr;
2629
2630 // TODO: The one-use check is standard because we do not typically want to
2631 // create longer instruction sequences, but this might be a special-case
2632 // because srem is not good for analysis or codegen.
2633 if (!SRem->hasOneUse())
2634 return nullptr;
2635
2636 const APInt *DivisorC;
2637 if (!match(V: SRem->getOperand(i_nocapture: 1), P: m_Power2(V&: DivisorC)))
2638 return nullptr;
2639
2640 // For cmp_sgt/cmp_slt only zero valued C is handled.
2641 // For cmp_eq/cmp_ne only positive valued C is handled.
2642 if (((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT) &&
2643 !C.isZero()) ||
2644 ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE) &&
2645 !C.isStrictlyPositive()))
2646 return nullptr;
2647
2648 // Mask off the sign bit and the modulo bits (low-bits).
2649 Type *Ty = SRem->getType();
2650 APInt SignMask = APInt::getSignMask(BitWidth: Ty->getScalarSizeInBits());
2651 Constant *MaskC = ConstantInt::get(Ty, V: SignMask | (*DivisorC - 1));
2652 Value *And = Builder.CreateAnd(LHS: SRem->getOperand(i_nocapture: 0), RHS: MaskC);
2653
2654 if (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE)
2655 return new ICmpInst(Pred, And, ConstantInt::get(Ty, V: C));
2656
2657 // For 'is positive?' check that the sign-bit is clear and at least 1 masked
2658 // bit is set. Example:
2659 // (i8 X % 32) s> 0 --> (X & 159) s> 0
2660 if (Pred == ICmpInst::ICMP_SGT)
2661 return new ICmpInst(ICmpInst::ICMP_SGT, And, ConstantInt::getNullValue(Ty));
2662
2663 // For 'is negative?' check that the sign-bit is set and at least 1 masked
2664 // bit is set. Example:
2665 // (i16 X % 4) s< 0 --> (X & 32771) u> 32768
2666 return new ICmpInst(ICmpInst::ICMP_UGT, And, ConstantInt::get(Ty, V: SignMask));
2667}
2668
2669/// Fold icmp (udiv X, Y), C.
2670Instruction *InstCombinerImpl::foldICmpUDivConstant(ICmpInst &Cmp,
2671 BinaryOperator *UDiv,
2672 const APInt &C) {
2673 ICmpInst::Predicate Pred = Cmp.getPredicate();
2674 Value *X = UDiv->getOperand(i_nocapture: 0);
2675 Value *Y = UDiv->getOperand(i_nocapture: 1);
2676 Type *Ty = UDiv->getType();
2677
2678 const APInt *C2;
2679 if (!match(V: X, P: m_APInt(Res&: C2)))
2680 return nullptr;
2681
2682 assert(*C2 != 0 && "udiv 0, X should have been simplified already.");
2683
2684 // (icmp ugt (udiv C2, Y), C) -> (icmp ule Y, C2/(C+1))
2685 if (Pred == ICmpInst::ICMP_UGT) {
2686 assert(!C.isMaxValue() &&
2687 "icmp ugt X, UINT_MAX should have been simplified already.");
2688 return new ICmpInst(ICmpInst::ICMP_ULE, Y,
2689 ConstantInt::get(Ty, V: C2->udiv(RHS: C + 1)));
2690 }
2691
2692 // (icmp ult (udiv C2, Y), C) -> (icmp ugt Y, C2/C)
2693 if (Pred == ICmpInst::ICMP_ULT) {
2694 assert(C != 0 && "icmp ult X, 0 should have been simplified already.");
2695 return new ICmpInst(ICmpInst::ICMP_UGT, Y,
2696 ConstantInt::get(Ty, V: C2->udiv(RHS: C)));
2697 }
2698
2699 return nullptr;
2700}
2701
2702/// Fold icmp ({su}div X, Y), C.
2703Instruction *InstCombinerImpl::foldICmpDivConstant(ICmpInst &Cmp,
2704 BinaryOperator *Div,
2705 const APInt &C) {
2706 ICmpInst::Predicate Pred = Cmp.getPredicate();
2707 Value *X = Div->getOperand(i_nocapture: 0);
2708 Value *Y = Div->getOperand(i_nocapture: 1);
2709 Type *Ty = Div->getType();
2710 bool DivIsSigned = Div->getOpcode() == Instruction::SDiv;
2711
2712 // If unsigned division and the compare constant is bigger than
2713 // UMAX/2 (negative), there's only one pair of values that satisfies an
2714 // equality check, so eliminate the division:
2715 // (X u/ Y) == C --> (X == C) && (Y == 1)
2716 // (X u/ Y) != C --> (X != C) || (Y != 1)
2717 // Similarly, if signed division and the compare constant is exactly SMIN:
2718 // (X s/ Y) == SMIN --> (X == SMIN) && (Y == 1)
2719 // (X s/ Y) != SMIN --> (X != SMIN) || (Y != 1)
2720 if (Cmp.isEquality() && Div->hasOneUse() && C.isSignBitSet() &&
2721 (!DivIsSigned || C.isMinSignedValue())) {
2722 Value *XBig = Builder.CreateICmp(P: Pred, LHS: X, RHS: ConstantInt::get(Ty, V: C));
2723 Value *YOne = Builder.CreateICmp(P: Pred, LHS: Y, RHS: ConstantInt::get(Ty, V: 1));
2724 auto Logic = Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
2725 return BinaryOperator::Create(Op: Logic, S1: XBig, S2: YOne);
2726 }
2727
2728 // Fold: icmp pred ([us]div X, C2), C -> range test
2729 // Fold this div into the comparison, producing a range check.
2730 // Determine, based on the divide type, what the range is being
2731 // checked. If there is an overflow on the low or high side, remember
2732 // it, otherwise compute the range [low, hi) bounding the new value.
2733 // See: InsertRangeTest above for the kinds of replacements possible.
2734 const APInt *C2;
2735 if (!match(V: Y, P: m_APInt(Res&: C2)))
2736 return nullptr;
2737
2738 // FIXME: If the operand types don't match the type of the divide
2739 // then don't attempt this transform. The code below doesn't have the
2740 // logic to deal with a signed divide and an unsigned compare (and
2741 // vice versa). This is because (x /s C2) <s C produces different
2742 // results than (x /s C2) <u C or (x /u C2) <s C or even
2743 // (x /u C2) <u C. Simply casting the operands and result won't
2744 // work. :( The if statement below tests that condition and bails
2745 // if it finds it.
2746 if (!Cmp.isEquality() && DivIsSigned != Cmp.isSigned())
2747 return nullptr;
2748
2749 // The ProdOV computation fails on divide by 0 and divide by -1. Cases with
2750 // INT_MIN will also fail if the divisor is 1. Although folds of all these
2751 // division-by-constant cases should be present, we can not assert that they
2752 // have happened before we reach this icmp instruction.
2753 if (C2->isZero() || C2->isOne() || (DivIsSigned && C2->isAllOnes()))
2754 return nullptr;
2755
2756 // Compute Prod = C * C2. We are essentially solving an equation of
2757 // form X / C2 = C. We solve for X by multiplying C2 and C.
2758 // By solving for X, we can turn this into a range check instead of computing
2759 // a divide.
2760 APInt Prod = C * *C2;
2761
2762 // Determine if the product overflows by seeing if the product is not equal to
2763 // the divide. Make sure we do the same kind of divide as in the LHS
2764 // instruction that we're folding.
2765 bool ProdOV = (DivIsSigned ? Prod.sdiv(RHS: *C2) : Prod.udiv(RHS: *C2)) != C;
2766
2767 // If the division is known to be exact, then there is no remainder from the
2768 // divide, so the covered range size is unit, otherwise it is the divisor.
2769 APInt RangeSize = Div->isExact() ? APInt(C2->getBitWidth(), 1) : *C2;
2770
2771 // Figure out the interval that is being checked. For example, a comparison
2772 // like "X /u 5 == 0" is really checking that X is in the interval [0, 5).
2773 // Compute this interval based on the constants involved and the signedness of
2774 // the compare/divide. This computes a half-open interval, keeping track of
2775 // whether either value in the interval overflows. After analysis each
2776 // overflow variable is set to 0 if it's corresponding bound variable is valid
2777 // -1 if overflowed off the bottom end, or +1 if overflowed off the top end.
2778 int LoOverflow = 0, HiOverflow = 0;
2779 APInt LoBound, HiBound;
2780
2781 if (!DivIsSigned) { // udiv
2782 // e.g. X/5 op 3 --> [15, 20)
2783 LoBound = Prod;
2784 HiOverflow = LoOverflow = ProdOV;
2785 if (!HiOverflow) {
2786 // If this is not an exact divide, then many values in the range collapse
2787 // to the same result value.
2788 HiOverflow = addWithOverflow(Result&: HiBound, In1: LoBound, In2: RangeSize, IsSigned: false);
2789 }
2790 } else if (C2->isStrictlyPositive()) { // Divisor is > 0.
2791 if (C.isZero()) { // (X / pos) op 0
2792 // Can't overflow. e.g. X/2 op 0 --> [-1, 2)
2793 LoBound = -(RangeSize - 1);
2794 HiBound = RangeSize;
2795 } else if (C.isStrictlyPositive()) { // (X / pos) op pos
2796 LoBound = Prod; // e.g. X/5 op 3 --> [15, 20)
2797 HiOverflow = LoOverflow = ProdOV;
2798 if (!HiOverflow)
2799 HiOverflow = addWithOverflow(Result&: HiBound, In1: Prod, In2: RangeSize, IsSigned: true);
2800 } else { // (X / pos) op neg
2801 // e.g. X/5 op -3 --> [-15-4, -15+1) --> [-19, -14)
2802 HiBound = Prod + 1;
2803 LoOverflow = HiOverflow = ProdOV ? -1 : 0;
2804 if (!LoOverflow) {
2805 APInt DivNeg = -RangeSize;
2806 LoOverflow = addWithOverflow(Result&: LoBound, In1: HiBound, In2: DivNeg, IsSigned: true) ? -1 : 0;
2807 }
2808 }
2809 } else if (C2->isNegative()) { // Divisor is < 0.
2810 if (Div->isExact())
2811 RangeSize.negate();
2812 if (C.isZero()) { // (X / neg) op 0
2813 // e.g. X/-5 op 0 --> [-4, 5)
2814 LoBound = RangeSize + 1;
2815 HiBound = -RangeSize;
2816 if (HiBound == *C2) { // -INTMIN = INTMIN
2817 HiOverflow = 1; // [INTMIN+1, overflow)
2818 HiBound = APInt(); // e.g. X/INTMIN = 0 --> X > INTMIN
2819 }
2820 } else if (C.isStrictlyPositive()) { // (X / neg) op pos
2821 // e.g. X/-5 op 3 --> [-19, -14)
2822 HiBound = Prod + 1;
2823 HiOverflow = LoOverflow = ProdOV ? -1 : 0;
2824 if (!LoOverflow)
2825 LoOverflow =
2826 addWithOverflow(Result&: LoBound, In1: HiBound, In2: RangeSize, IsSigned: true) ? -1 : 0;
2827 } else { // (X / neg) op neg
2828 LoBound = Prod; // e.g. X/-5 op -3 --> [15, 20)
2829 LoOverflow = HiOverflow = ProdOV;
2830 if (!HiOverflow)
2831 HiOverflow = subWithOverflow(Result&: HiBound, In1: Prod, In2: RangeSize, IsSigned: true);
2832 }
2833
2834 // Dividing by a negative swaps the condition. LT <-> GT
2835 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
2836 }
2837
2838 switch (Pred) {
2839 default:
2840 llvm_unreachable("Unhandled icmp predicate!");
2841 case ICmpInst::ICMP_EQ:
2842 if (LoOverflow && HiOverflow)
2843 return replaceInstUsesWith(I&: Cmp, V: Builder.getFalse());
2844 if (HiOverflow)
2845 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
2846 X, ConstantInt::get(Ty, V: LoBound));
2847 if (LoOverflow)
2848 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
2849 X, ConstantInt::get(Ty, V: HiBound));
2850 return replaceInstUsesWith(
2851 I&: Cmp, V: insertRangeTest(V: X, Lo: LoBound, Hi: HiBound, isSigned: DivIsSigned, Inside: true));
2852 case ICmpInst::ICMP_NE:
2853 if (LoOverflow && HiOverflow)
2854 return replaceInstUsesWith(I&: Cmp, V: Builder.getTrue());
2855 if (HiOverflow)
2856 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT,
2857 X, ConstantInt::get(Ty, V: LoBound));
2858 if (LoOverflow)
2859 return new ICmpInst(DivIsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
2860 X, ConstantInt::get(Ty, V: HiBound));
2861 return replaceInstUsesWith(
2862 I&: Cmp, V: insertRangeTest(V: X, Lo: LoBound, Hi: HiBound, isSigned: DivIsSigned, Inside: false));
2863 case ICmpInst::ICMP_ULT:
2864 case ICmpInst::ICMP_SLT:
2865 if (LoOverflow == +1) // Low bound is greater than input range.
2866 return replaceInstUsesWith(I&: Cmp, V: Builder.getTrue());
2867 if (LoOverflow == -1) // Low bound is less than input range.
2868 return replaceInstUsesWith(I&: Cmp, V: Builder.getFalse());
2869 return new ICmpInst(Pred, X, ConstantInt::get(Ty, V: LoBound));
2870 case ICmpInst::ICMP_UGT:
2871 case ICmpInst::ICMP_SGT:
2872 if (HiOverflow == +1) // High bound greater than input range.
2873 return replaceInstUsesWith(I&: Cmp, V: Builder.getFalse());
2874 if (HiOverflow == -1) // High bound less than input range.
2875 return replaceInstUsesWith(I&: Cmp, V: Builder.getTrue());
2876 if (Pred == ICmpInst::ICMP_UGT)
2877 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, V: HiBound));
2878 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, V: HiBound));
2879 }
2880
2881 return nullptr;
2882}
2883
2884/// Fold icmp (sub X, Y), C.
2885Instruction *InstCombinerImpl::foldICmpSubConstant(ICmpInst &Cmp,
2886 BinaryOperator *Sub,
2887 const APInt &C) {
2888 Value *X = Sub->getOperand(i_nocapture: 0), *Y = Sub->getOperand(i_nocapture: 1);
2889 ICmpInst::Predicate Pred = Cmp.getPredicate();
2890 Type *Ty = Sub->getType();
2891
2892 // (SubC - Y) == C) --> Y == (SubC - C)
2893 // (SubC - Y) != C) --> Y != (SubC - C)
2894 Constant *SubC;
2895 if (Cmp.isEquality() && match(V: X, P: m_ImmConstant(C&: SubC))) {
2896 return new ICmpInst(Pred, Y,
2897 ConstantExpr::getSub(C1: SubC, C2: ConstantInt::get(Ty, V: C)));
2898 }
2899
2900 // (icmp P (sub nuw|nsw C2, Y), C) -> (icmp swap(P) Y, C2-C)
2901 const APInt *C2;
2902 APInt SubResult;
2903 ICmpInst::Predicate SwappedPred = Cmp.getSwappedPredicate();
2904 bool HasNSW = Sub->hasNoSignedWrap();
2905 bool HasNUW = Sub->hasNoUnsignedWrap();
2906 if (match(V: X, P: m_APInt(Res&: C2)) &&
2907 ((Cmp.isUnsigned() && HasNUW) || (Cmp.isSigned() && HasNSW)) &&
2908 !subWithOverflow(Result&: SubResult, In1: *C2, In2: C, IsSigned: Cmp.isSigned()))
2909 return new ICmpInst(SwappedPred, Y, ConstantInt::get(Ty, V: SubResult));
2910
2911 // X - Y == 0 --> X == Y.
2912 // X - Y != 0 --> X != Y.
2913 // TODO: We allow this with multiple uses as long as the other uses are not
2914 // in phis. The phi use check is guarding against a codegen regression
2915 // for a loop test. If the backend could undo this (and possibly
2916 // subsequent transforms), we would not need this hack.
2917 if (Cmp.isEquality() && C.isZero() &&
2918 none_of(Range: (Sub->users()), P: [](const User *U) { return isa<PHINode>(Val: U); }))
2919 return new ICmpInst(Pred, X, Y);
2920
2921 // The following transforms are only worth it if the only user of the subtract
2922 // is the icmp.
2923 // TODO: This is an artificial restriction for all of the transforms below
2924 // that only need a single replacement icmp. Can these use the phi test
2925 // like the transform above here?
2926 if (!Sub->hasOneUse())
2927 return nullptr;
2928
2929 if (Sub->hasNoSignedWrap()) {
2930 // (icmp sgt (sub nsw X, Y), -1) -> (icmp sge X, Y)
2931 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
2932 return new ICmpInst(ICmpInst::ICMP_SGE, X, Y);
2933
2934 // (icmp sgt (sub nsw X, Y), 0) -> (icmp sgt X, Y)
2935 if (Pred == ICmpInst::ICMP_SGT && C.isZero())
2936 return new ICmpInst(ICmpInst::ICMP_SGT, X, Y);
2937
2938 // (icmp slt (sub nsw X, Y), 0) -> (icmp slt X, Y)
2939 if (Pred == ICmpInst::ICMP_SLT && C.isZero())
2940 return new ICmpInst(ICmpInst::ICMP_SLT, X, Y);
2941
2942 // (icmp slt (sub nsw X, Y), 1) -> (icmp sle X, Y)
2943 if (Pred == ICmpInst::ICMP_SLT && C.isOne())
2944 return new ICmpInst(ICmpInst::ICMP_SLE, X, Y);
2945 }
2946
2947 if (!match(V: X, P: m_APInt(Res&: C2)))
2948 return nullptr;
2949
2950 // C2 - Y <u C -> (Y | (C - 1)) == C2
2951 // iff (C2 & (C - 1)) == C - 1 and C is a power of 2
2952 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() &&
2953 (*C2 & (C - 1)) == (C - 1))
2954 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateOr(LHS: Y, RHS: C - 1), X);
2955
2956 // C2 - Y >u C -> (Y | C) != C2
2957 // iff C2 & C == C and C + 1 is a power of 2
2958 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == C)
2959 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateOr(LHS: Y, RHS: C), X);
2960
2961 // We have handled special cases that reduce.
2962 // Canonicalize any remaining sub to add as:
2963 // (C2 - Y) > C --> (Y + ~C2) < ~C
2964 Value *Add = Builder.CreateAdd(LHS: Y, RHS: ConstantInt::get(Ty, V: ~(*C2)), Name: "notsub",
2965 HasNUW, HasNSW);
2966 return new ICmpInst(SwappedPred, Add, ConstantInt::get(Ty, V: ~C));
2967}
2968
2969static Value *createLogicFromTable(const std::bitset<4> &Table, Value *Op0,
2970 Value *Op1, IRBuilderBase &Builder,
2971 bool HasOneUse) {
2972 auto FoldConstant = [&](bool Val) {
2973 Constant *Res = Val ? Builder.getTrue() : Builder.getFalse();
2974 if (Op0->getType()->isVectorTy())
2975 Res = ConstantVector::getSplat(
2976 EC: cast<VectorType>(Val: Op0->getType())->getElementCount(), Elt: Res);
2977 return Res;
2978 };
2979
2980 switch (Table.to_ulong()) {
2981 case 0: // 0 0 0 0
2982 return FoldConstant(false);
2983 case 1: // 0 0 0 1
2984 return HasOneUse ? Builder.CreateNot(V: Builder.CreateOr(LHS: Op0, RHS: Op1)) : nullptr;
2985 case 2: // 0 0 1 0
2986 return HasOneUse ? Builder.CreateAnd(LHS: Builder.CreateNot(V: Op0), RHS: Op1) : nullptr;
2987 case 3: // 0 0 1 1
2988 return Builder.CreateNot(V: Op0);
2989 case 4: // 0 1 0 0
2990 return HasOneUse ? Builder.CreateAnd(LHS: Op0, RHS: Builder.CreateNot(V: Op1)) : nullptr;
2991 case 5: // 0 1 0 1
2992 return Builder.CreateNot(V: Op1);
2993 case 6: // 0 1 1 0
2994 return Builder.CreateXor(LHS: Op0, RHS: Op1);
2995 case 7: // 0 1 1 1
2996 return HasOneUse ? Builder.CreateNot(V: Builder.CreateAnd(LHS: Op0, RHS: Op1)) : nullptr;
2997 case 8: // 1 0 0 0
2998 return Builder.CreateAnd(LHS: Op0, RHS: Op1);
2999 case 9: // 1 0 0 1
3000 return HasOneUse ? Builder.CreateNot(V: Builder.CreateXor(LHS: Op0, RHS: Op1)) : nullptr;
3001 case 10: // 1 0 1 0
3002 return Op1;
3003 case 11: // 1 0 1 1
3004 return HasOneUse ? Builder.CreateOr(LHS: Builder.CreateNot(V: Op0), RHS: Op1) : nullptr;
3005 case 12: // 1 1 0 0
3006 return Op0;
3007 case 13: // 1 1 0 1
3008 return HasOneUse ? Builder.CreateOr(LHS: Op0, RHS: Builder.CreateNot(V: Op1)) : nullptr;
3009 case 14: // 1 1 1 0
3010 return Builder.CreateOr(LHS: Op0, RHS: Op1);
3011 case 15: // 1 1 1 1
3012 return FoldConstant(true);
3013 default:
3014 llvm_unreachable("Invalid Operation");
3015 }
3016 return nullptr;
3017}
3018
3019/// Fold icmp (add X, Y), C.
3020Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp,
3021 BinaryOperator *Add,
3022 const APInt &C) {
3023 Value *Y = Add->getOperand(i_nocapture: 1);
3024 Value *X = Add->getOperand(i_nocapture: 0);
3025
3026 Value *Op0, *Op1;
3027 Instruction *Ext0, *Ext1;
3028 const CmpInst::Predicate Pred = Cmp.getPredicate();
3029 if (match(V: Add,
3030 P: m_Add(L: m_CombineAnd(L: m_Instruction(I&: Ext0), R: m_ZExtOrSExt(Op: m_Value(V&: Op0))),
3031 R: m_CombineAnd(L: m_Instruction(I&: Ext1),
3032 R: m_ZExtOrSExt(Op: m_Value(V&: Op1))))) &&
3033 Op0->getType()->isIntOrIntVectorTy(BitWidth: 1) &&
3034 Op1->getType()->isIntOrIntVectorTy(BitWidth: 1)) {
3035 unsigned BW = C.getBitWidth();
3036 std::bitset<4> Table;
3037 auto ComputeTable = [&](bool Op0Val, bool Op1Val) {
3038 int Res = 0;
3039 if (Op0Val)
3040 Res += isa<ZExtInst>(Val: Ext0) ? 1 : -1;
3041 if (Op1Val)
3042 Res += isa<ZExtInst>(Val: Ext1) ? 1 : -1;
3043 return ICmpInst::compare(LHS: APInt(BW, Res, true), RHS: C, Pred);
3044 };
3045
3046 Table[0] = ComputeTable(false, false);
3047 Table[1] = ComputeTable(false, true);
3048 Table[2] = ComputeTable(true, false);
3049 Table[3] = ComputeTable(true, true);
3050 if (auto *Cond =
3051 createLogicFromTable(Table, Op0, Op1, Builder, HasOneUse: Add->hasOneUse()))
3052 return replaceInstUsesWith(I&: Cmp, V: Cond);
3053 }
3054 const APInt *C2;
3055 if (Cmp.isEquality() || !match(V: Y, P: m_APInt(Res&: C2)))
3056 return nullptr;
3057
3058 // Fold icmp pred (add X, C2), C.
3059 Type *Ty = Add->getType();
3060
3061 // If the add does not wrap, we can always adjust the compare by subtracting
3062 // the constants. Equality comparisons are handled elsewhere. SGE/SLE/UGE/ULE
3063 // are canonicalized to SGT/SLT/UGT/ULT.
3064 if ((Add->hasNoSignedWrap() &&
3065 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLT)) ||
3066 (Add->hasNoUnsignedWrap() &&
3067 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULT))) {
3068 bool Overflow;
3069 APInt NewC =
3070 Cmp.isSigned() ? C.ssub_ov(RHS: *C2, Overflow) : C.usub_ov(RHS: *C2, Overflow);
3071 // If there is overflow, the result must be true or false.
3072 // TODO: Can we assert there is no overflow because InstSimplify always
3073 // handles those cases?
3074 if (!Overflow)
3075 // icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2)
3076 return new ICmpInst(Pred, X, ConstantInt::get(Ty, V: NewC));
3077 }
3078
3079 auto CR = ConstantRange::makeExactICmpRegion(Pred, Other: C).subtract(CI: *C2);
3080 const APInt &Upper = CR.getUpper();
3081 const APInt &Lower = CR.getLower();
3082 if (Cmp.isSigned()) {
3083 if (Lower.isSignMask())
3084 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, V: Upper));
3085 if (Upper.isSignMask())
3086 return new ICmpInst(ICmpInst::ICMP_SGE, X, ConstantInt::get(Ty, V: Lower));
3087 } else {
3088 if (Lower.isMinValue())
3089 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, V: Upper));
3090 if (Upper.isMinValue())
3091 return new ICmpInst(ICmpInst::ICMP_UGE, X, ConstantInt::get(Ty, V: Lower));
3092 }
3093
3094 // This set of folds is intentionally placed after folds that use no-wrapping
3095 // flags because those folds are likely better for later analysis/codegen.
3096 const APInt SMax = APInt::getSignedMaxValue(numBits: Ty->getScalarSizeInBits());
3097 const APInt SMin = APInt::getSignedMinValue(numBits: Ty->getScalarSizeInBits());
3098
3099 // Fold compare with offset to opposite sign compare if it eliminates offset:
3100 // (X + C2) >u C --> X <s -C2 (if C == C2 + SMAX)
3101 if (Pred == CmpInst::ICMP_UGT && C == *C2 + SMax)
3102 return new ICmpInst(ICmpInst::ICMP_SLT, X, ConstantInt::get(Ty, V: -(*C2)));
3103
3104 // (X + C2) <u C --> X >s ~C2 (if C == C2 + SMIN)
3105 if (Pred == CmpInst::ICMP_ULT && C == *C2 + SMin)
3106 return new ICmpInst(ICmpInst::ICMP_SGT, X, ConstantInt::get(Ty, V: ~(*C2)));
3107
3108 // (X + C2) >s C --> X <u (SMAX - C) (if C == C2 - 1)
3109 if (Pred == CmpInst::ICMP_SGT && C == *C2 - 1)
3110 return new ICmpInst(ICmpInst::ICMP_ULT, X, ConstantInt::get(Ty, V: SMax - C));
3111
3112 // (X + C2) <s C --> X >u (C ^ SMAX) (if C == C2)
3113 if (Pred == CmpInst::ICMP_SLT && C == *C2)
3114 return new ICmpInst(ICmpInst::ICMP_UGT, X, ConstantInt::get(Ty, V: C ^ SMax));
3115
3116 // (X + -1) <u C --> X <=u C (if X is never null)
3117 if (Pred == CmpInst::ICMP_ULT && C2->isAllOnes()) {
3118 const SimplifyQuery Q = SQ.getWithInstruction(I: &Cmp);
3119 if (llvm::isKnownNonZero(V: X, Q))
3120 return new ICmpInst(ICmpInst::ICMP_ULE, X, ConstantInt::get(Ty, V: C));
3121 }
3122
3123 if (!Add->hasOneUse())
3124 return nullptr;
3125
3126 // X+C <u C2 -> (X & -C2) == C
3127 // iff C & (C2-1) == 0
3128 // C2 is a power of 2
3129 if (Pred == ICmpInst::ICMP_ULT && C.isPowerOf2() && (*C2 & (C - 1)) == 0)
3130 return new ICmpInst(ICmpInst::ICMP_EQ, Builder.CreateAnd(LHS: X, RHS: -C),
3131 ConstantExpr::getNeg(C: cast<Constant>(Val: Y)));
3132
3133 // X+C2 <u C -> (X & C) == 2C
3134 // iff C == -(C2)
3135 // C2 is a power of 2
3136 if (Pred == ICmpInst::ICMP_ULT && C2->isPowerOf2() && C == -*C2)
3137 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(LHS: X, RHS: C),
3138 ConstantInt::get(Ty, V: C * 2));
3139
3140 // X+C >u C2 -> (X & ~C2) != C
3141 // iff C & C2 == 0
3142 // C2+1 is a power of 2
3143 if (Pred == ICmpInst::ICMP_UGT && (C + 1).isPowerOf2() && (*C2 & C) == 0)
3144 return new ICmpInst(ICmpInst::ICMP_NE, Builder.CreateAnd(LHS: X, RHS: ~C),
3145 ConstantExpr::getNeg(C: cast<Constant>(Val: Y)));
3146
3147 // The range test idiom can use either ult or ugt. Arbitrarily canonicalize
3148 // to the ult form.
3149 // X+C2 >u C -> X+(C2-C-1) <u ~C
3150 if (Pred == ICmpInst::ICMP_UGT)
3151 return new ICmpInst(ICmpInst::ICMP_ULT,
3152 Builder.CreateAdd(LHS: X, RHS: ConstantInt::get(Ty, V: *C2 - C - 1)),
3153 ConstantInt::get(Ty, V: ~C));
3154
3155 return nullptr;
3156}
3157
3158bool InstCombinerImpl::matchThreeWayIntCompare(SelectInst *SI, Value *&LHS,
3159 Value *&RHS, ConstantInt *&Less,
3160 ConstantInt *&Equal,
3161 ConstantInt *&Greater) {
3162 // TODO: Generalize this to work with other comparison idioms or ensure
3163 // they get canonicalized into this form.
3164
3165 // select i1 (a == b),
3166 // i32 Equal,
3167 // i32 (select i1 (a < b), i32 Less, i32 Greater)
3168 // where Equal, Less and Greater are placeholders for any three constants.
3169 ICmpInst::Predicate PredA;
3170 if (!match(V: SI->getCondition(), P: m_ICmp(Pred&: PredA, L: m_Value(V&: LHS), R: m_Value(V&: RHS))) ||
3171 !ICmpInst::isEquality(P: PredA))
3172 return false;
3173 Value *EqualVal = SI->getTrueValue();
3174 Value *UnequalVal = SI->getFalseValue();
3175 // We still can get non-canonical predicate here, so canonicalize.
3176 if (PredA == ICmpInst::ICMP_NE)
3177 std::swap(a&: EqualVal, b&: UnequalVal);
3178 if (!match(V: EqualVal, P: m_ConstantInt(CI&: Equal)))
3179 return false;
3180 ICmpInst::Predicate PredB;
3181 Value *LHS2, *RHS2;
3182 if (!match(V: UnequalVal, P: m_Select(C: m_ICmp(Pred&: PredB, L: m_Value(V&: LHS2), R: m_Value(V&: RHS2)),
3183 L: m_ConstantInt(CI&: Less), R: m_ConstantInt(CI&: Greater))))
3184 return false;
3185 // We can get predicate mismatch here, so canonicalize if possible:
3186 // First, ensure that 'LHS' match.
3187 if (LHS2 != LHS) {
3188 // x sgt y <--> y slt x
3189 std::swap(a&: LHS2, b&: RHS2);
3190 PredB = ICmpInst::getSwappedPredicate(pred: PredB);
3191 }
3192 if (LHS2 != LHS)
3193 return false;
3194 // We also need to canonicalize 'RHS'.
3195 if (PredB == ICmpInst::ICMP_SGT && isa<Constant>(Val: RHS2)) {
3196 // x sgt C-1 <--> x sge C <--> not(x slt C)
3197 auto FlippedStrictness =
3198 InstCombiner::getFlippedStrictnessPredicateAndConstant(
3199 Pred: PredB, C: cast<Constant>(Val: RHS2));
3200 if (!FlippedStrictness)
3201 return false;
3202 assert(FlippedStrictness->first == ICmpInst::ICMP_SGE &&
3203 "basic correctness failure");
3204 RHS2 = FlippedStrictness->second;
3205 // And kind-of perform the result swap.
3206 std::swap(a&: Less, b&: Greater);
3207 PredB = ICmpInst::ICMP_SLT;
3208 }
3209 return PredB == ICmpInst::ICMP_SLT && RHS == RHS2;
3210}
3211
3212Instruction *InstCombinerImpl::foldICmpSelectConstant(ICmpInst &Cmp,
3213 SelectInst *Select,
3214 ConstantInt *C) {
3215
3216 assert(C && "Cmp RHS should be a constant int!");
3217 // If we're testing a constant value against the result of a three way
3218 // comparison, the result can be expressed directly in terms of the
3219 // original values being compared. Note: We could possibly be more
3220 // aggressive here and remove the hasOneUse test. The original select is
3221 // really likely to simplify or sink when we remove a test of the result.
3222 Value *OrigLHS, *OrigRHS;
3223 ConstantInt *C1LessThan, *C2Equal, *C3GreaterThan;
3224 if (Cmp.hasOneUse() &&
3225 matchThreeWayIntCompare(SI: Select, LHS&: OrigLHS, RHS&: OrigRHS, Less&: C1LessThan, Equal&: C2Equal,
3226 Greater&: C3GreaterThan)) {
3227 assert(C1LessThan && C2Equal && C3GreaterThan);
3228
3229 bool TrueWhenLessThan = ICmpInst::compare(
3230 LHS: C1LessThan->getValue(), RHS: C->getValue(), Pred: Cmp.getPredicate());
3231 bool TrueWhenEqual = ICmpInst::compare(LHS: C2Equal->getValue(), RHS: C->getValue(),
3232 Pred: Cmp.getPredicate());
3233 bool TrueWhenGreaterThan = ICmpInst::compare(
3234 LHS: C3GreaterThan->getValue(), RHS: C->getValue(), Pred: Cmp.getPredicate());
3235
3236 // This generates the new instruction that will replace the original Cmp
3237 // Instruction. Instead of enumerating the various combinations when
3238 // TrueWhenLessThan, TrueWhenEqual and TrueWhenGreaterThan are true versus
3239 // false, we rely on chaining of ORs and future passes of InstCombine to
3240 // simplify the OR further (i.e. a s< b || a == b becomes a s<= b).
3241
3242 // When none of the three constants satisfy the predicate for the RHS (C),
3243 // the entire original Cmp can be simplified to a false.
3244 Value *Cond = Builder.getFalse();
3245 if (TrueWhenLessThan)
3246 Cond = Builder.CreateOr(LHS: Cond, RHS: Builder.CreateICmp(P: ICmpInst::ICMP_SLT,
3247 LHS: OrigLHS, RHS: OrigRHS));
3248 if (TrueWhenEqual)
3249 Cond = Builder.CreateOr(LHS: Cond, RHS: Builder.CreateICmp(P: ICmpInst::ICMP_EQ,
3250 LHS: OrigLHS, RHS: OrigRHS));
3251 if (TrueWhenGreaterThan)
3252 Cond = Builder.CreateOr(LHS: Cond, RHS: Builder.CreateICmp(P: ICmpInst::ICMP_SGT,
3253 LHS: OrigLHS, RHS: OrigRHS));
3254
3255 return replaceInstUsesWith(I&: Cmp, V: Cond);
3256 }
3257 return nullptr;
3258}
3259
3260Instruction *InstCombinerImpl::foldICmpBitCast(ICmpInst &Cmp) {
3261 auto *Bitcast = dyn_cast<BitCastInst>(Val: Cmp.getOperand(i_nocapture: 0));
3262 if (!Bitcast)
3263 return nullptr;
3264
3265 ICmpInst::Predicate Pred = Cmp.getPredicate();
3266 Value *Op1 = Cmp.getOperand(i_nocapture: 1);
3267 Value *BCSrcOp = Bitcast->getOperand(i_nocapture: 0);
3268 Type *SrcType = Bitcast->getSrcTy();
3269 Type *DstType = Bitcast->getType();
3270
3271 // Make sure the bitcast doesn't change between scalar and vector and
3272 // doesn't change the number of vector elements.
3273 if (SrcType->isVectorTy() == DstType->isVectorTy() &&
3274 SrcType->getScalarSizeInBits() == DstType->getScalarSizeInBits()) {
3275 // Zero-equality and sign-bit checks are preserved through sitofp + bitcast.
3276 Value *X;
3277 if (match(V: BCSrcOp, P: m_SIToFP(Op: m_Value(V&: X)))) {
3278 // icmp eq (bitcast (sitofp X)), 0 --> icmp eq X, 0
3279 // icmp ne (bitcast (sitofp X)), 0 --> icmp ne X, 0
3280 // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0
3281 // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0
3282 if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT ||
3283 Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) &&
3284 match(V: Op1, P: m_Zero()))
3285 return new ICmpInst(Pred, X, ConstantInt::getNullValue(Ty: X->getType()));
3286
3287 // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1
3288 if (Pred == ICmpInst::ICMP_SLT && match(V: Op1, P: m_One()))
3289 return new ICmpInst(Pred, X, ConstantInt::get(Ty: X->getType(), V: 1));
3290
3291 // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1
3292 if (Pred == ICmpInst::ICMP_SGT && match(V: Op1, P: m_AllOnes()))
3293 return new ICmpInst(Pred, X,
3294 ConstantInt::getAllOnesValue(Ty: X->getType()));
3295 }
3296
3297 // Zero-equality checks are preserved through unsigned floating-point casts:
3298 // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0
3299 // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0
3300 if (match(V: BCSrcOp, P: m_UIToFP(Op: m_Value(V&: X))))
3301 if (Cmp.isEquality() && match(V: Op1, P: m_Zero()))
3302 return new ICmpInst(Pred, X, ConstantInt::getNullValue(Ty: X->getType()));
3303
3304 const APInt *C;
3305 bool TrueIfSigned;
3306 if (match(V: Op1, P: m_APInt(Res&: C)) && Bitcast->hasOneUse()) {
3307 // If this is a sign-bit test of a bitcast of a casted FP value, eliminate
3308 // the FP extend/truncate because that cast does not change the sign-bit.
3309 // This is true for all standard IEEE-754 types and the X86 80-bit type.
3310 // The sign-bit is always the most significant bit in those types.
3311 if (isSignBitCheck(Pred, RHS: *C, TrueIfSigned) &&
3312 (match(V: BCSrcOp, P: m_FPExt(Op: m_Value(V&: X))) ||
3313 match(V: BCSrcOp, P: m_FPTrunc(Op: m_Value(V&: X))))) {
3314 // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0
3315 // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1
3316 Type *XType = X->getType();
3317
3318 // We can't currently handle Power style floating point operations here.
3319 if (!(XType->isPPC_FP128Ty() || SrcType->isPPC_FP128Ty())) {
3320 Type *NewType = Builder.getIntNTy(N: XType->getScalarSizeInBits());
3321 if (auto *XVTy = dyn_cast<VectorType>(Val: XType))
3322 NewType = VectorType::get(ElementType: NewType, EC: XVTy->getElementCount());
3323 Value *NewBitcast = Builder.CreateBitCast(V: X, DestTy: NewType);
3324 if (TrueIfSigned)
3325 return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast,
3326 ConstantInt::getNullValue(Ty: NewType));
3327 else
3328 return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast,
3329 ConstantInt::getAllOnesValue(Ty: NewType));
3330 }
3331 }
3332
3333 // icmp eq/ne (bitcast X to int), special fp -> llvm.is.fpclass(X, class)
3334 Type *FPType = SrcType->getScalarType();
3335 if (!Cmp.getParent()->getParent()->hasFnAttribute(
3336 Kind: Attribute::NoImplicitFloat) &&
3337 Cmp.isEquality() && FPType->isIEEELikeFPTy()) {
3338 FPClassTest Mask = APFloat(FPType->getFltSemantics(), *C).classify();
3339 if (Mask & (fcInf | fcZero)) {
3340 if (Pred == ICmpInst::ICMP_NE)
3341 Mask = ~Mask;
3342 return replaceInstUsesWith(I&: Cmp,
3343 V: Builder.createIsFPClass(FPNum: BCSrcOp, Test: Mask));
3344 }
3345 }
3346 }
3347 }
3348
3349 const APInt *C;
3350 if (!match(V: Cmp.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) || !DstType->isIntegerTy() ||
3351 !SrcType->isIntOrIntVectorTy())
3352 return nullptr;
3353
3354 // If this is checking if all elements of a vector compare are set or not,
3355 // invert the casted vector equality compare and test if all compare
3356 // elements are clear or not. Compare against zero is generally easier for
3357 // analysis and codegen.
3358 // icmp eq/ne (bitcast (not X) to iN), -1 --> icmp eq/ne (bitcast X to iN), 0
3359 // Example: are all elements equal? --> are zero elements not equal?
3360 // TODO: Try harder to reduce compare of 2 freely invertible operands?
3361 if (Cmp.isEquality() && C->isAllOnes() && Bitcast->hasOneUse()) {
3362 if (Value *NotBCSrcOp =
3363 getFreelyInverted(V: BCSrcOp, WillInvertAllUses: BCSrcOp->hasOneUse(), Builder: &Builder)) {
3364 Value *Cast = Builder.CreateBitCast(V: NotBCSrcOp, DestTy: DstType);
3365 return new ICmpInst(Pred, Cast, ConstantInt::getNullValue(Ty: DstType));
3366 }
3367 }
3368
3369 // If this is checking if all elements of an extended vector are clear or not,
3370 // compare in a narrow type to eliminate the extend:
3371 // icmp eq/ne (bitcast (ext X) to iN), 0 --> icmp eq/ne (bitcast X to iM), 0
3372 Value *X;
3373 if (Cmp.isEquality() && C->isZero() && Bitcast->hasOneUse() &&
3374 match(V: BCSrcOp, P: m_ZExtOrSExt(Op: m_Value(V&: X)))) {
3375 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: X->getType())) {
3376 Type *NewType = Builder.getIntNTy(N: VecTy->getPrimitiveSizeInBits());
3377 Value *NewCast = Builder.CreateBitCast(V: X, DestTy: NewType);
3378 return new ICmpInst(Pred, NewCast, ConstantInt::getNullValue(Ty: NewType));
3379 }
3380 }
3381
3382 // Folding: icmp <pred> iN X, C
3383 // where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN
3384 // and C is a splat of a K-bit pattern
3385 // and SC is a constant vector = <C', C', C', ..., C'>
3386 // Into:
3387 // %E = extractelement <M x iK> %vec, i32 C'
3388 // icmp <pred> iK %E, trunc(C)
3389 Value *Vec;
3390 ArrayRef<int> Mask;
3391 if (match(V: BCSrcOp, P: m_Shuffle(v1: m_Value(V&: Vec), v2: m_Undef(), mask: m_Mask(Mask)))) {
3392 // Check whether every element of Mask is the same constant
3393 if (all_equal(Range&: Mask)) {
3394 auto *VecTy = cast<VectorType>(Val: SrcType);
3395 auto *EltTy = cast<IntegerType>(Val: VecTy->getElementType());
3396 if (C->isSplat(SplatSizeInBits: EltTy->getBitWidth())) {
3397 // Fold the icmp based on the value of C
3398 // If C is M copies of an iK sized bit pattern,
3399 // then:
3400 // => %E = extractelement <N x iK> %vec, i32 Elem
3401 // icmp <pred> iK %SplatVal, <pattern>
3402 Value *Elem = Builder.getInt32(C: Mask[0]);
3403 Value *Extract = Builder.CreateExtractElement(Vec, Idx: Elem);
3404 Value *NewC = ConstantInt::get(Ty: EltTy, V: C->trunc(width: EltTy->getBitWidth()));
3405 return new ICmpInst(Pred, Extract, NewC);
3406 }
3407 }
3408 }
3409 return nullptr;
3410}
3411
3412/// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3413/// where X is some kind of instruction.
3414Instruction *InstCombinerImpl::foldICmpInstWithConstant(ICmpInst &Cmp) {
3415 const APInt *C;
3416
3417 if (match(V: Cmp.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
3418 if (auto *BO = dyn_cast<BinaryOperator>(Val: Cmp.getOperand(i_nocapture: 0)))
3419 if (Instruction *I = foldICmpBinOpWithConstant(Cmp, BO, C: *C))
3420 return I;
3421
3422 if (auto *SI = dyn_cast<SelectInst>(Val: Cmp.getOperand(i_nocapture: 0)))
3423 // For now, we only support constant integers while folding the
3424 // ICMP(SELECT)) pattern. We can extend this to support vector of integers
3425 // similar to the cases handled by binary ops above.
3426 if (auto *ConstRHS = dyn_cast<ConstantInt>(Val: Cmp.getOperand(i_nocapture: 1)))
3427 if (Instruction *I = foldICmpSelectConstant(Cmp, Select: SI, C: ConstRHS))
3428 return I;
3429
3430 if (auto *TI = dyn_cast<TruncInst>(Val: Cmp.getOperand(i_nocapture: 0)))
3431 if (Instruction *I = foldICmpTruncConstant(Cmp, Trunc: TI, C: *C))
3432 return I;
3433
3434 if (auto *II = dyn_cast<IntrinsicInst>(Val: Cmp.getOperand(i_nocapture: 0)))
3435 if (Instruction *I = foldICmpIntrinsicWithConstant(ICI&: Cmp, II, C: *C))
3436 return I;
3437
3438 // (extractval ([s/u]subo X, Y), 0) == 0 --> X == Y
3439 // (extractval ([s/u]subo X, Y), 0) != 0 --> X != Y
3440 // TODO: This checks one-use, but that is not strictly necessary.
3441 Value *Cmp0 = Cmp.getOperand(i_nocapture: 0);
3442 Value *X, *Y;
3443 if (C->isZero() && Cmp.isEquality() && Cmp0->hasOneUse() &&
3444 (match(V: Cmp0,
3445 P: m_ExtractValue<0>(V: m_Intrinsic<Intrinsic::ssub_with_overflow>(
3446 Op0: m_Value(V&: X), Op1: m_Value(V&: Y)))) ||
3447 match(V: Cmp0,
3448 P: m_ExtractValue<0>(V: m_Intrinsic<Intrinsic::usub_with_overflow>(
3449 Op0: m_Value(V&: X), Op1: m_Value(V&: Y))))))
3450 return new ICmpInst(Cmp.getPredicate(), X, Y);
3451 }
3452
3453 if (match(V: Cmp.getOperand(i_nocapture: 1), P: m_APIntAllowPoison(Res&: C)))
3454 return foldICmpInstWithConstantAllowPoison(Cmp, C: *C);
3455
3456 return nullptr;
3457}
3458
3459/// Fold an icmp equality instruction with binary operator LHS and constant RHS:
3460/// icmp eq/ne BO, C.
3461Instruction *InstCombinerImpl::foldICmpBinOpEqualityWithConstant(
3462 ICmpInst &Cmp, BinaryOperator *BO, const APInt &C) {
3463 // TODO: Some of these folds could work with arbitrary constants, but this
3464 // function is limited to scalar and vector splat constants.
3465 if (!Cmp.isEquality())
3466 return nullptr;
3467
3468 ICmpInst::Predicate Pred = Cmp.getPredicate();
3469 bool isICMP_NE = Pred == ICmpInst::ICMP_NE;
3470 Constant *RHS = cast<Constant>(Val: Cmp.getOperand(i_nocapture: 1));
3471 Value *BOp0 = BO->getOperand(i_nocapture: 0), *BOp1 = BO->getOperand(i_nocapture: 1);
3472
3473 switch (BO->getOpcode()) {
3474 case Instruction::SRem:
3475 // If we have a signed (X % (2^c)) == 0, turn it into an unsigned one.
3476 if (C.isZero() && BO->hasOneUse()) {
3477 const APInt *BOC;
3478 if (match(V: BOp1, P: m_APInt(Res&: BOC)) && BOC->sgt(RHS: 1) && BOC->isPowerOf2()) {
3479 Value *NewRem = Builder.CreateURem(LHS: BOp0, RHS: BOp1, Name: BO->getName());
3480 return new ICmpInst(Pred, NewRem,
3481 Constant::getNullValue(Ty: BO->getType()));
3482 }
3483 }
3484 break;
3485 case Instruction::Add: {
3486 // (A + C2) == C --> A == (C - C2)
3487 // (A + C2) != C --> A != (C - C2)
3488 // TODO: Remove the one-use limitation? See discussion in D58633.
3489 if (Constant *C2 = dyn_cast<Constant>(Val: BOp1)) {
3490 if (BO->hasOneUse())
3491 return new ICmpInst(Pred, BOp0, ConstantExpr::getSub(C1: RHS, C2));
3492 } else if (C.isZero()) {
3493 // Replace ((add A, B) != 0) with (A != -B) if A or B is
3494 // efficiently invertible, or if the add has just this one use.
3495 if (Value *NegVal = dyn_castNegVal(V: BOp1))
3496 return new ICmpInst(Pred, BOp0, NegVal);
3497 if (Value *NegVal = dyn_castNegVal(V: BOp0))
3498 return new ICmpInst(Pred, NegVal, BOp1);
3499 if (BO->hasOneUse()) {
3500 // (add nuw A, B) != 0 -> (or A, B) != 0
3501 if (match(V: BO, P: m_NUWAdd(L: m_Value(), R: m_Value()))) {
3502 Value *Or = Builder.CreateOr(LHS: BOp0, RHS: BOp1);
3503 return new ICmpInst(Pred, Or, Constant::getNullValue(Ty: BO->getType()));
3504 }
3505 Value *Neg = Builder.CreateNeg(V: BOp1);
3506 Neg->takeName(V: BO);
3507 return new ICmpInst(Pred, BOp0, Neg);
3508 }
3509 }
3510 break;
3511 }
3512 case Instruction::Xor:
3513 if (Constant *BOC = dyn_cast<Constant>(Val: BOp1)) {
3514 // For the xor case, we can xor two constants together, eliminating
3515 // the explicit xor.
3516 return new ICmpInst(Pred, BOp0, ConstantExpr::getXor(C1: RHS, C2: BOC));
3517 } else if (C.isZero()) {
3518 // Replace ((xor A, B) != 0) with (A != B)
3519 return new ICmpInst(Pred, BOp0, BOp1);
3520 }
3521 break;
3522 case Instruction::Or: {
3523 const APInt *BOC;
3524 if (match(V: BOp1, P: m_APInt(Res&: BOC)) && BO->hasOneUse() && RHS->isAllOnesValue()) {
3525 // Comparing if all bits outside of a constant mask are set?
3526 // Replace (X | C) == -1 with (X & ~C) == ~C.
3527 // This removes the -1 constant.
3528 Constant *NotBOC = ConstantExpr::getNot(C: cast<Constant>(Val: BOp1));
3529 Value *And = Builder.CreateAnd(LHS: BOp0, RHS: NotBOC);
3530 return new ICmpInst(Pred, And, NotBOC);
3531 }
3532 break;
3533 }
3534 case Instruction::UDiv:
3535 case Instruction::SDiv:
3536 if (BO->isExact()) {
3537 // div exact X, Y eq/ne 0 -> X eq/ne 0
3538 // div exact X, Y eq/ne 1 -> X eq/ne Y
3539 // div exact X, Y eq/ne C ->
3540 // if Y * C never-overflow && OneUse:
3541 // -> Y * C eq/ne X
3542 if (C.isZero())
3543 return new ICmpInst(Pred, BOp0, Constant::getNullValue(Ty: BO->getType()));
3544 else if (C.isOne())
3545 return new ICmpInst(Pred, BOp0, BOp1);
3546 else if (BO->hasOneUse()) {
3547 OverflowResult OR = computeOverflow(
3548 BinaryOp: Instruction::Mul, IsSigned: BO->getOpcode() == Instruction::SDiv, LHS: BOp1,
3549 RHS: Cmp.getOperand(i_nocapture: 1), CxtI: BO);
3550 if (OR == OverflowResult::NeverOverflows) {
3551 Value *YC =
3552 Builder.CreateMul(LHS: BOp1, RHS: ConstantInt::get(Ty: BO->getType(), V: C));
3553 return new ICmpInst(Pred, YC, BOp0);
3554 }
3555 }
3556 }
3557 if (BO->getOpcode() == Instruction::UDiv && C.isZero()) {
3558 // (icmp eq/ne (udiv A, B), 0) -> (icmp ugt/ule i32 B, A)
3559 auto NewPred = isICMP_NE ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT;
3560 return new ICmpInst(NewPred, BOp1, BOp0);
3561 }
3562 break;
3563 default:
3564 break;
3565 }
3566 return nullptr;
3567}
3568
3569static Instruction *foldCtpopPow2Test(ICmpInst &I, IntrinsicInst *CtpopLhs,
3570 const APInt &CRhs,
3571 InstCombiner::BuilderTy &Builder,
3572 const SimplifyQuery &Q) {
3573 assert(CtpopLhs->getIntrinsicID() == Intrinsic::ctpop &&
3574 "Non-ctpop intrin in ctpop fold");
3575 if (!CtpopLhs->hasOneUse())
3576 return nullptr;
3577
3578 // Power of 2 test:
3579 // isPow2OrZero : ctpop(X) u< 2
3580 // isPow2 : ctpop(X) == 1
3581 // NotPow2OrZero: ctpop(X) u> 1
3582 // NotPow2 : ctpop(X) != 1
3583 // If we know any bit of X can be folded to:
3584 // IsPow2 : X & (~Bit) == 0
3585 // NotPow2 : X & (~Bit) != 0
3586 const ICmpInst::Predicate Pred = I.getPredicate();
3587 if (((I.isEquality() || Pred == ICmpInst::ICMP_UGT) && CRhs == 1) ||
3588 (Pred == ICmpInst::ICMP_ULT && CRhs == 2)) {
3589 Value *Op = CtpopLhs->getArgOperand(i: 0);
3590 KnownBits OpKnown = computeKnownBits(V: Op, DL: Q.DL,
3591 /*Depth*/ 0, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT);
3592 // No need to check for count > 1, that should be already constant folded.
3593 if (OpKnown.countMinPopulation() == 1) {
3594 Value *And = Builder.CreateAnd(
3595 LHS: Op, RHS: Constant::getIntegerValue(Ty: Op->getType(), V: ~(OpKnown.One)));
3596 return new ICmpInst(
3597 (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_ULT)
3598 ? ICmpInst::ICMP_EQ
3599 : ICmpInst::ICMP_NE,
3600 And, Constant::getNullValue(Ty: Op->getType()));
3601 }
3602 }
3603
3604 return nullptr;
3605}
3606
3607/// Fold an equality icmp with LLVM intrinsic and constant operand.
3608Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant(
3609 ICmpInst &Cmp, IntrinsicInst *II, const APInt &C) {
3610 Type *Ty = II->getType();
3611 unsigned BitWidth = C.getBitWidth();
3612 const ICmpInst::Predicate Pred = Cmp.getPredicate();
3613
3614 switch (II->getIntrinsicID()) {
3615 case Intrinsic::abs:
3616 // abs(A) == 0 -> A == 0
3617 // abs(A) == INT_MIN -> A == INT_MIN
3618 if (C.isZero() || C.isMinSignedValue())
3619 return new ICmpInst(Pred, II->getArgOperand(i: 0), ConstantInt::get(Ty, V: C));
3620 break;
3621
3622 case Intrinsic::bswap:
3623 // bswap(A) == C -> A == bswap(C)
3624 return new ICmpInst(Pred, II->getArgOperand(i: 0),
3625 ConstantInt::get(Ty, V: C.byteSwap()));
3626
3627 case Intrinsic::bitreverse:
3628 // bitreverse(A) == C -> A == bitreverse(C)
3629 return new ICmpInst(Pred, II->getArgOperand(i: 0),
3630 ConstantInt::get(Ty, V: C.reverseBits()));
3631
3632 case Intrinsic::ctlz:
3633 case Intrinsic::cttz: {
3634 // ctz(A) == bitwidth(A) -> A == 0 and likewise for !=
3635 if (C == BitWidth)
3636 return new ICmpInst(Pred, II->getArgOperand(i: 0),
3637 ConstantInt::getNullValue(Ty));
3638
3639 // ctz(A) == C -> A & Mask1 == Mask2, where Mask2 only has bit C set
3640 // and Mask1 has bits 0..C+1 set. Similar for ctl, but for high bits.
3641 // Limit to one use to ensure we don't increase instruction count.
3642 unsigned Num = C.getLimitedValue(Limit: BitWidth);
3643 if (Num != BitWidth && II->hasOneUse()) {
3644 bool IsTrailing = II->getIntrinsicID() == Intrinsic::cttz;
3645 APInt Mask1 = IsTrailing ? APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: Num + 1)
3646 : APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: Num + 1);
3647 APInt Mask2 = IsTrailing
3648 ? APInt::getOneBitSet(numBits: BitWidth, BitNo: Num)
3649 : APInt::getOneBitSet(numBits: BitWidth, BitNo: BitWidth - Num - 1);
3650 return new ICmpInst(Pred, Builder.CreateAnd(LHS: II->getArgOperand(i: 0), RHS: Mask1),
3651 ConstantInt::get(Ty, V: Mask2));
3652 }
3653 break;
3654 }
3655
3656 case Intrinsic::ctpop: {
3657 // popcount(A) == 0 -> A == 0 and likewise for !=
3658 // popcount(A) == bitwidth(A) -> A == -1 and likewise for !=
3659 bool IsZero = C.isZero();
3660 if (IsZero || C == BitWidth)
3661 return new ICmpInst(Pred, II->getArgOperand(i: 0),
3662 IsZero ? Constant::getNullValue(Ty)
3663 : Constant::getAllOnesValue(Ty));
3664
3665 break;
3666 }
3667
3668 case Intrinsic::fshl:
3669 case Intrinsic::fshr:
3670 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) {
3671 const APInt *RotAmtC;
3672 // ror(X, RotAmtC) == C --> X == rol(C, RotAmtC)
3673 // rol(X, RotAmtC) == C --> X == ror(C, RotAmtC)
3674 if (match(V: II->getArgOperand(i: 2), P: m_APInt(Res&: RotAmtC)))
3675 return new ICmpInst(Pred, II->getArgOperand(i: 0),
3676 II->getIntrinsicID() == Intrinsic::fshl
3677 ? ConstantInt::get(Ty, V: C.rotr(rotateAmt: *RotAmtC))
3678 : ConstantInt::get(Ty, V: C.rotl(rotateAmt: *RotAmtC)));
3679 }
3680 break;
3681
3682 case Intrinsic::umax:
3683 case Intrinsic::uadd_sat: {
3684 // uadd.sat(a, b) == 0 -> (a | b) == 0
3685 // umax(a, b) == 0 -> (a | b) == 0
3686 if (C.isZero() && II->hasOneUse()) {
3687 Value *Or = Builder.CreateOr(LHS: II->getArgOperand(i: 0), RHS: II->getArgOperand(i: 1));
3688 return new ICmpInst(Pred, Or, Constant::getNullValue(Ty));
3689 }
3690 break;
3691 }
3692
3693 case Intrinsic::ssub_sat:
3694 // ssub.sat(a, b) == 0 -> a == b
3695 if (C.isZero())
3696 return new ICmpInst(Pred, II->getArgOperand(i: 0), II->getArgOperand(i: 1));
3697 break;
3698 case Intrinsic::usub_sat: {
3699 // usub.sat(a, b) == 0 -> a <= b
3700 if (C.isZero()) {
3701 ICmpInst::Predicate NewPred =
3702 Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_UGT;
3703 return new ICmpInst(NewPred, II->getArgOperand(i: 0), II->getArgOperand(i: 1));
3704 }
3705 break;
3706 }
3707 default:
3708 break;
3709 }
3710
3711 return nullptr;
3712}
3713
3714/// Fold an icmp with LLVM intrinsics
3715static Instruction *
3716foldICmpIntrinsicWithIntrinsic(ICmpInst &Cmp,
3717 InstCombiner::BuilderTy &Builder) {
3718 assert(Cmp.isEquality());
3719
3720 ICmpInst::Predicate Pred = Cmp.getPredicate();
3721 Value *Op0 = Cmp.getOperand(i_nocapture: 0);
3722 Value *Op1 = Cmp.getOperand(i_nocapture: 1);
3723 const auto *IIOp0 = dyn_cast<IntrinsicInst>(Val: Op0);
3724 const auto *IIOp1 = dyn_cast<IntrinsicInst>(Val: Op1);
3725 if (!IIOp0 || !IIOp1 || IIOp0->getIntrinsicID() != IIOp1->getIntrinsicID())
3726 return nullptr;
3727
3728 switch (IIOp0->getIntrinsicID()) {
3729 case Intrinsic::bswap:
3730 case Intrinsic::bitreverse:
3731 // If both operands are byte-swapped or bit-reversed, just compare the
3732 // original values.
3733 return new ICmpInst(Pred, IIOp0->getOperand(i_nocapture: 0), IIOp1->getOperand(i_nocapture: 0));
3734 case Intrinsic::fshl:
3735 case Intrinsic::fshr: {
3736 // If both operands are rotated by same amount, just compare the
3737 // original values.
3738 if (IIOp0->getOperand(i_nocapture: 0) != IIOp0->getOperand(i_nocapture: 1))
3739 break;
3740 if (IIOp1->getOperand(i_nocapture: 0) != IIOp1->getOperand(i_nocapture: 1))
3741 break;
3742 if (IIOp0->getOperand(i_nocapture: 2) == IIOp1->getOperand(i_nocapture: 2))
3743 return new ICmpInst(Pred, IIOp0->getOperand(i_nocapture: 0), IIOp1->getOperand(i_nocapture: 0));
3744
3745 // rotate(X, AmtX) == rotate(Y, AmtY)
3746 // -> rotate(X, AmtX - AmtY) == Y
3747 // Do this if either both rotates have one use or if only one has one use
3748 // and AmtX/AmtY are constants.
3749 unsigned OneUses = IIOp0->hasOneUse() + IIOp1->hasOneUse();
3750 if (OneUses == 2 ||
3751 (OneUses == 1 && match(V: IIOp0->getOperand(i_nocapture: 2), P: m_ImmConstant()) &&
3752 match(V: IIOp1->getOperand(i_nocapture: 2), P: m_ImmConstant()))) {
3753 Value *SubAmt =
3754 Builder.CreateSub(LHS: IIOp0->getOperand(i_nocapture: 2), RHS: IIOp1->getOperand(i_nocapture: 2));
3755 Value *CombinedRotate = Builder.CreateIntrinsic(
3756 RetTy: Op0->getType(), ID: IIOp0->getIntrinsicID(),
3757 Args: {IIOp0->getOperand(i_nocapture: 0), IIOp0->getOperand(i_nocapture: 0), SubAmt});
3758 return new ICmpInst(Pred, IIOp1->getOperand(i_nocapture: 0), CombinedRotate);
3759 }
3760 } break;
3761 default:
3762 break;
3763 }
3764
3765 return nullptr;
3766}
3767
3768/// Try to fold integer comparisons with a constant operand: icmp Pred X, C
3769/// where X is some kind of instruction and C is AllowPoison.
3770/// TODO: Move more folds which allow poison to this function.
3771Instruction *
3772InstCombinerImpl::foldICmpInstWithConstantAllowPoison(ICmpInst &Cmp,
3773 const APInt &C) {
3774 const ICmpInst::Predicate Pred = Cmp.getPredicate();
3775 if (auto *II = dyn_cast<IntrinsicInst>(Val: Cmp.getOperand(i_nocapture: 0))) {
3776 switch (II->getIntrinsicID()) {
3777 default:
3778 break;
3779 case Intrinsic::fshl:
3780 case Intrinsic::fshr:
3781 if (Cmp.isEquality() && II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) {
3782 // (rot X, ?) == 0/-1 --> X == 0/-1
3783 if (C.isZero() || C.isAllOnes())
3784 return new ICmpInst(Pred, II->getArgOperand(i: 0), Cmp.getOperand(i_nocapture: 1));
3785 }
3786 break;
3787 }
3788 }
3789
3790 return nullptr;
3791}
3792
3793/// Fold an icmp with BinaryOp and constant operand: icmp Pred BO, C.
3794Instruction *InstCombinerImpl::foldICmpBinOpWithConstant(ICmpInst &Cmp,
3795 BinaryOperator *BO,
3796 const APInt &C) {
3797 switch (BO->getOpcode()) {
3798 case Instruction::Xor:
3799 if (Instruction *I = foldICmpXorConstant(Cmp, Xor: BO, C))
3800 return I;
3801 break;
3802 case Instruction::And:
3803 if (Instruction *I = foldICmpAndConstant(Cmp, And: BO, C))
3804 return I;
3805 break;
3806 case Instruction::Or:
3807 if (Instruction *I = foldICmpOrConstant(Cmp, Or: BO, C))
3808 return I;
3809 break;
3810 case Instruction::Mul:
3811 if (Instruction *I = foldICmpMulConstant(Cmp, Mul: BO, C))
3812 return I;
3813 break;
3814 case Instruction::Shl:
3815 if (Instruction *I = foldICmpShlConstant(Cmp, Shl: BO, C))
3816 return I;
3817 break;
3818 case Instruction::LShr:
3819 case Instruction::AShr:
3820 if (Instruction *I = foldICmpShrConstant(Cmp, Shr: BO, C))
3821 return I;
3822 break;
3823 case Instruction::SRem:
3824 if (Instruction *I = foldICmpSRemConstant(Cmp, SRem: BO, C))
3825 return I;
3826 break;
3827 case Instruction::UDiv:
3828 if (Instruction *I = foldICmpUDivConstant(Cmp, UDiv: BO, C))
3829 return I;
3830 [[fallthrough]];
3831 case Instruction::SDiv:
3832 if (Instruction *I = foldICmpDivConstant(Cmp, Div: BO, C))
3833 return I;
3834 break;
3835 case Instruction::Sub:
3836 if (Instruction *I = foldICmpSubConstant(Cmp, Sub: BO, C))
3837 return I;
3838 break;
3839 case Instruction::Add:
3840 if (Instruction *I = foldICmpAddConstant(Cmp, Add: BO, C))
3841 return I;
3842 break;
3843 default:
3844 break;
3845 }
3846
3847 // TODO: These folds could be refactored to be part of the above calls.
3848 return foldICmpBinOpEqualityWithConstant(Cmp, BO, C);
3849}
3850
3851static Instruction *
3852foldICmpUSubSatOrUAddSatWithConstant(ICmpInst::Predicate Pred,
3853 SaturatingInst *II, const APInt &C,
3854 InstCombiner::BuilderTy &Builder) {
3855 // This transform may end up producing more than one instruction for the
3856 // intrinsic, so limit it to one user of the intrinsic.
3857 if (!II->hasOneUse())
3858 return nullptr;
3859
3860 // Let Y = [add/sub]_sat(X, C) pred C2
3861 // SatVal = The saturating value for the operation
3862 // WillWrap = Whether or not the operation will underflow / overflow
3863 // => Y = (WillWrap ? SatVal : (X binop C)) pred C2
3864 // => Y = WillWrap ? (SatVal pred C2) : ((X binop C) pred C2)
3865 //
3866 // When (SatVal pred C2) is true, then
3867 // Y = WillWrap ? true : ((X binop C) pred C2)
3868 // => Y = WillWrap || ((X binop C) pred C2)
3869 // else
3870 // Y = WillWrap ? false : ((X binop C) pred C2)
3871 // => Y = !WillWrap ? ((X binop C) pred C2) : false
3872 // => Y = !WillWrap && ((X binop C) pred C2)
3873 Value *Op0 = II->getOperand(i_nocapture: 0);
3874 Value *Op1 = II->getOperand(i_nocapture: 1);
3875
3876 const APInt *COp1;
3877 // This transform only works when the intrinsic has an integral constant or
3878 // splat vector as the second operand.
3879 if (!match(V: Op1, P: m_APInt(Res&: COp1)))
3880 return nullptr;
3881
3882 APInt SatVal;
3883 switch (II->getIntrinsicID()) {
3884 default:
3885 llvm_unreachable(
3886 "This function only works with usub_sat and uadd_sat for now!");
3887 case Intrinsic::uadd_sat:
3888 SatVal = APInt::getAllOnes(numBits: C.getBitWidth());
3889 break;
3890 case Intrinsic::usub_sat:
3891 SatVal = APInt::getZero(numBits: C.getBitWidth());
3892 break;
3893 }
3894
3895 // Check (SatVal pred C2)
3896 bool SatValCheck = ICmpInst::compare(LHS: SatVal, RHS: C, Pred);
3897
3898 // !WillWrap.
3899 ConstantRange C1 = ConstantRange::makeExactNoWrapRegion(
3900 BinOp: II->getBinaryOp(), Other: *COp1, NoWrapKind: II->getNoWrapKind());
3901
3902 // WillWrap.
3903 if (SatValCheck)
3904 C1 = C1.inverse();
3905
3906 ConstantRange C2 = ConstantRange::makeExactICmpRegion(Pred, Other: C);
3907 if (II->getBinaryOp() == Instruction::Add)
3908 C2 = C2.sub(Other: *COp1);
3909 else
3910 C2 = C2.add(Other: *COp1);
3911
3912 Instruction::BinaryOps CombiningOp =
3913 SatValCheck ? Instruction::BinaryOps::Or : Instruction::BinaryOps::And;
3914
3915 std::optional<ConstantRange> Combination;
3916 if (CombiningOp == Instruction::BinaryOps::Or)
3917 Combination = C1.exactUnionWith(CR: C2);
3918 else /* CombiningOp == Instruction::BinaryOps::And */
3919 Combination = C1.exactIntersectWith(CR: C2);
3920
3921 if (!Combination)
3922 return nullptr;
3923
3924 CmpInst::Predicate EquivPred;
3925 APInt EquivInt;
3926 APInt EquivOffset;
3927
3928 Combination->getEquivalentICmp(Pred&: EquivPred, RHS&: EquivInt, Offset&: EquivOffset);
3929
3930 return new ICmpInst(
3931 EquivPred,
3932 Builder.CreateAdd(LHS: Op0, RHS: ConstantInt::get(Ty: Op1->getType(), V: EquivOffset)),
3933 ConstantInt::get(Ty: Op1->getType(), V: EquivInt));
3934}
3935
3936static Instruction *
3937foldICmpOfCmpIntrinsicWithConstant(ICmpInst::Predicate Pred, IntrinsicInst *I,
3938 const APInt &C,
3939 InstCombiner::BuilderTy &Builder) {
3940 std::optional<ICmpInst::Predicate> NewPredicate = std::nullopt;
3941 switch (Pred) {
3942 case ICmpInst::ICMP_EQ:
3943 case ICmpInst::ICMP_NE:
3944 if (C.isZero())
3945 NewPredicate = Pred;
3946 else if (C.isOne())
3947 NewPredicate =
3948 Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_ULE;
3949 else if (C.isAllOnes())
3950 NewPredicate =
3951 Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE;
3952 break;
3953
3954 case ICmpInst::ICMP_SGT:
3955 if (C.isAllOnes())
3956 NewPredicate = ICmpInst::ICMP_UGE;
3957 else if (C.isZero())
3958 NewPredicate = ICmpInst::ICMP_UGT;
3959 break;
3960
3961 case ICmpInst::ICMP_SLT:
3962 if (C.isZero())
3963 NewPredicate = ICmpInst::ICMP_ULT;
3964 else if (C.isOne())
3965 NewPredicate = ICmpInst::ICMP_ULE;
3966 break;
3967
3968 default:
3969 break;
3970 }
3971
3972 if (!NewPredicate)
3973 return nullptr;
3974
3975 if (I->getIntrinsicID() == Intrinsic::scmp)
3976 NewPredicate = ICmpInst::getSignedPredicate(pred: *NewPredicate);
3977 Value *LHS = I->getOperand(i_nocapture: 0);
3978 Value *RHS = I->getOperand(i_nocapture: 1);
3979 return new ICmpInst(*NewPredicate, LHS, RHS);
3980}
3981
3982/// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
3983Instruction *InstCombinerImpl::foldICmpIntrinsicWithConstant(ICmpInst &Cmp,
3984 IntrinsicInst *II,
3985 const APInt &C) {
3986 ICmpInst::Predicate Pred = Cmp.getPredicate();
3987
3988 // Handle folds that apply for any kind of icmp.
3989 switch (II->getIntrinsicID()) {
3990 default:
3991 break;
3992 case Intrinsic::uadd_sat:
3993 case Intrinsic::usub_sat:
3994 if (auto *Folded = foldICmpUSubSatOrUAddSatWithConstant(
3995 Pred, II: cast<SaturatingInst>(Val: II), C, Builder))
3996 return Folded;
3997 break;
3998 case Intrinsic::ctpop: {
3999 const SimplifyQuery Q = SQ.getWithInstruction(I: &Cmp);
4000 if (Instruction *R = foldCtpopPow2Test(I&: Cmp, CtpopLhs: II, CRhs: C, Builder, Q))
4001 return R;
4002 } break;
4003 case Intrinsic::scmp:
4004 case Intrinsic::ucmp:
4005 if (auto *Folded = foldICmpOfCmpIntrinsicWithConstant(Pred, I: II, C, Builder))
4006 return Folded;
4007 break;
4008 }
4009
4010 if (Cmp.isEquality())
4011 return foldICmpEqIntrinsicWithConstant(Cmp, II, C);
4012
4013 Type *Ty = II->getType();
4014 unsigned BitWidth = C.getBitWidth();
4015 switch (II->getIntrinsicID()) {
4016 case Intrinsic::ctpop: {
4017 // (ctpop X > BitWidth - 1) --> X == -1
4018 Value *X = II->getArgOperand(i: 0);
4019 if (C == BitWidth - 1 && Pred == ICmpInst::ICMP_UGT)
4020 return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_EQ, S1: X,
4021 S2: ConstantInt::getAllOnesValue(Ty));
4022 // (ctpop X < BitWidth) --> X != -1
4023 if (C == BitWidth && Pred == ICmpInst::ICMP_ULT)
4024 return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_NE, S1: X,
4025 S2: ConstantInt::getAllOnesValue(Ty));
4026 break;
4027 }
4028 case Intrinsic::ctlz: {
4029 // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000
4030 if (Pred == ICmpInst::ICMP_UGT && C.ult(RHS: BitWidth)) {
4031 unsigned Num = C.getLimitedValue();
4032 APInt Limit = APInt::getOneBitSet(numBits: BitWidth, BitNo: BitWidth - Num - 1);
4033 return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_ULT,
4034 S1: II->getArgOperand(i: 0), S2: ConstantInt::get(Ty, V: Limit));
4035 }
4036
4037 // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111
4038 if (Pred == ICmpInst::ICMP_ULT && C.uge(RHS: 1) && C.ule(RHS: BitWidth)) {
4039 unsigned Num = C.getLimitedValue();
4040 APInt Limit = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - Num);
4041 return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_UGT,
4042 S1: II->getArgOperand(i: 0), S2: ConstantInt::get(Ty, V: Limit));
4043 }
4044 break;
4045 }
4046 case Intrinsic::cttz: {
4047 // Limit to one use to ensure we don't increase instruction count.
4048 if (!II->hasOneUse())
4049 return nullptr;
4050
4051 // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0
4052 if (Pred == ICmpInst::ICMP_UGT && C.ult(RHS: BitWidth)) {
4053 APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: C.getLimitedValue() + 1);
4054 return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_EQ,
4055 S1: Builder.CreateAnd(LHS: II->getArgOperand(i: 0), RHS: Mask),
4056 S2: ConstantInt::getNullValue(Ty));
4057 }
4058
4059 // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0
4060 if (Pred == ICmpInst::ICMP_ULT && C.uge(RHS: 1) && C.ule(RHS: BitWidth)) {
4061 APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: C.getLimitedValue());
4062 return CmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_NE,
4063 S1: Builder.CreateAnd(LHS: II->getArgOperand(i: 0), RHS: Mask),
4064 S2: ConstantInt::getNullValue(Ty));
4065 }
4066 break;
4067 }
4068 case Intrinsic::ssub_sat:
4069 // ssub.sat(a, b) spred 0 -> a spred b
4070 if (ICmpInst::isSigned(predicate: Pred)) {
4071 if (C.isZero())
4072 return new ICmpInst(Pred, II->getArgOperand(i: 0), II->getArgOperand(i: 1));
4073 // X s<= 0 is cannonicalized to X s< 1
4074 if (Pred == ICmpInst::ICMP_SLT && C.isOne())
4075 return new ICmpInst(ICmpInst::ICMP_SLE, II->getArgOperand(i: 0),
4076 II->getArgOperand(i: 1));
4077 // X s>= 0 is cannonicalized to X s> -1
4078 if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
4079 return new ICmpInst(ICmpInst::ICMP_SGE, II->getArgOperand(i: 0),
4080 II->getArgOperand(i: 1));
4081 }
4082 break;
4083 default:
4084 break;
4085 }
4086
4087 return nullptr;
4088}
4089
4090/// Handle icmp with constant (but not simple integer constant) RHS.
4091Instruction *InstCombinerImpl::foldICmpInstWithConstantNotInt(ICmpInst &I) {
4092 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
4093 Constant *RHSC = dyn_cast<Constant>(Val: Op1);
4094 Instruction *LHSI = dyn_cast<Instruction>(Val: Op0);
4095 if (!RHSC || !LHSI)
4096 return nullptr;
4097
4098 switch (LHSI->getOpcode()) {
4099 case Instruction::PHI:
4100 if (Instruction *NV = foldOpIntoPhi(I, PN: cast<PHINode>(Val: LHSI)))
4101 return NV;
4102 break;
4103 case Instruction::IntToPtr:
4104 // icmp pred inttoptr(X), null -> icmp pred X, 0
4105 if (RHSC->isNullValue() &&
4106 DL.getIntPtrType(RHSC->getType()) == LHSI->getOperand(i: 0)->getType())
4107 return new ICmpInst(
4108 I.getPredicate(), LHSI->getOperand(i: 0),
4109 Constant::getNullValue(Ty: LHSI->getOperand(i: 0)->getType()));
4110 break;
4111
4112 case Instruction::Load:
4113 // Try to optimize things like "A[i] > 4" to index computations.
4114 if (GetElementPtrInst *GEP =
4115 dyn_cast<GetElementPtrInst>(Val: LHSI->getOperand(i: 0)))
4116 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0)))
4117 if (Instruction *Res =
4118 foldCmpLoadFromIndexedGlobal(LI: cast<LoadInst>(Val: LHSI), GEP, GV, ICI&: I))
4119 return Res;
4120 break;
4121 }
4122
4123 return nullptr;
4124}
4125
4126Instruction *InstCombinerImpl::foldSelectICmp(ICmpInst::Predicate Pred,
4127 SelectInst *SI, Value *RHS,
4128 const ICmpInst &I) {
4129 // Try to fold the comparison into the select arms, which will cause the
4130 // select to be converted into a logical and/or.
4131 auto SimplifyOp = [&](Value *Op, bool SelectCondIsTrue) -> Value * {
4132 if (Value *Res = simplifyICmpInst(Predicate: Pred, LHS: Op, RHS, Q: SQ))
4133 return Res;
4134 if (std::optional<bool> Impl = isImpliedCondition(
4135 LHS: SI->getCondition(), RHSPred: Pred, RHSOp0: Op, RHSOp1: RHS, DL, LHSIsTrue: SelectCondIsTrue))
4136 return ConstantInt::get(Ty: I.getType(), V: *Impl);
4137 return nullptr;
4138 };
4139
4140 ConstantInt *CI = nullptr;
4141 Value *Op1 = SimplifyOp(SI->getOperand(i_nocapture: 1), true);
4142 if (Op1)
4143 CI = dyn_cast<ConstantInt>(Val: Op1);
4144
4145 Value *Op2 = SimplifyOp(SI->getOperand(i_nocapture: 2), false);
4146 if (Op2)
4147 CI = dyn_cast<ConstantInt>(Val: Op2);
4148
4149 // We only want to perform this transformation if it will not lead to
4150 // additional code. This is true if either both sides of the select
4151 // fold to a constant (in which case the icmp is replaced with a select
4152 // which will usually simplify) or this is the only user of the
4153 // select (in which case we are trading a select+icmp for a simpler
4154 // select+icmp) or all uses of the select can be replaced based on
4155 // dominance information ("Global cases").
4156 bool Transform = false;
4157 if (Op1 && Op2)
4158 Transform = true;
4159 else if (Op1 || Op2) {
4160 // Local case
4161 if (SI->hasOneUse())
4162 Transform = true;
4163 // Global cases
4164 else if (CI && !CI->isZero())
4165 // When Op1 is constant try replacing select with second operand.
4166 // Otherwise Op2 is constant and try replacing select with first
4167 // operand.
4168 Transform = replacedSelectWithOperand(SI, Icmp: &I, SIOpd: Op1 ? 2 : 1);
4169 }
4170 if (Transform) {
4171 if (!Op1)
4172 Op1 = Builder.CreateICmp(P: Pred, LHS: SI->getOperand(i_nocapture: 1), RHS, Name: I.getName());
4173 if (!Op2)
4174 Op2 = Builder.CreateICmp(P: Pred, LHS: SI->getOperand(i_nocapture: 2), RHS, Name: I.getName());
4175 return SelectInst::Create(C: SI->getOperand(i_nocapture: 0), S1: Op1, S2: Op2);
4176 }
4177
4178 return nullptr;
4179}
4180
4181// Returns whether V is a Mask ((X + 1) & X == 0) or ~Mask (-Pow2OrZero)
4182static bool isMaskOrZero(const Value *V, bool Not, const SimplifyQuery &Q,
4183 unsigned Depth = 0) {
4184 if (Not ? match(V, P: m_NegatedPower2OrZero()) : match(V, P: m_LowBitMaskOrZero()))
4185 return true;
4186 if (V->getType()->getScalarSizeInBits() == 1)
4187 return true;
4188 if (Depth++ >= MaxAnalysisRecursionDepth)
4189 return false;
4190 Value *X;
4191 const Instruction *I = dyn_cast<Instruction>(Val: V);
4192 if (!I)
4193 return false;
4194 switch (I->getOpcode()) {
4195 case Instruction::ZExt:
4196 // ZExt(Mask) is a Mask.
4197 return !Not && isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth);
4198 case Instruction::SExt:
4199 // SExt(Mask) is a Mask.
4200 // SExt(~Mask) is a ~Mask.
4201 return isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth);
4202 case Instruction::And:
4203 case Instruction::Or:
4204 // Mask0 | Mask1 is a Mask.
4205 // Mask0 & Mask1 is a Mask.
4206 // ~Mask0 | ~Mask1 is a ~Mask.
4207 // ~Mask0 & ~Mask1 is a ~Mask.
4208 return isMaskOrZero(V: I->getOperand(i: 1), Not, Q, Depth) &&
4209 isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth);
4210 case Instruction::Xor:
4211 if (match(V, P: m_Not(V: m_Value(V&: X))))
4212 return isMaskOrZero(V: X, Not: !Not, Q, Depth);
4213
4214 // (X ^ -X) is a ~Mask
4215 if (Not)
4216 return match(V, P: m_c_Xor(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X))));
4217 // (X ^ (X - 1)) is a Mask
4218 else
4219 return match(V, P: m_c_Xor(L: m_Value(V&: X), R: m_Add(L: m_Deferred(V: X), R: m_AllOnes())));
4220 case Instruction::Select:
4221 // c ? Mask0 : Mask1 is a Mask.
4222 return isMaskOrZero(V: I->getOperand(i: 1), Not, Q, Depth) &&
4223 isMaskOrZero(V: I->getOperand(i: 2), Not, Q, Depth);
4224 case Instruction::Shl:
4225 // (~Mask) << X is a ~Mask.
4226 return Not && isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth);
4227 case Instruction::LShr:
4228 // Mask >> X is a Mask.
4229 return !Not && isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth);
4230 case Instruction::AShr:
4231 // Mask s>> X is a Mask.
4232 // ~Mask s>> X is a ~Mask.
4233 return isMaskOrZero(V: I->getOperand(i: 0), Not, Q, Depth);
4234 case Instruction::Add:
4235 // Pow2 - 1 is a Mask.
4236 if (!Not && match(V: I->getOperand(i: 1), P: m_AllOnes()))
4237 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), DL: Q.DL, /*OrZero*/ true,
4238 Depth, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT);
4239 break;
4240 case Instruction::Sub:
4241 // -Pow2 is a ~Mask.
4242 if (Not && match(V: I->getOperand(i: 0), P: m_Zero()))
4243 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), DL: Q.DL, /*OrZero*/ true,
4244 Depth, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT);
4245 break;
4246 case Instruction::Call: {
4247 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
4248 switch (II->getIntrinsicID()) {
4249 // min/max(Mask0, Mask1) is a Mask.
4250 // min/max(~Mask0, ~Mask1) is a ~Mask.
4251 case Intrinsic::umax:
4252 case Intrinsic::smax:
4253 case Intrinsic::umin:
4254 case Intrinsic::smin:
4255 return isMaskOrZero(V: II->getArgOperand(i: 1), Not, Q, Depth) &&
4256 isMaskOrZero(V: II->getArgOperand(i: 0), Not, Q, Depth);
4257
4258 // In the context of masks, bitreverse(Mask) == ~Mask
4259 case Intrinsic::bitreverse:
4260 return isMaskOrZero(V: II->getArgOperand(i: 0), Not: !Not, Q, Depth);
4261 default:
4262 break;
4263 }
4264 }
4265 break;
4266 }
4267 default:
4268 break;
4269 }
4270 return false;
4271}
4272
4273/// Some comparisons can be simplified.
4274/// In this case, we are looking for comparisons that look like
4275/// a check for a lossy truncation.
4276/// Folds:
4277/// icmp SrcPred (x & Mask), x to icmp DstPred x, Mask
4278/// icmp SrcPred (x & ~Mask), ~Mask to icmp DstPred x, ~Mask
4279/// icmp eq/ne (x & ~Mask), 0 to icmp DstPred x, Mask
4280/// icmp eq/ne (~x | Mask), -1 to icmp DstPred x, Mask
4281/// Where Mask is some pattern that produces all-ones in low bits:
4282/// (-1 >> y)
4283/// ((-1 << y) >> y) <- non-canonical, has extra uses
4284/// ~(-1 << y)
4285/// ((1 << y) + (-1)) <- non-canonical, has extra uses
4286/// The Mask can be a constant, too.
4287/// For some predicates, the operands are commutative.
4288/// For others, x can only be on a specific side.
4289static Value *foldICmpWithLowBitMaskedVal(ICmpInst::Predicate Pred, Value *Op0,
4290 Value *Op1, const SimplifyQuery &Q,
4291 InstCombiner &IC) {
4292
4293 ICmpInst::Predicate DstPred;
4294 switch (Pred) {
4295 case ICmpInst::Predicate::ICMP_EQ:
4296 // x & Mask == x
4297 // x & ~Mask == 0
4298 // ~x | Mask == -1
4299 // -> x u<= Mask
4300 // x & ~Mask == ~Mask
4301 // -> ~Mask u<= x
4302 DstPred = ICmpInst::Predicate::ICMP_ULE;
4303 break;
4304 case ICmpInst::Predicate::ICMP_NE:
4305 // x & Mask != x
4306 // x & ~Mask != 0
4307 // ~x | Mask != -1
4308 // -> x u> Mask
4309 // x & ~Mask != ~Mask
4310 // -> ~Mask u> x
4311 DstPred = ICmpInst::Predicate::ICMP_UGT;
4312 break;
4313 case ICmpInst::Predicate::ICMP_ULT:
4314 // x & Mask u< x
4315 // -> x u> Mask
4316 // x & ~Mask u< ~Mask
4317 // -> ~Mask u> x
4318 DstPred = ICmpInst::Predicate::ICMP_UGT;
4319 break;
4320 case ICmpInst::Predicate::ICMP_UGE:
4321 // x & Mask u>= x
4322 // -> x u<= Mask
4323 // x & ~Mask u>= ~Mask
4324 // -> ~Mask u<= x
4325 DstPred = ICmpInst::Predicate::ICMP_ULE;
4326 break;
4327 case ICmpInst::Predicate::ICMP_SLT:
4328 // x & Mask s< x [iff Mask s>= 0]
4329 // -> x s> Mask
4330 // x & ~Mask s< ~Mask [iff ~Mask != 0]
4331 // -> ~Mask s> x
4332 DstPred = ICmpInst::Predicate::ICMP_SGT;
4333 break;
4334 case ICmpInst::Predicate::ICMP_SGE:
4335 // x & Mask s>= x [iff Mask s>= 0]
4336 // -> x s<= Mask
4337 // x & ~Mask s>= ~Mask [iff ~Mask != 0]
4338 // -> ~Mask s<= x
4339 DstPred = ICmpInst::Predicate::ICMP_SLE;
4340 break;
4341 default:
4342 // We don't support sgt,sle
4343 // ult/ugt are simplified to true/false respectively.
4344 return nullptr;
4345 }
4346
4347 Value *X, *M;
4348 // Put search code in lambda for early positive returns.
4349 auto IsLowBitMask = [&]() {
4350 if (match(V: Op0, P: m_c_And(L: m_Specific(V: Op1), R: m_Value(V&: M)))) {
4351 X = Op1;
4352 // Look for: x & Mask pred x
4353 if (isMaskOrZero(V: M, /*Not=*/false, Q)) {
4354 return !ICmpInst::isSigned(predicate: Pred) ||
4355 (match(V: M, P: m_NonNegative()) || isKnownNonNegative(V: M, SQ: Q));
4356 }
4357
4358 // Look for: x & ~Mask pred ~Mask
4359 if (isMaskOrZero(V: X, /*Not=*/true, Q)) {
4360 return !ICmpInst::isSigned(predicate: Pred) || isKnownNonZero(V: X, Q);
4361 }
4362 return false;
4363 }
4364 if (ICmpInst::isEquality(P: Pred) && match(V: Op1, P: m_AllOnes()) &&
4365 match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: X), R: m_Value(V&: M))))) {
4366
4367 auto Check = [&]() {
4368 // Look for: ~x | Mask == -1
4369 if (isMaskOrZero(V: M, /*Not=*/false, Q)) {
4370 if (Value *NotX =
4371 IC.getFreelyInverted(V: X, WillInvertAllUses: X->hasOneUse(), Builder: &IC.Builder)) {
4372 X = NotX;
4373 return true;
4374 }
4375 }
4376 return false;
4377 };
4378 if (Check())
4379 return true;
4380 std::swap(a&: X, b&: M);
4381 return Check();
4382 }
4383 if (ICmpInst::isEquality(P: Pred) && match(V: Op1, P: m_Zero()) &&
4384 match(V: Op0, P: m_OneUse(SubPattern: m_And(L: m_Value(V&: X), R: m_Value(V&: M))))) {
4385 auto Check = [&]() {
4386 // Look for: x & ~Mask == 0
4387 if (isMaskOrZero(V: M, /*Not=*/true, Q)) {
4388 if (Value *NotM =
4389 IC.getFreelyInverted(V: M, WillInvertAllUses: M->hasOneUse(), Builder: &IC.Builder)) {
4390 M = NotM;
4391 return true;
4392 }
4393 }
4394 return false;
4395 };
4396 if (Check())
4397 return true;
4398 std::swap(a&: X, b&: M);
4399 return Check();
4400 }
4401 return false;
4402 };
4403
4404 if (!IsLowBitMask())
4405 return nullptr;
4406
4407 return IC.Builder.CreateICmp(P: DstPred, LHS: X, RHS: M);
4408}
4409
4410/// Some comparisons can be simplified.
4411/// In this case, we are looking for comparisons that look like
4412/// a check for a lossy signed truncation.
4413/// Folds: (MaskedBits is a constant.)
4414/// ((%x << MaskedBits) a>> MaskedBits) SrcPred %x
4415/// Into:
4416/// (add %x, (1 << (KeptBits-1))) DstPred (1 << KeptBits)
4417/// Where KeptBits = bitwidth(%x) - MaskedBits
4418static Value *
4419foldICmpWithTruncSignExtendedVal(ICmpInst &I,
4420 InstCombiner::BuilderTy &Builder) {
4421 ICmpInst::Predicate SrcPred;
4422 Value *X;
4423 const APInt *C0, *C1; // FIXME: non-splats, potentially with undef.
4424 // We are ok with 'shl' having multiple uses, but 'ashr' must be one-use.
4425 if (!match(V: &I, P: m_c_ICmp(Pred&: SrcPred,
4426 L: m_OneUse(SubPattern: m_AShr(L: m_Shl(L: m_Value(V&: X), R: m_APInt(Res&: C0)),
4427 R: m_APInt(Res&: C1))),
4428 R: m_Deferred(V: X))))
4429 return nullptr;
4430
4431 // Potential handling of non-splats: for each element:
4432 // * if both are undef, replace with constant 0.
4433 // Because (1<<0) is OK and is 1, and ((1<<0)>>1) is also OK and is 0.
4434 // * if both are not undef, and are different, bailout.
4435 // * else, only one is undef, then pick the non-undef one.
4436
4437 // The shift amount must be equal.
4438 if (*C0 != *C1)
4439 return nullptr;
4440 const APInt &MaskedBits = *C0;
4441 assert(MaskedBits != 0 && "shift by zero should be folded away already.");
4442
4443 ICmpInst::Predicate DstPred;
4444 switch (SrcPred) {
4445 case ICmpInst::Predicate::ICMP_EQ:
4446 // ((%x << MaskedBits) a>> MaskedBits) == %x
4447 // =>
4448 // (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits)
4449 DstPred = ICmpInst::Predicate::ICMP_ULT;
4450 break;
4451 case ICmpInst::Predicate::ICMP_NE:
4452 // ((%x << MaskedBits) a>> MaskedBits) != %x
4453 // =>
4454 // (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits)
4455 DstPred = ICmpInst::Predicate::ICMP_UGE;
4456 break;
4457 // FIXME: are more folds possible?
4458 default:
4459 return nullptr;
4460 }
4461
4462 auto *XType = X->getType();
4463 const unsigned XBitWidth = XType->getScalarSizeInBits();
4464 const APInt BitWidth = APInt(XBitWidth, XBitWidth);
4465 assert(BitWidth.ugt(MaskedBits) && "shifts should leave some bits untouched");
4466
4467 // KeptBits = bitwidth(%x) - MaskedBits
4468 const APInt KeptBits = BitWidth - MaskedBits;
4469 assert(KeptBits.ugt(0) && KeptBits.ult(BitWidth) && "unreachable");
4470 // ICmpCst = (1 << KeptBits)
4471 const APInt ICmpCst = APInt(XBitWidth, 1).shl(ShiftAmt: KeptBits);
4472 assert(ICmpCst.isPowerOf2());
4473 // AddCst = (1 << (KeptBits-1))
4474 const APInt AddCst = ICmpCst.lshr(shiftAmt: 1);
4475 assert(AddCst.ult(ICmpCst) && AddCst.isPowerOf2());
4476
4477 // T0 = add %x, AddCst
4478 Value *T0 = Builder.CreateAdd(LHS: X, RHS: ConstantInt::get(Ty: XType, V: AddCst));
4479 // T1 = T0 DstPred ICmpCst
4480 Value *T1 = Builder.CreateICmp(P: DstPred, LHS: T0, RHS: ConstantInt::get(Ty: XType, V: ICmpCst));
4481
4482 return T1;
4483}
4484
4485// Given pattern:
4486// icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
4487// we should move shifts to the same hand of 'and', i.e. rewrite as
4488// icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x)
4489// We are only interested in opposite logical shifts here.
4490// One of the shifts can be truncated.
4491// If we can, we want to end up creating 'lshr' shift.
4492static Value *
4493foldShiftIntoShiftInAnotherHandOfAndInICmp(ICmpInst &I, const SimplifyQuery SQ,
4494 InstCombiner::BuilderTy &Builder) {
4495 if (!I.isEquality() || !match(V: I.getOperand(i_nocapture: 1), P: m_Zero()) ||
4496 !I.getOperand(i_nocapture: 0)->hasOneUse())
4497 return nullptr;
4498
4499 auto m_AnyLogicalShift = m_LogicalShift(L: m_Value(), R: m_Value());
4500
4501 // Look for an 'and' of two logical shifts, one of which may be truncated.
4502 // We use m_TruncOrSelf() on the RHS to correctly handle commutative case.
4503 Instruction *XShift, *MaybeTruncation, *YShift;
4504 if (!match(
4505 V: I.getOperand(i_nocapture: 0),
4506 P: m_c_And(L: m_CombineAnd(L: m_AnyLogicalShift, R: m_Instruction(I&: XShift)),
4507 R: m_CombineAnd(L: m_TruncOrSelf(Op: m_CombineAnd(
4508 L: m_AnyLogicalShift, R: m_Instruction(I&: YShift))),
4509 R: m_Instruction(I&: MaybeTruncation)))))
4510 return nullptr;
4511
4512 // We potentially looked past 'trunc', but only when matching YShift,
4513 // therefore YShift must have the widest type.
4514 Instruction *WidestShift = YShift;
4515 // Therefore XShift must have the shallowest type.
4516 // Or they both have identical types if there was no truncation.
4517 Instruction *NarrowestShift = XShift;
4518
4519 Type *WidestTy = WidestShift->getType();
4520 Type *NarrowestTy = NarrowestShift->getType();
4521 assert(NarrowestTy == I.getOperand(0)->getType() &&
4522 "We did not look past any shifts while matching XShift though.");
4523 bool HadTrunc = WidestTy != I.getOperand(i_nocapture: 0)->getType();
4524
4525 // If YShift is a 'lshr', swap the shifts around.
4526 if (match(V: YShift, P: m_LShr(L: m_Value(), R: m_Value())))
4527 std::swap(a&: XShift, b&: YShift);
4528
4529 // The shifts must be in opposite directions.
4530 auto XShiftOpcode = XShift->getOpcode();
4531 if (XShiftOpcode == YShift->getOpcode())
4532 return nullptr; // Do not care about same-direction shifts here.
4533
4534 Value *X, *XShAmt, *Y, *YShAmt;
4535 match(V: XShift, P: m_BinOp(L: m_Value(V&: X), R: m_ZExtOrSelf(Op: m_Value(V&: XShAmt))));
4536 match(V: YShift, P: m_BinOp(L: m_Value(V&: Y), R: m_ZExtOrSelf(Op: m_Value(V&: YShAmt))));
4537
4538 // If one of the values being shifted is a constant, then we will end with
4539 // and+icmp, and [zext+]shift instrs will be constant-folded. If they are not,
4540 // however, we will need to ensure that we won't increase instruction count.
4541 if (!isa<Constant>(Val: X) && !isa<Constant>(Val: Y)) {
4542 // At least one of the hands of the 'and' should be one-use shift.
4543 if (!match(V: I.getOperand(i_nocapture: 0),
4544 P: m_c_And(L: m_OneUse(SubPattern: m_AnyLogicalShift), R: m_Value())))
4545 return nullptr;
4546 if (HadTrunc) {
4547 // Due to the 'trunc', we will need to widen X. For that either the old
4548 // 'trunc' or the shift amt in the non-truncated shift should be one-use.
4549 if (!MaybeTruncation->hasOneUse() &&
4550 !NarrowestShift->getOperand(i: 1)->hasOneUse())
4551 return nullptr;
4552 }
4553 }
4554
4555 // We have two shift amounts from two different shifts. The types of those
4556 // shift amounts may not match. If that's the case let's bailout now.
4557 if (XShAmt->getType() != YShAmt->getType())
4558 return nullptr;
4559
4560 // As input, we have the following pattern:
4561 // icmp eq/ne (and ((x shift Q), (y oppositeshift K))), 0
4562 // We want to rewrite that as:
4563 // icmp eq/ne (and (x shift (Q+K)), y), 0 iff (Q+K) u< bitwidth(x)
4564 // While we know that originally (Q+K) would not overflow
4565 // (because 2 * (N-1) u<= iN -1), we have looked past extensions of
4566 // shift amounts. so it may now overflow in smaller bitwidth.
4567 // To ensure that does not happen, we need to ensure that the total maximal
4568 // shift amount is still representable in that smaller bit width.
4569 unsigned MaximalPossibleTotalShiftAmount =
4570 (WidestTy->getScalarSizeInBits() - 1) +
4571 (NarrowestTy->getScalarSizeInBits() - 1);
4572 APInt MaximalRepresentableShiftAmount =
4573 APInt::getAllOnes(numBits: XShAmt->getType()->getScalarSizeInBits());
4574 if (MaximalRepresentableShiftAmount.ult(RHS: MaximalPossibleTotalShiftAmount))
4575 return nullptr;
4576
4577 // Can we fold (XShAmt+YShAmt) ?
4578 auto *NewShAmt = dyn_cast_or_null<Constant>(
4579 Val: simplifyAddInst(LHS: XShAmt, RHS: YShAmt, /*isNSW=*/IsNSW: false,
4580 /*isNUW=*/IsNUW: false, Q: SQ.getWithInstruction(I: &I)));
4581 if (!NewShAmt)
4582 return nullptr;
4583 if (NewShAmt->getType() != WidestTy) {
4584 NewShAmt =
4585 ConstantFoldCastOperand(Opcode: Instruction::ZExt, C: NewShAmt, DestTy: WidestTy, DL: SQ.DL);
4586 if (!NewShAmt)
4587 return nullptr;
4588 }
4589 unsigned WidestBitWidth = WidestTy->getScalarSizeInBits();
4590
4591 // Is the new shift amount smaller than the bit width?
4592 // FIXME: could also rely on ConstantRange.
4593 if (!match(V: NewShAmt,
4594 P: m_SpecificInt_ICMP(Predicate: ICmpInst::Predicate::ICMP_ULT,
4595 Threshold: APInt(WidestBitWidth, WidestBitWidth))))
4596 return nullptr;
4597
4598 // An extra legality check is needed if we had trunc-of-lshr.
4599 if (HadTrunc && match(V: WidestShift, P: m_LShr(L: m_Value(), R: m_Value()))) {
4600 auto CanFold = [NewShAmt, WidestBitWidth, NarrowestShift, SQ,
4601 WidestShift]() {
4602 // It isn't obvious whether it's worth it to analyze non-constants here.
4603 // Also, let's basically give up on non-splat cases, pessimizing vectors.
4604 // If *any* of these preconditions matches we can perform the fold.
4605 Constant *NewShAmtSplat = NewShAmt->getType()->isVectorTy()
4606 ? NewShAmt->getSplatValue()
4607 : NewShAmt;
4608 // If it's edge-case shift (by 0 or by WidestBitWidth-1) we can fold.
4609 if (NewShAmtSplat &&
4610 (NewShAmtSplat->isNullValue() ||
4611 NewShAmtSplat->getUniqueInteger() == WidestBitWidth - 1))
4612 return true;
4613 // We consider *min* leading zeros so a single outlier
4614 // blocks the transform as opposed to allowing it.
4615 if (auto *C = dyn_cast<Constant>(Val: NarrowestShift->getOperand(i: 0))) {
4616 KnownBits Known = computeKnownBits(V: C, DL: SQ.DL);
4617 unsigned MinLeadZero = Known.countMinLeadingZeros();
4618 // If the value being shifted has at most lowest bit set we can fold.
4619 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
4620 if (MaxActiveBits <= 1)
4621 return true;
4622 // Precondition: NewShAmt u<= countLeadingZeros(C)
4623 if (NewShAmtSplat && NewShAmtSplat->getUniqueInteger().ule(RHS: MinLeadZero))
4624 return true;
4625 }
4626 if (auto *C = dyn_cast<Constant>(Val: WidestShift->getOperand(i: 0))) {
4627 KnownBits Known = computeKnownBits(V: C, DL: SQ.DL);
4628 unsigned MinLeadZero = Known.countMinLeadingZeros();
4629 // If the value being shifted has at most lowest bit set we can fold.
4630 unsigned MaxActiveBits = Known.getBitWidth() - MinLeadZero;
4631 if (MaxActiveBits <= 1)
4632 return true;
4633 // Precondition: ((WidestBitWidth-1)-NewShAmt) u<= countLeadingZeros(C)
4634 if (NewShAmtSplat) {
4635 APInt AdjNewShAmt =
4636 (WidestBitWidth - 1) - NewShAmtSplat->getUniqueInteger();
4637 if (AdjNewShAmt.ule(RHS: MinLeadZero))
4638 return true;
4639 }
4640 }
4641 return false; // Can't tell if it's ok.
4642 };
4643 if (!CanFold())
4644 return nullptr;
4645 }
4646
4647 // All good, we can do this fold.
4648 X = Builder.CreateZExt(V: X, DestTy: WidestTy);
4649 Y = Builder.CreateZExt(V: Y, DestTy: WidestTy);
4650 // The shift is the same that was for X.
4651 Value *T0 = XShiftOpcode == Instruction::BinaryOps::LShr
4652 ? Builder.CreateLShr(LHS: X, RHS: NewShAmt)
4653 : Builder.CreateShl(LHS: X, RHS: NewShAmt);
4654 Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Y);
4655 return Builder.CreateICmp(P: I.getPredicate(), LHS: T1,
4656 RHS: Constant::getNullValue(Ty: WidestTy));
4657}
4658
4659/// Fold
4660/// (-1 u/ x) u< y
4661/// ((x * y) ?/ x) != y
4662/// to
4663/// @llvm.?mul.with.overflow(x, y) plus extraction of overflow bit
4664/// Note that the comparison is commutative, while inverted (u>=, ==) predicate
4665/// will mean that we are looking for the opposite answer.
4666Value *InstCombinerImpl::foldMultiplicationOverflowCheck(ICmpInst &I) {
4667 ICmpInst::Predicate Pred;
4668 Value *X, *Y;
4669 Instruction *Mul;
4670 Instruction *Div;
4671 bool NeedNegation;
4672 // Look for: (-1 u/ x) u</u>= y
4673 if (!I.isEquality() &&
4674 match(V: &I, P: m_c_ICmp(Pred,
4675 L: m_CombineAnd(L: m_OneUse(SubPattern: m_UDiv(L: m_AllOnes(), R: m_Value(V&: X))),
4676 R: m_Instruction(I&: Div)),
4677 R: m_Value(V&: Y)))) {
4678 Mul = nullptr;
4679
4680 // Are we checking that overflow does not happen, or does happen?
4681 switch (Pred) {
4682 case ICmpInst::Predicate::ICMP_ULT:
4683 NeedNegation = false;
4684 break; // OK
4685 case ICmpInst::Predicate::ICMP_UGE:
4686 NeedNegation = true;
4687 break; // OK
4688 default:
4689 return nullptr; // Wrong predicate.
4690 }
4691 } else // Look for: ((x * y) / x) !=/== y
4692 if (I.isEquality() &&
4693 match(V: &I,
4694 P: m_c_ICmp(Pred, L: m_Value(V&: Y),
4695 R: m_CombineAnd(
4696 L: m_OneUse(SubPattern: m_IDiv(L: m_CombineAnd(L: m_c_Mul(L: m_Deferred(V: Y),
4697 R: m_Value(V&: X)),
4698 R: m_Instruction(I&: Mul)),
4699 R: m_Deferred(V: X))),
4700 R: m_Instruction(I&: Div))))) {
4701 NeedNegation = Pred == ICmpInst::Predicate::ICMP_EQ;
4702 } else
4703 return nullptr;
4704
4705 BuilderTy::InsertPointGuard Guard(Builder);
4706 // If the pattern included (x * y), we'll want to insert new instructions
4707 // right before that original multiplication so that we can replace it.
4708 bool MulHadOtherUses = Mul && !Mul->hasOneUse();
4709 if (MulHadOtherUses)
4710 Builder.SetInsertPoint(Mul);
4711
4712 Function *F = Intrinsic::getDeclaration(M: I.getModule(),
4713 id: Div->getOpcode() == Instruction::UDiv
4714 ? Intrinsic::umul_with_overflow
4715 : Intrinsic::smul_with_overflow,
4716 Tys: X->getType());
4717 CallInst *Call = Builder.CreateCall(Callee: F, Args: {X, Y}, Name: "mul");
4718
4719 // If the multiplication was used elsewhere, to ensure that we don't leave
4720 // "duplicate" instructions, replace uses of that original multiplication
4721 // with the multiplication result from the with.overflow intrinsic.
4722 if (MulHadOtherUses)
4723 replaceInstUsesWith(I&: *Mul, V: Builder.CreateExtractValue(Agg: Call, Idxs: 0, Name: "mul.val"));
4724
4725 Value *Res = Builder.CreateExtractValue(Agg: Call, Idxs: 1, Name: "mul.ov");
4726 if (NeedNegation) // This technically increases instruction count.
4727 Res = Builder.CreateNot(V: Res, Name: "mul.not.ov");
4728
4729 // If we replaced the mul, erase it. Do this after all uses of Builder,
4730 // as the mul is used as insertion point.
4731 if (MulHadOtherUses)
4732 eraseInstFromFunction(I&: *Mul);
4733
4734 return Res;
4735}
4736
4737static Instruction *foldICmpXNegX(ICmpInst &I,
4738 InstCombiner::BuilderTy &Builder) {
4739 CmpInst::Predicate Pred;
4740 Value *X;
4741 if (match(V: &I, P: m_c_ICmp(Pred, L: m_NSWNeg(V: m_Value(V&: X)), R: m_Deferred(V: X)))) {
4742
4743 if (ICmpInst::isSigned(predicate: Pred))
4744 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
4745 else if (ICmpInst::isUnsigned(predicate: Pred))
4746 Pred = ICmpInst::getSignedPredicate(pred: Pred);
4747 // else for equality-comparisons just keep the predicate.
4748
4749 return ICmpInst::Create(Op: Instruction::ICmp, Pred, S1: X,
4750 S2: Constant::getNullValue(Ty: X->getType()), Name: I.getName());
4751 }
4752
4753 // A value is not equal to its negation unless that value is 0 or
4754 // MinSignedValue, ie: a != -a --> (a & MaxSignedVal) != 0
4755 if (match(V: &I, P: m_c_ICmp(Pred, L: m_OneUse(SubPattern: m_Neg(V: m_Value(V&: X))), R: m_Deferred(V: X))) &&
4756 ICmpInst::isEquality(P: Pred)) {
4757 Type *Ty = X->getType();
4758 uint32_t BitWidth = Ty->getScalarSizeInBits();
4759 Constant *MaxSignedVal =
4760 ConstantInt::get(Ty, V: APInt::getSignedMaxValue(numBits: BitWidth));
4761 Value *And = Builder.CreateAnd(LHS: X, RHS: MaxSignedVal);
4762 Constant *Zero = Constant::getNullValue(Ty);
4763 return CmpInst::Create(Op: Instruction::ICmp, Pred, S1: And, S2: Zero);
4764 }
4765
4766 return nullptr;
4767}
4768
4769static Instruction *foldICmpAndXX(ICmpInst &I, const SimplifyQuery &Q,
4770 InstCombinerImpl &IC) {
4771 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1), *A;
4772 // Normalize and operand as operand 0.
4773 CmpInst::Predicate Pred = I.getPredicate();
4774 if (match(V: Op1, P: m_c_And(L: m_Specific(V: Op0), R: m_Value()))) {
4775 std::swap(a&: Op0, b&: Op1);
4776 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
4777 }
4778
4779 if (!match(V: Op0, P: m_c_And(L: m_Specific(V: Op1), R: m_Value(V&: A))))
4780 return nullptr;
4781
4782 // (icmp (X & Y) u< X --> (X & Y) != X
4783 if (Pred == ICmpInst::ICMP_ULT)
4784 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
4785
4786 // (icmp (X & Y) u>= X --> (X & Y) == X
4787 if (Pred == ICmpInst::ICMP_UGE)
4788 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
4789
4790 if (ICmpInst::isEquality(P: Pred) && Op0->hasOneUse()) {
4791 // icmp (X & Y) eq/ne Y --> (X | ~Y) eq/ne -1 if Y is freely invertible and
4792 // Y is non-constant. If Y is constant the `X & C == C` form is preferable
4793 // so don't do this fold.
4794 if (!match(V: Op1, P: m_ImmConstant()))
4795 if (auto *NotOp1 =
4796 IC.getFreelyInverted(V: Op1, WillInvertAllUses: !Op1->hasNUsesOrMore(N: 3), Builder: &IC.Builder))
4797 return new ICmpInst(Pred, IC.Builder.CreateOr(LHS: A, RHS: NotOp1),
4798 Constant::getAllOnesValue(Ty: Op1->getType()));
4799 // icmp (X & Y) eq/ne Y --> (~X & Y) eq/ne 0 if X is freely invertible.
4800 if (auto *NotA = IC.getFreelyInverted(V: A, WillInvertAllUses: A->hasOneUse(), Builder: &IC.Builder))
4801 return new ICmpInst(Pred, IC.Builder.CreateAnd(LHS: Op1, RHS: NotA),
4802 Constant::getNullValue(Ty: Op1->getType()));
4803 }
4804
4805 if (!ICmpInst::isSigned(predicate: Pred))
4806 return nullptr;
4807
4808 KnownBits KnownY = IC.computeKnownBits(V: A, /*Depth=*/0, CxtI: &I);
4809 // (X & NegY) spred X --> (X & NegY) upred X
4810 if (KnownY.isNegative())
4811 return new ICmpInst(ICmpInst::getUnsignedPredicate(pred: Pred), Op0, Op1);
4812
4813 if (Pred != ICmpInst::ICMP_SLE && Pred != ICmpInst::ICMP_SGT)
4814 return nullptr;
4815
4816 if (KnownY.isNonNegative())
4817 // (X & PosY) s<= X --> X s>= 0
4818 // (X & PosY) s> X --> X s< 0
4819 return new ICmpInst(ICmpInst::getSwappedPredicate(pred: Pred), Op1,
4820 Constant::getNullValue(Ty: Op1->getType()));
4821
4822 if (isKnownNegative(V: Op1, DL: IC.getSimplifyQuery().getWithInstruction(I: &I)))
4823 // (NegX & Y) s<= NegX --> Y s< 0
4824 // (NegX & Y) s> NegX --> Y s>= 0
4825 return new ICmpInst(ICmpInst::getFlippedStrictnessPredicate(pred: Pred), A,
4826 Constant::getNullValue(Ty: A->getType()));
4827
4828 return nullptr;
4829}
4830
4831static Instruction *foldICmpOrXX(ICmpInst &I, const SimplifyQuery &Q,
4832 InstCombinerImpl &IC) {
4833 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1), *A;
4834
4835 // Normalize or operand as operand 0.
4836 CmpInst::Predicate Pred = I.getPredicate();
4837 if (match(V: Op1, P: m_c_Or(L: m_Specific(V: Op0), R: m_Value(V&: A)))) {
4838 std::swap(a&: Op0, b&: Op1);
4839 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
4840 } else if (!match(V: Op0, P: m_c_Or(L: m_Specific(V: Op1), R: m_Value(V&: A)))) {
4841 return nullptr;
4842 }
4843
4844 // icmp (X | Y) u<= X --> (X | Y) == X
4845 if (Pred == ICmpInst::ICMP_ULE)
4846 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
4847
4848 // icmp (X | Y) u> X --> (X | Y) != X
4849 if (Pred == ICmpInst::ICMP_UGT)
4850 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
4851
4852 if (ICmpInst::isEquality(P: Pred) && Op0->hasOneUse()) {
4853 // icmp (X | Y) eq/ne Y --> (X & ~Y) eq/ne 0 if Y is freely invertible
4854 if (Value *NotOp1 =
4855 IC.getFreelyInverted(V: Op1, WillInvertAllUses: !Op1->hasNUsesOrMore(N: 3), Builder: &IC.Builder))
4856 return new ICmpInst(Pred, IC.Builder.CreateAnd(LHS: A, RHS: NotOp1),
4857 Constant::getNullValue(Ty: Op1->getType()));
4858 // icmp (X | Y) eq/ne Y --> (~X | Y) eq/ne -1 if X is freely invertible.
4859 if (Value *NotA = IC.getFreelyInverted(V: A, WillInvertAllUses: A->hasOneUse(), Builder: &IC.Builder))
4860 return new ICmpInst(Pred, IC.Builder.CreateOr(LHS: Op1, RHS: NotA),
4861 Constant::getAllOnesValue(Ty: Op1->getType()));
4862 }
4863 return nullptr;
4864}
4865
4866static Instruction *foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q,
4867 InstCombinerImpl &IC) {
4868 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1), *A;
4869 // Normalize xor operand as operand 0.
4870 CmpInst::Predicate Pred = I.getPredicate();
4871 if (match(V: Op1, P: m_c_Xor(L: m_Specific(V: Op0), R: m_Value()))) {
4872 std::swap(a&: Op0, b&: Op1);
4873 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
4874 }
4875 if (!match(V: Op0, P: m_c_Xor(L: m_Specific(V: Op1), R: m_Value(V&: A))))
4876 return nullptr;
4877
4878 // icmp (X ^ Y_NonZero) u>= X --> icmp (X ^ Y_NonZero) u> X
4879 // icmp (X ^ Y_NonZero) u<= X --> icmp (X ^ Y_NonZero) u< X
4880 // icmp (X ^ Y_NonZero) s>= X --> icmp (X ^ Y_NonZero) s> X
4881 // icmp (X ^ Y_NonZero) s<= X --> icmp (X ^ Y_NonZero) s< X
4882 CmpInst::Predicate PredOut = CmpInst::getStrictPredicate(pred: Pred);
4883 if (PredOut != Pred && isKnownNonZero(V: A, Q))
4884 return new ICmpInst(PredOut, Op0, Op1);
4885
4886 return nullptr;
4887}
4888
4889/// Try to fold icmp (binop), X or icmp X, (binop).
4890/// TODO: A large part of this logic is duplicated in InstSimplify's
4891/// simplifyICmpWithBinOp(). We should be able to share that and avoid the code
4892/// duplication.
4893Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
4894 const SimplifyQuery &SQ) {
4895 const SimplifyQuery Q = SQ.getWithInstruction(I: &I);
4896 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
4897
4898 // Special logic for binary operators.
4899 BinaryOperator *BO0 = dyn_cast<BinaryOperator>(Val: Op0);
4900 BinaryOperator *BO1 = dyn_cast<BinaryOperator>(Val: Op1);
4901 if (!BO0 && !BO1)
4902 return nullptr;
4903
4904 if (Instruction *NewICmp = foldICmpXNegX(I, Builder))
4905 return NewICmp;
4906
4907 const CmpInst::Predicate Pred = I.getPredicate();
4908 Value *X;
4909
4910 // Convert add-with-unsigned-overflow comparisons into a 'not' with compare.
4911 // (Op1 + X) u</u>= Op1 --> ~Op1 u</u>= X
4912 if (match(V: Op0, P: m_OneUse(SubPattern: m_c_Add(L: m_Specific(V: Op1), R: m_Value(V&: X)))) &&
4913 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
4914 return new ICmpInst(Pred, Builder.CreateNot(V: Op1), X);
4915 // Op0 u>/u<= (Op0 + X) --> X u>/u<= ~Op0
4916 if (match(V: Op1, P: m_OneUse(SubPattern: m_c_Add(L: m_Specific(V: Op0), R: m_Value(V&: X)))) &&
4917 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
4918 return new ICmpInst(Pred, X, Builder.CreateNot(V: Op0));
4919
4920 {
4921 // (Op1 + X) + C u</u>= Op1 --> ~C - X u</u>= Op1
4922 Constant *C;
4923 if (match(V: Op0, P: m_OneUse(SubPattern: m_Add(L: m_c_Add(L: m_Specific(V: Op1), R: m_Value(V&: X)),
4924 R: m_ImmConstant(C)))) &&
4925 (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE)) {
4926 Constant *C2 = ConstantExpr::getNot(C);
4927 return new ICmpInst(Pred, Builder.CreateSub(LHS: C2, RHS: X), Op1);
4928 }
4929 // Op0 u>/u<= (Op0 + X) + C --> Op0 u>/u<= ~C - X
4930 if (match(V: Op1, P: m_OneUse(SubPattern: m_Add(L: m_c_Add(L: m_Specific(V: Op0), R: m_Value(V&: X)),
4931 R: m_ImmConstant(C)))) &&
4932 (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE)) {
4933 Constant *C2 = ConstantExpr::getNot(C);
4934 return new ICmpInst(Pred, Op0, Builder.CreateSub(LHS: C2, RHS: X));
4935 }
4936 }
4937
4938 {
4939 // Similar to above: an unsigned overflow comparison may use offset + mask:
4940 // ((Op1 + C) & C) u< Op1 --> Op1 != 0
4941 // ((Op1 + C) & C) u>= Op1 --> Op1 == 0
4942 // Op0 u> ((Op0 + C) & C) --> Op0 != 0
4943 // Op0 u<= ((Op0 + C) & C) --> Op0 == 0
4944 BinaryOperator *BO;
4945 const APInt *C;
4946 if ((Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE) &&
4947 match(V: Op0, P: m_And(L: m_BinOp(I&: BO), R: m_LowBitMask(V&: C))) &&
4948 match(V: BO, P: m_Add(L: m_Specific(V: Op1), R: m_SpecificIntAllowPoison(V: *C)))) {
4949 CmpInst::Predicate NewPred =
4950 Pred == ICmpInst::ICMP_ULT ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
4951 Constant *Zero = ConstantInt::getNullValue(Ty: Op1->getType());
4952 return new ICmpInst(NewPred, Op1, Zero);
4953 }
4954
4955 if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
4956 match(V: Op1, P: m_And(L: m_BinOp(I&: BO), R: m_LowBitMask(V&: C))) &&
4957 match(V: BO, P: m_Add(L: m_Specific(V: Op0), R: m_SpecificIntAllowPoison(V: *C)))) {
4958 CmpInst::Predicate NewPred =
4959 Pred == ICmpInst::ICMP_UGT ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
4960 Constant *Zero = ConstantInt::getNullValue(Ty: Op1->getType());
4961 return new ICmpInst(NewPred, Op0, Zero);
4962 }
4963 }
4964
4965 bool NoOp0WrapProblem = false, NoOp1WrapProblem = false;
4966 bool Op0HasNUW = false, Op1HasNUW = false;
4967 bool Op0HasNSW = false, Op1HasNSW = false;
4968 // Analyze the case when either Op0 or Op1 is an add instruction.
4969 // Op0 = A + B (or A and B are null); Op1 = C + D (or C and D are null).
4970 auto hasNoWrapProblem = [](const BinaryOperator &BO, CmpInst::Predicate Pred,
4971 bool &HasNSW, bool &HasNUW) -> bool {
4972 if (isa<OverflowingBinaryOperator>(Val: BO)) {
4973 HasNUW = BO.hasNoUnsignedWrap();
4974 HasNSW = BO.hasNoSignedWrap();
4975 return ICmpInst::isEquality(P: Pred) ||
4976 (CmpInst::isUnsigned(predicate: Pred) && HasNUW) ||
4977 (CmpInst::isSigned(predicate: Pred) && HasNSW);
4978 } else if (BO.getOpcode() == Instruction::Or) {
4979 HasNUW = true;
4980 HasNSW = true;
4981 return true;
4982 } else {
4983 return false;
4984 }
4985 };
4986 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
4987
4988 if (BO0) {
4989 match(V: BO0, P: m_AddLike(L: m_Value(V&: A), R: m_Value(V&: B)));
4990 NoOp0WrapProblem = hasNoWrapProblem(*BO0, Pred, Op0HasNSW, Op0HasNUW);
4991 }
4992 if (BO1) {
4993 match(V: BO1, P: m_AddLike(L: m_Value(V&: C), R: m_Value(V&: D)));
4994 NoOp1WrapProblem = hasNoWrapProblem(*BO1, Pred, Op1HasNSW, Op1HasNUW);
4995 }
4996
4997 // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow.
4998 // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow.
4999 if ((A == Op1 || B == Op1) && NoOp0WrapProblem)
5000 return new ICmpInst(Pred, A == Op1 ? B : A,
5001 Constant::getNullValue(Ty: Op1->getType()));
5002
5003 // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow.
5004 // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow.
5005 if ((C == Op0 || D == Op0) && NoOp1WrapProblem)
5006 return new ICmpInst(Pred, Constant::getNullValue(Ty: Op0->getType()),
5007 C == Op0 ? D : C);
5008
5009 // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow.
5010 if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem &&
5011 NoOp1WrapProblem) {
5012 // Determine Y and Z in the form icmp (X+Y), (X+Z).
5013 Value *Y, *Z;
5014 if (A == C) {
5015 // C + B == C + D -> B == D
5016 Y = B;
5017 Z = D;
5018 } else if (A == D) {
5019 // D + B == C + D -> B == C
5020 Y = B;
5021 Z = C;
5022 } else if (B == C) {
5023 // A + C == C + D -> A == D
5024 Y = A;
5025 Z = D;
5026 } else {
5027 assert(B == D);
5028 // A + D == C + D -> A == C
5029 Y = A;
5030 Z = C;
5031 }
5032 return new ICmpInst(Pred, Y, Z);
5033 }
5034
5035 // icmp slt (A + -1), Op1 -> icmp sle A, Op1
5036 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT &&
5037 match(V: B, P: m_AllOnes()))
5038 return new ICmpInst(CmpInst::ICMP_SLE, A, Op1);
5039
5040 // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
5041 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE &&
5042 match(V: B, P: m_AllOnes()))
5043 return new ICmpInst(CmpInst::ICMP_SGT, A, Op1);
5044
5045 // icmp sle (A + 1), Op1 -> icmp slt A, Op1
5046 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(V: B, P: m_One()))
5047 return new ICmpInst(CmpInst::ICMP_SLT, A, Op1);
5048
5049 // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
5050 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(V: B, P: m_One()))
5051 return new ICmpInst(CmpInst::ICMP_SGE, A, Op1);
5052
5053 // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
5054 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT &&
5055 match(V: D, P: m_AllOnes()))
5056 return new ICmpInst(CmpInst::ICMP_SGE, Op0, C);
5057
5058 // icmp sle Op0, (C + -1) -> icmp slt Op0, C
5059 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE &&
5060 match(V: D, P: m_AllOnes()))
5061 return new ICmpInst(CmpInst::ICMP_SLT, Op0, C);
5062
5063 // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
5064 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(V: D, P: m_One()))
5065 return new ICmpInst(CmpInst::ICMP_SGT, Op0, C);
5066
5067 // icmp slt Op0, (C + 1) -> icmp sle Op0, C
5068 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(V: D, P: m_One()))
5069 return new ICmpInst(CmpInst::ICMP_SLE, Op0, C);
5070
5071 // TODO: The subtraction-related identities shown below also hold, but
5072 // canonicalization from (X -nuw 1) to (X + -1) means that the combinations
5073 // wouldn't happen even if they were implemented.
5074 //
5075 // icmp ult (A - 1), Op1 -> icmp ule A, Op1
5076 // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
5077 // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
5078 // icmp ule Op0, (C - 1) -> icmp ult Op0, C
5079
5080 // icmp ule (A + 1), Op0 -> icmp ult A, Op1
5081 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(V: B, P: m_One()))
5082 return new ICmpInst(CmpInst::ICMP_ULT, A, Op1);
5083
5084 // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
5085 if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(V: B, P: m_One()))
5086 return new ICmpInst(CmpInst::ICMP_UGE, A, Op1);
5087
5088 // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
5089 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(V: D, P: m_One()))
5090 return new ICmpInst(CmpInst::ICMP_UGT, Op0, C);
5091
5092 // icmp ult Op0, (C + 1) -> icmp ule Op0, C
5093 if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(V: D, P: m_One()))
5094 return new ICmpInst(CmpInst::ICMP_ULE, Op0, C);
5095
5096 // if C1 has greater magnitude than C2:
5097 // icmp (A + C1), (C + C2) -> icmp (A + C3), C
5098 // s.t. C3 = C1 - C2
5099 //
5100 // if C2 has greater magnitude than C1:
5101 // icmp (A + C1), (C + C2) -> icmp A, (C + C3)
5102 // s.t. C3 = C2 - C1
5103 if (A && C && NoOp0WrapProblem && NoOp1WrapProblem &&
5104 (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned()) {
5105 const APInt *AP1, *AP2;
5106 // TODO: Support non-uniform vectors.
5107 // TODO: Allow poison passthrough if B or D's element is poison.
5108 if (match(V: B, P: m_APIntAllowPoison(Res&: AP1)) &&
5109 match(V: D, P: m_APIntAllowPoison(Res&: AP2)) &&
5110 AP1->isNegative() == AP2->isNegative()) {
5111 APInt AP1Abs = AP1->abs();
5112 APInt AP2Abs = AP2->abs();
5113 if (AP1Abs.uge(RHS: AP2Abs)) {
5114 APInt Diff = *AP1 - *AP2;
5115 Constant *C3 = Constant::getIntegerValue(Ty: BO0->getType(), V: Diff);
5116 Value *NewAdd = Builder.CreateAdd(
5117 LHS: A, RHS: C3, Name: "", HasNUW: Op0HasNUW && Diff.ule(RHS: *AP1), HasNSW: Op0HasNSW);
5118 return new ICmpInst(Pred, NewAdd, C);
5119 } else {
5120 APInt Diff = *AP2 - *AP1;
5121 Constant *C3 = Constant::getIntegerValue(Ty: BO0->getType(), V: Diff);
5122 Value *NewAdd = Builder.CreateAdd(
5123 LHS: C, RHS: C3, Name: "", HasNUW: Op1HasNUW && Diff.ule(RHS: *AP2), HasNSW: Op1HasNSW);
5124 return new ICmpInst(Pred, A, NewAdd);
5125 }
5126 }
5127 Constant *Cst1, *Cst2;
5128 if (match(V: B, P: m_ImmConstant(C&: Cst1)) && match(V: D, P: m_ImmConstant(C&: Cst2)) &&
5129 ICmpInst::isEquality(P: Pred)) {
5130 Constant *Diff = ConstantExpr::getSub(C1: Cst2, C2: Cst1);
5131 Value *NewAdd = Builder.CreateAdd(LHS: C, RHS: Diff);
5132 return new ICmpInst(Pred, A, NewAdd);
5133 }
5134 }
5135
5136 // Analyze the case when either Op0 or Op1 is a sub instruction.
5137 // Op0 = A - B (or A and B are null); Op1 = C - D (or C and D are null).
5138 A = nullptr;
5139 B = nullptr;
5140 C = nullptr;
5141 D = nullptr;
5142 if (BO0 && BO0->getOpcode() == Instruction::Sub) {
5143 A = BO0->getOperand(i_nocapture: 0);
5144 B = BO0->getOperand(i_nocapture: 1);
5145 }
5146 if (BO1 && BO1->getOpcode() == Instruction::Sub) {
5147 C = BO1->getOperand(i_nocapture: 0);
5148 D = BO1->getOperand(i_nocapture: 1);
5149 }
5150
5151 // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow.
5152 if (A == Op1 && NoOp0WrapProblem)
5153 return new ICmpInst(Pred, Constant::getNullValue(Ty: Op1->getType()), B);
5154 // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow.
5155 if (C == Op0 && NoOp1WrapProblem)
5156 return new ICmpInst(Pred, D, Constant::getNullValue(Ty: Op0->getType()));
5157
5158 // Convert sub-with-unsigned-overflow comparisons into a comparison of args.
5159 // (A - B) u>/u<= A --> B u>/u<= A
5160 if (A == Op1 && (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE))
5161 return new ICmpInst(Pred, B, A);
5162 // C u</u>= (C - D) --> C u</u>= D
5163 if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
5164 return new ICmpInst(Pred, C, D);
5165 // (A - B) u>=/u< A --> B u>/u<= A iff B != 0
5166 if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
5167 isKnownNonZero(V: B, Q))
5168 return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(pred: Pred), B, A);
5169 // C u<=/u> (C - D) --> C u</u>= D iff B != 0
5170 if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
5171 isKnownNonZero(V: D, Q))
5172 return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(pred: Pred), C, D);
5173
5174 // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow.
5175 if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem)
5176 return new ICmpInst(Pred, A, C);
5177
5178 // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow.
5179 if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem)
5180 return new ICmpInst(Pred, D, B);
5181
5182 // icmp (0-X) < cst --> x > -cst
5183 if (NoOp0WrapProblem && ICmpInst::isSigned(predicate: Pred)) {
5184 Value *X;
5185 if (match(V: BO0, P: m_Neg(V: m_Value(V&: X))))
5186 if (Constant *RHSC = dyn_cast<Constant>(Val: Op1))
5187 if (RHSC->isNotMinSignedValue())
5188 return new ICmpInst(I.getSwappedPredicate(), X,
5189 ConstantExpr::getNeg(C: RHSC));
5190 }
5191
5192 if (Instruction * R = foldICmpXorXX(I, Q, IC&: *this))
5193 return R;
5194 if (Instruction *R = foldICmpOrXX(I, Q, IC&: *this))
5195 return R;
5196
5197 {
5198 // Try to remove shared multiplier from comparison:
5199 // X * Z u{lt/le/gt/ge}/eq/ne Y * Z
5200 Value *X, *Y, *Z;
5201 if (Pred == ICmpInst::getUnsignedPredicate(pred: Pred) &&
5202 ((match(V: Op0, P: m_Mul(L: m_Value(V&: X), R: m_Value(V&: Z))) &&
5203 match(V: Op1, P: m_c_Mul(L: m_Specific(V: Z), R: m_Value(V&: Y)))) ||
5204 (match(V: Op0, P: m_Mul(L: m_Value(V&: Z), R: m_Value(V&: X))) &&
5205 match(V: Op1, P: m_c_Mul(L: m_Specific(V: Z), R: m_Value(V&: Y)))))) {
5206 bool NonZero;
5207 if (ICmpInst::isEquality(P: Pred)) {
5208 KnownBits ZKnown = computeKnownBits(V: Z, Depth: 0, CxtI: &I);
5209 // if Z % 2 != 0
5210 // X * Z eq/ne Y * Z -> X eq/ne Y
5211 if (ZKnown.countMaxTrailingZeros() == 0)
5212 return new ICmpInst(Pred, X, Y);
5213 NonZero = !ZKnown.One.isZero() || isKnownNonZero(V: Z, Q);
5214 // if Z != 0 and nsw(X * Z) and nsw(Y * Z)
5215 // X * Z eq/ne Y * Z -> X eq/ne Y
5216 if (NonZero && BO0 && BO1 && Op0HasNSW && Op1HasNSW)
5217 return new ICmpInst(Pred, X, Y);
5218 } else
5219 NonZero = isKnownNonZero(V: Z, Q);
5220
5221 // If Z != 0 and nuw(X * Z) and nuw(Y * Z)
5222 // X * Z u{lt/le/gt/ge}/eq/ne Y * Z -> X u{lt/le/gt/ge}/eq/ne Y
5223 if (NonZero && BO0 && BO1 && Op0HasNUW && Op1HasNUW)
5224 return new ICmpInst(Pred, X, Y);
5225 }
5226 }
5227
5228 BinaryOperator *SRem = nullptr;
5229 // icmp (srem X, Y), Y
5230 if (BO0 && BO0->getOpcode() == Instruction::SRem && Op1 == BO0->getOperand(i_nocapture: 1))
5231 SRem = BO0;
5232 // icmp Y, (srem X, Y)
5233 else if (BO1 && BO1->getOpcode() == Instruction::SRem &&
5234 Op0 == BO1->getOperand(i_nocapture: 1))
5235 SRem = BO1;
5236 if (SRem) {
5237 // We don't check hasOneUse to avoid increasing register pressure because
5238 // the value we use is the same value this instruction was already using.
5239 switch (SRem == BO0 ? ICmpInst::getSwappedPredicate(pred: Pred) : Pred) {
5240 default:
5241 break;
5242 case ICmpInst::ICMP_EQ:
5243 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
5244 case ICmpInst::ICMP_NE:
5245 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
5246 case ICmpInst::ICMP_SGT:
5247 case ICmpInst::ICMP_SGE:
5248 return new ICmpInst(ICmpInst::ICMP_SGT, SRem->getOperand(i_nocapture: 1),
5249 Constant::getAllOnesValue(Ty: SRem->getType()));
5250 case ICmpInst::ICMP_SLT:
5251 case ICmpInst::ICMP_SLE:
5252 return new ICmpInst(ICmpInst::ICMP_SLT, SRem->getOperand(i_nocapture: 1),
5253 Constant::getNullValue(Ty: SRem->getType()));
5254 }
5255 }
5256
5257 if (BO0 && BO1 && BO0->getOpcode() == BO1->getOpcode() &&
5258 (BO0->hasOneUse() || BO1->hasOneUse()) &&
5259 BO0->getOperand(i_nocapture: 1) == BO1->getOperand(i_nocapture: 1)) {
5260 switch (BO0->getOpcode()) {
5261 default:
5262 break;
5263 case Instruction::Add:
5264 case Instruction::Sub:
5265 case Instruction::Xor: {
5266 if (I.isEquality()) // a+x icmp eq/ne b+x --> a icmp b
5267 return new ICmpInst(Pred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0));
5268
5269 const APInt *C;
5270 if (match(V: BO0->getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
5271 // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b
5272 if (C->isSignMask()) {
5273 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
5274 return new ICmpInst(NewPred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0));
5275 }
5276
5277 // icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b
5278 if (BO0->getOpcode() == Instruction::Xor && C->isMaxSignedValue()) {
5279 ICmpInst::Predicate NewPred = I.getFlippedSignednessPredicate();
5280 NewPred = I.getSwappedPredicate(pred: NewPred);
5281 return new ICmpInst(NewPred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0));
5282 }
5283 }
5284 break;
5285 }
5286 case Instruction::Mul: {
5287 if (!I.isEquality())
5288 break;
5289
5290 const APInt *C;
5291 if (match(V: BO0->getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero() &&
5292 !C->isOne()) {
5293 // icmp eq/ne (X * C), (Y * C) --> icmp (X & Mask), (Y & Mask)
5294 // Mask = -1 >> count-trailing-zeros(C).
5295 if (unsigned TZs = C->countr_zero()) {
5296 Constant *Mask = ConstantInt::get(
5297 Ty: BO0->getType(),
5298 V: APInt::getLowBitsSet(numBits: C->getBitWidth(), loBitsSet: C->getBitWidth() - TZs));
5299 Value *And1 = Builder.CreateAnd(LHS: BO0->getOperand(i_nocapture: 0), RHS: Mask);
5300 Value *And2 = Builder.CreateAnd(LHS: BO1->getOperand(i_nocapture: 0), RHS: Mask);
5301 return new ICmpInst(Pred, And1, And2);
5302 }
5303 }
5304 break;
5305 }
5306 case Instruction::UDiv:
5307 case Instruction::LShr:
5308 if (I.isSigned() || !BO0->isExact() || !BO1->isExact())
5309 break;
5310 return new ICmpInst(Pred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0));
5311
5312 case Instruction::SDiv:
5313 if (!(I.isEquality() || match(V: BO0->getOperand(i_nocapture: 1), P: m_NonNegative())) ||
5314 !BO0->isExact() || !BO1->isExact())
5315 break;
5316 return new ICmpInst(Pred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0));
5317
5318 case Instruction::AShr:
5319 if (!BO0->isExact() || !BO1->isExact())
5320 break;
5321 return new ICmpInst(Pred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0));
5322
5323 case Instruction::Shl: {
5324 bool NUW = Op0HasNUW && Op1HasNUW;
5325 bool NSW = Op0HasNSW && Op1HasNSW;
5326 if (!NUW && !NSW)
5327 break;
5328 if (!NSW && I.isSigned())
5329 break;
5330 return new ICmpInst(Pred, BO0->getOperand(i_nocapture: 0), BO1->getOperand(i_nocapture: 0));
5331 }
5332 }
5333 }
5334
5335 if (BO0) {
5336 // Transform A & (L - 1) `ult` L --> L != 0
5337 auto LSubOne = m_Add(L: m_Specific(V: Op1), R: m_AllOnes());
5338 auto BitwiseAnd = m_c_And(L: m_Value(), R: LSubOne);
5339
5340 if (match(V: BO0, P: BitwiseAnd) && Pred == ICmpInst::ICMP_ULT) {
5341 auto *Zero = Constant::getNullValue(Ty: BO0->getType());
5342 return new ICmpInst(ICmpInst::ICMP_NE, Op1, Zero);
5343 }
5344 }
5345
5346 // For unsigned predicates / eq / ne:
5347 // icmp pred (x << 1), x --> icmp getSignedPredicate(pred) x, 0
5348 // icmp pred x, (x << 1) --> icmp getSignedPredicate(pred) 0, x
5349 if (!ICmpInst::isSigned(predicate: Pred)) {
5350 if (match(V: Op0, P: m_Shl(L: m_Specific(V: Op1), R: m_One())))
5351 return new ICmpInst(ICmpInst::getSignedPredicate(pred: Pred), Op1,
5352 Constant::getNullValue(Ty: Op1->getType()));
5353 else if (match(V: Op1, P: m_Shl(L: m_Specific(V: Op0), R: m_One())))
5354 return new ICmpInst(ICmpInst::getSignedPredicate(pred: Pred),
5355 Constant::getNullValue(Ty: Op0->getType()), Op0);
5356 }
5357
5358 if (Value *V = foldMultiplicationOverflowCheck(I))
5359 return replaceInstUsesWith(I, V);
5360
5361 if (Instruction *R = foldICmpAndXX(I, Q, IC&: *this))
5362 return R;
5363
5364 if (Value *V = foldICmpWithTruncSignExtendedVal(I, Builder))
5365 return replaceInstUsesWith(I, V);
5366
5367 if (Value *V = foldShiftIntoShiftInAnotherHandOfAndInICmp(I, SQ, Builder))
5368 return replaceInstUsesWith(I, V);
5369
5370 return nullptr;
5371}
5372
5373/// Fold icmp Pred min|max(X, Y), Z.
5374Instruction *InstCombinerImpl::foldICmpWithMinMax(Instruction &I,
5375 MinMaxIntrinsic *MinMax,
5376 Value *Z,
5377 ICmpInst::Predicate Pred) {
5378 Value *X = MinMax->getLHS();
5379 Value *Y = MinMax->getRHS();
5380 if (ICmpInst::isSigned(predicate: Pred) && !MinMax->isSigned())
5381 return nullptr;
5382 if (ICmpInst::isUnsigned(predicate: Pred) && MinMax->isSigned()) {
5383 // Revert the transform signed pred -> unsigned pred
5384 // TODO: We can flip the signedness of predicate if both operands of icmp
5385 // are negative.
5386 if (isKnownNonNegative(V: Z, SQ: SQ.getWithInstruction(I: &I)) &&
5387 isKnownNonNegative(V: MinMax, SQ: SQ.getWithInstruction(I: &I))) {
5388 Pred = ICmpInst::getFlippedSignednessPredicate(pred: Pred);
5389 } else
5390 return nullptr;
5391 }
5392 SimplifyQuery Q = SQ.getWithInstruction(I: &I);
5393 auto IsCondKnownTrue = [](Value *Val) -> std::optional<bool> {
5394 if (!Val)
5395 return std::nullopt;
5396 if (match(V: Val, P: m_One()))
5397 return true;
5398 if (match(V: Val, P: m_Zero()))
5399 return false;
5400 return std::nullopt;
5401 };
5402 auto CmpXZ = IsCondKnownTrue(simplifyICmpInst(Predicate: Pred, LHS: X, RHS: Z, Q));
5403 auto CmpYZ = IsCondKnownTrue(simplifyICmpInst(Predicate: Pred, LHS: Y, RHS: Z, Q));
5404 if (!CmpXZ.has_value() && !CmpYZ.has_value())
5405 return nullptr;
5406 if (!CmpXZ.has_value()) {
5407 std::swap(a&: X, b&: Y);
5408 std::swap(lhs&: CmpXZ, rhs&: CmpYZ);
5409 }
5410
5411 auto FoldIntoCmpYZ = [&]() -> Instruction * {
5412 if (CmpYZ.has_value())
5413 return replaceInstUsesWith(I, V: ConstantInt::getBool(Ty: I.getType(), V: *CmpYZ));
5414 return ICmpInst::Create(Op: Instruction::ICmp, Pred, S1: Y, S2: Z);
5415 };
5416
5417 switch (Pred) {
5418 case ICmpInst::ICMP_EQ:
5419 case ICmpInst::ICMP_NE: {
5420 // If X == Z:
5421 // Expr Result
5422 // min(X, Y) == Z X <= Y
5423 // max(X, Y) == Z X >= Y
5424 // min(X, Y) != Z X > Y
5425 // max(X, Y) != Z X < Y
5426 if ((Pred == ICmpInst::ICMP_EQ) == *CmpXZ) {
5427 ICmpInst::Predicate NewPred =
5428 ICmpInst::getNonStrictPredicate(pred: MinMax->getPredicate());
5429 if (Pred == ICmpInst::ICMP_NE)
5430 NewPred = ICmpInst::getInversePredicate(pred: NewPred);
5431 return ICmpInst::Create(Op: Instruction::ICmp, Pred: NewPred, S1: X, S2: Y);
5432 }
5433 // Otherwise (X != Z):
5434 ICmpInst::Predicate NewPred = MinMax->getPredicate();
5435 auto MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(Predicate: NewPred, LHS: X, RHS: Z, Q));
5436 if (!MinMaxCmpXZ.has_value()) {
5437 std::swap(a&: X, b&: Y);
5438 std::swap(lhs&: CmpXZ, rhs&: CmpYZ);
5439 // Re-check pre-condition X != Z
5440 if (!CmpXZ.has_value() || (Pred == ICmpInst::ICMP_EQ) == *CmpXZ)
5441 break;
5442 MinMaxCmpXZ = IsCondKnownTrue(simplifyICmpInst(Predicate: NewPred, LHS: X, RHS: Z, Q));
5443 }
5444 if (!MinMaxCmpXZ.has_value())
5445 break;
5446 if (*MinMaxCmpXZ) {
5447 // Expr Fact Result
5448 // min(X, Y) == Z X < Z false
5449 // max(X, Y) == Z X > Z false
5450 // min(X, Y) != Z X < Z true
5451 // max(X, Y) != Z X > Z true
5452 return replaceInstUsesWith(
5453 I, V: ConstantInt::getBool(Ty: I.getType(), V: Pred == ICmpInst::ICMP_NE));
5454 } else {
5455 // Expr Fact Result
5456 // min(X, Y) == Z X > Z Y == Z
5457 // max(X, Y) == Z X < Z Y == Z
5458 // min(X, Y) != Z X > Z Y != Z
5459 // max(X, Y) != Z X < Z Y != Z
5460 return FoldIntoCmpYZ();
5461 }
5462 break;
5463 }
5464 case ICmpInst::ICMP_SLT:
5465 case ICmpInst::ICMP_ULT:
5466 case ICmpInst::ICMP_SLE:
5467 case ICmpInst::ICMP_ULE:
5468 case ICmpInst::ICMP_SGT:
5469 case ICmpInst::ICMP_UGT:
5470 case ICmpInst::ICMP_SGE:
5471 case ICmpInst::ICMP_UGE: {
5472 bool IsSame = MinMax->getPredicate() == ICmpInst::getStrictPredicate(pred: Pred);
5473 if (*CmpXZ) {
5474 if (IsSame) {
5475 // Expr Fact Result
5476 // min(X, Y) < Z X < Z true
5477 // min(X, Y) <= Z X <= Z true
5478 // max(X, Y) > Z X > Z true
5479 // max(X, Y) >= Z X >= Z true
5480 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
5481 } else {
5482 // Expr Fact Result
5483 // max(X, Y) < Z X < Z Y < Z
5484 // max(X, Y) <= Z X <= Z Y <= Z
5485 // min(X, Y) > Z X > Z Y > Z
5486 // min(X, Y) >= Z X >= Z Y >= Z
5487 return FoldIntoCmpYZ();
5488 }
5489 } else {
5490 if (IsSame) {
5491 // Expr Fact Result
5492 // min(X, Y) < Z X >= Z Y < Z
5493 // min(X, Y) <= Z X > Z Y <= Z
5494 // max(X, Y) > Z X <= Z Y > Z
5495 // max(X, Y) >= Z X < Z Y >= Z
5496 return FoldIntoCmpYZ();
5497 } else {
5498 // Expr Fact Result
5499 // max(X, Y) < Z X >= Z false
5500 // max(X, Y) <= Z X > Z false
5501 // min(X, Y) > Z X <= Z false
5502 // min(X, Y) >= Z X < Z false
5503 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
5504 }
5505 }
5506 break;
5507 }
5508 default:
5509 break;
5510 }
5511
5512 return nullptr;
5513}
5514
5515// Canonicalize checking for a power-of-2-or-zero value:
5516static Instruction *foldICmpPow2Test(ICmpInst &I,
5517 InstCombiner::BuilderTy &Builder) {
5518 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
5519 const CmpInst::Predicate Pred = I.getPredicate();
5520 Value *A = nullptr;
5521 bool CheckIs;
5522 if (I.isEquality()) {
5523 // (A & (A-1)) == 0 --> ctpop(A) < 2 (two commuted variants)
5524 // ((A-1) & A) != 0 --> ctpop(A) > 1 (two commuted variants)
5525 if (!match(V: Op0, P: m_OneUse(SubPattern: m_c_And(L: m_Add(L: m_Value(V&: A), R: m_AllOnes()),
5526 R: m_Deferred(V: A)))) ||
5527 !match(V: Op1, P: m_ZeroInt()))
5528 A = nullptr;
5529
5530 // (A & -A) == A --> ctpop(A) < 2 (four commuted variants)
5531 // (-A & A) != A --> ctpop(A) > 1 (four commuted variants)
5532 if (match(V: Op0, P: m_OneUse(SubPattern: m_c_And(L: m_Neg(V: m_Specific(V: Op1)), R: m_Specific(V: Op1)))))
5533 A = Op1;
5534 else if (match(V: Op1,
5535 P: m_OneUse(SubPattern: m_c_And(L: m_Neg(V: m_Specific(V: Op0)), R: m_Specific(V: Op0)))))
5536 A = Op0;
5537
5538 CheckIs = Pred == ICmpInst::ICMP_EQ;
5539 } else if (ICmpInst::isUnsigned(predicate: Pred)) {
5540 // (A ^ (A-1)) u>= A --> ctpop(A) < 2 (two commuted variants)
5541 // ((A-1) ^ A) u< A --> ctpop(A) > 1 (two commuted variants)
5542
5543 if ((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
5544 match(V: Op0, P: m_OneUse(SubPattern: m_c_Xor(L: m_Add(L: m_Specific(V: Op1), R: m_AllOnes()),
5545 R: m_Specific(V: Op1))))) {
5546 A = Op1;
5547 CheckIs = Pred == ICmpInst::ICMP_UGE;
5548 } else if ((Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_ULE) &&
5549 match(V: Op1, P: m_OneUse(SubPattern: m_c_Xor(L: m_Add(L: m_Specific(V: Op0), R: m_AllOnes()),
5550 R: m_Specific(V: Op0))))) {
5551 A = Op0;
5552 CheckIs = Pred == ICmpInst::ICMP_ULE;
5553 }
5554 }
5555
5556 if (A) {
5557 Type *Ty = A->getType();
5558 CallInst *CtPop = Builder.CreateUnaryIntrinsic(ID: Intrinsic::ctpop, V: A);
5559 return CheckIs ? new ICmpInst(ICmpInst::ICMP_ULT, CtPop,
5560 ConstantInt::get(Ty, V: 2))
5561 : new ICmpInst(ICmpInst::ICMP_UGT, CtPop,
5562 ConstantInt::get(Ty, V: 1));
5563 }
5564
5565 return nullptr;
5566}
5567
5568Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) {
5569 if (!I.isEquality())
5570 return nullptr;
5571
5572 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
5573 const CmpInst::Predicate Pred = I.getPredicate();
5574 Value *A, *B, *C, *D;
5575 if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B)))) {
5576 if (A == Op1 || B == Op1) { // (A^B) == A -> B == 0
5577 Value *OtherVal = A == Op1 ? B : A;
5578 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(Ty: A->getType()));
5579 }
5580
5581 if (match(V: Op1, P: m_Xor(L: m_Value(V&: C), R: m_Value(V&: D)))) {
5582 // A^c1 == C^c2 --> A == C^(c1^c2)
5583 ConstantInt *C1, *C2;
5584 if (match(V: B, P: m_ConstantInt(CI&: C1)) && match(V: D, P: m_ConstantInt(CI&: C2)) &&
5585 Op1->hasOneUse()) {
5586 Constant *NC = Builder.getInt(AI: C1->getValue() ^ C2->getValue());
5587 Value *Xor = Builder.CreateXor(LHS: C, RHS: NC);
5588 return new ICmpInst(Pred, A, Xor);
5589 }
5590
5591 // A^B == A^D -> B == D
5592 if (A == C)
5593 return new ICmpInst(Pred, B, D);
5594 if (A == D)
5595 return new ICmpInst(Pred, B, C);
5596 if (B == C)
5597 return new ICmpInst(Pred, A, D);
5598 if (B == D)
5599 return new ICmpInst(Pred, A, C);
5600 }
5601 }
5602
5603 if (match(V: Op1, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))) && (A == Op0 || B == Op0)) {
5604 // A == (A^B) -> B == 0
5605 Value *OtherVal = A == Op0 ? B : A;
5606 return new ICmpInst(Pred, OtherVal, Constant::getNullValue(Ty: A->getType()));
5607 }
5608
5609 // (X&Z) == (Y&Z) -> (X^Y) & Z == 0
5610 if (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
5611 match(V: Op1, P: m_And(L: m_Value(V&: C), R: m_Value(V&: D)))) {
5612 Value *X = nullptr, *Y = nullptr, *Z = nullptr;
5613
5614 if (A == C) {
5615 X = B;
5616 Y = D;
5617 Z = A;
5618 } else if (A == D) {
5619 X = B;
5620 Y = C;
5621 Z = A;
5622 } else if (B == C) {
5623 X = A;
5624 Y = D;
5625 Z = B;
5626 } else if (B == D) {
5627 X = A;
5628 Y = C;
5629 Z = B;
5630 }
5631
5632 if (X) {
5633 // If X^Y is a negative power of two, then `icmp eq/ne (Z & NegP2), 0`
5634 // will fold to `icmp ult/uge Z, -NegP2` incurringb no additional
5635 // instructions.
5636 const APInt *C0, *C1;
5637 bool XorIsNegP2 = match(V: X, P: m_APInt(Res&: C0)) && match(V: Y, P: m_APInt(Res&: C1)) &&
5638 (*C0 ^ *C1).isNegatedPowerOf2();
5639
5640 // If either Op0/Op1 are both one use or X^Y will constant fold and one of
5641 // Op0/Op1 are one use, proceed. In those cases we are instruction neutral
5642 // but `icmp eq/ne A, 0` is easier to analyze than `icmp eq/ne A, B`.
5643 int UseCnt =
5644 int(Op0->hasOneUse()) + int(Op1->hasOneUse()) +
5645 (int(match(V: X, P: m_ImmConstant()) && match(V: Y, P: m_ImmConstant())));
5646 if (XorIsNegP2 || UseCnt >= 2) {
5647 // Build (X^Y) & Z
5648 Op1 = Builder.CreateXor(LHS: X, RHS: Y);
5649 Op1 = Builder.CreateAnd(LHS: Op1, RHS: Z);
5650 return new ICmpInst(Pred, Op1, Constant::getNullValue(Ty: Op1->getType()));
5651 }
5652 }
5653 }
5654
5655 {
5656 // Similar to above, but specialized for constant because invert is needed:
5657 // (X | C) == (Y | C) --> (X ^ Y) & ~C == 0
5658 Value *X, *Y;
5659 Constant *C;
5660 if (match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: X), R: m_Constant(C)))) &&
5661 match(V: Op1, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: Y), R: m_Specific(V: C))))) {
5662 Value *Xor = Builder.CreateXor(LHS: X, RHS: Y);
5663 Value *And = Builder.CreateAnd(LHS: Xor, RHS: ConstantExpr::getNot(C));
5664 return new ICmpInst(Pred, And, Constant::getNullValue(Ty: And->getType()));
5665 }
5666 }
5667
5668 if (match(V: Op1, P: m_ZExt(Op: m_Value(V&: A))) &&
5669 (Op0->hasOneUse() || Op1->hasOneUse())) {
5670 // (B & (Pow2C-1)) == zext A --> A == trunc B
5671 // (B & (Pow2C-1)) != zext A --> A != trunc B
5672 const APInt *MaskC;
5673 if (match(V: Op0, P: m_And(L: m_Value(V&: B), R: m_LowBitMask(V&: MaskC))) &&
5674 MaskC->countr_one() == A->getType()->getScalarSizeInBits())
5675 return new ICmpInst(Pred, A, Builder.CreateTrunc(V: B, DestTy: A->getType()));
5676 }
5677
5678 // (A >> C) == (B >> C) --> (A^B) u< (1 << C)
5679 // For lshr and ashr pairs.
5680 const APInt *AP1, *AP2;
5681 if ((match(V: Op0, P: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: A), R: m_APIntAllowPoison(Res&: AP1)))) &&
5682 match(V: Op1, P: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: B), R: m_APIntAllowPoison(Res&: AP2))))) ||
5683 (match(V: Op0, P: m_OneUse(SubPattern: m_AShr(L: m_Value(V&: A), R: m_APIntAllowPoison(Res&: AP1)))) &&
5684 match(V: Op1, P: m_OneUse(SubPattern: m_AShr(L: m_Value(V&: B), R: m_APIntAllowPoison(Res&: AP2)))))) {
5685 if (AP1 != AP2)
5686 return nullptr;
5687 unsigned TypeBits = AP1->getBitWidth();
5688 unsigned ShAmt = AP1->getLimitedValue(Limit: TypeBits);
5689 if (ShAmt < TypeBits && ShAmt != 0) {
5690 ICmpInst::Predicate NewPred =
5691 Pred == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_ULT;
5692 Value *Xor = Builder.CreateXor(LHS: A, RHS: B, Name: I.getName() + ".unshifted");
5693 APInt CmpVal = APInt::getOneBitSet(numBits: TypeBits, BitNo: ShAmt);
5694 return new ICmpInst(NewPred, Xor, ConstantInt::get(Ty: A->getType(), V: CmpVal));
5695 }
5696 }
5697
5698 // (A << C) == (B << C) --> ((A^B) & (~0U >> C)) == 0
5699 ConstantInt *Cst1;
5700 if (match(V: Op0, P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: A), R: m_ConstantInt(CI&: Cst1)))) &&
5701 match(V: Op1, P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: B), R: m_Specific(V: Cst1))))) {
5702 unsigned TypeBits = Cst1->getBitWidth();
5703 unsigned ShAmt = (unsigned)Cst1->getLimitedValue(Limit: TypeBits);
5704 if (ShAmt < TypeBits && ShAmt != 0) {
5705 Value *Xor = Builder.CreateXor(LHS: A, RHS: B, Name: I.getName() + ".unshifted");
5706 APInt AndVal = APInt::getLowBitsSet(numBits: TypeBits, loBitsSet: TypeBits - ShAmt);
5707 Value *And = Builder.CreateAnd(LHS: Xor, RHS: Builder.getInt(AI: AndVal),
5708 Name: I.getName() + ".mask");
5709 return new ICmpInst(Pred, And, Constant::getNullValue(Ty: Cst1->getType()));
5710 }
5711 }
5712
5713 // Transform "icmp eq (trunc (lshr(X, cst1)), cst" to
5714 // "icmp (and X, mask), cst"
5715 uint64_t ShAmt = 0;
5716 if (Op0->hasOneUse() &&
5717 match(V: Op0, P: m_Trunc(Op: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: A), R: m_ConstantInt(V&: ShAmt))))) &&
5718 match(V: Op1, P: m_ConstantInt(CI&: Cst1)) &&
5719 // Only do this when A has multiple uses. This is most important to do
5720 // when it exposes other optimizations.
5721 !A->hasOneUse()) {
5722 unsigned ASize = cast<IntegerType>(Val: A->getType())->getPrimitiveSizeInBits();
5723
5724 if (ShAmt < ASize) {
5725 APInt MaskV =
5726 APInt::getLowBitsSet(numBits: ASize, loBitsSet: Op0->getType()->getPrimitiveSizeInBits());
5727 MaskV <<= ShAmt;
5728
5729 APInt CmpV = Cst1->getValue().zext(width: ASize);
5730 CmpV <<= ShAmt;
5731
5732 Value *Mask = Builder.CreateAnd(LHS: A, RHS: Builder.getInt(AI: MaskV));
5733 return new ICmpInst(Pred, Mask, Builder.getInt(AI: CmpV));
5734 }
5735 }
5736
5737 if (Instruction *ICmp = foldICmpIntrinsicWithIntrinsic(Cmp&: I, Builder))
5738 return ICmp;
5739
5740 // Match icmp eq (trunc (lshr A, BW), (ashr (trunc A), BW-1)), which checks the
5741 // top BW/2 + 1 bits are all the same. Create "A >=s INT_MIN && A <=s INT_MAX",
5742 // which we generate as "icmp ult (add A, 2^(BW-1)), 2^BW" to skip a few steps
5743 // of instcombine.
5744 unsigned BitWidth = Op0->getType()->getScalarSizeInBits();
5745 if (match(V: Op0, P: m_AShr(L: m_Trunc(Op: m_Value(V&: A)), R: m_SpecificInt(V: BitWidth - 1))) &&
5746 match(V: Op1, P: m_Trunc(Op: m_LShr(L: m_Specific(V: A), R: m_SpecificInt(V: BitWidth)))) &&
5747 A->getType()->getScalarSizeInBits() == BitWidth * 2 &&
5748 (I.getOperand(i_nocapture: 0)->hasOneUse() || I.getOperand(i_nocapture: 1)->hasOneUse())) {
5749 APInt C = APInt::getOneBitSet(numBits: BitWidth * 2, BitNo: BitWidth - 1);
5750 Value *Add = Builder.CreateAdd(LHS: A, RHS: ConstantInt::get(Ty: A->getType(), V: C));
5751 return new ICmpInst(Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT
5752 : ICmpInst::ICMP_UGE,
5753 Add, ConstantInt::get(Ty: A->getType(), V: C.shl(shiftAmt: 1)));
5754 }
5755
5756 // Canonicalize:
5757 // Assume B_Pow2 != 0
5758 // 1. A & B_Pow2 != B_Pow2 -> A & B_Pow2 == 0
5759 // 2. A & B_Pow2 == B_Pow2 -> A & B_Pow2 != 0
5760 if (match(V: Op0, P: m_c_And(L: m_Specific(V: Op1), R: m_Value())) &&
5761 isKnownToBeAPowerOfTwo(V: Op1, /* OrZero */ false, Depth: 0, CxtI: &I))
5762 return new ICmpInst(CmpInst::getInversePredicate(pred: Pred), Op0,
5763 ConstantInt::getNullValue(Ty: Op0->getType()));
5764
5765 if (match(V: Op1, P: m_c_And(L: m_Specific(V: Op0), R: m_Value())) &&
5766 isKnownToBeAPowerOfTwo(V: Op0, /* OrZero */ false, Depth: 0, CxtI: &I))
5767 return new ICmpInst(CmpInst::getInversePredicate(pred: Pred), Op1,
5768 ConstantInt::getNullValue(Ty: Op1->getType()));
5769
5770 // Canonicalize:
5771 // icmp eq/ne X, OneUse(rotate-right(X))
5772 // -> icmp eq/ne X, rotate-left(X)
5773 // We generally try to convert rotate-right -> rotate-left, this just
5774 // canonicalizes another case.
5775 CmpInst::Predicate PredUnused = Pred;
5776 if (match(V: &I, P: m_c_ICmp(Pred&: PredUnused, L: m_Value(V&: A),
5777 R: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::fshr>(
5778 Op0: m_Deferred(V: A), Op1: m_Deferred(V: A), Op2: m_Value(V&: B))))))
5779 return new ICmpInst(
5780 Pred, A,
5781 Builder.CreateIntrinsic(RetTy: Op0->getType(), ID: Intrinsic::fshl, Args: {A, A, B}));
5782
5783 // Canonicalize:
5784 // icmp eq/ne OneUse(A ^ Cst), B --> icmp eq/ne (A ^ B), Cst
5785 Constant *Cst;
5786 if (match(V: &I, P: m_c_ICmp(Pred&: PredUnused,
5787 L: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: A), R: m_ImmConstant(C&: Cst))),
5788 R: m_CombineAnd(L: m_Value(V&: B), R: m_Unless(M: m_ImmConstant())))))
5789 return new ICmpInst(Pred, Builder.CreateXor(LHS: A, RHS: B), Cst);
5790
5791 {
5792 // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
5793 auto m_Matcher =
5794 m_CombineOr(L: m_CombineOr(L: m_c_Add(L: m_Value(V&: B), R: m_Deferred(V: A)),
5795 R: m_c_Xor(L: m_Value(V&: B), R: m_Deferred(V: A))),
5796 R: m_Sub(L: m_Value(V&: B), R: m_Deferred(V: A)));
5797 std::optional<bool> IsZero = std::nullopt;
5798 if (match(V: &I, P: m_c_ICmp(Pred&: PredUnused, L: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: A), R: m_Matcher)),
5799 R: m_Deferred(V: A))))
5800 IsZero = false;
5801 // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
5802 else if (match(V: &I,
5803 P: m_ICmp(Pred&: PredUnused, L: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: A), R: m_Matcher)),
5804 R: m_Zero())))
5805 IsZero = true;
5806
5807 if (IsZero && isKnownToBeAPowerOfTwo(V: A, /* OrZero */ true, /*Depth*/ 0, CxtI: &I))
5808 // (icmp eq/ne (and (add/sub/xor X, P2), P2), P2)
5809 // -> (icmp eq/ne (and X, P2), 0)
5810 // (icmp eq/ne (and (add/sub/xor X, P2), P2), 0)
5811 // -> (icmp eq/ne (and X, P2), P2)
5812 return new ICmpInst(Pred, Builder.CreateAnd(LHS: B, RHS: A),
5813 *IsZero ? A
5814 : ConstantInt::getNullValue(Ty: A->getType()));
5815 }
5816
5817 return nullptr;
5818}
5819
5820Instruction *InstCombinerImpl::foldICmpWithTrunc(ICmpInst &ICmp) {
5821 ICmpInst::Predicate Pred = ICmp.getPredicate();
5822 Value *Op0 = ICmp.getOperand(i_nocapture: 0), *Op1 = ICmp.getOperand(i_nocapture: 1);
5823
5824 // Try to canonicalize trunc + compare-to-constant into a mask + cmp.
5825 // The trunc masks high bits while the compare may effectively mask low bits.
5826 Value *X;
5827 const APInt *C;
5828 if (!match(V: Op0, P: m_OneUse(SubPattern: m_Trunc(Op: m_Value(V&: X)))) || !match(V: Op1, P: m_APInt(Res&: C)))
5829 return nullptr;
5830
5831 // This matches patterns corresponding to tests of the signbit as well as:
5832 // (trunc X) u< C --> (X & -C) == 0 (are all masked-high-bits clear?)
5833 // (trunc X) u> C --> (X & ~C) != 0 (are any masked-high-bits set?)
5834 APInt Mask;
5835 if (decomposeBitTestICmp(LHS: Op0, RHS: Op1, Pred, X, Mask, LookThroughTrunc: true /* WithTrunc */)) {
5836 Value *And = Builder.CreateAnd(LHS: X, RHS: Mask);
5837 Constant *Zero = ConstantInt::getNullValue(Ty: X->getType());
5838 return new ICmpInst(Pred, And, Zero);
5839 }
5840
5841 unsigned SrcBits = X->getType()->getScalarSizeInBits();
5842 if (Pred == ICmpInst::ICMP_ULT && C->isNegatedPowerOf2()) {
5843 // If C is a negative power-of-2 (high-bit mask):
5844 // (trunc X) u< C --> (X & C) != C (are any masked-high-bits clear?)
5845 Constant *MaskC = ConstantInt::get(Ty: X->getType(), V: C->zext(width: SrcBits));
5846 Value *And = Builder.CreateAnd(LHS: X, RHS: MaskC);
5847 return new ICmpInst(ICmpInst::ICMP_NE, And, MaskC);
5848 }
5849
5850 if (Pred == ICmpInst::ICMP_UGT && (~*C).isPowerOf2()) {
5851 // If C is not-of-power-of-2 (one clear bit):
5852 // (trunc X) u> C --> (X & (C+1)) == C+1 (are all masked-high-bits set?)
5853 Constant *MaskC = ConstantInt::get(Ty: X->getType(), V: (*C + 1).zext(width: SrcBits));
5854 Value *And = Builder.CreateAnd(LHS: X, RHS: MaskC);
5855 return new ICmpInst(ICmpInst::ICMP_EQ, And, MaskC);
5856 }
5857
5858 if (auto *II = dyn_cast<IntrinsicInst>(Val: X)) {
5859 if (II->getIntrinsicID() == Intrinsic::cttz ||
5860 II->getIntrinsicID() == Intrinsic::ctlz) {
5861 unsigned MaxRet = SrcBits;
5862 // If the "is_zero_poison" argument is set, then we know at least
5863 // one bit is set in the input, so the result is always at least one
5864 // less than the full bitwidth of that input.
5865 if (match(V: II->getArgOperand(i: 1), P: m_One()))
5866 MaxRet--;
5867
5868 // Make sure the destination is wide enough to hold the largest output of
5869 // the intrinsic.
5870 if (llvm::Log2_32(Value: MaxRet) + 1 <= Op0->getType()->getScalarSizeInBits())
5871 if (Instruction *I =
5872 foldICmpIntrinsicWithConstant(Cmp&: ICmp, II, C: C->zext(width: SrcBits)))
5873 return I;
5874 }
5875 }
5876
5877 return nullptr;
5878}
5879
5880Instruction *InstCombinerImpl::foldICmpWithZextOrSext(ICmpInst &ICmp) {
5881 assert(isa<CastInst>(ICmp.getOperand(0)) && "Expected cast for operand 0");
5882 auto *CastOp0 = cast<CastInst>(Val: ICmp.getOperand(i_nocapture: 0));
5883 Value *X;
5884 if (!match(V: CastOp0, P: m_ZExtOrSExt(Op: m_Value(V&: X))))
5885 return nullptr;
5886
5887 bool IsSignedExt = CastOp0->getOpcode() == Instruction::SExt;
5888 bool IsSignedCmp = ICmp.isSigned();
5889
5890 // icmp Pred (ext X), (ext Y)
5891 Value *Y;
5892 if (match(V: ICmp.getOperand(i_nocapture: 1), P: m_ZExtOrSExt(Op: m_Value(V&: Y)))) {
5893 bool IsZext0 = isa<ZExtInst>(Val: ICmp.getOperand(i_nocapture: 0));
5894 bool IsZext1 = isa<ZExtInst>(Val: ICmp.getOperand(i_nocapture: 1));
5895
5896 if (IsZext0 != IsZext1) {
5897 // If X and Y and both i1
5898 // (icmp eq/ne (zext X) (sext Y))
5899 // eq -> (icmp eq (or X, Y), 0)
5900 // ne -> (icmp ne (or X, Y), 0)
5901 if (ICmp.isEquality() && X->getType()->isIntOrIntVectorTy(BitWidth: 1) &&
5902 Y->getType()->isIntOrIntVectorTy(BitWidth: 1))
5903 return new ICmpInst(ICmp.getPredicate(), Builder.CreateOr(LHS: X, RHS: Y),
5904 Constant::getNullValue(Ty: X->getType()));
5905
5906 // If we have mismatched casts and zext has the nneg flag, we can
5907 // treat the "zext nneg" as "sext". Otherwise, we cannot fold and quit.
5908
5909 auto *NonNegInst0 = dyn_cast<PossiblyNonNegInst>(Val: ICmp.getOperand(i_nocapture: 0));
5910 auto *NonNegInst1 = dyn_cast<PossiblyNonNegInst>(Val: ICmp.getOperand(i_nocapture: 1));
5911
5912 bool IsNonNeg0 = NonNegInst0 && NonNegInst0->hasNonNeg();
5913 bool IsNonNeg1 = NonNegInst1 && NonNegInst1->hasNonNeg();
5914
5915 if ((IsZext0 && IsNonNeg0) || (IsZext1 && IsNonNeg1))
5916 IsSignedExt = true;
5917 else
5918 return nullptr;
5919 }
5920
5921 // Not an extension from the same type?
5922 Type *XTy = X->getType(), *YTy = Y->getType();
5923 if (XTy != YTy) {
5924 // One of the casts must have one use because we are creating a new cast.
5925 if (!ICmp.getOperand(i_nocapture: 0)->hasOneUse() && !ICmp.getOperand(i_nocapture: 1)->hasOneUse())
5926 return nullptr;
5927 // Extend the narrower operand to the type of the wider operand.
5928 CastInst::CastOps CastOpcode =
5929 IsSignedExt ? Instruction::SExt : Instruction::ZExt;
5930 if (XTy->getScalarSizeInBits() < YTy->getScalarSizeInBits())
5931 X = Builder.CreateCast(Op: CastOpcode, V: X, DestTy: YTy);
5932 else if (YTy->getScalarSizeInBits() < XTy->getScalarSizeInBits())
5933 Y = Builder.CreateCast(Op: CastOpcode, V: Y, DestTy: XTy);
5934 else
5935 return nullptr;
5936 }
5937
5938 // (zext X) == (zext Y) --> X == Y
5939 // (sext X) == (sext Y) --> X == Y
5940 if (ICmp.isEquality())
5941 return new ICmpInst(ICmp.getPredicate(), X, Y);
5942
5943 // A signed comparison of sign extended values simplifies into a
5944 // signed comparison.
5945 if (IsSignedCmp && IsSignedExt)
5946 return new ICmpInst(ICmp.getPredicate(), X, Y);
5947
5948 // The other three cases all fold into an unsigned comparison.
5949 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Y);
5950 }
5951
5952 // Below here, we are only folding a compare with constant.
5953 auto *C = dyn_cast<Constant>(Val: ICmp.getOperand(i_nocapture: 1));
5954 if (!C)
5955 return nullptr;
5956
5957 // If a lossless truncate is possible...
5958 Type *SrcTy = CastOp0->getSrcTy();
5959 Constant *Res = getLosslessTrunc(C, TruncTy: SrcTy, ExtOp: CastOp0->getOpcode());
5960 if (Res) {
5961 if (ICmp.isEquality())
5962 return new ICmpInst(ICmp.getPredicate(), X, Res);
5963
5964 // A signed comparison of sign extended values simplifies into a
5965 // signed comparison.
5966 if (IsSignedExt && IsSignedCmp)
5967 return new ICmpInst(ICmp.getPredicate(), X, Res);
5968
5969 // The other three cases all fold into an unsigned comparison.
5970 return new ICmpInst(ICmp.getUnsignedPredicate(), X, Res);
5971 }
5972
5973 // The re-extended constant changed, partly changed (in the case of a vector),
5974 // or could not be determined to be equal (in the case of a constant
5975 // expression), so the constant cannot be represented in the shorter type.
5976 // All the cases that fold to true or false will have already been handled
5977 // by simplifyICmpInst, so only deal with the tricky case.
5978 if (IsSignedCmp || !IsSignedExt || !isa<ConstantInt>(Val: C))
5979 return nullptr;
5980
5981 // Is source op positive?
5982 // icmp ult (sext X), C --> icmp sgt X, -1
5983 if (ICmp.getPredicate() == ICmpInst::ICMP_ULT)
5984 return new ICmpInst(CmpInst::ICMP_SGT, X, Constant::getAllOnesValue(Ty: SrcTy));
5985
5986 // Is source op negative?
5987 // icmp ugt (sext X), C --> icmp slt X, 0
5988 assert(ICmp.getPredicate() == ICmpInst::ICMP_UGT && "ICmp should be folded!");
5989 return new ICmpInst(CmpInst::ICMP_SLT, X, Constant::getNullValue(Ty: SrcTy));
5990}
5991
5992/// Handle icmp (cast x), (cast or constant).
5993Instruction *InstCombinerImpl::foldICmpWithCastOp(ICmpInst &ICmp) {
5994 // If any operand of ICmp is a inttoptr roundtrip cast then remove it as
5995 // icmp compares only pointer's value.
5996 // icmp (inttoptr (ptrtoint p1)), p2 --> icmp p1, p2.
5997 Value *SimplifiedOp0 = simplifyIntToPtrRoundTripCast(Val: ICmp.getOperand(i_nocapture: 0));
5998 Value *SimplifiedOp1 = simplifyIntToPtrRoundTripCast(Val: ICmp.getOperand(i_nocapture: 1));
5999 if (SimplifiedOp0 || SimplifiedOp1)
6000 return new ICmpInst(ICmp.getPredicate(),
6001 SimplifiedOp0 ? SimplifiedOp0 : ICmp.getOperand(i_nocapture: 0),
6002 SimplifiedOp1 ? SimplifiedOp1 : ICmp.getOperand(i_nocapture: 1));
6003
6004 auto *CastOp0 = dyn_cast<CastInst>(Val: ICmp.getOperand(i_nocapture: 0));
6005 if (!CastOp0)
6006 return nullptr;
6007 if (!isa<Constant>(Val: ICmp.getOperand(i_nocapture: 1)) && !isa<CastInst>(Val: ICmp.getOperand(i_nocapture: 1)))
6008 return nullptr;
6009
6010 Value *Op0Src = CastOp0->getOperand(i_nocapture: 0);
6011 Type *SrcTy = CastOp0->getSrcTy();
6012 Type *DestTy = CastOp0->getDestTy();
6013
6014 // Turn icmp (ptrtoint x), (ptrtoint/c) into a compare of the input if the
6015 // integer type is the same size as the pointer type.
6016 auto CompatibleSizes = [&](Type *SrcTy, Type *DestTy) {
6017 if (isa<VectorType>(Val: SrcTy)) {
6018 SrcTy = cast<VectorType>(Val: SrcTy)->getElementType();
6019 DestTy = cast<VectorType>(Val: DestTy)->getElementType();
6020 }
6021 return DL.getPointerTypeSizeInBits(SrcTy) == DestTy->getIntegerBitWidth();
6022 };
6023 if (CastOp0->getOpcode() == Instruction::PtrToInt &&
6024 CompatibleSizes(SrcTy, DestTy)) {
6025 Value *NewOp1 = nullptr;
6026 if (auto *PtrToIntOp1 = dyn_cast<PtrToIntOperator>(Val: ICmp.getOperand(i_nocapture: 1))) {
6027 Value *PtrSrc = PtrToIntOp1->getOperand(i_nocapture: 0);
6028 if (PtrSrc->getType() == Op0Src->getType())
6029 NewOp1 = PtrToIntOp1->getOperand(i_nocapture: 0);
6030 } else if (auto *RHSC = dyn_cast<Constant>(Val: ICmp.getOperand(i_nocapture: 1))) {
6031 NewOp1 = ConstantExpr::getIntToPtr(C: RHSC, Ty: SrcTy);
6032 }
6033
6034 if (NewOp1)
6035 return new ICmpInst(ICmp.getPredicate(), Op0Src, NewOp1);
6036 }
6037
6038 if (Instruction *R = foldICmpWithTrunc(ICmp))
6039 return R;
6040
6041 return foldICmpWithZextOrSext(ICmp);
6042}
6043
6044static bool isNeutralValue(Instruction::BinaryOps BinaryOp, Value *RHS, bool IsSigned) {
6045 switch (BinaryOp) {
6046 default:
6047 llvm_unreachable("Unsupported binary op");
6048 case Instruction::Add:
6049 case Instruction::Sub:
6050 return match(V: RHS, P: m_Zero());
6051 case Instruction::Mul:
6052 return !(RHS->getType()->isIntOrIntVectorTy(BitWidth: 1) && IsSigned) &&
6053 match(V: RHS, P: m_One());
6054 }
6055}
6056
6057OverflowResult
6058InstCombinerImpl::computeOverflow(Instruction::BinaryOps BinaryOp,
6059 bool IsSigned, Value *LHS, Value *RHS,
6060 Instruction *CxtI) const {
6061 switch (BinaryOp) {
6062 default:
6063 llvm_unreachable("Unsupported binary op");
6064 case Instruction::Add:
6065 if (IsSigned)
6066 return computeOverflowForSignedAdd(LHS, RHS, CxtI);
6067 else
6068 return computeOverflowForUnsignedAdd(LHS, RHS, CxtI);
6069 case Instruction::Sub:
6070 if (IsSigned)
6071 return computeOverflowForSignedSub(LHS, RHS, CxtI);
6072 else
6073 return computeOverflowForUnsignedSub(LHS, RHS, CxtI);
6074 case Instruction::Mul:
6075 if (IsSigned)
6076 return computeOverflowForSignedMul(LHS, RHS, CxtI);
6077 else
6078 return computeOverflowForUnsignedMul(LHS, RHS, CxtI);
6079 }
6080}
6081
6082bool InstCombinerImpl::OptimizeOverflowCheck(Instruction::BinaryOps BinaryOp,
6083 bool IsSigned, Value *LHS,
6084 Value *RHS, Instruction &OrigI,
6085 Value *&Result,
6086 Constant *&Overflow) {
6087 if (OrigI.isCommutative() && isa<Constant>(Val: LHS) && !isa<Constant>(Val: RHS))
6088 std::swap(a&: LHS, b&: RHS);
6089
6090 // If the overflow check was an add followed by a compare, the insertion point
6091 // may be pointing to the compare. We want to insert the new instructions
6092 // before the add in case there are uses of the add between the add and the
6093 // compare.
6094 Builder.SetInsertPoint(&OrigI);
6095
6096 Type *OverflowTy = Type::getInt1Ty(C&: LHS->getContext());
6097 if (auto *LHSTy = dyn_cast<VectorType>(Val: LHS->getType()))
6098 OverflowTy = VectorType::get(ElementType: OverflowTy, EC: LHSTy->getElementCount());
6099
6100 if (isNeutralValue(BinaryOp, RHS, IsSigned)) {
6101 Result = LHS;
6102 Overflow = ConstantInt::getFalse(Ty: OverflowTy);
6103 return true;
6104 }
6105
6106 switch (computeOverflow(BinaryOp, IsSigned, LHS, RHS, CxtI: &OrigI)) {
6107 case OverflowResult::MayOverflow:
6108 return false;
6109 case OverflowResult::AlwaysOverflowsLow:
6110 case OverflowResult::AlwaysOverflowsHigh:
6111 Result = Builder.CreateBinOp(Opc: BinaryOp, LHS, RHS);
6112 Result->takeName(V: &OrigI);
6113 Overflow = ConstantInt::getTrue(Ty: OverflowTy);
6114 return true;
6115 case OverflowResult::NeverOverflows:
6116 Result = Builder.CreateBinOp(Opc: BinaryOp, LHS, RHS);
6117 Result->takeName(V: &OrigI);
6118 Overflow = ConstantInt::getFalse(Ty: OverflowTy);
6119 if (auto *Inst = dyn_cast<Instruction>(Val: Result)) {
6120 if (IsSigned)
6121 Inst->setHasNoSignedWrap();
6122 else
6123 Inst->setHasNoUnsignedWrap();
6124 }
6125 return true;
6126 }
6127
6128 llvm_unreachable("Unexpected overflow result");
6129}
6130
6131/// Recognize and process idiom involving test for multiplication
6132/// overflow.
6133///
6134/// The caller has matched a pattern of the form:
6135/// I = cmp u (mul(zext A, zext B), V
6136/// The function checks if this is a test for overflow and if so replaces
6137/// multiplication with call to 'mul.with.overflow' intrinsic.
6138///
6139/// \param I Compare instruction.
6140/// \param MulVal Result of 'mult' instruction. It is one of the arguments of
6141/// the compare instruction. Must be of integer type.
6142/// \param OtherVal The other argument of compare instruction.
6143/// \returns Instruction which must replace the compare instruction, NULL if no
6144/// replacement required.
6145static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal,
6146 const APInt *OtherVal,
6147 InstCombinerImpl &IC) {
6148 // Don't bother doing this transformation for pointers, don't do it for
6149 // vectors.
6150 if (!isa<IntegerType>(Val: MulVal->getType()))
6151 return nullptr;
6152
6153 auto *MulInstr = dyn_cast<Instruction>(Val: MulVal);
6154 if (!MulInstr)
6155 return nullptr;
6156 assert(MulInstr->getOpcode() == Instruction::Mul);
6157
6158 auto *LHS = cast<ZExtInst>(Val: MulInstr->getOperand(i: 0)),
6159 *RHS = cast<ZExtInst>(Val: MulInstr->getOperand(i: 1));
6160 assert(LHS->getOpcode() == Instruction::ZExt);
6161 assert(RHS->getOpcode() == Instruction::ZExt);
6162 Value *A = LHS->getOperand(i_nocapture: 0), *B = RHS->getOperand(i_nocapture: 0);
6163
6164 // Calculate type and width of the result produced by mul.with.overflow.
6165 Type *TyA = A->getType(), *TyB = B->getType();
6166 unsigned WidthA = TyA->getPrimitiveSizeInBits(),
6167 WidthB = TyB->getPrimitiveSizeInBits();
6168 unsigned MulWidth;
6169 Type *MulType;
6170 if (WidthB > WidthA) {
6171 MulWidth = WidthB;
6172 MulType = TyB;
6173 } else {
6174 MulWidth = WidthA;
6175 MulType = TyA;
6176 }
6177
6178 // In order to replace the original mul with a narrower mul.with.overflow,
6179 // all uses must ignore upper bits of the product. The number of used low
6180 // bits must be not greater than the width of mul.with.overflow.
6181 if (MulVal->hasNUsesOrMore(N: 2))
6182 for (User *U : MulVal->users()) {
6183 if (U == &I)
6184 continue;
6185 if (TruncInst *TI = dyn_cast<TruncInst>(Val: U)) {
6186 // Check if truncation ignores bits above MulWidth.
6187 unsigned TruncWidth = TI->getType()->getPrimitiveSizeInBits();
6188 if (TruncWidth > MulWidth)
6189 return nullptr;
6190 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: U)) {
6191 // Check if AND ignores bits above MulWidth.
6192 if (BO->getOpcode() != Instruction::And)
6193 return nullptr;
6194 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1))) {
6195 const APInt &CVal = CI->getValue();
6196 if (CVal.getBitWidth() - CVal.countl_zero() > MulWidth)
6197 return nullptr;
6198 } else {
6199 // In this case we could have the operand of the binary operation
6200 // being defined in another block, and performing the replacement
6201 // could break the dominance relation.
6202 return nullptr;
6203 }
6204 } else {
6205 // Other uses prohibit this transformation.
6206 return nullptr;
6207 }
6208 }
6209
6210 // Recognize patterns
6211 switch (I.getPredicate()) {
6212 case ICmpInst::ICMP_UGT: {
6213 // Recognize pattern:
6214 // mulval = mul(zext A, zext B)
6215 // cmp ugt mulval, max
6216 APInt MaxVal = APInt::getMaxValue(numBits: MulWidth);
6217 MaxVal = MaxVal.zext(width: OtherVal->getBitWidth());
6218 if (MaxVal.eq(RHS: *OtherVal))
6219 break; // Recognized
6220 return nullptr;
6221 }
6222
6223 case ICmpInst::ICMP_ULT: {
6224 // Recognize pattern:
6225 // mulval = mul(zext A, zext B)
6226 // cmp ule mulval, max + 1
6227 APInt MaxVal = APInt::getOneBitSet(numBits: OtherVal->getBitWidth(), BitNo: MulWidth);
6228 if (MaxVal.eq(RHS: *OtherVal))
6229 break; // Recognized
6230 return nullptr;
6231 }
6232
6233 default:
6234 return nullptr;
6235 }
6236
6237 InstCombiner::BuilderTy &Builder = IC.Builder;
6238 Builder.SetInsertPoint(MulInstr);
6239
6240 // Replace: mul(zext A, zext B) --> mul.with.overflow(A, B)
6241 Value *MulA = A, *MulB = B;
6242 if (WidthA < MulWidth)
6243 MulA = Builder.CreateZExt(V: A, DestTy: MulType);
6244 if (WidthB < MulWidth)
6245 MulB = Builder.CreateZExt(V: B, DestTy: MulType);
6246 Function *F = Intrinsic::getDeclaration(
6247 M: I.getModule(), id: Intrinsic::umul_with_overflow, Tys: MulType);
6248 CallInst *Call = Builder.CreateCall(Callee: F, Args: {MulA, MulB}, Name: "umul");
6249 IC.addToWorklist(I: MulInstr);
6250
6251 // If there are uses of mul result other than the comparison, we know that
6252 // they are truncation or binary AND. Change them to use result of
6253 // mul.with.overflow and adjust properly mask/size.
6254 if (MulVal->hasNUsesOrMore(N: 2)) {
6255 Value *Mul = Builder.CreateExtractValue(Agg: Call, Idxs: 0, Name: "umul.value");
6256 for (User *U : make_early_inc_range(Range: MulVal->users())) {
6257 if (U == &I)
6258 continue;
6259 if (TruncInst *TI = dyn_cast<TruncInst>(Val: U)) {
6260 if (TI->getType()->getPrimitiveSizeInBits() == MulWidth)
6261 IC.replaceInstUsesWith(I&: *TI, V: Mul);
6262 else
6263 TI->setOperand(i_nocapture: 0, Val_nocapture: Mul);
6264 } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: U)) {
6265 assert(BO->getOpcode() == Instruction::And);
6266 // Replace (mul & mask) --> zext (mul.with.overflow & short_mask)
6267 ConstantInt *CI = cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1));
6268 APInt ShortMask = CI->getValue().trunc(width: MulWidth);
6269 Value *ShortAnd = Builder.CreateAnd(LHS: Mul, RHS: ShortMask);
6270 Value *Zext = Builder.CreateZExt(V: ShortAnd, DestTy: BO->getType());
6271 IC.replaceInstUsesWith(I&: *BO, V: Zext);
6272 } else {
6273 llvm_unreachable("Unexpected Binary operation");
6274 }
6275 IC.addToWorklist(I: cast<Instruction>(Val: U));
6276 }
6277 }
6278
6279 // The original icmp gets replaced with the overflow value, maybe inverted
6280 // depending on predicate.
6281 if (I.getPredicate() == ICmpInst::ICMP_ULT) {
6282 Value *Res = Builder.CreateExtractValue(Agg: Call, Idxs: 1);
6283 return BinaryOperator::CreateNot(Op: Res);
6284 }
6285
6286 return ExtractValueInst::Create(Agg: Call, Idxs: 1);
6287}
6288
6289/// When performing a comparison against a constant, it is possible that not all
6290/// the bits in the LHS are demanded. This helper method computes the mask that
6291/// IS demanded.
6292static APInt getDemandedBitsLHSMask(ICmpInst &I, unsigned BitWidth) {
6293 const APInt *RHS;
6294 if (!match(V: I.getOperand(i_nocapture: 1), P: m_APInt(Res&: RHS)))
6295 return APInt::getAllOnes(numBits: BitWidth);
6296
6297 // If this is a normal comparison, it demands all bits. If it is a sign bit
6298 // comparison, it only demands the sign bit.
6299 bool UnusedBit;
6300 if (isSignBitCheck(Pred: I.getPredicate(), RHS: *RHS, TrueIfSigned&: UnusedBit))
6301 return APInt::getSignMask(BitWidth);
6302
6303 switch (I.getPredicate()) {
6304 // For a UGT comparison, we don't care about any bits that
6305 // correspond to the trailing ones of the comparand. The value of these
6306 // bits doesn't impact the outcome of the comparison, because any value
6307 // greater than the RHS must differ in a bit higher than these due to carry.
6308 case ICmpInst::ICMP_UGT:
6309 return APInt::getBitsSetFrom(numBits: BitWidth, loBit: RHS->countr_one());
6310
6311 // Similarly, for a ULT comparison, we don't care about the trailing zeros.
6312 // Any value less than the RHS must differ in a higher bit because of carries.
6313 case ICmpInst::ICMP_ULT:
6314 return APInt::getBitsSetFrom(numBits: BitWidth, loBit: RHS->countr_zero());
6315
6316 default:
6317 return APInt::getAllOnes(numBits: BitWidth);
6318 }
6319}
6320
6321/// Check that one use is in the same block as the definition and all
6322/// other uses are in blocks dominated by a given block.
6323///
6324/// \param DI Definition
6325/// \param UI Use
6326/// \param DB Block that must dominate all uses of \p DI outside
6327/// the parent block
6328/// \return true when \p UI is the only use of \p DI in the parent block
6329/// and all other uses of \p DI are in blocks dominated by \p DB.
6330///
6331bool InstCombinerImpl::dominatesAllUses(const Instruction *DI,
6332 const Instruction *UI,
6333 const BasicBlock *DB) const {
6334 assert(DI && UI && "Instruction not defined\n");
6335 // Ignore incomplete definitions.
6336 if (!DI->getParent())
6337 return false;
6338 // DI and UI must be in the same block.
6339 if (DI->getParent() != UI->getParent())
6340 return false;
6341 // Protect from self-referencing blocks.
6342 if (DI->getParent() == DB)
6343 return false;
6344 for (const User *U : DI->users()) {
6345 auto *Usr = cast<Instruction>(Val: U);
6346 if (Usr != UI && !DT.dominates(A: DB, B: Usr->getParent()))
6347 return false;
6348 }
6349 return true;
6350}
6351
6352/// Return true when the instruction sequence within a block is select-cmp-br.
6353static bool isChainSelectCmpBranch(const SelectInst *SI) {
6354 const BasicBlock *BB = SI->getParent();
6355 if (!BB)
6356 return false;
6357 auto *BI = dyn_cast_or_null<BranchInst>(Val: BB->getTerminator());
6358 if (!BI || BI->getNumSuccessors() != 2)
6359 return false;
6360 auto *IC = dyn_cast<ICmpInst>(Val: BI->getCondition());
6361 if (!IC || (IC->getOperand(i_nocapture: 0) != SI && IC->getOperand(i_nocapture: 1) != SI))
6362 return false;
6363 return true;
6364}
6365
6366/// True when a select result is replaced by one of its operands
6367/// in select-icmp sequence. This will eventually result in the elimination
6368/// of the select.
6369///
6370/// \param SI Select instruction
6371/// \param Icmp Compare instruction
6372/// \param SIOpd Operand that replaces the select
6373///
6374/// Notes:
6375/// - The replacement is global and requires dominator information
6376/// - The caller is responsible for the actual replacement
6377///
6378/// Example:
6379///
6380/// entry:
6381/// %4 = select i1 %3, %C* %0, %C* null
6382/// %5 = icmp eq %C* %4, null
6383/// br i1 %5, label %9, label %7
6384/// ...
6385/// ; <label>:7 ; preds = %entry
6386/// %8 = getelementptr inbounds %C* %4, i64 0, i32 0
6387/// ...
6388///
6389/// can be transformed to
6390///
6391/// %5 = icmp eq %C* %0, null
6392/// %6 = select i1 %3, i1 %5, i1 true
6393/// br i1 %6, label %9, label %7
6394/// ...
6395/// ; <label>:7 ; preds = %entry
6396/// %8 = getelementptr inbounds %C* %0, i64 0, i32 0 // replace by %0!
6397///
6398/// Similar when the first operand of the select is a constant or/and
6399/// the compare is for not equal rather than equal.
6400///
6401/// NOTE: The function is only called when the select and compare constants
6402/// are equal, the optimization can work only for EQ predicates. This is not a
6403/// major restriction since a NE compare should be 'normalized' to an equal
6404/// compare, which usually happens in the combiner and test case
6405/// select-cmp-br.ll checks for it.
6406bool InstCombinerImpl::replacedSelectWithOperand(SelectInst *SI,
6407 const ICmpInst *Icmp,
6408 const unsigned SIOpd) {
6409 assert((SIOpd == 1 || SIOpd == 2) && "Invalid select operand!");
6410 if (isChainSelectCmpBranch(SI) && Icmp->getPredicate() == ICmpInst::ICMP_EQ) {
6411 BasicBlock *Succ = SI->getParent()->getTerminator()->getSuccessor(Idx: 1);
6412 // The check for the single predecessor is not the best that can be
6413 // done. But it protects efficiently against cases like when SI's
6414 // home block has two successors, Succ and Succ1, and Succ1 predecessor
6415 // of Succ. Then SI can't be replaced by SIOpd because the use that gets
6416 // replaced can be reached on either path. So the uniqueness check
6417 // guarantees that the path all uses of SI (outside SI's parent) are on
6418 // is disjoint from all other paths out of SI. But that information
6419 // is more expensive to compute, and the trade-off here is in favor
6420 // of compile-time. It should also be noticed that we check for a single
6421 // predecessor and not only uniqueness. This to handle the situation when
6422 // Succ and Succ1 points to the same basic block.
6423 if (Succ->getSinglePredecessor() && dominatesAllUses(DI: SI, UI: Icmp, DB: Succ)) {
6424 NumSel++;
6425 SI->replaceUsesOutsideBlock(V: SI->getOperand(i_nocapture: SIOpd), BB: SI->getParent());
6426 return true;
6427 }
6428 }
6429 return false;
6430}
6431
6432/// Try to fold the comparison based on range information we can get by checking
6433/// whether bits are known to be zero or one in the inputs.
6434Instruction *InstCombinerImpl::foldICmpUsingKnownBits(ICmpInst &I) {
6435 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
6436 Type *Ty = Op0->getType();
6437 ICmpInst::Predicate Pred = I.getPredicate();
6438
6439 // Get scalar or pointer size.
6440 unsigned BitWidth = Ty->isIntOrIntVectorTy()
6441 ? Ty->getScalarSizeInBits()
6442 : DL.getPointerTypeSizeInBits(Ty->getScalarType());
6443
6444 if (!BitWidth)
6445 return nullptr;
6446
6447 KnownBits Op0Known(BitWidth);
6448 KnownBits Op1Known(BitWidth);
6449
6450 {
6451 // Don't use dominating conditions when folding icmp using known bits. This
6452 // may convert signed into unsigned predicates in ways that other passes
6453 // (especially IndVarSimplify) may not be able to reliably undo.
6454 SimplifyQuery Q = SQ.getWithoutDomCondCache().getWithInstruction(I: &I);
6455 if (SimplifyDemandedBits(I: &I, Op: 0, DemandedMask: getDemandedBitsLHSMask(I, BitWidth),
6456 Known&: Op0Known, /*Depth=*/0, Q))
6457 return &I;
6458
6459 if (SimplifyDemandedBits(I: &I, Op: 1, DemandedMask: APInt::getAllOnes(numBits: BitWidth), Known&: Op1Known,
6460 /*Depth=*/0, Q))
6461 return &I;
6462 }
6463
6464 // Given the known and unknown bits, compute a range that the LHS could be
6465 // in. Compute the Min, Max and RHS values based on the known bits. For the
6466 // EQ and NE we use unsigned values.
6467 APInt Op0Min(BitWidth, 0), Op0Max(BitWidth, 0);
6468 APInt Op1Min(BitWidth, 0), Op1Max(BitWidth, 0);
6469 if (I.isSigned()) {
6470 Op0Min = Op0Known.getSignedMinValue();
6471 Op0Max = Op0Known.getSignedMaxValue();
6472 Op1Min = Op1Known.getSignedMinValue();
6473 Op1Max = Op1Known.getSignedMaxValue();
6474 } else {
6475 Op0Min = Op0Known.getMinValue();
6476 Op0Max = Op0Known.getMaxValue();
6477 Op1Min = Op1Known.getMinValue();
6478 Op1Max = Op1Known.getMaxValue();
6479 }
6480
6481 // If Min and Max are known to be the same, then SimplifyDemandedBits figured
6482 // out that the LHS or RHS is a constant. Constant fold this now, so that
6483 // code below can assume that Min != Max.
6484 if (!isa<Constant>(Val: Op0) && Op0Min == Op0Max)
6485 return new ICmpInst(Pred, ConstantExpr::getIntegerValue(Ty, V: Op0Min), Op1);
6486 if (!isa<Constant>(Val: Op1) && Op1Min == Op1Max)
6487 return new ICmpInst(Pred, Op0, ConstantExpr::getIntegerValue(Ty, V: Op1Min));
6488
6489 // Don't break up a clamp pattern -- (min(max X, Y), Z) -- by replacing a
6490 // min/max canonical compare with some other compare. That could lead to
6491 // conflict with select canonicalization and infinite looping.
6492 // FIXME: This constraint may go away if min/max intrinsics are canonical.
6493 auto isMinMaxCmp = [&](Instruction &Cmp) {
6494 if (!Cmp.hasOneUse())
6495 return false;
6496 Value *A, *B;
6497 SelectPatternFlavor SPF = matchSelectPattern(V: Cmp.user_back(), LHS&: A, RHS&: B).Flavor;
6498 if (!SelectPatternResult::isMinOrMax(SPF))
6499 return false;
6500 return match(V: Op0, P: m_MaxOrMin(L: m_Value(), R: m_Value())) ||
6501 match(V: Op1, P: m_MaxOrMin(L: m_Value(), R: m_Value()));
6502 };
6503 if (!isMinMaxCmp(I)) {
6504 switch (Pred) {
6505 default:
6506 break;
6507 case ICmpInst::ICMP_ULT: {
6508 if (Op1Min == Op0Max) // A <u B -> A != B if max(A) == min(B)
6509 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6510 const APInt *CmpC;
6511 if (match(V: Op1, P: m_APInt(Res&: CmpC))) {
6512 // A <u C -> A == C-1 if min(A)+1 == C
6513 if (*CmpC == Op0Min + 1)
6514 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6515 ConstantInt::get(Ty: Op1->getType(), V: *CmpC - 1));
6516 // X <u C --> X == 0, if the number of zero bits in the bottom of X
6517 // exceeds the log2 of C.
6518 if (Op0Known.countMinTrailingZeros() >= CmpC->ceilLogBase2())
6519 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6520 Constant::getNullValue(Ty: Op1->getType()));
6521 }
6522 break;
6523 }
6524 case ICmpInst::ICMP_UGT: {
6525 if (Op1Max == Op0Min) // A >u B -> A != B if min(A) == max(B)
6526 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6527 const APInt *CmpC;
6528 if (match(V: Op1, P: m_APInt(Res&: CmpC))) {
6529 // A >u C -> A == C+1 if max(a)-1 == C
6530 if (*CmpC == Op0Max - 1)
6531 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6532 ConstantInt::get(Ty: Op1->getType(), V: *CmpC + 1));
6533 // X >u C --> X != 0, if the number of zero bits in the bottom of X
6534 // exceeds the log2 of C.
6535 if (Op0Known.countMinTrailingZeros() >= CmpC->getActiveBits())
6536 return new ICmpInst(ICmpInst::ICMP_NE, Op0,
6537 Constant::getNullValue(Ty: Op1->getType()));
6538 }
6539 break;
6540 }
6541 case ICmpInst::ICMP_SLT: {
6542 if (Op1Min == Op0Max) // A <s B -> A != B if max(A) == min(B)
6543 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6544 const APInt *CmpC;
6545 if (match(V: Op1, P: m_APInt(Res&: CmpC))) {
6546 if (*CmpC == Op0Min + 1) // A <s C -> A == C-1 if min(A)+1 == C
6547 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6548 ConstantInt::get(Ty: Op1->getType(), V: *CmpC - 1));
6549 }
6550 break;
6551 }
6552 case ICmpInst::ICMP_SGT: {
6553 if (Op1Max == Op0Min) // A >s B -> A != B if min(A) == max(B)
6554 return new ICmpInst(ICmpInst::ICMP_NE, Op0, Op1);
6555 const APInt *CmpC;
6556 if (match(V: Op1, P: m_APInt(Res&: CmpC))) {
6557 if (*CmpC == Op0Max - 1) // A >s C -> A == C+1 if max(A)-1 == C
6558 return new ICmpInst(ICmpInst::ICMP_EQ, Op0,
6559 ConstantInt::get(Ty: Op1->getType(), V: *CmpC + 1));
6560 }
6561 break;
6562 }
6563 }
6564 }
6565
6566 // Based on the range information we know about the LHS, see if we can
6567 // simplify this comparison. For example, (x&4) < 8 is always true.
6568 switch (Pred) {
6569 default:
6570 llvm_unreachable("Unknown icmp opcode!");
6571 case ICmpInst::ICMP_EQ:
6572 case ICmpInst::ICMP_NE: {
6573 if (Op0Max.ult(RHS: Op1Min) || Op0Min.ugt(RHS: Op1Max))
6574 return replaceInstUsesWith(
6575 I, V: ConstantInt::getBool(Ty: I.getType(), V: Pred == CmpInst::ICMP_NE));
6576
6577 // If all bits are known zero except for one, then we know at most one bit
6578 // is set. If the comparison is against zero, then this is a check to see if
6579 // *that* bit is set.
6580 APInt Op0KnownZeroInverted = ~Op0Known.Zero;
6581 if (Op1Known.isZero()) {
6582 // If the LHS is an AND with the same constant, look through it.
6583 Value *LHS = nullptr;
6584 const APInt *LHSC;
6585 if (!match(V: Op0, P: m_And(L: m_Value(V&: LHS), R: m_APInt(Res&: LHSC))) ||
6586 *LHSC != Op0KnownZeroInverted)
6587 LHS = Op0;
6588
6589 Value *X;
6590 const APInt *C1;
6591 if (match(V: LHS, P: m_Shl(L: m_Power2(V&: C1), R: m_Value(V&: X)))) {
6592 Type *XTy = X->getType();
6593 unsigned Log2C1 = C1->countr_zero();
6594 APInt C2 = Op0KnownZeroInverted;
6595 APInt C2Pow2 = (C2 & ~(*C1 - 1)) + *C1;
6596 if (C2Pow2.isPowerOf2()) {
6597 // iff (C1 is pow2) & ((C2 & ~(C1-1)) + C1) is pow2):
6598 // ((C1 << X) & C2) == 0 -> X >= (Log2(C2+C1) - Log2(C1))
6599 // ((C1 << X) & C2) != 0 -> X < (Log2(C2+C1) - Log2(C1))
6600 unsigned Log2C2 = C2Pow2.countr_zero();
6601 auto *CmpC = ConstantInt::get(Ty: XTy, V: Log2C2 - Log2C1);
6602 auto NewPred =
6603 Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGE : CmpInst::ICMP_ULT;
6604 return new ICmpInst(NewPred, X, CmpC);
6605 }
6606 }
6607 }
6608
6609 // Op0 eq C_Pow2 -> Op0 ne 0 if Op0 is known to be C_Pow2 or zero.
6610 if (Op1Known.isConstant() && Op1Known.getConstant().isPowerOf2() &&
6611 (Op0Known & Op1Known) == Op0Known)
6612 return new ICmpInst(CmpInst::getInversePredicate(pred: Pred), Op0,
6613 ConstantInt::getNullValue(Ty: Op1->getType()));
6614 break;
6615 }
6616 case ICmpInst::ICMP_ULT: {
6617 if (Op0Max.ult(RHS: Op1Min)) // A <u B -> true if max(A) < min(B)
6618 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
6619 if (Op0Min.uge(RHS: Op1Max)) // A <u B -> false if min(A) >= max(B)
6620 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
6621 break;
6622 }
6623 case ICmpInst::ICMP_UGT: {
6624 if (Op0Min.ugt(RHS: Op1Max)) // A >u B -> true if min(A) > max(B)
6625 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
6626 if (Op0Max.ule(RHS: Op1Min)) // A >u B -> false if max(A) <= max(B)
6627 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
6628 break;
6629 }
6630 case ICmpInst::ICMP_SLT: {
6631 if (Op0Max.slt(RHS: Op1Min)) // A <s B -> true if max(A) < min(C)
6632 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
6633 if (Op0Min.sge(RHS: Op1Max)) // A <s B -> false if min(A) >= max(C)
6634 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
6635 break;
6636 }
6637 case ICmpInst::ICMP_SGT: {
6638 if (Op0Min.sgt(RHS: Op1Max)) // A >s B -> true if min(A) > max(B)
6639 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
6640 if (Op0Max.sle(RHS: Op1Min)) // A >s B -> false if max(A) <= min(B)
6641 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
6642 break;
6643 }
6644 case ICmpInst::ICMP_SGE:
6645 assert(!isa<ConstantInt>(Op1) && "ICMP_SGE with ConstantInt not folded!");
6646 if (Op0Min.sge(RHS: Op1Max)) // A >=s B -> true if min(A) >= max(B)
6647 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
6648 if (Op0Max.slt(RHS: Op1Min)) // A >=s B -> false if max(A) < min(B)
6649 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
6650 if (Op1Min == Op0Max) // A >=s B -> A == B if max(A) == min(B)
6651 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6652 break;
6653 case ICmpInst::ICMP_SLE:
6654 assert(!isa<ConstantInt>(Op1) && "ICMP_SLE with ConstantInt not folded!");
6655 if (Op0Max.sle(RHS: Op1Min)) // A <=s B -> true if max(A) <= min(B)
6656 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
6657 if (Op0Min.sgt(RHS: Op1Max)) // A <=s B -> false if min(A) > max(B)
6658 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
6659 if (Op1Max == Op0Min) // A <=s B -> A == B if min(A) == max(B)
6660 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6661 break;
6662 case ICmpInst::ICMP_UGE:
6663 assert(!isa<ConstantInt>(Op1) && "ICMP_UGE with ConstantInt not folded!");
6664 if (Op0Min.uge(RHS: Op1Max)) // A >=u B -> true if min(A) >= max(B)
6665 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
6666 if (Op0Max.ult(RHS: Op1Min)) // A >=u B -> false if max(A) < min(B)
6667 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
6668 if (Op1Min == Op0Max) // A >=u B -> A == B if max(A) == min(B)
6669 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6670 break;
6671 case ICmpInst::ICMP_ULE:
6672 assert(!isa<ConstantInt>(Op1) && "ICMP_ULE with ConstantInt not folded!");
6673 if (Op0Max.ule(RHS: Op1Min)) // A <=u B -> true if max(A) <= min(B)
6674 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
6675 if (Op0Min.ugt(RHS: Op1Max)) // A <=u B -> false if min(A) > max(B)
6676 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
6677 if (Op1Max == Op0Min) // A <=u B -> A == B if min(A) == max(B)
6678 return new ICmpInst(ICmpInst::ICMP_EQ, Op0, Op1);
6679 break;
6680 }
6681
6682 // Turn a signed comparison into an unsigned one if both operands are known to
6683 // have the same sign.
6684 if (I.isSigned() &&
6685 ((Op0Known.Zero.isNegative() && Op1Known.Zero.isNegative()) ||
6686 (Op0Known.One.isNegative() && Op1Known.One.isNegative())))
6687 return new ICmpInst(I.getUnsignedPredicate(), Op0, Op1);
6688
6689 return nullptr;
6690}
6691
6692/// If one operand of an icmp is effectively a bool (value range of {0,1}),
6693/// then try to reduce patterns based on that limit.
6694Instruction *InstCombinerImpl::foldICmpUsingBoolRange(ICmpInst &I) {
6695 Value *X, *Y;
6696 ICmpInst::Predicate Pred;
6697
6698 // X must be 0 and bool must be true for "ULT":
6699 // X <u (zext i1 Y) --> (X == 0) & Y
6700 if (match(V: &I, P: m_c_ICmp(Pred, L: m_Value(V&: X), R: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: Y))))) &&
6701 Y->getType()->isIntOrIntVectorTy(BitWidth: 1) && Pred == ICmpInst::ICMP_ULT)
6702 return BinaryOperator::CreateAnd(V1: Builder.CreateIsNull(Arg: X), V2: Y);
6703
6704 // X must be 0 or bool must be true for "ULE":
6705 // X <=u (sext i1 Y) --> (X == 0) | Y
6706 if (match(V: &I, P: m_c_ICmp(Pred, L: m_Value(V&: X), R: m_OneUse(SubPattern: m_SExt(Op: m_Value(V&: Y))))) &&
6707 Y->getType()->isIntOrIntVectorTy(BitWidth: 1) && Pred == ICmpInst::ICMP_ULE)
6708 return BinaryOperator::CreateOr(V1: Builder.CreateIsNull(Arg: X), V2: Y);
6709
6710 // icmp eq/ne X, (zext/sext (icmp eq/ne X, C))
6711 ICmpInst::Predicate Pred1, Pred2;
6712 const APInt *C;
6713 Instruction *ExtI;
6714 if (match(V: &I, P: m_c_ICmp(Pred&: Pred1, L: m_Value(V&: X),
6715 R: m_CombineAnd(L: m_Instruction(I&: ExtI),
6716 R: m_ZExtOrSExt(Op: m_ICmp(Pred&: Pred2, L: m_Deferred(V: X),
6717 R: m_APInt(Res&: C)))))) &&
6718 ICmpInst::isEquality(P: Pred1) && ICmpInst::isEquality(P: Pred2)) {
6719 bool IsSExt = ExtI->getOpcode() == Instruction::SExt;
6720 bool HasOneUse = ExtI->hasOneUse() && ExtI->getOperand(i: 0)->hasOneUse();
6721 auto CreateRangeCheck = [&] {
6722 Value *CmpV1 =
6723 Builder.CreateICmp(P: Pred1, LHS: X, RHS: Constant::getNullValue(Ty: X->getType()));
6724 Value *CmpV2 = Builder.CreateICmp(
6725 P: Pred1, LHS: X, RHS: ConstantInt::getSigned(Ty: X->getType(), V: IsSExt ? -1 : 1));
6726 return BinaryOperator::Create(
6727 Op: Pred1 == ICmpInst::ICMP_EQ ? Instruction::Or : Instruction::And,
6728 S1: CmpV1, S2: CmpV2);
6729 };
6730 if (C->isZero()) {
6731 if (Pred2 == ICmpInst::ICMP_EQ) {
6732 // icmp eq X, (zext/sext (icmp eq X, 0)) --> false
6733 // icmp ne X, (zext/sext (icmp eq X, 0)) --> true
6734 return replaceInstUsesWith(
6735 I, V: ConstantInt::getBool(Ty: I.getType(), V: Pred1 == ICmpInst::ICMP_NE));
6736 } else if (!IsSExt || HasOneUse) {
6737 // icmp eq X, (zext (icmp ne X, 0)) --> X == 0 || X == 1
6738 // icmp ne X, (zext (icmp ne X, 0)) --> X != 0 && X != 1
6739 // icmp eq X, (sext (icmp ne X, 0)) --> X == 0 || X == -1
6740 // icmp ne X, (sext (icmp ne X, 0)) --> X != 0 && X == -1
6741 return CreateRangeCheck();
6742 }
6743 } else if (IsSExt ? C->isAllOnes() : C->isOne()) {
6744 if (Pred2 == ICmpInst::ICMP_NE) {
6745 // icmp eq X, (zext (icmp ne X, 1)) --> false
6746 // icmp ne X, (zext (icmp ne X, 1)) --> true
6747 // icmp eq X, (sext (icmp ne X, -1)) --> false
6748 // icmp ne X, (sext (icmp ne X, -1)) --> true
6749 return replaceInstUsesWith(
6750 I, V: ConstantInt::getBool(Ty: I.getType(), V: Pred1 == ICmpInst::ICMP_NE));
6751 } else if (!IsSExt || HasOneUse) {
6752 // icmp eq X, (zext (icmp eq X, 1)) --> X == 0 || X == 1
6753 // icmp ne X, (zext (icmp eq X, 1)) --> X != 0 && X != 1
6754 // icmp eq X, (sext (icmp eq X, -1)) --> X == 0 || X == -1
6755 // icmp ne X, (sext (icmp eq X, -1)) --> X != 0 && X == -1
6756 return CreateRangeCheck();
6757 }
6758 } else {
6759 // when C != 0 && C != 1:
6760 // icmp eq X, (zext (icmp eq X, C)) --> icmp eq X, 0
6761 // icmp eq X, (zext (icmp ne X, C)) --> icmp eq X, 1
6762 // icmp ne X, (zext (icmp eq X, C)) --> icmp ne X, 0
6763 // icmp ne X, (zext (icmp ne X, C)) --> icmp ne X, 1
6764 // when C != 0 && C != -1:
6765 // icmp eq X, (sext (icmp eq X, C)) --> icmp eq X, 0
6766 // icmp eq X, (sext (icmp ne X, C)) --> icmp eq X, -1
6767 // icmp ne X, (sext (icmp eq X, C)) --> icmp ne X, 0
6768 // icmp ne X, (sext (icmp ne X, C)) --> icmp ne X, -1
6769 return ICmpInst::Create(
6770 Op: Instruction::ICmp, Pred: Pred1, S1: X,
6771 S2: ConstantInt::getSigned(Ty: X->getType(), V: Pred2 == ICmpInst::ICMP_NE
6772 ? (IsSExt ? -1 : 1)
6773 : 0));
6774 }
6775 }
6776
6777 return nullptr;
6778}
6779
6780std::optional<std::pair<CmpInst::Predicate, Constant *>>
6781InstCombiner::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred,
6782 Constant *C) {
6783 assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
6784 "Only for relational integer predicates.");
6785
6786 Type *Type = C->getType();
6787 bool IsSigned = ICmpInst::isSigned(predicate: Pred);
6788
6789 CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(pred: Pred);
6790 bool WillIncrement =
6791 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
6792
6793 // Check if the constant operand can be safely incremented/decremented
6794 // without overflowing/underflowing.
6795 auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) {
6796 return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned);
6797 };
6798
6799 Constant *SafeReplacementConstant = nullptr;
6800 if (auto *CI = dyn_cast<ConstantInt>(Val: C)) {
6801 // Bail out if the constant can't be safely incremented/decremented.
6802 if (!ConstantIsOk(CI))
6803 return std::nullopt;
6804 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Type)) {
6805 unsigned NumElts = FVTy->getNumElements();
6806 for (unsigned i = 0; i != NumElts; ++i) {
6807 Constant *Elt = C->getAggregateElement(Elt: i);
6808 if (!Elt)
6809 return std::nullopt;
6810
6811 if (isa<UndefValue>(Val: Elt))
6812 continue;
6813
6814 // Bail out if we can't determine if this constant is min/max or if we
6815 // know that this constant is min/max.
6816 auto *CI = dyn_cast<ConstantInt>(Val: Elt);
6817 if (!CI || !ConstantIsOk(CI))
6818 return std::nullopt;
6819
6820 if (!SafeReplacementConstant)
6821 SafeReplacementConstant = CI;
6822 }
6823 } else if (isa<VectorType>(Val: C->getType())) {
6824 // Handle scalable splat
6825 Value *SplatC = C->getSplatValue();
6826 auto *CI = dyn_cast_or_null<ConstantInt>(Val: SplatC);
6827 // Bail out if the constant can't be safely incremented/decremented.
6828 if (!CI || !ConstantIsOk(CI))
6829 return std::nullopt;
6830 } else {
6831 // ConstantExpr?
6832 return std::nullopt;
6833 }
6834
6835 // It may not be safe to change a compare predicate in the presence of
6836 // undefined elements, so replace those elements with the first safe constant
6837 // that we found.
6838 // TODO: in case of poison, it is safe; let's replace undefs only.
6839 if (C->containsUndefOrPoisonElement()) {
6840 assert(SafeReplacementConstant && "Replacement constant not set");
6841 C = Constant::replaceUndefsWith(C, Replacement: SafeReplacementConstant);
6842 }
6843
6844 CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(pred: Pred);
6845
6846 // Increment or decrement the constant.
6847 Constant *OneOrNegOne = ConstantInt::get(Ty: Type, V: WillIncrement ? 1 : -1, IsSigned: true);
6848 Constant *NewC = ConstantExpr::getAdd(C1: C, C2: OneOrNegOne);
6849
6850 return std::make_pair(x&: NewPred, y&: NewC);
6851}
6852
6853/// If we have an icmp le or icmp ge instruction with a constant operand, turn
6854/// it into the appropriate icmp lt or icmp gt instruction. This transform
6855/// allows them to be folded in visitICmpInst.
6856static ICmpInst *canonicalizeCmpWithConstant(ICmpInst &I) {
6857 ICmpInst::Predicate Pred = I.getPredicate();
6858 if (ICmpInst::isEquality(P: Pred) || !ICmpInst::isIntPredicate(P: Pred) ||
6859 InstCombiner::isCanonicalPredicate(Pred))
6860 return nullptr;
6861
6862 Value *Op0 = I.getOperand(i_nocapture: 0);
6863 Value *Op1 = I.getOperand(i_nocapture: 1);
6864 auto *Op1C = dyn_cast<Constant>(Val: Op1);
6865 if (!Op1C)
6866 return nullptr;
6867
6868 auto FlippedStrictness =
6869 InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, C: Op1C);
6870 if (!FlippedStrictness)
6871 return nullptr;
6872
6873 return new ICmpInst(FlippedStrictness->first, Op0, FlippedStrictness->second);
6874}
6875
6876/// If we have a comparison with a non-canonical predicate, if we can update
6877/// all the users, invert the predicate and adjust all the users.
6878CmpInst *InstCombinerImpl::canonicalizeICmpPredicate(CmpInst &I) {
6879 // Is the predicate already canonical?
6880 CmpInst::Predicate Pred = I.getPredicate();
6881 if (InstCombiner::isCanonicalPredicate(Pred))
6882 return nullptr;
6883
6884 // Can all users be adjusted to predicate inversion?
6885 if (!InstCombiner::canFreelyInvertAllUsersOf(V: &I, /*IgnoredUser=*/nullptr))
6886 return nullptr;
6887
6888 // Ok, we can canonicalize comparison!
6889 // Let's first invert the comparison's predicate.
6890 I.setPredicate(CmpInst::getInversePredicate(pred: Pred));
6891 I.setName(I.getName() + ".not");
6892
6893 // And, adapt users.
6894 freelyInvertAllUsersOf(V: &I);
6895
6896 return &I;
6897}
6898
6899/// Integer compare with boolean values can always be turned into bitwise ops.
6900static Instruction *canonicalizeICmpBool(ICmpInst &I,
6901 InstCombiner::BuilderTy &Builder) {
6902 Value *A = I.getOperand(i_nocapture: 0), *B = I.getOperand(i_nocapture: 1);
6903 assert(A->getType()->isIntOrIntVectorTy(1) && "Bools only");
6904
6905 // A boolean compared to true/false can be simplified to Op0/true/false in
6906 // 14 out of the 20 (10 predicates * 2 constants) possible combinations.
6907 // Cases not handled by InstSimplify are always 'not' of Op0.
6908 if (match(V: B, P: m_Zero())) {
6909 switch (I.getPredicate()) {
6910 case CmpInst::ICMP_EQ: // A == 0 -> !A
6911 case CmpInst::ICMP_ULE: // A <=u 0 -> !A
6912 case CmpInst::ICMP_SGE: // A >=s 0 -> !A
6913 return BinaryOperator::CreateNot(Op: A);
6914 default:
6915 llvm_unreachable("ICmp i1 X, C not simplified as expected.");
6916 }
6917 } else if (match(V: B, P: m_One())) {
6918 switch (I.getPredicate()) {
6919 case CmpInst::ICMP_NE: // A != 1 -> !A
6920 case CmpInst::ICMP_ULT: // A <u 1 -> !A
6921 case CmpInst::ICMP_SGT: // A >s -1 -> !A
6922 return BinaryOperator::CreateNot(Op: A);
6923 default:
6924 llvm_unreachable("ICmp i1 X, C not simplified as expected.");
6925 }
6926 }
6927
6928 switch (I.getPredicate()) {
6929 default:
6930 llvm_unreachable("Invalid icmp instruction!");
6931 case ICmpInst::ICMP_EQ:
6932 // icmp eq i1 A, B -> ~(A ^ B)
6933 return BinaryOperator::CreateNot(Op: Builder.CreateXor(LHS: A, RHS: B));
6934
6935 case ICmpInst::ICMP_NE:
6936 // icmp ne i1 A, B -> A ^ B
6937 return BinaryOperator::CreateXor(V1: A, V2: B);
6938
6939 case ICmpInst::ICMP_UGT:
6940 // icmp ugt -> icmp ult
6941 std::swap(a&: A, b&: B);
6942 [[fallthrough]];
6943 case ICmpInst::ICMP_ULT:
6944 // icmp ult i1 A, B -> ~A & B
6945 return BinaryOperator::CreateAnd(V1: Builder.CreateNot(V: A), V2: B);
6946
6947 case ICmpInst::ICMP_SGT:
6948 // icmp sgt -> icmp slt
6949 std::swap(a&: A, b&: B);
6950 [[fallthrough]];
6951 case ICmpInst::ICMP_SLT:
6952 // icmp slt i1 A, B -> A & ~B
6953 return BinaryOperator::CreateAnd(V1: Builder.CreateNot(V: B), V2: A);
6954
6955 case ICmpInst::ICMP_UGE:
6956 // icmp uge -> icmp ule
6957 std::swap(a&: A, b&: B);
6958 [[fallthrough]];
6959 case ICmpInst::ICMP_ULE:
6960 // icmp ule i1 A, B -> ~A | B
6961 return BinaryOperator::CreateOr(V1: Builder.CreateNot(V: A), V2: B);
6962
6963 case ICmpInst::ICMP_SGE:
6964 // icmp sge -> icmp sle
6965 std::swap(a&: A, b&: B);
6966 [[fallthrough]];
6967 case ICmpInst::ICMP_SLE:
6968 // icmp sle i1 A, B -> A | ~B
6969 return BinaryOperator::CreateOr(V1: Builder.CreateNot(V: B), V2: A);
6970 }
6971}
6972
6973// Transform pattern like:
6974// (1 << Y) u<= X or ~(-1 << Y) u< X or ((1 << Y)+(-1)) u< X
6975// (1 << Y) u> X or ~(-1 << Y) u>= X or ((1 << Y)+(-1)) u>= X
6976// Into:
6977// (X l>> Y) != 0
6978// (X l>> Y) == 0
6979static Instruction *foldICmpWithHighBitMask(ICmpInst &Cmp,
6980 InstCombiner::BuilderTy &Builder) {
6981 ICmpInst::Predicate Pred, NewPred;
6982 Value *X, *Y;
6983 if (match(V: &Cmp,
6984 P: m_c_ICmp(Pred, L: m_OneUse(SubPattern: m_Shl(L: m_One(), R: m_Value(V&: Y))), R: m_Value(V&: X)))) {
6985 switch (Pred) {
6986 case ICmpInst::ICMP_ULE:
6987 NewPred = ICmpInst::ICMP_NE;
6988 break;
6989 case ICmpInst::ICMP_UGT:
6990 NewPred = ICmpInst::ICMP_EQ;
6991 break;
6992 default:
6993 return nullptr;
6994 }
6995 } else if (match(V: &Cmp, P: m_c_ICmp(Pred,
6996 L: m_OneUse(SubPattern: m_CombineOr(
6997 L: m_Not(V: m_Shl(L: m_AllOnes(), R: m_Value(V&: Y))),
6998 R: m_Add(L: m_Shl(L: m_One(), R: m_Value(V&: Y)),
6999 R: m_AllOnes()))),
7000 R: m_Value(V&: X)))) {
7001 // The variant with 'add' is not canonical, (the variant with 'not' is)
7002 // we only get it because it has extra uses, and can't be canonicalized,
7003
7004 switch (Pred) {
7005 case ICmpInst::ICMP_ULT:
7006 NewPred = ICmpInst::ICMP_NE;
7007 break;
7008 case ICmpInst::ICMP_UGE:
7009 NewPred = ICmpInst::ICMP_EQ;
7010 break;
7011 default:
7012 return nullptr;
7013 }
7014 } else
7015 return nullptr;
7016
7017 Value *NewX = Builder.CreateLShr(LHS: X, RHS: Y, Name: X->getName() + ".highbits");
7018 Constant *Zero = Constant::getNullValue(Ty: NewX->getType());
7019 return CmpInst::Create(Op: Instruction::ICmp, Pred: NewPred, S1: NewX, S2: Zero);
7020}
7021
7022static Instruction *foldVectorCmp(CmpInst &Cmp,
7023 InstCombiner::BuilderTy &Builder) {
7024 const CmpInst::Predicate Pred = Cmp.getPredicate();
7025 Value *LHS = Cmp.getOperand(i_nocapture: 0), *RHS = Cmp.getOperand(i_nocapture: 1);
7026 Value *V1, *V2;
7027
7028 auto createCmpReverse = [&](CmpInst::Predicate Pred, Value *X, Value *Y) {
7029 Value *V = Builder.CreateCmp(Pred, LHS: X, RHS: Y, Name: Cmp.getName());
7030 if (auto *I = dyn_cast<Instruction>(Val: V))
7031 I->copyIRFlags(V: &Cmp);
7032 Module *M = Cmp.getModule();
7033 Function *F =
7034 Intrinsic::getDeclaration(M, id: Intrinsic::vector_reverse, Tys: V->getType());
7035 return CallInst::Create(Func: F, Args: V);
7036 };
7037
7038 if (match(V: LHS, P: m_VecReverse(Op0: m_Value(V&: V1)))) {
7039 // cmp Pred, rev(V1), rev(V2) --> rev(cmp Pred, V1, V2)
7040 if (match(V: RHS, P: m_VecReverse(Op0: m_Value(V&: V2))) &&
7041 (LHS->hasOneUse() || RHS->hasOneUse()))
7042 return createCmpReverse(Pred, V1, V2);
7043
7044 // cmp Pred, rev(V1), RHSSplat --> rev(cmp Pred, V1, RHSSplat)
7045 if (LHS->hasOneUse() && isSplatValue(V: RHS))
7046 return createCmpReverse(Pred, V1, RHS);
7047 }
7048 // cmp Pred, LHSSplat, rev(V2) --> rev(cmp Pred, LHSSplat, V2)
7049 else if (isSplatValue(V: LHS) && match(V: RHS, P: m_OneUse(SubPattern: m_VecReverse(Op0: m_Value(V&: V2)))))
7050 return createCmpReverse(Pred, LHS, V2);
7051
7052 ArrayRef<int> M;
7053 if (!match(V: LHS, P: m_Shuffle(v1: m_Value(V&: V1), v2: m_Undef(), mask: m_Mask(M))))
7054 return nullptr;
7055
7056 // If both arguments of the cmp are shuffles that use the same mask and
7057 // shuffle within a single vector, move the shuffle after the cmp:
7058 // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
7059 Type *V1Ty = V1->getType();
7060 if (match(V: RHS, P: m_Shuffle(v1: m_Value(V&: V2), v2: m_Undef(), mask: m_SpecificMask(M))) &&
7061 V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) {
7062 Value *NewCmp = Builder.CreateCmp(Pred, LHS: V1, RHS: V2);
7063 return new ShuffleVectorInst(NewCmp, M);
7064 }
7065
7066 // Try to canonicalize compare with splatted operand and splat constant.
7067 // TODO: We could generalize this for more than splats. See/use the code in
7068 // InstCombiner::foldVectorBinop().
7069 Constant *C;
7070 if (!LHS->hasOneUse() || !match(V: RHS, P: m_Constant(C)))
7071 return nullptr;
7072
7073 // Length-changing splats are ok, so adjust the constants as needed:
7074 // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
7075 Constant *ScalarC = C->getSplatValue(/* AllowPoison */ true);
7076 int MaskSplatIndex;
7077 if (ScalarC && match(Mask: M, P: m_SplatOrPoisonMask(MaskSplatIndex))) {
7078 // We allow poison in matching, but this transform removes it for safety.
7079 // Demanded elements analysis should be able to recover some/all of that.
7080 C = ConstantVector::getSplat(EC: cast<VectorType>(Val: V1Ty)->getElementCount(),
7081 Elt: ScalarC);
7082 SmallVector<int, 8> NewM(M.size(), MaskSplatIndex);
7083 Value *NewCmp = Builder.CreateCmp(Pred, LHS: V1, RHS: C);
7084 return new ShuffleVectorInst(NewCmp, NewM);
7085 }
7086
7087 return nullptr;
7088}
7089
7090// extract(uadd.with.overflow(A, B), 0) ult A
7091// -> extract(uadd.with.overflow(A, B), 1)
7092static Instruction *foldICmpOfUAddOv(ICmpInst &I) {
7093 CmpInst::Predicate Pred = I.getPredicate();
7094 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
7095
7096 Value *UAddOv;
7097 Value *A, *B;
7098 auto UAddOvResultPat = m_ExtractValue<0>(
7099 V: m_Intrinsic<Intrinsic::uadd_with_overflow>(Op0: m_Value(V&: A), Op1: m_Value(V&: B)));
7100 if (match(V: Op0, P: UAddOvResultPat) &&
7101 ((Pred == ICmpInst::ICMP_ULT && (Op1 == A || Op1 == B)) ||
7102 (Pred == ICmpInst::ICMP_EQ && match(V: Op1, P: m_ZeroInt()) &&
7103 (match(V: A, P: m_One()) || match(V: B, P: m_One()))) ||
7104 (Pred == ICmpInst::ICMP_NE && match(V: Op1, P: m_AllOnes()) &&
7105 (match(V: A, P: m_AllOnes()) || match(V: B, P: m_AllOnes())))))
7106 // extract(uadd.with.overflow(A, B), 0) < A
7107 // extract(uadd.with.overflow(A, 1), 0) == 0
7108 // extract(uadd.with.overflow(A, -1), 0) != -1
7109 UAddOv = cast<ExtractValueInst>(Val: Op0)->getAggregateOperand();
7110 else if (match(V: Op1, P: UAddOvResultPat) &&
7111 Pred == ICmpInst::ICMP_UGT && (Op0 == A || Op0 == B))
7112 // A > extract(uadd.with.overflow(A, B), 0)
7113 UAddOv = cast<ExtractValueInst>(Val: Op1)->getAggregateOperand();
7114 else
7115 return nullptr;
7116
7117 return ExtractValueInst::Create(Agg: UAddOv, Idxs: 1);
7118}
7119
7120static Instruction *foldICmpInvariantGroup(ICmpInst &I) {
7121 if (!I.getOperand(i_nocapture: 0)->getType()->isPointerTy() ||
7122 NullPointerIsDefined(
7123 F: I.getParent()->getParent(),
7124 AS: I.getOperand(i_nocapture: 0)->getType()->getPointerAddressSpace())) {
7125 return nullptr;
7126 }
7127 Instruction *Op;
7128 if (match(V: I.getOperand(i_nocapture: 0), P: m_Instruction(I&: Op)) &&
7129 match(V: I.getOperand(i_nocapture: 1), P: m_Zero()) &&
7130 Op->isLaunderOrStripInvariantGroup()) {
7131 return ICmpInst::Create(Op: Instruction::ICmp, Pred: I.getPredicate(),
7132 S1: Op->getOperand(i: 0), S2: I.getOperand(i_nocapture: 1));
7133 }
7134 return nullptr;
7135}
7136
7137/// This function folds patterns produced by lowering of reduce idioms, such as
7138/// llvm.vector.reduce.and which are lowered into instruction chains. This code
7139/// attempts to generate fewer number of scalar comparisons instead of vector
7140/// comparisons when possible.
7141static Instruction *foldReductionIdiom(ICmpInst &I,
7142 InstCombiner::BuilderTy &Builder,
7143 const DataLayout &DL) {
7144 if (I.getType()->isVectorTy())
7145 return nullptr;
7146 ICmpInst::Predicate OuterPred, InnerPred;
7147 Value *LHS, *RHS;
7148
7149 // Match lowering of @llvm.vector.reduce.and. Turn
7150 /// %vec_ne = icmp ne <8 x i8> %lhs, %rhs
7151 /// %scalar_ne = bitcast <8 x i1> %vec_ne to i8
7152 /// %res = icmp <pred> i8 %scalar_ne, 0
7153 ///
7154 /// into
7155 ///
7156 /// %lhs.scalar = bitcast <8 x i8> %lhs to i64
7157 /// %rhs.scalar = bitcast <8 x i8> %rhs to i64
7158 /// %res = icmp <pred> i64 %lhs.scalar, %rhs.scalar
7159 ///
7160 /// for <pred> in {ne, eq}.
7161 if (!match(V: &I, P: m_ICmp(Pred&: OuterPred,
7162 L: m_OneUse(SubPattern: m_BitCast(Op: m_OneUse(
7163 SubPattern: m_ICmp(Pred&: InnerPred, L: m_Value(V&: LHS), R: m_Value(V&: RHS))))),
7164 R: m_Zero())))
7165 return nullptr;
7166 auto *LHSTy = dyn_cast<FixedVectorType>(Val: LHS->getType());
7167 if (!LHSTy || !LHSTy->getElementType()->isIntegerTy())
7168 return nullptr;
7169 unsigned NumBits =
7170 LHSTy->getNumElements() * LHSTy->getElementType()->getIntegerBitWidth();
7171 // TODO: Relax this to "not wider than max legal integer type"?
7172 if (!DL.isLegalInteger(Width: NumBits))
7173 return nullptr;
7174
7175 if (ICmpInst::isEquality(P: OuterPred) && InnerPred == ICmpInst::ICMP_NE) {
7176 auto *ScalarTy = Builder.getIntNTy(N: NumBits);
7177 LHS = Builder.CreateBitCast(V: LHS, DestTy: ScalarTy, Name: LHS->getName() + ".scalar");
7178 RHS = Builder.CreateBitCast(V: RHS, DestTy: ScalarTy, Name: RHS->getName() + ".scalar");
7179 return ICmpInst::Create(Op: Instruction::ICmp, Pred: OuterPred, S1: LHS, S2: RHS,
7180 Name: I.getName());
7181 }
7182
7183 return nullptr;
7184}
7185
7186// This helper will be called with icmp operands in both orders.
7187Instruction *InstCombinerImpl::foldICmpCommutative(ICmpInst::Predicate Pred,
7188 Value *Op0, Value *Op1,
7189 ICmpInst &CxtI) {
7190 // Try to optimize 'icmp GEP, P' or 'icmp P, GEP'.
7191 if (auto *GEP = dyn_cast<GEPOperator>(Val: Op0))
7192 if (Instruction *NI = foldGEPICmp(GEPLHS: GEP, RHS: Op1, Cond: Pred, I&: CxtI))
7193 return NI;
7194
7195 if (auto *SI = dyn_cast<SelectInst>(Val: Op0))
7196 if (Instruction *NI = foldSelectICmp(Pred, SI, RHS: Op1, I: CxtI))
7197 return NI;
7198
7199 if (auto *MinMax = dyn_cast<MinMaxIntrinsic>(Val: Op0))
7200 if (Instruction *Res = foldICmpWithMinMax(I&: CxtI, MinMax, Z: Op1, Pred))
7201 return Res;
7202
7203 {
7204 Value *X;
7205 const APInt *C;
7206 // icmp X+Cst, X
7207 if (match(V: Op0, P: m_Add(L: m_Value(V&: X), R: m_APInt(Res&: C))) && Op1 == X)
7208 return foldICmpAddOpConst(X, C: *C, Pred);
7209 }
7210
7211 // abs(X) >= X --> true
7212 // abs(X) u<= X --> true
7213 // abs(X) < X --> false
7214 // abs(X) u> X --> false
7215 // abs(X) u>= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7216 // abs(X) <= X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7217 // abs(X) == X --> IsIntMinPosion ? `X > -1`: `X u<= INTMIN`
7218 // abs(X) u< X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7219 // abs(X) > X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7220 // abs(X) != X --> IsIntMinPosion ? `X < 0` : `X > INTMIN`
7221 {
7222 Value *X;
7223 Constant *C;
7224 if (match(V: Op0, P: m_Intrinsic<Intrinsic::abs>(Op0: m_Value(V&: X), Op1: m_Constant(C))) &&
7225 match(V: Op1, P: m_Specific(V: X))) {
7226 Value *NullValue = Constant::getNullValue(Ty: X->getType());
7227 Value *AllOnesValue = Constant::getAllOnesValue(Ty: X->getType());
7228 const APInt SMin =
7229 APInt::getSignedMinValue(numBits: X->getType()->getScalarSizeInBits());
7230 bool IsIntMinPosion = C->isAllOnesValue();
7231 switch (Pred) {
7232 case CmpInst::ICMP_ULE:
7233 case CmpInst::ICMP_SGE:
7234 return replaceInstUsesWith(I&: CxtI, V: ConstantInt::getTrue(Ty: CxtI.getType()));
7235 case CmpInst::ICMP_UGT:
7236 case CmpInst::ICMP_SLT:
7237 return replaceInstUsesWith(I&: CxtI, V: ConstantInt::getFalse(Ty: CxtI.getType()));
7238 case CmpInst::ICMP_UGE:
7239 case CmpInst::ICMP_SLE:
7240 case CmpInst::ICMP_EQ: {
7241 return replaceInstUsesWith(
7242 I&: CxtI, V: IsIntMinPosion
7243 ? Builder.CreateICmpSGT(LHS: X, RHS: AllOnesValue)
7244 : Builder.CreateICmpULT(
7245 LHS: X, RHS: ConstantInt::get(Ty: X->getType(), V: SMin + 1)));
7246 }
7247 case CmpInst::ICMP_ULT:
7248 case CmpInst::ICMP_SGT:
7249 case CmpInst::ICMP_NE: {
7250 return replaceInstUsesWith(
7251 I&: CxtI, V: IsIntMinPosion
7252 ? Builder.CreateICmpSLT(LHS: X, RHS: NullValue)
7253 : Builder.CreateICmpUGT(
7254 LHS: X, RHS: ConstantInt::get(Ty: X->getType(), V: SMin)));
7255 }
7256 default:
7257 llvm_unreachable("Invalid predicate!");
7258 }
7259 }
7260 }
7261
7262 const SimplifyQuery Q = SQ.getWithInstruction(I: &CxtI);
7263 if (Value *V = foldICmpWithLowBitMaskedVal(Pred, Op0, Op1, Q, IC&: *this))
7264 return replaceInstUsesWith(I&: CxtI, V);
7265
7266 // Folding (X / Y) pred X => X swap(pred) 0 for constant Y other than 0 or 1
7267 auto CheckUGT1 = [](const APInt &Divisor) { return Divisor.ugt(RHS: 1); };
7268 {
7269 if (match(V: Op0, P: m_UDiv(L: m_Specific(V: Op1), R: m_CheckedInt(CheckFn: CheckUGT1)))) {
7270 return new ICmpInst(ICmpInst::getSwappedPredicate(pred: Pred), Op1,
7271 Constant::getNullValue(Ty: Op1->getType()));
7272 }
7273
7274 if (!ICmpInst::isUnsigned(predicate: Pred) &&
7275 match(V: Op0, P: m_SDiv(L: m_Specific(V: Op1), R: m_CheckedInt(CheckFn: CheckUGT1)))) {
7276 return new ICmpInst(ICmpInst::getSwappedPredicate(pred: Pred), Op1,
7277 Constant::getNullValue(Ty: Op1->getType()));
7278 }
7279 }
7280
7281 // Another case of this fold is (X >> Y) pred X => X swap(pred) 0 if Y != 0
7282 auto CheckNE0 = [](const APInt &Shift) { return !Shift.isZero(); };
7283 {
7284 if (match(V: Op0, P: m_LShr(L: m_Specific(V: Op1), R: m_CheckedInt(CheckFn: CheckNE0)))) {
7285 return new ICmpInst(ICmpInst::getSwappedPredicate(pred: Pred), Op1,
7286 Constant::getNullValue(Ty: Op1->getType()));
7287 }
7288
7289 if ((Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SGE) &&
7290 match(V: Op0, P: m_AShr(L: m_Specific(V: Op1), R: m_CheckedInt(CheckFn: CheckNE0)))) {
7291 return new ICmpInst(ICmpInst::getSwappedPredicate(pred: Pred), Op1,
7292 Constant::getNullValue(Ty: Op1->getType()));
7293 }
7294 }
7295
7296 return nullptr;
7297}
7298
7299Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
7300 bool Changed = false;
7301 const SimplifyQuery Q = SQ.getWithInstruction(I: &I);
7302 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
7303 unsigned Op0Cplxity = getComplexity(V: Op0);
7304 unsigned Op1Cplxity = getComplexity(V: Op1);
7305
7306 /// Orders the operands of the compare so that they are listed from most
7307 /// complex to least complex. This puts constants before unary operators,
7308 /// before binary operators.
7309 if (Op0Cplxity < Op1Cplxity) {
7310 I.swapOperands();
7311 std::swap(a&: Op0, b&: Op1);
7312 Changed = true;
7313 }
7314
7315 if (Value *V = simplifyICmpInst(Predicate: I.getPredicate(), LHS: Op0, RHS: Op1, Q))
7316 return replaceInstUsesWith(I, V);
7317
7318 // Comparing -val or val with non-zero is the same as just comparing val
7319 // ie, abs(val) != 0 -> val != 0
7320 if (I.getPredicate() == ICmpInst::ICMP_NE && match(V: Op1, P: m_Zero())) {
7321 Value *Cond, *SelectTrue, *SelectFalse;
7322 if (match(V: Op0, P: m_Select(C: m_Value(V&: Cond), L: m_Value(V&: SelectTrue),
7323 R: m_Value(V&: SelectFalse)))) {
7324 if (Value *V = dyn_castNegVal(V: SelectTrue)) {
7325 if (V == SelectFalse)
7326 return CmpInst::Create(Op: Instruction::ICmp, Pred: I.getPredicate(), S1: V, S2: Op1);
7327 }
7328 else if (Value *V = dyn_castNegVal(V: SelectFalse)) {
7329 if (V == SelectTrue)
7330 return CmpInst::Create(Op: Instruction::ICmp, Pred: I.getPredicate(), S1: V, S2: Op1);
7331 }
7332 }
7333 }
7334
7335 if (Op0->getType()->isIntOrIntVectorTy(BitWidth: 1))
7336 if (Instruction *Res = canonicalizeICmpBool(I, Builder))
7337 return Res;
7338
7339 if (Instruction *Res = canonicalizeCmpWithConstant(I))
7340 return Res;
7341
7342 if (Instruction *Res = canonicalizeICmpPredicate(I))
7343 return Res;
7344
7345 if (Instruction *Res = foldICmpWithConstant(Cmp&: I))
7346 return Res;
7347
7348 if (Instruction *Res = foldICmpWithDominatingICmp(Cmp&: I))
7349 return Res;
7350
7351 if (Instruction *Res = foldICmpUsingBoolRange(I))
7352 return Res;
7353
7354 if (Instruction *Res = foldICmpUsingKnownBits(I))
7355 return Res;
7356
7357 if (Instruction *Res = foldICmpTruncWithTruncOrExt(Cmp&: I, Q))
7358 return Res;
7359
7360 // Test if the ICmpInst instruction is used exclusively by a select as
7361 // part of a minimum or maximum operation. If so, refrain from doing
7362 // any other folding. This helps out other analyses which understand
7363 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
7364 // and CodeGen. And in this case, at least one of the comparison
7365 // operands has at least one user besides the compare (the select),
7366 // which would often largely negate the benefit of folding anyway.
7367 //
7368 // Do the same for the other patterns recognized by matchSelectPattern.
7369 if (I.hasOneUse())
7370 if (SelectInst *SI = dyn_cast<SelectInst>(Val: I.user_back())) {
7371 Value *A, *B;
7372 SelectPatternResult SPR = matchSelectPattern(V: SI, LHS&: A, RHS&: B);
7373 if (SPR.Flavor != SPF_UNKNOWN)
7374 return nullptr;
7375 }
7376
7377 // Do this after checking for min/max to prevent infinite looping.
7378 if (Instruction *Res = foldICmpWithZero(Cmp&: I))
7379 return Res;
7380
7381 // FIXME: We only do this after checking for min/max to prevent infinite
7382 // looping caused by a reverse canonicalization of these patterns for min/max.
7383 // FIXME: The organization of folds is a mess. These would naturally go into
7384 // canonicalizeCmpWithConstant(), but we can't move all of the above folds
7385 // down here after the min/max restriction.
7386 ICmpInst::Predicate Pred = I.getPredicate();
7387 const APInt *C;
7388 if (match(V: Op1, P: m_APInt(Res&: C))) {
7389 // For i32: x >u 2147483647 -> x <s 0 -> true if sign bit set
7390 if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) {
7391 Constant *Zero = Constant::getNullValue(Ty: Op0->getType());
7392 return new ICmpInst(ICmpInst::ICMP_SLT, Op0, Zero);
7393 }
7394
7395 // For i32: x <u 2147483648 -> x >s -1 -> true if sign bit clear
7396 if (Pred == ICmpInst::ICMP_ULT && C->isMinSignedValue()) {
7397 Constant *AllOnes = Constant::getAllOnesValue(Ty: Op0->getType());
7398 return new ICmpInst(ICmpInst::ICMP_SGT, Op0, AllOnes);
7399 }
7400 }
7401
7402 // The folds in here may rely on wrapping flags and special constants, so
7403 // they can break up min/max idioms in some cases but not seemingly similar
7404 // patterns.
7405 // FIXME: It may be possible to enhance select folding to make this
7406 // unnecessary. It may also be moot if we canonicalize to min/max
7407 // intrinsics.
7408 if (Instruction *Res = foldICmpBinOp(I, SQ: Q))
7409 return Res;
7410
7411 if (Instruction *Res = foldICmpInstWithConstant(Cmp&: I))
7412 return Res;
7413
7414 // Try to match comparison as a sign bit test. Intentionally do this after
7415 // foldICmpInstWithConstant() to potentially let other folds to happen first.
7416 if (Instruction *New = foldSignBitTest(I))
7417 return New;
7418
7419 if (Instruction *Res = foldICmpInstWithConstantNotInt(I))
7420 return Res;
7421
7422 if (Instruction *Res = foldICmpCommutative(Pred: I.getPredicate(), Op0, Op1, CxtI&: I))
7423 return Res;
7424 if (Instruction *Res =
7425 foldICmpCommutative(Pred: I.getSwappedPredicate(), Op0: Op1, Op1: Op0, CxtI&: I))
7426 return Res;
7427
7428 if (I.isCommutative()) {
7429 if (auto Pair = matchSymmetricPair(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1))) {
7430 replaceOperand(I, OpNum: 0, V: Pair->first);
7431 replaceOperand(I, OpNum: 1, V: Pair->second);
7432 return &I;
7433 }
7434 }
7435
7436 // In case of a comparison with two select instructions having the same
7437 // condition, check whether one of the resulting branches can be simplified.
7438 // If so, just compare the other branch and select the appropriate result.
7439 // For example:
7440 // %tmp1 = select i1 %cmp, i32 %y, i32 %x
7441 // %tmp2 = select i1 %cmp, i32 %z, i32 %x
7442 // %cmp2 = icmp slt i32 %tmp2, %tmp1
7443 // The icmp will result false for the false value of selects and the result
7444 // will depend upon the comparison of true values of selects if %cmp is
7445 // true. Thus, transform this into:
7446 // %cmp = icmp slt i32 %y, %z
7447 // %sel = select i1 %cond, i1 %cmp, i1 false
7448 // This handles similar cases to transform.
7449 {
7450 Value *Cond, *A, *B, *C, *D;
7451 if (match(V: Op0, P: m_Select(C: m_Value(V&: Cond), L: m_Value(V&: A), R: m_Value(V&: B))) &&
7452 match(V: Op1, P: m_Select(C: m_Specific(V: Cond), L: m_Value(V&: C), R: m_Value(V&: D))) &&
7453 (Op0->hasOneUse() || Op1->hasOneUse())) {
7454 // Check whether comparison of TrueValues can be simplified
7455 if (Value *Res = simplifyICmpInst(Predicate: Pred, LHS: A, RHS: C, Q: SQ)) {
7456 Value *NewICMP = Builder.CreateICmp(P: Pred, LHS: B, RHS: D);
7457 return SelectInst::Create(C: Cond, S1: Res, S2: NewICMP);
7458 }
7459 // Check whether comparison of FalseValues can be simplified
7460 if (Value *Res = simplifyICmpInst(Predicate: Pred, LHS: B, RHS: D, Q: SQ)) {
7461 Value *NewICMP = Builder.CreateICmp(P: Pred, LHS: A, RHS: C);
7462 return SelectInst::Create(C: Cond, S1: NewICMP, S2: Res);
7463 }
7464 }
7465 }
7466
7467 // Try to optimize equality comparisons against alloca-based pointers.
7468 if (Op0->getType()->isPointerTy() && I.isEquality()) {
7469 assert(Op1->getType()->isPointerTy() && "Comparing pointer with non-pointer?");
7470 if (auto *Alloca = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: Op0)))
7471 if (foldAllocaCmp(Alloca))
7472 return nullptr;
7473 if (auto *Alloca = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: Op1)))
7474 if (foldAllocaCmp(Alloca))
7475 return nullptr;
7476 }
7477
7478 if (Instruction *Res = foldICmpBitCast(Cmp&: I))
7479 return Res;
7480
7481 // TODO: Hoist this above the min/max bailout.
7482 if (Instruction *R = foldICmpWithCastOp(ICmp&: I))
7483 return R;
7484
7485 {
7486 Value *X, *Y;
7487 // Transform (X & ~Y) == 0 --> (X & Y) != 0
7488 // and (X & ~Y) != 0 --> (X & Y) == 0
7489 // if A is a power of 2.
7490 if (match(V: Op0, P: m_And(L: m_Value(V&: X), R: m_Not(V: m_Value(V&: Y)))) &&
7491 match(V: Op1, P: m_Zero()) && isKnownToBeAPowerOfTwo(V: X, OrZero: false, Depth: 0, CxtI: &I) &&
7492 I.isEquality())
7493 return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(LHS: X, RHS: Y),
7494 Op1);
7495
7496 // Op0 pred Op1 -> ~Op1 pred ~Op0, if this allows us to drop an instruction.
7497 if (Op0->getType()->isIntOrIntVectorTy()) {
7498 bool ConsumesOp0, ConsumesOp1;
7499 if (isFreeToInvert(V: Op0, WillInvertAllUses: Op0->hasOneUse(), DoesConsume&: ConsumesOp0) &&
7500 isFreeToInvert(V: Op1, WillInvertAllUses: Op1->hasOneUse(), DoesConsume&: ConsumesOp1) &&
7501 (ConsumesOp0 || ConsumesOp1)) {
7502 Value *InvOp0 = getFreelyInverted(V: Op0, WillInvertAllUses: Op0->hasOneUse(), Builder: &Builder);
7503 Value *InvOp1 = getFreelyInverted(V: Op1, WillInvertAllUses: Op1->hasOneUse(), Builder: &Builder);
7504 assert(InvOp0 && InvOp1 &&
7505 "Mismatch between isFreeToInvert and getFreelyInverted");
7506 return new ICmpInst(I.getSwappedPredicate(), InvOp0, InvOp1);
7507 }
7508 }
7509
7510 Instruction *AddI = nullptr;
7511 if (match(V: &I, P: m_UAddWithOverflow(L: m_Value(V&: X), R: m_Value(V&: Y),
7512 S: m_Instruction(I&: AddI))) &&
7513 isa<IntegerType>(Val: X->getType())) {
7514 Value *Result;
7515 Constant *Overflow;
7516 // m_UAddWithOverflow can match patterns that do not include an explicit
7517 // "add" instruction, so check the opcode of the matched op.
7518 if (AddI->getOpcode() == Instruction::Add &&
7519 OptimizeOverflowCheck(BinaryOp: Instruction::Add, /*Signed*/ IsSigned: false, LHS: X, RHS: Y, OrigI&: *AddI,
7520 Result, Overflow)) {
7521 replaceInstUsesWith(I&: *AddI, V: Result);
7522 eraseInstFromFunction(I&: *AddI);
7523 return replaceInstUsesWith(I, V: Overflow);
7524 }
7525 }
7526
7527 // (zext X) * (zext Y) --> llvm.umul.with.overflow.
7528 if (match(V: Op0, P: m_NUWMul(L: m_ZExt(Op: m_Value(V&: X)), R: m_ZExt(Op: m_Value(V&: Y)))) &&
7529 match(V: Op1, P: m_APInt(Res&: C))) {
7530 if (Instruction *R = processUMulZExtIdiom(I, MulVal: Op0, OtherVal: C, IC&: *this))
7531 return R;
7532 }
7533
7534 // Signbit test folds
7535 // Fold (X u>> BitWidth - 1 Pred ZExt(i1)) --> X s< 0 Pred i1
7536 // Fold (X s>> BitWidth - 1 Pred SExt(i1)) --> X s< 0 Pred i1
7537 Instruction *ExtI;
7538 if ((I.isUnsigned() || I.isEquality()) &&
7539 match(V: Op1,
7540 P: m_CombineAnd(L: m_Instruction(I&: ExtI), R: m_ZExtOrSExt(Op: m_Value(V&: Y)))) &&
7541 Y->getType()->getScalarSizeInBits() == 1 &&
7542 (Op0->hasOneUse() || Op1->hasOneUse())) {
7543 unsigned OpWidth = Op0->getType()->getScalarSizeInBits();
7544 Instruction *ShiftI;
7545 if (match(V: Op0, P: m_CombineAnd(L: m_Instruction(I&: ShiftI),
7546 R: m_Shr(L: m_Value(V&: X), R: m_SpecificIntAllowPoison(
7547 V: OpWidth - 1))))) {
7548 unsigned ExtOpc = ExtI->getOpcode();
7549 unsigned ShiftOpc = ShiftI->getOpcode();
7550 if ((ExtOpc == Instruction::ZExt && ShiftOpc == Instruction::LShr) ||
7551 (ExtOpc == Instruction::SExt && ShiftOpc == Instruction::AShr)) {
7552 Value *SLTZero =
7553 Builder.CreateICmpSLT(LHS: X, RHS: Constant::getNullValue(Ty: X->getType()));
7554 Value *Cmp = Builder.CreateICmp(P: Pred, LHS: SLTZero, RHS: Y, Name: I.getName());
7555 return replaceInstUsesWith(I, V: Cmp);
7556 }
7557 }
7558 }
7559 }
7560
7561 if (Instruction *Res = foldICmpEquality(I))
7562 return Res;
7563
7564 if (Instruction *Res = foldICmpPow2Test(I, Builder))
7565 return Res;
7566
7567 if (Instruction *Res = foldICmpOfUAddOv(I))
7568 return Res;
7569
7570 // The 'cmpxchg' instruction returns an aggregate containing the old value and
7571 // an i1 which indicates whether or not we successfully did the swap.
7572 //
7573 // Replace comparisons between the old value and the expected value with the
7574 // indicator that 'cmpxchg' returns.
7575 //
7576 // N.B. This transform is only valid when the 'cmpxchg' is not permitted to
7577 // spuriously fail. In those cases, the old value may equal the expected
7578 // value but it is possible for the swap to not occur.
7579 if (I.getPredicate() == ICmpInst::ICMP_EQ)
7580 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: Op0))
7581 if (auto *ACXI = dyn_cast<AtomicCmpXchgInst>(Val: EVI->getAggregateOperand()))
7582 if (EVI->getIndices()[0] == 0 && ACXI->getCompareOperand() == Op1 &&
7583 !ACXI->isWeak())
7584 return ExtractValueInst::Create(Agg: ACXI, Idxs: 1);
7585
7586 if (Instruction *Res = foldICmpWithHighBitMask(Cmp&: I, Builder))
7587 return Res;
7588
7589 if (I.getType()->isVectorTy())
7590 if (Instruction *Res = foldVectorCmp(Cmp&: I, Builder))
7591 return Res;
7592
7593 if (Instruction *Res = foldICmpInvariantGroup(I))
7594 return Res;
7595
7596 if (Instruction *Res = foldReductionIdiom(I, Builder, DL))
7597 return Res;
7598
7599 return Changed ? &I : nullptr;
7600}
7601
7602/// Fold fcmp ([us]itofp x, cst) if possible.
7603Instruction *InstCombinerImpl::foldFCmpIntToFPConst(FCmpInst &I,
7604 Instruction *LHSI,
7605 Constant *RHSC) {
7606 const APFloat *RHS;
7607 if (!match(V: RHSC, P: m_APFloat(Res&: RHS)))
7608 return nullptr;
7609
7610 // Get the width of the mantissa. We don't want to hack on conversions that
7611 // might lose information from the integer, e.g. "i64 -> float"
7612 int MantissaWidth = LHSI->getType()->getFPMantissaWidth();
7613 if (MantissaWidth == -1) return nullptr; // Unknown.
7614
7615 Type *IntTy = LHSI->getOperand(i: 0)->getType();
7616 unsigned IntWidth = IntTy->getScalarSizeInBits();
7617 bool LHSUnsigned = isa<UIToFPInst>(Val: LHSI);
7618
7619 if (I.isEquality()) {
7620 FCmpInst::Predicate P = I.getPredicate();
7621 bool IsExact = false;
7622 APSInt RHSCvt(IntWidth, LHSUnsigned);
7623 RHS->convertToInteger(Result&: RHSCvt, RM: APFloat::rmNearestTiesToEven, IsExact: &IsExact);
7624
7625 // If the floating point constant isn't an integer value, we know if we will
7626 // ever compare equal / not equal to it.
7627 if (!IsExact) {
7628 // TODO: Can never be -0.0 and other non-representable values
7629 APFloat RHSRoundInt(*RHS);
7630 RHSRoundInt.roundToIntegral(RM: APFloat::rmNearestTiesToEven);
7631 if (*RHS != RHSRoundInt) {
7632 if (P == FCmpInst::FCMP_OEQ || P == FCmpInst::FCMP_UEQ)
7633 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
7634
7635 assert(P == FCmpInst::FCMP_ONE || P == FCmpInst::FCMP_UNE);
7636 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
7637 }
7638 }
7639
7640 // TODO: If the constant is exactly representable, is it always OK to do
7641 // equality compares as integer?
7642 }
7643
7644 // Check to see that the input is converted from an integer type that is small
7645 // enough that preserves all bits. TODO: check here for "known" sign bits.
7646 // This would allow us to handle (fptosi (x >>s 62) to float) if x is i64 f.e.
7647
7648 // Following test does NOT adjust IntWidth downwards for signed inputs,
7649 // because the most negative value still requires all the mantissa bits
7650 // to distinguish it from one less than that value.
7651 if ((int)IntWidth > MantissaWidth) {
7652 // Conversion would lose accuracy. Check if loss can impact comparison.
7653 int Exp = ilogb(Arg: *RHS);
7654 if (Exp == APFloat::IEK_Inf) {
7655 int MaxExponent = ilogb(Arg: APFloat::getLargest(Sem: RHS->getSemantics()));
7656 if (MaxExponent < (int)IntWidth - !LHSUnsigned)
7657 // Conversion could create infinity.
7658 return nullptr;
7659 } else {
7660 // Note that if RHS is zero or NaN, then Exp is negative
7661 // and first condition is trivially false.
7662 if (MantissaWidth <= Exp && Exp <= (int)IntWidth - !LHSUnsigned)
7663 // Conversion could affect comparison.
7664 return nullptr;
7665 }
7666 }
7667
7668 // Otherwise, we can potentially simplify the comparison. We know that it
7669 // will always come through as an integer value and we know the constant is
7670 // not a NAN (it would have been previously simplified).
7671 assert(!RHS->isNaN() && "NaN comparison not already folded!");
7672
7673 ICmpInst::Predicate Pred;
7674 switch (I.getPredicate()) {
7675 default: llvm_unreachable("Unexpected predicate!");
7676 case FCmpInst::FCMP_UEQ:
7677 case FCmpInst::FCMP_OEQ:
7678 Pred = ICmpInst::ICMP_EQ;
7679 break;
7680 case FCmpInst::FCMP_UGT:
7681 case FCmpInst::FCMP_OGT:
7682 Pred = LHSUnsigned ? ICmpInst::ICMP_UGT : ICmpInst::ICMP_SGT;
7683 break;
7684 case FCmpInst::FCMP_UGE:
7685 case FCmpInst::FCMP_OGE:
7686 Pred = LHSUnsigned ? ICmpInst::ICMP_UGE : ICmpInst::ICMP_SGE;
7687 break;
7688 case FCmpInst::FCMP_ULT:
7689 case FCmpInst::FCMP_OLT:
7690 Pred = LHSUnsigned ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_SLT;
7691 break;
7692 case FCmpInst::FCMP_ULE:
7693 case FCmpInst::FCMP_OLE:
7694 Pred = LHSUnsigned ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_SLE;
7695 break;
7696 case FCmpInst::FCMP_UNE:
7697 case FCmpInst::FCMP_ONE:
7698 Pred = ICmpInst::ICMP_NE;
7699 break;
7700 case FCmpInst::FCMP_ORD:
7701 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
7702 case FCmpInst::FCMP_UNO:
7703 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
7704 }
7705
7706 // Now we know that the APFloat is a normal number, zero or inf.
7707
7708 // See if the FP constant is too large for the integer. For example,
7709 // comparing an i8 to 300.0.
7710 if (!LHSUnsigned) {
7711 // If the RHS value is > SignedMax, fold the comparison. This handles +INF
7712 // and large values.
7713 APFloat SMax(RHS->getSemantics());
7714 SMax.convertFromAPInt(Input: APInt::getSignedMaxValue(numBits: IntWidth), IsSigned: true,
7715 RM: APFloat::rmNearestTiesToEven);
7716 if (SMax < *RHS) { // smax < 13123.0
7717 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SLT ||
7718 Pred == ICmpInst::ICMP_SLE)
7719 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
7720 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
7721 }
7722 } else {
7723 // If the RHS value is > UnsignedMax, fold the comparison. This handles
7724 // +INF and large values.
7725 APFloat UMax(RHS->getSemantics());
7726 UMax.convertFromAPInt(Input: APInt::getMaxValue(numBits: IntWidth), IsSigned: false,
7727 RM: APFloat::rmNearestTiesToEven);
7728 if (UMax < *RHS) { // umax < 13123.0
7729 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_ULT ||
7730 Pred == ICmpInst::ICMP_ULE)
7731 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
7732 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
7733 }
7734 }
7735
7736 if (!LHSUnsigned) {
7737 // See if the RHS value is < SignedMin.
7738 APFloat SMin(RHS->getSemantics());
7739 SMin.convertFromAPInt(Input: APInt::getSignedMinValue(numBits: IntWidth), IsSigned: true,
7740 RM: APFloat::rmNearestTiesToEven);
7741 if (SMin > *RHS) { // smin > 12312.0
7742 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT ||
7743 Pred == ICmpInst::ICMP_SGE)
7744 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
7745 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
7746 }
7747 } else {
7748 // See if the RHS value is < UnsignedMin.
7749 APFloat UMin(RHS->getSemantics());
7750 UMin.convertFromAPInt(Input: APInt::getMinValue(numBits: IntWidth), IsSigned: false,
7751 RM: APFloat::rmNearestTiesToEven);
7752 if (UMin > *RHS) { // umin > 12312.0
7753 if (Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_UGT ||
7754 Pred == ICmpInst::ICMP_UGE)
7755 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
7756 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
7757 }
7758 }
7759
7760 // Okay, now we know that the FP constant fits in the range [SMIN, SMAX] or
7761 // [0, UMAX], but it may still be fractional. Check whether this is the case
7762 // using the IsExact flag.
7763 // Don't do this for zero, because -0.0 is not fractional.
7764 APSInt RHSInt(IntWidth, LHSUnsigned);
7765 bool IsExact;
7766 RHS->convertToInteger(Result&: RHSInt, RM: APFloat::rmTowardZero, IsExact: &IsExact);
7767 if (!RHS->isZero()) {
7768 if (!IsExact) {
7769 // If we had a comparison against a fractional value, we have to adjust
7770 // the compare predicate and sometimes the value. RHSC is rounded towards
7771 // zero at this point.
7772 switch (Pred) {
7773 default: llvm_unreachable("Unexpected integer comparison!");
7774 case ICmpInst::ICMP_NE: // (float)int != 4.4 --> true
7775 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
7776 case ICmpInst::ICMP_EQ: // (float)int == 4.4 --> false
7777 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
7778 case ICmpInst::ICMP_ULE:
7779 // (float)int <= 4.4 --> int <= 4
7780 // (float)int <= -4.4 --> false
7781 if (RHS->isNegative())
7782 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
7783 break;
7784 case ICmpInst::ICMP_SLE:
7785 // (float)int <= 4.4 --> int <= 4
7786 // (float)int <= -4.4 --> int < -4
7787 if (RHS->isNegative())
7788 Pred = ICmpInst::ICMP_SLT;
7789 break;
7790 case ICmpInst::ICMP_ULT:
7791 // (float)int < -4.4 --> false
7792 // (float)int < 4.4 --> int <= 4
7793 if (RHS->isNegative())
7794 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
7795 Pred = ICmpInst::ICMP_ULE;
7796 break;
7797 case ICmpInst::ICMP_SLT:
7798 // (float)int < -4.4 --> int < -4
7799 // (float)int < 4.4 --> int <= 4
7800 if (!RHS->isNegative())
7801 Pred = ICmpInst::ICMP_SLE;
7802 break;
7803 case ICmpInst::ICMP_UGT:
7804 // (float)int > 4.4 --> int > 4
7805 // (float)int > -4.4 --> true
7806 if (RHS->isNegative())
7807 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
7808 break;
7809 case ICmpInst::ICMP_SGT:
7810 // (float)int > 4.4 --> int > 4
7811 // (float)int > -4.4 --> int >= -4
7812 if (RHS->isNegative())
7813 Pred = ICmpInst::ICMP_SGE;
7814 break;
7815 case ICmpInst::ICMP_UGE:
7816 // (float)int >= -4.4 --> true
7817 // (float)int >= 4.4 --> int > 4
7818 if (RHS->isNegative())
7819 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
7820 Pred = ICmpInst::ICMP_UGT;
7821 break;
7822 case ICmpInst::ICMP_SGE:
7823 // (float)int >= -4.4 --> int >= -4
7824 // (float)int >= 4.4 --> int > 4
7825 if (!RHS->isNegative())
7826 Pred = ICmpInst::ICMP_SGT;
7827 break;
7828 }
7829 }
7830 }
7831
7832 // Lower this FP comparison into an appropriate integer version of the
7833 // comparison.
7834 return new ICmpInst(Pred, LHSI->getOperand(i: 0),
7835 ConstantInt::get(Ty: LHSI->getOperand(i: 0)->getType(), V: RHSInt));
7836}
7837
7838/// Fold (C / X) < 0.0 --> X < 0.0 if possible. Swap predicate if necessary.
7839static Instruction *foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI,
7840 Constant *RHSC) {
7841 // When C is not 0.0 and infinities are not allowed:
7842 // (C / X) < 0.0 is a sign-bit test of X
7843 // (C / X) < 0.0 --> X < 0.0 (if C is positive)
7844 // (C / X) < 0.0 --> X > 0.0 (if C is negative, swap the predicate)
7845 //
7846 // Proof:
7847 // Multiply (C / X) < 0.0 by X * X / C.
7848 // - X is non zero, if it is the flag 'ninf' is violated.
7849 // - C defines the sign of X * X * C. Thus it also defines whether to swap
7850 // the predicate. C is also non zero by definition.
7851 //
7852 // Thus X * X / C is non zero and the transformation is valid. [qed]
7853
7854 FCmpInst::Predicate Pred = I.getPredicate();
7855
7856 // Check that predicates are valid.
7857 if ((Pred != FCmpInst::FCMP_OGT) && (Pred != FCmpInst::FCMP_OLT) &&
7858 (Pred != FCmpInst::FCMP_OGE) && (Pred != FCmpInst::FCMP_OLE))
7859 return nullptr;
7860
7861 // Check that RHS operand is zero.
7862 if (!match(V: RHSC, P: m_AnyZeroFP()))
7863 return nullptr;
7864
7865 // Check fastmath flags ('ninf').
7866 if (!LHSI->hasNoInfs() || !I.hasNoInfs())
7867 return nullptr;
7868
7869 // Check the properties of the dividend. It must not be zero to avoid a
7870 // division by zero (see Proof).
7871 const APFloat *C;
7872 if (!match(V: LHSI->getOperand(i: 0), P: m_APFloat(Res&: C)))
7873 return nullptr;
7874
7875 if (C->isZero())
7876 return nullptr;
7877
7878 // Get swapped predicate if necessary.
7879 if (C->isNegative())
7880 Pred = I.getSwappedPredicate();
7881
7882 return new FCmpInst(Pred, LHSI->getOperand(i: 1), RHSC, "", &I);
7883}
7884
7885/// Optimize fabs(X) compared with zero.
7886static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombinerImpl &IC) {
7887 Value *X;
7888 if (!match(V: I.getOperand(i_nocapture: 0), P: m_FAbs(Op0: m_Value(V&: X))))
7889 return nullptr;
7890
7891 const APFloat *C;
7892 if (!match(V: I.getOperand(i_nocapture: 1), P: m_APFloat(Res&: C)))
7893 return nullptr;
7894
7895 if (!C->isPosZero()) {
7896 if (!C->isSmallestNormalized())
7897 return nullptr;
7898
7899 const Function *F = I.getFunction();
7900 DenormalMode Mode = F->getDenormalMode(FPType: C->getSemantics());
7901 if (Mode.Input == DenormalMode::PreserveSign ||
7902 Mode.Input == DenormalMode::PositiveZero) {
7903
7904 auto replaceFCmp = [](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
7905 Constant *Zero = ConstantFP::getZero(Ty: X->getType());
7906 return new FCmpInst(P, X, Zero, "", I);
7907 };
7908
7909 switch (I.getPredicate()) {
7910 case FCmpInst::FCMP_OLT:
7911 // fcmp olt fabs(x), smallest_normalized_number -> fcmp oeq x, 0.0
7912 return replaceFCmp(&I, FCmpInst::FCMP_OEQ, X);
7913 case FCmpInst::FCMP_UGE:
7914 // fcmp uge fabs(x), smallest_normalized_number -> fcmp une x, 0.0
7915 return replaceFCmp(&I, FCmpInst::FCMP_UNE, X);
7916 case FCmpInst::FCMP_OGE:
7917 // fcmp oge fabs(x), smallest_normalized_number -> fcmp one x, 0.0
7918 return replaceFCmp(&I, FCmpInst::FCMP_ONE, X);
7919 case FCmpInst::FCMP_ULT:
7920 // fcmp ult fabs(x), smallest_normalized_number -> fcmp ueq x, 0.0
7921 return replaceFCmp(&I, FCmpInst::FCMP_UEQ, X);
7922 default:
7923 break;
7924 }
7925 }
7926
7927 return nullptr;
7928 }
7929
7930 auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
7931 I->setPredicate(P);
7932 return IC.replaceOperand(I&: *I, OpNum: 0, V: X);
7933 };
7934
7935 switch (I.getPredicate()) {
7936 case FCmpInst::FCMP_UGE:
7937 case FCmpInst::FCMP_OLT:
7938 // fabs(X) >= 0.0 --> true
7939 // fabs(X) < 0.0 --> false
7940 llvm_unreachable("fcmp should have simplified");
7941
7942 case FCmpInst::FCMP_OGT:
7943 // fabs(X) > 0.0 --> X != 0.0
7944 return replacePredAndOp0(&I, FCmpInst::FCMP_ONE, X);
7945
7946 case FCmpInst::FCMP_UGT:
7947 // fabs(X) u> 0.0 --> X u!= 0.0
7948 return replacePredAndOp0(&I, FCmpInst::FCMP_UNE, X);
7949
7950 case FCmpInst::FCMP_OLE:
7951 // fabs(X) <= 0.0 --> X == 0.0
7952 return replacePredAndOp0(&I, FCmpInst::FCMP_OEQ, X);
7953
7954 case FCmpInst::FCMP_ULE:
7955 // fabs(X) u<= 0.0 --> X u== 0.0
7956 return replacePredAndOp0(&I, FCmpInst::FCMP_UEQ, X);
7957
7958 case FCmpInst::FCMP_OGE:
7959 // fabs(X) >= 0.0 --> !isnan(X)
7960 assert(!I.hasNoNaNs() && "fcmp should have simplified");
7961 return replacePredAndOp0(&I, FCmpInst::FCMP_ORD, X);
7962
7963 case FCmpInst::FCMP_ULT:
7964 // fabs(X) u< 0.0 --> isnan(X)
7965 assert(!I.hasNoNaNs() && "fcmp should have simplified");
7966 return replacePredAndOp0(&I, FCmpInst::FCMP_UNO, X);
7967
7968 case FCmpInst::FCMP_OEQ:
7969 case FCmpInst::FCMP_UEQ:
7970 case FCmpInst::FCMP_ONE:
7971 case FCmpInst::FCMP_UNE:
7972 case FCmpInst::FCMP_ORD:
7973 case FCmpInst::FCMP_UNO:
7974 // Look through the fabs() because it doesn't change anything but the sign.
7975 // fabs(X) == 0.0 --> X == 0.0,
7976 // fabs(X) != 0.0 --> X != 0.0
7977 // isnan(fabs(X)) --> isnan(X)
7978 // !isnan(fabs(X) --> !isnan(X)
7979 return replacePredAndOp0(&I, I.getPredicate(), X);
7980
7981 default:
7982 return nullptr;
7983 }
7984}
7985
7986static Instruction *foldFCmpFNegCommonOp(FCmpInst &I) {
7987 CmpInst::Predicate Pred = I.getPredicate();
7988 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
7989
7990 // Canonicalize fneg as Op1.
7991 if (match(V: Op0, P: m_FNeg(X: m_Value())) && !match(V: Op1, P: m_FNeg(X: m_Value()))) {
7992 std::swap(a&: Op0, b&: Op1);
7993 Pred = I.getSwappedPredicate();
7994 }
7995
7996 if (!match(V: Op1, P: m_FNeg(X: m_Specific(V: Op0))))
7997 return nullptr;
7998
7999 // Replace the negated operand with 0.0:
8000 // fcmp Pred Op0, -Op0 --> fcmp Pred Op0, 0.0
8001 Constant *Zero = ConstantFP::getZero(Ty: Op0->getType());
8002 return new FCmpInst(Pred, Op0, Zero, "", &I);
8003}
8004
8005static Instruction *foldFCmpFSubIntoFCmp(FCmpInst &I, Instruction *LHSI,
8006 Constant *RHSC, InstCombinerImpl &CI) {
8007 const CmpInst::Predicate Pred = I.getPredicate();
8008 Value *X = LHSI->getOperand(i: 0);
8009 Value *Y = LHSI->getOperand(i: 1);
8010 switch (Pred) {
8011 default:
8012 break;
8013 case FCmpInst::FCMP_UGT:
8014 case FCmpInst::FCMP_ULT:
8015 case FCmpInst::FCMP_UNE:
8016 case FCmpInst::FCMP_OEQ:
8017 case FCmpInst::FCMP_OGE:
8018 case FCmpInst::FCMP_OLE:
8019 // The optimization is not valid if X and Y are infinities of the same
8020 // sign, i.e. the inf - inf = nan case. If the fsub has the ninf or nnan
8021 // flag then we can assume we do not have that case. Otherwise we might be
8022 // able to prove that either X or Y is not infinity.
8023 if (!LHSI->hasNoNaNs() && !LHSI->hasNoInfs() &&
8024 !isKnownNeverInfinity(V: Y, /*Depth=*/0,
8025 SQ: CI.getSimplifyQuery().getWithInstruction(I: &I)) &&
8026 !isKnownNeverInfinity(V: X, /*Depth=*/0,
8027 SQ: CI.getSimplifyQuery().getWithInstruction(I: &I)))
8028 break;
8029
8030 [[fallthrough]];
8031 case FCmpInst::FCMP_OGT:
8032 case FCmpInst::FCMP_OLT:
8033 case FCmpInst::FCMP_ONE:
8034 case FCmpInst::FCMP_UEQ:
8035 case FCmpInst::FCMP_UGE:
8036 case FCmpInst::FCMP_ULE:
8037 // fcmp pred (x - y), 0 --> fcmp pred x, y
8038 if (match(V: RHSC, P: m_AnyZeroFP()) &&
8039 I.getFunction()->getDenormalMode(
8040 FPType: LHSI->getType()->getScalarType()->getFltSemantics()) ==
8041 DenormalMode::getIEEE()) {
8042 CI.replaceOperand(I, OpNum: 0, V: X);
8043 CI.replaceOperand(I, OpNum: 1, V: Y);
8044 return &I;
8045 }
8046 break;
8047 }
8048
8049 return nullptr;
8050}
8051
8052Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) {
8053 bool Changed = false;
8054
8055 /// Orders the operands of the compare so that they are listed from most
8056 /// complex to least complex. This puts constants before unary operators,
8057 /// before binary operators.
8058 if (getComplexity(V: I.getOperand(i_nocapture: 0)) < getComplexity(V: I.getOperand(i_nocapture: 1))) {
8059 I.swapOperands();
8060 Changed = true;
8061 }
8062
8063 const CmpInst::Predicate Pred = I.getPredicate();
8064 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
8065 if (Value *V = simplifyFCmpInst(Predicate: Pred, LHS: Op0, RHS: Op1, FMF: I.getFastMathFlags(),
8066 Q: SQ.getWithInstruction(I: &I)))
8067 return replaceInstUsesWith(I, V);
8068
8069 // Simplify 'fcmp pred X, X'
8070 Type *OpType = Op0->getType();
8071 assert(OpType == Op1->getType() && "fcmp with different-typed operands?");
8072 if (Op0 == Op1) {
8073 switch (Pred) {
8074 default: break;
8075 case FCmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y)
8076 case FCmpInst::FCMP_ULT: // True if unordered or less than
8077 case FCmpInst::FCMP_UGT: // True if unordered or greater than
8078 case FCmpInst::FCMP_UNE: // True if unordered or not equal
8079 // Canonicalize these to be 'fcmp uno %X, 0.0'.
8080 I.setPredicate(FCmpInst::FCMP_UNO);
8081 I.setOperand(i_nocapture: 1, Val_nocapture: Constant::getNullValue(Ty: OpType));
8082 return &I;
8083
8084 case FCmpInst::FCMP_ORD: // True if ordered (no nans)
8085 case FCmpInst::FCMP_OEQ: // True if ordered and equal
8086 case FCmpInst::FCMP_OGE: // True if ordered and greater than or equal
8087 case FCmpInst::FCMP_OLE: // True if ordered and less than or equal
8088 // Canonicalize these to be 'fcmp ord %X, 0.0'.
8089 I.setPredicate(FCmpInst::FCMP_ORD);
8090 I.setOperand(i_nocapture: 1, Val_nocapture: Constant::getNullValue(Ty: OpType));
8091 return &I;
8092 }
8093 }
8094
8095 if (I.isCommutative()) {
8096 if (auto Pair = matchSymmetricPair(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1))) {
8097 replaceOperand(I, OpNum: 0, V: Pair->first);
8098 replaceOperand(I, OpNum: 1, V: Pair->second);
8099 return &I;
8100 }
8101 }
8102
8103 // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand,
8104 // then canonicalize the operand to 0.0.
8105 if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
8106 if (!match(V: Op0, P: m_PosZeroFP()) &&
8107 isKnownNeverNaN(V: Op0, Depth: 0, SQ: getSimplifyQuery().getWithInstruction(I: &I)))
8108 return replaceOperand(I, OpNum: 0, V: ConstantFP::getZero(Ty: OpType));
8109
8110 if (!match(V: Op1, P: m_PosZeroFP()) &&
8111 isKnownNeverNaN(V: Op1, Depth: 0, SQ: getSimplifyQuery().getWithInstruction(I: &I)))
8112 return replaceOperand(I, OpNum: 1, V: ConstantFP::getZero(Ty: OpType));
8113 }
8114
8115 // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y
8116 Value *X, *Y;
8117 if (match(V: Op0, P: m_FNeg(X: m_Value(V&: X))) && match(V: Op1, P: m_FNeg(X: m_Value(V&: Y))))
8118 return new FCmpInst(I.getSwappedPredicate(), X, Y, "", &I);
8119
8120 if (Instruction *R = foldFCmpFNegCommonOp(I))
8121 return R;
8122
8123 // Test if the FCmpInst instruction is used exclusively by a select as
8124 // part of a minimum or maximum operation. If so, refrain from doing
8125 // any other folding. This helps out other analyses which understand
8126 // non-obfuscated minimum and maximum idioms, such as ScalarEvolution
8127 // and CodeGen. And in this case, at least one of the comparison
8128 // operands has at least one user besides the compare (the select),
8129 // which would often largely negate the benefit of folding anyway.
8130 if (I.hasOneUse())
8131 if (SelectInst *SI = dyn_cast<SelectInst>(Val: I.user_back())) {
8132 Value *A, *B;
8133 SelectPatternResult SPR = matchSelectPattern(V: SI, LHS&: A, RHS&: B);
8134 if (SPR.Flavor != SPF_UNKNOWN)
8135 return nullptr;
8136 }
8137
8138 // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0:
8139 // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0
8140 if (match(V: Op1, P: m_AnyZeroFP()) && !match(V: Op1, P: m_PosZeroFP()))
8141 return replaceOperand(I, OpNum: 1, V: ConstantFP::getZero(Ty: OpType));
8142
8143 // Canonicalize:
8144 // fcmp olt X, +inf -> fcmp one X, +inf
8145 // fcmp ole X, +inf -> fcmp ord X, 0
8146 // fcmp ogt X, +inf -> false
8147 // fcmp oge X, +inf -> fcmp oeq X, +inf
8148 // fcmp ult X, +inf -> fcmp une X, +inf
8149 // fcmp ule X, +inf -> true
8150 // fcmp ugt X, +inf -> fcmp uno X, 0
8151 // fcmp uge X, +inf -> fcmp ueq X, +inf
8152 // fcmp olt X, -inf -> false
8153 // fcmp ole X, -inf -> fcmp oeq X, -inf
8154 // fcmp ogt X, -inf -> fcmp one X, -inf
8155 // fcmp oge X, -inf -> fcmp ord X, 0
8156 // fcmp ult X, -inf -> fcmp uno X, 0
8157 // fcmp ule X, -inf -> fcmp ueq X, -inf
8158 // fcmp ugt X, -inf -> fcmp une X, -inf
8159 // fcmp uge X, -inf -> true
8160 const APFloat *C;
8161 if (match(V: Op1, P: m_APFloat(Res&: C)) && C->isInfinity()) {
8162 switch (C->isNegative() ? FCmpInst::getSwappedPredicate(pred: Pred) : Pred) {
8163 default:
8164 break;
8165 case FCmpInst::FCMP_ORD:
8166 case FCmpInst::FCMP_UNO:
8167 case FCmpInst::FCMP_TRUE:
8168 case FCmpInst::FCMP_FALSE:
8169 case FCmpInst::FCMP_OGT:
8170 case FCmpInst::FCMP_ULE:
8171 llvm_unreachable("Should be simplified by InstSimplify");
8172 case FCmpInst::FCMP_OLT:
8173 return new FCmpInst(FCmpInst::FCMP_ONE, Op0, Op1, "", &I);
8174 case FCmpInst::FCMP_OLE:
8175 return new FCmpInst(FCmpInst::FCMP_ORD, Op0, ConstantFP::getZero(Ty: OpType),
8176 "", &I);
8177 case FCmpInst::FCMP_OGE:
8178 return new FCmpInst(FCmpInst::FCMP_OEQ, Op0, Op1, "", &I);
8179 case FCmpInst::FCMP_ULT:
8180 return new FCmpInst(FCmpInst::FCMP_UNE, Op0, Op1, "", &I);
8181 case FCmpInst::FCMP_UGT:
8182 return new FCmpInst(FCmpInst::FCMP_UNO, Op0, ConstantFP::getZero(Ty: OpType),
8183 "", &I);
8184 case FCmpInst::FCMP_UGE:
8185 return new FCmpInst(FCmpInst::FCMP_UEQ, Op0, Op1, "", &I);
8186 }
8187 }
8188
8189 // Ignore signbit of bitcasted int when comparing equality to FP 0.0:
8190 // fcmp oeq/une (bitcast X), 0.0 --> (and X, SignMaskC) ==/!= 0
8191 if (match(V: Op1, P: m_PosZeroFP()) &&
8192 match(V: Op0, P: m_OneUse(SubPattern: m_ElementWiseBitCast(Op: m_Value(V&: X))))) {
8193 ICmpInst::Predicate IntPred = ICmpInst::BAD_ICMP_PREDICATE;
8194 if (Pred == FCmpInst::FCMP_OEQ)
8195 IntPred = ICmpInst::ICMP_EQ;
8196 else if (Pred == FCmpInst::FCMP_UNE)
8197 IntPred = ICmpInst::ICMP_NE;
8198
8199 if (IntPred != ICmpInst::BAD_ICMP_PREDICATE) {
8200 Type *IntTy = X->getType();
8201 const APInt &SignMask = ~APInt::getSignMask(BitWidth: IntTy->getScalarSizeInBits());
8202 Value *MaskX = Builder.CreateAnd(LHS: X, RHS: ConstantInt::get(Ty: IntTy, V: SignMask));
8203 return new ICmpInst(IntPred, MaskX, ConstantInt::getNullValue(Ty: IntTy));
8204 }
8205 }
8206
8207 // Handle fcmp with instruction LHS and constant RHS.
8208 Instruction *LHSI;
8209 Constant *RHSC;
8210 if (match(V: Op0, P: m_Instruction(I&: LHSI)) && match(V: Op1, P: m_Constant(C&: RHSC))) {
8211 switch (LHSI->getOpcode()) {
8212 case Instruction::Select:
8213 // fcmp eq (cond ? x : -x), 0 --> fcmp eq x, 0
8214 if (FCmpInst::isEquality(Pred) && match(V: RHSC, P: m_AnyZeroFP()) &&
8215 (match(V: LHSI,
8216 P: m_Select(C: m_Value(), L: m_Value(V&: X), R: m_FNeg(X: m_Deferred(V: X)))) ||
8217 match(V: LHSI, P: m_Select(C: m_Value(), L: m_FNeg(X: m_Value(V&: X)), R: m_Deferred(V: X)))))
8218 return replaceOperand(I, OpNum: 0, V: X);
8219 if (Instruction *NV = FoldOpIntoSelect(Op&: I, SI: cast<SelectInst>(Val: LHSI)))
8220 return NV;
8221 break;
8222 case Instruction::FSub:
8223 if (LHSI->hasOneUse())
8224 if (Instruction *NV = foldFCmpFSubIntoFCmp(I, LHSI, RHSC, CI&: *this))
8225 return NV;
8226 break;
8227 case Instruction::PHI:
8228 if (Instruction *NV = foldOpIntoPhi(I, PN: cast<PHINode>(Val: LHSI)))
8229 return NV;
8230 break;
8231 case Instruction::SIToFP:
8232 case Instruction::UIToFP:
8233 if (Instruction *NV = foldFCmpIntToFPConst(I, LHSI, RHSC))
8234 return NV;
8235 break;
8236 case Instruction::FDiv:
8237 if (Instruction *NV = foldFCmpReciprocalAndZero(I, LHSI, RHSC))
8238 return NV;
8239 break;
8240 case Instruction::Load:
8241 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: LHSI->getOperand(i: 0)))
8242 if (auto *GV = dyn_cast<GlobalVariable>(Val: GEP->getOperand(i_nocapture: 0)))
8243 if (Instruction *Res = foldCmpLoadFromIndexedGlobal(
8244 LI: cast<LoadInst>(Val: LHSI), GEP, GV, ICI&: I))
8245 return Res;
8246 break;
8247 }
8248 }
8249
8250 if (Instruction *R = foldFabsWithFcmpZero(I, IC&: *this))
8251 return R;
8252
8253 if (match(V: Op0, P: m_FNeg(X: m_Value(V&: X)))) {
8254 // fcmp pred (fneg X), C --> fcmp swap(pred) X, -C
8255 Constant *C;
8256 if (match(V: Op1, P: m_Constant(C)))
8257 if (Constant *NegC = ConstantFoldUnaryOpOperand(Opcode: Instruction::FNeg, Op: C, DL))
8258 return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I);
8259 }
8260
8261 // fcmp (fadd X, 0.0), Y --> fcmp X, Y
8262 if (match(V: Op0, P: m_FAdd(L: m_Value(V&: X), R: m_AnyZeroFP())))
8263 return new FCmpInst(Pred, X, Op1, "", &I);
8264
8265 // fcmp X, (fadd Y, 0.0) --> fcmp X, Y
8266 if (match(V: Op1, P: m_FAdd(L: m_Value(V&: Y), R: m_AnyZeroFP())))
8267 return new FCmpInst(Pred, Op0, Y, "", &I);
8268
8269 if (match(V: Op0, P: m_FPExt(Op: m_Value(V&: X)))) {
8270 // fcmp (fpext X), (fpext Y) -> fcmp X, Y
8271 if (match(V: Op1, P: m_FPExt(Op: m_Value(V&: Y))) && X->getType() == Y->getType())
8272 return new FCmpInst(Pred, X, Y, "", &I);
8273
8274 const APFloat *C;
8275 if (match(V: Op1, P: m_APFloat(Res&: C))) {
8276 const fltSemantics &FPSem =
8277 X->getType()->getScalarType()->getFltSemantics();
8278 bool Lossy;
8279 APFloat TruncC = *C;
8280 TruncC.convert(ToSemantics: FPSem, RM: APFloat::rmNearestTiesToEven, losesInfo: &Lossy);
8281
8282 if (Lossy) {
8283 // X can't possibly equal the higher-precision constant, so reduce any
8284 // equality comparison.
8285 // TODO: Other predicates can be handled via getFCmpCode().
8286 switch (Pred) {
8287 case FCmpInst::FCMP_OEQ:
8288 // X is ordered and equal to an impossible constant --> false
8289 return replaceInstUsesWith(I, V: ConstantInt::getFalse(Ty: I.getType()));
8290 case FCmpInst::FCMP_ONE:
8291 // X is ordered and not equal to an impossible constant --> ordered
8292 return new FCmpInst(FCmpInst::FCMP_ORD, X,
8293 ConstantFP::getZero(Ty: X->getType()));
8294 case FCmpInst::FCMP_UEQ:
8295 // X is unordered or equal to an impossible constant --> unordered
8296 return new FCmpInst(FCmpInst::FCMP_UNO, X,
8297 ConstantFP::getZero(Ty: X->getType()));
8298 case FCmpInst::FCMP_UNE:
8299 // X is unordered or not equal to an impossible constant --> true
8300 return replaceInstUsesWith(I, V: ConstantInt::getTrue(Ty: I.getType()));
8301 default:
8302 break;
8303 }
8304 }
8305
8306 // fcmp (fpext X), C -> fcmp X, (fptrunc C) if fptrunc is lossless
8307 // Avoid lossy conversions and denormals.
8308 // Zero is a special case that's OK to convert.
8309 APFloat Fabs = TruncC;
8310 Fabs.clearSign();
8311 if (!Lossy &&
8312 (Fabs.isZero() || !(Fabs < APFloat::getSmallestNormalized(Sem: FPSem)))) {
8313 Constant *NewC = ConstantFP::get(Ty: X->getType(), V: TruncC);
8314 return new FCmpInst(Pred, X, NewC, "", &I);
8315 }
8316 }
8317 }
8318
8319 // Convert a sign-bit test of an FP value into a cast and integer compare.
8320 // TODO: Simplify if the copysign constant is 0.0 or NaN.
8321 // TODO: Handle non-zero compare constants.
8322 // TODO: Handle other predicates.
8323 if (match(V: Op0, P: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::copysign>(Op0: m_APFloat(Res&: C),
8324 Op1: m_Value(V&: X)))) &&
8325 match(V: Op1, P: m_AnyZeroFP()) && !C->isZero() && !C->isNaN()) {
8326 Type *IntType = Builder.getIntNTy(N: X->getType()->getScalarSizeInBits());
8327 if (auto *VecTy = dyn_cast<VectorType>(Val: OpType))
8328 IntType = VectorType::get(ElementType: IntType, EC: VecTy->getElementCount());
8329
8330 // copysign(non-zero constant, X) < 0.0 --> (bitcast X) < 0
8331 if (Pred == FCmpInst::FCMP_OLT) {
8332 Value *IntX = Builder.CreateBitCast(V: X, DestTy: IntType);
8333 return new ICmpInst(ICmpInst::ICMP_SLT, IntX,
8334 ConstantInt::getNullValue(Ty: IntType));
8335 }
8336 }
8337
8338 {
8339 Value *CanonLHS = nullptr, *CanonRHS = nullptr;
8340 match(V: Op0, P: m_Intrinsic<Intrinsic::canonicalize>(Op0: m_Value(V&: CanonLHS)));
8341 match(V: Op1, P: m_Intrinsic<Intrinsic::canonicalize>(Op0: m_Value(V&: CanonRHS)));
8342
8343 // (canonicalize(x) == x) => (x == x)
8344 if (CanonLHS == Op1)
8345 return new FCmpInst(Pred, Op1, Op1, "", &I);
8346
8347 // (x == canonicalize(x)) => (x == x)
8348 if (CanonRHS == Op0)
8349 return new FCmpInst(Pred, Op0, Op0, "", &I);
8350
8351 // (canonicalize(x) == canonicalize(y)) => (x == y)
8352 if (CanonLHS && CanonRHS)
8353 return new FCmpInst(Pred, CanonLHS, CanonRHS, "", &I);
8354 }
8355
8356 if (I.getType()->isVectorTy())
8357 if (Instruction *Res = foldVectorCmp(Cmp&: I, Builder))
8358 return Res;
8359
8360 return Changed ? &I : nullptr;
8361}
8362