1//===- InstCombineAndOrXor.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitAnd, visitOr, and visitXor functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/SmallBitVector.h"
15#include "llvm/Analysis/CmpInstAnalysis.h"
16#include "llvm/Analysis/FloatingPointPredicateUtils.h"
17#include "llvm/Analysis/InstructionSimplify.h"
18#include "llvm/IR/ConstantRange.h"
19#include "llvm/IR/DerivedTypes.h"
20#include "llvm/IR/Instructions.h"
21#include "llvm/IR/Intrinsics.h"
22#include "llvm/IR/PatternMatch.h"
23#include "llvm/Transforms/InstCombine/InstCombiner.h"
24#include "llvm/Transforms/Utils/Local.h"
25
26using namespace llvm;
27using namespace PatternMatch;
28
29#define DEBUG_TYPE "instcombine"
30
31namespace llvm {
32extern cl::opt<bool> ProfcheckDisableMetadataFixes;
33}
34
35/// This is the complement of getICmpCode, which turns an opcode and two
36/// operands into either a constant true or false, or a brand new ICmp
37/// instruction. The sign is passed in to determine which kind of predicate to
38/// use in the new icmp instruction.
39static Value *getNewICmpValue(unsigned Code, bool Sign, Value *LHS, Value *RHS,
40 InstCombiner::BuilderTy &Builder) {
41 ICmpInst::Predicate NewPred;
42 if (Constant *TorF = getPredForICmpCode(Code, Sign, OpTy: LHS->getType(), Pred&: NewPred))
43 return TorF;
44 return Builder.CreateICmp(P: NewPred, LHS, RHS);
45}
46
47/// This is the complement of getFCmpCode, which turns an opcode and two
48/// operands into either a FCmp instruction, or a true/false constant.
49static Value *getFCmpValue(unsigned Code, Value *LHS, Value *RHS,
50 InstCombiner::BuilderTy &Builder, FMFSource FMF) {
51 FCmpInst::Predicate NewPred;
52 if (Constant *TorF = getPredForFCmpCode(Code, OpTy: LHS->getType(), Pred&: NewPred))
53 return TorF;
54 return Builder.CreateFCmpFMF(P: NewPred, LHS, RHS, FMFSource: FMF);
55}
56
57/// Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise
58/// (V < Lo || V >= Hi). This method expects that Lo < Hi. IsSigned indicates
59/// whether to treat V, Lo, and Hi as signed or not.
60Value *InstCombinerImpl::insertRangeTest(Value *V, const APInt &Lo,
61 const APInt &Hi, bool isSigned,
62 bool Inside) {
63 assert((isSigned ? Lo.slt(Hi) : Lo.ult(Hi)) &&
64 "Lo is not < Hi in range emission code!");
65
66 Type *Ty = V->getType();
67
68 // V >= Min && V < Hi --> V < Hi
69 // V < Min || V >= Hi --> V >= Hi
70 ICmpInst::Predicate Pred = Inside ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE;
71 if (isSigned ? Lo.isMinSignedValue() : Lo.isMinValue()) {
72 Pred = isSigned ? ICmpInst::getSignedPredicate(Pred) : Pred;
73 return Builder.CreateICmp(P: Pred, LHS: V, RHS: ConstantInt::get(Ty, V: Hi));
74 }
75
76 // V >= Lo && V < Hi --> V - Lo u< Hi - Lo
77 // V < Lo || V >= Hi --> V - Lo u>= Hi - Lo
78 Value *VMinusLo =
79 Builder.CreateSub(LHS: V, RHS: ConstantInt::get(Ty, V: Lo), Name: V->getName() + ".off");
80 Constant *HiMinusLo = ConstantInt::get(Ty, V: Hi - Lo);
81 return Builder.CreateICmp(P: Pred, LHS: VMinusLo, RHS: HiMinusLo);
82}
83
84/// Classify (icmp eq (A & B), C) and (icmp ne (A & B), C) as matching patterns
85/// that can be simplified.
86/// One of A and B is considered the mask. The other is the value. This is
87/// described as the "AMask" or "BMask" part of the enum. If the enum contains
88/// only "Mask", then both A and B can be considered masks. If A is the mask,
89/// then it was proven that (A & C) == C. This is trivial if C == A or C == 0.
90/// If both A and C are constants, this proof is also easy.
91/// For the following explanations, we assume that A is the mask.
92///
93/// "AllOnes" declares that the comparison is true only if (A & B) == A or all
94/// bits of A are set in B.
95/// Example: (icmp eq (A & 3), 3) -> AMask_AllOnes
96///
97/// "AllZeros" declares that the comparison is true only if (A & B) == 0 or all
98/// bits of A are cleared in B.
99/// Example: (icmp eq (A & 3), 0) -> Mask_AllZeroes
100///
101/// "Mixed" declares that (A & B) == C and C might or might not contain any
102/// number of one bits and zero bits.
103/// Example: (icmp eq (A & 3), 1) -> AMask_Mixed
104///
105/// "Not" means that in above descriptions "==" should be replaced by "!=".
106/// Example: (icmp ne (A & 3), 3) -> AMask_NotAllOnes
107///
108/// If the mask A contains a single bit, then the following is equivalent:
109/// (icmp eq (A & B), A) equals (icmp ne (A & B), 0)
110/// (icmp ne (A & B), A) equals (icmp eq (A & B), 0)
111enum MaskedICmpType {
112 AMask_AllOnes = 1,
113 AMask_NotAllOnes = 2,
114 BMask_AllOnes = 4,
115 BMask_NotAllOnes = 8,
116 Mask_AllZeros = 16,
117 Mask_NotAllZeros = 32,
118 AMask_Mixed = 64,
119 AMask_NotMixed = 128,
120 BMask_Mixed = 256,
121 BMask_NotMixed = 512
122};
123
124/// Return the set of patterns (from MaskedICmpType) that (icmp SCC (A & B), C)
125/// satisfies.
126static unsigned getMaskedICmpType(Value *A, Value *B, Value *C,
127 ICmpInst::Predicate Pred) {
128 const APInt *ConstA = nullptr, *ConstB = nullptr, *ConstC = nullptr;
129 match(V: A, P: m_APInt(Res&: ConstA));
130 match(V: B, P: m_APInt(Res&: ConstB));
131 match(V: C, P: m_APInt(Res&: ConstC));
132 bool IsEq = (Pred == ICmpInst::ICMP_EQ);
133 bool IsAPow2 = ConstA && ConstA->isPowerOf2();
134 bool IsBPow2 = ConstB && ConstB->isPowerOf2();
135 unsigned MaskVal = 0;
136 if (ConstC && ConstC->isZero()) {
137 // if C is zero, then both A and B qualify as mask
138 MaskVal |= (IsEq ? (Mask_AllZeros | AMask_Mixed | BMask_Mixed)
139 : (Mask_NotAllZeros | AMask_NotMixed | BMask_NotMixed));
140 if (IsAPow2)
141 MaskVal |= (IsEq ? (AMask_NotAllOnes | AMask_NotMixed)
142 : (AMask_AllOnes | AMask_Mixed));
143 if (IsBPow2)
144 MaskVal |= (IsEq ? (BMask_NotAllOnes | BMask_NotMixed)
145 : (BMask_AllOnes | BMask_Mixed));
146 return MaskVal;
147 }
148
149 if (A == C) {
150 MaskVal |= (IsEq ? (AMask_AllOnes | AMask_Mixed)
151 : (AMask_NotAllOnes | AMask_NotMixed));
152 if (IsAPow2)
153 MaskVal |= (IsEq ? (Mask_NotAllZeros | AMask_NotMixed)
154 : (Mask_AllZeros | AMask_Mixed));
155 } else if (ConstA && ConstC && ConstC->isSubsetOf(RHS: *ConstA)) {
156 MaskVal |= (IsEq ? AMask_Mixed : AMask_NotMixed);
157 }
158
159 if (B == C) {
160 MaskVal |= (IsEq ? (BMask_AllOnes | BMask_Mixed)
161 : (BMask_NotAllOnes | BMask_NotMixed));
162 if (IsBPow2)
163 MaskVal |= (IsEq ? (Mask_NotAllZeros | BMask_NotMixed)
164 : (Mask_AllZeros | BMask_Mixed));
165 } else if (ConstB && ConstC && ConstC->isSubsetOf(RHS: *ConstB)) {
166 MaskVal |= (IsEq ? BMask_Mixed : BMask_NotMixed);
167 }
168
169 return MaskVal;
170}
171
172/// Convert an analysis of a masked ICmp into its equivalent if all boolean
173/// operations had the opposite sense. Since each "NotXXX" flag (recording !=)
174/// is adjacent to the corresponding normal flag (recording ==), this just
175/// involves swapping those bits over.
176static unsigned conjugateICmpMask(unsigned Mask) {
177 unsigned NewMask;
178 NewMask = (Mask & (AMask_AllOnes | BMask_AllOnes | Mask_AllZeros |
179 AMask_Mixed | BMask_Mixed))
180 << 1;
181
182 NewMask |= (Mask & (AMask_NotAllOnes | BMask_NotAllOnes | Mask_NotAllZeros |
183 AMask_NotMixed | BMask_NotMixed))
184 >> 1;
185
186 return NewMask;
187}
188
189// Adapts the external decomposeBitTest for local use.
190static bool decomposeBitTest(Value *Cond, CmpInst::Predicate &Pred, Value *&X,
191 Value *&Y, Value *&Z) {
192 auto Res =
193 llvm::decomposeBitTest(Cond, /*LookThroughTrunc=*/true,
194 /*AllowNonZeroC=*/true, /*DecomposeAnd=*/true);
195 if (!Res)
196 return false;
197
198 Pred = Res->Pred;
199 X = Res->X;
200 Y = ConstantInt::get(Ty: X->getType(), V: Res->Mask);
201 Z = ConstantInt::get(Ty: X->getType(), V: Res->C);
202 return true;
203}
204
205/// Handle (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E).
206/// Return the pattern classes (from MaskedICmpType) for the left hand side and
207/// the right hand side as a pair.
208/// LHS and RHS are the left hand side and the right hand side ICmps and PredL
209/// and PredR are their predicates, respectively.
210static std::optional<std::pair<unsigned, unsigned>>
211getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C, Value *&D, Value *&E,
212 Value *LHS, Value *RHS, ICmpInst::Predicate &PredL,
213 ICmpInst::Predicate &PredR) {
214
215 // Here comes the tricky part:
216 // LHS might be of the form L11 & L12 == X, X == L21 & L22,
217 // and L11 & L12 == L21 & L22. The same goes for RHS.
218 // Now we must find those components L** and R**, that are equal, so
219 // that we can extract the parameters A, B, C, D, and E for the canonical
220 // above.
221
222 // Check whether the icmp can be decomposed into a bit test.
223 Value *L1, *L11, *L12, *L2, *L21, *L22;
224 if (decomposeBitTest(Cond: LHS, Pred&: PredL, X&: L11, Y&: L12, Z&: L2)) {
225 L21 = L22 = L1 = nullptr;
226 } else {
227 auto *LHSCMP = dyn_cast<ICmpInst>(Val: LHS);
228 if (!LHSCMP)
229 return std::nullopt;
230
231 // Don't allow pointers. Splat vectors are fine.
232 if (!LHSCMP->getOperand(i_nocapture: 0)->getType()->isIntOrIntVectorTy())
233 return std::nullopt;
234
235 PredL = LHSCMP->getPredicate();
236 L1 = LHSCMP->getOperand(i_nocapture: 0);
237 L2 = LHSCMP->getOperand(i_nocapture: 1);
238 // Look for ANDs in the LHS icmp.
239 if (!match(V: L1, P: m_And(L: m_Value(V&: L11), R: m_Value(V&: L12)))) {
240 // Any icmp can be viewed as being trivially masked; if it allows us to
241 // remove one, it's worth it.
242 L11 = L1;
243 L12 = Constant::getAllOnesValue(Ty: L1->getType());
244 }
245
246 if (!match(V: L2, P: m_And(L: m_Value(V&: L21), R: m_Value(V&: L22)))) {
247 L21 = L2;
248 L22 = Constant::getAllOnesValue(Ty: L2->getType());
249 }
250 }
251
252 // Bail if LHS was a icmp that can't be decomposed into an equality.
253 if (!ICmpInst::isEquality(P: PredL))
254 return std::nullopt;
255
256 Value *R11, *R12, *R2;
257 if (decomposeBitTest(Cond: RHS, Pred&: PredR, X&: R11, Y&: R12, Z&: R2)) {
258 if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
259 A = R11;
260 D = R12;
261 } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
262 A = R12;
263 D = R11;
264 } else {
265 return std::nullopt;
266 }
267 E = R2;
268 } else {
269 auto *RHSCMP = dyn_cast<ICmpInst>(Val: RHS);
270 if (!RHSCMP)
271 return std::nullopt;
272 // Don't allow pointers. Splat vectors are fine.
273 if (!RHSCMP->getOperand(i_nocapture: 0)->getType()->isIntOrIntVectorTy())
274 return std::nullopt;
275
276 PredR = RHSCMP->getPredicate();
277
278 Value *R1 = RHSCMP->getOperand(i_nocapture: 0);
279 R2 = RHSCMP->getOperand(i_nocapture: 1);
280 bool Ok = false;
281 if (!match(V: R1, P: m_And(L: m_Value(V&: R11), R: m_Value(V&: R12)))) {
282 // As before, model no mask as a trivial mask if it'll let us do an
283 // optimization.
284 R11 = R1;
285 R12 = Constant::getAllOnesValue(Ty: R1->getType());
286 }
287
288 if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
289 A = R11;
290 D = R12;
291 E = R2;
292 Ok = true;
293 } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
294 A = R12;
295 D = R11;
296 E = R2;
297 Ok = true;
298 }
299
300 // Avoid matching against the -1 value we created for unmasked operand.
301 if (Ok && match(V: A, P: m_AllOnes()))
302 Ok = false;
303
304 // Look for ANDs on the right side of the RHS icmp.
305 if (!Ok) {
306 if (!match(V: R2, P: m_And(L: m_Value(V&: R11), R: m_Value(V&: R12)))) {
307 R11 = R2;
308 R12 = Constant::getAllOnesValue(Ty: R2->getType());
309 }
310
311 if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
312 A = R11;
313 D = R12;
314 E = R1;
315 } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
316 A = R12;
317 D = R11;
318 E = R1;
319 } else {
320 return std::nullopt;
321 }
322 }
323 }
324
325 // Bail if RHS was a icmp that can't be decomposed into an equality.
326 if (!ICmpInst::isEquality(P: PredR))
327 return std::nullopt;
328
329 if (L11 == A) {
330 B = L12;
331 C = L2;
332 } else if (L12 == A) {
333 B = L11;
334 C = L2;
335 } else if (L21 == A) {
336 B = L22;
337 C = L1;
338 } else if (L22 == A) {
339 B = L21;
340 C = L1;
341 }
342
343 unsigned LeftType = getMaskedICmpType(A, B, C, Pred: PredL);
344 unsigned RightType = getMaskedICmpType(A, B: D, C: E, Pred: PredR);
345 return std::optional<std::pair<unsigned, unsigned>>(
346 std::make_pair(x&: LeftType, y&: RightType));
347}
348
349/// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E) into a single
350/// (icmp(A & X) ==/!= Y), where the left-hand side is of type Mask_NotAllZeros
351/// and the right hand side is of type BMask_Mixed. For example,
352/// (icmp (A & 12) != 0) & (icmp (A & 15) == 8) -> (icmp (A & 15) == 8).
353/// Also used for logical and/or, must be poison safe.
354static Value *foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
355 Value *LHS, Value *RHS, bool IsAnd, Value *A, Value *B, Value *D, Value *E,
356 ICmpInst::Predicate PredL, ICmpInst::Predicate PredR,
357 InstCombiner::BuilderTy &Builder) {
358 // We are given the canonical form:
359 // (icmp ne (A & B), 0) & (icmp eq (A & D), E).
360 // where D & E == E.
361 //
362 // If IsAnd is false, we get it in negated form:
363 // (icmp eq (A & B), 0) | (icmp ne (A & D), E) ->
364 // !((icmp ne (A & B), 0) & (icmp eq (A & D), E)).
365 //
366 // We currently handle the case of B, C, D, E are constant.
367 //
368 const APInt *BCst, *DCst, *OrigECst;
369 if (!match(V: B, P: m_APInt(Res&: BCst)) || !match(V: D, P: m_APInt(Res&: DCst)) ||
370 !match(V: E, P: m_APInt(Res&: OrigECst)))
371 return nullptr;
372
373 ICmpInst::Predicate NewCC = IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
374
375 // Update E to the canonical form when D is a power of two and RHS is
376 // canonicalized as,
377 // (icmp ne (A & D), 0) -> (icmp eq (A & D), D) or
378 // (icmp ne (A & D), D) -> (icmp eq (A & D), 0).
379 APInt ECst = *OrigECst;
380 if (PredR != NewCC)
381 ECst ^= *DCst;
382
383 // If B or D is zero, skip because if LHS or RHS can be trivially folded by
384 // other folding rules and this pattern won't apply any more.
385 if (*BCst == 0 || *DCst == 0)
386 return nullptr;
387
388 // If B and D don't intersect, ie. (B & D) == 0, try to fold isNaN idiom:
389 // (icmp ne (A & FractionBits), 0) & (icmp eq (A & ExpBits), ExpBits)
390 // -> isNaN(A)
391 // Otherwise, we cannot deduce anything from it.
392 if (!BCst->intersects(RHS: *DCst)) {
393 Value *Src;
394 if (*DCst == ECst && match(V: A, P: m_ElementWiseBitCast(Op: m_Value(V&: Src))) &&
395 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
396 Kind: Attribute::StrictFP)) {
397 Type *Ty = Src->getType()->getScalarType();
398 if (!Ty->isIEEELikeFPTy())
399 return nullptr;
400
401 APInt ExpBits = APFloat::getInf(Sem: Ty->getFltSemantics()).bitcastToAPInt();
402 if (ECst != ExpBits)
403 return nullptr;
404 APInt FractionBits = ~ExpBits;
405 FractionBits.clearSignBit();
406 if (*BCst != FractionBits)
407 return nullptr;
408
409 return Builder.CreateFCmp(P: IsAnd ? FCmpInst::FCMP_UNO : FCmpInst::FCMP_ORD,
410 LHS: Src, RHS: ConstantFP::getZero(Ty: Src->getType()));
411 }
412 return nullptr;
413 }
414
415 // If the following two conditions are met:
416 //
417 // 1. mask B covers only a single bit that's not covered by mask D, that is,
418 // (B & (B ^ D)) is a power of 2 (in other words, B minus the intersection of
419 // B and D has only one bit set) and,
420 //
421 // 2. RHS (and E) indicates that the rest of B's bits are zero (in other
422 // words, the intersection of B and D is zero), that is, ((B & D) & E) == 0
423 //
424 // then that single bit in B must be one and thus the whole expression can be
425 // folded to
426 // (A & (B | D)) == (B & (B ^ D)) | E.
427 //
428 // For example,
429 // (icmp ne (A & 12), 0) & (icmp eq (A & 7), 1) -> (icmp eq (A & 15), 9)
430 // (icmp ne (A & 15), 0) & (icmp eq (A & 7), 0) -> (icmp eq (A & 15), 8)
431 if ((((*BCst & *DCst) & ECst) == 0) &&
432 (*BCst & (*BCst ^ *DCst)).isPowerOf2()) {
433 APInt BorD = *BCst | *DCst;
434 APInt BandBxorDorE = (*BCst & (*BCst ^ *DCst)) | ECst;
435 Value *NewMask = ConstantInt::get(Ty: A->getType(), V: BorD);
436 Value *NewMaskedValue = ConstantInt::get(Ty: A->getType(), V: BandBxorDorE);
437 Value *NewAnd = Builder.CreateAnd(LHS: A, RHS: NewMask);
438 return Builder.CreateICmp(P: NewCC, LHS: NewAnd, RHS: NewMaskedValue);
439 }
440
441 auto IsSubSetOrEqual = [](const APInt *C1, const APInt *C2) {
442 return (*C1 & *C2) == *C1;
443 };
444 auto IsSuperSetOrEqual = [](const APInt *C1, const APInt *C2) {
445 return (*C1 & *C2) == *C2;
446 };
447
448 // In the following, we consider only the cases where B is a superset of D, B
449 // is a subset of D, or B == D because otherwise there's at least one bit
450 // covered by B but not D, in which case we can't deduce much from it, so
451 // no folding (aside from the single must-be-one bit case right above.)
452 // For example,
453 // (icmp ne (A & 14), 0) & (icmp eq (A & 3), 1) -> no folding.
454 if (!IsSubSetOrEqual(BCst, DCst) && !IsSuperSetOrEqual(BCst, DCst))
455 return nullptr;
456
457 // At this point, either B is a superset of D, B is a subset of D or B == D.
458
459 // If E is zero, if B is a subset of (or equal to) D, LHS and RHS contradict
460 // and the whole expression becomes false (or true if negated), otherwise, no
461 // folding.
462 // For example,
463 // (icmp ne (A & 3), 0) & (icmp eq (A & 7), 0) -> false.
464 // (icmp ne (A & 15), 0) & (icmp eq (A & 3), 0) -> no folding.
465 if (ECst.isZero()) {
466 if (IsSubSetOrEqual(BCst, DCst))
467 return ConstantInt::get(Ty: LHS->getType(), V: !IsAnd);
468 return nullptr;
469 }
470
471 // At this point, B, D, E aren't zero and (B & D) == B, (B & D) == D or B ==
472 // D. If B is a superset of (or equal to) D, since E is not zero, LHS is
473 // subsumed by RHS (RHS implies LHS.) So the whole expression becomes
474 // RHS. For example,
475 // (icmp ne (A & 255), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
476 // (icmp ne (A & 15), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
477 if (IsSuperSetOrEqual(BCst, DCst)) {
478 // We can't guarantee that samesign hold after this fold.
479 if (auto *ICmp = dyn_cast<ICmpInst>(Val: RHS))
480 ICmp->setSameSign(false);
481 return RHS;
482 }
483 // Otherwise, B is a subset of D. If B and E have a common bit set,
484 // ie. (B & E) != 0, then LHS is subsumed by RHS. For example.
485 // (icmp ne (A & 12), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
486 assert(IsSubSetOrEqual(BCst, DCst) && "Precondition due to above code");
487 if ((*BCst & ECst) != 0) {
488 // We can't guarantee that samesign hold after this fold.
489 if (auto *ICmp = dyn_cast<ICmpInst>(Val: RHS))
490 ICmp->setSameSign(false);
491 return RHS;
492 }
493 // Otherwise, LHS and RHS contradict and the whole expression becomes false
494 // (or true if negated.) For example,
495 // (icmp ne (A & 7), 0) & (icmp eq (A & 15), 8) -> false.
496 // (icmp ne (A & 6), 0) & (icmp eq (A & 15), 8) -> false.
497 return ConstantInt::get(Ty: LHS->getType(), V: !IsAnd);
498}
499
500/// Try to fold (icmp(A & B) ==/!= 0) &/| (icmp(A & D) ==/!= E) into a single
501/// (icmp(A & X) ==/!= Y), where the left-hand side and the right hand side
502/// aren't of the common mask pattern type.
503/// Also used for logical and/or, must be poison safe.
504static Value *foldLogOpOfMaskedICmpsAsymmetric(
505 Value *LHS, Value *RHS, bool IsAnd, Value *A, Value *B, Value *C, Value *D,
506 Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR,
507 unsigned LHSMask, unsigned RHSMask, InstCombiner::BuilderTy &Builder) {
508 assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
509 "Expected equality predicates for masked type of icmps.");
510 // Handle Mask_NotAllZeros-BMask_Mixed cases.
511 // (icmp ne/eq (A & B), C) &/| (icmp eq/ne (A & D), E), or
512 // (icmp eq/ne (A & B), C) &/| (icmp ne/eq (A & D), E)
513 // which gets swapped to
514 // (icmp ne/eq (A & D), E) &/| (icmp eq/ne (A & B), C).
515 if (!IsAnd) {
516 LHSMask = conjugateICmpMask(Mask: LHSMask);
517 RHSMask = conjugateICmpMask(Mask: RHSMask);
518 }
519 if ((LHSMask & Mask_NotAllZeros) && (RHSMask & BMask_Mixed)) {
520 if (Value *V = foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
521 LHS, RHS, IsAnd, A, B, D, E, PredL, PredR, Builder)) {
522 return V;
523 }
524 } else if ((LHSMask & BMask_Mixed) && (RHSMask & Mask_NotAllZeros)) {
525 if (Value *V = foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
526 LHS: RHS, RHS: LHS, IsAnd, A, B: D, D: B, E: C, PredL: PredR, PredR: PredL, Builder)) {
527 return V;
528 }
529 }
530 return nullptr;
531}
532
533/// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E)
534/// into a single (icmp(A & X) ==/!= Y).
535static Value *foldLogOpOfMaskedICmps(Value *LHS, Value *RHS, bool IsAnd,
536 bool IsLogical,
537 InstCombiner::BuilderTy &Builder,
538 const SimplifyQuery &Q) {
539 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr, *E = nullptr;
540 ICmpInst::Predicate PredL, PredR;
541 std::optional<std::pair<unsigned, unsigned>> MaskPair =
542 getMaskedTypeForICmpPair(A, B, C, D, E, LHS, RHS, PredL, PredR);
543 if (!MaskPair)
544 return nullptr;
545 assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
546 "Expected equality predicates for masked type of icmps.");
547 unsigned LHSMask = MaskPair->first;
548 unsigned RHSMask = MaskPair->second;
549 unsigned Mask = LHSMask & RHSMask;
550 if (Mask == 0) {
551 // Even if the two sides don't share a common pattern, check if folding can
552 // still happen.
553 if (Value *V = foldLogOpOfMaskedICmpsAsymmetric(
554 LHS, RHS, IsAnd, A, B, C, D, E, PredL, PredR, LHSMask, RHSMask,
555 Builder))
556 return V;
557 return nullptr;
558 }
559
560 // In full generality:
561 // (icmp (A & B) Op C) | (icmp (A & D) Op E)
562 // == ![ (icmp (A & B) !Op C) & (icmp (A & D) !Op E) ]
563 //
564 // If the latter can be converted into (icmp (A & X) Op Y) then the former is
565 // equivalent to (icmp (A & X) !Op Y).
566 //
567 // Therefore, we can pretend for the rest of this function that we're dealing
568 // with the conjunction, provided we flip the sense of any comparisons (both
569 // input and output).
570
571 // In most cases we're going to produce an EQ for the "&&" case.
572 ICmpInst::Predicate NewCC = IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
573 if (!IsAnd) {
574 // Convert the masking analysis into its equivalent with negated
575 // comparisons.
576 Mask = conjugateICmpMask(Mask);
577 }
578
579 if (Mask & Mask_AllZeros) {
580 // (icmp eq (A & B), 0) & (icmp eq (A & D), 0)
581 // -> (icmp eq (A & (B|D)), 0)
582 if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(V: D))
583 return nullptr; // TODO: Use freeze?
584 Value *NewOr = Builder.CreateOr(LHS: B, RHS: D);
585 Value *NewAnd = Builder.CreateAnd(LHS: A, RHS: NewOr);
586 // We can't use C as zero because we might actually handle
587 // (icmp ne (A & B), B) & (icmp ne (A & D), D)
588 // with B and D, having a single bit set.
589 Value *Zero = Constant::getNullValue(Ty: A->getType());
590 return Builder.CreateICmp(P: NewCC, LHS: NewAnd, RHS: Zero);
591 }
592 if (Mask & BMask_AllOnes) {
593 // (icmp eq (A & B), B) & (icmp eq (A & D), D)
594 // -> (icmp eq (A & (B|D)), (B|D))
595 if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(V: D))
596 return nullptr; // TODO: Use freeze?
597 Value *NewOr = Builder.CreateOr(LHS: B, RHS: D);
598 Value *NewAnd = Builder.CreateAnd(LHS: A, RHS: NewOr);
599 return Builder.CreateICmp(P: NewCC, LHS: NewAnd, RHS: NewOr);
600 }
601 if (Mask & AMask_AllOnes) {
602 // (icmp eq (A & B), A) & (icmp eq (A & D), A)
603 // -> (icmp eq (A & (B&D)), A)
604 if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(V: D))
605 return nullptr; // TODO: Use freeze?
606 Value *NewAnd1 = Builder.CreateAnd(LHS: B, RHS: D);
607 Value *NewAnd2 = Builder.CreateAnd(LHS: A, RHS: NewAnd1);
608 return Builder.CreateICmp(P: NewCC, LHS: NewAnd2, RHS: A);
609 }
610
611 const APInt *ConstB, *ConstD;
612 if (match(V: B, P: m_APInt(Res&: ConstB)) && match(V: D, P: m_APInt(Res&: ConstD))) {
613 if (Mask & (Mask_NotAllZeros | BMask_NotAllOnes)) {
614 // (icmp ne (A & B), 0) & (icmp ne (A & D), 0) and
615 // (icmp ne (A & B), B) & (icmp ne (A & D), D)
616 // -> (icmp ne (A & B), 0) or (icmp ne (A & D), 0)
617 // Only valid if one of the masks is a superset of the other (check "B&D"
618 // is the same as either B or D).
619 APInt NewMask = *ConstB & *ConstD;
620 if (NewMask == *ConstB)
621 return LHS;
622 if (NewMask == *ConstD) {
623 if (IsLogical) {
624 if (auto *RHSI = dyn_cast<Instruction>(Val: RHS))
625 RHSI->dropPoisonGeneratingFlags();
626 }
627 return RHS;
628 }
629 }
630
631 if (Mask & AMask_NotAllOnes) {
632 // (icmp ne (A & B), B) & (icmp ne (A & D), D)
633 // -> (icmp ne (A & B), A) or (icmp ne (A & D), A)
634 // Only valid if one of the masks is a superset of the other (check "B|D"
635 // is the same as either B or D).
636 APInt NewMask = *ConstB | *ConstD;
637 if (NewMask == *ConstB)
638 return LHS;
639 if (NewMask == *ConstD)
640 return RHS;
641 }
642
643 if (Mask & (BMask_Mixed | BMask_NotMixed)) {
644 // Mixed:
645 // (icmp eq (A & B), C) & (icmp eq (A & D), E)
646 // We already know that B & C == C && D & E == E.
647 // If we can prove that (B & D) & (C ^ E) == 0, that is, the bits of
648 // C and E, which are shared by both the mask B and the mask D, don't
649 // contradict, then we can transform to
650 // -> (icmp eq (A & (B|D)), (C|E))
651 // Currently, we only handle the case of B, C, D, and E being constant.
652 // We can't simply use C and E because we might actually handle
653 // (icmp ne (A & B), B) & (icmp eq (A & D), D)
654 // with B and D, having a single bit set.
655
656 // NotMixed:
657 // (icmp ne (A & B), C) & (icmp ne (A & D), E)
658 // -> (icmp ne (A & (B & D)), (C & E))
659 // Check the intersection (B & D) for inequality.
660 // Assume that (B & D) == B || (B & D) == D, i.e B/D is a subset of D/B
661 // and (B & D) & (C ^ E) == 0, bits of C and E, which are shared by both
662 // the B and the D, don't contradict. Note that we can assume (~B & C) ==
663 // 0 && (~D & E) == 0, previous operation should delete these icmps if it
664 // hadn't been met.
665
666 const APInt *OldConstC, *OldConstE;
667 if (!match(V: C, P: m_APInt(Res&: OldConstC)) || !match(V: E, P: m_APInt(Res&: OldConstE)))
668 return nullptr;
669
670 auto FoldBMixed = [&](ICmpInst::Predicate CC, bool IsNot) -> Value * {
671 CC = IsNot ? CmpInst::getInversePredicate(pred: CC) : CC;
672 const APInt ConstC = PredL != CC ? *ConstB ^ *OldConstC : *OldConstC;
673 const APInt ConstE = PredR != CC ? *ConstD ^ *OldConstE : *OldConstE;
674
675 if (((*ConstB & *ConstD) & (ConstC ^ ConstE)).getBoolValue())
676 return IsNot ? nullptr : ConstantInt::get(Ty: LHS->getType(), V: !IsAnd);
677
678 if (IsNot && !ConstB->isSubsetOf(RHS: *ConstD) &&
679 !ConstD->isSubsetOf(RHS: *ConstB))
680 return nullptr;
681
682 APInt BD, CE;
683 if (IsNot) {
684 BD = *ConstB & *ConstD;
685 CE = ConstC & ConstE;
686 } else {
687 BD = *ConstB | *ConstD;
688 CE = ConstC | ConstE;
689 }
690 Value *NewAnd = Builder.CreateAnd(LHS: A, RHS: BD);
691 Value *CEVal = ConstantInt::get(Ty: A->getType(), V: CE);
692 return Builder.CreateICmp(P: CC, LHS: NewAnd, RHS: CEVal);
693 };
694
695 if (Mask & BMask_Mixed)
696 return FoldBMixed(NewCC, false);
697 if (Mask & BMask_NotMixed) // can be else also
698 return FoldBMixed(NewCC, true);
699 }
700 }
701
702 // (icmp eq (A & B), 0) | (icmp eq (A & D), 0)
703 // -> (icmp ne (A & (B|D)), (B|D))
704 // (icmp ne (A & B), 0) & (icmp ne (A & D), 0)
705 // -> (icmp eq (A & (B|D)), (B|D))
706 // iff B and D is known to be a power of two
707 if (Mask & Mask_NotAllZeros &&
708 isKnownToBeAPowerOfTwo(V: B, /*OrZero=*/false, Q) &&
709 isKnownToBeAPowerOfTwo(V: D, /*OrZero=*/false, Q)) {
710 // If this is a logical and/or, then we must prevent propagation of a
711 // poison value from the RHS by inserting freeze.
712 if (IsLogical)
713 D = Builder.CreateFreeze(V: D);
714 Value *Mask = Builder.CreateOr(LHS: B, RHS: D);
715 Value *Masked = Builder.CreateAnd(LHS: A, RHS: Mask);
716 return Builder.CreateICmp(P: NewCC, LHS: Masked, RHS: Mask);
717 }
718 return nullptr;
719}
720
721/// Try to fold a signed range checked with lower bound 0 to an unsigned icmp.
722/// Example: (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n
723/// If \p Inverted is true then the check is for the inverted range, e.g.
724/// (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n
725Value *InstCombinerImpl::simplifyRangeCheck(ICmpInst *Cmp0, ICmpInst *Cmp1,
726 bool Inverted) {
727 // Check the lower range comparison, e.g. x >= 0
728 // InstCombine already ensured that if there is a constant it's on the RHS.
729 ConstantInt *RangeStart = dyn_cast<ConstantInt>(Val: Cmp0->getOperand(i_nocapture: 1));
730 if (!RangeStart)
731 return nullptr;
732
733 ICmpInst::Predicate Pred0 = (Inverted ? Cmp0->getInversePredicate() :
734 Cmp0->getPredicate());
735
736 // Accept x > -1 or x >= 0 (after potentially inverting the predicate).
737 if (!((Pred0 == ICmpInst::ICMP_SGT && RangeStart->isMinusOne()) ||
738 (Pred0 == ICmpInst::ICMP_SGE && RangeStart->isZero())))
739 return nullptr;
740
741 ICmpInst::Predicate Pred1 = (Inverted ? Cmp1->getInversePredicate() :
742 Cmp1->getPredicate());
743
744 Value *Input = Cmp0->getOperand(i_nocapture: 0);
745 Value *Cmp1Op0 = Cmp1->getOperand(i_nocapture: 0);
746 Value *Cmp1Op1 = Cmp1->getOperand(i_nocapture: 1);
747 Value *RangeEnd;
748 if (match(V: Cmp1Op0, P: m_SExtOrSelf(Op: m_Specific(V: Input)))) {
749 // For the upper range compare we have: icmp x, n
750 Input = Cmp1Op0;
751 RangeEnd = Cmp1Op1;
752 } else if (match(V: Cmp1Op1, P: m_SExtOrSelf(Op: m_Specific(V: Input)))) {
753 // For the upper range compare we have: icmp n, x
754 Input = Cmp1Op1;
755 RangeEnd = Cmp1Op0;
756 Pred1 = ICmpInst::getSwappedPredicate(pred: Pred1);
757 } else {
758 return nullptr;
759 }
760
761 // Check the upper range comparison, e.g. x < n
762 ICmpInst::Predicate NewPred;
763 switch (Pred1) {
764 case ICmpInst::ICMP_SLT: NewPred = ICmpInst::ICMP_ULT; break;
765 case ICmpInst::ICMP_SLE: NewPred = ICmpInst::ICMP_ULE; break;
766 default: return nullptr;
767 }
768
769 // This simplification is only valid if the upper range is not negative.
770 KnownBits Known = computeKnownBits(V: RangeEnd, CxtI: Cmp1);
771 if (!Known.isNonNegative())
772 return nullptr;
773
774 if (Inverted)
775 NewPred = ICmpInst::getInversePredicate(pred: NewPred);
776
777 return Builder.CreateICmp(P: NewPred, LHS: Input, RHS: RangeEnd);
778}
779
780// (or (icmp eq X, 0), (icmp eq X, Pow2OrZero))
781// -> (icmp eq (and X, Pow2OrZero), X)
782// (and (icmp ne X, 0), (icmp ne X, Pow2OrZero))
783// -> (icmp ne (and X, Pow2OrZero), X)
784static Value *
785foldAndOrOfICmpsWithPow2AndWithZero(InstCombiner::BuilderTy &Builder,
786 ICmpInst *LHS, ICmpInst *RHS, bool IsAnd,
787 const SimplifyQuery &Q) {
788 CmpPredicate Pred = IsAnd ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
789 // Make sure we have right compares for our op.
790 if (LHS->getPredicate() != Pred || RHS->getPredicate() != Pred)
791 return nullptr;
792
793 // Make it so we can match LHS against the (icmp eq/ne X, 0) just for
794 // simplicity.
795 if (match(V: RHS->getOperand(i_nocapture: 1), P: m_Zero()))
796 std::swap(a&: LHS, b&: RHS);
797
798 Value *Pow2, *Op;
799 // Match the desired pattern:
800 // LHS: (icmp eq/ne X, 0)
801 // RHS: (icmp eq/ne X, Pow2OrZero)
802 // Skip if Pow2OrZero is 1. Either way it gets folded to (icmp ugt X, 1) but
803 // this form ends up slightly less canonical.
804 // We could potentially be more sophisticated than requiring LHS/RHS
805 // be one-use. We don't create additional instructions if only one
806 // of them is one-use. So cases where one is one-use and the other
807 // is two-use might be profitable.
808 if (!match(V: LHS, P: m_OneUse(SubPattern: m_ICmp(Pred, L: m_Value(V&: Op), R: m_Zero()))) ||
809 !match(V: RHS, P: m_OneUse(SubPattern: m_c_ICmp(Pred, L: m_Specific(V: Op), R: m_Value(V&: Pow2)))) ||
810 match(V: Pow2, P: m_One()) ||
811 !isKnownToBeAPowerOfTwo(V: Pow2, DL: Q.DL, /*OrZero=*/true, AC: Q.AC, CxtI: Q.CxtI, DT: Q.DT))
812 return nullptr;
813
814 Value *And = Builder.CreateAnd(LHS: Op, RHS: Pow2);
815 return Builder.CreateICmp(P: Pred, LHS: And, RHS: Op);
816}
817
818/// General pattern:
819/// X & Y
820///
821/// Where Y is checking that all the high bits (covered by a mask 4294967168)
822/// are uniform, i.e. %arg & 4294967168 can be either 4294967168 or 0
823/// Pattern can be one of:
824/// %t = add i32 %arg, 128
825/// %r = icmp ult i32 %t, 256
826/// Or
827/// %t0 = shl i32 %arg, 24
828/// %t1 = ashr i32 %t0, 24
829/// %r = icmp eq i32 %t1, %arg
830/// Or
831/// %t0 = trunc i32 %arg to i8
832/// %t1 = sext i8 %t0 to i32
833/// %r = icmp eq i32 %t1, %arg
834/// This pattern is a signed truncation check.
835///
836/// And X is checking that some bit in that same mask is zero.
837/// I.e. can be one of:
838/// %r = icmp sgt i32 %arg, -1
839/// Or
840/// %t = and i32 %arg, 2147483648
841/// %r = icmp eq i32 %t, 0
842///
843/// Since we are checking that all the bits in that mask are the same,
844/// and a particular bit is zero, what we are really checking is that all the
845/// masked bits are zero.
846/// So this should be transformed to:
847/// %r = icmp ult i32 %arg, 128
848static Value *foldSignedTruncationCheck(ICmpInst *ICmp0, ICmpInst *ICmp1,
849 Instruction &CxtI,
850 InstCombiner::BuilderTy &Builder) {
851 assert(CxtI.getOpcode() == Instruction::And);
852
853 // Match icmp ult (add %arg, C01), C1 (C1 == C01 << 1; powers of two)
854 auto tryToMatchSignedTruncationCheck = [](ICmpInst *ICmp, Value *&X,
855 APInt &SignBitMask) -> bool {
856 const APInt *I01, *I1; // powers of two; I1 == I01 << 1
857 if (!(match(V: ICmp, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_ULT,
858 L: m_Add(L: m_Value(V&: X), R: m_Power2(V&: I01)),
859 R: m_Power2(V&: I1))) &&
860 I1->ugt(RHS: *I01) && I01->shl(shiftAmt: 1) == *I1))
861 return false;
862 // Which bit is the new sign bit as per the 'signed truncation' pattern?
863 SignBitMask = *I01;
864 return true;
865 };
866
867 // One icmp needs to be 'signed truncation check'.
868 // We need to match this first, else we will mismatch commutative cases.
869 Value *X1;
870 APInt HighestBit;
871 ICmpInst *OtherICmp;
872 if (tryToMatchSignedTruncationCheck(ICmp1, X1, HighestBit))
873 OtherICmp = ICmp0;
874 else if (tryToMatchSignedTruncationCheck(ICmp0, X1, HighestBit))
875 OtherICmp = ICmp1;
876 else
877 return nullptr;
878
879 assert(HighestBit.isPowerOf2() && "expected to be power of two (non-zero)");
880
881 // Try to match/decompose into: icmp eq (X & Mask), 0
882 auto tryToDecompose = [](ICmpInst *ICmp, Value *&X,
883 APInt &UnsetBitsMask) -> bool {
884 CmpPredicate Pred = ICmp->getPredicate();
885 // Can it be decomposed into icmp eq (X & Mask), 0 ?
886 auto Res = llvm::decomposeBitTestICmp(
887 LHS: ICmp->getOperand(i_nocapture: 0), RHS: ICmp->getOperand(i_nocapture: 1), Pred,
888 /*LookThroughTrunc=*/false, /*AllowNonZeroC=*/false,
889 /*DecomposeAnd=*/true);
890 if (Res && Res->Pred == ICmpInst::ICMP_EQ) {
891 X = Res->X;
892 UnsetBitsMask = Res->Mask;
893 return true;
894 }
895
896 return false;
897 };
898
899 // And the other icmp needs to be decomposable into a bit test.
900 Value *X0;
901 APInt UnsetBitsMask;
902 if (!tryToDecompose(OtherICmp, X0, UnsetBitsMask))
903 return nullptr;
904
905 assert(!UnsetBitsMask.isZero() && "empty mask makes no sense.");
906
907 // Are they working on the same value?
908 Value *X;
909 if (X1 == X0) {
910 // Ok as is.
911 X = X1;
912 } else if (match(V: X0, P: m_Trunc(Op: m_Specific(V: X1)))) {
913 UnsetBitsMask = UnsetBitsMask.zext(width: X1->getType()->getScalarSizeInBits());
914 X = X1;
915 } else
916 return nullptr;
917
918 // So which bits should be uniform as per the 'signed truncation check'?
919 // (all the bits starting with (i.e. including) HighestBit)
920 APInt SignBitsMask = ~(HighestBit - 1U);
921
922 // UnsetBitsMask must have some common bits with SignBitsMask,
923 if (!UnsetBitsMask.intersects(RHS: SignBitsMask))
924 return nullptr;
925
926 // Does UnsetBitsMask contain any bits outside of SignBitsMask?
927 if (!UnsetBitsMask.isSubsetOf(RHS: SignBitsMask)) {
928 APInt OtherHighestBit = (~UnsetBitsMask) + 1U;
929 if (!OtherHighestBit.isPowerOf2())
930 return nullptr;
931 HighestBit = APIntOps::umin(A: HighestBit, B: OtherHighestBit);
932 }
933 // Else, if it does not, then all is ok as-is.
934
935 // %r = icmp ult %X, SignBit
936 return Builder.CreateICmpULT(LHS: X, RHS: ConstantInt::get(Ty: X->getType(), V: HighestBit),
937 Name: CxtI.getName() + ".simplified");
938}
939
940/// Fold (icmp eq ctpop(X) 1) | (icmp eq X 0) into (icmp ult ctpop(X) 2) and
941/// fold (icmp ne ctpop(X) 1) & (icmp ne X 0) into (icmp ugt ctpop(X) 1).
942/// Also used for logical and/or, must be poison safe if range attributes are
943/// dropped.
944static Value *foldIsPowerOf2OrZero(ICmpInst *Cmp0, ICmpInst *Cmp1, bool IsAnd,
945 InstCombiner::BuilderTy &Builder,
946 InstCombinerImpl &IC) {
947 CmpPredicate Pred0, Pred1;
948 Value *X;
949 if (!match(V: Cmp0, P: m_ICmp(Pred&: Pred0, L: m_Intrinsic<Intrinsic::ctpop>(Op0: m_Value(V&: X)),
950 R: m_SpecificInt(V: 1))) ||
951 !match(V: Cmp1, P: m_ICmp(Pred&: Pred1, L: m_Specific(V: X), R: m_ZeroInt())))
952 return nullptr;
953
954 auto *CtPop = cast<Instruction>(Val: Cmp0->getOperand(i_nocapture: 0));
955 if (IsAnd && Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_NE) {
956 // Drop range attributes and re-infer them in the next iteration.
957 CtPop->dropPoisonGeneratingAnnotations();
958 IC.addToWorklist(I: CtPop);
959 return Builder.CreateICmpUGT(LHS: CtPop, RHS: ConstantInt::get(Ty: CtPop->getType(), V: 1));
960 }
961 if (!IsAnd && Pred0 == ICmpInst::ICMP_EQ && Pred1 == ICmpInst::ICMP_EQ) {
962 // Drop range attributes and re-infer them in the next iteration.
963 CtPop->dropPoisonGeneratingAnnotations();
964 IC.addToWorklist(I: CtPop);
965 return Builder.CreateICmpULT(LHS: CtPop, RHS: ConstantInt::get(Ty: CtPop->getType(), V: 2));
966 }
967
968 return nullptr;
969}
970
971/// Reduce a pair of compares that check if a value has exactly 1 bit set.
972/// Also used for logical and/or, must be poison safe if range attributes are
973/// dropped.
974static Value *foldIsPowerOf2(ICmpInst *Cmp0, ICmpInst *Cmp1, bool JoinedByAnd,
975 InstCombiner::BuilderTy &Builder,
976 InstCombinerImpl &IC) {
977 // Handle 'and' / 'or' commutation: make the equality check the first operand.
978 if (JoinedByAnd && Cmp1->getPredicate() == ICmpInst::ICMP_NE)
979 std::swap(a&: Cmp0, b&: Cmp1);
980 else if (!JoinedByAnd && Cmp1->getPredicate() == ICmpInst::ICMP_EQ)
981 std::swap(a&: Cmp0, b&: Cmp1);
982
983 // (X != 0) && (ctpop(X) u< 2) --> ctpop(X) == 1
984 Value *X;
985 if (JoinedByAnd &&
986 match(V: Cmp0, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_NE, L: m_Value(V&: X), R: m_ZeroInt())) &&
987 match(V: Cmp1, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_ULT,
988 L: m_Intrinsic<Intrinsic::ctpop>(Op0: m_Specific(V: X)),
989 R: m_SpecificInt(V: 2)))) {
990 auto *CtPop = cast<Instruction>(Val: Cmp1->getOperand(i_nocapture: 0));
991 // Drop range attributes and re-infer them in the next iteration.
992 CtPop->dropPoisonGeneratingAnnotations();
993 IC.addToWorklist(I: CtPop);
994 return Builder.CreateICmpEQ(LHS: CtPop, RHS: ConstantInt::get(Ty: CtPop->getType(), V: 1));
995 }
996 // (X == 0) || (ctpop(X) u> 1) --> ctpop(X) != 1
997 if (!JoinedByAnd &&
998 match(V: Cmp0, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ, L: m_Value(V&: X), R: m_ZeroInt())) &&
999 match(V: Cmp1, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_UGT,
1000 L: m_Intrinsic<Intrinsic::ctpop>(Op0: m_Specific(V: X)),
1001 R: m_SpecificInt(V: 1)))) {
1002 auto *CtPop = cast<Instruction>(Val: Cmp1->getOperand(i_nocapture: 0));
1003 // Drop range attributes and re-infer them in the next iteration.
1004 CtPop->dropPoisonGeneratingAnnotations();
1005 IC.addToWorklist(I: CtPop);
1006 return Builder.CreateICmpNE(LHS: CtPop, RHS: ConstantInt::get(Ty: CtPop->getType(), V: 1));
1007 }
1008 return nullptr;
1009}
1010
1011/// Try to fold (icmp(A & B) == 0) & (icmp(A & D) != E) into (icmp A u< D) iff
1012/// B is a contiguous set of ones starting from the most significant bit
1013/// (negative power of 2), D and E are equal, and D is a contiguous set of ones
1014/// starting at the most significant zero bit in B. Parameter B supports masking
1015/// using undef/poison in either scalar or vector values.
1016static Value *foldNegativePower2AndShiftedMask(
1017 Value *A, Value *B, Value *D, Value *E, ICmpInst::Predicate PredL,
1018 ICmpInst::Predicate PredR, InstCombiner::BuilderTy &Builder) {
1019 assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
1020 "Expected equality predicates for masked type of icmps.");
1021 if (PredL != ICmpInst::ICMP_EQ || PredR != ICmpInst::ICMP_NE)
1022 return nullptr;
1023
1024 if (!match(V: B, P: m_NegatedPower2()) || !match(V: D, P: m_ShiftedMask()) ||
1025 !match(V: E, P: m_ShiftedMask()))
1026 return nullptr;
1027
1028 // Test scalar arguments for conversion. B has been validated earlier to be a
1029 // negative power of two and thus is guaranteed to have one or more contiguous
1030 // ones starting from the MSB followed by zero or more contiguous zeros. D has
1031 // been validated earlier to be a shifted set of one or more contiguous ones.
1032 // In order to match, B leading ones and D leading zeros should be equal. The
1033 // predicate that B be a negative power of 2 prevents the condition of there
1034 // ever being zero leading ones. Thus 0 == 0 cannot occur. The predicate that
1035 // D always be a shifted mask prevents the condition of D equaling 0. This
1036 // prevents matching the condition where B contains the maximum number of
1037 // leading one bits (-1) and D contains the maximum number of leading zero
1038 // bits (0).
1039 auto isReducible = [](const Value *B, const Value *D, const Value *E) {
1040 const APInt *BCst, *DCst, *ECst;
1041 return match(V: B, P: m_APIntAllowPoison(Res&: BCst)) && match(V: D, P: m_APInt(Res&: DCst)) &&
1042 match(V: E, P: m_APInt(Res&: ECst)) && *DCst == *ECst &&
1043 (isa<PoisonValue>(Val: B) ||
1044 (BCst->countLeadingOnes() == DCst->countLeadingZeros()));
1045 };
1046
1047 // Test vector type arguments for conversion.
1048 if (const auto *BVTy = dyn_cast<VectorType>(Val: B->getType())) {
1049 const auto *BFVTy = dyn_cast<FixedVectorType>(Val: BVTy);
1050 const auto *BConst = dyn_cast<Constant>(Val: B);
1051 const auto *DConst = dyn_cast<Constant>(Val: D);
1052 const auto *EConst = dyn_cast<Constant>(Val: E);
1053
1054 if (!BFVTy || !BConst || !DConst || !EConst)
1055 return nullptr;
1056
1057 for (unsigned I = 0; I != BFVTy->getNumElements(); ++I) {
1058 const auto *BElt = BConst->getAggregateElement(Elt: I);
1059 const auto *DElt = DConst->getAggregateElement(Elt: I);
1060 const auto *EElt = EConst->getAggregateElement(Elt: I);
1061
1062 if (!BElt || !DElt || !EElt)
1063 return nullptr;
1064 if (!isReducible(BElt, DElt, EElt))
1065 return nullptr;
1066 }
1067 } else {
1068 // Test scalar type arguments for conversion.
1069 if (!isReducible(B, D, E))
1070 return nullptr;
1071 }
1072 return Builder.CreateICmp(P: ICmpInst::ICMP_ULT, LHS: A, RHS: D);
1073}
1074
1075/// Try to fold ((icmp X u< P) & (icmp(X & M) != M)) or ((icmp X s> -1) &
1076/// (icmp(X & M) != M)) into (icmp X u< M). Where P is a power of 2, M < P, and
1077/// M is a contiguous shifted mask starting at the right most significant zero
1078/// bit in P. SGT is supported as when P is the largest representable power of
1079/// 2, an earlier optimization converts the expression into (icmp X s> -1).
1080/// Parameter P supports masking using undef/poison in either scalar or vector
1081/// values.
1082static Value *foldPowerOf2AndShiftedMask(ICmpInst *Cmp0, ICmpInst *Cmp1,
1083 bool JoinedByAnd,
1084 InstCombiner::BuilderTy &Builder) {
1085 if (!JoinedByAnd)
1086 return nullptr;
1087 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr, *E = nullptr;
1088 ICmpInst::Predicate CmpPred0, CmpPred1;
1089 // Assuming P is a 2^n, getMaskedTypeForICmpPair will normalize (icmp X u<
1090 // 2^n) into (icmp (X & ~(2^n-1)) == 0) and (icmp X s> -1) into (icmp (X &
1091 // SignMask) == 0).
1092 std::optional<std::pair<unsigned, unsigned>> MaskPair =
1093 getMaskedTypeForICmpPair(A, B, C, D, E, LHS: Cmp0, RHS: Cmp1, PredL&: CmpPred0, PredR&: CmpPred1);
1094 if (!MaskPair)
1095 return nullptr;
1096
1097 const auto compareBMask = BMask_NotMixed | BMask_NotAllOnes;
1098 unsigned CmpMask0 = MaskPair->first;
1099 unsigned CmpMask1 = MaskPair->second;
1100 if ((CmpMask0 & Mask_AllZeros) && (CmpMask1 == compareBMask)) {
1101 if (Value *V = foldNegativePower2AndShiftedMask(A, B, D, E, PredL: CmpPred0,
1102 PredR: CmpPred1, Builder))
1103 return V;
1104 } else if ((CmpMask0 == compareBMask) && (CmpMask1 & Mask_AllZeros)) {
1105 if (Value *V = foldNegativePower2AndShiftedMask(A, B: D, D: B, E: C, PredL: CmpPred1,
1106 PredR: CmpPred0, Builder))
1107 return V;
1108 }
1109 return nullptr;
1110}
1111
1112/// Commuted variants are assumed to be handled by calling this function again
1113/// with the parameters swapped.
1114static Value *foldUnsignedUnderflowCheck(ICmpInst *ZeroICmp,
1115 ICmpInst *UnsignedICmp, bool IsAnd,
1116 const SimplifyQuery &Q,
1117 InstCombiner::BuilderTy &Builder) {
1118 Value *ZeroCmpOp;
1119 CmpPredicate EqPred;
1120 if (!match(V: ZeroICmp, P: m_ICmp(Pred&: EqPred, L: m_Value(V&: ZeroCmpOp), R: m_Zero())) ||
1121 !ICmpInst::isEquality(P: EqPred))
1122 return nullptr;
1123
1124 CmpPredicate UnsignedPred;
1125
1126 Value *A, *B;
1127 if (match(V: UnsignedICmp,
1128 P: m_c_ICmp(Pred&: UnsignedPred, L: m_Specific(V: ZeroCmpOp), R: m_Value(V&: A))) &&
1129 match(V: ZeroCmpOp, P: m_c_Add(L: m_Specific(V: A), R: m_Value(V&: B))) &&
1130 (ZeroICmp->hasOneUse() || UnsignedICmp->hasOneUse())) {
1131 auto GetKnownNonZeroAndOther = [&](Value *&NonZero, Value *&Other) {
1132 if (!isKnownNonZero(V: NonZero, Q))
1133 std::swap(a&: NonZero, b&: Other);
1134 return isKnownNonZero(V: NonZero, Q);
1135 };
1136
1137 // Given ZeroCmpOp = (A + B)
1138 // ZeroCmpOp < A && ZeroCmpOp != 0 --> (0-X) < Y iff
1139 // ZeroCmpOp >= A || ZeroCmpOp == 0 --> (0-X) >= Y iff
1140 // with X being the value (A/B) that is known to be non-zero,
1141 // and Y being remaining value.
1142 if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_NE &&
1143 IsAnd && GetKnownNonZeroAndOther(B, A))
1144 return Builder.CreateICmpULT(LHS: Builder.CreateNeg(V: B), RHS: A);
1145 if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_EQ &&
1146 !IsAnd && GetKnownNonZeroAndOther(B, A))
1147 return Builder.CreateICmpUGE(LHS: Builder.CreateNeg(V: B), RHS: A);
1148 }
1149
1150 return nullptr;
1151}
1152
1153struct IntPart {
1154 Value *From;
1155 unsigned StartBit;
1156 unsigned NumBits;
1157};
1158
1159/// Match an extraction of bits from an integer.
1160static std::optional<IntPart> matchIntPart(Value *V) {
1161 Value *X;
1162 if (!match(V, P: m_OneUse(SubPattern: m_Trunc(Op: m_Value(V&: X)))))
1163 return std::nullopt;
1164
1165 unsigned NumOriginalBits = X->getType()->getScalarSizeInBits();
1166 unsigned NumExtractedBits = V->getType()->getScalarSizeInBits();
1167 Value *Y;
1168 const APInt *Shift;
1169 // For a trunc(lshr Y, Shift) pattern, make sure we're only extracting bits
1170 // from Y, not any shifted-in zeroes.
1171 if (match(V: X, P: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: Y), R: m_APInt(Res&: Shift)))) &&
1172 Shift->ule(RHS: NumOriginalBits - NumExtractedBits))
1173 return {{.From: Y, .StartBit: (unsigned)Shift->getZExtValue(), .NumBits: NumExtractedBits}};
1174 return {{.From: X, .StartBit: 0, .NumBits: NumExtractedBits}};
1175}
1176
1177/// Materialize an extraction of bits from an integer in IR.
1178static Value *extractIntPart(const IntPart &P, IRBuilderBase &Builder) {
1179 Value *V = P.From;
1180 if (P.StartBit)
1181 V = Builder.CreateLShr(LHS: V, RHS: P.StartBit);
1182 Type *TruncTy = V->getType()->getWithNewBitWidth(NewBitWidth: P.NumBits);
1183 if (TruncTy != V->getType())
1184 V = Builder.CreateTrunc(V, DestTy: TruncTy);
1185 return V;
1186}
1187
1188/// (icmp eq X0, Y0) & (icmp eq X1, Y1) -> icmp eq X01, Y01
1189/// (icmp ne X0, Y0) | (icmp ne X1, Y1) -> icmp ne X01, Y01
1190/// where X0, X1 and Y0, Y1 are adjacent parts extracted from an integer.
1191Value *InstCombinerImpl::foldEqOfParts(Value *Cmp0, Value *Cmp1, bool IsAnd) {
1192 if (!Cmp0->hasOneUse() || !Cmp1->hasOneUse())
1193 return nullptr;
1194
1195 CmpInst::Predicate Pred = IsAnd ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1196 auto GetMatchPart = [&](Value *CmpV,
1197 unsigned OpNo) -> std::optional<IntPart> {
1198 assert(CmpV->getType()->isIntOrIntVectorTy(1) && "Must be bool");
1199
1200 Value *X, *Y;
1201 // icmp ne (and x, 1), (and y, 1) <=> trunc (xor x, y) to i1
1202 // icmp eq (and x, 1), (and y, 1) <=> not (trunc (xor x, y) to i1)
1203 if (Pred == CmpInst::ICMP_NE
1204 ? match(V: CmpV, P: m_Trunc(Op: m_Xor(L: m_Value(V&: X), R: m_Value(V&: Y))))
1205 : match(V: CmpV, P: m_Not(V: m_Trunc(Op: m_Xor(L: m_Value(V&: X), R: m_Value(V&: Y))))))
1206 return {{.From: OpNo == 0 ? X : Y, .StartBit: 0, .NumBits: 1}};
1207
1208 auto *Cmp = dyn_cast<ICmpInst>(Val: CmpV);
1209 if (!Cmp)
1210 return std::nullopt;
1211
1212 if (Pred == Cmp->getPredicate())
1213 return matchIntPart(V: Cmp->getOperand(i_nocapture: OpNo));
1214
1215 const APInt *C;
1216 // (icmp eq (lshr x, C), (lshr y, C)) gets optimized to:
1217 // (icmp ult (xor x, y), 1 << C) so also look for that.
1218 if (Pred == CmpInst::ICMP_EQ && Cmp->getPredicate() == CmpInst::ICMP_ULT) {
1219 if (!match(V: Cmp->getOperand(i_nocapture: 1), P: m_Power2(V&: C)) ||
1220 !match(V: Cmp->getOperand(i_nocapture: 0), P: m_Xor(L: m_Value(), R: m_Value())))
1221 return std::nullopt;
1222 }
1223
1224 // (icmp ne (lshr x, C), (lshr y, C)) gets optimized to:
1225 // (icmp ugt (xor x, y), (1 << C) - 1) so also look for that.
1226 else if (Pred == CmpInst::ICMP_NE &&
1227 Cmp->getPredicate() == CmpInst::ICMP_UGT) {
1228 if (!match(V: Cmp->getOperand(i_nocapture: 1), P: m_LowBitMask(V&: C)) ||
1229 !match(V: Cmp->getOperand(i_nocapture: 0), P: m_Xor(L: m_Value(), R: m_Value())))
1230 return std::nullopt;
1231 } else {
1232 return std::nullopt;
1233 }
1234
1235 unsigned From = Pred == CmpInst::ICMP_NE ? C->popcount() : C->countr_zero();
1236 Instruction *I = cast<Instruction>(Val: Cmp->getOperand(i_nocapture: 0));
1237 return {{.From: I->getOperand(i: OpNo), .StartBit: From, .NumBits: C->getBitWidth() - From}};
1238 };
1239
1240 std::optional<IntPart> L0 = GetMatchPart(Cmp0, 0);
1241 std::optional<IntPart> R0 = GetMatchPart(Cmp0, 1);
1242 std::optional<IntPart> L1 = GetMatchPart(Cmp1, 0);
1243 std::optional<IntPart> R1 = GetMatchPart(Cmp1, 1);
1244 if (!L0 || !R0 || !L1 || !R1)
1245 return nullptr;
1246
1247 // Make sure the LHS/RHS compare a part of the same value, possibly after
1248 // an operand swap.
1249 if (L0->From != L1->From || R0->From != R1->From) {
1250 if (L0->From != R1->From || R0->From != L1->From)
1251 return nullptr;
1252 std::swap(lhs&: L1, rhs&: R1);
1253 }
1254
1255 // Make sure the extracted parts are adjacent, canonicalizing to L0/R0 being
1256 // the low part and L1/R1 being the high part.
1257 if (L0->StartBit + L0->NumBits != L1->StartBit ||
1258 R0->StartBit + R0->NumBits != R1->StartBit) {
1259 if (L1->StartBit + L1->NumBits != L0->StartBit ||
1260 R1->StartBit + R1->NumBits != R0->StartBit)
1261 return nullptr;
1262 std::swap(lhs&: L0, rhs&: L1);
1263 std::swap(lhs&: R0, rhs&: R1);
1264 }
1265
1266 // We can simplify to a comparison of these larger parts of the integers.
1267 IntPart L = {.From: L0->From, .StartBit: L0->StartBit, .NumBits: L0->NumBits + L1->NumBits};
1268 IntPart R = {.From: R0->From, .StartBit: R0->StartBit, .NumBits: R0->NumBits + R1->NumBits};
1269 Value *LValue = extractIntPart(P: L, Builder);
1270 Value *RValue = extractIntPart(P: R, Builder);
1271 return Builder.CreateICmp(P: Pred, LHS: LValue, RHS: RValue);
1272}
1273
1274/// Reduce logic-of-compares with equality to a constant by substituting a
1275/// common operand with the constant. Callers are expected to call this with
1276/// Cmp0/Cmp1 switched to handle logic op commutativity.
1277static Value *foldAndOrOfICmpsWithConstEq(ICmpInst *Cmp0, ICmpInst *Cmp1,
1278 bool IsAnd, bool IsLogical,
1279 InstCombiner::BuilderTy &Builder,
1280 const SimplifyQuery &Q,
1281 Instruction &I) {
1282 // Match an equality compare with a non-poison constant as Cmp0.
1283 // Also, give up if the compare can be constant-folded to avoid looping.
1284 CmpPredicate Pred0;
1285 Value *X;
1286 Constant *C;
1287 if (!match(V: Cmp0, P: m_ICmp(Pred&: Pred0, L: m_Value(V&: X), R: m_Constant(C))) ||
1288 !isGuaranteedNotToBeUndefOrPoison(V: C) || isa<Constant>(Val: X))
1289 return nullptr;
1290 if ((IsAnd && Pred0 != ICmpInst::ICMP_EQ) ||
1291 (!IsAnd && Pred0 != ICmpInst::ICMP_NE))
1292 return nullptr;
1293
1294 // The other compare must include a common operand (X). Canonicalize the
1295 // common operand as operand 1 (Pred1 is swapped if the common operand was
1296 // operand 0).
1297 Value *Y;
1298 CmpPredicate Pred1;
1299 if (!match(V: Cmp1, P: m_c_ICmp(Pred&: Pred1, L: m_Value(V&: Y), R: m_Specific(V: X))))
1300 return nullptr;
1301
1302 // Replace variable with constant value equivalence to remove a variable use:
1303 // (X == C) && (Y Pred1 X) --> (X == C) && (Y Pred1 C)
1304 // (X != C) || (Y Pred1 X) --> (X != C) || (Y Pred1 C)
1305 // Can think of the 'or' substitution with the 'and' bool equivalent:
1306 // A || B --> A || (!A && B)
1307 Value *SubstituteCmp = simplifyICmpInst(Pred: Pred1, LHS: Y, RHS: C, Q);
1308 if (!SubstituteCmp) {
1309 // If we need to create a new instruction, require that the old compare can
1310 // be removed.
1311 if (!Cmp1->hasOneUse())
1312 return nullptr;
1313 SubstituteCmp = Builder.CreateICmp(P: Pred1, LHS: Y, RHS: C);
1314 }
1315 if (IsLogical) {
1316 Instruction *MDFrom =
1317 ProfcheckDisableMetadataFixes && isa<SelectInst>(Val: I) ? nullptr : &I;
1318 return IsAnd ? Builder.CreateLogicalAnd(Cond1: Cmp0, Cond2: SubstituteCmp, Name: "", MDFrom)
1319 : Builder.CreateLogicalOr(Cond1: Cmp0, Cond2: SubstituteCmp, Name: "", MDFrom);
1320 }
1321 return Builder.CreateBinOp(Opc: IsAnd ? Instruction::And : Instruction::Or, LHS: Cmp0,
1322 RHS: SubstituteCmp);
1323}
1324
1325/// Fold (icmp Pred1 V1, C1) & (icmp Pred2 V2, C2)
1326/// or (icmp Pred1 V1, C1) | (icmp Pred2 V2, C2)
1327/// into a single comparison using range-based reasoning.
1328/// NOTE: This is also used for logical and/or, must be poison-safe!
1329Value *InstCombinerImpl::foldAndOrOfICmpsUsingRanges(ICmpInst *ICmp1,
1330 ICmpInst *ICmp2,
1331 bool IsAnd) {
1332 // Return (V, CR) for a range check idiom V in CR.
1333 auto MatchExactRangeCheck =
1334 [](ICmpInst *ICmp) -> std::optional<std::pair<Value *, ConstantRange>> {
1335 const APInt *C;
1336 if (!match(V: ICmp->getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
1337 return std::nullopt;
1338 Value *LHS = ICmp->getOperand(i_nocapture: 0);
1339 CmpPredicate Pred = ICmp->getPredicate();
1340 Value *X;
1341 // Match (x & NegPow2) ==/!= C
1342 const APInt *Mask;
1343 if (ICmpInst::isEquality(P: Pred) &&
1344 match(V: LHS, P: m_OneUse(SubPattern: m_And(L: m_Value(V&: X), R: m_NegatedPower2(V&: Mask)))) &&
1345 C->countr_zero() >= Mask->countr_zero()) {
1346 ConstantRange CR(*C, *C - *Mask);
1347 if (Pred == ICmpInst::ICMP_NE)
1348 CR = CR.inverse();
1349 return std::make_pair(x&: X, y&: CR);
1350 }
1351 ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
1352 // Match (add X, C1) pred C
1353 // TODO: investigate whether we should apply the one-use check on m_AddLike.
1354 const APInt *C1;
1355 if (match(V: LHS, P: m_AddLike(L: m_Value(V&: X), R: m_APInt(Res&: C1))))
1356 return std::make_pair(x&: X, y: CR.subtract(CI: *C1));
1357 return std::make_pair(x&: LHS, y&: CR);
1358 };
1359
1360 auto RC1 = MatchExactRangeCheck(ICmp1);
1361 if (!RC1)
1362 return nullptr;
1363
1364 auto RC2 = MatchExactRangeCheck(ICmp2);
1365 if (!RC2)
1366 return nullptr;
1367
1368 auto &[V1, CR1] = *RC1;
1369 auto &[V2, CR2] = *RC2;
1370 if (V1 != V2)
1371 return nullptr;
1372
1373 // For 'and', we use the De Morgan's Laws to simplify the implementation.
1374 if (IsAnd) {
1375 CR1 = CR1.inverse();
1376 CR2 = CR2.inverse();
1377 }
1378
1379 Type *Ty = V1->getType();
1380 Value *NewV = V1;
1381 std::optional<ConstantRange> CR = CR1.exactUnionWith(CR: CR2);
1382 if (!CR) {
1383 if (!(ICmp1->hasOneUse() && ICmp2->hasOneUse()) || CR1.isWrappedSet() ||
1384 CR2.isWrappedSet())
1385 return nullptr;
1386
1387 // Check whether we have equal-size ranges that only differ by one bit.
1388 // In that case we can apply a mask to map one range onto the other.
1389 APInt LowerDiff = CR1.getLower() ^ CR2.getLower();
1390 APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1);
1391 APInt CR1Size = CR1.getUpper() - CR1.getLower();
1392 if (!LowerDiff.isPowerOf2() || LowerDiff != UpperDiff ||
1393 CR1Size != CR2.getUpper() - CR2.getLower())
1394 return nullptr;
1395
1396 CR = CR1.getLower().ult(RHS: CR2.getLower()) ? CR1 : CR2;
1397 NewV = Builder.CreateAnd(LHS: NewV, RHS: ConstantInt::get(Ty, V: ~LowerDiff));
1398 }
1399
1400 if (IsAnd)
1401 CR = CR->inverse();
1402
1403 CmpInst::Predicate NewPred;
1404 APInt NewC, Offset;
1405 CR->getEquivalentICmp(Pred&: NewPred, RHS&: NewC, Offset);
1406
1407 if (Offset != 0)
1408 NewV = Builder.CreateAdd(LHS: NewV, RHS: ConstantInt::get(Ty, V: Offset));
1409 return Builder.CreateICmp(P: NewPred, LHS: NewV, RHS: ConstantInt::get(Ty, V: NewC));
1410}
1411
1412/// Ignore all operations which only change the sign of a value, returning the
1413/// underlying magnitude value.
1414static Value *stripSignOnlyFPOps(Value *Val) {
1415 match(V: Val, P: m_FNeg(X: m_Value(V&: Val)));
1416 match(V: Val, P: m_FAbs(Op0: m_Value(V&: Val)));
1417 match(V: Val, P: m_CopySign(Op0: m_Value(V&: Val), Op1: m_Value()));
1418 return Val;
1419}
1420
1421/// Matches canonical form of isnan, fcmp ord x, 0
1422static bool matchIsNotNaN(FCmpInst::Predicate P, Value *LHS, Value *RHS) {
1423 return P == FCmpInst::FCMP_ORD && match(V: RHS, P: m_AnyZeroFP());
1424}
1425
1426/// Matches fcmp u__ x, +/-inf
1427static bool matchUnorderedInfCompare(FCmpInst::Predicate P, Value *LHS,
1428 Value *RHS) {
1429 return FCmpInst::isUnordered(predicate: P) && match(V: RHS, P: m_Inf());
1430}
1431
1432/// and (fcmp ord x, 0), (fcmp u* x, inf) -> fcmp o* x, inf
1433///
1434/// Clang emits this pattern for doing an isfinite check in __builtin_isnormal.
1435static Value *matchIsFiniteTest(InstCombiner::BuilderTy &Builder, FCmpInst *LHS,
1436 FCmpInst *RHS) {
1437 Value *LHS0 = LHS->getOperand(i_nocapture: 0), *LHS1 = LHS->getOperand(i_nocapture: 1);
1438 Value *RHS0 = RHS->getOperand(i_nocapture: 0), *RHS1 = RHS->getOperand(i_nocapture: 1);
1439 FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
1440
1441 if (!matchIsNotNaN(P: PredL, LHS: LHS0, RHS: LHS1) ||
1442 !matchUnorderedInfCompare(P: PredR, LHS: RHS0, RHS: RHS1))
1443 return nullptr;
1444
1445 return Builder.CreateFCmpFMF(P: FCmpInst::getOrderedPredicate(Pred: PredR), LHS: RHS0, RHS: RHS1,
1446 FMFSource: FMFSource::intersect(A: LHS, B: RHS));
1447}
1448
1449Value *InstCombinerImpl::foldLogicOfFCmps(FCmpInst *LHS, FCmpInst *RHS,
1450 bool IsAnd, bool IsLogicalSelect) {
1451 Value *LHS0 = LHS->getOperand(i_nocapture: 0), *LHS1 = LHS->getOperand(i_nocapture: 1);
1452 Value *RHS0 = RHS->getOperand(i_nocapture: 0), *RHS1 = RHS->getOperand(i_nocapture: 1);
1453 FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
1454
1455 if (LHS0 == RHS1 && RHS0 == LHS1) {
1456 // Swap RHS operands to match LHS.
1457 PredR = FCmpInst::getSwappedPredicate(pred: PredR);
1458 std::swap(a&: RHS0, b&: RHS1);
1459 }
1460
1461 // Simplify (fcmp cc0 x, y) & (fcmp cc1 x, y).
1462 // Suppose the relation between x and y is R, where R is one of
1463 // U(1000), L(0100), G(0010) or E(0001), and CC0 and CC1 are the bitmasks for
1464 // testing the desired relations.
1465 //
1466 // Since (R & CC0) and (R & CC1) are either R or 0, we actually have this:
1467 // bool(R & CC0) && bool(R & CC1)
1468 // = bool((R & CC0) & (R & CC1))
1469 // = bool(R & (CC0 & CC1)) <= by re-association, commutation, and idempotency
1470 //
1471 // Since (R & CC0) and (R & CC1) are either R or 0, we actually have this:
1472 // bool(R & CC0) || bool(R & CC1)
1473 // = bool((R & CC0) | (R & CC1))
1474 // = bool(R & (CC0 | CC1)) <= by reversed distribution (contribution? ;)
1475 if (LHS0 == RHS0 && LHS1 == RHS1) {
1476 unsigned FCmpCodeL = getFCmpCode(CC: PredL);
1477 unsigned FCmpCodeR = getFCmpCode(CC: PredR);
1478 unsigned NewPred = IsAnd ? FCmpCodeL & FCmpCodeR : FCmpCodeL | FCmpCodeR;
1479
1480 // Intersect the fast math flags.
1481 // TODO: We can union the fast math flags unless this is a logical select.
1482 return getFCmpValue(Code: NewPred, LHS: LHS0, RHS: LHS1, Builder,
1483 FMF: FMFSource::intersect(A: LHS, B: RHS));
1484 }
1485
1486 // This transform is not valid for a logical select.
1487 if (!IsLogicalSelect &&
1488 ((PredL == FCmpInst::FCMP_ORD && PredR == FCmpInst::FCMP_ORD && IsAnd) ||
1489 (PredL == FCmpInst::FCMP_UNO && PredR == FCmpInst::FCMP_UNO &&
1490 !IsAnd))) {
1491 if (LHS0->getType() != RHS0->getType())
1492 return nullptr;
1493
1494 // FCmp canonicalization ensures that (fcmp ord/uno X, X) and
1495 // (fcmp ord/uno X, C) will be transformed to (fcmp X, +0.0).
1496 if (match(V: LHS1, P: m_PosZeroFP()) && match(V: RHS1, P: m_PosZeroFP())) {
1497 // Ignore the constants because they are obviously not NANs:
1498 // (fcmp ord x, 0.0) & (fcmp ord y, 0.0) -> (fcmp ord x, y)
1499 // (fcmp uno x, 0.0) | (fcmp uno y, 0.0) -> (fcmp uno x, y)
1500 return Builder.CreateFCmpFMF(P: PredL, LHS: LHS0, RHS: RHS0,
1501 FMFSource: FMFSource::intersect(A: LHS, B: RHS));
1502 }
1503 }
1504
1505 // This transform is not valid for a logical select.
1506 if (!IsLogicalSelect && IsAnd &&
1507 stripSignOnlyFPOps(Val: LHS0) == stripSignOnlyFPOps(Val: RHS0)) {
1508 // and (fcmp ord x, 0), (fcmp u* x, inf) -> fcmp o* x, inf
1509 // and (fcmp ord x, 0), (fcmp u* fabs(x), inf) -> fcmp o* x, inf
1510 if (Value *Left = matchIsFiniteTest(Builder, LHS, RHS))
1511 return Left;
1512 if (Value *Right = matchIsFiniteTest(Builder, LHS: RHS, RHS: LHS))
1513 return Right;
1514 }
1515
1516 // Turn at least two fcmps with constants into llvm.is.fpclass.
1517 //
1518 // If we can represent a combined value test with one class call, we can
1519 // potentially eliminate 4-6 instructions. If we can represent a test with a
1520 // single fcmp with fneg and fabs, that's likely a better canonical form.
1521 if (LHS->hasOneUse() && RHS->hasOneUse()) {
1522 auto [ClassValRHS, ClassMaskRHS] =
1523 fcmpToClassTest(Pred: PredR, F: *RHS->getFunction(), LHS: RHS0, RHS: RHS1);
1524 if (ClassValRHS) {
1525 auto [ClassValLHS, ClassMaskLHS] =
1526 fcmpToClassTest(Pred: PredL, F: *LHS->getFunction(), LHS: LHS0, RHS: LHS1);
1527 if (ClassValLHS == ClassValRHS) {
1528 unsigned CombinedMask = IsAnd ? (ClassMaskLHS & ClassMaskRHS)
1529 : (ClassMaskLHS | ClassMaskRHS);
1530 return Builder.CreateIntrinsic(
1531 ID: Intrinsic::is_fpclass, Types: {ClassValLHS->getType()},
1532 Args: {ClassValLHS, Builder.getInt32(C: CombinedMask)});
1533 }
1534 }
1535 }
1536
1537 // Canonicalize the range check idiom:
1538 // and (fcmp olt/ole/ult/ule x, C), (fcmp ogt/oge/ugt/uge x, -C)
1539 // --> fabs(x) olt/ole/ult/ule C
1540 // or (fcmp ogt/oge/ugt/uge x, C), (fcmp olt/ole/ult/ule x, -C)
1541 // --> fabs(x) ogt/oge/ugt/uge C
1542 // TODO: Generalize to handle a negated variable operand?
1543 const APFloat *LHSC, *RHSC;
1544 if (LHS0 == RHS0 && LHS->hasOneUse() && RHS->hasOneUse() &&
1545 FCmpInst::getSwappedPredicate(pred: PredL) == PredR &&
1546 match(V: LHS1, P: m_APFloatAllowPoison(Res&: LHSC)) &&
1547 match(V: RHS1, P: m_APFloatAllowPoison(Res&: RHSC)) &&
1548 LHSC->bitwiseIsEqual(RHS: neg(X: *RHSC))) {
1549 auto IsLessThanOrLessEqual = [](FCmpInst::Predicate Pred) {
1550 switch (Pred) {
1551 case FCmpInst::FCMP_OLT:
1552 case FCmpInst::FCMP_OLE:
1553 case FCmpInst::FCMP_ULT:
1554 case FCmpInst::FCMP_ULE:
1555 return true;
1556 default:
1557 return false;
1558 }
1559 };
1560 if (IsLessThanOrLessEqual(IsAnd ? PredR : PredL)) {
1561 std::swap(a&: LHSC, b&: RHSC);
1562 std::swap(a&: PredL, b&: PredR);
1563 }
1564 if (IsLessThanOrLessEqual(IsAnd ? PredL : PredR)) {
1565 FastMathFlags NewFlag = LHS->getFastMathFlags();
1566 if (!IsLogicalSelect)
1567 NewFlag |= RHS->getFastMathFlags();
1568
1569 Value *FAbs =
1570 Builder.CreateUnaryIntrinsic(ID: Intrinsic::fabs, V: LHS0, FMFSource: NewFlag);
1571 return Builder.CreateFCmpFMF(
1572 P: PredL, LHS: FAbs, RHS: ConstantFP::get(Ty: LHS0->getType(), V: *LHSC), FMFSource: NewFlag);
1573 }
1574 }
1575
1576 return nullptr;
1577}
1578
1579/// Match an fcmp against a special value that performs a test possible by
1580/// llvm.is.fpclass.
1581static bool matchIsFPClassLikeFCmp(Value *Op, Value *&ClassVal,
1582 uint64_t &ClassMask) {
1583 auto *FCmp = dyn_cast<FCmpInst>(Val: Op);
1584 if (!FCmp || !FCmp->hasOneUse())
1585 return false;
1586
1587 std::tie(args&: ClassVal, args&: ClassMask) =
1588 fcmpToClassTest(Pred: FCmp->getPredicate(), F: *FCmp->getParent()->getParent(),
1589 LHS: FCmp->getOperand(i_nocapture: 0), RHS: FCmp->getOperand(i_nocapture: 1));
1590 return ClassVal != nullptr;
1591}
1592
1593/// or (is_fpclass x, mask0), (is_fpclass x, mask1)
1594/// -> is_fpclass x, (mask0 | mask1)
1595/// and (is_fpclass x, mask0), (is_fpclass x, mask1)
1596/// -> is_fpclass x, (mask0 & mask1)
1597/// xor (is_fpclass x, mask0), (is_fpclass x, mask1)
1598/// -> is_fpclass x, (mask0 ^ mask1)
1599Instruction *InstCombinerImpl::foldLogicOfIsFPClass(BinaryOperator &BO,
1600 Value *Op0, Value *Op1) {
1601 Value *ClassVal0 = nullptr;
1602 Value *ClassVal1 = nullptr;
1603 uint64_t ClassMask0, ClassMask1;
1604
1605 // Restrict to folding one fcmp into one is.fpclass for now, don't introduce a
1606 // new class.
1607 //
1608 // TODO: Support forming is.fpclass out of 2 separate fcmps when codegen is
1609 // better.
1610
1611 bool IsLHSClass =
1612 match(V: Op0, P: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::is_fpclass>(
1613 Op0: m_Value(V&: ClassVal0), Op1: m_ConstantInt(V&: ClassMask0))));
1614 bool IsRHSClass =
1615 match(V: Op1, P: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::is_fpclass>(
1616 Op0: m_Value(V&: ClassVal1), Op1: m_ConstantInt(V&: ClassMask1))));
1617 if ((((IsLHSClass || matchIsFPClassLikeFCmp(Op: Op0, ClassVal&: ClassVal0, ClassMask&: ClassMask0)) &&
1618 (IsRHSClass || matchIsFPClassLikeFCmp(Op: Op1, ClassVal&: ClassVal1, ClassMask&: ClassMask1)))) &&
1619 ClassVal0 == ClassVal1) {
1620 unsigned NewClassMask;
1621 switch (BO.getOpcode()) {
1622 case Instruction::And:
1623 NewClassMask = ClassMask0 & ClassMask1;
1624 break;
1625 case Instruction::Or:
1626 NewClassMask = ClassMask0 | ClassMask1;
1627 break;
1628 case Instruction::Xor:
1629 NewClassMask = ClassMask0 ^ ClassMask1;
1630 break;
1631 default:
1632 llvm_unreachable("not a binary logic operator");
1633 }
1634
1635 if (IsLHSClass) {
1636 auto *II = cast<IntrinsicInst>(Val: Op0);
1637 II->setArgOperand(
1638 i: 1, v: ConstantInt::get(Ty: II->getArgOperand(i: 1)->getType(), V: NewClassMask));
1639 return replaceInstUsesWith(I&: BO, V: II);
1640 }
1641
1642 if (IsRHSClass) {
1643 auto *II = cast<IntrinsicInst>(Val: Op1);
1644 II->setArgOperand(
1645 i: 1, v: ConstantInt::get(Ty: II->getArgOperand(i: 1)->getType(), V: NewClassMask));
1646 return replaceInstUsesWith(I&: BO, V: II);
1647 }
1648
1649 CallInst *NewClass =
1650 Builder.CreateIntrinsic(ID: Intrinsic::is_fpclass, Types: {ClassVal0->getType()},
1651 Args: {ClassVal0, Builder.getInt32(C: NewClassMask)});
1652 return replaceInstUsesWith(I&: BO, V: NewClass);
1653 }
1654
1655 return nullptr;
1656}
1657
1658/// Look for the pattern that conditionally negates a value via math operations:
1659/// cond.splat = sext i1 cond
1660/// sub = add cond.splat, x
1661/// xor = xor sub, cond.splat
1662/// and rewrite it to do the same, but via logical operations:
1663/// value.neg = sub 0, value
1664/// cond = select i1 neg, value.neg, value
1665Instruction *InstCombinerImpl::canonicalizeConditionalNegationViaMathToSelect(
1666 BinaryOperator &I) {
1667 assert(I.getOpcode() == BinaryOperator::Xor && "Only for xor!");
1668 Value *Cond, *X;
1669 // As per complexity ordering, `xor` is not commutative here.
1670 if (!match(V: &I, P: m_c_BinOp(L: m_OneUse(SubPattern: m_Value()), R: m_Value())) ||
1671 !match(V: I.getOperand(i_nocapture: 1), P: m_SExt(Op: m_Value(V&: Cond))) ||
1672 !Cond->getType()->isIntOrIntVectorTy(BitWidth: 1) ||
1673 !match(V: I.getOperand(i_nocapture: 0), P: m_c_Add(L: m_SExt(Op: m_Specific(V: Cond)), R: m_Value(V&: X))))
1674 return nullptr;
1675 return createSelectInstWithUnknownProfile(
1676 C: Cond, S1: Builder.CreateNeg(V: X, Name: X->getName() + ".neg"), S2: X);
1677}
1678
1679/// This a limited reassociation for a special case (see above) where we are
1680/// checking if two values are either both NAN (unordered) or not-NAN (ordered).
1681/// This could be handled more generally in '-reassociation', but it seems like
1682/// an unlikely pattern for a large number of logic ops and fcmps.
1683static Instruction *reassociateFCmps(BinaryOperator &BO,
1684 InstCombiner::BuilderTy &Builder) {
1685 Instruction::BinaryOps Opcode = BO.getOpcode();
1686 assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&
1687 "Expecting and/or op for fcmp transform");
1688
1689 // There are 4 commuted variants of the pattern. Canonicalize operands of this
1690 // logic op so an fcmp is operand 0 and a matching logic op is operand 1.
1691 Value *Op0 = BO.getOperand(i_nocapture: 0), *Op1 = BO.getOperand(i_nocapture: 1), *X;
1692 if (match(V: Op1, P: m_FCmp(L: m_Value(), R: m_AnyZeroFP())))
1693 std::swap(a&: Op0, b&: Op1);
1694
1695 // Match inner binop and the predicate for combining 2 NAN checks into 1.
1696 Value *BO10, *BO11;
1697 FCmpInst::Predicate NanPred = Opcode == Instruction::And ? FCmpInst::FCMP_ORD
1698 : FCmpInst::FCMP_UNO;
1699 if (!match(V: Op0, P: m_SpecificFCmp(MatchPred: NanPred, L: m_Value(V&: X), R: m_AnyZeroFP())) ||
1700 !match(V: Op1, P: m_BinOp(Opcode, L: m_Value(V&: BO10), R: m_Value(V&: BO11))))
1701 return nullptr;
1702
1703 // The inner logic op must have a matching fcmp operand.
1704 Value *Y;
1705 if (!match(V: BO10, P: m_SpecificFCmp(MatchPred: NanPred, L: m_Value(V&: Y), R: m_AnyZeroFP())) ||
1706 X->getType() != Y->getType())
1707 std::swap(a&: BO10, b&: BO11);
1708
1709 if (!match(V: BO10, P: m_SpecificFCmp(MatchPred: NanPred, L: m_Value(V&: Y), R: m_AnyZeroFP())) ||
1710 X->getType() != Y->getType())
1711 return nullptr;
1712
1713 // and (fcmp ord X, 0), (and (fcmp ord Y, 0), Z) --> and (fcmp ord X, Y), Z
1714 // or (fcmp uno X, 0), (or (fcmp uno Y, 0), Z) --> or (fcmp uno X, Y), Z
1715 // Intersect FMF from the 2 source fcmps.
1716 Value *NewFCmp =
1717 Builder.CreateFCmpFMF(P: NanPred, LHS: X, RHS: Y, FMFSource: FMFSource::intersect(A: Op0, B: BO10));
1718 return BinaryOperator::Create(Op: Opcode, S1: NewFCmp, S2: BO11);
1719}
1720
1721/// Match variations of De Morgan's Laws:
1722/// (~A & ~B) == (~(A | B))
1723/// (~A | ~B) == (~(A & B))
1724static Instruction *matchDeMorgansLaws(BinaryOperator &I,
1725 InstCombiner &IC) {
1726 const Instruction::BinaryOps Opcode = I.getOpcode();
1727 assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&
1728 "Trying to match De Morgan's Laws with something other than and/or");
1729
1730 // Flip the logic operation.
1731 const Instruction::BinaryOps FlippedOpcode =
1732 (Opcode == Instruction::And) ? Instruction::Or : Instruction::And;
1733
1734 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
1735 Value *A, *B;
1736 if (match(V: Op0, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: A)))) &&
1737 match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: B)))) &&
1738 !IC.isFreeToInvert(V: A, WillInvertAllUses: A->hasOneUse()) &&
1739 !IC.isFreeToInvert(V: B, WillInvertAllUses: B->hasOneUse())) {
1740 Value *AndOr =
1741 IC.Builder.CreateBinOp(Opc: FlippedOpcode, LHS: A, RHS: B, Name: I.getName() + ".demorgan");
1742 return BinaryOperator::CreateNot(Op: AndOr);
1743 }
1744
1745 // The 'not' ops may require reassociation.
1746 // (A & ~B) & ~C --> A & ~(B | C)
1747 // (~B & A) & ~C --> A & ~(B | C)
1748 // (A | ~B) | ~C --> A | ~(B & C)
1749 // (~B | A) | ~C --> A | ~(B & C)
1750 Value *C;
1751 if (match(V: Op0, P: m_OneUse(SubPattern: m_c_BinOp(Opcode, L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B))))) &&
1752 match(V: Op1, P: m_Not(V: m_Value(V&: C)))) {
1753 Value *FlippedBO = IC.Builder.CreateBinOp(Opc: FlippedOpcode, LHS: B, RHS: C);
1754 return BinaryOperator::Create(Op: Opcode, S1: A, S2: IC.Builder.CreateNot(V: FlippedBO));
1755 }
1756
1757 return nullptr;
1758}
1759
1760bool InstCombinerImpl::shouldOptimizeCast(CastInst *CI) {
1761 Value *CastSrc = CI->getOperand(i_nocapture: 0);
1762
1763 // Noop casts and casts of constants should be eliminated trivially.
1764 if (CI->getSrcTy() == CI->getDestTy() || isa<Constant>(Val: CastSrc))
1765 return false;
1766
1767 // If this cast is paired with another cast that can be eliminated, we prefer
1768 // to have it eliminated.
1769 if (const auto *PrecedingCI = dyn_cast<CastInst>(Val: CastSrc))
1770 if (isEliminableCastPair(CI1: PrecedingCI, CI2: CI))
1771 return false;
1772
1773 return true;
1774}
1775
1776/// Fold {and,or,xor} (cast X), C.
1777static Instruction *foldLogicCastConstant(BinaryOperator &Logic, CastInst *Cast,
1778 InstCombinerImpl &IC) {
1779 Constant *C = dyn_cast<Constant>(Val: Logic.getOperand(i_nocapture: 1));
1780 if (!C)
1781 return nullptr;
1782
1783 auto LogicOpc = Logic.getOpcode();
1784 Type *DestTy = Logic.getType();
1785 Type *SrcTy = Cast->getSrcTy();
1786
1787 // Move the logic operation ahead of a zext or sext if the constant is
1788 // unchanged in the smaller source type. Performing the logic in a smaller
1789 // type may provide more information to later folds, and the smaller logic
1790 // instruction may be cheaper (particularly in the case of vectors).
1791 Value *X;
1792 auto &DL = IC.getDataLayout();
1793 if (match(V: Cast, P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X))))) {
1794 PreservedCastFlags Flags;
1795 if (Constant *TruncC = getLosslessUnsignedTrunc(C, DestTy: SrcTy, DL, Flags: &Flags)) {
1796 // LogicOpc (zext X), C --> zext (LogicOpc X, C)
1797 Value *NewOp = IC.Builder.CreateBinOp(Opc: LogicOpc, LHS: X, RHS: TruncC);
1798 auto *ZExt = new ZExtInst(NewOp, DestTy);
1799 ZExt->setNonNeg(Flags.NNeg);
1800 ZExt->andIRFlags(V: Cast);
1801 return ZExt;
1802 }
1803 }
1804
1805 if (match(V: Cast, P: m_OneUse(SubPattern: m_SExtLike(Op: m_Value(V&: X))))) {
1806 if (Constant *TruncC = getLosslessSignedTrunc(C, DestTy: SrcTy, DL)) {
1807 // LogicOpc (sext X), C --> sext (LogicOpc X, C)
1808 Value *NewOp = IC.Builder.CreateBinOp(Opc: LogicOpc, LHS: X, RHS: TruncC);
1809 return new SExtInst(NewOp, DestTy);
1810 }
1811 }
1812
1813 return nullptr;
1814}
1815
1816/// Fold {and,or,xor} (cast X), Y.
1817Instruction *InstCombinerImpl::foldCastedBitwiseLogic(BinaryOperator &I) {
1818 auto LogicOpc = I.getOpcode();
1819 assert(I.isBitwiseLogicOp() && "Unexpected opcode for bitwise logic folding");
1820
1821 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
1822
1823 // fold bitwise(A >> BW - 1, zext(icmp)) (BW is the scalar bits of the
1824 // type of A)
1825 // -> bitwise(zext(A < 0), zext(icmp))
1826 // -> zext(bitwise(A < 0, icmp))
1827 auto FoldBitwiseICmpZeroWithICmp = [&](Value *Op0,
1828 Value *Op1) -> Instruction * {
1829 Value *A;
1830 bool IsMatched =
1831 match(V: Op0,
1832 P: m_OneUse(SubPattern: m_LShr(
1833 L: m_Value(V&: A),
1834 R: m_SpecificInt(V: Op0->getType()->getScalarSizeInBits() - 1)))) &&
1835 match(V: Op1, P: m_OneUse(SubPattern: m_ZExt(Op: m_ICmp(L: m_Value(), R: m_Value()))));
1836
1837 if (!IsMatched)
1838 return nullptr;
1839
1840 auto *ICmpL =
1841 Builder.CreateICmpSLT(LHS: A, RHS: Constant::getNullValue(Ty: A->getType()));
1842 auto *ICmpR = cast<ZExtInst>(Val: Op1)->getOperand(i_nocapture: 0);
1843 auto *BitwiseOp = Builder.CreateBinOp(Opc: LogicOpc, LHS: ICmpL, RHS: ICmpR);
1844
1845 return new ZExtInst(BitwiseOp, Op0->getType());
1846 };
1847
1848 if (auto *Ret = FoldBitwiseICmpZeroWithICmp(Op0, Op1))
1849 return Ret;
1850
1851 if (auto *Ret = FoldBitwiseICmpZeroWithICmp(Op1, Op0))
1852 return Ret;
1853
1854 CastInst *Cast0 = dyn_cast<CastInst>(Val: Op0);
1855 if (!Cast0)
1856 return nullptr;
1857
1858 // This must be a cast from an integer or integer vector source type to allow
1859 // transformation of the logic operation to the source type.
1860 Type *DestTy = I.getType();
1861 Type *SrcTy = Cast0->getSrcTy();
1862 if (!SrcTy->isIntOrIntVectorTy())
1863 return nullptr;
1864
1865 if (Instruction *Ret = foldLogicCastConstant(Logic&: I, Cast: Cast0, IC&: *this))
1866 return Ret;
1867
1868 CastInst *Cast1 = dyn_cast<CastInst>(Val: Op1);
1869 if (!Cast1)
1870 return nullptr;
1871
1872 // Both operands of the logic operation are casts. The casts must be the
1873 // same kind for reduction.
1874 Instruction::CastOps CastOpcode = Cast0->getOpcode();
1875 if (CastOpcode != Cast1->getOpcode())
1876 return nullptr;
1877
1878 // Can't fold it profitably if no one of casts has one use.
1879 if (!Cast0->hasOneUse() && !Cast1->hasOneUse())
1880 return nullptr;
1881
1882 Value *X, *Y;
1883 if (match(V: Cast0, P: m_ZExtOrSExt(Op: m_Value(V&: X))) &&
1884 match(V: Cast1, P: m_ZExtOrSExt(Op: m_Value(V&: Y)))) {
1885 // Cast the narrower source to the wider source type.
1886 unsigned XNumBits = X->getType()->getScalarSizeInBits();
1887 unsigned YNumBits = Y->getType()->getScalarSizeInBits();
1888 if (XNumBits != YNumBits) {
1889 // Cast the narrower source to the wider source type only if both of casts
1890 // have one use to avoid creating an extra instruction.
1891 if (!Cast0->hasOneUse() || !Cast1->hasOneUse())
1892 return nullptr;
1893
1894 // If the source types do not match, but the casts are matching extends,
1895 // we can still narrow the logic op.
1896 if (XNumBits < YNumBits) {
1897 X = Builder.CreateCast(Op: CastOpcode, V: X, DestTy: Y->getType());
1898 } else if (YNumBits < XNumBits) {
1899 Y = Builder.CreateCast(Op: CastOpcode, V: Y, DestTy: X->getType());
1900 }
1901 }
1902
1903 // Do the logic op in the intermediate width, then widen more.
1904 Value *NarrowLogic = Builder.CreateBinOp(Opc: LogicOpc, LHS: X, RHS: Y, Name: I.getName());
1905 auto *Disjoint = dyn_cast<PossiblyDisjointInst>(Val: &I);
1906 auto *NewDisjoint = dyn_cast<PossiblyDisjointInst>(Val: NarrowLogic);
1907 if (Disjoint && NewDisjoint)
1908 NewDisjoint->setIsDisjoint(Disjoint->isDisjoint());
1909 return CastInst::Create(CastOpcode, S: NarrowLogic, Ty: DestTy);
1910 }
1911
1912 // If the src type of casts are different, give up for other cast opcodes.
1913 if (SrcTy != Cast1->getSrcTy())
1914 return nullptr;
1915
1916 Value *Cast0Src = Cast0->getOperand(i_nocapture: 0);
1917 Value *Cast1Src = Cast1->getOperand(i_nocapture: 0);
1918
1919 // fold logic(cast(A), cast(B)) -> cast(logic(A, B))
1920 if (shouldOptimizeCast(CI: Cast0) && shouldOptimizeCast(CI: Cast1)) {
1921 Value *NewOp = Builder.CreateBinOp(Opc: LogicOpc, LHS: Cast0Src, RHS: Cast1Src,
1922 Name: I.getName());
1923 return CastInst::Create(CastOpcode, S: NewOp, Ty: DestTy);
1924 }
1925
1926 return nullptr;
1927}
1928
1929static Instruction *foldAndToXor(BinaryOperator &I,
1930 InstCombiner::BuilderTy &Builder) {
1931 assert(I.getOpcode() == Instruction::And);
1932 Value *Op0 = I.getOperand(i_nocapture: 0);
1933 Value *Op1 = I.getOperand(i_nocapture: 1);
1934 Value *A, *B;
1935
1936 // Operand complexity canonicalization guarantees that the 'or' is Op0.
1937 // (A | B) & ~(A & B) --> A ^ B
1938 // (A | B) & ~(B & A) --> A ^ B
1939 if (match(V: &I, P: m_BinOp(L: m_Or(L: m_Value(V&: A), R: m_Value(V&: B)),
1940 R: m_Not(V: m_c_And(L: m_Deferred(V: A), R: m_Deferred(V: B))))))
1941 return BinaryOperator::CreateXor(V1: A, V2: B);
1942
1943 // (A | ~B) & (~A | B) --> ~(A ^ B)
1944 // (A | ~B) & (B | ~A) --> ~(A ^ B)
1945 // (~B | A) & (~A | B) --> ~(A ^ B)
1946 // (~B | A) & (B | ~A) --> ~(A ^ B)
1947 if (Op0->hasOneUse() || Op1->hasOneUse())
1948 if (match(V: &I, P: m_BinOp(L: m_c_Or(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B))),
1949 R: m_c_Or(L: m_Not(V: m_Deferred(V: A)), R: m_Deferred(V: B)))))
1950 return BinaryOperator::CreateNot(Op: Builder.CreateXor(LHS: A, RHS: B));
1951
1952 return nullptr;
1953}
1954
1955static Instruction *foldOrToXor(BinaryOperator &I,
1956 InstCombiner::BuilderTy &Builder) {
1957 assert(I.getOpcode() == Instruction::Or);
1958 Value *Op0 = I.getOperand(i_nocapture: 0);
1959 Value *Op1 = I.getOperand(i_nocapture: 1);
1960 Value *A, *B;
1961
1962 // Operand complexity canonicalization guarantees that the 'and' is Op0.
1963 // (A & B) | ~(A | B) --> ~(A ^ B)
1964 // (A & B) | ~(B | A) --> ~(A ^ B)
1965 if (Op0->hasOneUse() || Op1->hasOneUse())
1966 if (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
1967 match(V: Op1, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B)))))
1968 return BinaryOperator::CreateNot(Op: Builder.CreateXor(LHS: A, RHS: B));
1969
1970 // Operand complexity canonicalization guarantees that the 'xor' is Op0.
1971 // (A ^ B) | ~(A | B) --> ~(A & B)
1972 // (A ^ B) | ~(B | A) --> ~(A & B)
1973 if (Op0->hasOneUse() || Op1->hasOneUse())
1974 if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))) &&
1975 match(V: Op1, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B)))))
1976 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: A, RHS: B));
1977
1978 // (A & ~B) | (~A & B) --> A ^ B
1979 // (A & ~B) | (B & ~A) --> A ^ B
1980 // (~B & A) | (~A & B) --> A ^ B
1981 // (~B & A) | (B & ~A) --> A ^ B
1982 if (match(V: Op0, P: m_c_And(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B)))) &&
1983 match(V: Op1, P: m_c_And(L: m_Not(V: m_Specific(V: A)), R: m_Specific(V: B))))
1984 return BinaryOperator::CreateXor(V1: A, V2: B);
1985
1986 return nullptr;
1987}
1988
1989/// Return true if a constant shift amount is always less than the specified
1990/// bit-width. If not, the shift could create poison in the narrower type.
1991static bool canNarrowShiftAmt(Constant *C, unsigned BitWidth) {
1992 APInt Threshold(C->getType()->getScalarSizeInBits(), BitWidth);
1993 return match(V: C, P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_ULT, Threshold));
1994}
1995
1996/// Try to use narrower ops (sink zext ops) for an 'and' with binop operand and
1997/// a common zext operand: and (binop (zext X), C), (zext X).
1998Instruction *InstCombinerImpl::narrowMaskedBinOp(BinaryOperator &And) {
1999 // This transform could also apply to {or, and, xor}, but there are better
2000 // folds for those cases, so we don't expect those patterns here. AShr is not
2001 // handled because it should always be transformed to LShr in this sequence.
2002 // The subtract transform is different because it has a constant on the left.
2003 // Add/mul commute the constant to RHS; sub with constant RHS becomes add.
2004 Value *Op0 = And.getOperand(i_nocapture: 0), *Op1 = And.getOperand(i_nocapture: 1);
2005 Constant *C;
2006 if (!match(V: Op0, P: m_OneUse(SubPattern: m_Add(L: m_Specific(V: Op1), R: m_Constant(C)))) &&
2007 !match(V: Op0, P: m_OneUse(SubPattern: m_Mul(L: m_Specific(V: Op1), R: m_Constant(C)))) &&
2008 !match(V: Op0, P: m_OneUse(SubPattern: m_LShr(L: m_Specific(V: Op1), R: m_Constant(C)))) &&
2009 !match(V: Op0, P: m_OneUse(SubPattern: m_Shl(L: m_Specific(V: Op1), R: m_Constant(C)))) &&
2010 !match(V: Op0, P: m_OneUse(SubPattern: m_Sub(L: m_Constant(C), R: m_Specific(V: Op1)))))
2011 return nullptr;
2012
2013 Value *X;
2014 if (!match(V: Op1, P: m_ZExt(Op: m_Value(V&: X))) || Op1->hasNUsesOrMore(N: 3))
2015 return nullptr;
2016
2017 Type *Ty = And.getType();
2018 if (!isa<VectorType>(Val: Ty) && !shouldChangeType(From: Ty, To: X->getType()))
2019 return nullptr;
2020
2021 // If we're narrowing a shift, the shift amount must be safe (less than the
2022 // width) in the narrower type. If the shift amount is greater, instsimplify
2023 // usually handles that case, but we can't guarantee/assert it.
2024 Instruction::BinaryOps Opc = cast<BinaryOperator>(Val: Op0)->getOpcode();
2025 if (Opc == Instruction::LShr || Opc == Instruction::Shl)
2026 if (!canNarrowShiftAmt(C, BitWidth: X->getType()->getScalarSizeInBits()))
2027 return nullptr;
2028
2029 // and (sub C, (zext X)), (zext X) --> zext (and (sub C', X), X)
2030 // and (binop (zext X), C), (zext X) --> zext (and (binop X, C'), X)
2031 Value *NewC = ConstantExpr::getTrunc(C, Ty: X->getType());
2032 Value *NewBO = Opc == Instruction::Sub ? Builder.CreateBinOp(Opc, LHS: NewC, RHS: X)
2033 : Builder.CreateBinOp(Opc, LHS: X, RHS: NewC);
2034 return new ZExtInst(Builder.CreateAnd(LHS: NewBO, RHS: X), Ty);
2035}
2036
2037/// Try folding relatively complex patterns for both And and Or operations
2038/// with all And and Or swapped.
2039static Instruction *foldComplexAndOrPatterns(BinaryOperator &I,
2040 InstCombiner::BuilderTy &Builder) {
2041 const Instruction::BinaryOps Opcode = I.getOpcode();
2042 assert(Opcode == Instruction::And || Opcode == Instruction::Or);
2043
2044 // Flip the logic operation.
2045 const Instruction::BinaryOps FlippedOpcode =
2046 (Opcode == Instruction::And) ? Instruction::Or : Instruction::And;
2047
2048 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
2049 Value *A, *B, *C, *X, *Y, *Dummy;
2050
2051 // Match following expressions:
2052 // (~(A | B) & C)
2053 // (~(A & B) | C)
2054 // Captures X = ~(A | B) or ~(A & B)
2055 const auto matchNotOrAnd =
2056 [Opcode, FlippedOpcode](Value *Op, auto m_A, auto m_B, auto m_C,
2057 Value *&X, bool CountUses = false) -> bool {
2058 if (CountUses && !Op->hasOneUse())
2059 return false;
2060
2061 if (match(Op,
2062 m_c_BinOp(FlippedOpcode,
2063 m_Value(X, m_Not(m_c_BinOp(Opcode, m_A, m_B))), m_C)))
2064 return !CountUses || X->hasOneUse();
2065
2066 return false;
2067 };
2068
2069 // (~(A | B) & C) | ... --> ...
2070 // (~(A & B) | C) & ... --> ...
2071 // TODO: One use checks are conservative. We just need to check that a total
2072 // number of multiple used values does not exceed reduction
2073 // in operations.
2074 if (matchNotOrAnd(Op0, m_Value(V&: A), m_Value(V&: B), m_Value(V&: C), X)) {
2075 // (~(A | B) & C) | (~(A | C) & B) --> (B ^ C) & ~A
2076 // (~(A & B) | C) & (~(A & C) | B) --> ~((B ^ C) & A)
2077 if (matchNotOrAnd(Op1, m_Specific(V: A), m_Specific(V: C), m_Specific(V: B), Dummy,
2078 true)) {
2079 Value *Xor = Builder.CreateXor(LHS: B, RHS: C);
2080 return (Opcode == Instruction::Or)
2081 ? BinaryOperator::CreateAnd(V1: Xor, V2: Builder.CreateNot(V: A))
2082 : BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: Xor, RHS: A));
2083 }
2084
2085 // (~(A | B) & C) | (~(B | C) & A) --> (A ^ C) & ~B
2086 // (~(A & B) | C) & (~(B & C) | A) --> ~((A ^ C) & B)
2087 if (matchNotOrAnd(Op1, m_Specific(V: B), m_Specific(V: C), m_Specific(V: A), Dummy,
2088 true)) {
2089 Value *Xor = Builder.CreateXor(LHS: A, RHS: C);
2090 return (Opcode == Instruction::Or)
2091 ? BinaryOperator::CreateAnd(V1: Xor, V2: Builder.CreateNot(V: B))
2092 : BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: Xor, RHS: B));
2093 }
2094
2095 // (~(A | B) & C) | ~(A | C) --> ~((B & C) | A)
2096 // (~(A & B) | C) & ~(A & C) --> ~((B | C) & A)
2097 if (match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_OneUse(
2098 SubPattern: m_c_BinOp(Opcode, L: m_Specific(V: A), R: m_Specific(V: C)))))))
2099 return BinaryOperator::CreateNot(Op: Builder.CreateBinOp(
2100 Opc: Opcode, LHS: Builder.CreateBinOp(Opc: FlippedOpcode, LHS: B, RHS: C), RHS: A));
2101
2102 // (~(A | B) & C) | ~(B | C) --> ~((A & C) | B)
2103 // (~(A & B) | C) & ~(B & C) --> ~((A | C) & B)
2104 if (match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_OneUse(
2105 SubPattern: m_c_BinOp(Opcode, L: m_Specific(V: B), R: m_Specific(V: C)))))))
2106 return BinaryOperator::CreateNot(Op: Builder.CreateBinOp(
2107 Opc: Opcode, LHS: Builder.CreateBinOp(Opc: FlippedOpcode, LHS: A, RHS: C), RHS: B));
2108
2109 // (~(A | B) & C) | ~(C | (A ^ B)) --> ~((A | B) & (C | (A ^ B)))
2110 // Note, the pattern with swapped and/or is not handled because the
2111 // result is more undefined than a source:
2112 // (~(A & B) | C) & ~(C & (A ^ B)) --> (A ^ B ^ C) | ~(A | C) is invalid.
2113 if (Opcode == Instruction::Or && Op0->hasOneUse() &&
2114 match(V: Op1,
2115 P: m_OneUse(SubPattern: m_Not(V: m_Value(
2116 V&: Y, Match: m_c_BinOp(Opcode, L: m_Specific(V: C),
2117 R: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B)))))))) {
2118 // X = ~(A | B)
2119 // Y = (C | (A ^ B)
2120 Value *Or = cast<BinaryOperator>(Val: X)->getOperand(i_nocapture: 0);
2121 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: Or, RHS: Y));
2122 }
2123 }
2124
2125 // (~A & B & C) | ... --> ...
2126 // (~A | B | C) | ... --> ...
2127 // TODO: One use checks are conservative. We just need to check that a total
2128 // number of multiple used values does not exceed reduction
2129 // in operations.
2130 if (match(V: Op0,
2131 P: m_OneUse(SubPattern: m_c_BinOp(Opcode: FlippedOpcode,
2132 L: m_BinOp(Opcode: FlippedOpcode, L: m_Value(V&: B), R: m_Value(V&: C)),
2133 R: m_Value(V&: X, Match: m_Not(V: m_Value(V&: A)))))) ||
2134 match(V: Op0, P: m_OneUse(SubPattern: m_c_BinOp(Opcode: FlippedOpcode,
2135 L: m_c_BinOp(Opcode: FlippedOpcode, L: m_Value(V&: C),
2136 R: m_Value(V&: X, Match: m_Not(V: m_Value(V&: A)))),
2137 R: m_Value(V&: B))))) {
2138 // X = ~A
2139 // (~A & B & C) | ~(A | B | C) --> ~(A | (B ^ C))
2140 // (~A | B | C) & ~(A & B & C) --> (~A | (B ^ C))
2141 if (match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_c_BinOp(
2142 Opcode, L: m_c_BinOp(Opcode, L: m_Specific(V: A), R: m_Specific(V: B)),
2143 R: m_Specific(V: C))))) ||
2144 match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_c_BinOp(
2145 Opcode, L: m_c_BinOp(Opcode, L: m_Specific(V: B), R: m_Specific(V: C)),
2146 R: m_Specific(V: A))))) ||
2147 match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_c_BinOp(
2148 Opcode, L: m_c_BinOp(Opcode, L: m_Specific(V: A), R: m_Specific(V: C)),
2149 R: m_Specific(V: B)))))) {
2150 Value *Xor = Builder.CreateXor(LHS: B, RHS: C);
2151 return (Opcode == Instruction::Or)
2152 ? BinaryOperator::CreateNot(Op: Builder.CreateOr(LHS: Xor, RHS: A))
2153 : BinaryOperator::CreateOr(V1: Xor, V2: X);
2154 }
2155
2156 // (~A & B & C) | ~(A | B) --> (C | ~B) & ~A
2157 // (~A | B | C) & ~(A & B) --> (C & ~B) | ~A
2158 if (match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_OneUse(
2159 SubPattern: m_c_BinOp(Opcode, L: m_Specific(V: A), R: m_Specific(V: B)))))))
2160 return BinaryOperator::Create(
2161 Op: FlippedOpcode, S1: Builder.CreateBinOp(Opc: Opcode, LHS: C, RHS: Builder.CreateNot(V: B)),
2162 S2: X);
2163
2164 // (~A & B & C) | ~(A | C) --> (B | ~C) & ~A
2165 // (~A | B | C) & ~(A & C) --> (B & ~C) | ~A
2166 if (match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_OneUse(
2167 SubPattern: m_c_BinOp(Opcode, L: m_Specific(V: A), R: m_Specific(V: C)))))))
2168 return BinaryOperator::Create(
2169 Op: FlippedOpcode, S1: Builder.CreateBinOp(Opc: Opcode, LHS: B, RHS: Builder.CreateNot(V: C)),
2170 S2: X);
2171 }
2172
2173 return nullptr;
2174}
2175
2176/// Try to reassociate a pair of binops so that values with one use only are
2177/// part of the same instruction. This may enable folds that are limited with
2178/// multi-use restrictions and makes it more likely to match other patterns that
2179/// are looking for a common operand.
2180static Instruction *reassociateForUses(BinaryOperator &BO,
2181 InstCombinerImpl::BuilderTy &Builder) {
2182 Instruction::BinaryOps Opcode = BO.getOpcode();
2183 Value *X, *Y, *Z;
2184 if (match(V: &BO,
2185 P: m_c_BinOp(Opcode, L: m_OneUse(SubPattern: m_BinOp(Opcode, L: m_Value(V&: X), R: m_Value(V&: Y))),
2186 R: m_OneUse(SubPattern: m_Value(V&: Z))))) {
2187 if (!isa<Constant>(Val: X) && !isa<Constant>(Val: Y) && !isa<Constant>(Val: Z)) {
2188 // (X op Y) op Z --> (Y op Z) op X
2189 if (!X->hasOneUse()) {
2190 Value *YZ = Builder.CreateBinOp(Opc: Opcode, LHS: Y, RHS: Z);
2191 return BinaryOperator::Create(Op: Opcode, S1: YZ, S2: X);
2192 }
2193 // (X op Y) op Z --> (X op Z) op Y
2194 if (!Y->hasOneUse()) {
2195 Value *XZ = Builder.CreateBinOp(Opc: Opcode, LHS: X, RHS: Z);
2196 return BinaryOperator::Create(Op: Opcode, S1: XZ, S2: Y);
2197 }
2198 }
2199 }
2200
2201 return nullptr;
2202}
2203
2204// Match
2205// (X + C2) | C
2206// (X + C2) ^ C
2207// (X + C2) & C
2208// and convert to do the bitwise logic first:
2209// (X | C) + C2
2210// (X ^ C) + C2
2211// (X & C) + C2
2212// iff bits affected by logic op are lower than last bit affected by math op
2213static Instruction *canonicalizeLogicFirst(BinaryOperator &I,
2214 InstCombiner::BuilderTy &Builder) {
2215 Type *Ty = I.getType();
2216 Instruction::BinaryOps OpC = I.getOpcode();
2217 Value *Op0 = I.getOperand(i_nocapture: 0);
2218 Value *Op1 = I.getOperand(i_nocapture: 1);
2219 Value *X;
2220 const APInt *C, *C2;
2221
2222 if (!(match(V: Op0, P: m_OneUse(SubPattern: m_Add(L: m_Value(V&: X), R: m_APInt(Res&: C2)))) &&
2223 match(V: Op1, P: m_APInt(Res&: C))))
2224 return nullptr;
2225
2226 unsigned Width = Ty->getScalarSizeInBits();
2227 unsigned LastOneMath = Width - C2->countr_zero();
2228
2229 switch (OpC) {
2230 case Instruction::And:
2231 if (C->countl_one() < LastOneMath)
2232 return nullptr;
2233 break;
2234 case Instruction::Xor:
2235 case Instruction::Or:
2236 if (C->countl_zero() < LastOneMath)
2237 return nullptr;
2238 break;
2239 default:
2240 llvm_unreachable("Unexpected BinaryOp!");
2241 }
2242
2243 Value *NewBinOp = Builder.CreateBinOp(Opc: OpC, LHS: X, RHS: ConstantInt::get(Ty, V: *C));
2244 return BinaryOperator::CreateWithCopiedFlags(Opc: Instruction::Add, V1: NewBinOp,
2245 V2: ConstantInt::get(Ty, V: *C2), CopyO: Op0);
2246}
2247
2248// binop(shift(ShiftedC1, ShAmt), shift(ShiftedC2, add(ShAmt, AddC))) ->
2249// shift(binop(ShiftedC1, shift(ShiftedC2, AddC)), ShAmt)
2250// where both shifts are the same and AddC is a valid shift amount.
2251Instruction *InstCombinerImpl::foldBinOpOfDisplacedShifts(BinaryOperator &I) {
2252 assert((I.isBitwiseLogicOp() || I.getOpcode() == Instruction::Add) &&
2253 "Unexpected opcode");
2254
2255 Value *ShAmt;
2256 Constant *ShiftedC1, *ShiftedC2, *AddC;
2257 Type *Ty = I.getType();
2258 unsigned BitWidth = Ty->getScalarSizeInBits();
2259 if (!match(V: &I, P: m_c_BinOp(L: m_Shift(L: m_ImmConstant(C&: ShiftedC1), R: m_Value(V&: ShAmt)),
2260 R: m_Shift(L: m_ImmConstant(C&: ShiftedC2),
2261 R: m_AddLike(L: m_Deferred(V: ShAmt),
2262 R: m_ImmConstant(C&: AddC))))))
2263 return nullptr;
2264
2265 // Make sure the add constant is a valid shift amount.
2266 if (!match(V: AddC,
2267 P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_ULT, Threshold: APInt(BitWidth, BitWidth))))
2268 return nullptr;
2269
2270 // Avoid constant expressions.
2271 auto *Op0Inst = dyn_cast<Instruction>(Val: I.getOperand(i_nocapture: 0));
2272 auto *Op1Inst = dyn_cast<Instruction>(Val: I.getOperand(i_nocapture: 1));
2273 if (!Op0Inst || !Op1Inst)
2274 return nullptr;
2275
2276 // Both shifts must be the same.
2277 Instruction::BinaryOps ShiftOp =
2278 static_cast<Instruction::BinaryOps>(Op0Inst->getOpcode());
2279 if (ShiftOp != Op1Inst->getOpcode())
2280 return nullptr;
2281
2282 // For adds, only left shifts are supported.
2283 if (I.getOpcode() == Instruction::Add && ShiftOp != Instruction::Shl)
2284 return nullptr;
2285
2286 Value *NewC = Builder.CreateBinOp(
2287 Opc: I.getOpcode(), LHS: ShiftedC1, RHS: Builder.CreateBinOp(Opc: ShiftOp, LHS: ShiftedC2, RHS: AddC));
2288 return BinaryOperator::Create(Op: ShiftOp, S1: NewC, S2: ShAmt);
2289}
2290
2291// Fold and/or/xor with two equal intrinsic IDs:
2292// bitwise(fshl (A, B, ShAmt), fshl(C, D, ShAmt))
2293// -> fshl(bitwise(A, C), bitwise(B, D), ShAmt)
2294// bitwise(fshr (A, B, ShAmt), fshr(C, D, ShAmt))
2295// -> fshr(bitwise(A, C), bitwise(B, D), ShAmt)
2296// bitwise(bswap(A), bswap(B)) -> bswap(bitwise(A, B))
2297// bitwise(bswap(A), C) -> bswap(bitwise(A, bswap(C)))
2298// bitwise(bitreverse(A), bitreverse(B)) -> bitreverse(bitwise(A, B))
2299// bitwise(bitreverse(A), C) -> bitreverse(bitwise(A, bitreverse(C)))
2300static Instruction *
2301foldBitwiseLogicWithIntrinsics(BinaryOperator &I,
2302 InstCombiner::BuilderTy &Builder) {
2303 assert(I.isBitwiseLogicOp() && "Should and/or/xor");
2304 if (!I.getOperand(i_nocapture: 0)->hasOneUse())
2305 return nullptr;
2306 IntrinsicInst *X = dyn_cast<IntrinsicInst>(Val: I.getOperand(i_nocapture: 0));
2307 if (!X)
2308 return nullptr;
2309
2310 IntrinsicInst *Y = dyn_cast<IntrinsicInst>(Val: I.getOperand(i_nocapture: 1));
2311 if (Y && (!Y->hasOneUse() || X->getIntrinsicID() != Y->getIntrinsicID()))
2312 return nullptr;
2313
2314 Intrinsic::ID IID = X->getIntrinsicID();
2315 const APInt *RHSC;
2316 // Try to match constant RHS.
2317 if (!Y && (!(IID == Intrinsic::bswap || IID == Intrinsic::bitreverse) ||
2318 !match(V: I.getOperand(i_nocapture: 1), P: m_APInt(Res&: RHSC))))
2319 return nullptr;
2320
2321 switch (IID) {
2322 case Intrinsic::fshl:
2323 case Intrinsic::fshr: {
2324 if (X->getOperand(i_nocapture: 2) != Y->getOperand(i_nocapture: 2))
2325 return nullptr;
2326 Value *NewOp0 =
2327 Builder.CreateBinOp(Opc: I.getOpcode(), LHS: X->getOperand(i_nocapture: 0), RHS: Y->getOperand(i_nocapture: 0));
2328 Value *NewOp1 =
2329 Builder.CreateBinOp(Opc: I.getOpcode(), LHS: X->getOperand(i_nocapture: 1), RHS: Y->getOperand(i_nocapture: 1));
2330 Function *F =
2331 Intrinsic::getOrInsertDeclaration(M: I.getModule(), id: IID, Tys: I.getType());
2332 return CallInst::Create(Func: F, Args: {NewOp0, NewOp1, X->getOperand(i_nocapture: 2)});
2333 }
2334 case Intrinsic::bswap:
2335 case Intrinsic::bitreverse: {
2336 Value *NewOp0 = Builder.CreateBinOp(
2337 Opc: I.getOpcode(), LHS: X->getOperand(i_nocapture: 0),
2338 RHS: Y ? Y->getOperand(i_nocapture: 0)
2339 : ConstantInt::get(Ty: I.getType(), V: IID == Intrinsic::bswap
2340 ? RHSC->byteSwap()
2341 : RHSC->reverseBits()));
2342 Function *F =
2343 Intrinsic::getOrInsertDeclaration(M: I.getModule(), id: IID, Tys: I.getType());
2344 return CallInst::Create(Func: F, Args: {NewOp0});
2345 }
2346 default:
2347 return nullptr;
2348 }
2349}
2350
2351// Try to simplify V by replacing occurrences of Op with RepOp, but only look
2352// through bitwise operations. In particular, for X | Y we try to replace Y with
2353// 0 inside X and for X & Y we try to replace Y with -1 inside X.
2354// Return the simplified result of X if successful, and nullptr otherwise.
2355// If SimplifyOnly is true, no new instructions will be created.
2356static Value *simplifyAndOrWithOpReplaced(Value *V, Value *Op, Value *RepOp,
2357 bool SimplifyOnly,
2358 InstCombinerImpl &IC,
2359 unsigned Depth = 0) {
2360 if (Op == RepOp)
2361 return nullptr;
2362
2363 if (V == Op)
2364 return RepOp;
2365
2366 auto *I = dyn_cast<BinaryOperator>(Val: V);
2367 if (!I || !I->isBitwiseLogicOp() || Depth >= 3)
2368 return nullptr;
2369
2370 if (!I->hasOneUse())
2371 SimplifyOnly = true;
2372
2373 Value *NewOp0 = simplifyAndOrWithOpReplaced(V: I->getOperand(i_nocapture: 0), Op, RepOp,
2374 SimplifyOnly, IC, Depth: Depth + 1);
2375 Value *NewOp1 = simplifyAndOrWithOpReplaced(V: I->getOperand(i_nocapture: 1), Op, RepOp,
2376 SimplifyOnly, IC, Depth: Depth + 1);
2377 if (!NewOp0 && !NewOp1)
2378 return nullptr;
2379
2380 if (!NewOp0)
2381 NewOp0 = I->getOperand(i_nocapture: 0);
2382 if (!NewOp1)
2383 NewOp1 = I->getOperand(i_nocapture: 1);
2384
2385 if (Value *Res = simplifyBinOp(Opcode: I->getOpcode(), LHS: NewOp0, RHS: NewOp1,
2386 Q: IC.getSimplifyQuery().getWithInstruction(I)))
2387 return Res;
2388
2389 if (SimplifyOnly)
2390 return nullptr;
2391 return IC.Builder.CreateBinOp(Opc: I->getOpcode(), LHS: NewOp0, RHS: NewOp1);
2392}
2393
2394/// Reassociate and/or expressions to see if we can fold the inner and/or ops.
2395/// TODO: Make this recursive; it's a little tricky because an arbitrary
2396/// number of and/or instructions might have to be created.
2397Value *InstCombinerImpl::reassociateBooleanAndOr(Value *LHS, Value *X, Value *Y,
2398 Instruction &I, bool IsAnd,
2399 bool RHSIsLogical) {
2400 Instruction::BinaryOps Opcode = IsAnd ? Instruction::And : Instruction::Or;
2401 // LHS bop (X lop Y) --> (LHS bop X) lop Y
2402 // LHS bop (X bop Y) --> (LHS bop X) bop Y
2403 if (Value *Res = foldBooleanAndOr(LHS, RHS: X, I, IsAnd, /*IsLogical=*/false))
2404 return RHSIsLogical ? Builder.CreateLogicalOp(Opc: Opcode, Cond1: Res, Cond2: Y)
2405 : Builder.CreateBinOp(Opc: Opcode, LHS: Res, RHS: Y);
2406 // LHS bop (X bop Y) --> X bop (LHS bop Y)
2407 // LHS bop (X lop Y) --> X lop (LHS bop Y)
2408 if (Value *Res = foldBooleanAndOr(LHS, RHS: Y, I, IsAnd, /*IsLogical=*/false))
2409 return RHSIsLogical ? Builder.CreateLogicalOp(Opc: Opcode, Cond1: X, Cond2: Res)
2410 : Builder.CreateBinOp(Opc: Opcode, LHS: X, RHS: Res);
2411 return nullptr;
2412}
2413
2414// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
2415// here. We should standardize that construct where it is needed or choose some
2416// other way to ensure that commutated variants of patterns are not missed.
2417Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
2418 Type *Ty = I.getType();
2419
2420 if (Value *V = simplifyAndInst(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1),
2421 Q: SQ.getWithInstruction(I: &I)))
2422 return replaceInstUsesWith(I, V);
2423
2424 if (SimplifyAssociativeOrCommutative(I))
2425 return &I;
2426
2427 if (Instruction *X = foldVectorBinop(Inst&: I))
2428 return X;
2429
2430 if (Instruction *Phi = foldBinopWithPhiOperands(BO&: I))
2431 return Phi;
2432
2433 // See if we can simplify any instructions used by the instruction whose sole
2434 // purpose is to compute bits we don't care about.
2435 if (SimplifyDemandedInstructionBits(Inst&: I))
2436 return &I;
2437
2438 // Do this before using distributive laws to catch simple and/or/not patterns.
2439 if (Instruction *Xor = foldAndToXor(I, Builder))
2440 return Xor;
2441
2442 if (Instruction *X = foldComplexAndOrPatterns(I, Builder))
2443 return X;
2444
2445 // (A|B)&(A|C) -> A|(B&C) etc
2446 if (Value *V = foldUsingDistributiveLaws(I))
2447 return replaceInstUsesWith(I, V);
2448
2449 if (Instruction *R = foldBinOpShiftWithShift(I))
2450 return R;
2451
2452 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
2453
2454 Value *X, *Y;
2455 const APInt *C;
2456 if ((match(V: Op0, P: m_OneUse(SubPattern: m_LogicalShift(L: m_One(), R: m_Value(V&: X)))) ||
2457 (match(V: Op0, P: m_OneUse(SubPattern: m_Shl(L: m_APInt(Res&: C), R: m_Value(V&: X)))) && (*C)[0])) &&
2458 match(V: Op1, P: m_One())) {
2459 // (1 >> X) & 1 --> zext(X == 0)
2460 // (C << X) & 1 --> zext(X == 0), when C is odd
2461 Value *IsZero = Builder.CreateICmpEQ(LHS: X, RHS: ConstantInt::get(Ty, V: 0));
2462 return new ZExtInst(IsZero, Ty);
2463 }
2464
2465 // (-(X & 1)) & Y --> (X & 1) == 0 ? 0 : Y
2466 Value *Neg;
2467 if (match(V: &I,
2468 P: m_c_And(L: m_Value(V&: Neg, Match: m_OneUse(SubPattern: m_Neg(V: m_And(L: m_Value(), R: m_One())))),
2469 R: m_Value(V&: Y)))) {
2470 Value *Cmp = Builder.CreateIsNull(Arg: Neg);
2471 return createSelectInstWithUnknownProfile(C: Cmp,
2472 S1: ConstantInt::getNullValue(Ty), S2: Y);
2473 }
2474
2475 // Canonicalize:
2476 // (X +/- Y) & Y --> ~X & Y when Y is a power of 2.
2477 if (match(V: &I, P: m_c_And(L: m_Value(V&: Y), R: m_OneUse(SubPattern: m_CombineOr(
2478 L: m_c_Add(L: m_Value(V&: X), R: m_Deferred(V: Y)),
2479 R: m_Sub(L: m_Value(V&: X), R: m_Deferred(V: Y)))))) &&
2480 isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ true, CxtI: &I))
2481 return BinaryOperator::CreateAnd(V1: Builder.CreateNot(V: X), V2: Y);
2482
2483 if (match(V: Op1, P: m_APInt(Res&: C))) {
2484 const APInt *XorC;
2485 if (match(V: Op0, P: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: X), R: m_APInt(Res&: XorC))))) {
2486 // (X ^ C1) & C2 --> (X & C2) ^ (C1&C2)
2487 Constant *NewC = ConstantInt::get(Ty, V: *C & *XorC);
2488 Value *And = Builder.CreateAnd(LHS: X, RHS: Op1);
2489 And->takeName(V: Op0);
2490 return BinaryOperator::CreateXor(V1: And, V2: NewC);
2491 }
2492
2493 const APInt *OrC;
2494 if (match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: X), R: m_APInt(Res&: OrC))))) {
2495 // (X | C1) & C2 --> (X & C2^(C1&C2)) | (C1&C2)
2496 // NOTE: This reduces the number of bits set in the & mask, which
2497 // can expose opportunities for store narrowing for scalars.
2498 // NOTE: SimplifyDemandedBits should have already removed bits from C1
2499 // that aren't set in C2. Meaning we can replace (C1&C2) with C1 in
2500 // above, but this feels safer.
2501 APInt Together = *C & *OrC;
2502 Value *And = Builder.CreateAnd(LHS: X, RHS: ConstantInt::get(Ty, V: Together ^ *C));
2503 And->takeName(V: Op0);
2504 return BinaryOperator::CreateOr(V1: And, V2: ConstantInt::get(Ty, V: Together));
2505 }
2506
2507 unsigned Width = Ty->getScalarSizeInBits();
2508 const APInt *ShiftC;
2509 if (match(V: Op0, P: m_OneUse(SubPattern: m_SExt(Op: m_AShr(L: m_Value(V&: X), R: m_APInt(Res&: ShiftC))))) &&
2510 ShiftC->ult(RHS: Width)) {
2511 if (*C == APInt::getLowBitsSet(numBits: Width, loBitsSet: Width - ShiftC->getZExtValue())) {
2512 // We are clearing high bits that were potentially set by sext+ashr:
2513 // and (sext (ashr X, ShiftC)), C --> lshr (sext X), ShiftC
2514 Value *Sext = Builder.CreateSExt(V: X, DestTy: Ty);
2515 Constant *ShAmtC = ConstantInt::get(Ty, V: ShiftC->zext(width: Width));
2516 return BinaryOperator::CreateLShr(V1: Sext, V2: ShAmtC);
2517 }
2518 }
2519
2520 // If this 'and' clears the sign-bits added by ashr, replace with lshr:
2521 // and (ashr X, ShiftC), C --> lshr X, ShiftC
2522 if (match(V: Op0, P: m_AShr(L: m_Value(V&: X), R: m_APInt(Res&: ShiftC))) && ShiftC->ult(RHS: Width) &&
2523 C->isMask(numBits: Width - ShiftC->getZExtValue()))
2524 return BinaryOperator::CreateLShr(V1: X, V2: ConstantInt::get(Ty, V: *ShiftC));
2525
2526 const APInt *AddC;
2527 if (match(V: Op0, P: m_Add(L: m_Value(V&: X), R: m_APInt(Res&: AddC)))) {
2528 // If we are masking the result of the add down to exactly one bit and
2529 // the constant we are adding has no bits set below that bit, then the
2530 // add is flipping a single bit. Example:
2531 // (X + 4) & 4 --> (X & 4) ^ 4
2532 if (Op0->hasOneUse() && C->isPowerOf2() && (*AddC & (*C - 1)) == 0) {
2533 assert((*C & *AddC) != 0 && "Expected common bit");
2534 Value *NewAnd = Builder.CreateAnd(LHS: X, RHS: Op1);
2535 return BinaryOperator::CreateXor(V1: NewAnd, V2: Op1);
2536 }
2537 }
2538
2539 // ((C1 OP zext(X)) & C2) -> zext((C1 OP X) & C2) if C2 fits in the
2540 // bitwidth of X and OP behaves well when given trunc(C1) and X.
2541 auto isNarrowableBinOpcode = [](BinaryOperator *B) {
2542 switch (B->getOpcode()) {
2543 case Instruction::Xor:
2544 case Instruction::Or:
2545 case Instruction::Mul:
2546 case Instruction::Add:
2547 case Instruction::Sub:
2548 return true;
2549 default:
2550 return false;
2551 }
2552 };
2553 BinaryOperator *BO;
2554 if (match(V: Op0, P: m_OneUse(SubPattern: m_BinOp(I&: BO))) && isNarrowableBinOpcode(BO)) {
2555 Instruction::BinaryOps BOpcode = BO->getOpcode();
2556 Value *X;
2557 const APInt *C1;
2558 // TODO: The one-use restrictions could be relaxed a little if the AND
2559 // is going to be removed.
2560 // Try to narrow the 'and' and a binop with constant operand:
2561 // and (bo (zext X), C1), C --> zext (and (bo X, TruncC1), TruncC)
2562 if (match(V: BO, P: m_c_BinOp(L: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X))), R: m_APInt(Res&: C1))) &&
2563 C->isIntN(N: X->getType()->getScalarSizeInBits())) {
2564 unsigned XWidth = X->getType()->getScalarSizeInBits();
2565 Constant *TruncC1 = ConstantInt::get(Ty: X->getType(), V: C1->trunc(width: XWidth));
2566 Value *BinOp = isa<ZExtInst>(Val: BO->getOperand(i_nocapture: 0))
2567 ? Builder.CreateBinOp(Opc: BOpcode, LHS: X, RHS: TruncC1)
2568 : Builder.CreateBinOp(Opc: BOpcode, LHS: TruncC1, RHS: X);
2569 Constant *TruncC = ConstantInt::get(Ty: X->getType(), V: C->trunc(width: XWidth));
2570 Value *And = Builder.CreateAnd(LHS: BinOp, RHS: TruncC);
2571 return new ZExtInst(And, Ty);
2572 }
2573
2574 // Similar to above: if the mask matches the zext input width, then the
2575 // 'and' can be eliminated, so we can truncate the other variable op:
2576 // and (bo (zext X), Y), C --> zext (bo X, (trunc Y))
2577 if (isa<Instruction>(Val: BO->getOperand(i_nocapture: 0)) &&
2578 match(V: BO->getOperand(i_nocapture: 0), P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X)))) &&
2579 C->isMask(numBits: X->getType()->getScalarSizeInBits())) {
2580 Y = BO->getOperand(i_nocapture: 1);
2581 Value *TrY = Builder.CreateTrunc(V: Y, DestTy: X->getType(), Name: Y->getName() + ".tr");
2582 Value *NewBO =
2583 Builder.CreateBinOp(Opc: BOpcode, LHS: X, RHS: TrY, Name: BO->getName() + ".narrow");
2584 return new ZExtInst(NewBO, Ty);
2585 }
2586 // and (bo Y, (zext X)), C --> zext (bo (trunc Y), X)
2587 if (isa<Instruction>(Val: BO->getOperand(i_nocapture: 1)) &&
2588 match(V: BO->getOperand(i_nocapture: 1), P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X)))) &&
2589 C->isMask(numBits: X->getType()->getScalarSizeInBits())) {
2590 Y = BO->getOperand(i_nocapture: 0);
2591 Value *TrY = Builder.CreateTrunc(V: Y, DestTy: X->getType(), Name: Y->getName() + ".tr");
2592 Value *NewBO =
2593 Builder.CreateBinOp(Opc: BOpcode, LHS: TrY, RHS: X, Name: BO->getName() + ".narrow");
2594 return new ZExtInst(NewBO, Ty);
2595 }
2596 }
2597
2598 // This is intentionally placed after the narrowing transforms for
2599 // efficiency (transform directly to the narrow logic op if possible).
2600 // If the mask is only needed on one incoming arm, push the 'and' op up.
2601 if (match(V: Op0, P: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: X), R: m_Value(V&: Y)))) ||
2602 match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
2603 APInt NotAndMask(~(*C));
2604 BinaryOperator::BinaryOps BinOp = cast<BinaryOperator>(Val: Op0)->getOpcode();
2605 if (MaskedValueIsZero(V: X, Mask: NotAndMask, CxtI: &I)) {
2606 // Not masking anything out for the LHS, move mask to RHS.
2607 // and ({x}or X, Y), C --> {x}or X, (and Y, C)
2608 Value *NewRHS = Builder.CreateAnd(LHS: Y, RHS: Op1, Name: Y->getName() + ".masked");
2609 return BinaryOperator::Create(Op: BinOp, S1: X, S2: NewRHS);
2610 }
2611 if (!isa<Constant>(Val: Y) && MaskedValueIsZero(V: Y, Mask: NotAndMask, CxtI: &I)) {
2612 // Not masking anything out for the RHS, move mask to LHS.
2613 // and ({x}or X, Y), C --> {x}or (and X, C), Y
2614 Value *NewLHS = Builder.CreateAnd(LHS: X, RHS: Op1, Name: X->getName() + ".masked");
2615 return BinaryOperator::Create(Op: BinOp, S1: NewLHS, S2: Y);
2616 }
2617 }
2618
2619 // When the mask is a power-of-2 constant and op0 is a shifted-power-of-2
2620 // constant, test if the shift amount equals the offset bit index:
2621 // (ShiftC << X) & C --> X == (log2(C) - log2(ShiftC)) ? C : 0
2622 // (ShiftC >> X) & C --> X == (log2(ShiftC) - log2(C)) ? C : 0
2623 if (C->isPowerOf2() &&
2624 match(V: Op0, P: m_OneUse(SubPattern: m_LogicalShift(L: m_Power2(V&: ShiftC), R: m_Value(V&: X))))) {
2625 int Log2ShiftC = ShiftC->exactLogBase2();
2626 int Log2C = C->exactLogBase2();
2627 bool IsShiftLeft =
2628 cast<BinaryOperator>(Val: Op0)->getOpcode() == Instruction::Shl;
2629 int BitNum = IsShiftLeft ? Log2C - Log2ShiftC : Log2ShiftC - Log2C;
2630 assert(BitNum >= 0 && "Expected demanded bits to handle impossible mask");
2631 Value *Cmp = Builder.CreateICmpEQ(LHS: X, RHS: ConstantInt::get(Ty, V: BitNum));
2632 return createSelectInstWithUnknownProfile(C: Cmp, S1: ConstantInt::get(Ty, V: *C),
2633 S2: ConstantInt::getNullValue(Ty));
2634 }
2635
2636 Constant *C1, *C2;
2637 const APInt *C3 = C;
2638 Value *X;
2639 if (C3->isPowerOf2()) {
2640 Constant *Log2C3 = ConstantInt::get(Ty, V: C3->countr_zero());
2641 if (match(V: Op0, P: m_OneUse(SubPattern: m_LShr(L: m_Shl(L: m_ImmConstant(C&: C1), R: m_Value(V&: X)),
2642 R: m_ImmConstant(C&: C2)))) &&
2643 match(V: C1, P: m_Power2())) {
2644 Constant *Log2C1 = ConstantExpr::getExactLogBase2(C: C1);
2645 Constant *LshrC = ConstantExpr::getAdd(C1: C2, C2: Log2C3);
2646 KnownBits KnownLShrc = computeKnownBits(V: LshrC, CxtI: nullptr);
2647 if (KnownLShrc.getMaxValue().ult(RHS: Width)) {
2648 // iff C1,C3 is pow2 and C2 + cttz(C3) < BitWidth:
2649 // ((C1 << X) >> C2) & C3 -> X == (cttz(C3)+C2-cttz(C1)) ? C3 : 0
2650 Constant *CmpC = ConstantExpr::getSub(C1: LshrC, C2: Log2C1);
2651 Value *Cmp = Builder.CreateICmpEQ(LHS: X, RHS: CmpC);
2652 return createSelectInstWithUnknownProfile(
2653 C: Cmp, S1: ConstantInt::get(Ty, V: *C3), S2: ConstantInt::getNullValue(Ty));
2654 }
2655 }
2656
2657 if (match(V: Op0, P: m_OneUse(SubPattern: m_Shl(L: m_LShr(L: m_ImmConstant(C&: C1), R: m_Value(V&: X)),
2658 R: m_ImmConstant(C&: C2)))) &&
2659 match(V: C1, P: m_Power2())) {
2660 Constant *Log2C1 = ConstantExpr::getExactLogBase2(C: C1);
2661 Constant *Cmp =
2662 ConstantFoldCompareInstOperands(Predicate: ICmpInst::ICMP_ULT, LHS: Log2C3, RHS: C2, DL);
2663 if (Cmp && Cmp->isZeroValue()) {
2664 // iff C1,C3 is pow2 and Log2(C3) >= C2:
2665 // ((C1 >> X) << C2) & C3 -> X == (cttz(C1)+C2-cttz(C3)) ? C3 : 0
2666 Constant *ShlC = ConstantExpr::getAdd(C1: C2, C2: Log2C1);
2667 Constant *CmpC = ConstantExpr::getSub(C1: ShlC, C2: Log2C3);
2668 Value *Cmp = Builder.CreateICmpEQ(LHS: X, RHS: CmpC);
2669 return createSelectInstWithUnknownProfile(
2670 C: Cmp, S1: ConstantInt::get(Ty, V: *C3), S2: ConstantInt::getNullValue(Ty));
2671 }
2672 }
2673 }
2674 }
2675
2676 // If we are clearing the sign bit of a floating-point value, convert this to
2677 // fabs, then cast back to integer.
2678 //
2679 // This is a generous interpretation for noimplicitfloat, this is not a true
2680 // floating-point operation.
2681 //
2682 // Assumes any IEEE-represented type has the sign bit in the high bit.
2683 // TODO: Unify with APInt matcher. This version allows undef unlike m_APInt
2684 Value *CastOp;
2685 if (match(V: Op0, P: m_ElementWiseBitCast(Op: m_Value(V&: CastOp))) &&
2686 match(V: Op1, P: m_MaxSignedValue()) &&
2687 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
2688 Kind: Attribute::NoImplicitFloat)) {
2689 Type *EltTy = CastOp->getType()->getScalarType();
2690 if (EltTy->isFloatingPointTy() &&
2691 APFloat::hasSignBitInMSB(EltTy->getFltSemantics())) {
2692 Value *FAbs = Builder.CreateUnaryIntrinsic(ID: Intrinsic::fabs, V: CastOp);
2693 return new BitCastInst(FAbs, I.getType());
2694 }
2695 }
2696
2697 // and(shl(zext(X), Y), SignMask) -> and(sext(X), SignMask)
2698 // where Y is a valid shift amount.
2699 if (match(V: &I, P: m_And(L: m_OneUse(SubPattern: m_Shl(L: m_ZExt(Op: m_Value(V&: X)), R: m_Value(V&: Y))),
2700 R: m_SignMask())) &&
2701 match(V: Y, P: m_SpecificInt_ICMP(
2702 Predicate: ICmpInst::Predicate::ICMP_EQ,
2703 Threshold: APInt(Ty->getScalarSizeInBits(),
2704 Ty->getScalarSizeInBits() -
2705 X->getType()->getScalarSizeInBits())))) {
2706 auto *SExt = Builder.CreateSExt(V: X, DestTy: Ty, Name: X->getName() + ".signext");
2707 return BinaryOperator::CreateAnd(V1: SExt, V2: Op1);
2708 }
2709
2710 if (Instruction *Z = narrowMaskedBinOp(And&: I))
2711 return Z;
2712
2713 if (I.getType()->isIntOrIntVectorTy(BitWidth: 1)) {
2714 if (auto *SI0 = dyn_cast<SelectInst>(Val: Op0)) {
2715 if (auto *R =
2716 foldAndOrOfSelectUsingImpliedCond(Op: Op1, SI&: *SI0, /* IsAnd */ true))
2717 return R;
2718 }
2719 if (auto *SI1 = dyn_cast<SelectInst>(Val: Op1)) {
2720 if (auto *R =
2721 foldAndOrOfSelectUsingImpliedCond(Op: Op0, SI&: *SI1, /* IsAnd */ true))
2722 return R;
2723 }
2724 }
2725
2726 if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
2727 return FoldedLogic;
2728
2729 if (Instruction *DeMorgan = matchDeMorgansLaws(I, IC&: *this))
2730 return DeMorgan;
2731
2732 {
2733 Value *A, *B, *C;
2734 // A & ~(A ^ B) --> A & B
2735 if (match(V: Op1, P: m_Not(V: m_c_Xor(L: m_Specific(V: Op0), R: m_Value(V&: B)))))
2736 return BinaryOperator::CreateAnd(V1: Op0, V2: B);
2737 // ~(A ^ B) & A --> A & B
2738 if (match(V: Op0, P: m_Not(V: m_c_Xor(L: m_Specific(V: Op1), R: m_Value(V&: B)))))
2739 return BinaryOperator::CreateAnd(V1: Op1, V2: B);
2740
2741 // (A ^ B) & ((B ^ C) ^ A) -> (A ^ B) & ~C
2742 if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))) &&
2743 match(V: Op1, P: m_Xor(L: m_Xor(L: m_Specific(V: B), R: m_Value(V&: C)), R: m_Specific(V: A)))) {
2744 Value *NotC = Op1->hasOneUse()
2745 ? Builder.CreateNot(V: C)
2746 : getFreelyInverted(V: C, WillInvertAllUses: C->hasOneUse(), Builder: &Builder);
2747 if (NotC != nullptr)
2748 return BinaryOperator::CreateAnd(V1: Op0, V2: NotC);
2749 }
2750
2751 // ((A ^ C) ^ B) & (B ^ A) -> (B ^ A) & ~C
2752 if (match(V: Op0, P: m_Xor(L: m_Xor(L: m_Value(V&: A), R: m_Value(V&: C)), R: m_Value(V&: B))) &&
2753 match(V: Op1, P: m_Xor(L: m_Specific(V: B), R: m_Specific(V: A)))) {
2754 Value *NotC = Op0->hasOneUse()
2755 ? Builder.CreateNot(V: C)
2756 : getFreelyInverted(V: C, WillInvertAllUses: C->hasOneUse(), Builder: &Builder);
2757 if (NotC != nullptr)
2758 return BinaryOperator::CreateAnd(V1: Op1, V2: Builder.CreateNot(V: C));
2759 }
2760
2761 // (A | B) & (~A ^ B) -> A & B
2762 // (A | B) & (B ^ ~A) -> A & B
2763 // (B | A) & (~A ^ B) -> A & B
2764 // (B | A) & (B ^ ~A) -> A & B
2765 if (match(V: Op1, P: m_c_Xor(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2766 match(V: Op0, P: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))
2767 return BinaryOperator::CreateAnd(V1: A, V2: B);
2768
2769 // (~A ^ B) & (A | B) -> A & B
2770 // (~A ^ B) & (B | A) -> A & B
2771 // (B ^ ~A) & (A | B) -> A & B
2772 // (B ^ ~A) & (B | A) -> A & B
2773 if (match(V: Op0, P: m_c_Xor(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2774 match(V: Op1, P: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))
2775 return BinaryOperator::CreateAnd(V1: A, V2: B);
2776
2777 // (~A | B) & (A ^ B) -> ~A & B
2778 // (~A | B) & (B ^ A) -> ~A & B
2779 // (B | ~A) & (A ^ B) -> ~A & B
2780 // (B | ~A) & (B ^ A) -> ~A & B
2781 if (match(V: Op0, P: m_c_Or(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2782 match(V: Op1, P: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B))))
2783 return BinaryOperator::CreateAnd(V1: Builder.CreateNot(V: A), V2: B);
2784
2785 // (A ^ B) & (~A | B) -> ~A & B
2786 // (B ^ A) & (~A | B) -> ~A & B
2787 // (A ^ B) & (B | ~A) -> ~A & B
2788 // (B ^ A) & (B | ~A) -> ~A & B
2789 if (match(V: Op1, P: m_c_Or(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2790 match(V: Op0, P: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B))))
2791 return BinaryOperator::CreateAnd(V1: Builder.CreateNot(V: A), V2: B);
2792 }
2793
2794 if (Value *Res =
2795 foldBooleanAndOr(LHS: Op0, RHS: Op1, I, /*IsAnd=*/true, /*IsLogical=*/false))
2796 return replaceInstUsesWith(I, V: Res);
2797
2798 if (match(V: Op1, P: m_OneUse(SubPattern: m_LogicalAnd(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
2799 bool IsLogical = isa<SelectInst>(Val: Op1);
2800 if (auto *V = reassociateBooleanAndOr(LHS: Op0, X, Y, I, /*IsAnd=*/true,
2801 /*RHSIsLogical=*/IsLogical))
2802 return replaceInstUsesWith(I, V);
2803 }
2804 if (match(V: Op0, P: m_OneUse(SubPattern: m_LogicalAnd(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
2805 bool IsLogical = isa<SelectInst>(Val: Op0);
2806 if (auto *V = reassociateBooleanAndOr(LHS: Op1, X, Y, I, /*IsAnd=*/true,
2807 /*RHSIsLogical=*/IsLogical))
2808 return replaceInstUsesWith(I, V);
2809 }
2810
2811 if (Instruction *FoldedFCmps = reassociateFCmps(BO&: I, Builder))
2812 return FoldedFCmps;
2813
2814 if (Instruction *CastedAnd = foldCastedBitwiseLogic(I))
2815 return CastedAnd;
2816
2817 if (Instruction *Sel = foldBinopOfSextBoolToSelect(I))
2818 return Sel;
2819
2820 // and(sext(A), B) / and(B, sext(A)) --> A ? B : 0, where A is i1 or <N x i1>.
2821 // TODO: Move this into foldBinopOfSextBoolToSelect as a more generalized fold
2822 // with binop identity constant. But creating a select with non-constant
2823 // arm may not be reversible due to poison semantics. Is that a good
2824 // canonicalization?
2825 Value *A, *B;
2826 if (match(V: &I, P: m_c_And(L: m_SExt(Op: m_Value(V&: A)), R: m_Value(V&: B))) &&
2827 A->getType()->isIntOrIntVectorTy(BitWidth: 1))
2828 return createSelectInstWithUnknownProfile(C: A, S1: B, S2: Constant::getNullValue(Ty));
2829
2830 // Similarly, a 'not' of the bool translates to a swap of the select arms:
2831 // ~sext(A) & B / B & ~sext(A) --> A ? 0 : B
2832 if (match(V: &I, P: m_c_And(L: m_Not(V: m_SExt(Op: m_Value(V&: A))), R: m_Value(V&: B))) &&
2833 A->getType()->isIntOrIntVectorTy(BitWidth: 1))
2834 return createSelectInstWithUnknownProfile(C: A, S1: Constant::getNullValue(Ty), S2: B);
2835
2836 // and(zext(A), B) -> A ? (B & 1) : 0
2837 if (match(V: &I, P: m_c_And(L: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: A))), R: m_Value(V&: B))) &&
2838 A->getType()->isIntOrIntVectorTy(BitWidth: 1))
2839 return createSelectInstWithUnknownProfile(
2840 C: A, S1: Builder.CreateAnd(LHS: B, RHS: ConstantInt::get(Ty, V: 1)),
2841 S2: Constant::getNullValue(Ty));
2842
2843 // (-1 + A) & B --> A ? 0 : B where A is 0/1.
2844 if (match(V: &I, P: m_c_And(L: m_OneUse(SubPattern: m_Add(L: m_ZExtOrSelf(Op: m_Value(V&: A)), R: m_AllOnes())),
2845 R: m_Value(V&: B)))) {
2846 if (A->getType()->isIntOrIntVectorTy(BitWidth: 1))
2847 return createSelectInstWithUnknownProfile(C: A, S1: Constant::getNullValue(Ty),
2848 S2: B);
2849 if (computeKnownBits(V: A, CxtI: &I).countMaxActiveBits() <= 1) {
2850 return createSelectInstWithUnknownProfile(
2851 C: Builder.CreateICmpEQ(LHS: A, RHS: Constant::getNullValue(Ty: A->getType())), S1: B,
2852 S2: Constant::getNullValue(Ty));
2853 }
2854 }
2855
2856 // (iN X s>> (N-1)) & Y --> (X s< 0) ? Y : 0 -- with optional sext
2857 if (match(V: &I, P: m_c_And(L: m_OneUse(SubPattern: m_SExtOrSelf(
2858 Op: m_AShr(L: m_Value(V&: X), R: m_APIntAllowPoison(Res&: C)))),
2859 R: m_Value(V&: Y))) &&
2860 *C == X->getType()->getScalarSizeInBits() - 1) {
2861 Value *IsNeg = Builder.CreateIsNeg(Arg: X, Name: "isneg");
2862 return createSelectInstWithUnknownProfile(C: IsNeg, S1: Y,
2863 S2: ConstantInt::getNullValue(Ty));
2864 }
2865 // If there's a 'not' of the shifted value, swap the select operands:
2866 // ~(iN X s>> (N-1)) & Y --> (X s< 0) ? 0 : Y -- with optional sext
2867 if (match(V: &I, P: m_c_And(L: m_OneUse(SubPattern: m_SExtOrSelf(
2868 Op: m_Not(V: m_AShr(L: m_Value(V&: X), R: m_APIntAllowPoison(Res&: C))))),
2869 R: m_Value(V&: Y))) &&
2870 *C == X->getType()->getScalarSizeInBits() - 1) {
2871 Value *IsNeg = Builder.CreateIsNeg(Arg: X, Name: "isneg");
2872 return createSelectInstWithUnknownProfile(C: IsNeg,
2873 S1: ConstantInt::getNullValue(Ty), S2: Y);
2874 }
2875
2876 // (~x) & y --> ~(x | (~y)) iff that gets rid of inversions
2877 if (sinkNotIntoOtherHandOfLogicalOp(I))
2878 return &I;
2879
2880 // An and recurrence w/loop invariant step is equivelent to (and start, step)
2881 PHINode *PN = nullptr;
2882 Value *Start = nullptr, *Step = nullptr;
2883 if (matchSimpleRecurrence(I: &I, P&: PN, Start, Step) && DT.dominates(Def: Step, User: PN))
2884 return replaceInstUsesWith(I, V: Builder.CreateAnd(LHS: Start, RHS: Step));
2885
2886 if (Instruction *R = reassociateForUses(BO&: I, Builder))
2887 return R;
2888
2889 if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder))
2890 return Canonicalized;
2891
2892 if (Instruction *Folded = foldLogicOfIsFPClass(BO&: I, Op0, Op1))
2893 return Folded;
2894
2895 if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
2896 return Res;
2897
2898 if (Instruction *Res = foldBitwiseLogicWithIntrinsics(I, Builder))
2899 return Res;
2900
2901 if (Value *V =
2902 simplifyAndOrWithOpReplaced(V: Op0, Op: Op1, RepOp: Constant::getAllOnesValue(Ty),
2903 /*SimplifyOnly*/ false, IC&: *this))
2904 return BinaryOperator::CreateAnd(V1: V, V2: Op1);
2905 if (Value *V =
2906 simplifyAndOrWithOpReplaced(V: Op1, Op: Op0, RepOp: Constant::getAllOnesValue(Ty),
2907 /*SimplifyOnly*/ false, IC&: *this))
2908 return BinaryOperator::CreateAnd(V1: Op0, V2: V);
2909
2910 return nullptr;
2911}
2912
2913Instruction *InstCombinerImpl::matchBSwapOrBitReverse(Instruction &I,
2914 bool MatchBSwaps,
2915 bool MatchBitReversals) {
2916 SmallVector<Instruction *, 4> Insts;
2917 if (!recognizeBSwapOrBitReverseIdiom(I: &I, MatchBSwaps, MatchBitReversals,
2918 InsertedInsts&: Insts))
2919 return nullptr;
2920 Instruction *LastInst = Insts.pop_back_val();
2921 LastInst->removeFromParent();
2922
2923 for (auto *Inst : Insts) {
2924 Inst->setDebugLoc(I.getDebugLoc());
2925 Worklist.push(I: Inst);
2926 }
2927 return LastInst;
2928}
2929
2930std::optional<std::pair<Intrinsic::ID, SmallVector<Value *, 3>>>
2931InstCombinerImpl::convertOrOfShiftsToFunnelShift(Instruction &Or) {
2932 // TODO: Can we reduce the code duplication between this and the related
2933 // rotate matching code under visitSelect and visitTrunc?
2934 assert(Or.getOpcode() == BinaryOperator::Or && "Expecting or instruction");
2935
2936 unsigned Width = Or.getType()->getScalarSizeInBits();
2937
2938 Instruction *Or0, *Or1;
2939 if (!match(V: Or.getOperand(i: 0), P: m_Instruction(I&: Or0)) ||
2940 !match(V: Or.getOperand(i: 1), P: m_Instruction(I&: Or1)))
2941 return std::nullopt;
2942
2943 bool IsFshl = true; // Sub on LSHR.
2944 SmallVector<Value *, 3> FShiftArgs;
2945
2946 // First, find an or'd pair of opposite shifts:
2947 // or (lshr ShVal0, ShAmt0), (shl ShVal1, ShAmt1)
2948 if (isa<BinaryOperator>(Val: Or0) && isa<BinaryOperator>(Val: Or1)) {
2949 Value *ShVal0, *ShVal1, *ShAmt0, *ShAmt1;
2950 if (!match(V: Or0,
2951 P: m_OneUse(SubPattern: m_LogicalShift(L: m_Value(V&: ShVal0), R: m_Value(V&: ShAmt0)))) ||
2952 !match(V: Or1,
2953 P: m_OneUse(SubPattern: m_LogicalShift(L: m_Value(V&: ShVal1), R: m_Value(V&: ShAmt1)))) ||
2954 Or0->getOpcode() == Or1->getOpcode())
2955 return std::nullopt;
2956
2957 // Canonicalize to or(shl(ShVal0, ShAmt0), lshr(ShVal1, ShAmt1)).
2958 if (Or0->getOpcode() == BinaryOperator::LShr) {
2959 std::swap(a&: Or0, b&: Or1);
2960 std::swap(a&: ShVal0, b&: ShVal1);
2961 std::swap(a&: ShAmt0, b&: ShAmt1);
2962 }
2963 assert(Or0->getOpcode() == BinaryOperator::Shl &&
2964 Or1->getOpcode() == BinaryOperator::LShr &&
2965 "Illegal or(shift,shift) pair");
2966
2967 // Match the shift amount operands for a funnel shift pattern. This always
2968 // matches a subtraction on the R operand.
2969 auto matchShiftAmount = [&](Value *L, Value *R, unsigned Width) -> Value * {
2970 // Check for constant shift amounts that sum to the bitwidth.
2971 const APInt *LI, *RI;
2972 if (match(V: L, P: m_APIntAllowPoison(Res&: LI)) && match(V: R, P: m_APIntAllowPoison(Res&: RI)))
2973 if (LI->ult(RHS: Width) && RI->ult(RHS: Width) && (*LI + *RI) == Width)
2974 return ConstantInt::get(Ty: L->getType(), V: *LI);
2975
2976 Constant *LC, *RC;
2977 if (match(V: L, P: m_Constant(C&: LC)) && match(V: R, P: m_Constant(C&: RC)) &&
2978 match(V: L,
2979 P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_ULT, Threshold: APInt(Width, Width))) &&
2980 match(V: R,
2981 P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_ULT, Threshold: APInt(Width, Width))) &&
2982 match(V: ConstantExpr::getAdd(C1: LC, C2: RC), P: m_SpecificIntAllowPoison(V: Width)))
2983 return ConstantExpr::mergeUndefsWith(C: LC, Other: RC);
2984
2985 // (shl ShVal, X) | (lshr ShVal, (Width - x)) iff X < Width.
2986 // We limit this to X < Width in case the backend re-expands the
2987 // intrinsic, and has to reintroduce a shift modulo operation (InstCombine
2988 // might remove it after this fold). This still doesn't guarantee that the
2989 // final codegen will match this original pattern.
2990 if (match(V: R, P: m_OneUse(SubPattern: m_Sub(L: m_SpecificInt(V: Width), R: m_Specific(V: L))))) {
2991 KnownBits KnownL = computeKnownBits(V: L, CxtI: &Or);
2992 return KnownL.getMaxValue().ult(RHS: Width) ? L : nullptr;
2993 }
2994
2995 // For non-constant cases, the following patterns currently only work for
2996 // rotation patterns.
2997 // TODO: Add general funnel-shift compatible patterns.
2998 if (ShVal0 != ShVal1)
2999 return nullptr;
3000
3001 // For non-constant cases we don't support non-pow2 shift masks.
3002 // TODO: Is it worth matching urem as well?
3003 if (!isPowerOf2_32(Value: Width))
3004 return nullptr;
3005
3006 // The shift amount may be masked with negation:
3007 // (shl ShVal, (X & (Width - 1))) | (lshr ShVal, ((-X) & (Width - 1)))
3008 Value *X;
3009 unsigned Mask = Width - 1;
3010 if (match(V: L, P: m_And(L: m_Value(V&: X), R: m_SpecificInt(V: Mask))) &&
3011 match(V: R, P: m_And(L: m_Neg(V: m_Specific(V: X)), R: m_SpecificInt(V: Mask))))
3012 return X;
3013
3014 // (shl ShVal, X) | (lshr ShVal, ((-X) & (Width - 1)))
3015 if (match(V: R, P: m_And(L: m_Neg(V: m_Specific(V: L)), R: m_SpecificInt(V: Mask))))
3016 return L;
3017
3018 // Similar to above, but the shift amount may be extended after masking,
3019 // so return the extended value as the parameter for the intrinsic.
3020 if (match(V: L, P: m_ZExt(Op: m_And(L: m_Value(V&: X), R: m_SpecificInt(V: Mask)))) &&
3021 match(V: R,
3022 P: m_And(L: m_Neg(V: m_ZExt(Op: m_And(L: m_Specific(V: X), R: m_SpecificInt(V: Mask)))),
3023 R: m_SpecificInt(V: Mask))))
3024 return L;
3025
3026 if (match(V: L, P: m_ZExt(Op: m_And(L: m_Value(V&: X), R: m_SpecificInt(V: Mask)))) &&
3027 match(V: R, P: m_ZExt(Op: m_And(L: m_Neg(V: m_Specific(V: X)), R: m_SpecificInt(V: Mask)))))
3028 return L;
3029
3030 return nullptr;
3031 };
3032
3033 Value *ShAmt = matchShiftAmount(ShAmt0, ShAmt1, Width);
3034 if (!ShAmt) {
3035 ShAmt = matchShiftAmount(ShAmt1, ShAmt0, Width);
3036 IsFshl = false; // Sub on SHL.
3037 }
3038 if (!ShAmt)
3039 return std::nullopt;
3040
3041 FShiftArgs = {ShVal0, ShVal1, ShAmt};
3042 } else if (isa<ZExtInst>(Val: Or0) || isa<ZExtInst>(Val: Or1)) {
3043 // If there are two 'or' instructions concat variables in opposite order:
3044 //
3045 // Slot1 and Slot2 are all zero bits.
3046 // | Slot1 | Low | Slot2 | High |
3047 // LowHigh = or (shl (zext Low), ZextLowShlAmt), (zext High)
3048 // | Slot2 | High | Slot1 | Low |
3049 // HighLow = or (shl (zext High), ZextHighShlAmt), (zext Low)
3050 //
3051 // the latter 'or' can be safely convert to
3052 // -> HighLow = fshl LowHigh, LowHigh, ZextHighShlAmt
3053 // if ZextLowShlAmt + ZextHighShlAmt == Width.
3054 if (!isa<ZExtInst>(Val: Or1))
3055 std::swap(a&: Or0, b&: Or1);
3056
3057 Value *High, *ZextHigh, *Low;
3058 const APInt *ZextHighShlAmt;
3059 if (!match(V: Or0,
3060 P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: ZextHigh), R: m_APInt(Res&: ZextHighShlAmt)))))
3061 return std::nullopt;
3062
3063 if (!match(V: Or1, P: m_ZExt(Op: m_Value(V&: Low))) ||
3064 !match(V: ZextHigh, P: m_ZExt(Op: m_Value(V&: High))))
3065 return std::nullopt;
3066
3067 unsigned HighSize = High->getType()->getScalarSizeInBits();
3068 unsigned LowSize = Low->getType()->getScalarSizeInBits();
3069 // Make sure High does not overlap with Low and most significant bits of
3070 // High aren't shifted out.
3071 if (ZextHighShlAmt->ult(RHS: LowSize) || ZextHighShlAmt->ugt(RHS: Width - HighSize))
3072 return std::nullopt;
3073
3074 for (User *U : ZextHigh->users()) {
3075 Value *X, *Y;
3076 if (!match(V: U, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))))
3077 continue;
3078
3079 if (!isa<ZExtInst>(Val: Y))
3080 std::swap(a&: X, b&: Y);
3081
3082 const APInt *ZextLowShlAmt;
3083 if (!match(V: X, P: m_Shl(L: m_Specific(V: Or1), R: m_APInt(Res&: ZextLowShlAmt))) ||
3084 !match(V: Y, P: m_Specific(V: ZextHigh)) || !DT.dominates(Def: U, User: &Or))
3085 continue;
3086
3087 // HighLow is good concat. If sum of two shifts amount equals to Width,
3088 // LowHigh must also be a good concat.
3089 if (*ZextLowShlAmt + *ZextHighShlAmt != Width)
3090 continue;
3091
3092 // Low must not overlap with High and most significant bits of Low must
3093 // not be shifted out.
3094 assert(ZextLowShlAmt->uge(HighSize) &&
3095 ZextLowShlAmt->ule(Width - LowSize) && "Invalid concat");
3096
3097 // We cannot reuse the result if it may produce poison.
3098 // Drop poison generating flags in the expression tree.
3099 // Or
3100 cast<Instruction>(Val: U)->dropPoisonGeneratingFlags();
3101 // Shl
3102 cast<Instruction>(Val: X)->dropPoisonGeneratingFlags();
3103
3104 FShiftArgs = {U, U, ConstantInt::get(Ty: Or0->getType(), V: *ZextHighShlAmt)};
3105 break;
3106 }
3107 }
3108
3109 if (FShiftArgs.empty())
3110 return std::nullopt;
3111
3112 Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
3113 return std::make_pair(x&: IID, y&: FShiftArgs);
3114}
3115
3116/// Match UB-safe variants of the funnel shift intrinsic.
3117static Instruction *matchFunnelShift(Instruction &Or, InstCombinerImpl &IC) {
3118 if (auto Opt = IC.convertOrOfShiftsToFunnelShift(Or)) {
3119 auto [IID, FShiftArgs] = *Opt;
3120 Function *F =
3121 Intrinsic::getOrInsertDeclaration(M: Or.getModule(), id: IID, Tys: Or.getType());
3122 return CallInst::Create(Func: F, Args: FShiftArgs);
3123 }
3124
3125 return nullptr;
3126}
3127
3128/// Attempt to combine or(zext(x),shl(zext(y),bw/2) concat packing patterns.
3129static Value *matchOrConcat(Instruction &Or, InstCombiner::BuilderTy &Builder) {
3130 assert(Or.getOpcode() == Instruction::Or && "bswap requires an 'or'");
3131 Value *Op0 = Or.getOperand(i: 0), *Op1 = Or.getOperand(i: 1);
3132 Type *Ty = Or.getType();
3133
3134 unsigned Width = Ty->getScalarSizeInBits();
3135 if ((Width & 1) != 0)
3136 return nullptr;
3137 unsigned HalfWidth = Width / 2;
3138
3139 // Canonicalize zext (lower half) to LHS.
3140 if (!isa<ZExtInst>(Val: Op0))
3141 std::swap(a&: Op0, b&: Op1);
3142
3143 // Find lower/upper half.
3144 Value *LowerSrc, *ShlVal, *UpperSrc;
3145 const APInt *C;
3146 if (!match(V: Op0, P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: LowerSrc)))) ||
3147 !match(V: Op1, P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: ShlVal), R: m_APInt(Res&: C)))) ||
3148 !match(V: ShlVal, P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: UpperSrc)))))
3149 return nullptr;
3150 if (*C != HalfWidth || LowerSrc->getType() != UpperSrc->getType() ||
3151 LowerSrc->getType()->getScalarSizeInBits() != HalfWidth)
3152 return nullptr;
3153
3154 auto ConcatIntrinsicCalls = [&](Intrinsic::ID id, Value *Lo, Value *Hi) {
3155 Value *NewLower = Builder.CreateZExt(V: Lo, DestTy: Ty);
3156 Value *NewUpper = Builder.CreateZExt(V: Hi, DestTy: Ty);
3157 NewUpper = Builder.CreateShl(LHS: NewUpper, RHS: HalfWidth);
3158 Value *BinOp = Builder.CreateOr(LHS: NewLower, RHS: NewUpper);
3159 return Builder.CreateIntrinsic(ID: id, Types: Ty, Args: BinOp);
3160 };
3161
3162 // BSWAP: Push the concat down, swapping the lower/upper sources.
3163 // concat(bswap(x),bswap(y)) -> bswap(concat(x,y))
3164 Value *LowerBSwap, *UpperBSwap;
3165 if (match(V: LowerSrc, P: m_BSwap(Op0: m_Value(V&: LowerBSwap))) &&
3166 match(V: UpperSrc, P: m_BSwap(Op0: m_Value(V&: UpperBSwap))))
3167 return ConcatIntrinsicCalls(Intrinsic::bswap, UpperBSwap, LowerBSwap);
3168
3169 // BITREVERSE: Push the concat down, swapping the lower/upper sources.
3170 // concat(bitreverse(x),bitreverse(y)) -> bitreverse(concat(x,y))
3171 Value *LowerBRev, *UpperBRev;
3172 if (match(V: LowerSrc, P: m_BitReverse(Op0: m_Value(V&: LowerBRev))) &&
3173 match(V: UpperSrc, P: m_BitReverse(Op0: m_Value(V&: UpperBRev))))
3174 return ConcatIntrinsicCalls(Intrinsic::bitreverse, UpperBRev, LowerBRev);
3175
3176 // iX ext split: extending or(zext(x),shl(zext(y),bw/2) pattern
3177 // to consume sext/ashr:
3178 // or(zext(sext(x)),shl(zext(sext(ashr(x,xbw-1))),bw/2)
3179 // or(zext(x),shl(zext(ashr(x,xbw-1)),bw/2)
3180 Value *X;
3181 if (match(V: LowerSrc, P: m_SExtOrSelf(Op: m_Value(V&: X))) &&
3182 match(V: UpperSrc,
3183 P: m_SExtOrSelf(Op: m_AShr(
3184 L: m_Specific(V: X),
3185 R: m_SpecificInt(V: X->getType()->getScalarSizeInBits() - 1)))))
3186 return Builder.CreateSExt(V: X, DestTy: Ty);
3187
3188 return nullptr;
3189}
3190
3191/// If all elements of two constant vectors are 0/-1 and inverses, return true.
3192static bool areInverseVectorBitmasks(Constant *C1, Constant *C2) {
3193 unsigned NumElts = cast<FixedVectorType>(Val: C1->getType())->getNumElements();
3194 for (unsigned i = 0; i != NumElts; ++i) {
3195 Constant *EltC1 = C1->getAggregateElement(Elt: i);
3196 Constant *EltC2 = C2->getAggregateElement(Elt: i);
3197 if (!EltC1 || !EltC2)
3198 return false;
3199
3200 // One element must be all ones, and the other must be all zeros.
3201 if (!((match(V: EltC1, P: m_Zero()) && match(V: EltC2, P: m_AllOnes())) ||
3202 (match(V: EltC2, P: m_Zero()) && match(V: EltC1, P: m_AllOnes()))))
3203 return false;
3204 }
3205 return true;
3206}
3207
3208/// We have an expression of the form (A & C) | (B & D). If A is a scalar or
3209/// vector composed of all-zeros or all-ones values and is the bitwise 'not' of
3210/// B, it can be used as the condition operand of a select instruction.
3211/// We will detect (A & C) | ~(B | D) when the flag ABIsTheSame enabled.
3212Value *InstCombinerImpl::getSelectCondition(Value *A, Value *B,
3213 bool ABIsTheSame) {
3214 // We may have peeked through bitcasts in the caller.
3215 // Exit immediately if we don't have (vector) integer types.
3216 Type *Ty = A->getType();
3217 if (!Ty->isIntOrIntVectorTy() || !B->getType()->isIntOrIntVectorTy())
3218 return nullptr;
3219
3220 // If A is the 'not' operand of B and has enough signbits, we have our answer.
3221 if (ABIsTheSame ? (A == B) : match(V: B, P: m_Not(V: m_Specific(V: A)))) {
3222 // If these are scalars or vectors of i1, A can be used directly.
3223 if (Ty->isIntOrIntVectorTy(BitWidth: 1))
3224 return A;
3225
3226 // If we look through a vector bitcast, the caller will bitcast the operands
3227 // to match the condition's number of bits (N x i1).
3228 // To make this poison-safe, disallow bitcast from wide element to narrow
3229 // element. That could allow poison in lanes where it was not present in the
3230 // original code.
3231 A = peekThroughBitcast(V: A);
3232 if (A->getType()->isIntOrIntVectorTy()) {
3233 unsigned NumSignBits = ComputeNumSignBits(Op: A);
3234 if (NumSignBits == A->getType()->getScalarSizeInBits() &&
3235 NumSignBits <= Ty->getScalarSizeInBits())
3236 return Builder.CreateTrunc(V: A, DestTy: CmpInst::makeCmpResultType(opnd_type: A->getType()));
3237 }
3238 return nullptr;
3239 }
3240
3241 // TODO: add support for sext and constant case
3242 if (ABIsTheSame)
3243 return nullptr;
3244
3245 // If both operands are constants, see if the constants are inverse bitmasks.
3246 Constant *AConst, *BConst;
3247 if (match(V: A, P: m_Constant(C&: AConst)) && match(V: B, P: m_Constant(C&: BConst)))
3248 if (AConst == ConstantExpr::getNot(C: BConst) &&
3249 ComputeNumSignBits(Op: A) == Ty->getScalarSizeInBits())
3250 return Builder.CreateZExtOrTrunc(V: A, DestTy: CmpInst::makeCmpResultType(opnd_type: Ty));
3251
3252 // Look for more complex patterns. The 'not' op may be hidden behind various
3253 // casts. Look through sexts and bitcasts to find the booleans.
3254 Value *Cond;
3255 Value *NotB;
3256 if (match(V: A, P: m_SExt(Op: m_Value(V&: Cond))) &&
3257 Cond->getType()->isIntOrIntVectorTy(BitWidth: 1)) {
3258 // A = sext i1 Cond; B = sext (not (i1 Cond))
3259 if (match(V: B, P: m_SExt(Op: m_Not(V: m_Specific(V: Cond)))))
3260 return Cond;
3261
3262 // A = sext i1 Cond; B = not ({bitcast} (sext (i1 Cond)))
3263 // TODO: The one-use checks are unnecessary or misplaced. If the caller
3264 // checked for uses on logic ops/casts, that should be enough to
3265 // make this transform worthwhile.
3266 if (match(V: B, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotB))))) {
3267 NotB = peekThroughBitcast(V: NotB, OneUseOnly: true);
3268 if (match(V: NotB, P: m_SExt(Op: m_Specific(V: Cond))))
3269 return Cond;
3270 }
3271 }
3272
3273 // All scalar (and most vector) possibilities should be handled now.
3274 // Try more matches that only apply to non-splat constant vectors.
3275 if (!Ty->isVectorTy())
3276 return nullptr;
3277
3278 // If both operands are xor'd with constants using the same sexted boolean
3279 // operand, see if the constants are inverse bitmasks.
3280 // TODO: Use ConstantExpr::getNot()?
3281 if (match(V: A, P: (m_Xor(L: m_SExt(Op: m_Value(V&: Cond)), R: m_Constant(C&: AConst)))) &&
3282 match(V: B, P: (m_Xor(L: m_SExt(Op: m_Specific(V: Cond)), R: m_Constant(C&: BConst)))) &&
3283 Cond->getType()->isIntOrIntVectorTy(BitWidth: 1) &&
3284 areInverseVectorBitmasks(C1: AConst, C2: BConst)) {
3285 AConst = ConstantExpr::getTrunc(C: AConst, Ty: CmpInst::makeCmpResultType(opnd_type: Ty));
3286 return Builder.CreateXor(LHS: Cond, RHS: AConst);
3287 }
3288 return nullptr;
3289}
3290
3291/// We have an expression of the form (A & B) | (C & D). Try to simplify this
3292/// to "A' ? B : D", where A' is a boolean or vector of booleans.
3293/// When InvertFalseVal is set to true, we try to match the pattern
3294/// where we have peeked through a 'not' op and A and C are the same:
3295/// (A & B) | ~(A | D) --> (A & B) | (~A & ~D) --> A' ? B : ~D
3296Value *InstCombinerImpl::matchSelectFromAndOr(Value *A, Value *B, Value *C,
3297 Value *D, bool InvertFalseVal) {
3298 // The potential condition of the select may be bitcasted. In that case, look
3299 // through its bitcast and the corresponding bitcast of the 'not' condition.
3300 Type *OrigType = A->getType();
3301 A = peekThroughBitcast(V: A, OneUseOnly: true);
3302 C = peekThroughBitcast(V: C, OneUseOnly: true);
3303 if (Value *Cond = getSelectCondition(A, B: C, ABIsTheSame: InvertFalseVal)) {
3304 // ((bc Cond) & B) | ((bc ~Cond) & D) --> bc (select Cond, (bc B), (bc D))
3305 // If this is a vector, we may need to cast to match the condition's length.
3306 // The bitcasts will either all exist or all not exist. The builder will
3307 // not create unnecessary casts if the types already match.
3308 Type *SelTy = A->getType();
3309 if (auto *VecTy = dyn_cast<VectorType>(Val: Cond->getType())) {
3310 // For a fixed or scalable vector get N from <{vscale x} N x iM>
3311 unsigned Elts = VecTy->getElementCount().getKnownMinValue();
3312 // For a fixed or scalable vector, get the size in bits of N x iM; for a
3313 // scalar this is just M.
3314 unsigned SelEltSize = SelTy->getPrimitiveSizeInBits().getKnownMinValue();
3315 Type *EltTy = Builder.getIntNTy(N: SelEltSize / Elts);
3316 SelTy = VectorType::get(ElementType: EltTy, EC: VecTy->getElementCount());
3317 }
3318 Value *BitcastB = Builder.CreateBitCast(V: B, DestTy: SelTy);
3319 if (InvertFalseVal)
3320 D = Builder.CreateNot(V: D);
3321 Value *BitcastD = Builder.CreateBitCast(V: D, DestTy: SelTy);
3322 Value *Select = Builder.CreateSelect(C: Cond, True: BitcastB, False: BitcastD);
3323 return Builder.CreateBitCast(V: Select, DestTy: OrigType);
3324 }
3325
3326 return nullptr;
3327}
3328
3329// (icmp eq X, C) | (icmp ult Other, (X - C)) -> (icmp ule Other, (X - (C + 1)))
3330// (icmp ne X, C) & (icmp uge Other, (X - C)) -> (icmp ugt Other, (X - (C + 1)))
3331static Value *foldAndOrOfICmpEqConstantAndICmp(ICmpInst *LHS, ICmpInst *RHS,
3332 bool IsAnd, bool IsLogical,
3333 IRBuilderBase &Builder) {
3334 Value *LHS0 = LHS->getOperand(i_nocapture: 0);
3335 Value *RHS0 = RHS->getOperand(i_nocapture: 0);
3336 Value *RHS1 = RHS->getOperand(i_nocapture: 1);
3337
3338 ICmpInst::Predicate LPred =
3339 IsAnd ? LHS->getInversePredicate() : LHS->getPredicate();
3340 ICmpInst::Predicate RPred =
3341 IsAnd ? RHS->getInversePredicate() : RHS->getPredicate();
3342
3343 const APInt *CInt;
3344 if (LPred != ICmpInst::ICMP_EQ ||
3345 !match(V: LHS->getOperand(i_nocapture: 1), P: m_APIntAllowPoison(Res&: CInt)) ||
3346 !LHS0->getType()->isIntOrIntVectorTy() ||
3347 !(LHS->hasOneUse() || RHS->hasOneUse()))
3348 return nullptr;
3349
3350 auto MatchRHSOp = [LHS0, CInt](const Value *RHSOp) {
3351 return match(V: RHSOp,
3352 P: m_Add(L: m_Specific(V: LHS0), R: m_SpecificIntAllowPoison(V: -*CInt))) ||
3353 (CInt->isZero() && RHSOp == LHS0);
3354 };
3355
3356 Value *Other;
3357 if (RPred == ICmpInst::ICMP_ULT && MatchRHSOp(RHS1))
3358 Other = RHS0;
3359 else if (RPred == ICmpInst::ICMP_UGT && MatchRHSOp(RHS0))
3360 Other = RHS1;
3361 else
3362 return nullptr;
3363
3364 if (IsLogical)
3365 Other = Builder.CreateFreeze(V: Other);
3366
3367 return Builder.CreateICmp(
3368 P: IsAnd ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE,
3369 LHS: Builder.CreateSub(LHS: LHS0, RHS: ConstantInt::get(Ty: LHS0->getType(), V: *CInt + 1)),
3370 RHS: Other);
3371}
3372
3373/// Fold (icmp)&(icmp) or (icmp)|(icmp) if possible.
3374/// If IsLogical is true, then the and/or is in select form and the transform
3375/// must be poison-safe.
3376Value *InstCombinerImpl::foldAndOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
3377 Instruction &I, bool IsAnd,
3378 bool IsLogical) {
3379 const SimplifyQuery Q = SQ.getWithInstruction(I: &I);
3380
3381 ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
3382 Value *LHS0 = LHS->getOperand(i_nocapture: 0), *RHS0 = RHS->getOperand(i_nocapture: 0);
3383 Value *LHS1 = LHS->getOperand(i_nocapture: 1), *RHS1 = RHS->getOperand(i_nocapture: 1);
3384
3385 const APInt *LHSC = nullptr, *RHSC = nullptr;
3386 match(V: LHS1, P: m_APInt(Res&: LHSC));
3387 match(V: RHS1, P: m_APInt(Res&: RHSC));
3388
3389 // (icmp1 A, B) | (icmp2 A, B) --> (icmp3 A, B)
3390 // (icmp1 A, B) & (icmp2 A, B) --> (icmp3 A, B)
3391 if (predicatesFoldable(P1: PredL, P2: PredR)) {
3392 if (LHS0 == RHS1 && LHS1 == RHS0) {
3393 PredL = ICmpInst::getSwappedPredicate(pred: PredL);
3394 std::swap(a&: LHS0, b&: LHS1);
3395 }
3396 if (LHS0 == RHS0 && LHS1 == RHS1) {
3397 unsigned Code = IsAnd ? getICmpCode(Pred: PredL) & getICmpCode(Pred: PredR)
3398 : getICmpCode(Pred: PredL) | getICmpCode(Pred: PredR);
3399 bool IsSigned = LHS->isSigned() || RHS->isSigned();
3400 return getNewICmpValue(Code, Sign: IsSigned, LHS: LHS0, RHS: LHS1, Builder);
3401 }
3402 }
3403
3404 if (Value *V =
3405 foldAndOrOfICmpEqConstantAndICmp(LHS, RHS, IsAnd, IsLogical, Builder))
3406 return V;
3407 // We can treat logical like bitwise here, because both operands are used on
3408 // the LHS, and as such poison from both will propagate.
3409 if (Value *V = foldAndOrOfICmpEqConstantAndICmp(LHS: RHS, RHS: LHS, IsAnd,
3410 /*IsLogical*/ false, Builder))
3411 return V;
3412
3413 if (Value *V = foldAndOrOfICmpsWithConstEq(Cmp0: LHS, Cmp1: RHS, IsAnd, IsLogical,
3414 Builder, Q, I))
3415 return V;
3416 // We can convert this case to bitwise and, because both operands are used
3417 // on the LHS, and as such poison from both will propagate.
3418 if (Value *V = foldAndOrOfICmpsWithConstEq(
3419 Cmp0: RHS, Cmp1: LHS, IsAnd, /*IsLogical=*/false, Builder, Q, I)) {
3420 // If RHS is still used, we should drop samesign flag.
3421 if (IsLogical && RHS->hasSameSign() && !RHS->use_empty()) {
3422 RHS->setSameSign(false);
3423 addToWorklist(I: RHS);
3424 }
3425 return V;
3426 }
3427
3428 if (Value *V = foldIsPowerOf2OrZero(Cmp0: LHS, Cmp1: RHS, IsAnd, Builder, IC&: *this))
3429 return V;
3430 if (Value *V = foldIsPowerOf2OrZero(Cmp0: RHS, Cmp1: LHS, IsAnd, Builder, IC&: *this))
3431 return V;
3432
3433 // TODO: One of these directions is fine with logical and/or, the other could
3434 // be supported by inserting freeze.
3435 if (!IsLogical) {
3436 // E.g. (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n
3437 // E.g. (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n
3438 if (Value *V = simplifyRangeCheck(Cmp0: LHS, Cmp1: RHS, /*Inverted=*/!IsAnd))
3439 return V;
3440
3441 // E.g. (icmp sgt x, n) | (icmp slt x, 0) --> icmp ugt x, n
3442 // E.g. (icmp slt x, n) & (icmp sge x, 0) --> icmp ult x, n
3443 if (Value *V = simplifyRangeCheck(Cmp0: RHS, Cmp1: LHS, /*Inverted=*/!IsAnd))
3444 return V;
3445 }
3446
3447 // TODO: Add conjugated or fold, check whether it is safe for logical and/or.
3448 if (IsAnd && !IsLogical)
3449 if (Value *V = foldSignedTruncationCheck(ICmp0: LHS, ICmp1: RHS, CxtI&: I, Builder))
3450 return V;
3451
3452 if (Value *V = foldIsPowerOf2(Cmp0: LHS, Cmp1: RHS, JoinedByAnd: IsAnd, Builder, IC&: *this))
3453 return V;
3454
3455 if (Value *V = foldPowerOf2AndShiftedMask(Cmp0: LHS, Cmp1: RHS, JoinedByAnd: IsAnd, Builder))
3456 return V;
3457
3458 // TODO: Verify whether this is safe for logical and/or.
3459 if (!IsLogical) {
3460 if (Value *X = foldUnsignedUnderflowCheck(ZeroICmp: LHS, UnsignedICmp: RHS, IsAnd, Q, Builder))
3461 return X;
3462 if (Value *X = foldUnsignedUnderflowCheck(ZeroICmp: RHS, UnsignedICmp: LHS, IsAnd, Q, Builder))
3463 return X;
3464 }
3465
3466 // (icmp ne A, 0) | (icmp ne B, 0) --> (icmp ne (A|B), 0)
3467 // (icmp eq A, 0) & (icmp eq B, 0) --> (icmp eq (A|B), 0)
3468 // TODO: Remove this and below when foldLogOpOfMaskedICmps can handle undefs.
3469 if (PredL == (IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE) &&
3470 PredL == PredR && match(V: LHS1, P: m_ZeroInt()) && match(V: RHS1, P: m_ZeroInt()) &&
3471 LHS0->getType() == RHS0->getType() &&
3472 (!IsLogical || isGuaranteedNotToBePoison(V: RHS0))) {
3473 Value *NewOr = Builder.CreateOr(LHS: LHS0, RHS: RHS0);
3474 return Builder.CreateICmp(P: PredL, LHS: NewOr,
3475 RHS: Constant::getNullValue(Ty: NewOr->getType()));
3476 }
3477
3478 // (icmp ne A, -1) | (icmp ne B, -1) --> (icmp ne (A&B), -1)
3479 // (icmp eq A, -1) & (icmp eq B, -1) --> (icmp eq (A&B), -1)
3480 if (PredL == (IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE) &&
3481 PredL == PredR && match(V: LHS1, P: m_AllOnes()) && match(V: RHS1, P: m_AllOnes()) &&
3482 LHS0->getType() == RHS0->getType() &&
3483 (!IsLogical || isGuaranteedNotToBePoison(V: RHS0))) {
3484 Value *NewAnd = Builder.CreateAnd(LHS: LHS0, RHS: RHS0);
3485 return Builder.CreateICmp(P: PredL, LHS: NewAnd,
3486 RHS: Constant::getAllOnesValue(Ty: LHS0->getType()));
3487 }
3488
3489 if (!IsLogical)
3490 if (Value *V =
3491 foldAndOrOfICmpsWithPow2AndWithZero(Builder, LHS, RHS, IsAnd, Q))
3492 return V;
3493
3494 // This only handles icmp of constants: (icmp1 A, C1) | (icmp2 B, C2).
3495 if (!LHSC || !RHSC)
3496 return nullptr;
3497
3498 // (trunc x) == C1 & (and x, CA) == C2 -> (and x, CA|CMAX) == C1|C2
3499 // (trunc x) != C1 | (and x, CA) != C2 -> (and x, CA|CMAX) != C1|C2
3500 // where CMAX is the all ones value for the truncated type,
3501 // iff the lower bits of C2 and CA are zero.
3502 if (PredL == (IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE) &&
3503 PredL == PredR && LHS->hasOneUse() && RHS->hasOneUse()) {
3504 Value *V;
3505 const APInt *AndC, *SmallC = nullptr, *BigC = nullptr;
3506
3507 // (trunc x) == C1 & (and x, CA) == C2
3508 // (and x, CA) == C2 & (trunc x) == C1
3509 if (match(V: RHS0, P: m_Trunc(Op: m_Value(V))) &&
3510 match(V: LHS0, P: m_And(L: m_Specific(V), R: m_APInt(Res&: AndC)))) {
3511 SmallC = RHSC;
3512 BigC = LHSC;
3513 } else if (match(V: LHS0, P: m_Trunc(Op: m_Value(V))) &&
3514 match(V: RHS0, P: m_And(L: m_Specific(V), R: m_APInt(Res&: AndC)))) {
3515 SmallC = LHSC;
3516 BigC = RHSC;
3517 }
3518
3519 if (SmallC && BigC) {
3520 unsigned BigBitSize = BigC->getBitWidth();
3521 unsigned SmallBitSize = SmallC->getBitWidth();
3522
3523 // Check that the low bits are zero.
3524 APInt Low = APInt::getLowBitsSet(numBits: BigBitSize, loBitsSet: SmallBitSize);
3525 if ((Low & *AndC).isZero() && (Low & *BigC).isZero()) {
3526 Value *NewAnd = Builder.CreateAnd(LHS: V, RHS: Low | *AndC);
3527 APInt N = SmallC->zext(width: BigBitSize) | *BigC;
3528 Value *NewVal = ConstantInt::get(Ty: NewAnd->getType(), V: N);
3529 return Builder.CreateICmp(P: PredL, LHS: NewAnd, RHS: NewVal);
3530 }
3531 }
3532 }
3533
3534 // Match naive pattern (and its inverted form) for checking if two values
3535 // share same sign. An example of the pattern:
3536 // (icmp slt (X & Y), 0) | (icmp sgt (X | Y), -1) -> (icmp sgt (X ^ Y), -1)
3537 // Inverted form (example):
3538 // (icmp slt (X | Y), 0) & (icmp sgt (X & Y), -1) -> (icmp slt (X ^ Y), 0)
3539 bool TrueIfSignedL, TrueIfSignedR;
3540 if (isSignBitCheck(Pred: PredL, RHS: *LHSC, TrueIfSigned&: TrueIfSignedL) &&
3541 isSignBitCheck(Pred: PredR, RHS: *RHSC, TrueIfSigned&: TrueIfSignedR) &&
3542 (RHS->hasOneUse() || LHS->hasOneUse())) {
3543 Value *X, *Y;
3544 if (IsAnd) {
3545 if ((TrueIfSignedL && !TrueIfSignedR &&
3546 match(V: LHS0, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
3547 match(V: RHS0, P: m_c_And(L: m_Specific(V: X), R: m_Specific(V: Y)))) ||
3548 (!TrueIfSignedL && TrueIfSignedR &&
3549 match(V: LHS0, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
3550 match(V: RHS0, P: m_c_Or(L: m_Specific(V: X), R: m_Specific(V: Y))))) {
3551 Value *NewXor = Builder.CreateXor(LHS: X, RHS: Y);
3552 return Builder.CreateIsNeg(Arg: NewXor);
3553 }
3554 } else {
3555 if ((TrueIfSignedL && !TrueIfSignedR &&
3556 match(V: LHS0, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
3557 match(V: RHS0, P: m_c_Or(L: m_Specific(V: X), R: m_Specific(V: Y)))) ||
3558 (!TrueIfSignedL && TrueIfSignedR &&
3559 match(V: LHS0, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
3560 match(V: RHS0, P: m_c_And(L: m_Specific(V: X), R: m_Specific(V: Y))))) {
3561 Value *NewXor = Builder.CreateXor(LHS: X, RHS: Y);
3562 return Builder.CreateIsNotNeg(Arg: NewXor);
3563 }
3564 }
3565 }
3566
3567 // (X & ExpMask) != 0 && (X & ExpMask) != ExpMask -> isnormal(X)
3568 // (X & ExpMask) == 0 || (X & ExpMask) == ExpMask -> !isnormal(X)
3569 Value *X;
3570 const APInt *MaskC;
3571 if (LHS0 == RHS0 && PredL == PredR &&
3572 PredL == (IsAnd ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ) &&
3573 !I.getFunction()->hasFnAttribute(Kind: Attribute::NoImplicitFloat) &&
3574 LHS->hasOneUse() && RHS->hasOneUse() &&
3575 match(V: LHS0, P: m_And(L: m_ElementWiseBitCast(Op: m_Value(V&: X)), R: m_APInt(Res&: MaskC))) &&
3576 X->getType()->getScalarType()->isIEEELikeFPTy() &&
3577 APFloat(X->getType()->getScalarType()->getFltSemantics(), *MaskC)
3578 .isPosInfinity() &&
3579 ((LHSC->isZero() && *RHSC == *MaskC) ||
3580 (RHSC->isZero() && *LHSC == *MaskC)))
3581 return Builder.createIsFPClass(FPNum: X, Test: IsAnd ? FPClassTest::fcNormal
3582 : ~FPClassTest::fcNormal);
3583
3584 return foldAndOrOfICmpsUsingRanges(ICmp1: LHS, ICmp2: RHS, IsAnd);
3585}
3586
3587/// If IsLogical is true, then the and/or is in select form and the transform
3588/// must be poison-safe.
3589Value *InstCombinerImpl::foldBooleanAndOr(Value *LHS, Value *RHS,
3590 Instruction &I, bool IsAnd,
3591 bool IsLogical) {
3592 if (!LHS->getType()->isIntOrIntVectorTy(BitWidth: 1))
3593 return nullptr;
3594
3595 // handle (roughly):
3596 // (icmp ne (A & B), C) | (icmp ne (A & D), E)
3597 // (icmp eq (A & B), C) & (icmp eq (A & D), E)
3598 if (Value *V = foldLogOpOfMaskedICmps(LHS, RHS, IsAnd, IsLogical, Builder,
3599 Q: SQ.getWithInstruction(I: &I)))
3600 return V;
3601
3602 if (auto *LHSCmp = dyn_cast<ICmpInst>(Val: LHS))
3603 if (auto *RHSCmp = dyn_cast<ICmpInst>(Val: RHS))
3604 if (Value *Res = foldAndOrOfICmps(LHS: LHSCmp, RHS: RHSCmp, I, IsAnd, IsLogical))
3605 return Res;
3606
3607 if (auto *LHSCmp = dyn_cast<FCmpInst>(Val: LHS))
3608 if (auto *RHSCmp = dyn_cast<FCmpInst>(Val: RHS))
3609 if (Value *Res = foldLogicOfFCmps(LHS: LHSCmp, RHS: RHSCmp, IsAnd, IsLogicalSelect: IsLogical))
3610 return Res;
3611
3612 if (Value *Res = foldEqOfParts(Cmp0: LHS, Cmp1: RHS, IsAnd))
3613 return Res;
3614
3615 return nullptr;
3616}
3617
3618static Value *foldOrOfInversions(BinaryOperator &I,
3619 InstCombiner::BuilderTy &Builder) {
3620 assert(I.getOpcode() == Instruction::Or &&
3621 "Simplification only supports or at the moment.");
3622
3623 Value *Cmp1, *Cmp2, *Cmp3, *Cmp4;
3624 if (!match(V: I.getOperand(i_nocapture: 0), P: m_And(L: m_Value(V&: Cmp1), R: m_Value(V&: Cmp2))) ||
3625 !match(V: I.getOperand(i_nocapture: 1), P: m_And(L: m_Value(V&: Cmp3), R: m_Value(V&: Cmp4))))
3626 return nullptr;
3627
3628 // Check if any two pairs of the and operations are inversions of each other.
3629 if (isKnownInversion(X: Cmp1, Y: Cmp3) && isKnownInversion(X: Cmp2, Y: Cmp4))
3630 return Builder.CreateXor(LHS: Cmp1, RHS: Cmp4);
3631 if (isKnownInversion(X: Cmp1, Y: Cmp4) && isKnownInversion(X: Cmp2, Y: Cmp3))
3632 return Builder.CreateXor(LHS: Cmp1, RHS: Cmp3);
3633
3634 return nullptr;
3635}
3636
3637/// Match \p V as "shufflevector -> bitcast" or "extractelement -> zext -> shl"
3638/// patterns, which extract vector elements and pack them in the same relative
3639/// positions.
3640///
3641/// \p Vec is the underlying vector being extracted from.
3642/// \p Mask is a bitmask identifying which packed elements are obtained from the
3643/// vector.
3644/// \p VecOffset is the vector element corresponding to index 0 of the
3645/// mask.
3646static bool matchSubIntegerPackFromVector(Value *V, Value *&Vec,
3647 int64_t &VecOffset,
3648 SmallBitVector &Mask,
3649 const DataLayout &DL) {
3650 // First try to match extractelement -> zext -> shl
3651 uint64_t VecIdx, ShlAmt;
3652 if (match(V, P: m_ShlOrSelf(L: m_ZExtOrSelf(Op: m_ExtractElt(Val: m_Value(V&: Vec),
3653 Idx: m_ConstantInt(V&: VecIdx))),
3654 R&: ShlAmt))) {
3655 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
3656 if (!VecTy)
3657 return false;
3658 auto *EltTy = dyn_cast<IntegerType>(Val: VecTy->getElementType());
3659 if (!EltTy)
3660 return false;
3661
3662 const unsigned EltBitWidth = EltTy->getBitWidth();
3663 const unsigned TargetBitWidth = V->getType()->getIntegerBitWidth();
3664 if (TargetBitWidth % EltBitWidth != 0 || ShlAmt % EltBitWidth != 0)
3665 return false;
3666 const unsigned TargetEltWidth = TargetBitWidth / EltBitWidth;
3667 const unsigned ShlEltAmt = ShlAmt / EltBitWidth;
3668
3669 const unsigned MaskIdx =
3670 DL.isLittleEndian() ? ShlEltAmt : TargetEltWidth - ShlEltAmt - 1;
3671
3672 VecOffset = static_cast<int64_t>(VecIdx) - static_cast<int64_t>(MaskIdx);
3673 Mask.resize(N: TargetEltWidth);
3674 Mask.set(MaskIdx);
3675 return true;
3676 }
3677
3678 // Now try to match a bitcasted subvector.
3679 Instruction *SrcVecI;
3680 if (!match(V, P: m_BitCast(Op: m_Instruction(I&: SrcVecI))))
3681 return false;
3682
3683 auto *SrcTy = dyn_cast<FixedVectorType>(Val: SrcVecI->getType());
3684 if (!SrcTy)
3685 return false;
3686
3687 Mask.resize(N: SrcTy->getNumElements());
3688
3689 // First check for a subvector obtained from a shufflevector.
3690 if (isa<ShuffleVectorInst>(Val: SrcVecI)) {
3691 Constant *ConstVec;
3692 ArrayRef<int> ShuffleMask;
3693 if (!match(V: SrcVecI, P: m_Shuffle(v1: m_Value(V&: Vec), v2: m_Constant(C&: ConstVec),
3694 mask: m_Mask(ShuffleMask))))
3695 return false;
3696
3697 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
3698 if (!VecTy)
3699 return false;
3700
3701 const unsigned NumVecElts = VecTy->getNumElements();
3702 bool FoundVecOffset = false;
3703 for (unsigned Idx = 0; Idx < ShuffleMask.size(); ++Idx) {
3704 if (ShuffleMask[Idx] == PoisonMaskElem)
3705 return false;
3706 const unsigned ShuffleIdx = ShuffleMask[Idx];
3707 if (ShuffleIdx >= NumVecElts) {
3708 const unsigned ConstIdx = ShuffleIdx - NumVecElts;
3709 auto *ConstElt =
3710 dyn_cast<ConstantInt>(Val: ConstVec->getAggregateElement(Elt: ConstIdx));
3711 if (!ConstElt || !ConstElt->isNullValue())
3712 return false;
3713 continue;
3714 }
3715
3716 if (FoundVecOffset) {
3717 if (VecOffset + Idx != ShuffleIdx)
3718 return false;
3719 } else {
3720 if (ShuffleIdx < Idx)
3721 return false;
3722 VecOffset = ShuffleIdx - Idx;
3723 FoundVecOffset = true;
3724 }
3725 Mask.set(Idx);
3726 }
3727 return FoundVecOffset;
3728 }
3729
3730 // Check for a subvector obtained as an (insertelement V, 0, idx)
3731 uint64_t InsertIdx;
3732 if (!match(V: SrcVecI,
3733 P: m_InsertElt(Val: m_Value(V&: Vec), Elt: m_Zero(), Idx: m_ConstantInt(V&: InsertIdx))))
3734 return false;
3735
3736 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
3737 if (!VecTy)
3738 return false;
3739 VecOffset = 0;
3740 bool AlreadyInsertedMaskedElt = Mask.test(Idx: InsertIdx);
3741 Mask.set();
3742 if (!AlreadyInsertedMaskedElt)
3743 Mask.reset(Idx: InsertIdx);
3744 return true;
3745}
3746
3747/// Try to fold the join of two scalar integers whose contents are packed
3748/// elements of the same vector.
3749static Instruction *foldIntegerPackFromVector(Instruction &I,
3750 InstCombiner::BuilderTy &Builder,
3751 const DataLayout &DL) {
3752 assert(I.getOpcode() == Instruction::Or);
3753 Value *LhsVec, *RhsVec;
3754 int64_t LhsVecOffset, RhsVecOffset;
3755 SmallBitVector Mask;
3756 if (!matchSubIntegerPackFromVector(V: I.getOperand(i: 0), Vec&: LhsVec, VecOffset&: LhsVecOffset,
3757 Mask, DL))
3758 return nullptr;
3759 if (!matchSubIntegerPackFromVector(V: I.getOperand(i: 1), Vec&: RhsVec, VecOffset&: RhsVecOffset,
3760 Mask, DL))
3761 return nullptr;
3762 if (LhsVec != RhsVec || LhsVecOffset != RhsVecOffset)
3763 return nullptr;
3764
3765 // Convert into shufflevector -> bitcast;
3766 const unsigned ZeroVecIdx =
3767 cast<FixedVectorType>(Val: LhsVec->getType())->getNumElements();
3768 SmallVector<int> ShuffleMask(Mask.size(), ZeroVecIdx);
3769 for (unsigned Idx : Mask.set_bits()) {
3770 assert(LhsVecOffset + Idx >= 0);
3771 ShuffleMask[Idx] = LhsVecOffset + Idx;
3772 }
3773
3774 Value *MaskedVec = Builder.CreateShuffleVector(
3775 V1: LhsVec, V2: Constant::getNullValue(Ty: LhsVec->getType()), Mask: ShuffleMask,
3776 Name: I.getName() + ".v");
3777 return CastInst::Create(Instruction::BitCast, S: MaskedVec, Ty: I.getType());
3778}
3779
3780/// Match \p V as "lshr -> mask -> zext -> shl".
3781///
3782/// \p Int is the underlying integer being extracted from.
3783/// \p Mask is a bitmask identifying which bits of the integer are being
3784/// extracted. \p Offset identifies which bit of the result \p V corresponds to
3785/// the least significant bit of \p Int
3786static bool matchZExtedSubInteger(Value *V, Value *&Int, APInt &Mask,
3787 uint64_t &Offset, bool &IsShlNUW,
3788 bool &IsShlNSW) {
3789 Value *ShlOp0;
3790 uint64_t ShlAmt = 0;
3791 if (!match(V, P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: ShlOp0), R: m_ConstantInt(V&: ShlAmt)))))
3792 return false;
3793
3794 IsShlNUW = cast<BinaryOperator>(Val: V)->hasNoUnsignedWrap();
3795 IsShlNSW = cast<BinaryOperator>(Val: V)->hasNoSignedWrap();
3796
3797 Value *ZExtOp0;
3798 if (!match(V: ShlOp0, P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: ZExtOp0)))))
3799 return false;
3800
3801 Value *MaskedOp0;
3802 const APInt *ShiftedMaskConst = nullptr;
3803 if (!match(V: ZExtOp0, P: m_CombineOr(L: m_OneUse(SubPattern: m_And(L: m_Value(V&: MaskedOp0),
3804 R: m_APInt(Res&: ShiftedMaskConst))),
3805 R: m_Value(V&: MaskedOp0))))
3806 return false;
3807
3808 uint64_t LShrAmt = 0;
3809 if (!match(V: MaskedOp0,
3810 P: m_CombineOr(L: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: Int), R: m_ConstantInt(V&: LShrAmt))),
3811 R: m_Value(V&: Int))))
3812 return false;
3813
3814 if (LShrAmt > ShlAmt)
3815 return false;
3816 Offset = ShlAmt - LShrAmt;
3817
3818 Mask = ShiftedMaskConst ? ShiftedMaskConst->shl(shiftAmt: LShrAmt)
3819 : APInt::getBitsSetFrom(
3820 numBits: Int->getType()->getScalarSizeInBits(), loBit: LShrAmt);
3821
3822 return true;
3823}
3824
3825/// Try to fold the join of two scalar integers whose bits are unpacked and
3826/// zexted from the same source integer.
3827static Value *foldIntegerRepackThroughZExt(Value *Lhs, Value *Rhs,
3828 InstCombiner::BuilderTy &Builder) {
3829
3830 Value *LhsInt, *RhsInt;
3831 APInt LhsMask, RhsMask;
3832 uint64_t LhsOffset, RhsOffset;
3833 bool IsLhsShlNUW, IsLhsShlNSW, IsRhsShlNUW, IsRhsShlNSW;
3834 if (!matchZExtedSubInteger(V: Lhs, Int&: LhsInt, Mask&: LhsMask, Offset&: LhsOffset, IsShlNUW&: IsLhsShlNUW,
3835 IsShlNSW&: IsLhsShlNSW))
3836 return nullptr;
3837 if (!matchZExtedSubInteger(V: Rhs, Int&: RhsInt, Mask&: RhsMask, Offset&: RhsOffset, IsShlNUW&: IsRhsShlNUW,
3838 IsShlNSW&: IsRhsShlNSW))
3839 return nullptr;
3840 if (LhsInt != RhsInt || LhsOffset != RhsOffset)
3841 return nullptr;
3842
3843 APInt Mask = LhsMask | RhsMask;
3844
3845 Type *DestTy = Lhs->getType();
3846 Value *Res = Builder.CreateShl(
3847 LHS: Builder.CreateZExt(
3848 V: Builder.CreateAnd(LHS: LhsInt, RHS: Mask, Name: LhsInt->getName() + ".mask"), DestTy,
3849 Name: LhsInt->getName() + ".zext"),
3850 RHS: ConstantInt::get(Ty: DestTy, V: LhsOffset), Name: "", HasNUW: IsLhsShlNUW && IsRhsShlNUW,
3851 HasNSW: IsLhsShlNSW && IsRhsShlNSW);
3852 Res->takeName(V: Lhs);
3853 return Res;
3854}
3855
3856// A decomposition of ((X & Mask) * Factor). The NUW / NSW bools
3857// track these properities for preservation. Note that we can decompose
3858// equivalent select form of this expression (e.g. (!(X & Mask) ? 0 : Mask *
3859// Factor))
3860struct DecomposedBitMaskMul {
3861 Value *X;
3862 APInt Factor;
3863 APInt Mask;
3864 bool NUW;
3865 bool NSW;
3866
3867 bool isCombineableWith(const DecomposedBitMaskMul Other) {
3868 return X == Other.X && !Mask.intersects(RHS: Other.Mask) &&
3869 Factor == Other.Factor;
3870 }
3871};
3872
3873static std::optional<DecomposedBitMaskMul> matchBitmaskMul(Value *V) {
3874 Instruction *Op = dyn_cast<Instruction>(Val: V);
3875 if (!Op)
3876 return std::nullopt;
3877
3878 // Decompose (A & N) * C) into BitMaskMul
3879 Value *Original = nullptr;
3880 const APInt *Mask = nullptr;
3881 const APInt *MulConst = nullptr;
3882 if (match(V: Op, P: m_Mul(L: m_And(L: m_Value(V&: Original), R: m_APInt(Res&: Mask)),
3883 R: m_APInt(Res&: MulConst)))) {
3884 if (MulConst->isZero() || Mask->isZero())
3885 return std::nullopt;
3886
3887 return std::optional<DecomposedBitMaskMul>(
3888 {.X: Original, .Factor: *MulConst, .Mask: *Mask,
3889 .NUW: cast<BinaryOperator>(Val: Op)->hasNoUnsignedWrap(),
3890 .NSW: cast<BinaryOperator>(Val: Op)->hasNoSignedWrap()});
3891 }
3892
3893 Value *Cond = nullptr;
3894 const APInt *EqZero = nullptr, *NeZero = nullptr;
3895
3896 // Decompose ((A & N) ? 0 : N * C) into BitMaskMul
3897 if (match(V: Op, P: m_Select(C: m_Value(V&: Cond), L: m_APInt(Res&: EqZero), R: m_APInt(Res&: NeZero)))) {
3898 auto ICmpDecompose =
3899 decomposeBitTest(Cond, /*LookThroughTrunc=*/true,
3900 /*AllowNonZeroC=*/false, /*DecomposeBitMask=*/DecomposeAnd: true);
3901 if (!ICmpDecompose.has_value())
3902 return std::nullopt;
3903
3904 assert(ICmpInst::isEquality(ICmpDecompose->Pred) &&
3905 ICmpDecompose->C.isZero());
3906
3907 if (ICmpDecompose->Pred == ICmpInst::ICMP_NE)
3908 std::swap(a&: EqZero, b&: NeZero);
3909
3910 if (!EqZero->isZero() || NeZero->isZero())
3911 return std::nullopt;
3912
3913 if (!ICmpDecompose->Mask.isPowerOf2() || ICmpDecompose->Mask.isZero() ||
3914 NeZero->getBitWidth() != ICmpDecompose->Mask.getBitWidth())
3915 return std::nullopt;
3916
3917 if (!NeZero->urem(RHS: ICmpDecompose->Mask).isZero())
3918 return std::nullopt;
3919
3920 return std::optional<DecomposedBitMaskMul>(
3921 {.X: ICmpDecompose->X, .Factor: NeZero->udiv(RHS: ICmpDecompose->Mask),
3922 .Mask: ICmpDecompose->Mask, /*NUW=*/false, /*NSW=*/false});
3923 }
3924
3925 return std::nullopt;
3926}
3927
3928/// (A & N) * C + (A & M) * C -> (A & (N + M)) & C
3929/// This also accepts the equivalent select form of (A & N) * C
3930/// expressions i.e. !(A & N) ? 0 : N * C)
3931static Value *foldBitmaskMul(Value *Op0, Value *Op1,
3932 InstCombiner::BuilderTy &Builder) {
3933 auto Decomp1 = matchBitmaskMul(V: Op1);
3934 if (!Decomp1)
3935 return nullptr;
3936
3937 auto Decomp0 = matchBitmaskMul(V: Op0);
3938 if (!Decomp0)
3939 return nullptr;
3940
3941 if (Decomp0->isCombineableWith(Other: *Decomp1)) {
3942 Value *NewAnd = Builder.CreateAnd(
3943 LHS: Decomp0->X,
3944 RHS: ConstantInt::get(Ty: Decomp0->X->getType(), V: Decomp0->Mask + Decomp1->Mask));
3945
3946 return Builder.CreateMul(
3947 LHS: NewAnd, RHS: ConstantInt::get(Ty: NewAnd->getType(), V: Decomp1->Factor), Name: "",
3948 HasNUW: Decomp0->NUW && Decomp1->NUW, HasNSW: Decomp0->NSW && Decomp1->NSW);
3949 }
3950
3951 return nullptr;
3952}
3953
3954Value *InstCombinerImpl::foldDisjointOr(Value *LHS, Value *RHS) {
3955 if (Value *Res = foldBitmaskMul(Op0: LHS, Op1: RHS, Builder))
3956 return Res;
3957 if (Value *Res = foldIntegerRepackThroughZExt(Lhs: LHS, Rhs: RHS, Builder))
3958 return Res;
3959
3960 return nullptr;
3961}
3962
3963Value *InstCombinerImpl::reassociateDisjointOr(Value *LHS, Value *RHS) {
3964
3965 Value *X, *Y;
3966 if (match(V: RHS, P: m_OneUse(SubPattern: m_DisjointOr(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
3967 if (Value *Res = foldDisjointOr(LHS, RHS: X))
3968 return Builder.CreateOr(LHS: Res, RHS: Y, Name: "", /*IsDisjoint=*/true);
3969 if (Value *Res = foldDisjointOr(LHS, RHS: Y))
3970 return Builder.CreateOr(LHS: Res, RHS: X, Name: "", /*IsDisjoint=*/true);
3971 }
3972
3973 if (match(V: LHS, P: m_OneUse(SubPattern: m_DisjointOr(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
3974 if (Value *Res = foldDisjointOr(LHS: X, RHS))
3975 return Builder.CreateOr(LHS: Res, RHS: Y, Name: "", /*IsDisjoint=*/true);
3976 if (Value *Res = foldDisjointOr(LHS: Y, RHS))
3977 return Builder.CreateOr(LHS: Res, RHS: X, Name: "", /*IsDisjoint=*/true);
3978 }
3979
3980 return nullptr;
3981}
3982
3983/// Fold Res, Overflow = (umul.with.overflow x c1); (or Overflow (ugt Res c2))
3984/// --> (ugt x (c2/c1)). This code checks whether a multiplication of two
3985/// unsigned numbers (one is a constant) is mathematically greater than a
3986/// second constant.
3987static Value *foldOrUnsignedUMulOverflowICmp(BinaryOperator &I,
3988 InstCombiner::BuilderTy &Builder,
3989 const DataLayout &DL) {
3990 Value *WOV, *X;
3991 const APInt *C1, *C2;
3992 if (match(V: &I,
3993 P: m_c_Or(L: m_ExtractValue<1>(
3994 V: m_Value(V&: WOV, Match: m_Intrinsic<Intrinsic::umul_with_overflow>(
3995 Op0: m_Value(V&: X), Op1: m_APInt(Res&: C1)))),
3996 R: m_OneUse(SubPattern: m_SpecificCmp(MatchPred: ICmpInst::ICMP_UGT,
3997 L: m_ExtractValue<0>(V: m_Deferred(V: WOV)),
3998 R: m_APInt(Res&: C2))))) &&
3999 !C1->isZero()) {
4000 Constant *NewC = ConstantInt::get(Ty: X->getType(), V: C2->udiv(RHS: *C1));
4001 return Builder.CreateICmp(P: ICmpInst::ICMP_UGT, LHS: X, RHS: NewC);
4002 }
4003 return nullptr;
4004}
4005
4006/// Fold select(X >s 0, 0, -X) | smax(X, 0) --> abs(X)
4007/// select(X <s 0, -X, 0) | smax(X, 0) --> abs(X)
4008static Value *FoldOrOfSelectSmaxToAbs(BinaryOperator &I,
4009 InstCombiner::BuilderTy &Builder) {
4010 Value *X;
4011 Value *Sel;
4012 if (match(V: &I,
4013 P: m_c_Or(L: m_Value(V&: Sel), R: m_OneUse(SubPattern: m_SMax(L: m_Value(V&: X), R: m_ZeroInt()))))) {
4014 auto NegX = m_Neg(V: m_Specific(V: X));
4015 if (match(V: Sel, P: m_Select(C: m_SpecificICmp(MatchPred: ICmpInst::ICMP_SGT, L: m_Specific(V: X),
4016 R: m_ZeroInt()),
4017 L: m_ZeroInt(), R: NegX)) ||
4018 match(V: Sel, P: m_Select(C: m_SpecificICmp(MatchPred: ICmpInst::ICMP_SLT, L: m_Specific(V: X),
4019 R: m_ZeroInt()),
4020 L: NegX, R: m_ZeroInt())))
4021 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: X,
4022 RHS: Builder.getFalse());
4023 }
4024 return nullptr;
4025}
4026
4027// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
4028// here. We should standardize that construct where it is needed or choose some
4029// other way to ensure that commutated variants of patterns are not missed.
4030Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
4031 if (Value *V = simplifyOrInst(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1),
4032 Q: SQ.getWithInstruction(I: &I)))
4033 return replaceInstUsesWith(I, V);
4034
4035 if (SimplifyAssociativeOrCommutative(I))
4036 return &I;
4037
4038 if (Instruction *X = foldVectorBinop(Inst&: I))
4039 return X;
4040
4041 if (Instruction *Phi = foldBinopWithPhiOperands(BO&: I))
4042 return Phi;
4043
4044 // See if we can simplify any instructions used by the instruction whose sole
4045 // purpose is to compute bits we don't care about.
4046 if (SimplifyDemandedInstructionBits(Inst&: I))
4047 return &I;
4048
4049 // Do this before using distributive laws to catch simple and/or/not patterns.
4050 if (Instruction *Xor = foldOrToXor(I, Builder))
4051 return Xor;
4052
4053 if (Instruction *X = foldComplexAndOrPatterns(I, Builder))
4054 return X;
4055
4056 if (Instruction *X = foldIntegerPackFromVector(I, Builder, DL))
4057 return X;
4058
4059 // (A & B) | (C & D) -> A ^ D where A == ~C && B == ~D
4060 // (A & B) | (C & D) -> A ^ C where A == ~D && B == ~C
4061 if (Value *V = foldOrOfInversions(I, Builder))
4062 return replaceInstUsesWith(I, V);
4063
4064 // (A&B)|(A&C) -> A&(B|C) etc
4065 if (Value *V = foldUsingDistributiveLaws(I))
4066 return replaceInstUsesWith(I, V);
4067
4068 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
4069 Type *Ty = I.getType();
4070 if (Ty->isIntOrIntVectorTy(BitWidth: 1)) {
4071 if (auto *SI0 = dyn_cast<SelectInst>(Val: Op0)) {
4072 if (auto *R =
4073 foldAndOrOfSelectUsingImpliedCond(Op: Op1, SI&: *SI0, /* IsAnd */ false))
4074 return R;
4075 }
4076 if (auto *SI1 = dyn_cast<SelectInst>(Val: Op1)) {
4077 if (auto *R =
4078 foldAndOrOfSelectUsingImpliedCond(Op: Op0, SI&: *SI1, /* IsAnd */ false))
4079 return R;
4080 }
4081 }
4082
4083 if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
4084 return FoldedLogic;
4085
4086 if (Instruction *FoldedLogic = foldBinOpSelectBinOp(Op&: I))
4087 return FoldedLogic;
4088
4089 if (Instruction *BitOp = matchBSwapOrBitReverse(I, /*MatchBSwaps*/ true,
4090 /*MatchBitReversals*/ true))
4091 return BitOp;
4092
4093 if (Instruction *Funnel = matchFunnelShift(Or&: I, IC&: *this))
4094 return Funnel;
4095
4096 if (Value *Concat = matchOrConcat(Or&: I, Builder))
4097 return replaceInstUsesWith(I, V: Concat);
4098
4099 if (Instruction *R = foldBinOpShiftWithShift(I))
4100 return R;
4101
4102 if (Instruction *R = tryFoldInstWithCtpopWithNot(I: &I))
4103 return R;
4104
4105 if (cast<PossiblyDisjointInst>(Val&: I).isDisjoint()) {
4106 if (Instruction *R =
4107 foldAddLikeCommutative(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1),
4108 /*NSW=*/true, /*NUW=*/true))
4109 return R;
4110 if (Instruction *R =
4111 foldAddLikeCommutative(LHS: I.getOperand(i_nocapture: 1), RHS: I.getOperand(i_nocapture: 0),
4112 /*NSW=*/true, /*NUW=*/true))
4113 return R;
4114
4115 if (Value *Res = foldDisjointOr(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1)))
4116 return replaceInstUsesWith(I, V: Res);
4117
4118 if (Value *Res = reassociateDisjointOr(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1)))
4119 return replaceInstUsesWith(I, V: Res);
4120 }
4121
4122 Value *X, *Y;
4123 const APInt *CV;
4124 if (match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: X), R: m_APInt(Res&: CV))), R: m_Value(V&: Y))) &&
4125 !CV->isAllOnes() && MaskedValueIsZero(V: Y, Mask: *CV, CxtI: &I)) {
4126 // (X ^ C) | Y -> (X | Y) ^ C iff Y & C == 0
4127 // The check for a 'not' op is for efficiency (if Y is known zero --> ~X).
4128 Value *Or = Builder.CreateOr(LHS: X, RHS: Y);
4129 return BinaryOperator::CreateXor(V1: Or, V2: ConstantInt::get(Ty, V: *CV));
4130 }
4131
4132 // If the operands have no common bits set:
4133 // or (mul X, Y), X --> add (mul X, Y), X --> mul X, (Y + 1)
4134 if (match(V: &I, P: m_c_DisjointOr(L: m_OneUse(SubPattern: m_Mul(L: m_Value(V&: X), R: m_Value(V&: Y))),
4135 R: m_Deferred(V: X)))) {
4136 Value *IncrementY = Builder.CreateAdd(LHS: Y, RHS: ConstantInt::get(Ty, V: 1));
4137 return BinaryOperator::CreateMul(V1: X, V2: IncrementY);
4138 }
4139
4140 // (A & C) | (B & D)
4141 Value *A, *B, *C, *D;
4142 if (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: C))) &&
4143 match(V: Op1, P: m_And(L: m_Value(V&: B), R: m_Value(V&: D)))) {
4144
4145 // (A & C0) | (B & C1)
4146 const APInt *C0, *C1;
4147 if (match(V: C, P: m_APInt(Res&: C0)) && match(V: D, P: m_APInt(Res&: C1))) {
4148 Value *X;
4149 if (*C0 == ~*C1) {
4150 // ((X | B) & MaskC) | (B & ~MaskC) -> (X & MaskC) | B
4151 if (match(V: A, P: m_c_Or(L: m_Value(V&: X), R: m_Specific(V: B))))
4152 return BinaryOperator::CreateOr(V1: Builder.CreateAnd(LHS: X, RHS: *C0), V2: B);
4153 // (A & MaskC) | ((X | A) & ~MaskC) -> (X & ~MaskC) | A
4154 if (match(V: B, P: m_c_Or(L: m_Specific(V: A), R: m_Value(V&: X))))
4155 return BinaryOperator::CreateOr(V1: Builder.CreateAnd(LHS: X, RHS: *C1), V2: A);
4156
4157 // ((X ^ B) & MaskC) | (B & ~MaskC) -> (X & MaskC) ^ B
4158 if (match(V: A, P: m_c_Xor(L: m_Value(V&: X), R: m_Specific(V: B))))
4159 return BinaryOperator::CreateXor(V1: Builder.CreateAnd(LHS: X, RHS: *C0), V2: B);
4160 // (A & MaskC) | ((X ^ A) & ~MaskC) -> (X & ~MaskC) ^ A
4161 if (match(V: B, P: m_c_Xor(L: m_Specific(V: A), R: m_Value(V&: X))))
4162 return BinaryOperator::CreateXor(V1: Builder.CreateAnd(LHS: X, RHS: *C1), V2: A);
4163 }
4164
4165 if ((*C0 & *C1).isZero()) {
4166 // ((X | B) & C0) | (B & C1) --> (X | B) & (C0 | C1)
4167 // iff (C0 & C1) == 0 and (X & ~C0) == 0
4168 if (match(V: A, P: m_c_Or(L: m_Value(V&: X), R: m_Specific(V: B))) &&
4169 MaskedValueIsZero(V: X, Mask: ~*C0, CxtI: &I)) {
4170 Constant *C01 = ConstantInt::get(Ty, V: *C0 | *C1);
4171 return BinaryOperator::CreateAnd(V1: A, V2: C01);
4172 }
4173 // (A & C0) | ((X | A) & C1) --> (X | A) & (C0 | C1)
4174 // iff (C0 & C1) == 0 and (X & ~C1) == 0
4175 if (match(V: B, P: m_c_Or(L: m_Value(V&: X), R: m_Specific(V: A))) &&
4176 MaskedValueIsZero(V: X, Mask: ~*C1, CxtI: &I)) {
4177 Constant *C01 = ConstantInt::get(Ty, V: *C0 | *C1);
4178 return BinaryOperator::CreateAnd(V1: B, V2: C01);
4179 }
4180 // ((X | C2) & C0) | ((X | C3) & C1) --> (X | C2 | C3) & (C0 | C1)
4181 // iff (C0 & C1) == 0 and (C2 & ~C0) == 0 and (C3 & ~C1) == 0.
4182 const APInt *C2, *C3;
4183 if (match(V: A, P: m_Or(L: m_Value(V&: X), R: m_APInt(Res&: C2))) &&
4184 match(V: B, P: m_Or(L: m_Specific(V: X), R: m_APInt(Res&: C3))) &&
4185 (*C2 & ~*C0).isZero() && (*C3 & ~*C1).isZero()) {
4186 Value *Or = Builder.CreateOr(LHS: X, RHS: *C2 | *C3, Name: "bitfield");
4187 Constant *C01 = ConstantInt::get(Ty, V: *C0 | *C1);
4188 return BinaryOperator::CreateAnd(V1: Or, V2: C01);
4189 }
4190 }
4191 }
4192
4193 // Don't try to form a select if it's unlikely that we'll get rid of at
4194 // least one of the operands. A select is generally more expensive than the
4195 // 'or' that it is replacing.
4196 if (Op0->hasOneUse() || Op1->hasOneUse()) {
4197 // (Cond & C) | (~Cond & D) -> Cond ? C : D, and commuted variants.
4198 if (Value *V = matchSelectFromAndOr(A, B: C, C: B, D))
4199 return replaceInstUsesWith(I, V);
4200 if (Value *V = matchSelectFromAndOr(A, B: C, C: D, D: B))
4201 return replaceInstUsesWith(I, V);
4202 if (Value *V = matchSelectFromAndOr(A: C, B: A, C: B, D))
4203 return replaceInstUsesWith(I, V);
4204 if (Value *V = matchSelectFromAndOr(A: C, B: A, C: D, D: B))
4205 return replaceInstUsesWith(I, V);
4206 if (Value *V = matchSelectFromAndOr(A: B, B: D, C: A, D: C))
4207 return replaceInstUsesWith(I, V);
4208 if (Value *V = matchSelectFromAndOr(A: B, B: D, C, D: A))
4209 return replaceInstUsesWith(I, V);
4210 if (Value *V = matchSelectFromAndOr(A: D, B, C: A, D: C))
4211 return replaceInstUsesWith(I, V);
4212 if (Value *V = matchSelectFromAndOr(A: D, B, C, D: A))
4213 return replaceInstUsesWith(I, V);
4214 }
4215 }
4216
4217 if (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: C))) &&
4218 match(V: Op1, P: m_Not(V: m_Or(L: m_Value(V&: B), R: m_Value(V&: D)))) &&
4219 (Op0->hasOneUse() || Op1->hasOneUse())) {
4220 // (Cond & C) | ~(Cond | D) -> Cond ? C : ~D
4221 if (Value *V = matchSelectFromAndOr(A, B: C, C: B, D, InvertFalseVal: true))
4222 return replaceInstUsesWith(I, V);
4223 if (Value *V = matchSelectFromAndOr(A, B: C, C: D, D: B, InvertFalseVal: true))
4224 return replaceInstUsesWith(I, V);
4225 if (Value *V = matchSelectFromAndOr(A: C, B: A, C: B, D, InvertFalseVal: true))
4226 return replaceInstUsesWith(I, V);
4227 if (Value *V = matchSelectFromAndOr(A: C, B: A, C: D, D: B, InvertFalseVal: true))
4228 return replaceInstUsesWith(I, V);
4229 }
4230
4231 // (A ^ B) | ((B ^ C) ^ A) -> (A ^ B) | C
4232 if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))))
4233 if (match(V: Op1,
4234 P: m_c_Xor(L: m_c_Xor(L: m_Specific(V: B), R: m_Value(V&: C)), R: m_Specific(V: A))) ||
4235 match(V: Op1, P: m_c_Xor(L: m_c_Xor(L: m_Specific(V: A), R: m_Value(V&: C)), R: m_Specific(V: B))))
4236 return BinaryOperator::CreateOr(V1: Op0, V2: C);
4237
4238 // ((B ^ C) ^ A) | (A ^ B) -> (A ^ B) | C
4239 if (match(V: Op1, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))))
4240 if (match(V: Op0,
4241 P: m_c_Xor(L: m_c_Xor(L: m_Specific(V: B), R: m_Value(V&: C)), R: m_Specific(V: A))) ||
4242 match(V: Op0, P: m_c_Xor(L: m_c_Xor(L: m_Specific(V: A), R: m_Value(V&: C)), R: m_Specific(V: B))))
4243 return BinaryOperator::CreateOr(V1: Op1, V2: C);
4244
4245 if (Instruction *DeMorgan = matchDeMorgansLaws(I, IC&: *this))
4246 return DeMorgan;
4247
4248 // Canonicalize xor to the RHS.
4249 bool SwappedForXor = false;
4250 if (match(V: Op0, P: m_Xor(L: m_Value(), R: m_Value()))) {
4251 std::swap(a&: Op0, b&: Op1);
4252 SwappedForXor = true;
4253 }
4254
4255 if (match(V: Op1, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B)))) {
4256 // (A | ?) | (A ^ B) --> (A | ?) | B
4257 // (B | ?) | (A ^ B) --> (B | ?) | A
4258 if (match(V: Op0, P: m_c_Or(L: m_Specific(V: A), R: m_Value())))
4259 return BinaryOperator::CreateOr(V1: Op0, V2: B);
4260 if (match(V: Op0, P: m_c_Or(L: m_Specific(V: B), R: m_Value())))
4261 return BinaryOperator::CreateOr(V1: Op0, V2: A);
4262
4263 // (A & B) | (A ^ B) --> A | B
4264 // (B & A) | (A ^ B) --> A | B
4265 if (match(V: Op0, P: m_c_And(L: m_Specific(V: A), R: m_Specific(V: B))))
4266 return BinaryOperator::CreateOr(V1: A, V2: B);
4267
4268 // ~A | (A ^ B) --> ~(A & B)
4269 // ~B | (A ^ B) --> ~(A & B)
4270 // The swap above should always make Op0 the 'not'.
4271 if ((Op0->hasOneUse() || Op1->hasOneUse()) &&
4272 (match(V: Op0, P: m_Not(V: m_Specific(V: A))) || match(V: Op0, P: m_Not(V: m_Specific(V: B)))))
4273 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: A, RHS: B));
4274
4275 // Same as above, but peek through an 'and' to the common operand:
4276 // ~(A & ?) | (A ^ B) --> ~((A & ?) & B)
4277 // ~(B & ?) | (A ^ B) --> ~((B & ?) & A)
4278 Instruction *And;
4279 if ((Op0->hasOneUse() || Op1->hasOneUse()) &&
4280 match(V: Op0,
4281 P: m_Not(V: m_Instruction(I&: And, Match: m_c_And(L: m_Specific(V: A), R: m_Value())))))
4282 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: And, RHS: B));
4283 if ((Op0->hasOneUse() || Op1->hasOneUse()) &&
4284 match(V: Op0,
4285 P: m_Not(V: m_Instruction(I&: And, Match: m_c_And(L: m_Specific(V: B), R: m_Value())))))
4286 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: And, RHS: A));
4287
4288 // (~A | C) | (A ^ B) --> ~(A & B) | C
4289 // (~B | C) | (A ^ B) --> ~(A & B) | C
4290 if (Op0->hasOneUse() && Op1->hasOneUse() &&
4291 (match(V: Op0, P: m_c_Or(L: m_Not(V: m_Specific(V: A)), R: m_Value(V&: C))) ||
4292 match(V: Op0, P: m_c_Or(L: m_Not(V: m_Specific(V: B)), R: m_Value(V&: C))))) {
4293 Value *Nand = Builder.CreateNot(V: Builder.CreateAnd(LHS: A, RHS: B), Name: "nand");
4294 return BinaryOperator::CreateOr(V1: Nand, V2: C);
4295 }
4296 }
4297
4298 if (SwappedForXor)
4299 std::swap(a&: Op0, b&: Op1);
4300
4301 if (Value *Res =
4302 foldBooleanAndOr(LHS: Op0, RHS: Op1, I, /*IsAnd=*/false, /*IsLogical=*/false))
4303 return replaceInstUsesWith(I, V: Res);
4304
4305 if (match(V: Op1, P: m_OneUse(SubPattern: m_LogicalOr(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
4306 bool IsLogical = isa<SelectInst>(Val: Op1);
4307 if (auto *V = reassociateBooleanAndOr(LHS: Op0, X, Y, I, /*IsAnd=*/false,
4308 /*RHSIsLogical=*/IsLogical))
4309 return replaceInstUsesWith(I, V);
4310 }
4311 if (match(V: Op0, P: m_OneUse(SubPattern: m_LogicalOr(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
4312 bool IsLogical = isa<SelectInst>(Val: Op0);
4313 if (auto *V = reassociateBooleanAndOr(LHS: Op1, X, Y, I, /*IsAnd=*/false,
4314 /*RHSIsLogical=*/IsLogical))
4315 return replaceInstUsesWith(I, V);
4316 }
4317
4318 if (Instruction *FoldedFCmps = reassociateFCmps(BO&: I, Builder))
4319 return FoldedFCmps;
4320
4321 if (Instruction *CastedOr = foldCastedBitwiseLogic(I))
4322 return CastedOr;
4323
4324 if (Instruction *Sel = foldBinopOfSextBoolToSelect(I))
4325 return Sel;
4326
4327 // or(sext(A), B) / or(B, sext(A)) --> A ? -1 : B, where A is i1 or <N x i1>.
4328 // TODO: Move this into foldBinopOfSextBoolToSelect as a more generalized fold
4329 // with binop identity constant. But creating a select with non-constant
4330 // arm may not be reversible due to poison semantics. Is that a good
4331 // canonicalization?
4332 if (match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_SExt(Op: m_Value(V&: A))), R: m_Value(V&: B))) &&
4333 A->getType()->isIntOrIntVectorTy(BitWidth: 1))
4334 return createSelectInstWithUnknownProfile(
4335 C: A, S1: ConstantInt::getAllOnesValue(Ty), S2: B);
4336
4337 // Note: If we've gotten to the point of visiting the outer OR, then the
4338 // inner one couldn't be simplified. If it was a constant, then it won't
4339 // be simplified by a later pass either, so we try swapping the inner/outer
4340 // ORs in the hopes that we'll be able to simplify it this way.
4341 // (X|C) | V --> (X|V) | C
4342 // Pass the disjoint flag in the following two patterns:
4343 // 1. or-disjoint (or-disjoint X, C), V -->
4344 // or-disjoint (or-disjoint X, V), C
4345 //
4346 // 2. or-disjoint (or X, C), V -->
4347 // or (or-disjoint X, V), C
4348 ConstantInt *CI;
4349 if (Op0->hasOneUse() && !match(V: Op1, P: m_ConstantInt()) &&
4350 match(V: Op0, P: m_Or(L: m_Value(V&: A), R: m_ConstantInt(CI)))) {
4351 bool IsDisjointOuter = cast<PossiblyDisjointInst>(Val&: I).isDisjoint();
4352 bool IsDisjointInner = cast<PossiblyDisjointInst>(Val: Op0)->isDisjoint();
4353 Value *Inner = Builder.CreateOr(LHS: A, RHS: Op1);
4354 cast<PossiblyDisjointInst>(Val: Inner)->setIsDisjoint(IsDisjointOuter);
4355 Inner->takeName(V: Op0);
4356 return IsDisjointOuter && IsDisjointInner
4357 ? BinaryOperator::CreateDisjointOr(V1: Inner, V2: CI)
4358 : BinaryOperator::CreateOr(V1: Inner, V2: CI);
4359 }
4360
4361 // Change (or (bool?A:B),(bool?C:D)) --> (bool?(or A,C):(or B,D))
4362 // Since this OR statement hasn't been optimized further yet, we hope
4363 // that this transformation will allow the new ORs to be optimized.
4364 {
4365 Value *X = nullptr, *Y = nullptr;
4366 if (Op0->hasOneUse() && Op1->hasOneUse() &&
4367 match(V: Op0, P: m_Select(C: m_Value(V&: X), L: m_Value(V&: A), R: m_Value(V&: B))) &&
4368 match(V: Op1, P: m_Select(C: m_Value(V&: Y), L: m_Value(V&: C), R: m_Value(V&: D))) && X == Y) {
4369 Value *orTrue = Builder.CreateOr(LHS: A, RHS: C);
4370 Value *orFalse = Builder.CreateOr(LHS: B, RHS: D);
4371 return SelectInst::Create(C: X, S1: orTrue, S2: orFalse);
4372 }
4373 }
4374
4375 // or(ashr(subNSW(Y, X), ScalarSizeInBits(Y) - 1), X) --> X s> Y ? -1 : X.
4376 {
4377 Value *X, *Y;
4378 if (match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_AShr(
4379 L: m_NSWSub(L: m_Value(V&: Y), R: m_Value(V&: X)),
4380 R: m_SpecificInt(V: Ty->getScalarSizeInBits() - 1))),
4381 R: m_Deferred(V: X)))) {
4382 Value *NewICmpInst = Builder.CreateICmpSGT(LHS: X, RHS: Y);
4383 Value *AllOnes = ConstantInt::getAllOnesValue(Ty);
4384 return createSelectInstWithUnknownProfile(C: NewICmpInst, S1: AllOnes, S2: X);
4385 }
4386 }
4387
4388 {
4389 // ((A & B) ^ A) | ((A & B) ^ B) -> A ^ B
4390 // (A ^ (A & B)) | (B ^ (A & B)) -> A ^ B
4391 // ((A & B) ^ B) | ((A & B) ^ A) -> A ^ B
4392 // (B ^ (A & B)) | (A ^ (A & B)) -> A ^ B
4393 const auto TryXorOpt = [&](Value *Lhs, Value *Rhs) -> Instruction * {
4394 if (match(V: Lhs, P: m_c_Xor(L: m_And(L: m_Value(V&: A), R: m_Value(V&: B)), R: m_Deferred(V: A))) &&
4395 match(V: Rhs,
4396 P: m_c_Xor(L: m_And(L: m_Specific(V: A), R: m_Specific(V: B)), R: m_Specific(V: B)))) {
4397 return BinaryOperator::CreateXor(V1: A, V2: B);
4398 }
4399 return nullptr;
4400 };
4401
4402 if (Instruction *Result = TryXorOpt(Op0, Op1))
4403 return Result;
4404 if (Instruction *Result = TryXorOpt(Op1, Op0))
4405 return Result;
4406 }
4407
4408 if (Instruction *V =
4409 canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(I))
4410 return V;
4411
4412 CmpPredicate Pred;
4413 Value *Mul, *Ov, *MulIsNotZero, *UMulWithOv;
4414 // Check if the OR weakens the overflow condition for umul.with.overflow by
4415 // treating any non-zero result as overflow. In that case, we overflow if both
4416 // umul.with.overflow operands are != 0, as in that case the result can only
4417 // be 0, iff the multiplication overflows.
4418 if (match(V: &I, P: m_c_Or(L: m_Value(V&: Ov, Match: m_ExtractValue<1>(V: m_Value(V&: UMulWithOv))),
4419 R: m_Value(V&: MulIsNotZero,
4420 Match: m_SpecificICmp(
4421 MatchPred: ICmpInst::ICMP_NE,
4422 L: m_Value(V&: Mul, Match: m_ExtractValue<0>(
4423 V: m_Deferred(V: UMulWithOv))),
4424 R: m_ZeroInt())))) &&
4425 (Ov->hasOneUse() || (MulIsNotZero->hasOneUse() && Mul->hasOneUse()))) {
4426 Value *A, *B;
4427 if (match(V: UMulWithOv, P: m_Intrinsic<Intrinsic::umul_with_overflow>(
4428 Op0: m_Value(V&: A), Op1: m_Value(V&: B)))) {
4429 Value *NotNullA = Builder.CreateIsNotNull(Arg: A);
4430 Value *NotNullB = Builder.CreateIsNotNull(Arg: B);
4431 return BinaryOperator::CreateAnd(V1: NotNullA, V2: NotNullB);
4432 }
4433 }
4434
4435 /// Res, Overflow = xxx_with_overflow X, C1
4436 /// Try to canonicalize the pattern "Overflow | icmp pred Res, C2" into
4437 /// "Overflow | icmp pred X, C2 +/- C1".
4438 const WithOverflowInst *WO;
4439 const Value *WOV;
4440 const APInt *C1, *C2;
4441 if (match(V: &I, P: m_c_Or(L: m_Value(V&: Ov, Match: m_ExtractValue<1>(
4442 V: m_Value(V&: WOV, Match: m_WithOverflowInst(I&: WO)))),
4443 R: m_OneUse(SubPattern: m_ICmp(Pred, L: m_ExtractValue<0>(V: m_Deferred(V: WOV)),
4444 R: m_APInt(Res&: C2))))) &&
4445 (WO->getBinaryOp() == Instruction::Add ||
4446 WO->getBinaryOp() == Instruction::Sub) &&
4447 (ICmpInst::isEquality(P: Pred) ||
4448 WO->isSigned() == ICmpInst::isSigned(predicate: Pred)) &&
4449 match(V: WO->getRHS(), P: m_APInt(Res&: C1))) {
4450 bool Overflow;
4451 APInt NewC = WO->getBinaryOp() == Instruction::Add
4452 ? (ICmpInst::isSigned(predicate: Pred) ? C2->ssub_ov(RHS: *C1, Overflow)
4453 : C2->usub_ov(RHS: *C1, Overflow))
4454 : (ICmpInst::isSigned(predicate: Pred) ? C2->sadd_ov(RHS: *C1, Overflow)
4455 : C2->uadd_ov(RHS: *C1, Overflow));
4456 if (!Overflow || ICmpInst::isEquality(P: Pred)) {
4457 Value *NewCmp = Builder.CreateICmp(
4458 P: Pred, LHS: WO->getLHS(), RHS: ConstantInt::get(Ty: WO->getLHS()->getType(), V: NewC));
4459 return BinaryOperator::CreateOr(V1: Ov, V2: NewCmp);
4460 }
4461 }
4462
4463 // Try to fold the pattern "Overflow | icmp pred Res, C2" into a single
4464 // comparison instruction for umul.with.overflow.
4465 if (Value *R = foldOrUnsignedUMulOverflowICmp(I, Builder, DL))
4466 return replaceInstUsesWith(I, V: R);
4467
4468 // (~x) | y --> ~(x & (~y)) iff that gets rid of inversions
4469 if (sinkNotIntoOtherHandOfLogicalOp(I))
4470 return &I;
4471
4472 // Improve "get low bit mask up to and including bit X" pattern:
4473 // (1 << X) | ((1 << X) + -1) --> -1 l>> (bitwidth(x) - 1 - X)
4474 if (match(V: &I, P: m_c_Or(L: m_Add(L: m_Shl(L: m_One(), R: m_Value(V&: X)), R: m_AllOnes()),
4475 R: m_Shl(L: m_One(), R: m_Deferred(V: X)))) &&
4476 match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_Value()), R: m_Value()))) {
4477 Value *Sub = Builder.CreateSub(
4478 LHS: ConstantInt::get(Ty, V: Ty->getScalarSizeInBits() - 1), RHS: X);
4479 return BinaryOperator::CreateLShr(V1: Constant::getAllOnesValue(Ty), V2: Sub);
4480 }
4481
4482 // An or recurrence w/loop invariant step is equivelent to (or start, step)
4483 PHINode *PN = nullptr;
4484 Value *Start = nullptr, *Step = nullptr;
4485 if (matchSimpleRecurrence(I: &I, P&: PN, Start, Step) && DT.dominates(Def: Step, User: PN))
4486 return replaceInstUsesWith(I, V: Builder.CreateOr(LHS: Start, RHS: Step));
4487
4488 // (A & B) | (C | D) or (C | D) | (A & B)
4489 // Can be combined if C or D is of type (A/B & X)
4490 if (match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_And(L: m_Value(V&: A), R: m_Value(V&: B))),
4491 R: m_OneUse(SubPattern: m_Or(L: m_Value(V&: C), R: m_Value(V&: D)))))) {
4492 // (A & B) | (C | ?) -> C | (? | (A & B))
4493 // (A & B) | (C | ?) -> C | (? | (A & B))
4494 // (A & B) | (C | ?) -> C | (? | (A & B))
4495 // (A & B) | (C | ?) -> C | (? | (A & B))
4496 // (C | ?) | (A & B) -> C | (? | (A & B))
4497 // (C | ?) | (A & B) -> C | (? | (A & B))
4498 // (C | ?) | (A & B) -> C | (? | (A & B))
4499 // (C | ?) | (A & B) -> C | (? | (A & B))
4500 if (match(V: D, P: m_OneUse(SubPattern: m_c_And(L: m_Specific(V: A), R: m_Value()))) ||
4501 match(V: D, P: m_OneUse(SubPattern: m_c_And(L: m_Specific(V: B), R: m_Value()))))
4502 return BinaryOperator::CreateOr(
4503 V1: C, V2: Builder.CreateOr(LHS: D, RHS: Builder.CreateAnd(LHS: A, RHS: B)));
4504 // (A & B) | (? | D) -> (? | (A & B)) | D
4505 // (A & B) | (? | D) -> (? | (A & B)) | D
4506 // (A & B) | (? | D) -> (? | (A & B)) | D
4507 // (A & B) | (? | D) -> (? | (A & B)) | D
4508 // (? | D) | (A & B) -> (? | (A & B)) | D
4509 // (? | D) | (A & B) -> (? | (A & B)) | D
4510 // (? | D) | (A & B) -> (? | (A & B)) | D
4511 // (? | D) | (A & B) -> (? | (A & B)) | D
4512 if (match(V: C, P: m_OneUse(SubPattern: m_c_And(L: m_Specific(V: A), R: m_Value()))) ||
4513 match(V: C, P: m_OneUse(SubPattern: m_c_And(L: m_Specific(V: B), R: m_Value()))))
4514 return BinaryOperator::CreateOr(
4515 V1: Builder.CreateOr(LHS: C, RHS: Builder.CreateAnd(LHS: A, RHS: B)), V2: D);
4516 }
4517
4518 if (Instruction *R = reassociateForUses(BO&: I, Builder))
4519 return R;
4520
4521 if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder))
4522 return Canonicalized;
4523
4524 if (Instruction *Folded = foldLogicOfIsFPClass(BO&: I, Op0, Op1))
4525 return Folded;
4526
4527 if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
4528 return Res;
4529
4530 // If we are setting the sign bit of a floating-point value, convert
4531 // this to fneg(fabs), then cast back to integer.
4532 //
4533 // If the result isn't immediately cast back to a float, this will increase
4534 // the number of instructions. This is still probably a better canonical form
4535 // as it enables FP value tracking.
4536 //
4537 // Assumes any IEEE-represented type has the sign bit in the high bit.
4538 //
4539 // This is generous interpretation of noimplicitfloat, this is not a true
4540 // floating-point operation.
4541 Value *CastOp;
4542 if (match(V: Op0, P: m_ElementWiseBitCast(Op: m_Value(V&: CastOp))) &&
4543 match(V: Op1, P: m_SignMask()) &&
4544 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
4545 Kind: Attribute::NoImplicitFloat)) {
4546 Type *EltTy = CastOp->getType()->getScalarType();
4547 if (EltTy->isFloatingPointTy() &&
4548 APFloat::hasSignBitInMSB(EltTy->getFltSemantics())) {
4549 Value *FAbs = Builder.CreateUnaryIntrinsic(ID: Intrinsic::fabs, V: CastOp);
4550 Value *FNegFAbs = Builder.CreateFNeg(V: FAbs);
4551 return new BitCastInst(FNegFAbs, I.getType());
4552 }
4553 }
4554
4555 // (X & C1) | C2 -> X & (C1 | C2) iff (X & C2) == C2
4556 if (match(V: Op0, P: m_OneUse(SubPattern: m_And(L: m_Value(V&: X), R: m_APInt(Res&: C1)))) &&
4557 match(V: Op1, P: m_APInt(Res&: C2))) {
4558 KnownBits KnownX = computeKnownBits(V: X, CxtI: &I);
4559 if ((KnownX.One & *C2) == *C2)
4560 return BinaryOperator::CreateAnd(V1: X, V2: ConstantInt::get(Ty, V: *C1 | *C2));
4561 }
4562
4563 if (Instruction *Res = foldBitwiseLogicWithIntrinsics(I, Builder))
4564 return Res;
4565
4566 if (Value *V =
4567 simplifyAndOrWithOpReplaced(V: Op0, Op: Op1, RepOp: Constant::getNullValue(Ty),
4568 /*SimplifyOnly*/ false, IC&: *this))
4569 return BinaryOperator::CreateOr(V1: V, V2: Op1);
4570 if (Value *V =
4571 simplifyAndOrWithOpReplaced(V: Op1, Op: Op0, RepOp: Constant::getNullValue(Ty),
4572 /*SimplifyOnly*/ false, IC&: *this))
4573 return BinaryOperator::CreateOr(V1: Op0, V2: V);
4574
4575 if (cast<PossiblyDisjointInst>(Val&: I).isDisjoint())
4576 if (Value *V = SimplifyAddWithRemainder(I))
4577 return replaceInstUsesWith(I, V);
4578
4579 if (Value *Res = FoldOrOfSelectSmaxToAbs(I, Builder))
4580 return replaceInstUsesWith(I, V: Res);
4581
4582 return nullptr;
4583}
4584
4585/// A ^ B can be specified using other logic ops in a variety of patterns. We
4586/// can fold these early and efficiently by morphing an existing instruction.
4587static Instruction *foldXorToXor(BinaryOperator &I,
4588 InstCombiner::BuilderTy &Builder) {
4589 assert(I.getOpcode() == Instruction::Xor);
4590 Value *Op0 = I.getOperand(i_nocapture: 0);
4591 Value *Op1 = I.getOperand(i_nocapture: 1);
4592 Value *A, *B;
4593
4594 // There are 4 commuted variants for each of the basic patterns.
4595
4596 // (A & B) ^ (A | B) -> A ^ B
4597 // (A & B) ^ (B | A) -> A ^ B
4598 // (A | B) ^ (A & B) -> A ^ B
4599 // (A | B) ^ (B & A) -> A ^ B
4600 if (match(V: &I, P: m_c_Xor(L: m_And(L: m_Value(V&: A), R: m_Value(V&: B)),
4601 R: m_c_Or(L: m_Deferred(V: A), R: m_Deferred(V: B)))))
4602 return BinaryOperator::CreateXor(V1: A, V2: B);
4603
4604 // (A | ~B) ^ (~A | B) -> A ^ B
4605 // (~B | A) ^ (~A | B) -> A ^ B
4606 // (~A | B) ^ (A | ~B) -> A ^ B
4607 // (B | ~A) ^ (A | ~B) -> A ^ B
4608 if (match(V: &I, P: m_Xor(L: m_c_Or(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B))),
4609 R: m_c_Or(L: m_Not(V: m_Deferred(V: A)), R: m_Deferred(V: B)))))
4610 return BinaryOperator::CreateXor(V1: A, V2: B);
4611
4612 // (A & ~B) ^ (~A & B) -> A ^ B
4613 // (~B & A) ^ (~A & B) -> A ^ B
4614 // (~A & B) ^ (A & ~B) -> A ^ B
4615 // (B & ~A) ^ (A & ~B) -> A ^ B
4616 if (match(V: &I, P: m_Xor(L: m_c_And(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B))),
4617 R: m_c_And(L: m_Not(V: m_Deferred(V: A)), R: m_Deferred(V: B)))))
4618 return BinaryOperator::CreateXor(V1: A, V2: B);
4619
4620 // For the remaining cases we need to get rid of one of the operands.
4621 if (!Op0->hasOneUse() && !Op1->hasOneUse())
4622 return nullptr;
4623
4624 // (A | B) ^ ~(A & B) -> ~(A ^ B)
4625 // (A | B) ^ ~(B & A) -> ~(A ^ B)
4626 // (A & B) ^ ~(A | B) -> ~(A ^ B)
4627 // (A & B) ^ ~(B | A) -> ~(A ^ B)
4628 // Complexity sorting ensures the not will be on the right side.
4629 if ((match(V: Op0, P: m_Or(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4630 match(V: Op1, P: m_Not(V: m_c_And(L: m_Specific(V: A), R: m_Specific(V: B))))) ||
4631 (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4632 match(V: Op1, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))))
4633 return BinaryOperator::CreateNot(Op: Builder.CreateXor(LHS: A, RHS: B));
4634
4635 return nullptr;
4636}
4637
4638Value *InstCombinerImpl::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS,
4639 BinaryOperator &I) {
4640 assert(I.getOpcode() == Instruction::Xor && I.getOperand(0) == LHS &&
4641 I.getOperand(1) == RHS && "Should be 'xor' with these operands");
4642
4643 ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
4644 Value *LHS0 = LHS->getOperand(i_nocapture: 0), *LHS1 = LHS->getOperand(i_nocapture: 1);
4645 Value *RHS0 = RHS->getOperand(i_nocapture: 0), *RHS1 = RHS->getOperand(i_nocapture: 1);
4646
4647 if (predicatesFoldable(P1: PredL, P2: PredR)) {
4648 if (LHS0 == RHS1 && LHS1 == RHS0) {
4649 std::swap(a&: LHS0, b&: LHS1);
4650 PredL = ICmpInst::getSwappedPredicate(pred: PredL);
4651 }
4652 if (LHS0 == RHS0 && LHS1 == RHS1) {
4653 // (icmp1 A, B) ^ (icmp2 A, B) --> (icmp3 A, B)
4654 unsigned Code = getICmpCode(Pred: PredL) ^ getICmpCode(Pred: PredR);
4655 bool IsSigned = LHS->isSigned() || RHS->isSigned();
4656 return getNewICmpValue(Code, Sign: IsSigned, LHS: LHS0, RHS: LHS1, Builder);
4657 }
4658 }
4659
4660 const APInt *LC, *RC;
4661 if (match(V: LHS1, P: m_APInt(Res&: LC)) && match(V: RHS1, P: m_APInt(Res&: RC)) &&
4662 LHS0->getType() == RHS0->getType() &&
4663 LHS0->getType()->isIntOrIntVectorTy()) {
4664 // Convert xor of signbit tests to signbit test of xor'd values:
4665 // (X > -1) ^ (Y > -1) --> (X ^ Y) < 0
4666 // (X < 0) ^ (Y < 0) --> (X ^ Y) < 0
4667 // (X > -1) ^ (Y < 0) --> (X ^ Y) > -1
4668 // (X < 0) ^ (Y > -1) --> (X ^ Y) > -1
4669 bool TrueIfSignedL, TrueIfSignedR;
4670 if ((LHS->hasOneUse() || RHS->hasOneUse()) &&
4671 isSignBitCheck(Pred: PredL, RHS: *LC, TrueIfSigned&: TrueIfSignedL) &&
4672 isSignBitCheck(Pred: PredR, RHS: *RC, TrueIfSigned&: TrueIfSignedR)) {
4673 Value *XorLR = Builder.CreateXor(LHS: LHS0, RHS: RHS0);
4674 return TrueIfSignedL == TrueIfSignedR ? Builder.CreateIsNeg(Arg: XorLR) :
4675 Builder.CreateIsNotNeg(Arg: XorLR);
4676 }
4677
4678 // Fold (icmp pred1 X, C1) ^ (icmp pred2 X, C2)
4679 // into a single comparison using range-based reasoning.
4680 if (LHS0 == RHS0) {
4681 ConstantRange CR1 = ConstantRange::makeExactICmpRegion(Pred: PredL, Other: *LC);
4682 ConstantRange CR2 = ConstantRange::makeExactICmpRegion(Pred: PredR, Other: *RC);
4683 auto CRUnion = CR1.exactUnionWith(CR: CR2);
4684 auto CRIntersect = CR1.exactIntersectWith(CR: CR2);
4685 if (CRUnion && CRIntersect)
4686 if (auto CR = CRUnion->exactIntersectWith(CR: CRIntersect->inverse())) {
4687 if (CR->isFullSet())
4688 return ConstantInt::getTrue(Ty: I.getType());
4689 if (CR->isEmptySet())
4690 return ConstantInt::getFalse(Ty: I.getType());
4691
4692 CmpInst::Predicate NewPred;
4693 APInt NewC, Offset;
4694 CR->getEquivalentICmp(Pred&: NewPred, RHS&: NewC, Offset);
4695
4696 if ((Offset.isZero() && (LHS->hasOneUse() || RHS->hasOneUse())) ||
4697 (LHS->hasOneUse() && RHS->hasOneUse())) {
4698 Value *NewV = LHS0;
4699 Type *Ty = LHS0->getType();
4700 if (!Offset.isZero())
4701 NewV = Builder.CreateAdd(LHS: NewV, RHS: ConstantInt::get(Ty, V: Offset));
4702 return Builder.CreateICmp(P: NewPred, LHS: NewV,
4703 RHS: ConstantInt::get(Ty, V: NewC));
4704 }
4705 }
4706 }
4707
4708 // Fold (icmp eq/ne (X & Pow2), 0) ^ (icmp eq/ne (Y & Pow2), 0) into
4709 // (icmp eq/ne ((X ^ Y) & Pow2), 0)
4710 Value *X, *Y, *Pow2;
4711 if (ICmpInst::isEquality(P: PredL) && ICmpInst::isEquality(P: PredR) &&
4712 LC->isZero() && RC->isZero() && LHS->hasOneUse() && RHS->hasOneUse() &&
4713 match(V: LHS0, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Pow2))) &&
4714 match(V: RHS0, P: m_And(L: m_Value(V&: Y), R: m_Specific(V: Pow2))) &&
4715 isKnownToBeAPowerOfTwo(V: Pow2, /*OrZero=*/true, CxtI: &I)) {
4716 Value *Xor = Builder.CreateXor(LHS: X, RHS: Y);
4717 Value *And = Builder.CreateAnd(LHS: Xor, RHS: Pow2);
4718 return Builder.CreateICmp(P: PredL == PredR ? ICmpInst::ICMP_NE
4719 : ICmpInst::ICMP_EQ,
4720 LHS: And, RHS: ConstantInt::getNullValue(Ty: Xor->getType()));
4721 }
4722 }
4723
4724 // Instead of trying to imitate the folds for and/or, decompose this 'xor'
4725 // into those logic ops. That is, try to turn this into an and-of-icmps
4726 // because we have many folds for that pattern.
4727 //
4728 // This is based on a truth table definition of xor:
4729 // X ^ Y --> (X | Y) & !(X & Y)
4730 if (Value *OrICmp = simplifyBinOp(Opcode: Instruction::Or, LHS, RHS, Q: SQ)) {
4731 // TODO: If OrICmp is true, then the definition of xor simplifies to !(X&Y).
4732 // TODO: If OrICmp is false, the whole thing is false (InstSimplify?).
4733 if (Value *AndICmp = simplifyBinOp(Opcode: Instruction::And, LHS, RHS, Q: SQ)) {
4734 // TODO: Independently handle cases where the 'and' side is a constant.
4735 ICmpInst *X = nullptr, *Y = nullptr;
4736 if (OrICmp == LHS && AndICmp == RHS) {
4737 // (LHS | RHS) & !(LHS & RHS) --> LHS & !RHS --> X & !Y
4738 X = LHS;
4739 Y = RHS;
4740 }
4741 if (OrICmp == RHS && AndICmp == LHS) {
4742 // !(LHS & RHS) & (LHS | RHS) --> !LHS & RHS --> !Y & X
4743 X = RHS;
4744 Y = LHS;
4745 }
4746 if (X && Y && (Y->hasOneUse() || canFreelyInvertAllUsersOf(V: Y, IgnoredUser: &I))) {
4747 // Invert the predicate of 'Y', thus inverting its output.
4748 Y->setPredicate(Y->getInversePredicate());
4749 // So, are there other uses of Y?
4750 if (!Y->hasOneUse()) {
4751 // We need to adapt other uses of Y though. Get a value that matches
4752 // the original value of Y before inversion. While this increases
4753 // immediate instruction count, we have just ensured that all the
4754 // users are freely-invertible, so that 'not' *will* get folded away.
4755 BuilderTy::InsertPointGuard Guard(Builder);
4756 // Set insertion point to right after the Y.
4757 Builder.SetInsertPoint(TheBB: Y->getParent(), IP: ++(Y->getIterator()));
4758 Value *NotY = Builder.CreateNot(V: Y, Name: Y->getName() + ".not");
4759 // Replace all uses of Y (excluding the one in NotY!) with NotY.
4760 Worklist.pushUsersToWorkList(I&: *Y);
4761 Y->replaceUsesWithIf(New: NotY,
4762 ShouldReplace: [NotY](Use &U) { return U.getUser() != NotY; });
4763 }
4764 // All done.
4765 return Builder.CreateAnd(LHS, RHS);
4766 }
4767 }
4768 }
4769
4770 return nullptr;
4771}
4772
4773/// If we have a masked merge, in the canonical form of:
4774/// (assuming that A only has one use.)
4775/// | A | |B|
4776/// ((x ^ y) & M) ^ y
4777/// | D |
4778/// * If M is inverted:
4779/// | D |
4780/// ((x ^ y) & ~M) ^ y
4781/// We can canonicalize by swapping the final xor operand
4782/// to eliminate the 'not' of the mask.
4783/// ((x ^ y) & M) ^ x
4784/// * If M is a constant, and D has one use, we transform to 'and' / 'or' ops
4785/// because that shortens the dependency chain and improves analysis:
4786/// (x & M) | (y & ~M)
4787static Instruction *visitMaskedMerge(BinaryOperator &I,
4788 InstCombiner::BuilderTy &Builder) {
4789 Value *B, *X, *D;
4790 Value *M;
4791 if (!match(V: &I, P: m_c_Xor(L: m_Value(V&: B),
4792 R: m_OneUse(SubPattern: m_c_And(
4793 L: m_Value(V&: D, Match: m_c_Xor(L: m_Deferred(V: B), R: m_Value(V&: X))),
4794 R: m_Value(V&: M))))))
4795 return nullptr;
4796
4797 Value *NotM;
4798 if (match(V: M, P: m_Not(V: m_Value(V&: NotM)))) {
4799 // De-invert the mask and swap the value in B part.
4800 Value *NewA = Builder.CreateAnd(LHS: D, RHS: NotM);
4801 return BinaryOperator::CreateXor(V1: NewA, V2: X);
4802 }
4803
4804 Constant *C;
4805 if (D->hasOneUse() && match(V: M, P: m_Constant(C))) {
4806 // Propagating undef is unsafe. Clamp undef elements to -1.
4807 Type *EltTy = C->getType()->getScalarType();
4808 C = Constant::replaceUndefsWith(C, Replacement: ConstantInt::getAllOnesValue(Ty: EltTy));
4809 // Unfold.
4810 Value *LHS = Builder.CreateAnd(LHS: X, RHS: C);
4811 Value *NotC = Builder.CreateNot(V: C);
4812 Value *RHS = Builder.CreateAnd(LHS: B, RHS: NotC);
4813 return BinaryOperator::CreateOr(V1: LHS, V2: RHS);
4814 }
4815
4816 return nullptr;
4817}
4818
4819static Instruction *foldNotXor(BinaryOperator &I,
4820 InstCombiner::BuilderTy &Builder) {
4821 Value *X, *Y;
4822 // FIXME: one-use check is not needed in general, but currently we are unable
4823 // to fold 'not' into 'icmp', if that 'icmp' has multiple uses. (D35182)
4824 if (!match(V: &I, P: m_Not(V: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: X), R: m_Value(V&: Y))))))
4825 return nullptr;
4826
4827 auto hasCommonOperand = [](Value *A, Value *B, Value *C, Value *D) {
4828 return A == C || A == D || B == C || B == D;
4829 };
4830
4831 Value *A, *B, *C, *D;
4832 // Canonicalize ~((A & B) ^ (A | ?)) -> (A & B) | ~(A | ?)
4833 // 4 commuted variants
4834 if (match(V: X, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4835 match(V: Y, P: m_Or(L: m_Value(V&: C), R: m_Value(V&: D))) && hasCommonOperand(A, B, C, D)) {
4836 Value *NotY = Builder.CreateNot(V: Y);
4837 return BinaryOperator::CreateOr(V1: X, V2: NotY);
4838 };
4839
4840 // Canonicalize ~((A | ?) ^ (A & B)) -> (A & B) | ~(A | ?)
4841 // 4 commuted variants
4842 if (match(V: Y, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4843 match(V: X, P: m_Or(L: m_Value(V&: C), R: m_Value(V&: D))) && hasCommonOperand(A, B, C, D)) {
4844 Value *NotX = Builder.CreateNot(V: X);
4845 return BinaryOperator::CreateOr(V1: Y, V2: NotX);
4846 };
4847
4848 return nullptr;
4849}
4850
4851/// Canonicalize a shifty way to code absolute value to the more common pattern
4852/// that uses negation and select.
4853static Instruction *canonicalizeAbs(BinaryOperator &Xor,
4854 InstCombiner::BuilderTy &Builder) {
4855 assert(Xor.getOpcode() == Instruction::Xor && "Expected an xor instruction.");
4856
4857 // There are 4 potential commuted variants. Move the 'ashr' candidate to Op1.
4858 // We're relying on the fact that we only do this transform when the shift has
4859 // exactly 2 uses and the add has exactly 1 use (otherwise, we might increase
4860 // instructions).
4861 Value *Op0 = Xor.getOperand(i_nocapture: 0), *Op1 = Xor.getOperand(i_nocapture: 1);
4862 if (Op0->hasNUses(N: 2))
4863 std::swap(a&: Op0, b&: Op1);
4864
4865 Type *Ty = Xor.getType();
4866 Value *A;
4867 const APInt *ShAmt;
4868 if (match(V: Op1, P: m_AShr(L: m_Value(V&: A), R: m_APInt(Res&: ShAmt))) &&
4869 Op1->hasNUses(N: 2) && *ShAmt == Ty->getScalarSizeInBits() - 1 &&
4870 match(V: Op0, P: m_OneUse(SubPattern: m_c_Add(L: m_Specific(V: A), R: m_Specific(V: Op1))))) {
4871 // Op1 = ashr i32 A, 31 ; smear the sign bit
4872 // xor (add A, Op1), Op1 ; add -1 and flip bits if negative
4873 // --> (A < 0) ? -A : A
4874 Value *IsNeg = Builder.CreateIsNeg(Arg: A);
4875 // Copy the nsw flags from the add to the negate.
4876 auto *Add = cast<BinaryOperator>(Val: Op0);
4877 Value *NegA = Add->hasNoUnsignedWrap()
4878 ? Constant::getNullValue(Ty: A->getType())
4879 : Builder.CreateNeg(V: A, Name: "", HasNSW: Add->hasNoSignedWrap());
4880 return SelectInst::Create(C: IsNeg, S1: NegA, S2: A);
4881 }
4882 return nullptr;
4883}
4884
4885static bool canFreelyInvert(InstCombiner &IC, Value *Op,
4886 Instruction *IgnoredUser) {
4887 auto *I = dyn_cast<Instruction>(Val: Op);
4888 return I && IC.isFreeToInvert(V: I, /*WillInvertAllUses=*/true) &&
4889 IC.canFreelyInvertAllUsersOf(V: I, IgnoredUser);
4890}
4891
4892static Value *freelyInvert(InstCombinerImpl &IC, Value *Op,
4893 Instruction *IgnoredUser) {
4894 auto *I = cast<Instruction>(Val: Op);
4895 IC.Builder.SetInsertPoint(*I->getInsertionPointAfterDef());
4896 Value *NotOp = IC.Builder.CreateNot(V: Op, Name: Op->getName() + ".not");
4897 Op->replaceUsesWithIf(New: NotOp,
4898 ShouldReplace: [NotOp](Use &U) { return U.getUser() != NotOp; });
4899 IC.freelyInvertAllUsersOf(V: NotOp, IgnoredUser);
4900 return NotOp;
4901}
4902
4903// Transform
4904// z = ~(x &/| y)
4905// into:
4906// z = ((~x) |/& (~y))
4907// iff both x and y are free to invert and all uses of z can be freely updated.
4908bool InstCombinerImpl::sinkNotIntoLogicalOp(Instruction &I) {
4909 Value *Op0, *Op1;
4910 if (!match(V: &I, P: m_LogicalOp(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
4911 return false;
4912
4913 // If this logic op has not been simplified yet, just bail out and let that
4914 // happen first. Otherwise, the code below may wrongly invert.
4915 if (Op0 == Op1)
4916 return false;
4917
4918 // If one of the operands is a user of the other,
4919 // freelyInvert->freelyInvertAllUsersOf will change the operands of I, which
4920 // may cause miscompilation.
4921 if (match(V: Op0, P: m_Not(V: m_Specific(V: Op1))) || match(V: Op1, P: m_Not(V: m_Specific(V: Op0))))
4922 return false;
4923
4924 Instruction::BinaryOps NewOpc =
4925 match(V: &I, P: m_LogicalAnd()) ? Instruction::Or : Instruction::And;
4926 bool IsBinaryOp = isa<BinaryOperator>(Val: I);
4927
4928 // Can our users be adapted?
4929 if (!InstCombiner::canFreelyInvertAllUsersOf(V: &I, /*IgnoredUser=*/nullptr))
4930 return false;
4931
4932 // And can the operands be adapted?
4933 if (!canFreelyInvert(IC&: *this, Op: Op0, IgnoredUser: &I) || !canFreelyInvert(IC&: *this, Op: Op1, IgnoredUser: &I))
4934 return false;
4935
4936 Op0 = freelyInvert(IC&: *this, Op: Op0, IgnoredUser: &I);
4937 Op1 = freelyInvert(IC&: *this, Op: Op1, IgnoredUser: &I);
4938
4939 Builder.SetInsertPoint(*I.getInsertionPointAfterDef());
4940 Value *NewLogicOp;
4941 if (IsBinaryOp) {
4942 NewLogicOp = Builder.CreateBinOp(Opc: NewOpc, LHS: Op0, RHS: Op1, Name: I.getName() + ".not");
4943 } else {
4944 NewLogicOp =
4945 Builder.CreateLogicalOp(Opc: NewOpc, Cond1: Op0, Cond2: Op1, Name: I.getName() + ".not",
4946 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &I);
4947 if (SelectInst *SI = dyn_cast<SelectInst>(Val: NewLogicOp))
4948 SI->swapProfMetadata();
4949 }
4950
4951 replaceInstUsesWith(I, V: NewLogicOp);
4952 // We can not just create an outer `not`, it will most likely be immediately
4953 // folded back, reconstructing our initial pattern, and causing an
4954 // infinite combine loop, so immediately manually fold it away.
4955 freelyInvertAllUsersOf(V: NewLogicOp);
4956 return true;
4957}
4958
4959// Transform
4960// z = (~x) &/| y
4961// into:
4962// z = ~(x |/& (~y))
4963// iff y is free to invert and all uses of z can be freely updated.
4964bool InstCombinerImpl::sinkNotIntoOtherHandOfLogicalOp(Instruction &I) {
4965 Value *Op0, *Op1;
4966 if (!match(V: &I, P: m_LogicalOp(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
4967 return false;
4968 Instruction::BinaryOps NewOpc =
4969 match(V: &I, P: m_LogicalAnd()) ? Instruction::Or : Instruction::And;
4970 bool IsBinaryOp = isa<BinaryOperator>(Val: I);
4971
4972 Value *NotOp0 = nullptr;
4973 Value *NotOp1 = nullptr;
4974 Value **OpToInvert = nullptr;
4975 if (match(V: Op0, P: m_Not(V: m_Value(V&: NotOp0))) && canFreelyInvert(IC&: *this, Op: Op1, IgnoredUser: &I)) {
4976 Op0 = NotOp0;
4977 OpToInvert = &Op1;
4978 } else if (match(V: Op1, P: m_Not(V: m_Value(V&: NotOp1))) &&
4979 canFreelyInvert(IC&: *this, Op: Op0, IgnoredUser: &I)) {
4980 Op1 = NotOp1;
4981 OpToInvert = &Op0;
4982 } else
4983 return false;
4984
4985 // And can our users be adapted?
4986 if (!InstCombiner::canFreelyInvertAllUsersOf(V: &I, /*IgnoredUser=*/nullptr))
4987 return false;
4988
4989 *OpToInvert = freelyInvert(IC&: *this, Op: *OpToInvert, IgnoredUser: &I);
4990
4991 Builder.SetInsertPoint(*I.getInsertionPointAfterDef());
4992 Value *NewBinOp;
4993 if (IsBinaryOp)
4994 NewBinOp = Builder.CreateBinOp(Opc: NewOpc, LHS: Op0, RHS: Op1, Name: I.getName() + ".not");
4995 else
4996 NewBinOp = Builder.CreateLogicalOp(Opc: NewOpc, Cond1: Op0, Cond2: Op1, Name: I.getName() + ".not");
4997 replaceInstUsesWith(I, V: NewBinOp);
4998 // We can not just create an outer `not`, it will most likely be immediately
4999 // folded back, reconstructing our initial pattern, and causing an
5000 // infinite combine loop, so immediately manually fold it away.
5001 freelyInvertAllUsersOf(V: NewBinOp);
5002 return true;
5003}
5004
5005Instruction *InstCombinerImpl::foldNot(BinaryOperator &I) {
5006 Value *NotOp;
5007 if (!match(V: &I, P: m_Not(V: m_Value(V&: NotOp))))
5008 return nullptr;
5009
5010 // Apply DeMorgan's Law for 'nand' / 'nor' logic with an inverted operand.
5011 // We must eliminate the and/or (one-use) for these transforms to not increase
5012 // the instruction count.
5013 //
5014 // ~(~X & Y) --> (X | ~Y)
5015 // ~(Y & ~X) --> (X | ~Y)
5016 //
5017 // Note: The logical matches do not check for the commuted patterns because
5018 // those are handled via SimplifySelectsFeedingBinaryOp().
5019 Type *Ty = I.getType();
5020 Value *X, *Y;
5021 if (match(V: NotOp, P: m_OneUse(SubPattern: m_c_And(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))) {
5022 Value *NotY = Builder.CreateNot(V: Y, Name: Y->getName() + ".not");
5023 return BinaryOperator::CreateOr(V1: X, V2: NotY);
5024 }
5025 if (match(V: NotOp, P: m_OneUse(SubPattern: m_LogicalAnd(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))) {
5026 Value *NotY = Builder.CreateNot(V: Y, Name: Y->getName() + ".not");
5027 SelectInst *SI = SelectInst::Create(
5028 C: X, S1: ConstantInt::getTrue(Ty), S2: NotY, NameStr: "", InsertBefore: nullptr,
5029 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : cast<Instruction>(Val: NotOp));
5030 SI->swapProfMetadata();
5031 return SI;
5032 }
5033
5034 // ~(~X | Y) --> (X & ~Y)
5035 // ~(Y | ~X) --> (X & ~Y)
5036 if (match(V: NotOp, P: m_OneUse(SubPattern: m_c_Or(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))) {
5037 Value *NotY = Builder.CreateNot(V: Y, Name: Y->getName() + ".not");
5038 return BinaryOperator::CreateAnd(V1: X, V2: NotY);
5039 }
5040 if (match(V: NotOp, P: m_OneUse(SubPattern: m_LogicalOr(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))) {
5041 Value *NotY = Builder.CreateNot(V: Y, Name: Y->getName() + ".not");
5042 SelectInst *SI = SelectInst::Create(
5043 C: X, S1: NotY, S2: ConstantInt::getFalse(Ty), NameStr: "", InsertBefore: nullptr,
5044 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : cast<Instruction>(Val: NotOp));
5045 SI->swapProfMetadata();
5046 return SI;
5047 }
5048
5049 // Is this a 'not' (~) fed by a binary operator?
5050 BinaryOperator *NotVal;
5051 if (match(V: NotOp, P: m_BinOp(I&: NotVal))) {
5052 // ~((-X) | Y) --> (X - 1) & (~Y)
5053 if (match(V: NotVal,
5054 P: m_OneUse(SubPattern: m_c_Or(L: m_OneUse(SubPattern: m_Neg(V: m_Value(V&: X))), R: m_Value(V&: Y))))) {
5055 Value *DecX = Builder.CreateAdd(LHS: X, RHS: ConstantInt::getAllOnesValue(Ty));
5056 Value *NotY = Builder.CreateNot(V: Y);
5057 return BinaryOperator::CreateAnd(V1: DecX, V2: NotY);
5058 }
5059
5060 // ~(~X >>s Y) --> (X >>s Y)
5061 if (match(V: NotVal, P: m_AShr(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))
5062 return BinaryOperator::CreateAShr(V1: X, V2: Y);
5063
5064 // Treat lshr with non-negative operand as ashr.
5065 // ~(~X >>u Y) --> (X >>s Y) iff X is known negative
5066 if (match(V: NotVal, P: m_LShr(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))) &&
5067 isKnownNegative(V: X, SQ: SQ.getWithInstruction(I: NotVal)))
5068 return BinaryOperator::CreateAShr(V1: X, V2: Y);
5069
5070 // Bit-hack form of a signbit test for iN type:
5071 // ~(X >>s (N - 1)) --> sext i1 (X > -1) to iN
5072 unsigned FullShift = Ty->getScalarSizeInBits() - 1;
5073 if (match(V: NotVal, P: m_OneUse(SubPattern: m_AShr(L: m_Value(V&: X), R: m_SpecificInt(V: FullShift))))) {
5074 Value *IsNotNeg = Builder.CreateIsNotNeg(Arg: X, Name: "isnotneg");
5075 return new SExtInst(IsNotNeg, Ty);
5076 }
5077
5078 // If we are inverting a right-shifted constant, we may be able to eliminate
5079 // the 'not' by inverting the constant and using the opposite shift type.
5080 // Canonicalization rules ensure that only a negative constant uses 'ashr',
5081 // but we must check that in case that transform has not fired yet.
5082
5083 // ~(C >>s Y) --> ~C >>u Y (when inverting the replicated sign bits)
5084 Constant *C;
5085 if (match(V: NotVal, P: m_AShr(L: m_Constant(C), R: m_Value(V&: Y))) &&
5086 match(V: C, P: m_Negative()))
5087 return BinaryOperator::CreateLShr(V1: ConstantExpr::getNot(C), V2: Y);
5088
5089 // ~(C >>u Y) --> ~C >>s Y (when inverting the replicated sign bits)
5090 if (match(V: NotVal, P: m_LShr(L: m_Constant(C), R: m_Value(V&: Y))) &&
5091 match(V: C, P: m_NonNegative()))
5092 return BinaryOperator::CreateAShr(V1: ConstantExpr::getNot(C), V2: Y);
5093
5094 // ~(X + C) --> ~C - X
5095 if (match(V: NotVal, P: m_Add(L: m_Value(V&: X), R: m_ImmConstant(C))))
5096 return BinaryOperator::CreateSub(V1: ConstantExpr::getNot(C), V2: X);
5097
5098 // ~(X - Y) --> ~X + Y
5099 // FIXME: is it really beneficial to sink the `not` here?
5100 if (match(V: NotVal, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y))))
5101 if (isa<Constant>(Val: X) || NotVal->hasOneUse())
5102 return BinaryOperator::CreateAdd(V1: Builder.CreateNot(V: X), V2: Y);
5103
5104 // ~(~X + Y) --> X - Y
5105 if (match(V: NotVal, P: m_c_Add(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))
5106 return BinaryOperator::CreateWithCopiedFlags(Opc: Instruction::Sub, V1: X, V2: Y,
5107 CopyO: NotVal);
5108 }
5109
5110 // not (cmp A, B) = !cmp A, B
5111 CmpPredicate Pred;
5112 if (match(V: NotOp, P: m_Cmp(Pred, L: m_Value(), R: m_Value())) &&
5113 (NotOp->hasOneUse() ||
5114 InstCombiner::canFreelyInvertAllUsersOf(V: cast<Instruction>(Val: NotOp),
5115 /*IgnoredUser=*/nullptr))) {
5116 cast<CmpInst>(Val: NotOp)->setPredicate(CmpInst::getInversePredicate(pred: Pred));
5117 freelyInvertAllUsersOf(V: NotOp);
5118 return &I;
5119 }
5120
5121 // not (bitcast (cmp A, B) --> bitcast (!cmp A, B)
5122 if (match(V: NotOp, P: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: X)))) &&
5123 match(V: X, P: m_OneUse(SubPattern: m_Cmp(Pred, L: m_Value(), R: m_Value())))) {
5124 cast<CmpInst>(Val: X)->setPredicate(CmpInst::getInversePredicate(pred: Pred));
5125 return new BitCastInst(X, Ty);
5126 }
5127
5128 // Move a 'not' ahead of casts of a bool to enable logic reduction:
5129 // not (bitcast (sext i1 X)) --> bitcast (sext (not i1 X))
5130 if (match(V: NotOp, P: m_OneUse(SubPattern: m_BitCast(Op: m_OneUse(SubPattern: m_SExt(Op: m_Value(V&: X)))))) &&
5131 X->getType()->isIntOrIntVectorTy(BitWidth: 1)) {
5132 Type *SextTy = cast<BitCastOperator>(Val: NotOp)->getSrcTy();
5133 Value *NotX = Builder.CreateNot(V: X);
5134 Value *Sext = Builder.CreateSExt(V: NotX, DestTy: SextTy);
5135 return new BitCastInst(Sext, Ty);
5136 }
5137
5138 if (auto *NotOpI = dyn_cast<Instruction>(Val: NotOp))
5139 if (sinkNotIntoLogicalOp(I&: *NotOpI))
5140 return &I;
5141
5142 // Eliminate a bitwise 'not' op of 'not' min/max by inverting the min/max:
5143 // ~min(~X, ~Y) --> max(X, Y)
5144 // ~max(~X, Y) --> min(X, ~Y)
5145 auto *II = dyn_cast<IntrinsicInst>(Val: NotOp);
5146 if (II && II->hasOneUse()) {
5147 if (match(V: NotOp, P: m_c_MaxOrMin(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y)))) {
5148 Intrinsic::ID InvID = getInverseMinMaxIntrinsic(MinMaxID: II->getIntrinsicID());
5149 Value *NotY = Builder.CreateNot(V: Y);
5150 Value *InvMaxMin = Builder.CreateBinaryIntrinsic(ID: InvID, LHS: X, RHS: NotY);
5151 return replaceInstUsesWith(I, V: InvMaxMin);
5152 }
5153
5154 if (II->getIntrinsicID() == Intrinsic::is_fpclass) {
5155 ConstantInt *ClassMask = cast<ConstantInt>(Val: II->getArgOperand(i: 1));
5156 II->setArgOperand(
5157 i: 1, v: ConstantInt::get(Ty: ClassMask->getType(),
5158 V: ~ClassMask->getZExtValue() & fcAllFlags));
5159 return replaceInstUsesWith(I, V: II);
5160 }
5161 }
5162
5163 if (NotOp->hasOneUse()) {
5164 // Pull 'not' into operands of select if both operands are one-use compares
5165 // or one is one-use compare and the other one is a constant.
5166 // Inverting the predicates eliminates the 'not' operation.
5167 // Example:
5168 // not (select ?, (cmp TPred, ?, ?), (cmp FPred, ?, ?) -->
5169 // select ?, (cmp InvTPred, ?, ?), (cmp InvFPred, ?, ?)
5170 // not (select ?, (cmp TPred, ?, ?), true -->
5171 // select ?, (cmp InvTPred, ?, ?), false
5172 if (auto *Sel = dyn_cast<SelectInst>(Val: NotOp)) {
5173 Value *TV = Sel->getTrueValue();
5174 Value *FV = Sel->getFalseValue();
5175 auto *CmpT = dyn_cast<CmpInst>(Val: TV);
5176 auto *CmpF = dyn_cast<CmpInst>(Val: FV);
5177 bool InvertibleT = (CmpT && CmpT->hasOneUse()) || isa<Constant>(Val: TV);
5178 bool InvertibleF = (CmpF && CmpF->hasOneUse()) || isa<Constant>(Val: FV);
5179 if (InvertibleT && InvertibleF) {
5180 if (CmpT)
5181 CmpT->setPredicate(CmpT->getInversePredicate());
5182 else
5183 Sel->setTrueValue(ConstantExpr::getNot(C: cast<Constant>(Val: TV)));
5184 if (CmpF)
5185 CmpF->setPredicate(CmpF->getInversePredicate());
5186 else
5187 Sel->setFalseValue(ConstantExpr::getNot(C: cast<Constant>(Val: FV)));
5188 return replaceInstUsesWith(I, V: Sel);
5189 }
5190 }
5191 }
5192
5193 if (Instruction *NewXor = foldNotXor(I, Builder))
5194 return NewXor;
5195
5196 // TODO: Could handle multi-use better by checking if all uses of NotOp (other
5197 // than I) can be inverted.
5198 if (Value *R = getFreelyInverted(V: NotOp, WillInvertAllUses: NotOp->hasOneUse(), Builder: &Builder))
5199 return replaceInstUsesWith(I, V: R);
5200
5201 return nullptr;
5202}
5203
5204// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
5205// here. We should standardize that construct where it is needed or choose some
5206// other way to ensure that commutated variants of patterns are not missed.
5207Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
5208 if (Value *V = simplifyXorInst(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1),
5209 Q: SQ.getWithInstruction(I: &I)))
5210 return replaceInstUsesWith(I, V);
5211
5212 if (SimplifyAssociativeOrCommutative(I))
5213 return &I;
5214
5215 if (Instruction *X = foldVectorBinop(Inst&: I))
5216 return X;
5217
5218 if (Instruction *Phi = foldBinopWithPhiOperands(BO&: I))
5219 return Phi;
5220
5221 if (Instruction *NewXor = foldXorToXor(I, Builder))
5222 return NewXor;
5223
5224 // (A&B)^(A&C) -> A&(B^C) etc
5225 if (Value *V = foldUsingDistributiveLaws(I))
5226 return replaceInstUsesWith(I, V);
5227
5228 // See if we can simplify any instructions used by the instruction whose sole
5229 // purpose is to compute bits we don't care about.
5230 if (SimplifyDemandedInstructionBits(Inst&: I))
5231 return &I;
5232
5233 if (Instruction *R = foldNot(I))
5234 return R;
5235
5236 if (Instruction *R = foldBinOpShiftWithShift(I))
5237 return R;
5238
5239 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
5240 Value *X, *Y, *M;
5241
5242 // (X | Y) ^ M -> (X ^ M) ^ Y
5243 // (X | Y) ^ M -> (Y ^ M) ^ X
5244 if (match(V: &I, P: m_c_Xor(L: m_OneUse(SubPattern: m_DisjointOr(L: m_Value(V&: X), R: m_Value(V&: Y))),
5245 R: m_Value(V&: M)))) {
5246 if (Value *XorAC = simplifyXorInst(LHS: X, RHS: M, Q: SQ.getWithInstruction(I: &I)))
5247 return BinaryOperator::CreateXor(V1: XorAC, V2: Y);
5248
5249 if (Value *XorBC = simplifyXorInst(LHS: Y, RHS: M, Q: SQ.getWithInstruction(I: &I)))
5250 return BinaryOperator::CreateXor(V1: XorBC, V2: X);
5251 }
5252
5253 // Fold (X & M) ^ (Y & ~M) -> (X & M) | (Y & ~M)
5254 // This it a special case in haveNoCommonBitsSet, but the computeKnownBits
5255 // calls in there are unnecessary as SimplifyDemandedInstructionBits should
5256 // have already taken care of those cases.
5257 if (match(V: &I, P: m_c_Xor(L: m_c_And(L: m_Not(V: m_Value(V&: M)), R: m_Value()),
5258 R: m_c_And(L: m_Deferred(V: M), R: m_Value())))) {
5259 if (isGuaranteedNotToBeUndef(V: M))
5260 return BinaryOperator::CreateDisjointOr(V1: Op0, V2: Op1);
5261 else
5262 return BinaryOperator::CreateOr(V1: Op0, V2: Op1);
5263 }
5264
5265 if (Instruction *Xor = visitMaskedMerge(I, Builder))
5266 return Xor;
5267
5268 Constant *C1;
5269 if (match(V: Op1, P: m_Constant(C&: C1))) {
5270 Constant *C2;
5271
5272 if (match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: X), R: m_ImmConstant(C&: C2)))) &&
5273 match(V: C1, P: m_ImmConstant())) {
5274 // (X | C2) ^ C1 --> (X & ~C2) ^ (C1^C2)
5275 C2 = Constant::replaceUndefsWith(
5276 C: C2, Replacement: Constant::getAllOnesValue(Ty: C2->getType()->getScalarType()));
5277 Value *And = Builder.CreateAnd(
5278 LHS: X, RHS: Constant::mergeUndefsWith(C: ConstantExpr::getNot(C: C2), Other: C1));
5279 return BinaryOperator::CreateXor(
5280 V1: And, V2: Constant::mergeUndefsWith(C: ConstantExpr::getXor(C1, C2), Other: C1));
5281 }
5282
5283 // Use DeMorgan and reassociation to eliminate a 'not' op.
5284 if (match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Not(V: m_Value(V&: X)), R: m_Constant(C&: C2))))) {
5285 // (~X | C2) ^ C1 --> ((X & ~C2) ^ -1) ^ C1 --> (X & ~C2) ^ ~C1
5286 Value *And = Builder.CreateAnd(LHS: X, RHS: ConstantExpr::getNot(C: C2));
5287 return BinaryOperator::CreateXor(V1: And, V2: ConstantExpr::getNot(C: C1));
5288 }
5289 if (match(V: Op0, P: m_OneUse(SubPattern: m_And(L: m_Not(V: m_Value(V&: X)), R: m_Constant(C&: C2))))) {
5290 // (~X & C2) ^ C1 --> ((X | ~C2) ^ -1) ^ C1 --> (X | ~C2) ^ ~C1
5291 Value *Or = Builder.CreateOr(LHS: X, RHS: ConstantExpr::getNot(C: C2));
5292 return BinaryOperator::CreateXor(V1: Or, V2: ConstantExpr::getNot(C: C1));
5293 }
5294
5295 // Convert xor ([trunc] (ashr X, BW-1)), C =>
5296 // select(X >s -1, C, ~C)
5297 // The ashr creates "AllZeroOrAllOne's", which then optionally inverses the
5298 // constant depending on whether this input is less than 0.
5299 const APInt *CA;
5300 if (match(V: Op0, P: m_OneUse(SubPattern: m_TruncOrSelf(
5301 Op: m_AShr(L: m_Value(V&: X), R: m_APIntAllowPoison(Res&: CA))))) &&
5302 *CA == X->getType()->getScalarSizeInBits() - 1 &&
5303 !match(V: C1, P: m_AllOnes())) {
5304 assert(!C1->isZeroValue() && "Unexpected xor with 0");
5305 Value *IsNotNeg = Builder.CreateIsNotNeg(Arg: X);
5306 return createSelectInstWithUnknownProfile(C: IsNotNeg, S1: Op1,
5307 S2: Builder.CreateNot(V: Op1));
5308 }
5309 }
5310
5311 Type *Ty = I.getType();
5312 {
5313 const APInt *RHSC;
5314 if (match(V: Op1, P: m_APInt(Res&: RHSC))) {
5315 Value *X;
5316 const APInt *C;
5317 // (C - X) ^ signmaskC --> (C + signmaskC) - X
5318 if (RHSC->isSignMask() && match(V: Op0, P: m_Sub(L: m_APInt(Res&: C), R: m_Value(V&: X))))
5319 return BinaryOperator::CreateSub(V1: ConstantInt::get(Ty, V: *C + *RHSC), V2: X);
5320
5321 // (X + C) ^ signmaskC --> X + (C + signmaskC)
5322 if (RHSC->isSignMask() && match(V: Op0, P: m_Add(L: m_Value(V&: X), R: m_APInt(Res&: C))))
5323 return BinaryOperator::CreateAdd(V1: X, V2: ConstantInt::get(Ty, V: *C + *RHSC));
5324
5325 // (X | C) ^ RHSC --> X ^ (C ^ RHSC) iff X & C == 0
5326 if (match(V: Op0, P: m_Or(L: m_Value(V&: X), R: m_APInt(Res&: C))) &&
5327 MaskedValueIsZero(V: X, Mask: *C, CxtI: &I))
5328 return BinaryOperator::CreateXor(V1: X, V2: ConstantInt::get(Ty, V: *C ^ *RHSC));
5329
5330 // When X is a power-of-two or zero and zero input is poison:
5331 // ctlz(i32 X) ^ 31 --> cttz(X)
5332 // cttz(i32 X) ^ 31 --> ctlz(X)
5333 auto *II = dyn_cast<IntrinsicInst>(Val: Op0);
5334 if (II && II->hasOneUse() && *RHSC == Ty->getScalarSizeInBits() - 1) {
5335 Intrinsic::ID IID = II->getIntrinsicID();
5336 if ((IID == Intrinsic::ctlz || IID == Intrinsic::cttz) &&
5337 match(V: II->getArgOperand(i: 1), P: m_One()) &&
5338 isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), /*OrZero */ true)) {
5339 IID = (IID == Intrinsic::ctlz) ? Intrinsic::cttz : Intrinsic::ctlz;
5340 Function *F =
5341 Intrinsic::getOrInsertDeclaration(M: II->getModule(), id: IID, Tys: Ty);
5342 return CallInst::Create(Func: F, Args: {II->getArgOperand(i: 0), Builder.getTrue()});
5343 }
5344 }
5345
5346 // If RHSC is inverting the remaining bits of shifted X,
5347 // canonicalize to a 'not' before the shift to help SCEV and codegen:
5348 // (X << C) ^ RHSC --> ~X << C
5349 if (match(V: Op0, P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: X), R: m_APInt(Res&: C)))) &&
5350 *RHSC == APInt::getAllOnes(numBits: Ty->getScalarSizeInBits()).shl(ShiftAmt: *C)) {
5351 Value *NotX = Builder.CreateNot(V: X);
5352 return BinaryOperator::CreateShl(V1: NotX, V2: ConstantInt::get(Ty, V: *C));
5353 }
5354 // (X >>u C) ^ RHSC --> ~X >>u C
5355 if (match(V: Op0, P: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: C)))) &&
5356 *RHSC == APInt::getAllOnes(numBits: Ty->getScalarSizeInBits()).lshr(ShiftAmt: *C)) {
5357 Value *NotX = Builder.CreateNot(V: X);
5358 return BinaryOperator::CreateLShr(V1: NotX, V2: ConstantInt::get(Ty, V: *C));
5359 }
5360 // TODO: We could handle 'ashr' here as well. That would be matching
5361 // a 'not' op and moving it before the shift. Doing that requires
5362 // preventing the inverse fold in canShiftBinOpWithConstantRHS().
5363 }
5364
5365 // If we are XORing the sign bit of a floating-point value, convert
5366 // this to fneg, then cast back to integer.
5367 //
5368 // This is generous interpretation of noimplicitfloat, this is not a true
5369 // floating-point operation.
5370 //
5371 // Assumes any IEEE-represented type has the sign bit in the high bit.
5372 // TODO: Unify with APInt matcher. This version allows undef unlike m_APInt
5373 Value *CastOp;
5374 if (match(V: Op0, P: m_ElementWiseBitCast(Op: m_Value(V&: CastOp))) &&
5375 match(V: Op1, P: m_SignMask()) &&
5376 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
5377 Kind: Attribute::NoImplicitFloat)) {
5378 Type *EltTy = CastOp->getType()->getScalarType();
5379 if (EltTy->isFloatingPointTy() &&
5380 APFloat::hasSignBitInMSB(EltTy->getFltSemantics())) {
5381 Value *FNeg = Builder.CreateFNeg(V: CastOp);
5382 return new BitCastInst(FNeg, I.getType());
5383 }
5384 }
5385 }
5386
5387 // FIXME: This should not be limited to scalar (pull into APInt match above).
5388 {
5389 Value *X;
5390 ConstantInt *C1, *C2, *C3;
5391 // ((X^C1) >> C2) ^ C3 -> (X>>C2) ^ ((C1>>C2)^C3)
5392 if (match(V: Op1, P: m_ConstantInt(CI&: C3)) &&
5393 match(V: Op0, P: m_LShr(L: m_Xor(L: m_Value(V&: X), R: m_ConstantInt(CI&: C1)),
5394 R: m_ConstantInt(CI&: C2))) &&
5395 Op0->hasOneUse()) {
5396 // fold (C1 >> C2) ^ C3
5397 APInt FoldConst = C1->getValue().lshr(ShiftAmt: C2->getValue());
5398 FoldConst ^= C3->getValue();
5399 // Prepare the two operands.
5400 auto *Opnd0 = Builder.CreateLShr(LHS: X, RHS: C2);
5401 Opnd0->takeName(V: Op0);
5402 return BinaryOperator::CreateXor(V1: Opnd0, V2: ConstantInt::get(Ty, V: FoldConst));
5403 }
5404 }
5405
5406 if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
5407 return FoldedLogic;
5408
5409 if (Instruction *FoldedLogic = foldBinOpSelectBinOp(Op&: I))
5410 return FoldedLogic;
5411
5412 // Y ^ (X | Y) --> X & ~Y
5413 // Y ^ (Y | X) --> X & ~Y
5414 if (match(V: Op1, P: m_OneUse(SubPattern: m_c_Or(L: m_Value(V&: X), R: m_Specific(V: Op0)))))
5415 return BinaryOperator::CreateAnd(V1: X, V2: Builder.CreateNot(V: Op0));
5416 // (X | Y) ^ Y --> X & ~Y
5417 // (Y | X) ^ Y --> X & ~Y
5418 if (match(V: Op0, P: m_OneUse(SubPattern: m_c_Or(L: m_Value(V&: X), R: m_Specific(V: Op1)))))
5419 return BinaryOperator::CreateAnd(V1: X, V2: Builder.CreateNot(V: Op1));
5420
5421 // Y ^ (X & Y) --> ~X & Y
5422 // Y ^ (Y & X) --> ~X & Y
5423 if (match(V: Op1, P: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: X), R: m_Specific(V: Op0)))))
5424 return BinaryOperator::CreateAnd(V1: Op0, V2: Builder.CreateNot(V: X));
5425 // (X & Y) ^ Y --> ~X & Y
5426 // (Y & X) ^ Y --> ~X & Y
5427 // Canonical form is (X & C) ^ C; don't touch that.
5428 // TODO: A 'not' op is better for analysis and codegen, but demanded bits must
5429 // be fixed to prefer that (otherwise we get infinite looping).
5430 if (!match(V: Op1, P: m_Constant()) &&
5431 match(V: Op0, P: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: X), R: m_Specific(V: Op1)))))
5432 return BinaryOperator::CreateAnd(V1: Op1, V2: Builder.CreateNot(V: X));
5433
5434 Value *A, *B, *C;
5435 // (A ^ B) ^ (A | C) --> (~A & C) ^ B -- There are 4 commuted variants.
5436 if (match(V: &I, P: m_c_Xor(L: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))),
5437 R: m_OneUse(SubPattern: m_c_Or(L: m_Deferred(V: A), R: m_Value(V&: C))))))
5438 return BinaryOperator::CreateXor(
5439 V1: Builder.CreateAnd(LHS: Builder.CreateNot(V: A), RHS: C), V2: B);
5440
5441 // (A ^ B) ^ (B | C) --> (~B & C) ^ A -- There are 4 commuted variants.
5442 if (match(V: &I, P: m_c_Xor(L: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))),
5443 R: m_OneUse(SubPattern: m_c_Or(L: m_Deferred(V: B), R: m_Value(V&: C))))))
5444 return BinaryOperator::CreateXor(
5445 V1: Builder.CreateAnd(LHS: Builder.CreateNot(V: B), RHS: C), V2: A);
5446
5447 // (A & B) ^ (A ^ B) -> (A | B)
5448 if (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
5449 match(V: Op1, P: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B))))
5450 return BinaryOperator::CreateOr(V1: A, V2: B);
5451 // (A ^ B) ^ (A & B) -> (A | B)
5452 if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))) &&
5453 match(V: Op1, P: m_c_And(L: m_Specific(V: A), R: m_Specific(V: B))))
5454 return BinaryOperator::CreateOr(V1: A, V2: B);
5455
5456 // (A & ~B) ^ ~A -> ~(A & B)
5457 // (~B & A) ^ ~A -> ~(A & B)
5458 if (match(V: Op0, P: m_c_And(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B)))) &&
5459 match(V: Op1, P: m_Not(V: m_Specific(V: A))))
5460 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: A, RHS: B));
5461
5462 // (~A & B) ^ A --> A | B -- There are 4 commuted variants.
5463 if (match(V: &I, P: m_c_Xor(L: m_c_And(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B)), R: m_Deferred(V: A))))
5464 return BinaryOperator::CreateOr(V1: A, V2: B);
5465
5466 // (~A | B) ^ A --> ~(A & B)
5467 if (match(V: Op0, P: m_OneUse(SubPattern: m_c_Or(L: m_Not(V: m_Specific(V: Op1)), R: m_Value(V&: B)))))
5468 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: Op1, RHS: B));
5469
5470 // A ^ (~A | B) --> ~(A & B)
5471 if (match(V: Op1, P: m_OneUse(SubPattern: m_c_Or(L: m_Not(V: m_Specific(V: Op0)), R: m_Value(V&: B)))))
5472 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: Op0, RHS: B));
5473
5474 // (A | B) ^ (A | C) --> (B ^ C) & ~A -- There are 4 commuted variants.
5475 // TODO: Loosen one-use restriction if common operand is a constant.
5476 Value *D;
5477 if (match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: A), R: m_Value(V&: B)))) &&
5478 match(V: Op1, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: C), R: m_Value(V&: D))))) {
5479 if (B == C || B == D)
5480 std::swap(a&: A, b&: B);
5481 if (A == C)
5482 std::swap(a&: C, b&: D);
5483 if (A == D) {
5484 Value *NotA = Builder.CreateNot(V: A);
5485 return BinaryOperator::CreateAnd(V1: Builder.CreateXor(LHS: B, RHS: C), V2: NotA);
5486 }
5487 }
5488
5489 // (A & B) ^ (A | C) --> A ? ~B : C -- There are 4 commuted variants.
5490 if (I.getType()->isIntOrIntVectorTy(BitWidth: 1) &&
5491 match(V: &I, P: m_c_Xor(L: m_OneUse(SubPattern: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B))),
5492 R: m_OneUse(SubPattern: m_LogicalOr(L: m_Value(V&: C), R: m_Value(V&: D)))))) {
5493 bool NeedFreeze = isa<SelectInst>(Val: Op0) && isa<SelectInst>(Val: Op1) && B == D;
5494 Instruction *MDFrom = cast<Instruction>(Val: Op0);
5495 if (B == C || B == D) {
5496 std::swap(a&: A, b&: B);
5497 MDFrom = B == C ? cast<Instruction>(Val: Op1) : nullptr;
5498 }
5499 if (A == C)
5500 std::swap(a&: C, b&: D);
5501 if (A == D) {
5502 if (NeedFreeze)
5503 A = Builder.CreateFreeze(V: A);
5504 Value *NotB = Builder.CreateNot(V: B);
5505 return MDFrom == nullptr || ProfcheckDisableMetadataFixes
5506 ? createSelectInstWithUnknownProfile(C: A, S1: NotB, S2: C)
5507 : SelectInst::Create(C: A, S1: NotB, S2: C, NameStr: "", InsertBefore: nullptr, MDFrom);
5508 }
5509 }
5510
5511 if (auto *LHS = dyn_cast<ICmpInst>(Val: I.getOperand(i_nocapture: 0)))
5512 if (auto *RHS = dyn_cast<ICmpInst>(Val: I.getOperand(i_nocapture: 1)))
5513 if (Value *V = foldXorOfICmps(LHS, RHS, I))
5514 return replaceInstUsesWith(I, V);
5515
5516 if (Instruction *CastedXor = foldCastedBitwiseLogic(I))
5517 return CastedXor;
5518
5519 if (Instruction *Abs = canonicalizeAbs(Xor&: I, Builder))
5520 return Abs;
5521
5522 // Otherwise, if all else failed, try to hoist the xor-by-constant:
5523 // (X ^ C) ^ Y --> (X ^ Y) ^ C
5524 // Just like we do in other places, we completely avoid the fold
5525 // for constantexprs, at least to avoid endless combine loop.
5526 if (match(V: &I, P: m_c_Xor(L: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: X, Match: m_Unless(M: m_ConstantExpr())),
5527 R: m_ImmConstant(C&: C1))),
5528 R: m_Value(V&: Y))))
5529 return BinaryOperator::CreateXor(V1: Builder.CreateXor(LHS: X, RHS: Y), V2: C1);
5530
5531 if (Instruction *R = reassociateForUses(BO&: I, Builder))
5532 return R;
5533
5534 if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder))
5535 return Canonicalized;
5536
5537 if (Instruction *Folded = foldLogicOfIsFPClass(BO&: I, Op0, Op1))
5538 return Folded;
5539
5540 if (Instruction *Folded = canonicalizeConditionalNegationViaMathToSelect(I))
5541 return Folded;
5542
5543 if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
5544 return Res;
5545
5546 if (Instruction *Res = foldBitwiseLogicWithIntrinsics(I, Builder))
5547 return Res;
5548
5549 return nullptr;
5550}
5551