1//===- InstCombineAndOrXor.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitAnd, visitOr, and visitXor functions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/SmallBitVector.h"
15#include "llvm/Analysis/CmpInstAnalysis.h"
16#include "llvm/Analysis/FloatingPointPredicateUtils.h"
17#include "llvm/Analysis/InstructionSimplify.h"
18#include "llvm/IR/ConstantRange.h"
19#include "llvm/IR/DerivedTypes.h"
20#include "llvm/IR/Instructions.h"
21#include "llvm/IR/Intrinsics.h"
22#include "llvm/IR/PatternMatch.h"
23#include "llvm/IR/ProfDataUtils.h"
24#include "llvm/Transforms/InstCombine/InstCombiner.h"
25#include "llvm/Transforms/Utils/Local.h"
26
27using namespace llvm;
28using namespace PatternMatch;
29
30#define DEBUG_TYPE "instcombine"
31
32namespace llvm {
33extern cl::opt<bool> ProfcheckDisableMetadataFixes;
34}
35
36/// This is the complement of getICmpCode, which turns an opcode and two
37/// operands into either a constant true or false, or a brand new ICmp
38/// instruction. The sign is passed in to determine which kind of predicate to
39/// use in the new icmp instruction.
40static Value *getNewICmpValue(unsigned Code, bool Sign, Value *LHS, Value *RHS,
41 InstCombiner::BuilderTy &Builder) {
42 ICmpInst::Predicate NewPred;
43 if (Constant *TorF = getPredForICmpCode(Code, Sign, OpTy: LHS->getType(), Pred&: NewPred))
44 return TorF;
45 return Builder.CreateICmp(P: NewPred, LHS, RHS);
46}
47
48/// This is the complement of getFCmpCode, which turns an opcode and two
49/// operands into either a FCmp instruction, or a true/false constant.
50static Value *getFCmpValue(unsigned Code, Value *LHS, Value *RHS,
51 InstCombiner::BuilderTy &Builder, FMFSource FMF) {
52 FCmpInst::Predicate NewPred;
53 if (Constant *TorF = getPredForFCmpCode(Code, OpTy: LHS->getType(), Pred&: NewPred))
54 return TorF;
55 return Builder.CreateFCmpFMF(P: NewPred, LHS, RHS, FMFSource: FMF);
56}
57
58/// Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise
59/// (V < Lo || V >= Hi). This method expects that Lo < Hi. IsSigned indicates
60/// whether to treat V, Lo, and Hi as signed or not.
61Value *InstCombinerImpl::insertRangeTest(Value *V, const APInt &Lo,
62 const APInt &Hi, bool isSigned,
63 bool Inside) {
64 assert((isSigned ? Lo.slt(Hi) : Lo.ult(Hi)) &&
65 "Lo is not < Hi in range emission code!");
66
67 Type *Ty = V->getType();
68
69 // V >= Min && V < Hi --> V < Hi
70 // V < Min || V >= Hi --> V >= Hi
71 ICmpInst::Predicate Pred = Inside ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE;
72 if (isSigned ? Lo.isMinSignedValue() : Lo.isMinValue()) {
73 Pred = isSigned ? ICmpInst::getSignedPredicate(Pred) : Pred;
74 return Builder.CreateICmp(P: Pred, LHS: V, RHS: ConstantInt::get(Ty, V: Hi));
75 }
76
77 // V >= Lo && V < Hi --> V - Lo u< Hi - Lo
78 // V < Lo || V >= Hi --> V - Lo u>= Hi - Lo
79 Value *VMinusLo =
80 Builder.CreateSub(LHS: V, RHS: ConstantInt::get(Ty, V: Lo), Name: V->getName() + ".off");
81 Constant *HiMinusLo = ConstantInt::get(Ty, V: Hi - Lo);
82 return Builder.CreateICmp(P: Pred, LHS: VMinusLo, RHS: HiMinusLo);
83}
84
85/// Classify (icmp eq (A & B), C) and (icmp ne (A & B), C) as matching patterns
86/// that can be simplified.
87/// One of A and B is considered the mask. The other is the value. This is
88/// described as the "AMask" or "BMask" part of the enum. If the enum contains
89/// only "Mask", then both A and B can be considered masks. If A is the mask,
90/// then it was proven that (A & C) == C. This is trivial if C == A or C == 0.
91/// If both A and C are constants, this proof is also easy.
92/// For the following explanations, we assume that A is the mask.
93///
94/// "AllOnes" declares that the comparison is true only if (A & B) == A or all
95/// bits of A are set in B.
96/// Example: (icmp eq (A & 3), 3) -> AMask_AllOnes
97///
98/// "AllZeros" declares that the comparison is true only if (A & B) == 0 or all
99/// bits of A are cleared in B.
100/// Example: (icmp eq (A & 3), 0) -> Mask_AllZeroes
101///
102/// "Mixed" declares that (A & B) == C and C might or might not contain any
103/// number of one bits and zero bits.
104/// Example: (icmp eq (A & 3), 1) -> AMask_Mixed
105///
106/// "Not" means that in above descriptions "==" should be replaced by "!=".
107/// Example: (icmp ne (A & 3), 3) -> AMask_NotAllOnes
108///
109/// If the mask A contains a single bit, then the following is equivalent:
110/// (icmp eq (A & B), A) equals (icmp ne (A & B), 0)
111/// (icmp ne (A & B), A) equals (icmp eq (A & B), 0)
112enum MaskedICmpType {
113 AMask_AllOnes = 1,
114 AMask_NotAllOnes = 2,
115 BMask_AllOnes = 4,
116 BMask_NotAllOnes = 8,
117 Mask_AllZeros = 16,
118 Mask_NotAllZeros = 32,
119 AMask_Mixed = 64,
120 AMask_NotMixed = 128,
121 BMask_Mixed = 256,
122 BMask_NotMixed = 512
123};
124
125/// Return the set of patterns (from MaskedICmpType) that (icmp SCC (A & B), C)
126/// satisfies.
127static unsigned getMaskedICmpType(Value *A, Value *B, Value *C,
128 ICmpInst::Predicate Pred) {
129 const APInt *ConstA = nullptr, *ConstB = nullptr, *ConstC = nullptr;
130 match(V: A, P: m_APInt(Res&: ConstA));
131 match(V: B, P: m_APInt(Res&: ConstB));
132 match(V: C, P: m_APInt(Res&: ConstC));
133 bool IsEq = (Pred == ICmpInst::ICMP_EQ);
134 bool IsAPow2 = ConstA && ConstA->isPowerOf2();
135 bool IsBPow2 = ConstB && ConstB->isPowerOf2();
136 unsigned MaskVal = 0;
137 if (ConstC && ConstC->isZero()) {
138 // if C is zero, then both A and B qualify as mask
139 MaskVal |= (IsEq ? (Mask_AllZeros | AMask_Mixed | BMask_Mixed)
140 : (Mask_NotAllZeros | AMask_NotMixed | BMask_NotMixed));
141 if (IsAPow2)
142 MaskVal |= (IsEq ? (AMask_NotAllOnes | AMask_NotMixed)
143 : (AMask_AllOnes | AMask_Mixed));
144 if (IsBPow2)
145 MaskVal |= (IsEq ? (BMask_NotAllOnes | BMask_NotMixed)
146 : (BMask_AllOnes | BMask_Mixed));
147 return MaskVal;
148 }
149
150 if (A == C) {
151 MaskVal |= (IsEq ? (AMask_AllOnes | AMask_Mixed)
152 : (AMask_NotAllOnes | AMask_NotMixed));
153 if (IsAPow2)
154 MaskVal |= (IsEq ? (Mask_NotAllZeros | AMask_NotMixed)
155 : (Mask_AllZeros | AMask_Mixed));
156 } else if (ConstA && ConstC && ConstC->isSubsetOf(RHS: *ConstA)) {
157 MaskVal |= (IsEq ? AMask_Mixed : AMask_NotMixed);
158 }
159
160 if (B == C) {
161 MaskVal |= (IsEq ? (BMask_AllOnes | BMask_Mixed)
162 : (BMask_NotAllOnes | BMask_NotMixed));
163 if (IsBPow2)
164 MaskVal |= (IsEq ? (Mask_NotAllZeros | BMask_NotMixed)
165 : (Mask_AllZeros | BMask_Mixed));
166 } else if (ConstB && ConstC && ConstC->isSubsetOf(RHS: *ConstB)) {
167 MaskVal |= (IsEq ? BMask_Mixed : BMask_NotMixed);
168 }
169
170 return MaskVal;
171}
172
173/// Convert an analysis of a masked ICmp into its equivalent if all boolean
174/// operations had the opposite sense. Since each "NotXXX" flag (recording !=)
175/// is adjacent to the corresponding normal flag (recording ==), this just
176/// involves swapping those bits over.
177static unsigned conjugateICmpMask(unsigned Mask) {
178 unsigned NewMask;
179 NewMask = (Mask & (AMask_AllOnes | BMask_AllOnes | Mask_AllZeros |
180 AMask_Mixed | BMask_Mixed))
181 << 1;
182
183 NewMask |= (Mask & (AMask_NotAllOnes | BMask_NotAllOnes | Mask_NotAllZeros |
184 AMask_NotMixed | BMask_NotMixed))
185 >> 1;
186
187 return NewMask;
188}
189
190// Adapts the external decomposeBitTest for local use.
191static bool decomposeBitTest(Value *Cond, CmpInst::Predicate &Pred, Value *&X,
192 Value *&Y, Value *&Z) {
193 auto Res =
194 llvm::decomposeBitTest(Cond, /*LookThroughTrunc=*/true,
195 /*AllowNonZeroC=*/true, /*DecomposeAnd=*/true);
196 if (!Res)
197 return false;
198
199 Pred = Res->Pred;
200 X = Res->X;
201 Y = ConstantInt::get(Ty: X->getType(), V: Res->Mask);
202 Z = ConstantInt::get(Ty: X->getType(), V: Res->C);
203 return true;
204}
205
206/// Handle (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E).
207/// Return the pattern classes (from MaskedICmpType) for the left hand side and
208/// the right hand side as a pair.
209/// LHS and RHS are the left hand side and the right hand side ICmps and PredL
210/// and PredR are their predicates, respectively.
211static std::optional<std::pair<unsigned, unsigned>>
212getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C, Value *&D, Value *&E,
213 Value *LHS, Value *RHS, ICmpInst::Predicate &PredL,
214 ICmpInst::Predicate &PredR) {
215
216 // Here comes the tricky part:
217 // LHS might be of the form L11 & L12 == X, X == L21 & L22,
218 // and L11 & L12 == L21 & L22. The same goes for RHS.
219 // Now we must find those components L** and R**, that are equal, so
220 // that we can extract the parameters A, B, C, D, and E for the canonical
221 // above.
222
223 // Check whether the icmp can be decomposed into a bit test.
224 Value *L1, *L11, *L12, *L2, *L21, *L22;
225 if (decomposeBitTest(Cond: LHS, Pred&: PredL, X&: L11, Y&: L12, Z&: L2)) {
226 L21 = L22 = L1 = nullptr;
227 } else {
228 auto *LHSCMP = dyn_cast<ICmpInst>(Val: LHS);
229 if (!LHSCMP)
230 return std::nullopt;
231
232 // Don't allow pointers. Splat vectors are fine.
233 if (!LHSCMP->getOperand(i_nocapture: 0)->getType()->isIntOrIntVectorTy())
234 return std::nullopt;
235
236 PredL = LHSCMP->getPredicate();
237 L1 = LHSCMP->getOperand(i_nocapture: 0);
238 L2 = LHSCMP->getOperand(i_nocapture: 1);
239 // Look for ANDs in the LHS icmp.
240 if (!match(V: L1, P: m_And(L: m_Value(V&: L11), R: m_Value(V&: L12)))) {
241 // Any icmp can be viewed as being trivially masked; if it allows us to
242 // remove one, it's worth it.
243 L11 = L1;
244 L12 = Constant::getAllOnesValue(Ty: L1->getType());
245 }
246
247 if (!match(V: L2, P: m_And(L: m_Value(V&: L21), R: m_Value(V&: L22)))) {
248 L21 = L2;
249 L22 = Constant::getAllOnesValue(Ty: L2->getType());
250 }
251 }
252
253 // Bail if LHS was a icmp that can't be decomposed into an equality.
254 if (!ICmpInst::isEquality(P: PredL))
255 return std::nullopt;
256
257 Value *R11, *R12, *R2;
258 if (decomposeBitTest(Cond: RHS, Pred&: PredR, X&: R11, Y&: R12, Z&: R2)) {
259 if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
260 A = R11;
261 D = R12;
262 } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
263 A = R12;
264 D = R11;
265 } else {
266 return std::nullopt;
267 }
268 E = R2;
269 } else {
270 auto *RHSCMP = dyn_cast<ICmpInst>(Val: RHS);
271 if (!RHSCMP)
272 return std::nullopt;
273 // Don't allow pointers. Splat vectors are fine.
274 if (!RHSCMP->getOperand(i_nocapture: 0)->getType()->isIntOrIntVectorTy())
275 return std::nullopt;
276
277 PredR = RHSCMP->getPredicate();
278
279 Value *R1 = RHSCMP->getOperand(i_nocapture: 0);
280 R2 = RHSCMP->getOperand(i_nocapture: 1);
281 bool Ok = false;
282 if (!match(V: R1, P: m_And(L: m_Value(V&: R11), R: m_Value(V&: R12)))) {
283 // As before, model no mask as a trivial mask if it'll let us do an
284 // optimization.
285 R11 = R1;
286 R12 = Constant::getAllOnesValue(Ty: R1->getType());
287 }
288
289 if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
290 A = R11;
291 D = R12;
292 E = R2;
293 Ok = true;
294 } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
295 A = R12;
296 D = R11;
297 E = R2;
298 Ok = true;
299 }
300
301 // Avoid matching against the -1 value we created for unmasked operand.
302 if (Ok && match(V: A, P: m_AllOnes()))
303 Ok = false;
304
305 // Look for ANDs on the right side of the RHS icmp.
306 if (!Ok) {
307 if (!match(V: R2, P: m_And(L: m_Value(V&: R11), R: m_Value(V&: R12)))) {
308 R11 = R2;
309 R12 = Constant::getAllOnesValue(Ty: R2->getType());
310 }
311
312 if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
313 A = R11;
314 D = R12;
315 E = R1;
316 } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
317 A = R12;
318 D = R11;
319 E = R1;
320 } else {
321 return std::nullopt;
322 }
323 }
324 }
325
326 // Bail if RHS was a icmp that can't be decomposed into an equality.
327 if (!ICmpInst::isEquality(P: PredR))
328 return std::nullopt;
329
330 if (L11 == A) {
331 B = L12;
332 C = L2;
333 } else if (L12 == A) {
334 B = L11;
335 C = L2;
336 } else if (L21 == A) {
337 B = L22;
338 C = L1;
339 } else if (L22 == A) {
340 B = L21;
341 C = L1;
342 }
343
344 unsigned LeftType = getMaskedICmpType(A, B, C, Pred: PredL);
345 unsigned RightType = getMaskedICmpType(A, B: D, C: E, Pred: PredR);
346 return std::optional<std::pair<unsigned, unsigned>>(
347 std::make_pair(x&: LeftType, y&: RightType));
348}
349
350/// When foldLogOpOfMaskedICmps returns RHS for a logical and/or, we need to
351/// make sure that RHS is not more poisonous, e.g. as a result of poison flags
352/// on icmp or trunc, or via poison elements in vector constants. Using freeze
353/// is insufficient here, because we need the constants to match those the
354/// analysis was performed on.
355/// D and E are expected to be poison-free constants here.
356static Value *getReconstructedMaskedICmp(IRBuilderBase &Builder, bool IsLogical,
357 Value *RHS, Value *A, Value *D,
358 Value *E, ICmpInst::Predicate PredR) {
359 if (!IsLogical)
360 return RHS;
361
362 Value *And = Builder.CreateAnd(LHS: A, RHS: D);
363 return Builder.CreateICmp(P: PredR, LHS: And, RHS: E);
364}
365
366/// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E) into a single
367/// (icmp(A & X) ==/!= Y), where the left-hand side is of type Mask_NotAllZeros
368/// and the right hand side is of type BMask_Mixed. For example,
369/// (icmp (A & 12) != 0) & (icmp (A & 15) == 8) -> (icmp (A & 15) == 8).
370/// Also used for logical and/or, must be poison safe.
371static Value *foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
372 Value *LHS, Value *RHS, bool IsAnd, bool IsLogical, Value *A, Value *B,
373 Value *D, Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR,
374 InstCombiner::BuilderTy &Builder) {
375 // We are given the canonical form:
376 // (icmp ne (A & B), 0) & (icmp eq (A & D), E).
377 // where D & E == E.
378 //
379 // If IsAnd is false, we get it in negated form:
380 // (icmp eq (A & B), 0) | (icmp ne (A & D), E) ->
381 // !((icmp ne (A & B), 0) & (icmp eq (A & D), E)).
382 //
383 // We currently handle the case of B, C, D, E are constant.
384 //
385 const APInt *BCst, *DCst, *OrigECst;
386 if (!match(V: B, P: m_APInt(Res&: BCst)) || !match(V: D, P: m_APInt(Res&: DCst)) ||
387 !match(V: E, P: m_APInt(Res&: OrigECst)))
388 return nullptr;
389
390 ICmpInst::Predicate NewCC = IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
391
392 // Update E to the canonical form when D is a power of two and RHS is
393 // canonicalized as,
394 // (icmp ne (A & D), 0) -> (icmp eq (A & D), D) or
395 // (icmp ne (A & D), D) -> (icmp eq (A & D), 0).
396 APInt ECst = *OrigECst;
397 if (PredR != NewCC)
398 ECst ^= *DCst;
399
400 // If B or D is zero, skip because if LHS or RHS can be trivially folded by
401 // other folding rules and this pattern won't apply any more.
402 if (*BCst == 0 || *DCst == 0)
403 return nullptr;
404
405 // If B and D don't intersect, ie. (B & D) == 0, try to fold isNaN idiom:
406 // (icmp ne (A & FractionBits), 0) & (icmp eq (A & ExpBits), ExpBits)
407 // -> isNaN(A)
408 // Otherwise, we cannot deduce anything from it.
409 if (!BCst->intersects(RHS: *DCst)) {
410 Value *Src;
411 if (*DCst == ECst && match(V: A, P: m_ElementWiseBitCast(Op: m_Value(V&: Src))) &&
412 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
413 Kind: Attribute::StrictFP)) {
414 Type *Ty = Src->getType()->getScalarType();
415 if (!Ty->isIEEELikeFPTy())
416 return nullptr;
417
418 APInt ExpBits = APFloat::getInf(Sem: Ty->getFltSemantics()).bitcastToAPInt();
419 if (ECst != ExpBits)
420 return nullptr;
421 APInt FractionBits = ~ExpBits;
422 FractionBits.clearSignBit();
423 if (*BCst != FractionBits)
424 return nullptr;
425
426 return Builder.CreateFCmp(P: IsAnd ? FCmpInst::FCMP_UNO : FCmpInst::FCMP_ORD,
427 LHS: Src, RHS: ConstantFP::getZero(Ty: Src->getType()));
428 }
429 return nullptr;
430 }
431
432 // If the following two conditions are met:
433 //
434 // 1. mask B covers only a single bit that's not covered by mask D, that is,
435 // (B & (B ^ D)) is a power of 2 (in other words, B minus the intersection of
436 // B and D has only one bit set) and,
437 //
438 // 2. RHS (and E) indicates that the rest of B's bits are zero (in other
439 // words, the intersection of B and D is zero), that is, ((B & D) & E) == 0
440 //
441 // then that single bit in B must be one and thus the whole expression can be
442 // folded to
443 // (A & (B | D)) == (B & (B ^ D)) | E.
444 //
445 // For example,
446 // (icmp ne (A & 12), 0) & (icmp eq (A & 7), 1) -> (icmp eq (A & 15), 9)
447 // (icmp ne (A & 15), 0) & (icmp eq (A & 7), 0) -> (icmp eq (A & 15), 8)
448 if ((((*BCst & *DCst) & ECst) == 0) &&
449 (*BCst & (*BCst ^ *DCst)).isPowerOf2()) {
450 APInt BorD = *BCst | *DCst;
451 APInt BandBxorDorE = (*BCst & (*BCst ^ *DCst)) | ECst;
452 Value *NewMask = ConstantInt::get(Ty: A->getType(), V: BorD);
453 Value *NewMaskedValue = ConstantInt::get(Ty: A->getType(), V: BandBxorDorE);
454 Value *NewAnd = Builder.CreateAnd(LHS: A, RHS: NewMask);
455 return Builder.CreateICmp(P: NewCC, LHS: NewAnd, RHS: NewMaskedValue);
456 }
457
458 auto IsSubSetOrEqual = [](const APInt *C1, const APInt *C2) {
459 return (*C1 & *C2) == *C1;
460 };
461 auto IsSuperSetOrEqual = [](const APInt *C1, const APInt *C2) {
462 return (*C1 & *C2) == *C2;
463 };
464
465 // In the following, we consider only the cases where B is a superset of D, B
466 // is a subset of D, or B == D because otherwise there's at least one bit
467 // covered by B but not D, in which case we can't deduce much from it, so
468 // no folding (aside from the single must-be-one bit case right above.)
469 // For example,
470 // (icmp ne (A & 14), 0) & (icmp eq (A & 3), 1) -> no folding.
471 if (!IsSubSetOrEqual(BCst, DCst) && !IsSuperSetOrEqual(BCst, DCst))
472 return nullptr;
473
474 // At this point, either B is a superset of D, B is a subset of D or B == D.
475
476 // If E is zero, if B is a subset of (or equal to) D, LHS and RHS contradict
477 // and the whole expression becomes false (or true if negated), otherwise, no
478 // folding.
479 // For example,
480 // (icmp ne (A & 3), 0) & (icmp eq (A & 7), 0) -> false.
481 // (icmp ne (A & 15), 0) & (icmp eq (A & 3), 0) -> no folding.
482 if (ECst.isZero()) {
483 if (IsSubSetOrEqual(BCst, DCst))
484 return ConstantInt::get(Ty: LHS->getType(), V: !IsAnd);
485 return nullptr;
486 }
487
488 // At this point, B, D, E aren't zero and (B & D) == B, (B & D) == D or B ==
489 // D. If B is a superset of (or equal to) D, since E is not zero, LHS is
490 // subsumed by RHS (RHS implies LHS.) So the whole expression becomes
491 // RHS. For example,
492 // (icmp ne (A & 255), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
493 // (icmp ne (A & 15), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
494 if (IsSuperSetOrEqual(BCst, DCst))
495 return getReconstructedMaskedICmp(Builder, IsLogical, RHS, A, D, E, PredR);
496
497 // Otherwise, B is a subset of D. If B and E have a common bit set,
498 // ie. (B & E) != 0, then LHS is subsumed by RHS. For example.
499 // (icmp ne (A & 12), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
500 assert(IsSubSetOrEqual(BCst, DCst) && "Precondition due to above code");
501 if ((*BCst & ECst) != 0)
502 return getReconstructedMaskedICmp(Builder, IsLogical, RHS, A, D, E, PredR);
503
504 // Otherwise, LHS and RHS contradict and the whole expression becomes false
505 // (or true if negated.) For example,
506 // (icmp ne (A & 7), 0) & (icmp eq (A & 15), 8) -> false.
507 // (icmp ne (A & 6), 0) & (icmp eq (A & 15), 8) -> false.
508 return ConstantInt::get(Ty: LHS->getType(), V: !IsAnd);
509}
510
511/// Try to fold (icmp(A & B) ==/!= 0) &/| (icmp(A & D) ==/!= E) into a single
512/// (icmp(A & X) ==/!= Y), where the left-hand side and the right hand side
513/// aren't of the common mask pattern type.
514/// Also used for logical and/or, must be poison safe.
515static Value *foldLogOpOfMaskedICmpsAsymmetric(
516 Value *LHS, Value *RHS, bool IsAnd, bool IsLogical, Value *A, Value *B,
517 Value *C, Value *D, Value *E, ICmpInst::Predicate PredL,
518 ICmpInst::Predicate PredR, unsigned LHSMask, unsigned RHSMask,
519 InstCombiner::BuilderTy &Builder) {
520 assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
521 "Expected equality predicates for masked type of icmps.");
522 // Handle Mask_NotAllZeros-BMask_Mixed cases.
523 // (icmp ne/eq (A & B), C) &/| (icmp eq/ne (A & D), E), or
524 // (icmp eq/ne (A & B), C) &/| (icmp ne/eq (A & D), E)
525 // which gets swapped to
526 // (icmp ne/eq (A & D), E) &/| (icmp eq/ne (A & B), C).
527 if (!IsAnd) {
528 LHSMask = conjugateICmpMask(Mask: LHSMask);
529 RHSMask = conjugateICmpMask(Mask: RHSMask);
530 }
531 if ((LHSMask & Mask_NotAllZeros) && (RHSMask & BMask_Mixed)) {
532 if (Value *V = foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
533 LHS, RHS, IsAnd, IsLogical, A, B, D, E, PredL, PredR, Builder)) {
534 return V;
535 }
536 } else if ((LHSMask & BMask_Mixed) && (RHSMask & Mask_NotAllZeros)) {
537 if (Value *V = foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
538 LHS: RHS, RHS: LHS, IsAnd, /*IsLogical=*/false, A, B: D, D: B, E: C, PredL: PredR, PredR: PredL,
539 Builder)) {
540 return V;
541 }
542 }
543 return nullptr;
544}
545
546/// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E)
547/// into a single (icmp(A & X) ==/!= Y).
548static Value *foldLogOpOfMaskedICmps(Value *LHS, Value *RHS, bool IsAnd,
549 bool IsLogical,
550 InstCombiner::BuilderTy &Builder,
551 const SimplifyQuery &Q) {
552 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr, *E = nullptr;
553 ICmpInst::Predicate PredL, PredR;
554 std::optional<std::pair<unsigned, unsigned>> MaskPair =
555 getMaskedTypeForICmpPair(A, B, C, D, E, LHS, RHS, PredL, PredR);
556 if (!MaskPair)
557 return nullptr;
558 assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
559 "Expected equality predicates for masked type of icmps.");
560 unsigned LHSMask = MaskPair->first;
561 unsigned RHSMask = MaskPair->second;
562 unsigned Mask = LHSMask & RHSMask;
563 if (Mask == 0) {
564 // Even if the two sides don't share a common pattern, check if folding can
565 // still happen.
566 if (Value *V = foldLogOpOfMaskedICmpsAsymmetric(LHS, RHS, IsAnd, IsLogical,
567 A, B, C, D, E, PredL, PredR,
568 LHSMask, RHSMask, Builder))
569 return V;
570 return nullptr;
571 }
572
573 // In full generality:
574 // (icmp (A & B) Op C) | (icmp (A & D) Op E)
575 // == ![ (icmp (A & B) !Op C) & (icmp (A & D) !Op E) ]
576 //
577 // If the latter can be converted into (icmp (A & X) Op Y) then the former is
578 // equivalent to (icmp (A & X) !Op Y).
579 //
580 // Therefore, we can pretend for the rest of this function that we're dealing
581 // with the conjunction, provided we flip the sense of any comparisons (both
582 // input and output).
583
584 // In most cases we're going to produce an EQ for the "&&" case.
585 ICmpInst::Predicate NewCC = IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
586 if (!IsAnd) {
587 // Convert the masking analysis into its equivalent with negated
588 // comparisons.
589 Mask = conjugateICmpMask(Mask);
590 }
591
592 if (Mask & Mask_AllZeros) {
593 // (icmp eq (A & B), 0) & (icmp eq (A & D), 0)
594 // -> (icmp eq (A & (B|D)), 0)
595 if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(V: D))
596 return nullptr; // TODO: Use freeze?
597 Value *NewOr = Builder.CreateOr(LHS: B, RHS: D);
598 Value *NewAnd = Builder.CreateAnd(LHS: A, RHS: NewOr);
599 // We can't use C as zero because we might actually handle
600 // (icmp ne (A & B), B) & (icmp ne (A & D), D)
601 // with B and D, having a single bit set.
602 Value *Zero = Constant::getNullValue(Ty: A->getType());
603 return Builder.CreateICmp(P: NewCC, LHS: NewAnd, RHS: Zero);
604 }
605 if (Mask & BMask_AllOnes) {
606 // (icmp eq (A & B), B) & (icmp eq (A & D), D)
607 // -> (icmp eq (A & (B|D)), (B|D))
608 if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(V: D))
609 return nullptr; // TODO: Use freeze?
610 Value *NewOr = Builder.CreateOr(LHS: B, RHS: D);
611 Value *NewAnd = Builder.CreateAnd(LHS: A, RHS: NewOr);
612 return Builder.CreateICmp(P: NewCC, LHS: NewAnd, RHS: NewOr);
613 }
614 if (Mask & AMask_AllOnes) {
615 // (icmp eq (A & B), A) & (icmp eq (A & D), A)
616 // -> (icmp eq (A & (B&D)), A)
617 if (IsLogical && !isGuaranteedNotToBeUndefOrPoison(V: D))
618 return nullptr; // TODO: Use freeze?
619 Value *NewAnd1 = Builder.CreateAnd(LHS: B, RHS: D);
620 Value *NewAnd2 = Builder.CreateAnd(LHS: A, RHS: NewAnd1);
621 return Builder.CreateICmp(P: NewCC, LHS: NewAnd2, RHS: A);
622 }
623
624 const APInt *ConstB, *ConstD;
625 if (match(V: B, P: m_APInt(Res&: ConstB)) && match(V: D, P: m_APInt(Res&: ConstD))) {
626 if (Mask & (Mask_NotAllZeros | BMask_NotAllOnes)) {
627 // (icmp ne (A & B), 0) & (icmp ne (A & D), 0) and
628 // (icmp ne (A & B), B) & (icmp ne (A & D), D)
629 // -> (icmp ne (A & B), 0) or (icmp ne (A & D), 0)
630 // Only valid if one of the masks is a superset of the other (check "B&D"
631 // is the same as either B or D).
632 APInt NewMask = *ConstB & *ConstD;
633 if (NewMask == *ConstB)
634 return LHS;
635 if (NewMask == *ConstD)
636 return getReconstructedMaskedICmp(Builder, IsLogical, RHS, A, D, E,
637 PredR);
638 }
639
640 if (Mask & AMask_NotAllOnes) {
641 // (icmp ne (A & B), B) & (icmp ne (A & D), D)
642 // -> (icmp ne (A & B), A) or (icmp ne (A & D), A)
643 // Only valid if one of the masks is a superset of the other (check "B|D"
644 // is the same as either B or D).
645 APInt NewMask = *ConstB | *ConstD;
646 if (NewMask == *ConstB)
647 return LHS;
648 if (NewMask == *ConstD)
649 return RHS;
650 }
651
652 if (Mask & (BMask_Mixed | BMask_NotMixed)) {
653 // Mixed:
654 // (icmp eq (A & B), C) & (icmp eq (A & D), E)
655 // We already know that B & C == C && D & E == E.
656 // If we can prove that (B & D) & (C ^ E) == 0, that is, the bits of
657 // C and E, which are shared by both the mask B and the mask D, don't
658 // contradict, then we can transform to
659 // -> (icmp eq (A & (B|D)), (C|E))
660 // Currently, we only handle the case of B, C, D, and E being constant.
661 // We can't simply use C and E because we might actually handle
662 // (icmp ne (A & B), B) & (icmp eq (A & D), D)
663 // with B and D, having a single bit set.
664
665 // NotMixed:
666 // (icmp ne (A & B), C) & (icmp ne (A & D), E)
667 // -> (icmp ne (A & (B & D)), (C & E))
668 // Check the intersection (B & D) for inequality.
669 // Assume that (B & D) == B || (B & D) == D, i.e B/D is a subset of D/B
670 // and (B & D) & (C ^ E) == 0, bits of C and E, which are shared by both
671 // the B and the D, don't contradict. Note that we can assume (~B & C) ==
672 // 0 && (~D & E) == 0, previous operation should delete these icmps if it
673 // hadn't been met.
674
675 const APInt *OldConstC, *OldConstE;
676 if (!match(V: C, P: m_APInt(Res&: OldConstC)) || !match(V: E, P: m_APInt(Res&: OldConstE)))
677 return nullptr;
678
679 auto FoldBMixed = [&](ICmpInst::Predicate CC, bool IsNot) -> Value * {
680 CC = IsNot ? CmpInst::getInversePredicate(pred: CC) : CC;
681 const APInt ConstC = PredL != CC ? *ConstB ^ *OldConstC : *OldConstC;
682 const APInt ConstE = PredR != CC ? *ConstD ^ *OldConstE : *OldConstE;
683
684 if (((*ConstB & *ConstD) & (ConstC ^ ConstE)).getBoolValue())
685 return IsNot ? nullptr : ConstantInt::get(Ty: LHS->getType(), V: !IsAnd);
686
687 if (IsNot && !ConstB->isSubsetOf(RHS: *ConstD) &&
688 !ConstD->isSubsetOf(RHS: *ConstB))
689 return nullptr;
690
691 APInt BD, CE;
692 if (IsNot) {
693 BD = *ConstB & *ConstD;
694 CE = ConstC & ConstE;
695 } else {
696 BD = *ConstB | *ConstD;
697 CE = ConstC | ConstE;
698 }
699 Value *NewAnd = Builder.CreateAnd(LHS: A, RHS: BD);
700 Value *CEVal = ConstantInt::get(Ty: A->getType(), V: CE);
701 return Builder.CreateICmp(P: CC, LHS: NewAnd, RHS: CEVal);
702 };
703
704 if (Mask & BMask_Mixed)
705 return FoldBMixed(NewCC, false);
706 if (Mask & BMask_NotMixed) // can be else also
707 return FoldBMixed(NewCC, true);
708 }
709 }
710
711 // (icmp eq (A & B), 0) | (icmp eq (A & D), 0)
712 // -> (icmp ne (A & (B|D)), (B|D))
713 // (icmp ne (A & B), 0) & (icmp ne (A & D), 0)
714 // -> (icmp eq (A & (B|D)), (B|D))
715 // iff B and D is known to be a power of two
716 if (Mask & Mask_NotAllZeros &&
717 isKnownToBeAPowerOfTwo(V: B, /*OrZero=*/false, Q) &&
718 isKnownToBeAPowerOfTwo(V: D, /*OrZero=*/false, Q)) {
719 // If this is a logical and/or, then we must prevent propagation of a
720 // poison value from the RHS by inserting freeze.
721 if (IsLogical)
722 D = Builder.CreateFreeze(V: D);
723 Value *Mask = Builder.CreateOr(LHS: B, RHS: D);
724 Value *Masked = Builder.CreateAnd(LHS: A, RHS: Mask);
725 return Builder.CreateICmp(P: NewCC, LHS: Masked, RHS: Mask);
726 }
727 return nullptr;
728}
729
730/// Try to fold a signed range checked with lower bound 0 to an unsigned icmp.
731/// Example: (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n
732/// If \p Inverted is true then the check is for the inverted range, e.g.
733/// (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n
734Value *InstCombinerImpl::simplifyRangeCheck(CmpPredicate PredL, Value *LHS0,
735 Value *LHS1, CmpPredicate PredR,
736 Value *RHS0, Value *RHS1,
737 Instruction *CtxI, bool Inverted) {
738 // Check the lower range comparison, e.g. x >= 0
739 // InstCombine already ensured that if there is a constant it's on the RHS.
740 ConstantInt *RangeStart = dyn_cast<ConstantInt>(Val: LHS1);
741 if (!RangeStart)
742 return nullptr;
743
744 if (Inverted) {
745 PredL = CmpPredicate::getInverse(P: PredL);
746 PredR = CmpPredicate::getInverse(P: PredR);
747 }
748
749 // Accept x > -1 or x >= 0 (after potentially inverting the predicate).
750 if (!((PredL == ICmpInst::ICMP_SGT && RangeStart->isMinusOne()) ||
751 (PredL == ICmpInst::ICMP_SGE && RangeStart->isZero())))
752 return nullptr;
753
754 Value *Input = LHS0;
755 Value *RangeEnd;
756 if (match(V: RHS0, P: m_SExtOrSelf(Op: m_Specific(V: Input)))) {
757 // For the upper range compare we have: icmp x, n
758 Input = RHS0;
759 RangeEnd = RHS1;
760 } else if (match(V: RHS1, P: m_SExtOrSelf(Op: m_Specific(V: Input)))) {
761 // For the upper range compare we have: icmp n, x
762 Input = RHS1;
763 RangeEnd = RHS0;
764 PredR = CmpPredicate::getSwapped(P: PredR);
765 } else {
766 return nullptr;
767 }
768
769 // Check the upper range comparison, e.g. x < n
770 ICmpInst::Predicate NewPred;
771 switch (PredR) {
772 case ICmpInst::ICMP_SLT:
773 NewPred = ICmpInst::ICMP_ULT;
774 break;
775 case ICmpInst::ICMP_SLE:
776 NewPred = ICmpInst::ICMP_ULE;
777 break;
778 default:
779 return nullptr;
780 }
781
782 // This simplification is only valid if the upper range is not negative.
783 KnownBits Known = computeKnownBits(V: RangeEnd, CtxI);
784 if (!Known.isNonNegative())
785 return nullptr;
786
787 if (Inverted)
788 NewPred = ICmpInst::getInversePredicate(pred: NewPred);
789
790 return Builder.CreateICmp(P: NewPred, LHS: Input, RHS: RangeEnd);
791}
792
793// (or (icmp eq X, 0), (icmp eq X, Pow2OrZero))
794// -> (icmp eq (and X, Pow2OrZero), X)
795// (and (icmp ne X, 0), (icmp ne X, Pow2OrZero))
796// -> (icmp ne (and X, Pow2OrZero), X)
797static Value *foldAndOrOfICmpsWithPow2AndWithZero(
798 InstCombiner::BuilderTy &Builder, CmpPredicate PredL, Value *LHS0,
799 Value *LHS1, bool LHSOneUse, CmpPredicate PredR, Value *RHS0, Value *RHS1,
800 bool RHSOneUse, bool IsAnd, const SimplifyQuery &Q) {
801 CmpInst::Predicate Pred = IsAnd ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
802 // Make sure we have right compares for our op.
803 if (PredL != Pred || PredR != Pred)
804 return nullptr;
805
806 // Make it so we can match LHS against the (icmp eq/ne X, 0) just for
807 // simplicity.
808 if (match(V: RHS1, P: m_Zero())) {
809 std::swap(a&: PredL, b&: PredR);
810 std::swap(a&: LHS0, b&: RHS0);
811 std::swap(a&: LHS1, b&: RHS1);
812 }
813
814 if (RHS1 == LHS0)
815 std::swap(a&: RHS0, b&: RHS1);
816
817 // Match the desired pattern:
818 // LHS: (icmp eq/ne X, 0)
819 // RHS: (icmp eq/ne X, Pow2OrZero)
820 // Skip if Pow2OrZero is 1. Either way it gets folded to (icmp ugt X, 1) but
821 // this form ends up slightly less canonical.
822 // We could potentially be more sophisticated than requiring LHS/RHS
823 // be one-use. We don't create additional instructions if only one
824 // of them is one-use. So cases where one is one-use and the other
825 // is two-use might be profitable.
826 if (!LHSOneUse || !RHSOneUse || !match(V: LHS1, P: m_Zero()) || RHS0 != LHS0 ||
827 match(V: RHS1, P: m_One()) || !isKnownToBeAPowerOfTwo(V: RHS1, /*OrZero=*/true, Q))
828 return nullptr;
829
830 Value *And = Builder.CreateAnd(LHS: LHS0, RHS: RHS1);
831 return Builder.CreateICmp(P: Pred, LHS: And, RHS: LHS0);
832}
833
834/// General pattern:
835/// X & Y
836///
837/// Where Y is checking that all the high bits (covered by a mask 4294967168)
838/// are uniform, i.e. %arg & 4294967168 can be either 4294967168 or 0
839/// Pattern can be one of:
840/// %t = add i32 %arg, 128
841/// %r = icmp ult i32 %t, 256
842/// Or
843/// %t0 = shl i32 %arg, 24
844/// %t1 = ashr i32 %t0, 24
845/// %r = icmp eq i32 %t1, %arg
846/// Or
847/// %t0 = trunc i32 %arg to i8
848/// %t1 = sext i8 %t0 to i32
849/// %r = icmp eq i32 %t1, %arg
850/// This pattern is a signed truncation check.
851///
852/// And X is checking that some bit in that same mask is zero.
853/// I.e. can be one of:
854/// %r = icmp sgt i32 %arg, -1
855/// Or
856/// %t = and i32 %arg, 2147483648
857/// %r = icmp eq i32 %t, 0
858///
859/// Since we are checking that all the bits in that mask are the same,
860/// and a particular bit is zero, what we are really checking is that all the
861/// masked bits are zero.
862/// So this should be transformed to:
863/// %r = icmp ult i32 %arg, 128
864static Value *foldSignedTruncationCheck(CmpPredicate PredL, Value *LHS0,
865 Value *LHS1, CmpPredicate PredR,
866 Value *RHS0, Value *RHS1,
867 Instruction &CtxI,
868 InstCombiner::BuilderTy &Builder) {
869 assert(CtxI.getOpcode() == Instruction::And);
870
871 // Match icmp ult (add %arg, C01), C1 (C1 == C01 << 1; powers of two)
872 auto tryToMatchSignedTruncationCheck = [](CmpPredicate Pred, Value *LHS,
873 Value *RHS, Value *&X,
874 APInt &SignBitMask) -> bool {
875 const APInt *I01, *I1; // powers of two; I1 == I01 << 1
876 if (Pred != ICmpInst::ICMP_ULT ||
877 !match(V: LHS, P: m_Add(L: m_Value(V&: X), R: m_Power2(V&: I01))) ||
878 !match(V: RHS, P: m_Power2(V&: I1)) || I1->ule(RHS: *I01) || I01->shl(shiftAmt: 1) != *I1)
879 return false;
880 // Which bit is the new sign bit as per the 'signed truncation' pattern?
881 SignBitMask = *I01;
882 return true;
883 };
884
885 // One icmp needs to be 'signed truncation check'.
886 // We need to match this first, else we will mismatch commutative cases.
887 Value *X1;
888 APInt HighestBit;
889 if (tryToMatchSignedTruncationCheck(PredR, RHS0, RHS1, X1, HighestBit)) {
890 std::swap(a&: PredL, b&: PredR);
891 std::swap(a&: LHS0, b&: RHS0);
892 std::swap(a&: LHS1, b&: RHS1);
893 } else if (!tryToMatchSignedTruncationCheck(PredL, LHS0, LHS1, X1,
894 HighestBit))
895 return nullptr;
896
897 assert(HighestBit.isPowerOf2() && "expected to be power of two (non-zero)");
898
899 // Try to match/decompose into: icmp eq (X & Mask), 0
900 auto tryToDecompose = [](CmpPredicate Pred, Value *LHS, Value *RHS, Value *&X,
901 APInt &UnsetBitsMask) -> bool {
902 // Can it be decomposed into icmp eq (X & Mask), 0 ?
903 auto Res = llvm::decomposeBitTestICmp(LHS, RHS, Pred,
904 /*LookThroughTrunc=*/false,
905 /*AllowNonZeroC=*/false,
906 /*DecomposeAnd=*/true);
907 if (Res && Res->Pred == ICmpInst::ICMP_EQ) {
908 X = Res->X;
909 UnsetBitsMask = Res->Mask;
910 return true;
911 }
912
913 return false;
914 };
915
916 // And the other icmp needs to be decomposable into a bit test.
917 Value *X0;
918 APInt UnsetBitsMask;
919 if (!tryToDecompose(PredR, RHS0, RHS1, X0, UnsetBitsMask))
920 return nullptr;
921
922 assert(!UnsetBitsMask.isZero() && "empty mask makes no sense.");
923
924 // Are they working on the same value?
925 Value *X;
926 if (X1 == X0) {
927 // Ok as is.
928 X = X1;
929 } else if (match(V: X0, P: m_Trunc(Op: m_Specific(V: X1)))) {
930 UnsetBitsMask = UnsetBitsMask.zext(width: X1->getType()->getScalarSizeInBits());
931 X = X1;
932 } else
933 return nullptr;
934
935 // So which bits should be uniform as per the 'signed truncation check'?
936 // (all the bits starting with (i.e. including) HighestBit)
937 APInt SignBitsMask = ~(HighestBit - 1U);
938
939 // UnsetBitsMask must have some common bits with SignBitsMask,
940 if (!UnsetBitsMask.intersects(RHS: SignBitsMask))
941 return nullptr;
942
943 // Does UnsetBitsMask contain any bits outside of SignBitsMask?
944 if (!UnsetBitsMask.isSubsetOf(RHS: SignBitsMask)) {
945 APInt OtherHighestBit = (~UnsetBitsMask) + 1U;
946 if (!OtherHighestBit.isPowerOf2())
947 return nullptr;
948 HighestBit = APIntOps::umin(A: HighestBit, B: OtherHighestBit);
949 }
950 // Else, if it does not, then all is ok as-is.
951
952 // %r = icmp ult %X, SignBit
953 return Builder.CreateICmpULT(LHS: X, RHS: ConstantInt::get(Ty: X->getType(), V: HighestBit),
954 Name: CtxI.getName() + ".simplified");
955}
956
957/// Fold (icmp eq ctpop(X) 1) | (icmp eq X 0) into (icmp ult ctpop(X) 2) and
958/// fold (icmp ne ctpop(X) 1) & (icmp ne X 0) into (icmp ugt ctpop(X) 1).
959/// Also used for logical and/or, must be poison safe if range attributes are
960/// dropped.
961static Value *foldIsPowerOf2OrZero(CmpPredicate PredL, Value *LHS0, Value *LHS1,
962 CmpPredicate PredR, Value *RHS0, Value *RHS1,
963 bool IsAnd, InstCombiner::BuilderTy &Builder,
964 InstCombinerImpl &IC) {
965
966 Value *X;
967 if (!match(V: LHS0, P: m_Ctpop(Op0: m_Value(V&: X))) || !match(V: LHS1, P: m_SpecificInt(V: 1)) ||
968 RHS0 != X || !match(V: RHS1, P: m_ZeroInt()))
969 return nullptr;
970
971 auto *CtPop = cast<Instruction>(Val: LHS0);
972 if (IsAnd && PredL == ICmpInst::ICMP_NE && PredR == ICmpInst::ICMP_NE) {
973 // Drop range attributes and re-infer them in the next iteration.
974 CtPop->dropPoisonGeneratingAnnotations();
975 IC.addToWorklist(I: CtPop);
976 return Builder.CreateICmpUGT(LHS: CtPop, RHS: ConstantInt::get(Ty: CtPop->getType(), V: 1));
977 }
978 if (!IsAnd && PredL == ICmpInst::ICMP_EQ && PredR == ICmpInst::ICMP_EQ) {
979 // Drop range attributes and re-infer them in the next iteration.
980 CtPop->dropPoisonGeneratingAnnotations();
981 IC.addToWorklist(I: CtPop);
982 return Builder.CreateICmpULT(LHS: CtPop, RHS: ConstantInt::get(Ty: CtPop->getType(), V: 2));
983 }
984
985 return nullptr;
986}
987
988/// Reduce a pair of compares that check if a value has exactly 1 bit set.
989/// Also used for logical and/or, must be poison safe if range attributes are
990/// dropped.
991static Value *foldIsPowerOf2(CmpPredicate PredL, Value *LHS0, Value *LHS1,
992 CmpPredicate PredR, Value *RHS0, Value *RHS1,
993 bool JoinedByAnd, InstCombiner::BuilderTy &Builder,
994 InstCombinerImpl &IC) {
995 // Handle 'and' / 'or' commutation: make the equality check the first operand.
996 if (PredR == (JoinedByAnd ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ)) {
997 std::swap(a&: PredL, b&: PredR);
998 std::swap(a&: LHS0, b&: RHS0);
999 std::swap(a&: LHS1, b&: RHS1);
1000 }
1001
1002 // (X != 0) && (ctpop(X) u< 2) --> ctpop(X) == 1
1003 if (JoinedByAnd && PredL == ICmpInst::ICMP_NE && match(V: LHS1, P: m_ZeroInt()) &&
1004 PredR == ICmpInst::ICMP_ULT && match(V: RHS0, P: m_Ctpop(Op0: m_Specific(V: LHS0))) &&
1005 match(V: RHS1, P: m_SpecificInt(V: 2))) {
1006 auto *CtPop = cast<Instruction>(Val: RHS0);
1007 // Drop range attributes and re-infer them in the next iteration.
1008 CtPop->dropPoisonGeneratingAnnotations();
1009 IC.addToWorklist(I: CtPop);
1010 return Builder.CreateICmpEQ(LHS: CtPop, RHS: ConstantInt::get(Ty: CtPop->getType(), V: 1));
1011 }
1012 // (X == 0) || (ctpop(X) u> 1) --> ctpop(X) != 1
1013 if (!JoinedByAnd && PredL == ICmpInst::ICMP_EQ && match(V: LHS1, P: m_ZeroInt()) &&
1014 PredR == ICmpInst::ICMP_UGT && match(V: RHS0, P: m_Ctpop(Op0: m_Specific(V: LHS0))) &&
1015 match(V: RHS1, P: m_SpecificInt(V: 1))) {
1016 auto *CtPop = cast<Instruction>(Val: RHS0);
1017 // Drop range attributes and re-infer them in the next iteration.
1018 CtPop->dropPoisonGeneratingAnnotations();
1019 IC.addToWorklist(I: CtPop);
1020 return Builder.CreateICmpNE(LHS: CtPop, RHS: ConstantInt::get(Ty: CtPop->getType(), V: 1));
1021 }
1022 return nullptr;
1023}
1024
1025/// Try to fold (icmp(A & B) == 0) & (icmp(A & D) != E) into (icmp A u< D) iff
1026/// B is a contiguous set of ones starting from the most significant bit
1027/// (negative power of 2), D and E are equal, and D is a contiguous set of ones
1028/// starting at the most significant zero bit in B. Parameter B supports masking
1029/// using undef/poison in either scalar or vector values.
1030static Value *foldNegativePower2AndShiftedMask(
1031 Value *A, Value *B, Value *D, Value *E, ICmpInst::Predicate PredL,
1032 ICmpInst::Predicate PredR, InstCombiner::BuilderTy &Builder) {
1033 assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
1034 "Expected equality predicates for masked type of icmps.");
1035 if (PredL != ICmpInst::ICMP_EQ || PredR != ICmpInst::ICMP_NE)
1036 return nullptr;
1037
1038 if (!match(V: B, P: m_NegatedPower2()) || !match(V: D, P: m_ShiftedMask()) ||
1039 !match(V: E, P: m_ShiftedMask()))
1040 return nullptr;
1041
1042 // Test scalar arguments for conversion. B has been validated earlier to be a
1043 // negative power of two and thus is guaranteed to have one or more contiguous
1044 // ones starting from the MSB followed by zero or more contiguous zeros. D has
1045 // been validated earlier to be a shifted set of one or more contiguous ones.
1046 // In order to match, B leading ones and D leading zeros should be equal. The
1047 // predicate that B be a negative power of 2 prevents the condition of there
1048 // ever being zero leading ones. Thus 0 == 0 cannot occur. The predicate that
1049 // D always be a shifted mask prevents the condition of D equaling 0. This
1050 // prevents matching the condition where B contains the maximum number of
1051 // leading one bits (-1) and D contains the maximum number of leading zero
1052 // bits (0).
1053 auto isReducible = [](const Value *B, const Value *D, const Value *E) {
1054 const APInt *BCst, *DCst, *ECst;
1055 return match(V: B, P: m_APIntAllowPoison(Res&: BCst)) && match(V: D, P: m_APInt(Res&: DCst)) &&
1056 match(V: E, P: m_APInt(Res&: ECst)) && *DCst == *ECst &&
1057 (isa<PoisonValue>(Val: B) ||
1058 (BCst->countLeadingOnes() == DCst->countLeadingZeros()));
1059 };
1060
1061 // Test vector type arguments for conversion.
1062 if (const auto *BVTy = dyn_cast<VectorType>(Val: B->getType())) {
1063 const auto *BFVTy = dyn_cast<FixedVectorType>(Val: BVTy);
1064 const auto *BConst = dyn_cast<Constant>(Val: B);
1065 const auto *DConst = dyn_cast<Constant>(Val: D);
1066 const auto *EConst = dyn_cast<Constant>(Val: E);
1067
1068 if (!BFVTy || !BConst || !DConst || !EConst)
1069 return nullptr;
1070
1071 for (unsigned I = 0; I != BFVTy->getNumElements(); ++I) {
1072 const auto *BElt = BConst->getAggregateElement(Elt: I);
1073 const auto *DElt = DConst->getAggregateElement(Elt: I);
1074 const auto *EElt = EConst->getAggregateElement(Elt: I);
1075
1076 if (!BElt || !DElt || !EElt)
1077 return nullptr;
1078 if (!isReducible(BElt, DElt, EElt))
1079 return nullptr;
1080 }
1081 } else {
1082 // Test scalar type arguments for conversion.
1083 if (!isReducible(B, D, E))
1084 return nullptr;
1085 }
1086 return Builder.CreateICmp(P: ICmpInst::ICMP_ULT, LHS: A, RHS: D);
1087}
1088
1089/// Try to fold ((icmp X u< P) & (icmp(X & M) != M)) or ((icmp X s> -1) &
1090/// (icmp(X & M) != M)) into (icmp X u< M). Where P is a power of 2, M < P, and
1091/// M is a contiguous shifted mask starting at the right most significant zero
1092/// bit in P. SGT is supported as when P is the largest representable power of
1093/// 2, an earlier optimization converts the expression into (icmp X s> -1).
1094/// Parameter P supports masking using undef/poison in either scalar or vector
1095/// values.
1096static Value *foldPowerOf2AndShiftedMask(Value *Cmp0, Value *Cmp1,
1097 bool JoinedByAnd,
1098 InstCombiner::BuilderTy &Builder) {
1099 if (!JoinedByAnd)
1100 return nullptr;
1101 Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr, *E = nullptr;
1102 ICmpInst::Predicate CmpPred0, CmpPred1;
1103 // Assuming P is a 2^n, getMaskedTypeForICmpPair will normalize (icmp X u<
1104 // 2^n) into (icmp (X & ~(2^n-1)) == 0) and (icmp X s> -1) into (icmp (X &
1105 // SignMask) == 0).
1106 std::optional<std::pair<unsigned, unsigned>> MaskPair =
1107 getMaskedTypeForICmpPair(A, B, C, D, E, LHS: Cmp0, RHS: Cmp1, PredL&: CmpPred0, PredR&: CmpPred1);
1108 if (!MaskPair)
1109 return nullptr;
1110
1111 const auto compareBMask = BMask_NotMixed | BMask_NotAllOnes;
1112 unsigned CmpMask0 = MaskPair->first;
1113 unsigned CmpMask1 = MaskPair->second;
1114 if ((CmpMask0 & Mask_AllZeros) && (CmpMask1 == compareBMask)) {
1115 if (Value *V = foldNegativePower2AndShiftedMask(A, B, D, E, PredL: CmpPred0,
1116 PredR: CmpPred1, Builder))
1117 return V;
1118 } else if ((CmpMask0 == compareBMask) && (CmpMask1 & Mask_AllZeros)) {
1119 if (Value *V = foldNegativePower2AndShiftedMask(A, B: D, D: B, E: C, PredL: CmpPred1,
1120 PredR: CmpPred0, Builder))
1121 return V;
1122 }
1123 return nullptr;
1124}
1125
1126/// Commuted variants are assumed to be handled by calling this function again
1127/// with the parameters swapped.
1128static Value *foldUnsignedUnderflowCheck(CmpPredicate PredL, Value *LHS0,
1129 Value *LHS1, bool LHSOneUse,
1130 CmpPredicate PredR, Value *RHS0,
1131 Value *RHS1, bool RHSOneUse,
1132 bool IsAnd, const SimplifyQuery &Q,
1133 InstCombiner::BuilderTy &Builder) {
1134 if (!match(V: LHS1, P: m_Zero()) || !ICmpInst::isEquality(P: PredL))
1135 return nullptr;
1136
1137 Value *A, *B;
1138 if (RHS0 == LHS0)
1139 A = RHS1;
1140 else if (RHS1 == LHS0) {
1141 A = RHS0;
1142 PredR = CmpPredicate::getSwapped(P: PredR);
1143 } else
1144 return nullptr;
1145
1146 if (!match(V: LHS0, P: m_c_Add(L: m_Specific(V: A), R: m_Value(V&: B))) ||
1147 !(LHSOneUse || RHSOneUse))
1148 return nullptr;
1149
1150 auto GetKnownNonZeroAndOther = [&](Value *&NonZero, Value *&Other) {
1151 if (!isKnownNonZero(V: NonZero, Q))
1152 std::swap(a&: NonZero, b&: Other);
1153 return isKnownNonZero(V: NonZero, Q);
1154 };
1155
1156 // Given ZeroCmpOp = (A + B)
1157 // ZeroCmpOp < A && ZeroCmpOp != 0 --> (0-X) < Y iff
1158 // ZeroCmpOp >= A || ZeroCmpOp == 0 --> (0-X) >= Y iff
1159 // with X being the value (A/B) that is known to be non-zero,
1160 // and Y being remaining value.
1161 if (PredR == ICmpInst::ICMP_ULT && PredL == ICmpInst::ICMP_NE && IsAnd &&
1162 GetKnownNonZeroAndOther(B, A))
1163 return Builder.CreateICmpULT(LHS: Builder.CreateNeg(V: B), RHS: A);
1164 if (PredR == ICmpInst::ICMP_UGE && PredL == ICmpInst::ICMP_EQ && !IsAnd &&
1165 GetKnownNonZeroAndOther(B, A))
1166 return Builder.CreateICmpUGE(LHS: Builder.CreateNeg(V: B), RHS: A);
1167
1168 return nullptr;
1169}
1170
1171struct IntPart {
1172 Value *From;
1173 unsigned StartBit;
1174 unsigned NumBits;
1175};
1176
1177/// Match an extraction of bits from an integer.
1178static std::optional<IntPart> matchIntPart(Value *V) {
1179 Value *X;
1180 if (!match(V, P: m_OneUse(SubPattern: m_Trunc(Op: m_Value(V&: X)))))
1181 return std::nullopt;
1182
1183 unsigned NumOriginalBits = X->getType()->getScalarSizeInBits();
1184 unsigned NumExtractedBits = V->getType()->getScalarSizeInBits();
1185 Value *Y;
1186 const APInt *Shift;
1187 // For a trunc(lshr Y, Shift) pattern, make sure we're only extracting bits
1188 // from Y, not any shifted-in zeroes.
1189 if (match(V: X, P: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: Y), R: m_APInt(Res&: Shift)))) &&
1190 Shift->ule(RHS: NumOriginalBits - NumExtractedBits))
1191 return {{.From: Y, .StartBit: (unsigned)Shift->getZExtValue(), .NumBits: NumExtractedBits}};
1192 return {{.From: X, .StartBit: 0, .NumBits: NumExtractedBits}};
1193}
1194
1195/// Materialize an extraction of bits from an integer in IR.
1196static Value *extractIntPart(const IntPart &P, IRBuilderBase &Builder) {
1197 Value *V = P.From;
1198 if (P.StartBit)
1199 V = Builder.CreateLShr(LHS: V, RHS: P.StartBit);
1200 Type *TruncTy = V->getType()->getWithNewBitWidth(NewBitWidth: P.NumBits);
1201 if (TruncTy != V->getType())
1202 V = Builder.CreateTrunc(V, DestTy: TruncTy);
1203 return V;
1204}
1205
1206/// (icmp eq X0, Y0) & (icmp eq X1, Y1) -> icmp eq X01, Y01
1207/// (icmp ne X0, Y0) | (icmp ne X1, Y1) -> icmp ne X01, Y01
1208/// where X0, X1 and Y0, Y1 are adjacent parts extracted from an integer.
1209Value *InstCombinerImpl::foldEqOfParts(Value *Cmp0, Value *Cmp1, bool IsAnd) {
1210 if (!Cmp0->hasOneUse() || !Cmp1->hasOneUse())
1211 return nullptr;
1212
1213 CmpInst::Predicate Pred = IsAnd ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
1214 auto GetMatchPart = [&](Value *CmpV,
1215 unsigned OpNo) -> std::optional<IntPart> {
1216 assert(CmpV->getType()->isIntOrIntVectorTy(1) && "Must be bool");
1217
1218 Value *X, *Y;
1219 // icmp ne (and x, 1), (and y, 1) <=> trunc (xor x, y) to i1
1220 // icmp eq (and x, 1), (and y, 1) <=> not (trunc (xor x, y) to i1)
1221 if (Pred == CmpInst::ICMP_NE
1222 ? match(V: CmpV, P: m_Trunc(Op: m_Xor(L: m_Value(V&: X), R: m_Value(V&: Y))))
1223 : match(V: CmpV, P: m_Not(V: m_Trunc(Op: m_Xor(L: m_Value(V&: X), R: m_Value(V&: Y))))))
1224 return {{.From: OpNo == 0 ? X : Y, .StartBit: 0, .NumBits: 1}};
1225
1226 auto *Cmp = dyn_cast<ICmpInst>(Val: CmpV);
1227 if (!Cmp)
1228 return std::nullopt;
1229
1230 if (Pred == Cmp->getPredicate())
1231 return matchIntPart(V: Cmp->getOperand(i_nocapture: OpNo));
1232
1233 const APInt *C;
1234 // (icmp eq (lshr x, C), (lshr y, C)) gets optimized to:
1235 // (icmp ult (xor x, y), 1 << C) so also look for that.
1236 if (Pred == CmpInst::ICMP_EQ && Cmp->getPredicate() == CmpInst::ICMP_ULT) {
1237 if (!match(V: Cmp->getOperand(i_nocapture: 1), P: m_Power2(V&: C)) ||
1238 !match(V: Cmp->getOperand(i_nocapture: 0), P: m_Xor(L: m_Value(), R: m_Value())))
1239 return std::nullopt;
1240 }
1241
1242 // (icmp ne (lshr x, C), (lshr y, C)) gets optimized to:
1243 // (icmp ugt (xor x, y), (1 << C) - 1) so also look for that.
1244 else if (Pred == CmpInst::ICMP_NE &&
1245 Cmp->getPredicate() == CmpInst::ICMP_UGT) {
1246 if (!match(V: Cmp->getOperand(i_nocapture: 1), P: m_LowBitMask(V&: C)) ||
1247 !match(V: Cmp->getOperand(i_nocapture: 0), P: m_Xor(L: m_Value(), R: m_Value())))
1248 return std::nullopt;
1249 } else {
1250 return std::nullopt;
1251 }
1252
1253 unsigned From = Pred == CmpInst::ICMP_NE ? C->popcount() : C->countr_zero();
1254 Instruction *I = cast<Instruction>(Val: Cmp->getOperand(i_nocapture: 0));
1255 return {{.From: I->getOperand(i: OpNo), .StartBit: From, .NumBits: C->getBitWidth() - From}};
1256 };
1257
1258 std::optional<IntPart> L0 = GetMatchPart(Cmp0, 0);
1259 std::optional<IntPart> R0 = GetMatchPart(Cmp0, 1);
1260 std::optional<IntPart> L1 = GetMatchPart(Cmp1, 0);
1261 std::optional<IntPart> R1 = GetMatchPart(Cmp1, 1);
1262 if (!L0 || !R0 || !L1 || !R1)
1263 return nullptr;
1264
1265 // Make sure the LHS/RHS compare a part of the same value, possibly after
1266 // an operand swap.
1267 if (L0->From != L1->From || R0->From != R1->From) {
1268 if (L0->From != R1->From || R0->From != L1->From)
1269 return nullptr;
1270 std::swap(lhs&: L1, rhs&: R1);
1271 }
1272
1273 // Make sure the extracted parts are adjacent, canonicalizing to L0/R0 being
1274 // the low part and L1/R1 being the high part.
1275 if (L0->StartBit + L0->NumBits != L1->StartBit ||
1276 R0->StartBit + R0->NumBits != R1->StartBit) {
1277 if (L1->StartBit + L1->NumBits != L0->StartBit ||
1278 R1->StartBit + R1->NumBits != R0->StartBit)
1279 return nullptr;
1280 std::swap(lhs&: L0, rhs&: L1);
1281 std::swap(lhs&: R0, rhs&: R1);
1282 }
1283
1284 // We can simplify to a comparison of these larger parts of the integers.
1285 IntPart L = {.From: L0->From, .StartBit: L0->StartBit, .NumBits: L0->NumBits + L1->NumBits};
1286 IntPart R = {.From: R0->From, .StartBit: R0->StartBit, .NumBits: R0->NumBits + R1->NumBits};
1287 Value *LValue = extractIntPart(P: L, Builder);
1288 Value *RValue = extractIntPart(P: R, Builder);
1289 return Builder.CreateICmp(P: Pred, LHS: LValue, RHS: RValue);
1290}
1291
1292/// Reduce logic-of-compares with equality to a constant by substituting a
1293/// common operand with the constant. Callers are expected to call this with
1294/// Cmp0/Cmp1 switched to handle logic op commutativity.
1295static Value *
1296foldAndOrOfICmpsWithConstEq(CmpPredicate PredL, Value *LHS0, Value *LHS1,
1297 Value *LHS, CmpPredicate PredR, Value *RHS0,
1298 Value *RHS1, bool RHSOneUse, bool IsAnd,
1299 bool IsLogical, InstCombiner::BuilderTy &Builder,
1300 const SimplifyQuery &Q, Instruction &I) {
1301 // Match an equality compare with a non-poison constant as Cmp0.
1302 // Also, give up if the compare can be constant-folded to avoid looping.
1303 if (!isa<Constant>(Val: LHS1) || !isGuaranteedNotToBeUndefOrPoison(V: LHS1) ||
1304 isa<Constant>(Val: LHS0))
1305 return nullptr;
1306 if ((IsAnd && PredL != ICmpInst::ICMP_EQ) ||
1307 (!IsAnd && PredL != ICmpInst::ICMP_NE))
1308 return nullptr;
1309
1310 // The other compare must include a common operand (X). Canonicalize the
1311 // common operand as operand 1 (Pred1 is swapped if the common operand was
1312 // operand 0).
1313 Value *Y;
1314
1315 if (LHS0 == RHS0) {
1316 Y = RHS1;
1317 PredR = CmpPredicate::getSwapped(P: PredR);
1318 } else if (LHS0 == RHS1)
1319 Y = RHS0;
1320 else
1321 return nullptr;
1322
1323 // Replace variable with constant value equivalence to remove a variable use:
1324 // (X == C) && (Y Pred1 X) --> (X == C) && (Y Pred1 C)
1325 // (X != C) || (Y Pred1 X) --> (X != C) || (Y Pred1 C)
1326 // Can think of the 'or' substitution with the 'and' bool equivalent:
1327 // A || B --> A || (!A && B)
1328 Value *SubstituteCmp = simplifyICmpInst(Pred: PredR, LHS: Y, RHS: LHS1, Q);
1329 if (!SubstituteCmp) {
1330 // If we need to create a new instruction, require that the old compare can
1331 // be removed.
1332 if (!RHSOneUse)
1333 return nullptr;
1334 SubstituteCmp = Builder.CreateICmp(P: PredR, LHS: Y, RHS: LHS1);
1335 }
1336 if (IsLogical) {
1337 Instruction *MDFrom = isa<SelectInst>(Val: I) ? &I : nullptr;
1338 return IsAnd ? Builder.CreateLogicalAnd(Cond1: LHS, Cond2: SubstituteCmp, Name: "", MDFrom)
1339 : Builder.CreateLogicalOr(Cond1: LHS, Cond2: SubstituteCmp, Name: "", MDFrom);
1340 }
1341 return Builder.CreateBinOp(Opc: IsAnd ? Instruction::And : Instruction::Or, LHS,
1342 RHS: SubstituteCmp);
1343}
1344
1345/// Fold (icmp Pred1 V1, C1) & (icmp Pred2 V2, C2)
1346/// or (icmp Pred1 V1, C1) | (icmp Pred2 V2, C2)
1347/// into a single comparison using range-based reasoning.
1348/// NOTE: This is also used for logical and/or, must be poison-safe!
1349Value *InstCombinerImpl::foldAndOrOfICmpsUsingRanges(
1350 CmpPredicate PredL, Value *LHS0, Value *LHS1, bool LHSOneUse,
1351 CmpPredicate PredR, Value *RHS0, Value *RHS1, bool RHSOneUse, bool IsAnd) {
1352 // Return (V, CR) for a range check idiom V in CR.
1353 auto MatchExactRangeCheck =
1354 [](CmpPredicate Pred, Value *LHS,
1355 Value *RHS) -> std::optional<std::pair<Value *, ConstantRange>> {
1356 const APInt *C;
1357 if (!match(V: RHS, P: m_APInt(Res&: C)))
1358 return std::nullopt;
1359
1360 Value *X;
1361 // Match (x & NegPow2) ==/!= C
1362 const APInt *Mask;
1363 if (ICmpInst::isEquality(P: Pred) &&
1364 match(V: LHS, P: m_OneUse(SubPattern: m_And(L: m_Value(V&: X), R: m_NegatedPower2(V&: Mask)))) &&
1365 C->countr_zero() >= Mask->countr_zero()) {
1366 ConstantRange CR(*C, *C - *Mask);
1367 if (Pred == ICmpInst::ICMP_NE)
1368 CR = CR.inverse();
1369 return std::make_pair(x&: X, y&: CR);
1370 }
1371 ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
1372 // Match (add X, C1) pred C
1373 // TODO: investigate whether we should apply the one-use check on m_AddLike.
1374 const APInt *C1;
1375 if (match(V: LHS, P: m_AddLike(L: m_Value(V&: X), R: m_APInt(Res&: C1))))
1376 return std::make_pair(x&: X, y: CR.subtract(CI: *C1));
1377 return std::make_pair(x&: LHS, y&: CR);
1378 };
1379
1380 auto RC1 = MatchExactRangeCheck(PredL, LHS0, LHS1);
1381 if (!RC1)
1382 return nullptr;
1383
1384 auto RC2 = MatchExactRangeCheck(PredR, RHS0, RHS1);
1385 if (!RC2)
1386 return nullptr;
1387
1388 auto &[V1, CR1] = *RC1;
1389 auto &[V2, CR2] = *RC2;
1390 if (V1 != V2)
1391 return nullptr;
1392
1393 // For 'and', we use the De Morgan's Laws to simplify the implementation.
1394 if (IsAnd) {
1395 CR1 = CR1.inverse();
1396 CR2 = CR2.inverse();
1397 }
1398
1399 Type *Ty = V1->getType();
1400 Value *NewV = V1;
1401 std::optional<ConstantRange> CR = CR1.exactUnionWith(CR: CR2);
1402 if (!CR) {
1403 if (!LHSOneUse || !RHSOneUse || CR1.isWrappedSet() || CR2.isWrappedSet())
1404 return nullptr;
1405
1406 // Check whether we have equal-size ranges that only differ by one bit.
1407 // In that case we can apply a mask to map one range onto the other.
1408 APInt LowerDiff = CR1.getLower() ^ CR2.getLower();
1409 APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1);
1410 APInt CR1Size = CR1.getUpper() - CR1.getLower();
1411 if (!LowerDiff.isPowerOf2() || LowerDiff != UpperDiff ||
1412 CR1Size != CR2.getUpper() - CR2.getLower())
1413 return nullptr;
1414
1415 CR = CR1.getLower().ult(RHS: CR2.getLower()) ? CR1 : CR2;
1416 NewV = Builder.CreateAnd(LHS: NewV, RHS: ConstantInt::get(Ty, V: ~LowerDiff));
1417 }
1418
1419 if (IsAnd)
1420 CR = CR->inverse();
1421
1422 CmpInst::Predicate NewPred;
1423 APInt NewC, Offset;
1424 CR->getEquivalentICmp(Pred&: NewPred, RHS&: NewC, Offset);
1425
1426 if (Offset != 0)
1427 NewV = Builder.CreateAdd(LHS: NewV, RHS: ConstantInt::get(Ty, V: Offset));
1428 return Builder.CreateICmp(P: NewPred, LHS: NewV, RHS: ConstantInt::get(Ty, V: NewC));
1429}
1430
1431/// Matches canonical form of isnan, fcmp ord x, 0
1432static bool matchIsNotNaN(FCmpInst::Predicate P, Value *LHS, Value *RHS) {
1433 return P == FCmpInst::FCMP_ORD && match(V: RHS, P: m_AnyZeroFP());
1434}
1435
1436/// Matches fcmp u__ x, +/-inf
1437static bool matchUnorderedInfCompare(FCmpInst::Predicate P, Value *LHS,
1438 Value *RHS) {
1439 return FCmpInst::isUnordered(predicate: P) && match(V: RHS, P: m_Inf());
1440}
1441
1442/// and (fcmp ord x, 0), (fcmp u* x, inf) -> fcmp o* x, inf
1443///
1444/// Clang emits this pattern for doing an isfinite check in __builtin_isnormal.
1445static Value *matchIsFiniteTest(InstCombiner::BuilderTy &Builder, FCmpInst *LHS,
1446 FCmpInst *RHS) {
1447 Value *LHS0 = LHS->getOperand(i_nocapture: 0), *LHS1 = LHS->getOperand(i_nocapture: 1);
1448 Value *RHS0 = RHS->getOperand(i_nocapture: 0), *RHS1 = RHS->getOperand(i_nocapture: 1);
1449 FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
1450
1451 if (!matchIsNotNaN(P: PredL, LHS: LHS0, RHS: LHS1) ||
1452 !matchUnorderedInfCompare(P: PredR, LHS: RHS0, RHS: RHS1))
1453 return nullptr;
1454
1455 return Builder.CreateFCmpFMF(P: FCmpInst::getOrderedPredicate(Pred: PredR), LHS: RHS0, RHS: RHS1,
1456 FMFSource: FMFSource::intersect(A: LHS, B: RHS));
1457}
1458
1459Value *InstCombinerImpl::foldLogicOfFCmps(FCmpInst *LHS, FCmpInst *RHS,
1460 bool IsAnd, bool IsLogicalSelect) {
1461 Value *LHS0 = LHS->getOperand(i_nocapture: 0), *LHS1 = LHS->getOperand(i_nocapture: 1);
1462 Value *RHS0 = RHS->getOperand(i_nocapture: 0), *RHS1 = RHS->getOperand(i_nocapture: 1);
1463 FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
1464
1465 if (LHS0 == RHS1 && RHS0 == LHS1) {
1466 // Swap RHS operands to match LHS.
1467 PredR = FCmpInst::getSwappedPredicate(pred: PredR);
1468 std::swap(a&: RHS0, b&: RHS1);
1469 }
1470
1471 // Simplify (fcmp cc0 x, y) & (fcmp cc1 x, y).
1472 // Suppose the relation between x and y is R, where R is one of
1473 // U(1000), L(0100), G(0010) or E(0001), and CC0 and CC1 are the bitmasks for
1474 // testing the desired relations.
1475 //
1476 // Since (R & CC0) and (R & CC1) are either R or 0, we actually have this:
1477 // bool(R & CC0) && bool(R & CC1)
1478 // = bool((R & CC0) & (R & CC1))
1479 // = bool(R & (CC0 & CC1)) <= by re-association, commutation, and idempotency
1480 //
1481 // Since (R & CC0) and (R & CC1) are either R or 0, we actually have this:
1482 // bool(R & CC0) || bool(R & CC1)
1483 // = bool((R & CC0) | (R & CC1))
1484 // = bool(R & (CC0 | CC1)) <= by reversed distribution (contribution? ;)
1485 if (LHS0 == RHS0 && LHS1 == RHS1) {
1486 unsigned FCmpCodeL = getFCmpCode(CC: PredL);
1487 unsigned FCmpCodeR = getFCmpCode(CC: PredR);
1488 unsigned NewPred = IsAnd ? FCmpCodeL & FCmpCodeR : FCmpCodeL | FCmpCodeR;
1489
1490 // Intersect the fast math flags.
1491 // TODO: We can union the fast math flags unless this is a logical select.
1492 return getFCmpValue(Code: NewPred, LHS: LHS0, RHS: LHS1, Builder,
1493 FMF: FMFSource::intersect(A: LHS, B: RHS));
1494 }
1495
1496 if ((PredL == FCmpInst::FCMP_ORD && PredR == FCmpInst::FCMP_ORD && IsAnd) ||
1497 (PredL == FCmpInst::FCMP_UNO && PredR == FCmpInst::FCMP_UNO && !IsAnd)) {
1498 if (LHS0->getType() != RHS0->getType())
1499 return nullptr;
1500
1501 // FCmp canonicalization ensures that (fcmp ord/uno X, X) and
1502 // (fcmp ord/uno X, C) will be transformed to (fcmp X, +0.0).
1503 if (match(V: LHS1, P: m_PosZeroFP()) && match(V: RHS1, P: m_PosZeroFP())) {
1504 // Ignore the constants because they are obviously not NANs:
1505 // (fcmp ord x, 0.0) & (fcmp ord y, 0.0) -> (fcmp ord x, y)
1506 // (fcmp uno x, 0.0) | (fcmp uno y, 0.0) -> (fcmp uno x, y)
1507 Value *Y = RHS0;
1508 FastMathFlags FMF = LHS->getFastMathFlags() & RHS->getFastMathFlags();
1509 if (IsLogicalSelect) {
1510 Y = Builder.CreateFreeze(V: Y, Name: Y->getName() + ".fr");
1511 FMF.setNoNaNs(false);
1512 FMF.setNoInfs(false);
1513 }
1514 return Builder.CreateFCmpFMF(P: PredL, LHS: LHS0, RHS: Y, FMFSource: FMF);
1515 }
1516 }
1517
1518 // This transform is not valid for a logical select.
1519 if (!IsLogicalSelect && IsAnd &&
1520 stripSignOnlyFPOps(Val: LHS0) == stripSignOnlyFPOps(Val: RHS0)) {
1521 // and (fcmp ord x, 0), (fcmp u* x, inf) -> fcmp o* x, inf
1522 // and (fcmp ord x, 0), (fcmp u* fabs(x), inf) -> fcmp o* x, inf
1523 if (Value *Left = matchIsFiniteTest(Builder, LHS, RHS))
1524 return Left;
1525 if (Value *Right = matchIsFiniteTest(Builder, LHS: RHS, RHS: LHS))
1526 return Right;
1527 }
1528
1529 // Turn at least two fcmps with constants into llvm.is.fpclass.
1530 //
1531 // If we can represent a combined value test with one class call, we can
1532 // potentially eliminate 4-6 instructions. If we can represent a test with a
1533 // single fcmp with fneg and fabs, that's likely a better canonical form.
1534 if (LHS->hasOneUse() && RHS->hasOneUse()) {
1535 auto [ClassValRHS, ClassMaskRHS] =
1536 fcmpToClassTest(Pred: PredR, F: *RHS->getFunction(), LHS: RHS0, RHS: RHS1);
1537 if (ClassValRHS) {
1538 auto [ClassValLHS, ClassMaskLHS] =
1539 fcmpToClassTest(Pred: PredL, F: *LHS->getFunction(), LHS: LHS0, RHS: LHS1);
1540 if (ClassValLHS == ClassValRHS) {
1541 unsigned CombinedMask = IsAnd ? (ClassMaskLHS & ClassMaskRHS)
1542 : (ClassMaskLHS | ClassMaskRHS);
1543 return Builder.CreateIntrinsic(
1544 ID: Intrinsic::is_fpclass, OverloadTypes: {ClassValLHS->getType()},
1545 Args: {ClassValLHS, Builder.getInt32(C: CombinedMask)});
1546 }
1547 }
1548 }
1549
1550 // Canonicalize the range check idiom:
1551 // and (fcmp olt/ole/ult/ule x, C), (fcmp ogt/oge/ugt/uge x, -C)
1552 // --> fabs(x) olt/ole/ult/ule C
1553 // or (fcmp ogt/oge/ugt/uge x, C), (fcmp olt/ole/ult/ule x, -C)
1554 // --> fabs(x) ogt/oge/ugt/uge C
1555 // TODO: Generalize to handle a negated variable operand?
1556 const APFloat *LHSC, *RHSC;
1557 if (LHS0 == RHS0 && LHS->hasOneUse() && RHS->hasOneUse() &&
1558 FCmpInst::getSwappedPredicate(pred: PredL) == PredR &&
1559 match(V: LHS1, P: m_APFloatAllowPoison(Res&: LHSC)) &&
1560 match(V: RHS1, P: m_APFloatAllowPoison(Res&: RHSC)) &&
1561 LHSC->bitwiseIsEqual(RHS: neg(X: *RHSC))) {
1562 auto IsLessThanOrLessEqual = [](FCmpInst::Predicate Pred) {
1563 switch (Pred) {
1564 case FCmpInst::FCMP_OLT:
1565 case FCmpInst::FCMP_OLE:
1566 case FCmpInst::FCMP_ULT:
1567 case FCmpInst::FCMP_ULE:
1568 return true;
1569 default:
1570 return false;
1571 }
1572 };
1573 if (IsLessThanOrLessEqual(IsAnd ? PredR : PredL)) {
1574 std::swap(a&: LHSC, b&: RHSC);
1575 std::swap(a&: PredL, b&: PredR);
1576 }
1577 if (IsLessThanOrLessEqual(IsAnd ? PredL : PredR)) {
1578 FastMathFlags NewFlag = LHS->getFastMathFlags();
1579 if (!IsLogicalSelect)
1580 NewFlag |= RHS->getFastMathFlags();
1581
1582 Value *FAbs = Builder.CreateFAbs(V: LHS0, FMFSource: NewFlag);
1583 return Builder.CreateFCmpFMF(
1584 P: PredL, LHS: FAbs, RHS: ConstantFP::get(Ty: LHS0->getType(), V: *LHSC), FMFSource: NewFlag);
1585 }
1586 }
1587
1588 return nullptr;
1589}
1590
1591/// Match an fcmp against a special value that performs a test possible by
1592/// llvm.is.fpclass.
1593static bool matchIsFPClassLikeFCmp(Value *Op, Value *&ClassVal,
1594 uint64_t &ClassMask) {
1595 auto *FCmp = dyn_cast<FCmpInst>(Val: Op);
1596 if (!FCmp || !FCmp->hasOneUse())
1597 return false;
1598
1599 std::tie(args&: ClassVal, args&: ClassMask) =
1600 fcmpToClassTest(Pred: FCmp->getPredicate(), F: *FCmp->getParent()->getParent(),
1601 LHS: FCmp->getOperand(i_nocapture: 0), RHS: FCmp->getOperand(i_nocapture: 1));
1602 return ClassVal != nullptr;
1603}
1604
1605/// or (is_fpclass x, mask0), (is_fpclass x, mask1)
1606/// -> is_fpclass x, (mask0 | mask1)
1607/// and (is_fpclass x, mask0), (is_fpclass x, mask1)
1608/// -> is_fpclass x, (mask0 & mask1)
1609/// xor (is_fpclass x, mask0), (is_fpclass x, mask1)
1610/// -> is_fpclass x, (mask0 ^ mask1)
1611Instruction *InstCombinerImpl::foldLogicOfIsFPClass(BinaryOperator &BO,
1612 Value *Op0, Value *Op1) {
1613 Value *ClassVal0 = nullptr;
1614 Value *ClassVal1 = nullptr;
1615 uint64_t ClassMask0, ClassMask1;
1616
1617 // Restrict to folding one fcmp into one is.fpclass for now, don't introduce a
1618 // new class.
1619 //
1620 // TODO: Support forming is.fpclass out of 2 separate fcmps when codegen is
1621 // better.
1622
1623 bool IsLHSClass =
1624 match(V: Op0, P: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::is_fpclass>(
1625 Ops: m_Value(V&: ClassVal0), Ops: m_ConstantInt(V&: ClassMask0))));
1626 bool IsRHSClass =
1627 match(V: Op1, P: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::is_fpclass>(
1628 Ops: m_Value(V&: ClassVal1), Ops: m_ConstantInt(V&: ClassMask1))));
1629 if ((((IsLHSClass || matchIsFPClassLikeFCmp(Op: Op0, ClassVal&: ClassVal0, ClassMask&: ClassMask0)) &&
1630 (IsRHSClass || matchIsFPClassLikeFCmp(Op: Op1, ClassVal&: ClassVal1, ClassMask&: ClassMask1)))) &&
1631 ClassVal0 == ClassVal1) {
1632 unsigned NewClassMask;
1633 switch (BO.getOpcode()) {
1634 case Instruction::And:
1635 NewClassMask = ClassMask0 & ClassMask1;
1636 break;
1637 case Instruction::Or:
1638 NewClassMask = ClassMask0 | ClassMask1;
1639 break;
1640 case Instruction::Xor:
1641 NewClassMask = ClassMask0 ^ ClassMask1;
1642 break;
1643 default:
1644 llvm_unreachable("not a binary logic operator");
1645 }
1646
1647 if (IsLHSClass) {
1648 auto *II = cast<IntrinsicInst>(Val: Op0);
1649 II->setArgOperand(
1650 i: 1, v: ConstantInt::get(Ty: II->getArgOperand(i: 1)->getType(), V: NewClassMask));
1651 return replaceInstUsesWith(I&: BO, V: II);
1652 }
1653
1654 if (IsRHSClass) {
1655 auto *II = cast<IntrinsicInst>(Val: Op1);
1656 II->setArgOperand(
1657 i: 1, v: ConstantInt::get(Ty: II->getArgOperand(i: 1)->getType(), V: NewClassMask));
1658 return replaceInstUsesWith(I&: BO, V: II);
1659 }
1660
1661 Value *NewClass =
1662 Builder.CreateIntrinsic(ID: Intrinsic::is_fpclass, OverloadTypes: {ClassVal0->getType()},
1663 Args: {ClassVal0, Builder.getInt32(C: NewClassMask)});
1664 return replaceInstUsesWith(I&: BO, V: NewClass);
1665 }
1666
1667 return nullptr;
1668}
1669
1670/// Look for the pattern that conditionally negates a value via math operations:
1671/// cond.splat = sext i1 cond
1672/// sub = add cond.splat, x
1673/// xor = xor sub, cond.splat
1674/// and rewrite it to do the same, but via logical operations:
1675/// value.neg = sub 0, value
1676/// cond = select i1 neg, value.neg, value
1677Instruction *InstCombinerImpl::canonicalizeConditionalNegationViaMathToSelect(
1678 BinaryOperator &I) {
1679 assert(I.getOpcode() == BinaryOperator::Xor && "Only for xor!");
1680 Value *Cond, *X;
1681 // As per complexity ordering, `xor` is not commutative here.
1682 if (!match(V: &I, P: m_c_BinOp(L: m_OneUse(SubPattern: m_Value()), R: m_Value())) ||
1683 !match(V: I.getOperand(i_nocapture: 1), P: m_SExt(Op: m_Value(V&: Cond))) ||
1684 !Cond->getType()->isIntOrIntVectorTy(BitWidth: 1) ||
1685 !match(V: I.getOperand(i_nocapture: 0), P: m_c_Add(L: m_SExt(Op: m_Specific(V: Cond)), R: m_Value(V&: X))))
1686 return nullptr;
1687 return createSelectInstWithUnknownProfile(
1688 C: Cond, S1: Builder.CreateNeg(V: X, Name: X->getName() + ".neg"), S2: X);
1689}
1690
1691/// This a limited reassociation for a special case (see above) where we are
1692/// checking if two values are either both NAN (unordered) or not-NAN (ordered).
1693/// This could be handled more generally in '-reassociation', but it seems like
1694/// an unlikely pattern for a large number of logic ops and fcmps.
1695static Instruction *reassociateFCmps(BinaryOperator &BO,
1696 InstCombiner::BuilderTy &Builder) {
1697 Instruction::BinaryOps Opcode = BO.getOpcode();
1698 assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&
1699 "Expecting and/or op for fcmp transform");
1700
1701 // There are 4 commuted variants of the pattern. Canonicalize operands of this
1702 // logic op so an fcmp is operand 0 and a matching logic op is operand 1.
1703 Value *Op0 = BO.getOperand(i_nocapture: 0), *Op1 = BO.getOperand(i_nocapture: 1), *X;
1704 if (match(V: Op1, P: m_FCmp(L: m_Value(), R: m_AnyZeroFP())))
1705 std::swap(a&: Op0, b&: Op1);
1706
1707 // Match inner binop and the predicate for combining 2 NAN checks into 1.
1708 Value *BO10, *BO11;
1709 FCmpInst::Predicate NanPred = Opcode == Instruction::And ? FCmpInst::FCMP_ORD
1710 : FCmpInst::FCMP_UNO;
1711 if (!match(V: Op0, P: m_SpecificFCmp(MatchPred: NanPred, L: m_Value(V&: X), R: m_AnyZeroFP())) ||
1712 !match(V: Op1, P: m_BinOp(Opcode, L: m_Value(V&: BO10), R: m_Value(V&: BO11))))
1713 return nullptr;
1714
1715 // The inner logic op must have a matching fcmp operand.
1716 Value *Y;
1717 if (!match(V: BO10, P: m_SpecificFCmp(MatchPred: NanPred, L: m_Value(V&: Y), R: m_AnyZeroFP())) ||
1718 X->getType() != Y->getType())
1719 std::swap(a&: BO10, b&: BO11);
1720
1721 if (!match(V: BO10, P: m_SpecificFCmp(MatchPred: NanPred, L: m_Value(V&: Y), R: m_AnyZeroFP())) ||
1722 X->getType() != Y->getType())
1723 return nullptr;
1724
1725 // and (fcmp ord X, 0), (and (fcmp ord Y, 0), Z) --> and (fcmp ord X, Y), Z
1726 // or (fcmp uno X, 0), (or (fcmp uno Y, 0), Z) --> or (fcmp uno X, Y), Z
1727 // Intersect FMF from the 2 source fcmps.
1728 Value *NewFCmp =
1729 Builder.CreateFCmpFMF(P: NanPred, LHS: X, RHS: Y, FMFSource: FMFSource::intersect(A: Op0, B: BO10));
1730 return BinaryOperator::Create(Op: Opcode, S1: NewFCmp, S2: BO11);
1731}
1732
1733/// Match variations of De Morgan's Laws:
1734/// (~A & ~B) == (~(A | B))
1735/// (~A | ~B) == (~(A & B))
1736static Instruction *matchDeMorgansLaws(BinaryOperator &I,
1737 InstCombiner &IC) {
1738 const Instruction::BinaryOps Opcode = I.getOpcode();
1739 assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&
1740 "Trying to match De Morgan's Laws with something other than and/or");
1741
1742 // Flip the logic operation.
1743 const Instruction::BinaryOps FlippedOpcode =
1744 (Opcode == Instruction::And) ? Instruction::Or : Instruction::And;
1745
1746 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
1747 Value *A, *B;
1748 if (match(V: Op0, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: A)))) &&
1749 match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: B)))) &&
1750 !IC.isFreeToInvert(V: A, WillInvertAllUses: A->hasOneUse()) &&
1751 !IC.isFreeToInvert(V: B, WillInvertAllUses: B->hasOneUse())) {
1752 Value *AndOr =
1753 IC.Builder.CreateBinOp(Opc: FlippedOpcode, LHS: A, RHS: B, Name: I.getName() + ".demorgan");
1754 return BinaryOperator::CreateNot(Op: AndOr);
1755 }
1756
1757 // The 'not' ops may require reassociation.
1758 // (A & ~B) & ~C --> A & ~(B | C)
1759 // (~B & A) & ~C --> A & ~(B | C)
1760 // (A | ~B) | ~C --> A | ~(B & C)
1761 // (~B | A) | ~C --> A | ~(B & C)
1762 Value *C;
1763 if (match(V: Op0, P: m_OneUse(SubPattern: m_c_BinOp(Opcode, L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B))))) &&
1764 match(V: Op1, P: m_Not(V: m_Value(V&: C)))) {
1765 Value *FlippedBO = IC.Builder.CreateBinOp(Opc: FlippedOpcode, LHS: B, RHS: C);
1766 return BinaryOperator::Create(Op: Opcode, S1: A, S2: IC.Builder.CreateNot(V: FlippedBO));
1767 }
1768
1769 return nullptr;
1770}
1771
1772bool InstCombinerImpl::shouldOptimizeCast(CastInst *CI) {
1773 Value *CastSrc = CI->getOperand(i_nocapture: 0);
1774
1775 // Noop casts and casts of constants should be eliminated trivially.
1776 if (CI->getSrcTy() == CI->getDestTy() || isa<Constant>(Val: CastSrc))
1777 return false;
1778
1779 // If this cast is paired with another cast that can be eliminated, we prefer
1780 // to have it eliminated.
1781 if (const auto *PrecedingCI = dyn_cast<CastInst>(Val: CastSrc))
1782 if (isEliminableCastPair(CI1: PrecedingCI, CI2: CI))
1783 return false;
1784
1785 return true;
1786}
1787
1788/// Fold {and,or,xor} (cast X), C.
1789static Instruction *foldLogicCastConstant(BinaryOperator &Logic, CastInst *Cast,
1790 InstCombinerImpl &IC) {
1791 Constant *C = dyn_cast<Constant>(Val: Logic.getOperand(i_nocapture: 1));
1792 if (!C)
1793 return nullptr;
1794
1795 auto LogicOpc = Logic.getOpcode();
1796 Type *DestTy = Logic.getType();
1797 Type *SrcTy = Cast->getSrcTy();
1798
1799 // Move the logic operation ahead of a zext or sext if the constant is
1800 // unchanged in the smaller source type. Performing the logic in a smaller
1801 // type may provide more information to later folds, and the smaller logic
1802 // instruction may be cheaper (particularly in the case of vectors).
1803 Value *X;
1804 auto &DL = IC.getDataLayout();
1805 if (match(V: Cast, P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X))))) {
1806 PreservedCastFlags Flags;
1807 if (Constant *TruncC = getLosslessUnsignedTrunc(C, DestTy: SrcTy, DL, Flags: &Flags)) {
1808 // LogicOpc (zext X), C --> zext (LogicOpc X, C)
1809 Value *NewOp = IC.Builder.CreateBinOp(Opc: LogicOpc, LHS: X, RHS: TruncC);
1810 auto *ZExt = new ZExtInst(NewOp, DestTy);
1811 ZExt->setNonNeg(Flags.NNeg);
1812 ZExt->andIRFlags(V: Cast);
1813 return ZExt;
1814 }
1815 }
1816
1817 if (match(V: Cast, P: m_OneUse(SubPattern: m_SExtLike(Op: m_Value(V&: X))))) {
1818 if (Constant *TruncC = getLosslessSignedTrunc(C, DestTy: SrcTy, DL)) {
1819 // LogicOpc (sext X), C --> sext (LogicOpc X, C)
1820 Value *NewOp = IC.Builder.CreateBinOp(Opc: LogicOpc, LHS: X, RHS: TruncC);
1821 return new SExtInst(NewOp, DestTy);
1822 }
1823 }
1824
1825 return nullptr;
1826}
1827
1828/// Fold {and,or,xor} (cast X), Y.
1829Instruction *InstCombinerImpl::foldCastedBitwiseLogic(BinaryOperator &I) {
1830 auto LogicOpc = I.getOpcode();
1831 assert(I.isBitwiseLogicOp() && "Unexpected opcode for bitwise logic folding");
1832
1833 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
1834
1835 // fold bitwise(A >> BW - 1, zext(icmp)) (BW is the scalar bits of the
1836 // type of A)
1837 // -> bitwise(zext(A < 0), zext(icmp))
1838 // -> zext(bitwise(A < 0, icmp))
1839 auto FoldBitwiseICmpZeroWithICmp = [&](Value *Op0,
1840 Value *Op1) -> Instruction * {
1841 Value *A;
1842 bool IsMatched =
1843 match(V: Op0,
1844 P: m_OneUse(SubPattern: m_LShr(
1845 L: m_Value(V&: A),
1846 R: m_SpecificInt(V: Op0->getType()->getScalarSizeInBits() - 1)))) &&
1847 match(V: Op1, P: m_OneUse(SubPattern: m_ZExt(Op: m_ICmp(L: m_Value(), R: m_Value()))));
1848
1849 if (!IsMatched)
1850 return nullptr;
1851
1852 auto *ICmpL =
1853 Builder.CreateICmpSLT(LHS: A, RHS: Constant::getNullValue(Ty: A->getType()));
1854 auto *ICmpR = cast<ZExtInst>(Val: Op1)->getOperand(i_nocapture: 0);
1855 auto *BitwiseOp = Builder.CreateBinOp(Opc: LogicOpc, LHS: ICmpL, RHS: ICmpR);
1856
1857 return new ZExtInst(BitwiseOp, Op0->getType());
1858 };
1859
1860 if (auto *Ret = FoldBitwiseICmpZeroWithICmp(Op0, Op1))
1861 return Ret;
1862
1863 if (auto *Ret = FoldBitwiseICmpZeroWithICmp(Op1, Op0))
1864 return Ret;
1865
1866 CastInst *Cast0 = dyn_cast<CastInst>(Val: Op0);
1867 if (!Cast0)
1868 return nullptr;
1869
1870 // This must be a cast from an integer or integer vector source type to allow
1871 // transformation of the logic operation to the source type.
1872 Type *DestTy = I.getType();
1873 Type *SrcTy = Cast0->getSrcTy();
1874 if (!SrcTy->isIntOrIntVectorTy())
1875 return nullptr;
1876
1877 if (Instruction *Ret = foldLogicCastConstant(Logic&: I, Cast: Cast0, IC&: *this))
1878 return Ret;
1879
1880 CastInst *Cast1 = dyn_cast<CastInst>(Val: Op1);
1881 if (!Cast1)
1882 return nullptr;
1883
1884 // Both operands of the logic operation are casts. The casts must be the
1885 // same kind for reduction.
1886 Instruction::CastOps CastOpcode = Cast0->getOpcode();
1887 if (CastOpcode != Cast1->getOpcode())
1888 return nullptr;
1889
1890 // Can't fold it profitably if no one of casts has one use.
1891 if (!Cast0->hasOneUse() && !Cast1->hasOneUse())
1892 return nullptr;
1893
1894 Value *X, *Y;
1895 if (match(V: Cast0, P: m_ZExtOrSExt(Op: m_Value(V&: X))) &&
1896 match(V: Cast1, P: m_ZExtOrSExt(Op: m_Value(V&: Y)))) {
1897 // Cast the narrower source to the wider source type.
1898 unsigned XNumBits = X->getType()->getScalarSizeInBits();
1899 unsigned YNumBits = Y->getType()->getScalarSizeInBits();
1900 if (XNumBits != YNumBits) {
1901 // Cast the narrower source to the wider source type only if both of casts
1902 // have one use to avoid creating an extra instruction.
1903 if (!Cast0->hasOneUse() || !Cast1->hasOneUse())
1904 return nullptr;
1905
1906 // If the source types do not match, but the casts are matching extends,
1907 // we can still narrow the logic op.
1908 if (XNumBits < YNumBits) {
1909 X = Builder.CreateCast(Op: CastOpcode, V: X, DestTy: Y->getType());
1910 } else if (YNumBits < XNumBits) {
1911 Y = Builder.CreateCast(Op: CastOpcode, V: Y, DestTy: X->getType());
1912 }
1913 }
1914
1915 // Do the logic op in the intermediate width, then widen more.
1916 Value *NarrowLogic = Builder.CreateBinOp(Opc: LogicOpc, LHS: X, RHS: Y, Name: I.getName());
1917 auto *Disjoint = dyn_cast<PossiblyDisjointInst>(Val: &I);
1918 auto *NewDisjoint = dyn_cast<PossiblyDisjointInst>(Val: NarrowLogic);
1919 if (Disjoint && NewDisjoint)
1920 NewDisjoint->setIsDisjoint(Disjoint->isDisjoint());
1921 return CastInst::Create(CastOpcode, S: NarrowLogic, Ty: DestTy);
1922 }
1923
1924 // If the src type of casts are different, give up for other cast opcodes.
1925 if (SrcTy != Cast1->getSrcTy())
1926 return nullptr;
1927
1928 Value *Cast0Src = Cast0->getOperand(i_nocapture: 0);
1929 Value *Cast1Src = Cast1->getOperand(i_nocapture: 0);
1930
1931 // fold logic(cast(A), cast(B)) -> cast(logic(A, B))
1932 if (shouldOptimizeCast(CI: Cast0) && shouldOptimizeCast(CI: Cast1)) {
1933 Value *NewOp = Builder.CreateBinOp(Opc: LogicOpc, LHS: Cast0Src, RHS: Cast1Src,
1934 Name: I.getName());
1935 auto *NewCast = CastInst::Create(CastOpcode, S: NewOp, Ty: DestTy);
1936 if (auto *NewTrunc = dyn_cast<TruncInst>(Val: NewCast)) {
1937 auto *Trunc0 = cast<TruncInst>(Val: Cast0);
1938 auto *Trunc1 = cast<TruncInst>(Val: Cast1);
1939 NewTrunc->setHasNoUnsignedWrap(
1940 LogicOpc == Instruction::And
1941 ? Trunc0->hasNoUnsignedWrap() || Trunc1->hasNoUnsignedWrap()
1942 : Trunc0->hasNoUnsignedWrap() && Trunc1->hasNoUnsignedWrap());
1943 NewTrunc->setHasNoSignedWrap(Trunc0->hasNoSignedWrap() &&
1944 Trunc1->hasNoSignedWrap());
1945 }
1946 return NewCast;
1947 }
1948
1949 return nullptr;
1950}
1951
1952static Instruction *foldAndToXor(BinaryOperator &I,
1953 InstCombiner::BuilderTy &Builder) {
1954 assert(I.getOpcode() == Instruction::And);
1955 Value *Op0 = I.getOperand(i_nocapture: 0);
1956 Value *Op1 = I.getOperand(i_nocapture: 1);
1957 Value *A, *B;
1958
1959 // Operand complexity canonicalization guarantees that the 'or' is Op0.
1960 // (A | B) & ~(A & B) --> A ^ B
1961 // (A | B) & ~(B & A) --> A ^ B
1962 if (match(V: &I, P: m_BinOp(L: m_Or(L: m_Value(V&: A), R: m_Value(V&: B)),
1963 R: m_Not(V: m_c_And(L: m_Deferred(V: A), R: m_Deferred(V: B))))))
1964 return BinaryOperator::CreateXor(V1: A, V2: B);
1965
1966 // (A | ~B) & (~A | B) --> ~(A ^ B)
1967 // (A | ~B) & (B | ~A) --> ~(A ^ B)
1968 // (~B | A) & (~A | B) --> ~(A ^ B)
1969 // (~B | A) & (B | ~A) --> ~(A ^ B)
1970 if (Op0->hasOneUse() || Op1->hasOneUse())
1971 if (match(V: &I, P: m_BinOp(L: m_c_Or(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B))),
1972 R: m_c_Or(L: m_Not(V: m_Deferred(V: A)), R: m_Deferred(V: B)))))
1973 return BinaryOperator::CreateNot(Op: Builder.CreateXor(LHS: A, RHS: B));
1974
1975 return nullptr;
1976}
1977
1978static Instruction *foldOrToXor(BinaryOperator &I,
1979 InstCombiner::BuilderTy &Builder) {
1980 assert(I.getOpcode() == Instruction::Or);
1981 Value *Op0 = I.getOperand(i_nocapture: 0);
1982 Value *Op1 = I.getOperand(i_nocapture: 1);
1983 Value *A, *B;
1984
1985 // Operand complexity canonicalization guarantees that the 'and' is Op0.
1986 // (A & B) | ~(A | B) --> ~(A ^ B)
1987 // (A & B) | ~(B | A) --> ~(A ^ B)
1988 if (Op0->hasOneUse() || Op1->hasOneUse())
1989 if (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
1990 match(V: Op1, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B)))))
1991 return BinaryOperator::CreateNot(Op: Builder.CreateXor(LHS: A, RHS: B));
1992
1993 // Operand complexity canonicalization guarantees that the 'xor' is Op0.
1994 // (A ^ B) | ~(A | B) --> ~(A & B)
1995 // (A ^ B) | ~(B | A) --> ~(A & B)
1996 if (Op0->hasOneUse() || Op1->hasOneUse())
1997 if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))) &&
1998 match(V: Op1, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B)))))
1999 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: A, RHS: B));
2000
2001 // (A & ~B) | (~A & B) --> A ^ B
2002 // (A & ~B) | (B & ~A) --> A ^ B
2003 // (~B & A) | (~A & B) --> A ^ B
2004 // (~B & A) | (B & ~A) --> A ^ B
2005 if (match(V: Op0, P: m_c_And(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B)))) &&
2006 match(V: Op1, P: m_c_And(L: m_Not(V: m_Specific(V: A)), R: m_Specific(V: B))))
2007 return BinaryOperator::CreateXor(V1: A, V2: B);
2008
2009 return nullptr;
2010}
2011
2012/// Return true if a constant shift amount is always less than the specified
2013/// bit-width. If not, the shift could create poison in the narrower type.
2014static bool canNarrowShiftAmt(Constant *C, unsigned BitWidth) {
2015 APInt Threshold(C->getType()->getScalarSizeInBits(), BitWidth);
2016 return match(V: C, P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_ULT, Threshold));
2017}
2018
2019/// Try to use narrower ops (sink zext ops) for an 'and' with binop operand and
2020/// a common zext operand: and (binop (zext X), C), (zext X).
2021Instruction *InstCombinerImpl::narrowMaskedBinOp(BinaryOperator &And) {
2022 // This transform could also apply to {or, and, xor}, but there are better
2023 // folds for those cases, so we don't expect those patterns here. AShr is not
2024 // handled because it should always be transformed to LShr in this sequence.
2025 // The subtract transform is different because it has a constant on the left.
2026 // Add/mul commute the constant to RHS; sub with constant RHS becomes add.
2027 Value *Op0 = And.getOperand(i_nocapture: 0), *Op1 = And.getOperand(i_nocapture: 1);
2028 Constant *C;
2029 if (!match(V: Op0, P: m_OneUse(SubPattern: m_Add(L: m_Specific(V: Op1), R: m_Constant(C)))) &&
2030 !match(V: Op0, P: m_OneUse(SubPattern: m_Mul(L: m_Specific(V: Op1), R: m_Constant(C)))) &&
2031 !match(V: Op0, P: m_OneUse(SubPattern: m_LShr(L: m_Specific(V: Op1), R: m_Constant(C)))) &&
2032 !match(V: Op0, P: m_OneUse(SubPattern: m_Shl(L: m_Specific(V: Op1), R: m_Constant(C)))) &&
2033 !match(V: Op0, P: m_OneUse(SubPattern: m_Sub(L: m_Constant(C), R: m_Specific(V: Op1)))))
2034 return nullptr;
2035
2036 Value *X;
2037 if (!match(V: Op1, P: m_ZExt(Op: m_Value(V&: X))) || Op1->hasNUsesOrMore(N: 3))
2038 return nullptr;
2039
2040 Type *Ty = And.getType();
2041 if (!isa<VectorType>(Val: Ty) && !shouldChangeType(From: Ty, To: X->getType()))
2042 return nullptr;
2043
2044 // If we're narrowing a shift, the shift amount must be safe (less than the
2045 // width) in the narrower type. If the shift amount is greater, instsimplify
2046 // usually handles that case, but we can't guarantee/assert it.
2047 Instruction::BinaryOps Opc = cast<BinaryOperator>(Val: Op0)->getOpcode();
2048 if (Opc == Instruction::LShr || Opc == Instruction::Shl)
2049 if (!canNarrowShiftAmt(C, BitWidth: X->getType()->getScalarSizeInBits()))
2050 return nullptr;
2051
2052 // and (sub C, (zext X)), (zext X) --> zext (and (sub C', X), X)
2053 // and (binop (zext X), C), (zext X) --> zext (and (binop X, C'), X)
2054 Value *NewC = ConstantExpr::getTrunc(C, Ty: X->getType());
2055 Value *NewBO = Opc == Instruction::Sub ? Builder.CreateBinOp(Opc, LHS: NewC, RHS: X)
2056 : Builder.CreateBinOp(Opc, LHS: X, RHS: NewC);
2057 return new ZExtInst(Builder.CreateAnd(LHS: NewBO, RHS: X), Ty);
2058}
2059
2060/// Try folding relatively complex patterns for both And and Or operations
2061/// with all And and Or swapped.
2062static Instruction *foldComplexAndOrPatterns(BinaryOperator &I,
2063 InstCombiner::BuilderTy &Builder) {
2064 const Instruction::BinaryOps Opcode = I.getOpcode();
2065 assert(Opcode == Instruction::And || Opcode == Instruction::Or);
2066
2067 // Flip the logic operation.
2068 const Instruction::BinaryOps FlippedOpcode =
2069 (Opcode == Instruction::And) ? Instruction::Or : Instruction::And;
2070
2071 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
2072 Value *A, *B, *C, *X, *Y, *Dummy;
2073
2074 // Match following expressions:
2075 // (~(A | B) & C)
2076 // (~(A & B) | C)
2077 // Captures X = ~(A | B) or ~(A & B)
2078 const auto matchNotOrAnd =
2079 [Opcode, FlippedOpcode](Value *Op, auto m_A, auto m_B, auto m_C,
2080 Value *&X, bool CountUses = false) -> bool {
2081 if (CountUses && !Op->hasOneUse())
2082 return false;
2083
2084 if (match(Op,
2085 m_c_BinOp(FlippedOpcode,
2086 m_Value(X, m_Not(m_c_BinOp(Opcode, m_A, m_B))), m_C)))
2087 return !CountUses || X->hasOneUse();
2088
2089 return false;
2090 };
2091
2092 // (~(A | B) & C) | ... --> ...
2093 // (~(A & B) | C) & ... --> ...
2094 // TODO: One use checks are conservative. We just need to check that a total
2095 // number of multiple used values does not exceed reduction
2096 // in operations.
2097 if (matchNotOrAnd(Op0, m_Value(V&: A), m_Value(V&: B), m_Value(V&: C), X)) {
2098 // (~(A | B) & C) | (~(A | C) & B) --> (B ^ C) & ~A
2099 // (~(A & B) | C) & (~(A & C) | B) --> ~((B ^ C) & A)
2100 if (matchNotOrAnd(Op1, m_Specific(V: A), m_Specific(V: C), m_Specific(V: B), Dummy,
2101 true)) {
2102 Value *Xor = Builder.CreateXor(LHS: B, RHS: C);
2103 return (Opcode == Instruction::Or)
2104 ? BinaryOperator::CreateAnd(V1: Xor, V2: Builder.CreateNot(V: A))
2105 : BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: Xor, RHS: A));
2106 }
2107
2108 // (~(A | B) & C) | (~(B | C) & A) --> (A ^ C) & ~B
2109 // (~(A & B) | C) & (~(B & C) | A) --> ~((A ^ C) & B)
2110 if (matchNotOrAnd(Op1, m_Specific(V: B), m_Specific(V: C), m_Specific(V: A), Dummy,
2111 true)) {
2112 Value *Xor = Builder.CreateXor(LHS: A, RHS: C);
2113 return (Opcode == Instruction::Or)
2114 ? BinaryOperator::CreateAnd(V1: Xor, V2: Builder.CreateNot(V: B))
2115 : BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: Xor, RHS: B));
2116 }
2117
2118 // (~(A | B) & C) | ~(A | C) --> ~((B & C) | A)
2119 // (~(A & B) | C) & ~(A & C) --> ~((B | C) & A)
2120 if (match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_OneUse(
2121 SubPattern: m_c_BinOp(Opcode, L: m_Specific(V: A), R: m_Specific(V: C)))))))
2122 return BinaryOperator::CreateNot(Op: Builder.CreateBinOp(
2123 Opc: Opcode, LHS: Builder.CreateBinOp(Opc: FlippedOpcode, LHS: B, RHS: C), RHS: A));
2124
2125 // (~(A | B) & C) | ~(B | C) --> ~((A & C) | B)
2126 // (~(A & B) | C) & ~(B & C) --> ~((A | C) & B)
2127 if (match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_OneUse(
2128 SubPattern: m_c_BinOp(Opcode, L: m_Specific(V: B), R: m_Specific(V: C)))))))
2129 return BinaryOperator::CreateNot(Op: Builder.CreateBinOp(
2130 Opc: Opcode, LHS: Builder.CreateBinOp(Opc: FlippedOpcode, LHS: A, RHS: C), RHS: B));
2131
2132 // (~(A | B) & C) | ~(C | (A ^ B)) --> ~((A | B) & (C | (A ^ B)))
2133 // Note, the pattern with swapped and/or is not handled because the
2134 // result is more undefined than a source:
2135 // (~(A & B) | C) & ~(C & (A ^ B)) --> (A ^ B ^ C) | ~(A | C) is invalid.
2136 if (Opcode == Instruction::Or && Op0->hasOneUse() &&
2137 match(V: Op1,
2138 P: m_OneUse(SubPattern: m_Not(V: m_Value(
2139 V&: Y, P: m_c_BinOp(Opcode, L: m_Specific(V: C),
2140 R: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B)))))))) {
2141 // X = ~(A | B)
2142 // Y = (C | (A ^ B)
2143 Value *Or = cast<BinaryOperator>(Val: X)->getOperand(i_nocapture: 0);
2144 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: Or, RHS: Y));
2145 }
2146 }
2147
2148 // (~A & B & C) | ... --> ...
2149 // (~A | B | C) | ... --> ...
2150 // TODO: One use checks are conservative. We just need to check that a total
2151 // number of multiple used values does not exceed reduction
2152 // in operations.
2153 if (match(V: Op0,
2154 P: m_OneUse(SubPattern: m_c_BinOp(Opcode: FlippedOpcode,
2155 L: m_BinOp(Opcode: FlippedOpcode, L: m_Value(V&: B), R: m_Value(V&: C)),
2156 R: m_Value(V&: X, P: m_Not(V: m_Value(V&: A)))))) ||
2157 match(V: Op0, P: m_OneUse(SubPattern: m_c_BinOp(Opcode: FlippedOpcode,
2158 L: m_c_BinOp(Opcode: FlippedOpcode, L: m_Value(V&: C),
2159 R: m_Value(V&: X, P: m_Not(V: m_Value(V&: A)))),
2160 R: m_Value(V&: B))))) {
2161 // X = ~A
2162 // (~A & B & C) | ~(A | B | C) --> ~(A | (B ^ C))
2163 // (~A | B | C) & ~(A & B & C) --> (~A | (B ^ C))
2164 if (match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_c_BinOp(
2165 Opcode, L: m_c_BinOp(Opcode, L: m_Specific(V: A), R: m_Specific(V: B)),
2166 R: m_Specific(V: C))))) ||
2167 match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_c_BinOp(
2168 Opcode, L: m_c_BinOp(Opcode, L: m_Specific(V: B), R: m_Specific(V: C)),
2169 R: m_Specific(V: A))))) ||
2170 match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_c_BinOp(
2171 Opcode, L: m_c_BinOp(Opcode, L: m_Specific(V: A), R: m_Specific(V: C)),
2172 R: m_Specific(V: B)))))) {
2173 Value *Xor = Builder.CreateXor(LHS: B, RHS: C);
2174 return (Opcode == Instruction::Or)
2175 ? BinaryOperator::CreateNot(Op: Builder.CreateOr(LHS: Xor, RHS: A))
2176 : BinaryOperator::CreateOr(V1: Xor, V2: X);
2177 }
2178
2179 // (~A & B & C) | ~(A | B) --> (C | ~B) & ~A
2180 // (~A | B | C) & ~(A & B) --> (C & ~B) | ~A
2181 if (match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_OneUse(
2182 SubPattern: m_c_BinOp(Opcode, L: m_Specific(V: A), R: m_Specific(V: B)))))))
2183 return BinaryOperator::Create(
2184 Op: FlippedOpcode, S1: Builder.CreateBinOp(Opc: Opcode, LHS: C, RHS: Builder.CreateNot(V: B)),
2185 S2: X);
2186
2187 // (~A & B & C) | ~(A | C) --> (B | ~C) & ~A
2188 // (~A | B | C) & ~(A & C) --> (B & ~C) | ~A
2189 if (match(V: Op1, P: m_OneUse(SubPattern: m_Not(V: m_OneUse(
2190 SubPattern: m_c_BinOp(Opcode, L: m_Specific(V: A), R: m_Specific(V: C)))))))
2191 return BinaryOperator::Create(
2192 Op: FlippedOpcode, S1: Builder.CreateBinOp(Opc: Opcode, LHS: B, RHS: Builder.CreateNot(V: C)),
2193 S2: X);
2194 }
2195
2196 return nullptr;
2197}
2198
2199/// Try to reassociate a pair of binops so that values with one use only are
2200/// part of the same instruction. This may enable folds that are limited with
2201/// multi-use restrictions and makes it more likely to match other patterns that
2202/// are looking for a common operand.
2203static Instruction *reassociateForUses(BinaryOperator &BO,
2204 InstCombinerImpl::BuilderTy &Builder) {
2205 Instruction::BinaryOps Opcode = BO.getOpcode();
2206 Value *X, *Y, *Z;
2207 if (match(V: &BO,
2208 P: m_c_BinOp(Opcode, L: m_OneUse(SubPattern: m_BinOp(Opcode, L: m_Value(V&: X), R: m_Value(V&: Y))),
2209 R: m_OneUse(SubPattern: m_Value(V&: Z))))) {
2210 if (!isa<Constant>(Val: X) && !isa<Constant>(Val: Y) && !isa<Constant>(Val: Z)) {
2211 // (X op Y) op Z --> (Y op Z) op X
2212 if (!X->hasOneUse()) {
2213 Value *YZ = Builder.CreateBinOp(Opc: Opcode, LHS: Y, RHS: Z);
2214 return BinaryOperator::Create(Op: Opcode, S1: YZ, S2: X);
2215 }
2216 // (X op Y) op Z --> (X op Z) op Y
2217 if (!Y->hasOneUse()) {
2218 Value *XZ = Builder.CreateBinOp(Opc: Opcode, LHS: X, RHS: Z);
2219 return BinaryOperator::Create(Op: Opcode, S1: XZ, S2: Y);
2220 }
2221 }
2222 }
2223
2224 return nullptr;
2225}
2226
2227// Match
2228// (X + C2) | C
2229// (X + C2) ^ C
2230// (X + C2) & C
2231// and convert to do the bitwise logic first:
2232// (X | C) + C2
2233// (X ^ C) + C2
2234// (X & C) + C2
2235// iff bits affected by logic op are lower than last bit affected by math op
2236static Instruction *canonicalizeLogicFirst(BinaryOperator &I,
2237 InstCombiner::BuilderTy &Builder) {
2238 Type *Ty = I.getType();
2239 Instruction::BinaryOps OpC = I.getOpcode();
2240 Value *Op0 = I.getOperand(i_nocapture: 0);
2241 Value *Op1 = I.getOperand(i_nocapture: 1);
2242 Value *X;
2243 const APInt *C, *C2;
2244
2245 if (!(match(V: Op0, P: m_OneUse(SubPattern: m_Add(L: m_Value(V&: X), R: m_APInt(Res&: C2)))) &&
2246 match(V: Op1, P: m_APInt(Res&: C))))
2247 return nullptr;
2248
2249 unsigned Width = Ty->getScalarSizeInBits();
2250 unsigned LastOneMath = Width - C2->countr_zero();
2251
2252 switch (OpC) {
2253 case Instruction::And:
2254 if (C->countl_one() < LastOneMath)
2255 return nullptr;
2256 break;
2257 case Instruction::Xor:
2258 case Instruction::Or:
2259 if (C->countl_zero() < LastOneMath)
2260 return nullptr;
2261 break;
2262 default:
2263 llvm_unreachable("Unexpected BinaryOp!");
2264 }
2265
2266 Value *NewBinOp = Builder.CreateBinOp(Opc: OpC, LHS: X, RHS: ConstantInt::get(Ty, V: *C));
2267 return BinaryOperator::CreateWithCopiedFlags(Opc: Instruction::Add, V1: NewBinOp,
2268 V2: ConstantInt::get(Ty, V: *C2), CopyO: Op0);
2269}
2270
2271// binop(shift(ShiftedC1, ShAmt), shift(ShiftedC2, add(ShAmt, AddC))) ->
2272// shift(binop(ShiftedC1, shift(ShiftedC2, AddC)), ShAmt)
2273// where both shifts are the same and AddC is a valid shift amount.
2274Instruction *InstCombinerImpl::foldBinOpOfDisplacedShifts(BinaryOperator &I) {
2275 assert((I.isBitwiseLogicOp() || I.getOpcode() == Instruction::Add) &&
2276 "Unexpected opcode");
2277
2278 Value *ShAmt;
2279 Constant *ShiftedC1, *ShiftedC2, *AddC;
2280 Type *Ty = I.getType();
2281 unsigned BitWidth = Ty->getScalarSizeInBits();
2282 if (!match(V: &I, P: m_c_BinOp(L: m_Shift(L: m_ImmConstant(C&: ShiftedC1), R: m_Value(V&: ShAmt)),
2283 R: m_Shift(L: m_ImmConstant(C&: ShiftedC2),
2284 R: m_AddLike(L: m_Deferred(V: ShAmt),
2285 R: m_ImmConstant(C&: AddC))))))
2286 return nullptr;
2287
2288 // Make sure the add constant is a valid shift amount.
2289 if (!match(V: AddC,
2290 P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_ULT, Threshold: APInt(BitWidth, BitWidth))))
2291 return nullptr;
2292
2293 // Avoid constant expressions.
2294 auto *Op0Inst = dyn_cast<Instruction>(Val: I.getOperand(i_nocapture: 0));
2295 auto *Op1Inst = dyn_cast<Instruction>(Val: I.getOperand(i_nocapture: 1));
2296 if (!Op0Inst || !Op1Inst)
2297 return nullptr;
2298
2299 // Both shifts must be the same.
2300 Instruction::BinaryOps ShiftOp =
2301 static_cast<Instruction::BinaryOps>(Op0Inst->getOpcode());
2302 if (ShiftOp != Op1Inst->getOpcode())
2303 return nullptr;
2304
2305 // For adds, only left shifts are supported.
2306 if (I.getOpcode() == Instruction::Add && ShiftOp != Instruction::Shl)
2307 return nullptr;
2308
2309 Value *NewC = Builder.CreateBinOp(
2310 Opc: I.getOpcode(), LHS: ShiftedC1, RHS: Builder.CreateBinOp(Opc: ShiftOp, LHS: ShiftedC2, RHS: AddC));
2311 return BinaryOperator::Create(Op: ShiftOp, S1: NewC, S2: ShAmt);
2312}
2313
2314// Fold and/or/xor with two equal intrinsic IDs:
2315// bitwise(fshl (A, B, ShAmt), fshl(C, D, ShAmt))
2316// -> fshl(bitwise(A, C), bitwise(B, D), ShAmt)
2317// bitwise(fshr (A, B, ShAmt), fshr(C, D, ShAmt))
2318// -> fshr(bitwise(A, C), bitwise(B, D), ShAmt)
2319// bitwise(bswap(A), bswap(B)) -> bswap(bitwise(A, B))
2320// bitwise(bswap(A), C) -> bswap(bitwise(A, bswap(C)))
2321// bitwise(bitreverse(A), bitreverse(B)) -> bitreverse(bitwise(A, B))
2322// bitwise(bitreverse(A), C) -> bitreverse(bitwise(A, bitreverse(C)))
2323static Instruction *
2324foldBitwiseLogicWithIntrinsics(BinaryOperator &I,
2325 InstCombiner::BuilderTy &Builder) {
2326 assert(I.isBitwiseLogicOp() && "Should and/or/xor");
2327 if (!I.getOperand(i_nocapture: 0)->hasOneUse())
2328 return nullptr;
2329 IntrinsicInst *X = dyn_cast<IntrinsicInst>(Val: I.getOperand(i_nocapture: 0));
2330 if (!X)
2331 return nullptr;
2332
2333 IntrinsicInst *Y = dyn_cast<IntrinsicInst>(Val: I.getOperand(i_nocapture: 1));
2334 if (Y && (!Y->hasOneUse() || X->getIntrinsicID() != Y->getIntrinsicID()))
2335 return nullptr;
2336
2337 Intrinsic::ID IID = X->getIntrinsicID();
2338 const APInt *RHSC;
2339 // Try to match constant RHS.
2340 if (!Y && (!(IID == Intrinsic::bswap || IID == Intrinsic::bitreverse) ||
2341 !match(V: I.getOperand(i_nocapture: 1), P: m_APInt(Res&: RHSC))))
2342 return nullptr;
2343
2344 switch (IID) {
2345 case Intrinsic::fshl:
2346 case Intrinsic::fshr: {
2347 if (X->getOperand(i_nocapture: 2) != Y->getOperand(i_nocapture: 2))
2348 return nullptr;
2349 Value *NewOp0 =
2350 Builder.CreateBinOp(Opc: I.getOpcode(), LHS: X->getOperand(i_nocapture: 0), RHS: Y->getOperand(i_nocapture: 0));
2351 Value *NewOp1 =
2352 Builder.CreateBinOp(Opc: I.getOpcode(), LHS: X->getOperand(i_nocapture: 1), RHS: Y->getOperand(i_nocapture: 1));
2353 Function *F =
2354 Intrinsic::getOrInsertDeclaration(M: I.getModule(), id: IID, OverloadTys: I.getType());
2355 return CallInst::Create(Func: F, Args: {NewOp0, NewOp1, X->getOperand(i_nocapture: 2)});
2356 }
2357 case Intrinsic::bswap:
2358 case Intrinsic::bitreverse: {
2359 Value *NewOp0 = Builder.CreateBinOp(
2360 Opc: I.getOpcode(), LHS: X->getOperand(i_nocapture: 0),
2361 RHS: Y ? Y->getOperand(i_nocapture: 0)
2362 : ConstantInt::get(Ty: I.getType(), V: IID == Intrinsic::bswap
2363 ? RHSC->byteSwap()
2364 : RHSC->reverseBits()));
2365 Function *F =
2366 Intrinsic::getOrInsertDeclaration(M: I.getModule(), id: IID, OverloadTys: I.getType());
2367 return CallInst::Create(Func: F, Args: {NewOp0});
2368 }
2369 default:
2370 return nullptr;
2371 }
2372}
2373
2374// Try to simplify V by replacing occurrences of Op with RepOp, but only look
2375// through bitwise operations. In particular, for X | Y we try to replace Y with
2376// 0 inside X and for X & Y we try to replace Y with -1 inside X.
2377// Return the simplified result of X if successful, and nullptr otherwise.
2378// If SimplifyOnly is true, no new instructions will be created.
2379static Value *simplifyAndOrWithOpReplaced(Value *V, Value *Op, Value *RepOp,
2380 bool SimplifyOnly,
2381 InstCombinerImpl &IC,
2382 unsigned Depth = 0) {
2383 if (Op == RepOp)
2384 return nullptr;
2385
2386 if (V == Op)
2387 return RepOp;
2388
2389 auto *I = dyn_cast<BinaryOperator>(Val: V);
2390 if (!I || !I->isBitwiseLogicOp() || Depth >= 3)
2391 return nullptr;
2392
2393 if (!I->hasOneUse())
2394 SimplifyOnly = true;
2395
2396 Value *NewOp0 = simplifyAndOrWithOpReplaced(V: I->getOperand(i_nocapture: 0), Op, RepOp,
2397 SimplifyOnly, IC, Depth: Depth + 1);
2398 Value *NewOp1 = simplifyAndOrWithOpReplaced(V: I->getOperand(i_nocapture: 1), Op, RepOp,
2399 SimplifyOnly, IC, Depth: Depth + 1);
2400 if (!NewOp0 && !NewOp1)
2401 return nullptr;
2402
2403 if (!NewOp0)
2404 NewOp0 = I->getOperand(i_nocapture: 0);
2405 if (!NewOp1)
2406 NewOp1 = I->getOperand(i_nocapture: 1);
2407
2408 if (Value *Res = simplifyBinOp(Opcode: I->getOpcode(), LHS: NewOp0, RHS: NewOp1,
2409 Q: IC.getSimplifyQuery().getWithInstruction(I)))
2410 return Res;
2411
2412 if (SimplifyOnly)
2413 return nullptr;
2414 return IC.Builder.CreateBinOp(Opc: I->getOpcode(), LHS: NewOp0, RHS: NewOp1);
2415}
2416
2417/// The pattern div_ceil(X, P) * P, where P is a power of 2, lowers to the
2418/// following conditional round-up: (X + select(C, 0, Pow2)) & -Pow2, where
2419/// C is X % Pow2 == 0. This may be simplified to (X + (Pow2-1)) & -Pow2.
2420static Instruction *
2421foldRoundUpToPow2Alignment(BinaryOperator &I,
2422 InstCombiner::BuilderTy &Builder) {
2423 const APInt *NegP;
2424 Value *Add;
2425 if (!match(V: &I, P: m_And(L: m_Value(V&: Add), R: m_NegatedPower2(V&: NegP))))
2426 return nullptr;
2427
2428 Value *X, *Cond;
2429 APInt Mask = ~*NegP;
2430
2431 // Match the pattern. Ensure the true arm of the select is zero, and the false
2432 // one is the Pow2.
2433 if (!match(V: Add,
2434 P: m_OneUse(SubPattern: m_c_Add(L: m_Value(V&: X), R: m_Select(C: m_Value(V&: Cond), L: m_ZeroInt(),
2435 R: m_SpecificInt(V: -*NegP))))))
2436 return nullptr;
2437
2438 // icmp ne should have already been canonicalized to the eq form for this
2439 // pattern.
2440 if (!match(V: Cond, P: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
2441 L: m_And(L: m_Specific(V: X), R: m_SpecificInt(V: Mask)),
2442 R: m_Zero())))
2443 return nullptr;
2444
2445 Type *Ty = I.getType();
2446 Value *NewAdd = Builder.CreateAdd(LHS: X, RHS: ConstantInt::get(Ty, V: Mask));
2447 return BinaryOperator::CreateAnd(V1: NewAdd, V2: ConstantInt::get(Ty, V: *NegP));
2448}
2449
2450/// Reassociate and/or expressions to see if we can fold the inner and/or ops.
2451/// TODO: Make this recursive; it's a little tricky because an arbitrary
2452/// number of and/or instructions might have to be created.
2453Value *InstCombinerImpl::reassociateBooleanAndOr(Value *LHS, Value *X, Value *Y,
2454 Instruction &I, bool IsAnd,
2455 bool RHSIsLogical) {
2456 Instruction::BinaryOps Opcode = IsAnd ? Instruction::And : Instruction::Or;
2457 Value *Folded = nullptr;
2458 // LHS bop (X lop Y) --> (LHS bop X) lop Y
2459 // LHS bop (X bop Y) --> (LHS bop X) bop Y
2460 if (Value *Res = foldBooleanAndOr(LHS, RHS: X, I, IsAnd, /*IsLogical=*/false))
2461 Folded = RHSIsLogical ? Builder.CreateLogicalOp(Opc: Opcode, Cond1: Res, Cond2: Y)
2462 : Builder.CreateBinOp(Opc: Opcode, LHS: Res, RHS: Y);
2463 // LHS bop (X bop Y) --> X bop (LHS bop Y)
2464 // LHS bop (X lop Y) --> X lop (LHS bop Y)
2465 else if (Value *Res = foldBooleanAndOr(LHS, RHS: Y, I, IsAnd, /*IsLogical=*/false))
2466 Folded = RHSIsLogical ? Builder.CreateLogicalOp(Opc: Opcode, Cond1: X, Cond2: Res)
2467 : Builder.CreateBinOp(Opc: Opcode, LHS: X, RHS: Res);
2468 if (SelectInst *SI = dyn_cast_or_null<SelectInst>(Val: Folded); SI != nullptr)
2469 // If the bop I was originally a lop, we could recover branch weight
2470 // information using that lop's weights. However, InstCombine usually
2471 // replaces the lop with a bop by the time we get here, deleting the branch
2472 // weight information. Therefore, we can only assume unknown branch weights.
2473 // TODO: see if it's possible to recover branch weight information from the
2474 // original lop (https://github.com/llvm/llvm-project/issues/183864).
2475 setExplicitlyUnknownBranchWeightsIfProfiled(I&: *SI, DEBUG_TYPE,
2476 F: I.getFunction());
2477 return Folded;
2478}
2479
2480// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
2481// here. We should standardize that construct where it is needed or choose some
2482// other way to ensure that commutated variants of patterns are not missed.
2483Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
2484 Type *Ty = I.getType();
2485
2486 if (Value *V = simplifyAndInst(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1),
2487 Q: SQ.getWithInstruction(I: &I)))
2488 return replaceInstUsesWith(I, V);
2489
2490 if (SimplifyAssociativeOrCommutative(I))
2491 return &I;
2492
2493 if (Instruction *X = foldVectorBinop(Inst&: I))
2494 return X;
2495
2496 if (Instruction *Phi = foldBinopWithPhiOperands(BO&: I))
2497 return Phi;
2498
2499 // See if we can simplify any instructions used by the instruction whose sole
2500 // purpose is to compute bits we don't care about.
2501 if (SimplifyDemandedInstructionBits(Inst&: I))
2502 return &I;
2503
2504 // Do this before using distributive laws to catch simple and/or/not patterns.
2505 if (Instruction *Xor = foldAndToXor(I, Builder))
2506 return Xor;
2507
2508 if (Instruction *X = foldComplexAndOrPatterns(I, Builder))
2509 return X;
2510
2511 // (A|B)&(A|C) -> A|(B&C) etc
2512 if (Value *V = foldUsingDistributiveLaws(I))
2513 return replaceInstUsesWith(I, V);
2514
2515 if (Instruction *R = foldBinOpShiftWithShift(I))
2516 return R;
2517
2518 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
2519
2520 Value *X, *Y;
2521 const APInt *C;
2522 if ((match(V: Op0, P: m_OneUse(SubPattern: m_LogicalShift(L: m_One(), R: m_Value(V&: X)))) ||
2523 (match(V: Op0, P: m_OneUse(SubPattern: m_Shl(L: m_APInt(Res&: C), R: m_Value(V&: X)))) && (*C)[0])) &&
2524 match(V: Op1, P: m_One())) {
2525 // (1 >> X) & 1 --> zext(X == 0)
2526 // (C << X) & 1 --> zext(X == 0), when C is odd
2527 Value *IsZero = Builder.CreateICmpEQ(LHS: X, RHS: ConstantInt::get(Ty, V: 0));
2528 return new ZExtInst(IsZero, Ty);
2529 }
2530
2531 // (-(X & 1)) & Y --> (X & 1) == 0 ? 0 : Y
2532 Value *Neg;
2533 if (match(V: &I,
2534 P: m_c_And(L: m_Value(V&: Neg, P: m_OneUse(SubPattern: m_Neg(V: m_And(L: m_Value(), R: m_One())))),
2535 R: m_Value(V&: Y)))) {
2536 Value *Cmp = Builder.CreateIsNull(Arg: Neg);
2537 return createSelectInstWithUnknownProfile(C: Cmp,
2538 S1: ConstantInt::getNullValue(Ty), S2: Y);
2539 }
2540
2541 // Canonicalize:
2542 // (X +/- Y) & Y --> ~X & Y when Y is a power of 2.
2543 if (match(V: &I, P: m_c_And(L: m_Value(V&: Y), R: m_OneUse(SubPattern: m_CombineOr(
2544 Ps: m_c_Add(L: m_Value(V&: X), R: m_Deferred(V: Y)),
2545 Ps: m_Sub(L: m_Value(V&: X), R: m_Deferred(V: Y)))))) &&
2546 isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ true, CtxI: &I))
2547 return BinaryOperator::CreateAnd(V1: Builder.CreateNot(V: X), V2: Y);
2548
2549 if (match(V: Op1, P: m_APInt(Res&: C))) {
2550 const APInt *XorC;
2551 if (match(V: Op0, P: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: X), R: m_APInt(Res&: XorC))))) {
2552 // (X ^ C1) & C2 --> (X & C2) ^ (C1&C2)
2553 Constant *NewC = ConstantInt::get(Ty, V: *C & *XorC);
2554 Value *And = Builder.CreateAnd(LHS: X, RHS: Op1);
2555 And->takeName(V: Op0);
2556 return BinaryOperator::CreateXor(V1: And, V2: NewC);
2557 }
2558
2559 const APInt *OrC;
2560 if (match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: X), R: m_APInt(Res&: OrC))))) {
2561 // (X | C1) & C2 --> (X & C2^(C1&C2)) | (C1&C2)
2562 // NOTE: This reduces the number of bits set in the & mask, which
2563 // can expose opportunities for store narrowing for scalars.
2564 // NOTE: SimplifyDemandedBits should have already removed bits from C1
2565 // that aren't set in C2. Meaning we can replace (C1&C2) with C1 in
2566 // above, but this feels safer.
2567 APInt Together = *C & *OrC;
2568 Value *And = Builder.CreateAnd(LHS: X, RHS: ConstantInt::get(Ty, V: Together ^ *C));
2569 And->takeName(V: Op0);
2570 return BinaryOperator::CreateOr(V1: And, V2: ConstantInt::get(Ty, V: Together));
2571 }
2572
2573 unsigned Width = Ty->getScalarSizeInBits();
2574 const APInt *ShiftC;
2575 if (match(V: Op0, P: m_OneUse(SubPattern: m_SExt(Op: m_AShr(L: m_Value(V&: X), R: m_APInt(Res&: ShiftC))))) &&
2576 ShiftC->ult(RHS: Width)) {
2577 if (*C == APInt::getLowBitsSet(numBits: Width, loBitsSet: Width - ShiftC->getZExtValue())) {
2578 // We are clearing high bits that were potentially set by sext+ashr:
2579 // and (sext (ashr X, ShiftC)), C --> lshr (sext X), ShiftC
2580 Value *Sext = Builder.CreateSExt(V: X, DestTy: Ty);
2581 Constant *ShAmtC = ConstantInt::get(Ty, V: ShiftC->zext(width: Width));
2582 return BinaryOperator::CreateLShr(V1: Sext, V2: ShAmtC);
2583 }
2584 }
2585
2586 // If this 'and' clears the sign-bits added by ashr, replace with lshr:
2587 // and (ashr X, ShiftC), C --> lshr X, ShiftC
2588 if (match(V: Op0, P: m_AShr(L: m_Value(V&: X), R: m_APInt(Res&: ShiftC))) && ShiftC->ult(RHS: Width) &&
2589 C->isMask(numBits: Width - ShiftC->getZExtValue()))
2590 return BinaryOperator::CreateLShr(V1: X, V2: ConstantInt::get(Ty, V: *ShiftC));
2591
2592 const APInt *AddC;
2593 if (match(V: Op0, P: m_Add(L: m_Value(V&: X), R: m_APInt(Res&: AddC)))) {
2594 // If we are masking the result of the add down to exactly one bit and
2595 // the constant we are adding has no bits set below that bit, then the
2596 // add is flipping a single bit. Example:
2597 // (X + 4) & 4 --> (X & 4) ^ 4
2598 if (Op0->hasOneUse() && C->isPowerOf2() && (*AddC & (*C - 1)) == 0) {
2599 assert((*C & *AddC) != 0 && "Expected common bit");
2600 Value *NewAnd = Builder.CreateAnd(LHS: X, RHS: Op1);
2601 return BinaryOperator::CreateXor(V1: NewAnd, V2: Op1);
2602 }
2603 }
2604
2605 // ((C1 OP zext(X)) & C2) -> zext((C1 OP X) & C2) if C2 fits in the
2606 // bitwidth of X and OP behaves well when given trunc(C1) and X.
2607 auto isNarrowableBinOpcode = [](BinaryOperator *B) {
2608 switch (B->getOpcode()) {
2609 case Instruction::Xor:
2610 case Instruction::Or:
2611 case Instruction::Mul:
2612 case Instruction::Add:
2613 case Instruction::Sub:
2614 return true;
2615 default:
2616 return false;
2617 }
2618 };
2619 BinaryOperator *BO;
2620 if (match(V: Op0, P: m_OneUse(SubPattern: m_BinOp(I&: BO))) && isNarrowableBinOpcode(BO)) {
2621 Instruction::BinaryOps BOpcode = BO->getOpcode();
2622 Value *X;
2623 const APInt *C1;
2624 // TODO: The one-use restrictions could be relaxed a little if the AND
2625 // is going to be removed.
2626 // Try to narrow the 'and' and a binop with constant operand:
2627 // and (bo (zext X), C1), C --> zext (and (bo X, TruncC1), TruncC)
2628 if (match(V: BO, P: m_c_BinOp(L: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X))), R: m_APInt(Res&: C1))) &&
2629 C->isIntN(N: X->getType()->getScalarSizeInBits())) {
2630 unsigned XWidth = X->getType()->getScalarSizeInBits();
2631 Constant *TruncC1 = ConstantInt::get(Ty: X->getType(), V: C1->trunc(width: XWidth));
2632 Value *BinOp = isa<ZExtInst>(Val: BO->getOperand(i_nocapture: 0))
2633 ? Builder.CreateBinOp(Opc: BOpcode, LHS: X, RHS: TruncC1)
2634 : Builder.CreateBinOp(Opc: BOpcode, LHS: TruncC1, RHS: X);
2635 Constant *TruncC = ConstantInt::get(Ty: X->getType(), V: C->trunc(width: XWidth));
2636 Value *And = Builder.CreateAnd(LHS: BinOp, RHS: TruncC);
2637 return new ZExtInst(And, Ty);
2638 }
2639
2640 // Similar to above: if the mask matches the zext input width, then the
2641 // 'and' can be eliminated, so we can truncate the other variable op:
2642 // and (bo (zext X), Y), C --> zext (bo X, (trunc Y))
2643 if (isa<Instruction>(Val: BO->getOperand(i_nocapture: 0)) &&
2644 match(V: BO->getOperand(i_nocapture: 0), P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X)))) &&
2645 C->isMask(numBits: X->getType()->getScalarSizeInBits())) {
2646 Y = BO->getOperand(i_nocapture: 1);
2647 Value *TrY = Builder.CreateTrunc(V: Y, DestTy: X->getType(), Name: Y->getName() + ".tr");
2648 Value *NewBO =
2649 Builder.CreateBinOp(Opc: BOpcode, LHS: X, RHS: TrY, Name: BO->getName() + ".narrow");
2650 return new ZExtInst(NewBO, Ty);
2651 }
2652 // and (bo Y, (zext X)), C --> zext (bo (trunc Y), X)
2653 if (isa<Instruction>(Val: BO->getOperand(i_nocapture: 1)) &&
2654 match(V: BO->getOperand(i_nocapture: 1), P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X)))) &&
2655 C->isMask(numBits: X->getType()->getScalarSizeInBits())) {
2656 Y = BO->getOperand(i_nocapture: 0);
2657 Value *TrY = Builder.CreateTrunc(V: Y, DestTy: X->getType(), Name: Y->getName() + ".tr");
2658 Value *NewBO =
2659 Builder.CreateBinOp(Opc: BOpcode, LHS: TrY, RHS: X, Name: BO->getName() + ".narrow");
2660 return new ZExtInst(NewBO, Ty);
2661 }
2662 }
2663
2664 // This is intentionally placed after the narrowing transforms for
2665 // efficiency (transform directly to the narrow logic op if possible).
2666 // If the mask is only needed on one incoming arm, push the 'and' op up.
2667 if (match(V: Op0, P: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: X), R: m_Value(V&: Y)))) ||
2668 match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
2669 APInt NotAndMask(~(*C));
2670 BinaryOperator::BinaryOps BinOp = cast<BinaryOperator>(Val: Op0)->getOpcode();
2671 if (MaskedValueIsZero(V: X, Mask: NotAndMask, CtxI: &I)) {
2672 // Not masking anything out for the LHS, move mask to RHS.
2673 // and ({x}or X, Y), C --> {x}or X, (and Y, C)
2674 Value *NewRHS = Builder.CreateAnd(LHS: Y, RHS: Op1, Name: Y->getName() + ".masked");
2675 return BinaryOperator::Create(Op: BinOp, S1: X, S2: NewRHS);
2676 }
2677 if (!isa<Constant>(Val: Y) && MaskedValueIsZero(V: Y, Mask: NotAndMask, CtxI: &I)) {
2678 // Not masking anything out for the RHS, move mask to LHS.
2679 // and ({x}or X, Y), C --> {x}or (and X, C), Y
2680 Value *NewLHS = Builder.CreateAnd(LHS: X, RHS: Op1, Name: X->getName() + ".masked");
2681 return BinaryOperator::Create(Op: BinOp, S1: NewLHS, S2: Y);
2682 }
2683 }
2684
2685 // When the mask is a power-of-2 constant and op0 is a shifted-power-of-2
2686 // constant, test if the shift amount equals the offset bit index:
2687 // (ShiftC << X) & C --> X == (log2(C) - log2(ShiftC)) ? C : 0
2688 // (ShiftC >> X) & C --> X == (log2(ShiftC) - log2(C)) ? C : 0
2689 if (C->isPowerOf2() &&
2690 match(V: Op0, P: m_OneUse(SubPattern: m_LogicalShift(L: m_Power2(V&: ShiftC), R: m_Value(V&: X))))) {
2691 int Log2ShiftC = ShiftC->exactLogBase2();
2692 int Log2C = C->exactLogBase2();
2693 bool IsShiftLeft =
2694 cast<BinaryOperator>(Val: Op0)->getOpcode() == Instruction::Shl;
2695 int BitNum = IsShiftLeft ? Log2C - Log2ShiftC : Log2ShiftC - Log2C;
2696 assert(BitNum >= 0 && "Expected demanded bits to handle impossible mask");
2697 Value *Cmp = Builder.CreateICmpEQ(LHS: X, RHS: ConstantInt::get(Ty, V: BitNum));
2698 return createSelectInstWithUnknownProfile(C: Cmp, S1: ConstantInt::get(Ty, V: *C),
2699 S2: ConstantInt::getNullValue(Ty));
2700 }
2701
2702 Constant *C1, *C2;
2703 const APInt *C3 = C;
2704 Value *X;
2705 if (C3->isPowerOf2()) {
2706 Constant *Log2C3 = ConstantInt::get(Ty, V: C3->countr_zero());
2707 if (match(V: Op0, P: m_OneUse(SubPattern: m_LShr(L: m_Shl(L: m_ImmConstant(C&: C1), R: m_Value(V&: X)),
2708 R: m_ImmConstant(C&: C2)))) &&
2709 match(V: C1, P: m_Power2())) {
2710 Constant *Log2C1 = ConstantExpr::getExactLogBase2(C: C1);
2711 Constant *LshrC = ConstantExpr::getAdd(C1: C2, C2: Log2C3);
2712 KnownBits KnownLShrc = computeKnownBits(V: LshrC, CtxI: nullptr);
2713 if (KnownLShrc.getMaxValue().ult(RHS: Width)) {
2714 // iff C1,C3 is pow2 and C2 + cttz(C3) < BitWidth:
2715 // ((C1 << X) >> C2) & C3 -> X == (cttz(C3)+C2-cttz(C1)) ? C3 : 0
2716 Constant *CmpC = ConstantExpr::getSub(C1: LshrC, C2: Log2C1);
2717 Value *Cmp = Builder.CreateICmpEQ(LHS: X, RHS: CmpC);
2718 return createSelectInstWithUnknownProfile(
2719 C: Cmp, S1: ConstantInt::get(Ty, V: *C3), S2: ConstantInt::getNullValue(Ty));
2720 }
2721 }
2722
2723 if (match(V: Op0, P: m_OneUse(SubPattern: m_Shl(L: m_LShr(L: m_ImmConstant(C&: C1), R: m_Value(V&: X)),
2724 R: m_ImmConstant(C&: C2)))) &&
2725 match(V: C1, P: m_Power2())) {
2726 Constant *Log2C1 = ConstantExpr::getExactLogBase2(C: C1);
2727 Constant *Cmp =
2728 ConstantFoldCompareInstOperands(Predicate: ICmpInst::ICMP_ULT, LHS: Log2C3, RHS: C2, DL);
2729 if (Cmp && Cmp->isNullValue()) {
2730 // iff C1,C3 is pow2 and Log2(C3) >= C2:
2731 // ((C1 >> X) << C2) & C3 -> X == (cttz(C1)+C2-cttz(C3)) ? C3 : 0
2732 Constant *ShlC = ConstantExpr::getAdd(C1: C2, C2: Log2C1);
2733 Constant *CmpC = ConstantExpr::getSub(C1: ShlC, C2: Log2C3);
2734 Value *Cmp = Builder.CreateICmpEQ(LHS: X, RHS: CmpC);
2735 return createSelectInstWithUnknownProfile(
2736 C: Cmp, S1: ConstantInt::get(Ty, V: *C3), S2: ConstantInt::getNullValue(Ty));
2737 }
2738 }
2739 }
2740
2741 // and (or (shr (sub C, X), ShAmt), X), C --> umin(X, C)
2742 // The fold holds when all of:
2743 // - C is a low-bit mask: C = (1<<S)-1
2744 // - S u<= ShAmt -- When X u<= C, sub(C, X) u< 1<<ShAmt, so shr is 0
2745 // - ShAmt + S u<= BW -- the borrow's all-ones run reaches up to bit S-1
2746 // - X u< 1<<ShAmt -- bounds X - C u< 1<<ShAmt in the X u> C case
2747 // Then shr(sub(C, X), ShAmt) is 0 when X u<= C and has all of C's bits set
2748 // when X u> C, so (shr | X) & C clamps X to C. For ashr, the bound on X
2749 // also means sub(C, X) is negative when X u> C, so shr is all-ones and
2750 // ShAmt + S u<= BW is not needed.
2751 const APInt *ShAmtC;
2752 Instruction *Shr;
2753 if (C->isMask() &&
2754 match(V: Op0, P: m_OneUse(SubPattern: m_c_Or(
2755 L: m_CombineAnd(Ps: m_Instruction(I&: Shr),
2756 Ps: m_Shr(L: m_Sub(L: m_SpecificInt(V: *C), R: m_Value(V&: X)),
2757 R: m_APInt(Res&: ShAmtC))),
2758 R: m_Deferred(V: X)))) &&
2759 ShAmtC->ult(RHS: Width)) {
2760 unsigned S = C->popcount();
2761 unsigned ShAmt = ShAmtC->getZExtValue();
2762 bool IsAShr = Shr->getOpcode() == Instruction::AShr;
2763 if (S <= ShAmt && (IsAShr || ShAmt + S <= Width) &&
2764 MaskedValueIsZero(V: X, Mask: APInt::getBitsSetFrom(numBits: Width, loBit: ShAmt), CtxI: &I))
2765 return replaceInstUsesWith(
2766 I, V: Builder.CreateBinaryIntrinsic(ID: Intrinsic::umin, LHS: X, RHS: Op1));
2767 }
2768 }
2769
2770 // If we are clearing the sign bit of a floating-point value, convert this to
2771 // fabs, then cast back to integer.
2772 //
2773 // This is a generous interpretation for noimplicitfloat, this is not a true
2774 // floating-point operation.
2775 //
2776 // Assumes any IEEE-represented type has the sign bit in the high bit.
2777 // TODO: Unify with APInt matcher. This version allows undef unlike m_APInt
2778 Value *CastOp;
2779 if (match(V: Op0, P: m_ElementWiseBitCast(Op: m_Value(V&: CastOp))) &&
2780 match(V: Op1, P: m_MaxSignedValue()) &&
2781 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
2782 Kind: Attribute::NoImplicitFloat)) {
2783 Type *EltTy = CastOp->getType()->getScalarType();
2784 if (EltTy->isFloatingPointTy() &&
2785 APFloat::hasSignBitInMSB(EltTy->getFltSemantics())) {
2786 Value *FAbs = Builder.CreateFAbs(V: CastOp);
2787 return new BitCastInst(FAbs, I.getType());
2788 }
2789 }
2790
2791 // and(shl(zext(X), Y), SignMask) -> and(sext(X), SignMask)
2792 // where Y is a valid shift amount.
2793 if (match(V: &I, P: m_And(L: m_OneUse(SubPattern: m_Shl(L: m_ZExt(Op: m_Value(V&: X)), R: m_Value(V&: Y))),
2794 R: m_SignMask())) &&
2795 match(V: Y, P: m_SpecificInt_ICMP(
2796 Predicate: ICmpInst::Predicate::ICMP_EQ,
2797 Threshold: APInt(Ty->getScalarSizeInBits(),
2798 Ty->getScalarSizeInBits() -
2799 X->getType()->getScalarSizeInBits())))) {
2800 auto *SExt = Builder.CreateSExt(V: X, DestTy: Ty, Name: X->getName() + ".signext");
2801 return BinaryOperator::CreateAnd(V1: SExt, V2: Op1);
2802 }
2803
2804 if (Instruction *Z = narrowMaskedBinOp(And&: I))
2805 return Z;
2806
2807 if (I.getType()->isIntOrIntVectorTy(BitWidth: 1)) {
2808 if (auto *SI0 = dyn_cast<SelectInst>(Val: Op0)) {
2809 if (auto *R =
2810 foldAndOrOfSelectUsingImpliedCond(Op: Op1, SI&: *SI0, /* IsAnd */ true))
2811 return R;
2812 }
2813 if (auto *SI1 = dyn_cast<SelectInst>(Val: Op1)) {
2814 if (auto *R =
2815 foldAndOrOfSelectUsingImpliedCond(Op: Op0, SI&: *SI1, /* IsAnd */ true))
2816 return R;
2817 }
2818 }
2819
2820 if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
2821 return FoldedLogic;
2822
2823 if (Instruction *DeMorgan = matchDeMorgansLaws(I, IC&: *this))
2824 return DeMorgan;
2825
2826 {
2827 Value *A, *B, *C;
2828 // A & ~(A ^ B) --> A & B
2829 if (match(V: Op1, P: m_Not(V: m_c_Xor(L: m_Specific(V: Op0), R: m_Value(V&: B)))))
2830 return BinaryOperator::CreateAnd(V1: Op0, V2: B);
2831 // ~(A ^ B) & A --> A & B
2832 if (match(V: Op0, P: m_Not(V: m_c_Xor(L: m_Specific(V: Op1), R: m_Value(V&: B)))))
2833 return BinaryOperator::CreateAnd(V1: Op1, V2: B);
2834
2835 // (A ^ B) & ((B ^ C) ^ A) -> (A ^ B) & ~C
2836 if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))) &&
2837 match(V: Op1, P: m_Xor(L: m_Xor(L: m_Specific(V: B), R: m_Value(V&: C)), R: m_Specific(V: A)))) {
2838 Value *NotC = Op1->hasOneUse()
2839 ? Builder.CreateNot(V: C)
2840 : getFreelyInverted(V: C, WillInvertAllUses: C->hasOneUse(), Builder: &Builder);
2841 if (NotC != nullptr)
2842 return BinaryOperator::CreateAnd(V1: Op0, V2: NotC);
2843 }
2844
2845 // ((A ^ C) ^ B) & (B ^ A) -> (B ^ A) & ~C
2846 if (match(V: Op0, P: m_Xor(L: m_Xor(L: m_Value(V&: A), R: m_Value(V&: C)), R: m_Value(V&: B))) &&
2847 match(V: Op1, P: m_Xor(L: m_Specific(V: B), R: m_Specific(V: A)))) {
2848 Value *NotC = Op0->hasOneUse()
2849 ? Builder.CreateNot(V: C)
2850 : getFreelyInverted(V: C, WillInvertAllUses: C->hasOneUse(), Builder: &Builder);
2851 if (NotC != nullptr)
2852 return BinaryOperator::CreateAnd(V1: Op1, V2: NotC);
2853 }
2854
2855 // (A | B) & (~A ^ B) -> A & B
2856 // (A | B) & (B ^ ~A) -> A & B
2857 // (B | A) & (~A ^ B) -> A & B
2858 // (B | A) & (B ^ ~A) -> A & B
2859 if (match(V: Op1, P: m_c_Xor(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2860 match(V: Op0, P: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))
2861 return BinaryOperator::CreateAnd(V1: A, V2: B);
2862
2863 // (~A ^ B) & (A | B) -> A & B
2864 // (~A ^ B) & (B | A) -> A & B
2865 // (B ^ ~A) & (A | B) -> A & B
2866 // (B ^ ~A) & (B | A) -> A & B
2867 if (match(V: Op0, P: m_c_Xor(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2868 match(V: Op1, P: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))
2869 return BinaryOperator::CreateAnd(V1: A, V2: B);
2870
2871 // (~A | B) & (A ^ B) -> ~A & B
2872 // (~A | B) & (B ^ A) -> ~A & B
2873 // (B | ~A) & (A ^ B) -> ~A & B
2874 // (B | ~A) & (B ^ A) -> ~A & B
2875 if (match(V: Op0, P: m_c_Or(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2876 match(V: Op1, P: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B))))
2877 return BinaryOperator::CreateAnd(V1: Builder.CreateNot(V: A), V2: B);
2878
2879 // (A ^ B) & (~A | B) -> ~A & B
2880 // (B ^ A) & (~A | B) -> ~A & B
2881 // (A ^ B) & (B | ~A) -> ~A & B
2882 // (B ^ A) & (B | ~A) -> ~A & B
2883 if (match(V: Op1, P: m_c_Or(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B))) &&
2884 match(V: Op0, P: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B))))
2885 return BinaryOperator::CreateAnd(V1: Builder.CreateNot(V: A), V2: B);
2886 }
2887
2888 if (Value *Res =
2889 foldBooleanAndOr(LHS: Op0, RHS: Op1, I, /*IsAnd=*/true, /*IsLogical=*/false))
2890 return replaceInstUsesWith(I, V: Res);
2891
2892 if (match(V: Op1, P: m_OneUse(SubPattern: m_LogicalAnd(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
2893 bool IsLogical = isa<SelectInst>(Val: Op1);
2894 if (auto *V = reassociateBooleanAndOr(LHS: Op0, X, Y, I, /*IsAnd=*/true,
2895 /*RHSIsLogical=*/IsLogical))
2896 return replaceInstUsesWith(I, V);
2897 }
2898 if (match(V: Op0, P: m_OneUse(SubPattern: m_LogicalAnd(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
2899 bool IsLogical = isa<SelectInst>(Val: Op0);
2900 if (auto *V = reassociateBooleanAndOr(LHS: Op1, X, Y, I, /*IsAnd=*/true,
2901 /*RHSIsLogical=*/IsLogical))
2902 return replaceInstUsesWith(I, V);
2903 }
2904
2905 if (Instruction *FoldedFCmps = reassociateFCmps(BO&: I, Builder))
2906 return FoldedFCmps;
2907
2908 if (Instruction *CastedAnd = foldCastedBitwiseLogic(I))
2909 return CastedAnd;
2910
2911 if (Instruction *Sel = foldBinopOfSextBoolToSelect(I))
2912 return Sel;
2913
2914 // and(sext(A), B) / and(B, sext(A)) --> A ? B : 0, where A is i1 or <N x i1>.
2915 // TODO: Move this into foldBinopOfSextBoolToSelect as a more generalized fold
2916 // with binop identity constant. But creating a select with non-constant
2917 // arm may not be reversible due to poison semantics. Is that a good
2918 // canonicalization?
2919 Value *A, *B;
2920 if (match(V: &I, P: m_c_And(L: m_SExt(Op: m_Value(V&: A)), R: m_Value(V&: B))) &&
2921 A->getType()->isIntOrIntVectorTy(BitWidth: 1))
2922 return createSelectInstWithUnknownProfile(C: A, S1: B, S2: Constant::getNullValue(Ty));
2923
2924 // Similarly, a 'not' of the bool translates to a swap of the select arms:
2925 // ~sext(A) & B / B & ~sext(A) --> A ? 0 : B
2926 if (match(V: &I, P: m_c_And(L: m_Not(V: m_SExt(Op: m_Value(V&: A))), R: m_Value(V&: B))) &&
2927 A->getType()->isIntOrIntVectorTy(BitWidth: 1))
2928 return createSelectInstWithUnknownProfile(C: A, S1: Constant::getNullValue(Ty), S2: B);
2929
2930 // and(zext(A), B) -> A ? (B & 1) : 0
2931 if (match(V: &I, P: m_c_And(L: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: A))), R: m_Value(V&: B))) &&
2932 A->getType()->isIntOrIntVectorTy(BitWidth: 1))
2933 return createSelectInstWithUnknownProfile(
2934 C: A, S1: Builder.CreateAnd(LHS: B, RHS: ConstantInt::get(Ty, V: 1)),
2935 S2: Constant::getNullValue(Ty));
2936
2937 // (-1 + A) & B --> A ? 0 : B where A is 0/1.
2938 if (match(V: &I, P: m_c_And(L: m_OneUse(SubPattern: m_Add(L: m_ZExtOrSelf(Op: m_Value(V&: A)), R: m_AllOnes())),
2939 R: m_Value(V&: B)))) {
2940 if (A->getType()->isIntOrIntVectorTy(BitWidth: 1))
2941 return createSelectInstWithUnknownProfile(C: A, S1: Constant::getNullValue(Ty),
2942 S2: B);
2943 if (computeKnownBits(V: A, CtxI: &I).countMaxActiveBits() <= 1) {
2944 return createSelectInstWithUnknownProfile(
2945 C: Builder.CreateICmpEQ(LHS: A, RHS: Constant::getNullValue(Ty: A->getType())), S1: B,
2946 S2: Constant::getNullValue(Ty));
2947 }
2948 }
2949
2950 // (iN X s>> (N-1)) & Y --> (X s< 0) ? Y : 0 -- with optional sext
2951 if (match(V: &I, P: m_c_And(L: m_OneUse(SubPattern: m_SExtOrSelf(
2952 Op: m_AShr(L: m_Value(V&: X), R: m_APIntAllowPoison(Res&: C)))),
2953 R: m_Value(V&: Y))) &&
2954 *C == X->getType()->getScalarSizeInBits() - 1) {
2955 Value *IsNeg = Builder.CreateIsNeg(Arg: X, Name: "isneg");
2956 return createSelectInstWithUnknownProfile(C: IsNeg, S1: Y,
2957 S2: ConstantInt::getNullValue(Ty));
2958 }
2959 // If there's a 'not' of the shifted value, swap the select operands:
2960 // ~(iN X s>> (N-1)) & Y --> (X s< 0) ? 0 : Y -- with optional sext
2961 if (match(V: &I, P: m_c_And(L: m_OneUse(SubPattern: m_SExtOrSelf(
2962 Op: m_Not(V: m_AShr(L: m_Value(V&: X), R: m_APIntAllowPoison(Res&: C))))),
2963 R: m_Value(V&: Y))) &&
2964 *C == X->getType()->getScalarSizeInBits() - 1) {
2965 Value *IsNeg = Builder.CreateIsNeg(Arg: X, Name: "isneg");
2966 return createSelectInstWithUnknownProfile(C: IsNeg,
2967 S1: ConstantInt::getNullValue(Ty), S2: Y);
2968 }
2969
2970 // (~x) & y --> ~(x | (~y)) iff that gets rid of inversions
2971 if (sinkNotIntoOtherHandOfLogicalOp(I))
2972 return &I;
2973
2974 // An and recurrence w/loop invariant step is equivelent to (and start, step)
2975 PHINode *PN = nullptr;
2976 Value *Start = nullptr, *Step = nullptr;
2977 if (matchSimpleRecurrence(I: &I, P&: PN, Start, Step) && DT.dominates(Def: Step, User: PN))
2978 return replaceInstUsesWith(I, V: Builder.CreateAnd(LHS: Start, RHS: Step));
2979
2980 if (Instruction *R = reassociateForUses(BO&: I, Builder))
2981 return R;
2982
2983 if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder))
2984 return Canonicalized;
2985
2986 if (Instruction *Folded = foldLogicOfIsFPClass(BO&: I, Op0, Op1))
2987 return Folded;
2988
2989 if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
2990 return Res;
2991
2992 if (Instruction *Res = foldBitwiseLogicWithIntrinsics(I, Builder))
2993 return Res;
2994
2995 if (Value *V =
2996 simplifyAndOrWithOpReplaced(V: Op0, Op: Op1, RepOp: Constant::getAllOnesValue(Ty),
2997 /*SimplifyOnly*/ false, IC&: *this))
2998 return BinaryOperator::CreateAnd(V1: V, V2: Op1);
2999 if (Value *V =
3000 simplifyAndOrWithOpReplaced(V: Op1, Op: Op0, RepOp: Constant::getAllOnesValue(Ty),
3001 /*SimplifyOnly*/ false, IC&: *this))
3002 return BinaryOperator::CreateAnd(V1: Op0, V2: V);
3003
3004 if (Instruction *Res = foldRoundUpToPow2Alignment(I, Builder))
3005 return Res;
3006
3007 return nullptr;
3008}
3009
3010Instruction *InstCombinerImpl::matchBSwapOrBitReverse(Instruction &I,
3011 bool MatchBSwaps,
3012 bool MatchBitReversals) {
3013 SmallVector<Instruction *, 4> Insts;
3014 if (!recognizeBSwapOrBitReverseIdiom(I: &I, MatchBSwaps, MatchBitReversals,
3015 InsertedInsts&: Insts))
3016 return nullptr;
3017 Instruction *LastInst = Insts.pop_back_val();
3018 LastInst->removeFromParent();
3019
3020 for (auto *Inst : Insts) {
3021 Inst->setDebugLoc(I.getDebugLoc());
3022 Worklist.push(I: Inst);
3023 }
3024 return LastInst;
3025}
3026
3027std::optional<std::pair<Intrinsic::ID, SmallVector<Value *, 3>>>
3028InstCombinerImpl::convertOrOfShiftsToFunnelShift(Instruction &Or) {
3029 // TODO: Can we reduce the code duplication between this and the related
3030 // rotate matching code under visitSelect and visitTrunc?
3031 assert(Or.getOpcode() == BinaryOperator::Or && "Expecting or instruction");
3032
3033 unsigned Width = Or.getType()->getScalarSizeInBits();
3034
3035 Instruction *Or0, *Or1;
3036 if (!match(V: Or.getOperand(i: 0), P: m_Instruction(I&: Or0)) ||
3037 !match(V: Or.getOperand(i: 1), P: m_Instruction(I&: Or1)))
3038 return std::nullopt;
3039
3040 bool IsFshl = true; // Sub on LSHR.
3041 SmallVector<Value *, 3> FShiftArgs;
3042
3043 // First, find an or'd pair of opposite shifts:
3044 // or (lshr ShVal0, ShAmt0), (shl ShVal1, ShAmt1)
3045 if (isa<BinaryOperator>(Val: Or0) && isa<BinaryOperator>(Val: Or1)) {
3046 Value *ShVal0, *ShVal1, *ShAmt0, *ShAmt1;
3047 if (!match(V: Or0,
3048 P: m_OneUse(SubPattern: m_LogicalShift(L: m_Value(V&: ShVal0), R: m_Value(V&: ShAmt0)))) ||
3049 !match(V: Or1,
3050 P: m_OneUse(SubPattern: m_LogicalShift(L: m_Value(V&: ShVal1), R: m_Value(V&: ShAmt1)))) ||
3051 Or0->getOpcode() == Or1->getOpcode())
3052 return std::nullopt;
3053
3054 // Canonicalize to or(shl(ShVal0, ShAmt0), lshr(ShVal1, ShAmt1)).
3055 if (Or0->getOpcode() == BinaryOperator::LShr) {
3056 std::swap(a&: Or0, b&: Or1);
3057 std::swap(a&: ShVal0, b&: ShVal1);
3058 std::swap(a&: ShAmt0, b&: ShAmt1);
3059 }
3060 assert(Or0->getOpcode() == BinaryOperator::Shl &&
3061 Or1->getOpcode() == BinaryOperator::LShr &&
3062 "Illegal or(shift,shift) pair");
3063
3064 // Match the shift amount operands for a funnel shift pattern. This always
3065 // matches a subtraction on the R operand.
3066 auto matchShiftAmount = [&](Value *L, Value *R, unsigned Width) -> Value * {
3067 // Check for constant shift amounts that sum to the bitwidth.
3068 const APInt *LI, *RI;
3069 if (match(V: L, P: m_APIntAllowPoison(Res&: LI)) && match(V: R, P: m_APIntAllowPoison(Res&: RI)))
3070 if (LI->ult(RHS: Width) && RI->ult(RHS: Width) && (*LI + *RI) == Width)
3071 return ConstantInt::get(Ty: L->getType(), V: *LI);
3072
3073 Constant *LC, *RC;
3074 if (match(V: L, P: m_Constant(C&: LC)) && match(V: R, P: m_Constant(C&: RC)) &&
3075 match(V: L,
3076 P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_ULT, Threshold: APInt(Width, Width))) &&
3077 match(V: R,
3078 P: m_SpecificInt_ICMP(Predicate: ICmpInst::ICMP_ULT, Threshold: APInt(Width, Width))) &&
3079 match(V: ConstantExpr::getAdd(C1: LC, C2: RC), P: m_SpecificIntAllowPoison(V: Width)))
3080 return ConstantExpr::mergeUndefsWith(C: LC, Other: RC);
3081
3082 // (shl ShVal, X) | (lshr ShVal, (Width - x)) iff X < Width.
3083 // We limit this to X < Width in case the backend re-expands the
3084 // intrinsic, and has to reintroduce a shift modulo operation (InstCombine
3085 // might remove it after this fold). This still doesn't guarantee that the
3086 // final codegen will match this original pattern.
3087 if (match(V: R, P: m_OneUse(SubPattern: m_Sub(L: m_SpecificInt(V: Width), R: m_Specific(V: L))))) {
3088 KnownBits KnownL = computeKnownBits(V: L, CtxI: &Or);
3089 return KnownL.getMaxValue().ult(RHS: Width) ? L : nullptr;
3090 }
3091
3092 // For non-constant cases, the following patterns currently only work for
3093 // rotation patterns.
3094 // TODO: Add general funnel-shift compatible patterns.
3095 if (ShVal0 != ShVal1)
3096 return nullptr;
3097
3098 // For non-constant cases we don't support non-pow2 shift masks.
3099 // TODO: Is it worth matching urem as well?
3100 if (!isPowerOf2_32(Value: Width))
3101 return nullptr;
3102
3103 // The shift amount may be masked with negation:
3104 // (shl ShVal, (X & (Width - 1))) | (lshr ShVal, ((-X) & (Width - 1)))
3105 Value *X;
3106 unsigned Mask = Width - 1;
3107 if (match(V: L, P: m_And(L: m_Value(V&: X), R: m_SpecificInt(V: Mask))) &&
3108 match(V: R, P: m_And(L: m_Neg(V: m_Specific(V: X)), R: m_SpecificInt(V: Mask))))
3109 return X;
3110
3111 // (shl ShVal,(X+1) & (Width-1)) | (lshr ShVal,((X & (Width-1)) ^
3112 // (Width-1)))
3113 {
3114 Value *XPlusOne = nullptr;
3115 if (match(V: L, P: m_And(L: m_Value(V&: XPlusOne, P: m_Add(L: m_Value(V&: X), R: m_One())),
3116 R: m_SpecificInt(V: Mask))) &&
3117 match(V: R, P: m_Xor(L: m_And(L: m_Specific(V: X), R: m_SpecificInt(V: Mask)),
3118 R: m_SpecificInt(V: Mask))))
3119 return XPlusOne;
3120 }
3121
3122 // (shl ShVal, X) | (lshr ShVal, ((-X) & (Width - 1)))
3123 if (match(V: R, P: m_And(L: m_Neg(V: m_Specific(V: L)), R: m_SpecificInt(V: Mask))))
3124 return L;
3125
3126 // Similar to above, but the shift amount may be extended after masking,
3127 // so return the extended value as the parameter for the intrinsic.
3128 if (match(V: L, P: m_ZExt(Op: m_And(L: m_Value(V&: X), R: m_SpecificInt(V: Mask)))) &&
3129 match(V: R,
3130 P: m_And(L: m_Neg(V: m_ZExt(Op: m_And(L: m_Specific(V: X), R: m_SpecificInt(V: Mask)))),
3131 R: m_SpecificInt(V: Mask))))
3132 return L;
3133
3134 if (match(V: L, P: m_ZExt(Op: m_And(L: m_Value(V&: X), R: m_SpecificInt(V: Mask)))) &&
3135 match(V: R, P: m_ZExt(Op: m_And(L: m_Neg(V: m_Specific(V: X)), R: m_SpecificInt(V: Mask)))))
3136 return L;
3137
3138 return nullptr;
3139 };
3140
3141 Value *ShAmt = matchShiftAmount(ShAmt0, ShAmt1, Width);
3142 if (!ShAmt) {
3143 ShAmt = matchShiftAmount(ShAmt1, ShAmt0, Width);
3144 IsFshl = false; // Sub on SHL.
3145 }
3146 if (!ShAmt)
3147 return std::nullopt;
3148
3149 FShiftArgs = {ShVal0, ShVal1, ShAmt};
3150 } else if (isa<ZExtInst>(Val: Or0) || isa<ZExtInst>(Val: Or1)) {
3151 // If there are two 'or' instructions concat variables in opposite order:
3152 //
3153 // Slot1 and Slot2 are all zero bits.
3154 // | Slot1 | Low | Slot2 | High |
3155 // LowHigh = or (shl (zext Low), ZextLowShlAmt), (zext High)
3156 // | Slot2 | High | Slot1 | Low |
3157 // HighLow = or (shl (zext High), ZextHighShlAmt), (zext Low)
3158 //
3159 // the latter 'or' can be safely convert to
3160 // -> HighLow = fshl LowHigh, LowHigh, ZextHighShlAmt
3161 // if ZextLowShlAmt + ZextHighShlAmt == Width.
3162 if (!isa<ZExtInst>(Val: Or1))
3163 std::swap(a&: Or0, b&: Or1);
3164
3165 Value *High, *ZextHigh, *Low;
3166 const APInt *ZextHighShlAmt;
3167 if (!match(V: Or0,
3168 P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: ZextHigh), R: m_APInt(Res&: ZextHighShlAmt)))))
3169 return std::nullopt;
3170
3171 if (!match(V: Or1, P: m_ZExt(Op: m_Value(V&: Low))) ||
3172 !match(V: ZextHigh, P: m_ZExt(Op: m_Value(V&: High))))
3173 return std::nullopt;
3174
3175 unsigned HighSize = High->getType()->getScalarSizeInBits();
3176 unsigned LowSize = Low->getType()->getScalarSizeInBits();
3177 // Make sure High does not overlap with Low and most significant bits of
3178 // High aren't shifted out.
3179 if (ZextHighShlAmt->ult(RHS: LowSize) || ZextHighShlAmt->ugt(RHS: Width - HighSize))
3180 return std::nullopt;
3181
3182 for (User *U : ZextHigh->users()) {
3183 Value *X, *Y;
3184 if (!match(V: U, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))))
3185 continue;
3186
3187 if (!isa<ZExtInst>(Val: Y))
3188 std::swap(a&: X, b&: Y);
3189
3190 const APInt *ZextLowShlAmt;
3191 if (!match(V: X, P: m_Shl(L: m_Specific(V: Or1), R: m_APInt(Res&: ZextLowShlAmt))) ||
3192 !match(V: Y, P: m_Specific(V: ZextHigh)) || !DT.dominates(Def: U, User: &Or))
3193 continue;
3194
3195 // HighLow is good concat. If sum of two shifts amount equals to Width,
3196 // LowHigh must also be a good concat.
3197 if (*ZextLowShlAmt + *ZextHighShlAmt != Width)
3198 continue;
3199
3200 // Low must not overlap with High and most significant bits of Low must
3201 // not be shifted out.
3202 assert(ZextLowShlAmt->uge(HighSize) &&
3203 ZextLowShlAmt->ule(Width - LowSize) && "Invalid concat");
3204
3205 // We cannot reuse the result if it may produce poison.
3206 // Drop poison generating flags in the expression tree.
3207 // Or
3208 cast<Instruction>(Val: U)->dropPoisonGeneratingFlags();
3209 // Shl
3210 cast<Instruction>(Val: X)->dropPoisonGeneratingFlags();
3211
3212 FShiftArgs = {U, U, ConstantInt::get(Ty: Or0->getType(), V: *ZextHighShlAmt)};
3213 break;
3214 }
3215 }
3216
3217 if (FShiftArgs.empty())
3218 return std::nullopt;
3219
3220 Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
3221 return std::make_pair(x&: IID, y&: FShiftArgs);
3222}
3223
3224/// Match UB-safe variants of the funnel shift intrinsic.
3225static Instruction *matchFunnelShift(Instruction &Or, InstCombinerImpl &IC) {
3226 if (auto Opt = IC.convertOrOfShiftsToFunnelShift(Or)) {
3227 auto [IID, FShiftArgs] = *Opt;
3228 Function *F =
3229 Intrinsic::getOrInsertDeclaration(M: Or.getModule(), id: IID, OverloadTys: Or.getType());
3230 return CallInst::Create(Func: F, Args: FShiftArgs);
3231 }
3232
3233 return nullptr;
3234}
3235
3236/// Attempt to combine or(zext(x),shl(zext(y),bw/2) concat packing patterns.
3237static Value *matchOrConcat(Instruction &Or, InstCombiner::BuilderTy &Builder) {
3238 assert(Or.getOpcode() == Instruction::Or && "bswap requires an 'or'");
3239 Value *Op0 = Or.getOperand(i: 0), *Op1 = Or.getOperand(i: 1);
3240 Type *Ty = Or.getType();
3241
3242 unsigned Width = Ty->getScalarSizeInBits();
3243 if ((Width & 1) != 0)
3244 return nullptr;
3245 unsigned HalfWidth = Width / 2;
3246
3247 // Canonicalize zext (lower half) to LHS.
3248 if (!isa<ZExtInst>(Val: Op0))
3249 std::swap(a&: Op0, b&: Op1);
3250
3251 // Find lower/upper half.
3252 Value *LowerSrc, *ShlVal, *UpperSrc;
3253 const APInt *C;
3254 if (!match(V: Op0, P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: LowerSrc)))) ||
3255 !match(V: Op1, P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: ShlVal), R: m_APInt(Res&: C)))) ||
3256 !match(V: ShlVal, P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: UpperSrc)))))
3257 return nullptr;
3258 if (*C != HalfWidth || LowerSrc->getType() != UpperSrc->getType() ||
3259 LowerSrc->getType()->getScalarSizeInBits() != HalfWidth)
3260 return nullptr;
3261
3262 auto ConcatIntrinsicCalls = [&](Intrinsic::ID id, Value *Lo, Value *Hi) {
3263 Value *NewLower = Builder.CreateZExt(V: Lo, DestTy: Ty);
3264 Value *NewUpper = Builder.CreateZExt(V: Hi, DestTy: Ty);
3265 NewUpper = Builder.CreateShl(LHS: NewUpper, RHS: HalfWidth);
3266 Value *BinOp = Builder.CreateDisjointOr(LHS: NewLower, RHS: NewUpper);
3267 return Builder.CreateIntrinsic(ID: id, OverloadTypes: Ty, Args: BinOp);
3268 };
3269
3270 // BSWAP: Push the concat down, swapping the lower/upper sources.
3271 // concat(bswap(x),bswap(y)) -> bswap(concat(x,y))
3272 Value *LowerBSwap, *UpperBSwap;
3273 if (match(V: LowerSrc, P: m_BSwap(Op0: m_Value(V&: LowerBSwap))) &&
3274 match(V: UpperSrc, P: m_BSwap(Op0: m_Value(V&: UpperBSwap))))
3275 return ConcatIntrinsicCalls(Intrinsic::bswap, UpperBSwap, LowerBSwap);
3276
3277 // BITREVERSE: Push the concat down, swapping the lower/upper sources.
3278 // concat(bitreverse(x),bitreverse(y)) -> bitreverse(concat(x,y))
3279 Value *LowerBRev, *UpperBRev;
3280 if (match(V: LowerSrc, P: m_BitReverse(Op0: m_Value(V&: LowerBRev))) &&
3281 match(V: UpperSrc, P: m_BitReverse(Op0: m_Value(V&: UpperBRev))))
3282 return ConcatIntrinsicCalls(Intrinsic::bitreverse, UpperBRev, LowerBRev);
3283
3284 // iX ext split: extending or(zext(x),shl(zext(y),bw/2) pattern
3285 // to consume sext/ashr:
3286 // or(zext(sext(x)),shl(zext(sext(ashr(x,xbw-1))),bw/2)
3287 // or(zext(x),shl(zext(ashr(x,xbw-1)),bw/2)
3288 Value *X;
3289 if (match(V: LowerSrc, P: m_SExtOrSelf(Op: m_Value(V&: X))) &&
3290 match(V: UpperSrc,
3291 P: m_SExtOrSelf(Op: m_AShr(
3292 L: m_Specific(V: X),
3293 R: m_SpecificInt(V: X->getType()->getScalarSizeInBits() - 1)))))
3294 return Builder.CreateSExt(V: X, DestTy: Ty);
3295
3296 return nullptr;
3297}
3298
3299/// If all elements of two constant vectors are 0/-1 and inverses, return true.
3300static bool areInverseVectorBitmasks(Constant *C1, Constant *C2) {
3301 unsigned NumElts = cast<FixedVectorType>(Val: C1->getType())->getNumElements();
3302 for (unsigned i = 0; i != NumElts; ++i) {
3303 Constant *EltC1 = C1->getAggregateElement(Elt: i);
3304 Constant *EltC2 = C2->getAggregateElement(Elt: i);
3305 if (!EltC1 || !EltC2)
3306 return false;
3307
3308 // One element must be all ones, and the other must be all zeros.
3309 if (!((match(V: EltC1, P: m_Zero()) && match(V: EltC2, P: m_AllOnes())) ||
3310 (match(V: EltC2, P: m_Zero()) && match(V: EltC1, P: m_AllOnes()))))
3311 return false;
3312 }
3313 return true;
3314}
3315
3316/// We have an expression of the form (A & C) | (B & D). If A is a scalar or
3317/// vector composed of all-zeros or all-ones values and is the bitwise 'not' of
3318/// B, it can be used as the condition operand of a select instruction.
3319/// We will detect (A & C) | ~(B | D) when the flag ABIsTheSame enabled.
3320Value *InstCombinerImpl::getSelectCondition(Value *A, Value *B,
3321 bool ABIsTheSame) {
3322 // We may have peeked through bitcasts in the caller.
3323 // Exit immediately if we don't have (vector) integer types.
3324 Type *Ty = A->getType();
3325 if (!Ty->isIntOrIntVectorTy() || !B->getType()->isIntOrIntVectorTy())
3326 return nullptr;
3327
3328 // If A is the 'not' operand of B and has enough signbits, we have our answer.
3329 if (ABIsTheSame ? (A == B) : match(V: B, P: m_Not(V: m_Specific(V: A)))) {
3330 // If these are scalars or vectors of i1, A can be used directly.
3331 if (Ty->isIntOrIntVectorTy(BitWidth: 1))
3332 return A;
3333
3334 // If we look through a vector bitcast, the caller will bitcast the operands
3335 // to match the condition's number of bits (N x i1).
3336 // To make this poison-safe, disallow bitcast from wide element to narrow
3337 // element. That could allow poison in lanes where it was not present in the
3338 // original code.
3339 A = peekThroughBitcast(V: A);
3340 if (A->getType()->isIntOrIntVectorTy()) {
3341 unsigned NumSignBits = ComputeNumSignBits(Op: A);
3342 if (NumSignBits == A->getType()->getScalarSizeInBits() &&
3343 NumSignBits <= Ty->getScalarSizeInBits())
3344 return Builder.CreateTrunc(V: A, DestTy: CmpInst::makeCmpResultType(opnd_type: A->getType()));
3345 }
3346 return nullptr;
3347 }
3348
3349 // TODO: add support for sext and constant case
3350 if (ABIsTheSame)
3351 return nullptr;
3352
3353 // If both operands are constants, see if the constants are inverse bitmasks.
3354 Constant *AConst, *BConst;
3355 if (match(V: A, P: m_Constant(C&: AConst)) && match(V: B, P: m_Constant(C&: BConst)))
3356 if (AConst == ConstantExpr::getNot(C: BConst) &&
3357 ComputeNumSignBits(Op: A) == Ty->getScalarSizeInBits())
3358 return Builder.CreateZExtOrTrunc(V: A, DestTy: CmpInst::makeCmpResultType(opnd_type: Ty));
3359
3360 // Look for more complex patterns. The 'not' op may be hidden behind various
3361 // casts. Look through sexts and bitcasts to find the booleans.
3362 Value *Cond;
3363 Value *NotB;
3364 if (match(V: A, P: m_SExt(Op: m_Value(V&: Cond))) &&
3365 Cond->getType()->isIntOrIntVectorTy(BitWidth: 1)) {
3366 // A = sext i1 Cond; B = sext (not (i1 Cond))
3367 if (match(V: B, P: m_SExt(Op: m_Not(V: m_Specific(V: Cond)))))
3368 return Cond;
3369
3370 // A = sext i1 Cond; B = not ({bitcast} (sext (i1 Cond)))
3371 // TODO: The one-use checks are unnecessary or misplaced. If the caller
3372 // checked for uses on logic ops/casts, that should be enough to
3373 // make this transform worthwhile.
3374 if (match(V: B, P: m_OneUse(SubPattern: m_Not(V: m_Value(V&: NotB))))) {
3375 NotB = peekThroughBitcast(V: NotB, OneUseOnly: true);
3376 if (match(V: NotB, P: m_SExt(Op: m_Specific(V: Cond))))
3377 return Cond;
3378 }
3379 }
3380
3381 // All scalar (and most vector) possibilities should be handled now.
3382 // Try more matches that only apply to non-splat constant vectors.
3383 if (!Ty->isVectorTy())
3384 return nullptr;
3385
3386 // If both operands are xor'd with constants using the same sexted boolean
3387 // operand, see if the constants are inverse bitmasks.
3388 // TODO: Use ConstantExpr::getNot()?
3389 if (match(V: A, P: (m_Xor(L: m_SExt(Op: m_Value(V&: Cond)), R: m_Constant(C&: AConst)))) &&
3390 match(V: B, P: (m_Xor(L: m_SExt(Op: m_Specific(V: Cond)), R: m_Constant(C&: BConst)))) &&
3391 Cond->getType()->isIntOrIntVectorTy(BitWidth: 1) &&
3392 areInverseVectorBitmasks(C1: AConst, C2: BConst)) {
3393 AConst = ConstantExpr::getTrunc(C: AConst, Ty: CmpInst::makeCmpResultType(opnd_type: Ty));
3394 return Builder.CreateXor(LHS: Cond, RHS: AConst);
3395 }
3396 return nullptr;
3397}
3398
3399/// We have an expression of the form (A & B) | (C & D). Try to simplify this
3400/// to "A' ? B : D", where A' is a boolean or vector of booleans.
3401/// When InvertFalseVal is set to true, we try to match the pattern
3402/// where we have peeked through a 'not' op and A and C are the same:
3403/// (A & B) | ~(A | D) --> (A & B) | (~A & ~D) --> A' ? B : ~D
3404Value *InstCombinerImpl::matchSelectFromAndOr(Value *A, Value *B, Value *C,
3405 Value *D, bool InvertFalseVal) {
3406 // The potential condition of the select may be bitcasted. In that case, look
3407 // through its bitcast and the corresponding bitcast of the 'not' condition.
3408 Type *OrigType = A->getType();
3409 A = peekThroughBitcast(V: A, OneUseOnly: true);
3410 C = peekThroughBitcast(V: C, OneUseOnly: true);
3411 if (Value *Cond = getSelectCondition(A, B: C, ABIsTheSame: InvertFalseVal)) {
3412 // ((bc Cond) & B) | ((bc ~Cond) & D) --> bc (select Cond, (bc B), (bc D))
3413 // If this is a vector, we may need to cast to match the condition's length.
3414 // The bitcasts will either all exist or all not exist. The builder will
3415 // not create unnecessary casts if the types already match.
3416 Type *SelTy = A->getType();
3417 if (auto *VecTy = dyn_cast<VectorType>(Val: Cond->getType())) {
3418 // For a fixed or scalable vector get N from <{vscale x} N x iM>
3419 unsigned Elts = VecTy->getElementCount().getKnownMinValue();
3420 // For a fixed or scalable vector, get the size in bits of N x iM; for a
3421 // scalar this is just M.
3422 unsigned SelEltSize = SelTy->getPrimitiveSizeInBits().getKnownMinValue();
3423 Type *EltTy = Builder.getIntNTy(N: SelEltSize / Elts);
3424 SelTy = VectorType::get(ElementType: EltTy, EC: VecTy->getElementCount());
3425 }
3426 Value *BitcastB = Builder.CreateBitCast(V: B, DestTy: SelTy);
3427 if (InvertFalseVal)
3428 D = Builder.CreateNot(V: D);
3429 Value *BitcastD = Builder.CreateBitCast(V: D, DestTy: SelTy);
3430 // The condition here is synthesized and thus we have no way of knowing the
3431 // distribution in general. Thus, mark the branch weights of the created
3432 // select unknown.
3433 Value *Select = Builder.CreateSelectWithUnknownProfile(
3434 C: Cond, True: BitcastB, False: BitcastD, DEBUG_TYPE);
3435 return Builder.CreateBitCast(V: Select, DestTy: OrigType);
3436 }
3437
3438 return nullptr;
3439}
3440
3441// (icmp eq X, C) | (icmp ult Other, (X - C)) -> (icmp ule Other, (X - (C + 1)))
3442// (icmp ne X, C) & (icmp uge Other, (X - C)) -> (icmp ugt Other, (X - (C + 1)))
3443static Value *foldAndOrOfICmpEqConstantAndICmp(CmpPredicate PredL, Value *LHS0,
3444 Value *LHS1, bool LHSOneUse,
3445 CmpPredicate PredR, Value *RHS0,
3446 Value *RHS1, bool RHSOneUse,
3447 bool IsAnd, bool IsLogical,
3448 IRBuilderBase &Builder) {
3449 if (IsAnd) {
3450 PredL = CmpPredicate::getInverse(P: PredL);
3451 PredR = CmpPredicate::getInverse(P: PredR);
3452 }
3453
3454 const APInt *CInt;
3455 if (PredL != ICmpInst::ICMP_EQ || !match(V: LHS1, P: m_APIntAllowPoison(Res&: CInt)) ||
3456 !LHS0->getType()->isIntOrIntVectorTy() || !(LHSOneUse || RHSOneUse))
3457 return nullptr;
3458
3459 auto MatchRHSOp = [LHS0, CInt](const Value *RHSOp) {
3460 return match(V: RHSOp,
3461 P: m_Add(L: m_Specific(V: LHS0), R: m_SpecificIntAllowPoison(V: -*CInt))) ||
3462 (CInt->isZero() && RHSOp == LHS0);
3463 };
3464
3465 Value *Other;
3466 if (PredR == ICmpInst::ICMP_ULT && MatchRHSOp(RHS1))
3467 Other = RHS0;
3468 else if (PredR == ICmpInst::ICMP_UGT && MatchRHSOp(RHS0))
3469 Other = RHS1;
3470 else
3471 return nullptr;
3472
3473 if (IsLogical)
3474 Other = Builder.CreateFreeze(V: Other);
3475
3476 return Builder.CreateICmp(
3477 P: IsAnd ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE,
3478 LHS: Builder.CreateSub(LHS: LHS0, RHS: ConstantInt::get(Ty: LHS0->getType(), V: *CInt + 1)),
3479 RHS: Other);
3480}
3481
3482/// Fold (icmp)&(icmp) or (icmp)|(icmp) if possible.
3483/// If IsLogical is true, then the and/or is in select form and the transform
3484/// must be poison-safe.
3485Value *InstCombinerImpl::foldAndOrOfICmps(Value *LHS, Value *RHS,
3486 Instruction &I, bool IsAnd,
3487 bool IsLogical) {
3488 CmpPredicate PredL, PredR;
3489 Value *LHS0, *LHS1, *RHS0, *RHS1;
3490 if (!match(V: LHS, P: m_ICmpLike(Pred&: PredL, L: m_Value(V&: LHS0), R: m_Value(V&: LHS1))) ||
3491 !match(V: RHS, P: m_ICmpLike(Pred&: PredR, L: m_Value(V&: RHS0), R: m_Value(V&: RHS1))))
3492 return nullptr;
3493
3494 bool LHSOneUse = LHS->hasOneUse();
3495 bool RHSOneUse = RHS->hasOneUse();
3496
3497 const SimplifyQuery Q = SQ.getWithInstruction(I: &I);
3498
3499 const APInt *LHSC = nullptr, *RHSC = nullptr;
3500 match(V: LHS1, P: m_APInt(Res&: LHSC));
3501 match(V: RHS1, P: m_APInt(Res&: RHSC));
3502
3503 // (icmp1 A, B) | (icmp2 A, B) --> (icmp3 A, B)
3504 // (icmp1 A, B) & (icmp2 A, B) --> (icmp3 A, B)
3505 if (predicatesFoldable(P1: PredL, P2: PredR)) {
3506 if (LHS0 == RHS1 && LHS1 == RHS0) {
3507 PredL = ICmpInst::getSwappedPredicate(pred: PredL);
3508 std::swap(a&: LHS0, b&: LHS1);
3509 }
3510 if (LHS0 == RHS0 && LHS1 == RHS1) {
3511 unsigned Code = IsAnd ? getICmpCode(Pred: PredL) & getICmpCode(Pred: PredR)
3512 : getICmpCode(Pred: PredL) | getICmpCode(Pred: PredR);
3513 bool IsSigned = ICmpInst::isSigned(Pred: PredL) || ICmpInst::isSigned(Pred: PredR);
3514 return getNewICmpValue(Code, Sign: IsSigned, LHS: LHS0, RHS: LHS1, Builder);
3515 }
3516 }
3517
3518 if (Value *V = foldAndOrOfICmpEqConstantAndICmp(PredL, LHS0, LHS1, LHSOneUse,
3519 PredR, RHS0, RHS1, RHSOneUse,
3520 IsAnd, IsLogical, Builder))
3521 return V;
3522 // We can treat logical like bitwise here, because both operands are used on
3523 // the LHS, and as such poison from both will propagate.
3524 if (Value *V = foldAndOrOfICmpEqConstantAndICmp(
3525 PredL: PredR, LHS0: RHS0, LHS1: RHS1, LHSOneUse: RHSOneUse, PredR: PredL, RHS0: LHS0, RHS1: LHS1, RHSOneUse: LHSOneUse, IsAnd,
3526 /*IsLogical*/ false, Builder))
3527 return V;
3528
3529 if (Value *V = foldAndOrOfICmpsWithConstEq(PredL, LHS0, LHS1, LHS, PredR,
3530 RHS0, RHS1, RHSOneUse, IsAnd,
3531 IsLogical, Builder, Q, I))
3532 return V;
3533 // We can convert this case to bitwise and, because both operands are used
3534 // on the LHS, and as such poison from both will propagate.
3535 // Can not handle RHS = trunc nuw as it is not same as icmp ne 0 for all
3536 // values
3537 if (isa<ICmpInst>(Val: RHS))
3538 if (Value *V = foldAndOrOfICmpsWithConstEq(
3539 PredL: PredR, LHS0: RHS0, LHS1: RHS1, LHS: RHS, PredR: PredL, RHS0: LHS0, RHS1: LHS1, RHSOneUse: LHSOneUse, IsAnd,
3540 /*IsLogical=*/false, Builder, Q, I)) {
3541 // If RHS is still used, we should drop samesign flag.
3542 if (IsLogical && PredR.hasSameSign() && !RHS->use_empty()) {
3543 auto *CmpR = cast<ICmpInst>(Val: RHS);
3544 CmpR->setSameSign(false);
3545 addToWorklist(I: CmpR);
3546 }
3547 return V;
3548 }
3549
3550 if (Value *V = foldIsPowerOf2OrZero(PredL, LHS0, LHS1, PredR, RHS0, RHS1,
3551 IsAnd, Builder, IC&: *this))
3552 return V;
3553 if (Value *V = foldIsPowerOf2OrZero(PredL: PredR, LHS0: RHS0, LHS1: RHS1, PredR: PredL, RHS0: LHS0, RHS1: LHS1,
3554 IsAnd, Builder, IC&: *this))
3555 return V;
3556
3557 // TODO: One of these directions is fine with logical and/or, the other could
3558 // be supported by inserting freeze.
3559 if (!IsLogical) {
3560 // E.g. (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n
3561 // E.g. (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n
3562 if (Value *V = simplifyRangeCheck(PredL, LHS0, LHS1, PredR, RHS0, RHS1, CtxI: &I,
3563 /*Inverted=*/!IsAnd))
3564 return V;
3565
3566 // E.g. (icmp sgt x, n) | (icmp slt x, 0) --> icmp ugt x, n
3567 // E.g. (icmp slt x, n) & (icmp sge x, 0) --> icmp ult x, n
3568 if (Value *V = simplifyRangeCheck(PredL: PredR, LHS0: RHS0, LHS1: RHS1, PredR: PredL, RHS0: LHS0, RHS1: LHS1, CtxI: &I,
3569 /*Inverted=*/!IsAnd))
3570 return V;
3571 }
3572
3573 // TODO: Add conjugated or fold, check whether it is safe for logical and/or.
3574 if (IsAnd && !IsLogical)
3575 if (Value *V = foldSignedTruncationCheck(PredL, LHS0, LHS1, PredR, RHS0,
3576 RHS1, CtxI&: I, Builder))
3577 return V;
3578
3579 if (Value *V = foldIsPowerOf2(PredL, LHS0, LHS1, PredR, RHS0, RHS1, JoinedByAnd: IsAnd,
3580 Builder, IC&: *this))
3581 return V;
3582
3583 if (Value *V = foldPowerOf2AndShiftedMask(Cmp0: LHS, Cmp1: RHS, JoinedByAnd: IsAnd, Builder))
3584 return V;
3585
3586 // TODO: Verify whether this is safe for logical and/or.
3587 if (!IsLogical) {
3588 if (Value *X = foldUnsignedUnderflowCheck(PredL, LHS0, LHS1, LHSOneUse,
3589 PredR, RHS0, RHS1, RHSOneUse,
3590 IsAnd, Q, Builder))
3591 return X;
3592 if (Value *X = foldUnsignedUnderflowCheck(PredL: PredR, LHS0: RHS0, LHS1: RHS1, LHSOneUse: RHSOneUse,
3593 PredR: PredL, RHS0: LHS0, RHS1: LHS1, RHSOneUse: LHSOneUse,
3594 IsAnd, Q, Builder))
3595 return X;
3596 }
3597
3598 // (icmp ne A, 0) | (icmp ne B, 0) --> (icmp ne (A|B), 0)
3599 // (icmp eq A, 0) & (icmp eq B, 0) --> (icmp eq (A|B), 0)
3600 // TODO: Remove this and below when foldLogOpOfMaskedICmps can handle undefs.
3601 if (PredL == (IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE) &&
3602 PredL.dropSameSign() == PredR.dropSameSign() &&
3603 match(V: LHS1, P: m_ZeroInt()) && match(V: RHS1, P: m_ZeroInt()) &&
3604 LHS0->getType() == RHS0->getType() &&
3605 (!IsLogical || isGuaranteedNotToBePoison(V: RHS0))) {
3606 Value *NewOr = Builder.CreateOr(LHS: LHS0, RHS: RHS0);
3607 return Builder.CreateICmp(P: PredL, LHS: NewOr,
3608 RHS: Constant::getNullValue(Ty: NewOr->getType()));
3609 }
3610
3611 // (icmp ne A, -1) | (icmp ne B, -1) --> (icmp ne (A&B), -1)
3612 // (icmp eq A, -1) & (icmp eq B, -1) --> (icmp eq (A&B), -1)
3613 if (PredL == (IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE) &&
3614 PredL.dropSameSign() == PredR.dropSameSign() &&
3615 match(V: LHS1, P: m_AllOnes()) && match(V: RHS1, P: m_AllOnes()) &&
3616 LHS0->getType() == RHS0->getType() &&
3617 (!IsLogical || isGuaranteedNotToBePoison(V: RHS0))) {
3618 Value *NewAnd = Builder.CreateAnd(LHS: LHS0, RHS: RHS0);
3619 return Builder.CreateICmp(P: PredL, LHS: NewAnd,
3620 RHS: Constant::getAllOnesValue(Ty: LHS0->getType()));
3621 }
3622
3623 if (!IsLogical)
3624 if (Value *V = foldAndOrOfICmpsWithPow2AndWithZero(
3625 Builder, PredL, LHS0, LHS1, LHSOneUse, PredR, RHS0, RHS1, RHSOneUse,
3626 IsAnd, Q))
3627 return V;
3628
3629 // This only handles icmp of constants: (icmp1 A, C1) | (icmp2 B, C2).
3630 if (!LHSC || !RHSC)
3631 return nullptr;
3632
3633 // Match naive pattern (and its inverted form) for checking if two values
3634 // share same sign. An example of the pattern:
3635 // (icmp slt (X & Y), 0) | (icmp sgt (X | Y), -1) -> (icmp sgt (X ^ Y), -1)
3636 // Inverted form (example):
3637 // (icmp slt (X | Y), 0) & (icmp sgt (X & Y), -1) -> (icmp slt (X ^ Y), 0)
3638 bool TrueIfSignedL, TrueIfSignedR;
3639 if (isSignBitCheck(Pred: PredL, RHS: *LHSC, TrueIfSigned&: TrueIfSignedL) &&
3640 isSignBitCheck(Pred: PredR, RHS: *RHSC, TrueIfSigned&: TrueIfSignedR) &&
3641 (RHS->hasOneUse() || LHS->hasOneUse())) {
3642 Value *X, *Y;
3643 if (IsAnd) {
3644 if ((TrueIfSignedL && !TrueIfSignedR &&
3645 match(V: LHS0, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
3646 match(V: RHS0, P: m_c_And(L: m_Specific(V: X), R: m_Specific(V: Y)))) ||
3647 (!TrueIfSignedL && TrueIfSignedR &&
3648 match(V: LHS0, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
3649 match(V: RHS0, P: m_c_Or(L: m_Specific(V: X), R: m_Specific(V: Y))))) {
3650 Value *NewXor = Builder.CreateXor(LHS: X, RHS: Y);
3651 return Builder.CreateIsNeg(Arg: NewXor);
3652 }
3653 } else {
3654 if ((TrueIfSignedL && !TrueIfSignedR &&
3655 match(V: LHS0, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
3656 match(V: RHS0, P: m_c_Or(L: m_Specific(V: X), R: m_Specific(V: Y)))) ||
3657 (!TrueIfSignedL && TrueIfSignedR &&
3658 match(V: LHS0, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
3659 match(V: RHS0, P: m_c_And(L: m_Specific(V: X), R: m_Specific(V: Y))))) {
3660 Value *NewXor = Builder.CreateXor(LHS: X, RHS: Y);
3661 return Builder.CreateIsNotNeg(Arg: NewXor);
3662 }
3663 }
3664 }
3665
3666 // (X & ExpMask) != 0 && (X & ExpMask) != ExpMask -> isnormal(X)
3667 // (X & ExpMask) == 0 || (X & ExpMask) == ExpMask -> !isnormal(X)
3668 Value *X;
3669 const APInt *MaskC;
3670 if (LHS0 == RHS0 && PredL.dropSameSign() == PredR.dropSameSign() &&
3671 PredL == (IsAnd ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ) &&
3672 !I.getFunction()->hasFnAttribute(Kind: Attribute::NoImplicitFloat) &&
3673 LHSOneUse && RHSOneUse &&
3674 match(V: LHS0, P: m_And(L: m_ElementWiseBitCast(Op: m_Value(V&: X)), R: m_APInt(Res&: MaskC))) &&
3675 X->getType()->getScalarType()->isIEEELikeFPTy() &&
3676 APFloat(X->getType()->getScalarType()->getFltSemantics(), *MaskC)
3677 .isPosInfinity() &&
3678 ((LHSC->isZero() && *RHSC == *MaskC) ||
3679 (RHSC->isZero() && *LHSC == *MaskC)))
3680 return Builder.createIsFPClass(FPNum: X, Test: IsAnd ? FPClassTest::fcNormal
3681 : ~FPClassTest::fcNormal);
3682
3683 return foldAndOrOfICmpsUsingRanges(PredL, LHS0, LHS1, LHSOneUse, PredR, RHS0,
3684 RHS1, RHSOneUse, IsAnd);
3685}
3686
3687/// If IsLogical is true, then the and/or is in select form and the transform
3688/// must be poison-safe.
3689Value *InstCombinerImpl::foldBooleanAndOr(Value *LHS, Value *RHS,
3690 Instruction &I, bool IsAnd,
3691 bool IsLogical) {
3692 if (!LHS->getType()->isIntOrIntVectorTy(BitWidth: 1))
3693 return nullptr;
3694
3695 // handle (roughly):
3696 // (icmp ne (A & B), C) | (icmp ne (A & D), E)
3697 // (icmp eq (A & B), C) & (icmp eq (A & D), E)
3698 if (Value *V = foldLogOpOfMaskedICmps(LHS, RHS, IsAnd, IsLogical, Builder,
3699 Q: SQ.getWithInstruction(I: &I)))
3700 return V;
3701
3702 if (Value *Res = foldAndOrOfICmps(LHS, RHS, I, IsAnd, IsLogical))
3703 return Res;
3704
3705 if (auto *LHSCmp = dyn_cast<FCmpInst>(Val: LHS))
3706 if (auto *RHSCmp = dyn_cast<FCmpInst>(Val: RHS))
3707 if (Value *Res = foldLogicOfFCmps(LHS: LHSCmp, RHS: RHSCmp, IsAnd, IsLogicalSelect: IsLogical))
3708 return Res;
3709
3710 if (Value *Res = foldEqOfParts(Cmp0: LHS, Cmp1: RHS, IsAnd))
3711 return Res;
3712
3713 return nullptr;
3714}
3715
3716static Value *foldOrOfInversions(BinaryOperator &I,
3717 InstCombiner::BuilderTy &Builder) {
3718 assert(I.getOpcode() == Instruction::Or &&
3719 "Simplification only supports or at the moment.");
3720
3721 Value *Cmp1, *Cmp2, *Cmp3, *Cmp4;
3722 if (!match(V: I.getOperand(i_nocapture: 0), P: m_And(L: m_Value(V&: Cmp1), R: m_Value(V&: Cmp2))) ||
3723 !match(V: I.getOperand(i_nocapture: 1), P: m_And(L: m_Value(V&: Cmp3), R: m_Value(V&: Cmp4))))
3724 return nullptr;
3725
3726 // Check if any two pairs of the and operations are inversions of each other.
3727 if (isKnownInversion(X: Cmp1, Y: Cmp3) && isKnownInversion(X: Cmp2, Y: Cmp4))
3728 return Builder.CreateXor(LHS: Cmp1, RHS: Cmp4);
3729 if (isKnownInversion(X: Cmp1, Y: Cmp4) && isKnownInversion(X: Cmp2, Y: Cmp3))
3730 return Builder.CreateXor(LHS: Cmp1, RHS: Cmp3);
3731
3732 return nullptr;
3733}
3734
3735/// Match \p V as "shufflevector -> bitcast" or "extractelement -> zext -> shl"
3736/// patterns, which extract vector elements and pack them in the same relative
3737/// positions.
3738///
3739/// \p Vec is the underlying vector being extracted from.
3740/// \p Mask is a bitmask identifying which packed elements are obtained from the
3741/// vector.
3742/// \p VecOffset is the vector element corresponding to index 0 of the
3743/// mask.
3744static bool matchSubIntegerPackFromVector(Value *V, Value *&Vec,
3745 int64_t &VecOffset,
3746 SmallBitVector &Mask,
3747 const DataLayout &DL) {
3748 // First try to match extractelement -> zext -> shl
3749 uint64_t VecIdx, ShlAmt;
3750 if (match(V, P: m_ShlOrSelf(L: m_ZExtOrSelf(Op: m_ExtractElt(Val: m_Value(V&: Vec),
3751 Idx: m_ConstantInt(V&: VecIdx))),
3752 R&: ShlAmt))) {
3753 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
3754 if (!VecTy)
3755 return false;
3756 auto *EltTy = dyn_cast<IntegerType>(Val: VecTy->getElementType());
3757 if (!EltTy)
3758 return false;
3759
3760 const unsigned EltBitWidth = EltTy->getBitWidth();
3761 const unsigned TargetBitWidth = V->getType()->getIntegerBitWidth();
3762 if (TargetBitWidth % EltBitWidth != 0 || ShlAmt % EltBitWidth != 0)
3763 return false;
3764 const unsigned TargetEltWidth = TargetBitWidth / EltBitWidth;
3765 const unsigned ShlEltAmt = ShlAmt / EltBitWidth;
3766
3767 const unsigned MaskIdx =
3768 DL.isLittleEndian() ? ShlEltAmt : TargetEltWidth - ShlEltAmt - 1;
3769
3770 VecOffset = static_cast<int64_t>(VecIdx) - static_cast<int64_t>(MaskIdx);
3771 Mask.resize(N: TargetEltWidth);
3772 Mask.set(MaskIdx);
3773 return true;
3774 }
3775
3776 // Now try to match a bitcasted subvector.
3777 Instruction *SrcVecI;
3778 if (!match(V, P: m_BitCast(Op: m_Instruction(I&: SrcVecI))))
3779 return false;
3780
3781 auto *SrcTy = dyn_cast<FixedVectorType>(Val: SrcVecI->getType());
3782 if (!SrcTy)
3783 return false;
3784
3785 Mask.resize(N: SrcTy->getNumElements());
3786
3787 // First check for a subvector obtained from a shufflevector.
3788 if (isa<ShuffleVectorInst>(Val: SrcVecI)) {
3789 Constant *ConstVec;
3790 ArrayRef<int> ShuffleMask;
3791 if (!match(V: SrcVecI, P: m_Shuffle(v1: m_Value(V&: Vec), v2: m_Constant(C&: ConstVec),
3792 mask: m_Mask(ShuffleMask))))
3793 return false;
3794
3795 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
3796 if (!VecTy)
3797 return false;
3798
3799 const unsigned NumVecElts = VecTy->getNumElements();
3800 bool FoundVecOffset = false;
3801 for (unsigned Idx = 0; Idx < ShuffleMask.size(); ++Idx) {
3802 if (ShuffleMask[Idx] == PoisonMaskElem)
3803 return false;
3804 const unsigned ShuffleIdx = ShuffleMask[Idx];
3805 if (ShuffleIdx >= NumVecElts) {
3806 const unsigned ConstIdx = ShuffleIdx - NumVecElts;
3807 auto *ConstElt =
3808 dyn_cast<ConstantInt>(Val: ConstVec->getAggregateElement(Elt: ConstIdx));
3809 if (!ConstElt || !ConstElt->isNullValue())
3810 return false;
3811 continue;
3812 }
3813
3814 if (FoundVecOffset) {
3815 if (VecOffset + Idx != ShuffleIdx)
3816 return false;
3817 } else {
3818 if (ShuffleIdx < Idx)
3819 return false;
3820 VecOffset = ShuffleIdx - Idx;
3821 FoundVecOffset = true;
3822 }
3823 Mask.set(Idx);
3824 }
3825 return FoundVecOffset;
3826 }
3827
3828 // Check for a subvector obtained as an (insertelement V, 0, idx)
3829 uint64_t InsertIdx;
3830 if (!match(V: SrcVecI,
3831 P: m_InsertElt(Val: m_Value(V&: Vec), Elt: m_Zero(), Idx: m_ConstantInt(V&: InsertIdx))))
3832 return false;
3833
3834 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
3835 if (!VecTy)
3836 return false;
3837 VecOffset = 0;
3838 bool AlreadyInsertedMaskedElt = Mask.test(Idx: InsertIdx);
3839 Mask.set();
3840 if (!AlreadyInsertedMaskedElt)
3841 Mask.reset(Idx: InsertIdx);
3842 return true;
3843}
3844
3845/// Try to fold the join of two scalar integers whose contents are packed
3846/// elements of the same vector.
3847static Instruction *foldIntegerPackFromVector(Instruction &I,
3848 InstCombiner::BuilderTy &Builder,
3849 const DataLayout &DL) {
3850 assert(I.getOpcode() == Instruction::Or);
3851 Value *LhsVec, *RhsVec;
3852 int64_t LhsVecOffset, RhsVecOffset;
3853 SmallBitVector Mask;
3854 if (!matchSubIntegerPackFromVector(V: I.getOperand(i: 0), Vec&: LhsVec, VecOffset&: LhsVecOffset,
3855 Mask, DL))
3856 return nullptr;
3857 if (!matchSubIntegerPackFromVector(V: I.getOperand(i: 1), Vec&: RhsVec, VecOffset&: RhsVecOffset,
3858 Mask, DL))
3859 return nullptr;
3860 if (LhsVec != RhsVec || LhsVecOffset != RhsVecOffset)
3861 return nullptr;
3862
3863 // Convert into shufflevector -> bitcast;
3864 const unsigned ZeroVecIdx =
3865 cast<FixedVectorType>(Val: LhsVec->getType())->getNumElements();
3866 SmallVector<int> ShuffleMask(Mask.size(), ZeroVecIdx);
3867 for (unsigned Idx : Mask.set_bits()) {
3868 assert(LhsVecOffset + Idx >= 0);
3869 ShuffleMask[Idx] = LhsVecOffset + Idx;
3870 }
3871
3872 Value *MaskedVec = Builder.CreateShuffleVector(
3873 V1: LhsVec, V2: Constant::getNullValue(Ty: LhsVec->getType()), Mask: ShuffleMask,
3874 Name: I.getName() + ".v");
3875 return CastInst::Create(Instruction::BitCast, S: MaskedVec, Ty: I.getType());
3876}
3877
3878/// Match \p V as "lshr -> mask -> zext -> shl".
3879///
3880/// \p Int is the underlying integer being extracted from.
3881/// \p Mask is a bitmask identifying which bits of the integer are being
3882/// extracted. \p Offset identifies which bit of the result \p V corresponds to
3883/// the least significant bit of \p Int
3884static bool matchZExtedSubInteger(Value *V, Value *&Int, APInt &Mask,
3885 uint64_t &Offset, bool &IsShlNUW,
3886 bool &IsShlNSW) {
3887 Value *ShlOp0;
3888 uint64_t ShlAmt = 0;
3889 if (!match(V, P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: ShlOp0), R: m_ConstantInt(V&: ShlAmt)))))
3890 return false;
3891
3892 IsShlNUW = cast<BinaryOperator>(Val: V)->hasNoUnsignedWrap();
3893 IsShlNSW = cast<BinaryOperator>(Val: V)->hasNoSignedWrap();
3894
3895 Value *ZExtOp0;
3896 if (!match(V: ShlOp0, P: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: ZExtOp0)))))
3897 return false;
3898
3899 Value *MaskedOp0;
3900 const APInt *ShiftedMaskConst = nullptr;
3901 if (!match(V: ZExtOp0, P: m_CombineOr(Ps: m_OneUse(SubPattern: m_And(L: m_Value(V&: MaskedOp0),
3902 R: m_APInt(Res&: ShiftedMaskConst))),
3903 Ps: m_Value(V&: MaskedOp0))))
3904 return false;
3905
3906 uint64_t LShrAmt = 0;
3907 if (!match(V: MaskedOp0,
3908 P: m_CombineOr(Ps: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: Int), R: m_ConstantInt(V&: LShrAmt))),
3909 Ps: m_Value(V&: Int))))
3910 return false;
3911
3912 if (LShrAmt > ShlAmt)
3913 return false;
3914 Offset = ShlAmt - LShrAmt;
3915
3916 Mask = ShiftedMaskConst ? ShiftedMaskConst->shl(shiftAmt: LShrAmt)
3917 : APInt::getBitsSetFrom(
3918 numBits: Int->getType()->getScalarSizeInBits(), loBit: LShrAmt);
3919
3920 return true;
3921}
3922
3923/// Try to fold the join of two scalar integers whose bits are unpacked and
3924/// zexted from the same source integer.
3925static Value *foldIntegerRepackThroughZExt(Value *Lhs, Value *Rhs,
3926 InstCombiner::BuilderTy &Builder) {
3927
3928 Value *LhsInt, *RhsInt;
3929 APInt LhsMask, RhsMask;
3930 uint64_t LhsOffset, RhsOffset;
3931 bool IsLhsShlNUW, IsLhsShlNSW, IsRhsShlNUW, IsRhsShlNSW;
3932 if (!matchZExtedSubInteger(V: Lhs, Int&: LhsInt, Mask&: LhsMask, Offset&: LhsOffset, IsShlNUW&: IsLhsShlNUW,
3933 IsShlNSW&: IsLhsShlNSW))
3934 return nullptr;
3935 if (!matchZExtedSubInteger(V: Rhs, Int&: RhsInt, Mask&: RhsMask, Offset&: RhsOffset, IsShlNUW&: IsRhsShlNUW,
3936 IsShlNSW&: IsRhsShlNSW))
3937 return nullptr;
3938 if (LhsInt != RhsInt || LhsOffset != RhsOffset)
3939 return nullptr;
3940
3941 APInt Mask = LhsMask | RhsMask;
3942
3943 Type *DestTy = Lhs->getType();
3944 Value *Res = Builder.CreateShl(
3945 LHS: Builder.CreateZExt(
3946 V: Builder.CreateAnd(LHS: LhsInt, RHS: Mask, Name: LhsInt->getName() + ".mask"), DestTy,
3947 Name: LhsInt->getName() + ".zext"),
3948 RHS: ConstantInt::get(Ty: DestTy, V: LhsOffset), Name: "", HasNUW: IsLhsShlNUW && IsRhsShlNUW,
3949 HasNSW: IsLhsShlNSW && IsRhsShlNSW);
3950 Res->takeName(V: Lhs);
3951 return Res;
3952}
3953
3954// A decomposition of ((X & Mask) * Factor). The NUW / NSW bools
3955// track these properities for preservation. Note that we can decompose
3956// equivalent select form of this expression (e.g. (!(X & Mask) ? 0 : Mask *
3957// Factor))
3958struct DecomposedBitMaskMul {
3959 Value *X;
3960 APInt Factor;
3961 APInt Mask;
3962 bool NUW;
3963 bool NSW;
3964
3965 bool isCombineableWith(const DecomposedBitMaskMul Other) {
3966 return X == Other.X && !Mask.intersects(RHS: Other.Mask) &&
3967 Factor == Other.Factor;
3968 }
3969};
3970
3971static std::optional<DecomposedBitMaskMul> matchBitmaskMul(Value *V) {
3972 Instruction *Op = dyn_cast<Instruction>(Val: V);
3973 if (!Op)
3974 return std::nullopt;
3975
3976 // Decompose (A & N) * C) into BitMaskMul
3977 Value *Original = nullptr;
3978 const APInt *Mask = nullptr;
3979 const APInt *MulConst = nullptr;
3980 if (match(V: Op, P: m_Mul(L: m_And(L: m_Value(V&: Original), R: m_APInt(Res&: Mask)),
3981 R: m_APInt(Res&: MulConst)))) {
3982 if (MulConst->isZero() || Mask->isZero())
3983 return std::nullopt;
3984
3985 return std::optional<DecomposedBitMaskMul>(
3986 {.X: Original, .Factor: *MulConst, .Mask: *Mask,
3987 .NUW: cast<BinaryOperator>(Val: Op)->hasNoUnsignedWrap(),
3988 .NSW: cast<BinaryOperator>(Val: Op)->hasNoSignedWrap()});
3989 }
3990
3991 Value *Cond = nullptr;
3992 const APInt *EqZero = nullptr, *NeZero = nullptr;
3993
3994 // Decompose ((A & N) ? 0 : N * C) into BitMaskMul
3995 if (match(V: Op, P: m_Select(C: m_Value(V&: Cond), L: m_APInt(Res&: EqZero), R: m_APInt(Res&: NeZero)))) {
3996 auto ICmpDecompose =
3997 decomposeBitTest(Cond, /*LookThroughTrunc=*/true,
3998 /*AllowNonZeroC=*/false, /*DecomposeBitMask=*/DecomposeAnd: true);
3999 if (!ICmpDecompose.has_value())
4000 return std::nullopt;
4001
4002 // decomposeBitTest may provide a scalar bit test for a vector select.
4003 // Ensure the types match.
4004 if (ICmpDecompose->X->getType() != V->getType())
4005 return std::nullopt;
4006
4007 assert(ICmpInst::isEquality(ICmpDecompose->Pred) &&
4008 ICmpDecompose->C.isZero());
4009
4010 if (ICmpDecompose->Pred == ICmpInst::ICMP_NE)
4011 std::swap(a&: EqZero, b&: NeZero);
4012
4013 if (!EqZero->isZero() || NeZero->isZero())
4014 return std::nullopt;
4015
4016 if (!ICmpDecompose->Mask.isPowerOf2() || ICmpDecompose->Mask.isZero())
4017 return std::nullopt;
4018
4019 if (!NeZero->urem(RHS: ICmpDecompose->Mask).isZero())
4020 return std::nullopt;
4021
4022 return std::optional<DecomposedBitMaskMul>(
4023 {.X: ICmpDecompose->X, .Factor: NeZero->udiv(RHS: ICmpDecompose->Mask),
4024 .Mask: ICmpDecompose->Mask, /*NUW=*/false, /*NSW=*/false});
4025 }
4026
4027 return std::nullopt;
4028}
4029
4030/// (A & N) * C + (A & M) * C -> (A & (N + M)) & C
4031/// This also accepts the equivalent select form of (A & N) * C
4032/// expressions i.e. !(A & N) ? 0 : N * C)
4033static Value *foldBitmaskMul(Value *Op0, Value *Op1,
4034 InstCombiner::BuilderTy &Builder) {
4035 auto Decomp1 = matchBitmaskMul(V: Op1);
4036 if (!Decomp1)
4037 return nullptr;
4038
4039 auto Decomp0 = matchBitmaskMul(V: Op0);
4040 if (!Decomp0)
4041 return nullptr;
4042
4043 if (Decomp0->isCombineableWith(Other: *Decomp1)) {
4044 Value *NewAnd = Builder.CreateAnd(
4045 LHS: Decomp0->X,
4046 RHS: ConstantInt::get(Ty: Decomp0->X->getType(), V: Decomp0->Mask + Decomp1->Mask));
4047
4048 return Builder.CreateMul(
4049 LHS: NewAnd, RHS: ConstantInt::get(Ty: NewAnd->getType(), V: Decomp1->Factor), Name: "",
4050 HasNUW: Decomp0->NUW && Decomp1->NUW, HasNSW: Decomp0->NSW && Decomp1->NSW);
4051 }
4052
4053 return nullptr;
4054}
4055
4056Value *InstCombinerImpl::foldDisjointOr(Value *LHS, Value *RHS) {
4057 if (Value *Res = foldBitmaskMul(Op0: LHS, Op1: RHS, Builder))
4058 return Res;
4059 if (Value *Res = foldIntegerRepackThroughZExt(Lhs: LHS, Rhs: RHS, Builder))
4060 return Res;
4061
4062 return nullptr;
4063}
4064
4065Value *InstCombinerImpl::reassociateDisjointOr(Value *LHS, Value *RHS) {
4066
4067 Value *X, *Y;
4068 if (match(V: RHS, P: m_OneUse(SubPattern: m_DisjointOr(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
4069 if (Value *Res = foldDisjointOr(LHS, RHS: X))
4070 return Builder.CreateDisjointOr(LHS: Res, RHS: Y);
4071 if (Value *Res = foldDisjointOr(LHS, RHS: Y))
4072 return Builder.CreateDisjointOr(LHS: Res, RHS: X);
4073 }
4074
4075 if (match(V: LHS, P: m_OneUse(SubPattern: m_DisjointOr(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
4076 if (Value *Res = foldDisjointOr(LHS: X, RHS))
4077 return Builder.CreateDisjointOr(LHS: Res, RHS: Y);
4078 if (Value *Res = foldDisjointOr(LHS: Y, RHS))
4079 return Builder.CreateDisjointOr(LHS: Res, RHS: X);
4080 }
4081
4082 return nullptr;
4083}
4084
4085/// Fold Res, Overflow = (umul.with.overflow x c1); (or Overflow (ugt Res c2))
4086/// --> (ugt x (c2/c1)). This code checks whether a multiplication of two
4087/// unsigned numbers (one is a constant) is mathematically greater than a
4088/// second constant.
4089static Value *foldOrUnsignedUMulOverflowICmp(BinaryOperator &I,
4090 InstCombiner::BuilderTy &Builder,
4091 const DataLayout &DL) {
4092 Value *WOV, *X;
4093 const APInt *C1, *C2;
4094 if (match(V: &I,
4095 P: m_c_Or(L: m_ExtractValue<1>(
4096 V: m_Value(V&: WOV, P: m_Intrinsic<Intrinsic::umul_with_overflow>(
4097 Ops: m_Value(V&: X), Ops: m_APInt(Res&: C1)))),
4098 R: m_OneUse(SubPattern: m_SpecificCmp(MatchPred: ICmpInst::ICMP_UGT,
4099 L: m_ExtractValue<0>(V: m_Deferred(V: WOV)),
4100 R: m_APInt(Res&: C2))))) &&
4101 !C1->isZero()) {
4102 Constant *NewC = ConstantInt::get(Ty: X->getType(), V: C2->udiv(RHS: *C1));
4103 return Builder.CreateICmp(P: ICmpInst::ICMP_UGT, LHS: X, RHS: NewC);
4104 }
4105 return nullptr;
4106}
4107
4108/// Fold select(X >s 0, 0, -X) | smax(X, 0) --> abs(X)
4109/// select(X <s 0, -X, 0) | smax(X, 0) --> abs(X)
4110static Value *FoldOrOfSelectSmaxToAbs(BinaryOperator &I,
4111 InstCombiner::BuilderTy &Builder) {
4112 Value *X;
4113 Value *Sel;
4114 if (match(V: &I,
4115 P: m_c_Or(L: m_Value(V&: Sel), R: m_OneUse(SubPattern: m_SMax(Op0: m_Value(V&: X), Op1: m_ZeroInt()))))) {
4116 auto NegX = m_Neg(V: m_Specific(V: X));
4117 if (match(V: Sel, P: m_Select(C: m_SpecificICmp(MatchPred: ICmpInst::ICMP_SGT, L: m_Specific(V: X),
4118 R: m_ZeroInt()),
4119 L: m_ZeroInt(), R: NegX)) ||
4120 match(V: Sel, P: m_Select(C: m_SpecificICmp(MatchPred: ICmpInst::ICMP_SLT, L: m_Specific(V: X),
4121 R: m_ZeroInt()),
4122 L: NegX, R: m_ZeroInt())))
4123 return Builder.CreateBinaryIntrinsic(ID: Intrinsic::abs, LHS: X,
4124 RHS: Builder.getFalse());
4125 }
4126 return nullptr;
4127}
4128
4129Instruction *InstCombinerImpl::FoldOrOfLogicalAnds(Value *Op0, Value *Op1) {
4130 Value *C, *A, *B;
4131 // (C && A) || (!C && B)
4132 // (C && A) || (B && !C)
4133 // (A && C) || (!C && B)
4134 // (A && C) || (B && !C) (may require freeze)
4135 //
4136 // => select C, A, B
4137 if (match(V: Op1, P: m_c_LogicalAnd(L: m_Not(V: m_Value(V&: C)), R: m_Value(V&: B))) &&
4138 match(V: Op0, P: m_c_LogicalAnd(L: m_Specific(V: C), R: m_Value(V&: A)))) {
4139 auto *SelOp0 = dyn_cast<SelectInst>(Val: Op0);
4140 auto *SelOp1 = dyn_cast<SelectInst>(Val: Op1);
4141
4142 bool MayNeedFreeze = SelOp0 && SelOp1 &&
4143 match(V: SelOp1->getTrueValue(),
4144 P: m_Not(V: m_Specific(V: SelOp0->getTrueValue())));
4145 if (MayNeedFreeze)
4146 C = Builder.CreateFreeze(V: C);
4147 Value *C2 = nullptr, *A2 = nullptr, *B2 = nullptr;
4148 if (match(V: Op0, P: m_LogicalAnd(L: m_Specific(V: C), R: m_Value(V&: A2))) && SelOp0)
4149 return SelectInst::Create(C, S1: A, S2: B, NameStr: "", InsertBefore: nullptr, MDFrom: SelOp0);
4150 if (match(V: Op1, P: m_LogicalAnd(L: m_Not(V: m_Value(V&: C2)), R: m_Value(V&: B2))) && SelOp1) {
4151 SelectInst *NewSI = SelectInst::Create(C, S1: A, S2: B, NameStr: "", InsertBefore: nullptr, MDFrom: SelOp1);
4152 NewSI->swapProfMetadata();
4153 return NewSI;
4154 }
4155 return createSelectInstWithUnknownProfile(C, S1: A, S2: B);
4156 }
4157
4158 // (!C && A) || (C && B)
4159 // (A && !C) || (C && B)
4160 // (!C && A) || (B && C)
4161 // (A && !C) || (B && C) (may require freeze)
4162 //
4163 // => select C, B, A
4164 if (match(V: Op0, P: m_c_LogicalAnd(L: m_Not(V: m_Value(V&: C)), R: m_Value(V&: A))) &&
4165 match(V: Op1, P: m_c_LogicalAnd(L: m_Specific(V: C), R: m_Value(V&: B)))) {
4166 auto *SelOp0 = dyn_cast<SelectInst>(Val: Op0);
4167 auto *SelOp1 = dyn_cast<SelectInst>(Val: Op1);
4168 bool MayNeedFreeze = SelOp0 && SelOp1 &&
4169 match(V: SelOp0->getTrueValue(),
4170 P: m_Not(V: m_Specific(V: SelOp1->getTrueValue())));
4171 if (MayNeedFreeze)
4172 C = Builder.CreateFreeze(V: C);
4173 Value *C2 = nullptr, *A2 = nullptr, *B2 = nullptr;
4174 if (match(V: Op0, P: m_LogicalAnd(L: m_Not(V: m_Value(V&: C2)), R: m_Value(V&: A2))) && SelOp0) {
4175 SelectInst *NewSI = SelectInst::Create(C, S1: B, S2: A, NameStr: "", InsertBefore: nullptr, MDFrom: SelOp0);
4176 NewSI->swapProfMetadata();
4177 return NewSI;
4178 }
4179 if (match(V: Op1, P: m_LogicalAnd(L: m_Specific(V: C), R: m_Value(V&: B2))) && SelOp1)
4180 return SelectInst::Create(C, S1: B, S2: A, NameStr: "", InsertBefore: nullptr, MDFrom: SelOp1);
4181 return createSelectInstWithUnknownProfile(C, S1: B, S2: A);
4182 }
4183
4184 return nullptr;
4185}
4186
4187// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
4188// here. We should standardize that construct where it is needed or choose some
4189// other way to ensure that commutated variants of patterns are not missed.
4190Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
4191 if (Value *V = simplifyOrInst(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1),
4192 Q: SQ.getWithInstruction(I: &I)))
4193 return replaceInstUsesWith(I, V);
4194
4195 if (SimplifyAssociativeOrCommutative(I))
4196 return &I;
4197
4198 if (Instruction *X = foldVectorBinop(Inst&: I))
4199 return X;
4200
4201 if (Instruction *Phi = foldBinopWithPhiOperands(BO&: I))
4202 return Phi;
4203
4204 // See if we can simplify any instructions used by the instruction whose sole
4205 // purpose is to compute bits we don't care about.
4206 if (SimplifyDemandedInstructionBits(Inst&: I))
4207 return &I;
4208
4209 // Do this before using distributive laws to catch simple and/or/not patterns.
4210 if (Instruction *Xor = foldOrToXor(I, Builder))
4211 return Xor;
4212
4213 if (Instruction *X = foldComplexAndOrPatterns(I, Builder))
4214 return X;
4215
4216 if (Instruction *X = foldIntegerPackFromVector(I, Builder, DL))
4217 return X;
4218
4219 // (A & B) | (C & D) -> A ^ D where A == ~C && B == ~D
4220 // (A & B) | (C & D) -> A ^ C where A == ~D && B == ~C
4221 if (Value *V = foldOrOfInversions(I, Builder))
4222 return replaceInstUsesWith(I, V);
4223
4224 // (A&B)|(A&C) -> A&(B|C) etc
4225 if (Value *V = foldUsingDistributiveLaws(I))
4226 return replaceInstUsesWith(I, V);
4227
4228 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
4229 Type *Ty = I.getType();
4230 if (Ty->isIntOrIntVectorTy(BitWidth: 1)) {
4231 if (auto *SI0 = dyn_cast<SelectInst>(Val: Op0)) {
4232 if (auto *R =
4233 foldAndOrOfSelectUsingImpliedCond(Op: Op1, SI&: *SI0, /* IsAnd */ false))
4234 return R;
4235 }
4236 if (auto *SI1 = dyn_cast<SelectInst>(Val: Op1)) {
4237 if (auto *R =
4238 foldAndOrOfSelectUsingImpliedCond(Op: Op0, SI&: *SI1, /* IsAnd */ false))
4239 return R;
4240 }
4241 }
4242
4243 if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
4244 return FoldedLogic;
4245
4246 if (Instruction *FoldedLogic = foldBinOpSelectBinOp(Op&: I))
4247 return FoldedLogic;
4248
4249 if (Instruction *BitOp = matchBSwapOrBitReverse(I, /*MatchBSwaps*/ true,
4250 /*MatchBitReversals*/ true))
4251 return BitOp;
4252
4253 if (Instruction *Funnel = matchFunnelShift(Or&: I, IC&: *this))
4254 return Funnel;
4255
4256 if (Value *Concat = matchOrConcat(Or&: I, Builder))
4257 return replaceInstUsesWith(I, V: Concat);
4258
4259 if (Instruction *R = foldBinOpShiftWithShift(I))
4260 return R;
4261
4262 if (Instruction *R = tryFoldInstWithCtpopWithNot(I: &I))
4263 return R;
4264
4265 if (cast<PossiblyDisjointInst>(Val&: I).isDisjoint()) {
4266 if (Instruction *R =
4267 foldAddLikeCommutative(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1),
4268 /*NSW=*/true, /*NUW=*/true))
4269 return R;
4270 if (Instruction *R =
4271 foldAddLikeCommutative(LHS: I.getOperand(i_nocapture: 1), RHS: I.getOperand(i_nocapture: 0),
4272 /*NSW=*/true, /*NUW=*/true))
4273 return R;
4274
4275 if (Value *Res = foldDisjointOr(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1)))
4276 return replaceInstUsesWith(I, V: Res);
4277
4278 if (Value *Res = reassociateDisjointOr(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1)))
4279 return replaceInstUsesWith(I, V: Res);
4280 }
4281
4282 Value *X, *Y;
4283 const APInt *CV;
4284 if (match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: X), R: m_APInt(Res&: CV))), R: m_Value(V&: Y))) &&
4285 !CV->isAllOnes() && MaskedValueIsZero(V: Y, Mask: *CV, CtxI: &I)) {
4286 // (X ^ C) | Y -> (X | Y) ^ C iff Y & C == 0
4287 // The check for a 'not' op is for efficiency (if Y is known zero --> ~X).
4288 Value *Or = Builder.CreateOr(LHS: X, RHS: Y);
4289 return BinaryOperator::CreateXor(V1: Or, V2: ConstantInt::get(Ty, V: *CV));
4290 }
4291
4292 // If the operands have no common bits set:
4293 // or (mul X, Y), X --> add (mul X, Y), X --> mul X, (Y + 1)
4294 if (match(V: &I, P: m_c_DisjointOr(L: m_OneUse(SubPattern: m_Mul(L: m_Value(V&: X), R: m_Value(V&: Y))),
4295 R: m_Deferred(V: X)))) {
4296 Value *IncrementY = Builder.CreateAdd(LHS: Y, RHS: ConstantInt::get(Ty, V: 1));
4297 return BinaryOperator::CreateMul(V1: X, V2: IncrementY);
4298 }
4299
4300 // Canonicalization to achieve lowering to Bit Manipulation Instructions (BMI)
4301 // ~X | (X-1) => ~(X & -X)
4302 Value *Op;
4303 if (match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_Not(V: m_Value(V&: Op))),
4304 R: m_OneUse(SubPattern: m_Add(L: m_Deferred(V: Op), R: m_AllOnes()))))) {
4305 Value *NegX = Builder.CreateNeg(V: Op);
4306 Value *And = Builder.CreateAnd(LHS: Op, RHS: NegX);
4307 return BinaryOperator::CreateNot(Op: And);
4308 }
4309
4310 // (C && A) || (C && B) => select C, A, B (and similar cases)
4311 //
4312 // Note: This is the same transformation used in `foldSelectOfBools`,
4313 // except that it's an `or` instead of `select`.
4314 if (I.getType()->isIntOrIntVectorTy(BitWidth: 1) &&
4315 (Op0->hasOneUse() || Op1->hasOneUse())) {
4316 if (Instruction *V = FoldOrOfLogicalAnds(Op0, Op1)) {
4317 return V;
4318 }
4319 }
4320
4321 // (A & C) | (B & D)
4322 Value *A, *B, *C, *D;
4323 if (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: C))) &&
4324 match(V: Op1, P: m_And(L: m_Value(V&: B), R: m_Value(V&: D)))) {
4325
4326 // (A & C0) | (B & C1)
4327 const APInt *C0, *C1;
4328 if (match(V: C, P: m_APInt(Res&: C0)) && match(V: D, P: m_APInt(Res&: C1))) {
4329 Value *X;
4330 if (*C0 == ~*C1) {
4331 // ((X | B) & MaskC) | (B & ~MaskC) -> (X & MaskC) | B
4332 if (match(V: A, P: m_c_Or(L: m_Value(V&: X), R: m_Specific(V: B))))
4333 return BinaryOperator::CreateOr(V1: Builder.CreateAnd(LHS: X, RHS: *C0), V2: B);
4334 // (A & MaskC) | ((X | A) & ~MaskC) -> (X & ~MaskC) | A
4335 if (match(V: B, P: m_c_Or(L: m_Specific(V: A), R: m_Value(V&: X))))
4336 return BinaryOperator::CreateOr(V1: Builder.CreateAnd(LHS: X, RHS: *C1), V2: A);
4337
4338 // ((X ^ B) & MaskC) | (B & ~MaskC) -> (X & MaskC) ^ B
4339 if (match(V: A, P: m_c_Xor(L: m_Value(V&: X), R: m_Specific(V: B))))
4340 return BinaryOperator::CreateXor(V1: Builder.CreateAnd(LHS: X, RHS: *C0), V2: B);
4341 // (A & MaskC) | ((X ^ A) & ~MaskC) -> (X & ~MaskC) ^ A
4342 if (match(V: B, P: m_c_Xor(L: m_Specific(V: A), R: m_Value(V&: X))))
4343 return BinaryOperator::CreateXor(V1: Builder.CreateAnd(LHS: X, RHS: *C1), V2: A);
4344 }
4345
4346 if ((*C0 & *C1).isZero()) {
4347 // ((X | B) & C0) | (B & C1) --> (X | B) & (C0 | C1)
4348 // iff (C0 & C1) == 0 and (X & ~C0) == 0
4349 if (match(V: A, P: m_c_Or(L: m_Value(V&: X), R: m_Specific(V: B))) &&
4350 MaskedValueIsZero(V: X, Mask: ~*C0, CtxI: &I)) {
4351 Constant *C01 = ConstantInt::get(Ty, V: *C0 | *C1);
4352 return BinaryOperator::CreateAnd(V1: A, V2: C01);
4353 }
4354 // (A & C0) | ((X | A) & C1) --> (X | A) & (C0 | C1)
4355 // iff (C0 & C1) == 0 and (X & ~C1) == 0
4356 if (match(V: B, P: m_c_Or(L: m_Value(V&: X), R: m_Specific(V: A))) &&
4357 MaskedValueIsZero(V: X, Mask: ~*C1, CtxI: &I)) {
4358 Constant *C01 = ConstantInt::get(Ty, V: *C0 | *C1);
4359 return BinaryOperator::CreateAnd(V1: B, V2: C01);
4360 }
4361 // ((X | C2) & C0) | ((X | C3) & C1) --> (X | C2 | C3) & (C0 | C1)
4362 // iff (C0 & C1) == 0 and (C2 & ~C0) == 0 and (C3 & ~C1) == 0.
4363 const APInt *C2, *C3;
4364 if (match(V: A, P: m_Or(L: m_Value(V&: X), R: m_APInt(Res&: C2))) &&
4365 match(V: B, P: m_Or(L: m_Specific(V: X), R: m_APInt(Res&: C3))) &&
4366 (*C2 & ~*C0).isZero() && (*C3 & ~*C1).isZero()) {
4367 Value *Or = Builder.CreateOr(LHS: X, RHS: *C2 | *C3, Name: "bitfield");
4368 Constant *C01 = ConstantInt::get(Ty, V: *C0 | *C1);
4369 return BinaryOperator::CreateAnd(V1: Or, V2: C01);
4370 }
4371 }
4372
4373 // ((trunc (lshr X, S)) & C0) | ((lshr (trunc X), S) & C1)
4374 // --> ((trunc (lshr X, S)) & (C0 | C1)) (and similar cases)
4375 // A = trunc (lshr X, S) B = lshr (trunc X), S
4376 const APInt *ShiftAmt;
4377 if (match(V: A, P: m_Trunc(Op: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: ShiftAmt)))) &&
4378 match(V: B, P: m_LShr(L: m_Trunc(Op: m_Specific(V: X)), R: m_SpecificInt(V: *ShiftAmt))) &&
4379 ShiftAmt->ult(RHS: A->getType()->getScalarSizeInBits()) &&
4380 C1->isIntN(N: A->getType()->getScalarSizeInBits() -
4381 ShiftAmt->getZExtValue())) {
4382 return BinaryOperator::CreateAnd(
4383 V1: A, V2: ConstantInt::get(Ty: I.getType(), V: *C0 | *C1));
4384 }
4385 // A = lshr (trunc X), S
4386 // B = trunc (lshr X, S)
4387 if (match(V: B, P: m_Trunc(Op: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: ShiftAmt)))) &&
4388 match(V: A, P: m_LShr(L: m_Trunc(Op: m_Specific(V: X)), R: m_SpecificInt(V: *ShiftAmt))) &&
4389 ShiftAmt->ult(RHS: A->getType()->getScalarSizeInBits()) &&
4390 C0->isIntN(N: A->getType()->getScalarSizeInBits() -
4391 ShiftAmt->getZExtValue())) {
4392 return BinaryOperator::CreateAnd(
4393 V1: B, V2: ConstantInt::get(Ty: I.getType(), V: *C0 | *C1));
4394 }
4395 }
4396
4397 // Don't try to form a select if it's unlikely that we'll get rid of at
4398 // least one of the operands. A select is generally more expensive than the
4399 // 'or' that it is replacing.
4400 if (Op0->hasOneUse() || Op1->hasOneUse()) {
4401 // (Cond & C) | (~Cond & D) -> Cond ? C : D, and commuted variants.
4402 if (Value *V = matchSelectFromAndOr(A, B: C, C: B, D))
4403 return replaceInstUsesWith(I, V);
4404 if (Value *V = matchSelectFromAndOr(A, B: C, C: D, D: B))
4405 return replaceInstUsesWith(I, V);
4406 if (Value *V = matchSelectFromAndOr(A: C, B: A, C: B, D))
4407 return replaceInstUsesWith(I, V);
4408 if (Value *V = matchSelectFromAndOr(A: C, B: A, C: D, D: B))
4409 return replaceInstUsesWith(I, V);
4410 if (Value *V = matchSelectFromAndOr(A: B, B: D, C: A, D: C))
4411 return replaceInstUsesWith(I, V);
4412 if (Value *V = matchSelectFromAndOr(A: B, B: D, C, D: A))
4413 return replaceInstUsesWith(I, V);
4414 if (Value *V = matchSelectFromAndOr(A: D, B, C: A, D: C))
4415 return replaceInstUsesWith(I, V);
4416 if (Value *V = matchSelectFromAndOr(A: D, B, C, D: A))
4417 return replaceInstUsesWith(I, V);
4418 }
4419 }
4420
4421 if (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: C))) &&
4422 match(V: Op1, P: m_Not(V: m_Or(L: m_Value(V&: B), R: m_Value(V&: D)))) &&
4423 (Op0->hasOneUse() || Op1->hasOneUse())) {
4424 // (Cond & C) | ~(Cond | D) -> Cond ? C : ~D
4425 if (Value *V = matchSelectFromAndOr(A, B: C, C: B, D, InvertFalseVal: true))
4426 return replaceInstUsesWith(I, V);
4427 if (Value *V = matchSelectFromAndOr(A, B: C, C: D, D: B, InvertFalseVal: true))
4428 return replaceInstUsesWith(I, V);
4429 if (Value *V = matchSelectFromAndOr(A: C, B: A, C: B, D, InvertFalseVal: true))
4430 return replaceInstUsesWith(I, V);
4431 if (Value *V = matchSelectFromAndOr(A: C, B: A, C: D, D: B, InvertFalseVal: true))
4432 return replaceInstUsesWith(I, V);
4433 }
4434
4435 // (A ^ B) | ((B ^ C) ^ A) -> (A ^ B) | C
4436 if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))))
4437 if (match(V: Op1,
4438 P: m_c_Xor(L: m_c_Xor(L: m_Specific(V: B), R: m_Value(V&: C)), R: m_Specific(V: A))) ||
4439 match(V: Op1, P: m_c_Xor(L: m_c_Xor(L: m_Specific(V: A), R: m_Value(V&: C)), R: m_Specific(V: B))))
4440 return BinaryOperator::CreateOr(V1: Op0, V2: C);
4441
4442 // ((B ^ C) ^ A) | (A ^ B) -> (A ^ B) | C
4443 if (match(V: Op1, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))))
4444 if (match(V: Op0,
4445 P: m_c_Xor(L: m_c_Xor(L: m_Specific(V: B), R: m_Value(V&: C)), R: m_Specific(V: A))) ||
4446 match(V: Op0, P: m_c_Xor(L: m_c_Xor(L: m_Specific(V: A), R: m_Value(V&: C)), R: m_Specific(V: B))))
4447 return BinaryOperator::CreateOr(V1: Op1, V2: C);
4448
4449 if (Instruction *DeMorgan = matchDeMorgansLaws(I, IC&: *this))
4450 return DeMorgan;
4451
4452 // Canonicalize xor to the RHS.
4453 bool SwappedForXor = false;
4454 if (match(V: Op0, P: m_Xor(L: m_Value(), R: m_Value()))) {
4455 std::swap(a&: Op0, b&: Op1);
4456 SwappedForXor = true;
4457 }
4458
4459 if (match(V: Op1, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B)))) {
4460 // (A | ?) | (A ^ B) --> (A | ?) | B
4461 // (B | ?) | (A ^ B) --> (B | ?) | A
4462 if (match(V: Op0, P: m_c_Or(L: m_Specific(V: A), R: m_Value())))
4463 return BinaryOperator::CreateOr(V1: Op0, V2: B);
4464 if (match(V: Op0, P: m_c_Or(L: m_Specific(V: B), R: m_Value())))
4465 return BinaryOperator::CreateOr(V1: Op0, V2: A);
4466
4467 // (A & B) | (A ^ B) --> A | B
4468 // (B & A) | (A ^ B) --> A | B
4469 if (match(V: Op0, P: m_c_And(L: m_Specific(V: A), R: m_Specific(V: B))))
4470 return BinaryOperator::CreateOr(V1: A, V2: B);
4471
4472 // ~A | (A ^ B) --> ~(A & B)
4473 // ~B | (A ^ B) --> ~(A & B)
4474 // The swap above should always make Op0 the 'not'.
4475 if ((Op0->hasOneUse() || Op1->hasOneUse()) &&
4476 (match(V: Op0, P: m_Not(V: m_Specific(V: A))) || match(V: Op0, P: m_Not(V: m_Specific(V: B)))))
4477 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: A, RHS: B));
4478
4479 // Same as above, but peek through an 'and' to the common operand:
4480 // ~(A & ?) | (A ^ B) --> ~((A & ?) & B)
4481 // ~(B & ?) | (A ^ B) --> ~((B & ?) & A)
4482 Instruction *And;
4483 if ((Op0->hasOneUse() || Op1->hasOneUse()) &&
4484 match(V: Op0,
4485 P: m_Not(V: m_Instruction(I&: And, P: m_c_And(L: m_Specific(V: A), R: m_Value())))))
4486 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: And, RHS: B));
4487 if ((Op0->hasOneUse() || Op1->hasOneUse()) &&
4488 match(V: Op0,
4489 P: m_Not(V: m_Instruction(I&: And, P: m_c_And(L: m_Specific(V: B), R: m_Value())))))
4490 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: And, RHS: A));
4491
4492 // (~A | C) | (A ^ B) --> ~(A & B) | C
4493 // (~B | C) | (A ^ B) --> ~(A & B) | C
4494 if (Op0->hasOneUse() && Op1->hasOneUse() &&
4495 (match(V: Op0, P: m_c_Or(L: m_Not(V: m_Specific(V: A)), R: m_Value(V&: C))) ||
4496 match(V: Op0, P: m_c_Or(L: m_Not(V: m_Specific(V: B)), R: m_Value(V&: C))))) {
4497 Value *Nand = Builder.CreateNot(V: Builder.CreateAnd(LHS: A, RHS: B), Name: "nand");
4498 return BinaryOperator::CreateOr(V1: Nand, V2: C);
4499 }
4500 }
4501
4502 if (SwappedForXor)
4503 std::swap(a&: Op0, b&: Op1);
4504
4505 if (Value *Res =
4506 foldBooleanAndOr(LHS: Op0, RHS: Op1, I, /*IsAnd=*/false, /*IsLogical=*/false))
4507 return replaceInstUsesWith(I, V: Res);
4508
4509 if (match(V: Op1, P: m_OneUse(SubPattern: m_LogicalOr(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
4510 bool IsLogical = isa<SelectInst>(Val: Op1);
4511 if (auto *V = reassociateBooleanAndOr(LHS: Op0, X, Y, I, /*IsAnd=*/false,
4512 /*RHSIsLogical=*/IsLogical))
4513 return replaceInstUsesWith(I, V);
4514 }
4515 if (match(V: Op0, P: m_OneUse(SubPattern: m_LogicalOr(L: m_Value(V&: X), R: m_Value(V&: Y))))) {
4516 bool IsLogical = isa<SelectInst>(Val: Op0);
4517 if (auto *V = reassociateBooleanAndOr(LHS: Op1, X, Y, I, /*IsAnd=*/false,
4518 /*RHSIsLogical=*/IsLogical))
4519 return replaceInstUsesWith(I, V);
4520 }
4521
4522 if (Instruction *FoldedFCmps = reassociateFCmps(BO&: I, Builder))
4523 return FoldedFCmps;
4524
4525 if (Instruction *CastedOr = foldCastedBitwiseLogic(I))
4526 return CastedOr;
4527
4528 if (Instruction *Sel = foldBinopOfSextBoolToSelect(I))
4529 return Sel;
4530
4531 // or(sext(A), B) / or(B, sext(A)) --> A ? -1 : B, where A is i1 or <N x i1>.
4532 // TODO: Move this into foldBinopOfSextBoolToSelect as a more generalized fold
4533 // with binop identity constant. But creating a select with non-constant
4534 // arm may not be reversible due to poison semantics. Is that a good
4535 // canonicalization?
4536 if (match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_SExt(Op: m_Value(V&: A))), R: m_Value(V&: B))) &&
4537 A->getType()->isIntOrIntVectorTy(BitWidth: 1))
4538 return createSelectInstWithUnknownProfile(
4539 C: A, S1: ConstantInt::getAllOnesValue(Ty), S2: B);
4540
4541 // Note: If we've gotten to the point of visiting the outer OR, then the
4542 // inner one couldn't be simplified. If it was a constant, then it won't
4543 // be simplified by a later pass either, so we try swapping the inner/outer
4544 // ORs in the hopes that we'll be able to simplify it this way.
4545 // (X|C) | V --> (X|V) | C
4546 // V | (X|C) --> (X|V) | C
4547 // Pass the disjoint flag in the following two patterns:
4548 // 1. or-disjoint (or-disjoint X, C), V -->
4549 // or-disjoint (or-disjoint X, V), C
4550 //
4551 // 2. or-disjoint (or X, C), V -->
4552 // or (or-disjoint X, V), C
4553 ConstantInt *CI;
4554 Instruction *InnerOr;
4555 Value *V;
4556 if (match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_Instruction(
4557 I&: InnerOr, P: m_Or(L: m_Value(V&: A), R: m_ConstantInt(CI)))),
4558 R: m_Value(V))) &&
4559 !match(V, P: m_ConstantInt())) {
4560 bool IsDisjointOuter = cast<PossiblyDisjointInst>(Val&: I).isDisjoint();
4561 bool IsDisjointInner = cast<PossiblyDisjointInst>(Val: InnerOr)->isDisjoint();
4562 Value *Inner = Builder.CreateOr(LHS: A, RHS: V, Name: "", /*IsDisjoint=*/IsDisjointOuter);
4563 Inner->takeName(V: InnerOr);
4564 return IsDisjointOuter && IsDisjointInner
4565 ? BinaryOperator::CreateDisjointOr(V1: Inner, V2: CI)
4566 : BinaryOperator::CreateOr(V1: Inner, V2: CI);
4567 }
4568
4569 // Change (or (bool?A:B),(bool?C:D)) --> (bool?(or A,C):(or B,D))
4570 // Since this OR statement hasn't been optimized further yet, we hope
4571 // that this transformation will allow the new ORs to be optimized.
4572 {
4573 Value *X = nullptr, *Y = nullptr;
4574 if (Op0->hasOneUse() && Op1->hasOneUse() &&
4575 match(V: Op0, P: m_Select(C: m_Value(V&: X), L: m_Value(V&: A), R: m_Value(V&: B))) &&
4576 match(V: Op1, P: m_Select(C: m_Value(V&: Y), L: m_Value(V&: C), R: m_Value(V&: D))) && X == Y) {
4577 Value *orTrue = Builder.CreateOr(LHS: A, RHS: C);
4578 Value *orFalse = Builder.CreateOr(LHS: B, RHS: D);
4579 return SelectInst::Create(C: X, S1: orTrue, S2: orFalse);
4580 }
4581 }
4582
4583 // or(ashr(subNSW(Y, X), ScalarSizeInBits(Y) - 1), X) --> X s> Y ? -1 : X.
4584 {
4585 Value *X, *Y;
4586 if (match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_AShr(
4587 L: m_NSWSub(L: m_Value(V&: Y), R: m_Value(V&: X)),
4588 R: m_SpecificInt(V: Ty->getScalarSizeInBits() - 1))),
4589 R: m_Deferred(V: X)))) {
4590 Value *NewICmpInst = Builder.CreateICmpSGT(LHS: X, RHS: Y);
4591 Value *AllOnes = ConstantInt::getAllOnesValue(Ty);
4592 return createSelectInstWithUnknownProfile(C: NewICmpInst, S1: AllOnes, S2: X);
4593 }
4594 }
4595
4596 {
4597 // ((A & B) ^ A) | ((A & B) ^ B) -> A ^ B
4598 // (A ^ (A & B)) | (B ^ (A & B)) -> A ^ B
4599 // ((A & B) ^ B) | ((A & B) ^ A) -> A ^ B
4600 // (B ^ (A & B)) | (A ^ (A & B)) -> A ^ B
4601 const auto TryXorOpt = [&](Value *Lhs, Value *Rhs) -> Instruction * {
4602 if (match(V: Lhs, P: m_c_Xor(L: m_And(L: m_Value(V&: A), R: m_Value(V&: B)), R: m_Deferred(V: A))) &&
4603 match(V: Rhs,
4604 P: m_c_Xor(L: m_And(L: m_Specific(V: A), R: m_Specific(V: B)), R: m_Specific(V: B)))) {
4605 return BinaryOperator::CreateXor(V1: A, V2: B);
4606 }
4607 return nullptr;
4608 };
4609
4610 if (Instruction *Result = TryXorOpt(Op0, Op1))
4611 return Result;
4612 if (Instruction *Result = TryXorOpt(Op1, Op0))
4613 return Result;
4614 }
4615
4616 if (Instruction *V =
4617 canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(I))
4618 return V;
4619
4620 CmpPredicate Pred;
4621 Value *Mul, *Ov, *MulIsNotZero, *UMulWithOv;
4622 // Check if the OR weakens the overflow condition for umul.with.overflow by
4623 // treating any non-zero result as overflow. In that case, we overflow if both
4624 // umul.with.overflow operands are != 0, as in that case the result can only
4625 // be 0, iff the multiplication overflows.
4626 if (match(V: &I, P: m_c_Or(L: m_Value(V&: Ov, P: m_ExtractValue<1>(V: m_Value(V&: UMulWithOv))),
4627 R: m_Value(V&: MulIsNotZero,
4628 P: m_SpecificICmp(
4629 MatchPred: ICmpInst::ICMP_NE,
4630 L: m_Value(V&: Mul, P: m_ExtractValue<0>(
4631 V: m_Deferred(V: UMulWithOv))),
4632 R: m_ZeroInt())))) &&
4633 (Ov->hasOneUse() || (MulIsNotZero->hasOneUse() && Mul->hasOneUse()))) {
4634 Value *A, *B;
4635 if (match(V: UMulWithOv, P: m_Intrinsic<Intrinsic::umul_with_overflow>(
4636 Ops: m_Value(V&: A), Ops: m_Value(V&: B)))) {
4637 Value *NotNullA = Builder.CreateIsNotNull(Arg: A);
4638 Value *NotNullB = Builder.CreateIsNotNull(Arg: B);
4639 return BinaryOperator::CreateAnd(V1: NotNullA, V2: NotNullB);
4640 }
4641 }
4642
4643 /// Res, Overflow = xxx_with_overflow X, C1
4644 /// Try to canonicalize the pattern "Overflow | icmp pred Res, C2" into
4645 /// "Overflow | icmp pred X, C2 +/- C1".
4646 const WithOverflowInst *WO;
4647 const Value *WOV;
4648 const APInt *C1, *C2;
4649 if (match(V: &I, P: m_c_Or(L: m_Value(V&: Ov, P: m_ExtractValue<1>(
4650 V: m_Value(V&: WOV, P: m_WithOverflowInst(I&: WO)))),
4651 R: m_OneUse(SubPattern: m_ICmp(Pred, L: m_ExtractValue<0>(V: m_Deferred(V: WOV)),
4652 R: m_APInt(Res&: C2))))) &&
4653 (WO->getBinaryOp() == Instruction::Add ||
4654 WO->getBinaryOp() == Instruction::Sub) &&
4655 (ICmpInst::isEquality(P: Pred) ||
4656 WO->isSigned() == ICmpInst::isSigned(Pred)) &&
4657 match(V: WO->getRHS(), P: m_APInt(Res&: C1))) {
4658 bool Overflow;
4659 APInt NewC = WO->getBinaryOp() == Instruction::Add
4660 ? (ICmpInst::isSigned(Pred) ? C2->ssub_ov(RHS: *C1, Overflow)
4661 : C2->usub_ov(RHS: *C1, Overflow))
4662 : (ICmpInst::isSigned(Pred) ? C2->sadd_ov(RHS: *C1, Overflow)
4663 : C2->uadd_ov(RHS: *C1, Overflow));
4664 if (!Overflow || ICmpInst::isEquality(P: Pred)) {
4665 Value *NewCmp = Builder.CreateICmp(
4666 P: Pred, LHS: WO->getLHS(), RHS: ConstantInt::get(Ty: WO->getLHS()->getType(), V: NewC));
4667 return BinaryOperator::CreateOr(V1: Ov, V2: NewCmp);
4668 }
4669 }
4670
4671 // Try to fold the pattern "Overflow | icmp pred Res, C2" into a single
4672 // comparison instruction for umul.with.overflow.
4673 if (Value *R = foldOrUnsignedUMulOverflowICmp(I, Builder, DL))
4674 return replaceInstUsesWith(I, V: R);
4675
4676 // (~x) | y --> ~(x & (~y)) iff that gets rid of inversions
4677 if (sinkNotIntoOtherHandOfLogicalOp(I))
4678 return &I;
4679
4680 // Improve "get low bit mask up to and including bit X" pattern:
4681 // (1 << X) | ((1 << X) + -1) --> -1 l>> (bitwidth(x) - 1 - X)
4682 if (match(V: &I, P: m_c_Or(L: m_Add(L: m_Shl(L: m_One(), R: m_Value(V&: X)), R: m_AllOnes()),
4683 R: m_Shl(L: m_One(), R: m_Deferred(V: X)))) &&
4684 match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_Value()), R: m_Value()))) {
4685 Value *Sub = Builder.CreateSub(
4686 LHS: ConstantInt::get(Ty, V: Ty->getScalarSizeInBits() - 1), RHS: X);
4687 return BinaryOperator::CreateLShr(V1: Constant::getAllOnesValue(Ty), V2: Sub);
4688 }
4689
4690 // An or recurrence w/loop invariant step is equivelent to (or start, step)
4691 PHINode *PN = nullptr;
4692 Value *Start = nullptr, *Step = nullptr;
4693 if (matchSimpleRecurrence(I: &I, P&: PN, Start, Step) && DT.dominates(Def: Step, User: PN))
4694 return replaceInstUsesWith(I, V: Builder.CreateOr(LHS: Start, RHS: Step));
4695
4696 // (A & B) | (C | D) or (C | D) | (A & B)
4697 // Can be combined if C or D is of type (A/B & X)
4698 if (match(V: &I, P: m_c_Or(L: m_OneUse(SubPattern: m_And(L: m_Value(V&: A), R: m_Value(V&: B))),
4699 R: m_OneUse(SubPattern: m_Or(L: m_Value(V&: C), R: m_Value(V&: D)))))) {
4700 // (A & B) | (C | ?) -> C | (? | (A & B))
4701 // (A & B) | (C | ?) -> C | (? | (A & B))
4702 // (A & B) | (C | ?) -> C | (? | (A & B))
4703 // (A & B) | (C | ?) -> C | (? | (A & B))
4704 // (C | ?) | (A & B) -> C | (? | (A & B))
4705 // (C | ?) | (A & B) -> C | (? | (A & B))
4706 // (C | ?) | (A & B) -> C | (? | (A & B))
4707 // (C | ?) | (A & B) -> C | (? | (A & B))
4708 if (match(V: D, P: m_OneUse(SubPattern: m_c_And(L: m_Specific(V: A), R: m_Value()))) ||
4709 match(V: D, P: m_OneUse(SubPattern: m_c_And(L: m_Specific(V: B), R: m_Value()))))
4710 return BinaryOperator::CreateOr(
4711 V1: C, V2: Builder.CreateOr(LHS: D, RHS: Builder.CreateAnd(LHS: A, RHS: B)));
4712 // (A & B) | (? | D) -> (? | (A & B)) | D
4713 // (A & B) | (? | D) -> (? | (A & B)) | D
4714 // (A & B) | (? | D) -> (? | (A & B)) | D
4715 // (A & B) | (? | D) -> (? | (A & B)) | D
4716 // (? | D) | (A & B) -> (? | (A & B)) | D
4717 // (? | D) | (A & B) -> (? | (A & B)) | D
4718 // (? | D) | (A & B) -> (? | (A & B)) | D
4719 // (? | D) | (A & B) -> (? | (A & B)) | D
4720 if (match(V: C, P: m_OneUse(SubPattern: m_c_And(L: m_Specific(V: A), R: m_Value()))) ||
4721 match(V: C, P: m_OneUse(SubPattern: m_c_And(L: m_Specific(V: B), R: m_Value()))))
4722 return BinaryOperator::CreateOr(
4723 V1: Builder.CreateOr(LHS: C, RHS: Builder.CreateAnd(LHS: A, RHS: B)), V2: D);
4724 }
4725
4726 if (Instruction *R = reassociateForUses(BO&: I, Builder))
4727 return R;
4728
4729 if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder))
4730 return Canonicalized;
4731
4732 if (Instruction *Folded = foldLogicOfIsFPClass(BO&: I, Op0, Op1))
4733 return Folded;
4734
4735 if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
4736 return Res;
4737
4738 // If we are setting the sign bit of a floating-point value, convert
4739 // this to fneg(fabs), then cast back to integer.
4740 //
4741 // If the result isn't immediately cast back to a float, this will increase
4742 // the number of instructions. This is still probably a better canonical form
4743 // as it enables FP value tracking.
4744 //
4745 // Assumes any IEEE-represented type has the sign bit in the high bit.
4746 //
4747 // This is generous interpretation of noimplicitfloat, this is not a true
4748 // floating-point operation.
4749 Value *CastOp;
4750 if (match(V: Op0, P: m_ElementWiseBitCast(Op: m_Value(V&: CastOp))) &&
4751 match(V: Op1, P: m_SignMask()) &&
4752 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
4753 Kind: Attribute::NoImplicitFloat)) {
4754 Type *EltTy = CastOp->getType()->getScalarType();
4755 if (EltTy->isFloatingPointTy() &&
4756 APFloat::hasSignBitInMSB(EltTy->getFltSemantics())) {
4757 Value *FAbs = Builder.CreateFAbs(V: CastOp);
4758 Value *FNegFAbs = Builder.CreateFNeg(V: FAbs);
4759 return new BitCastInst(FNegFAbs, I.getType());
4760 }
4761 }
4762
4763 // (X & C1) | C2 -> X & (C1 | C2) iff (X & C2) == C2
4764 if (match(V: Op0, P: m_OneUse(SubPattern: m_And(L: m_Value(V&: X), R: m_APInt(Res&: C1)))) &&
4765 match(V: Op1, P: m_APInt(Res&: C2))) {
4766 KnownBits KnownX = computeKnownBits(V: X, CtxI: &I);
4767 if ((KnownX.One & *C2) == *C2)
4768 return BinaryOperator::CreateAnd(V1: X, V2: ConstantInt::get(Ty, V: *C1 | *C2));
4769 }
4770
4771 if (Instruction *Res = foldBitwiseLogicWithIntrinsics(I, Builder))
4772 return Res;
4773
4774 if (Value *V =
4775 simplifyAndOrWithOpReplaced(V: Op0, Op: Op1, RepOp: Constant::getNullValue(Ty),
4776 /*SimplifyOnly*/ false, IC&: *this))
4777 return BinaryOperator::CreateOr(V1: V, V2: Op1);
4778 if (Value *V =
4779 simplifyAndOrWithOpReplaced(V: Op1, Op: Op0, RepOp: Constant::getNullValue(Ty),
4780 /*SimplifyOnly*/ false, IC&: *this))
4781 return BinaryOperator::CreateOr(V1: Op0, V2: V);
4782
4783 if (cast<PossiblyDisjointInst>(Val&: I).isDisjoint())
4784 if (Value *V = SimplifyAddWithRemainder(I))
4785 return replaceInstUsesWith(I, V);
4786
4787 if (Value *Res = FoldOrOfSelectSmaxToAbs(I, Builder))
4788 return replaceInstUsesWith(I, V: Res);
4789
4790 // signum: or (ashr X, BW-1), zext (icmp ne|sgt X, 0) --> scmp(X, 0)
4791 // The ashr already supplies -1 for negative X, so any predicate that
4792 // produces 1 for positive X and 0 for X == 0 yields the same result here.
4793 {
4794 Value *X;
4795 CmpPredicate SignPred;
4796 unsigned BitWidth = Ty->getScalarSizeInBits();
4797 if (match(V: &I,
4798 P: m_c_Or(L: m_AShr(L: m_Value(V&: X), R: m_SpecificIntAllowPoison(V: BitWidth - 1)),
4799 R: m_ZExt(Op: m_ICmp(Pred&: SignPred, L: m_Deferred(V: X), R: m_ZeroInt())))) &&
4800 (SignPred == ICmpInst::ICMP_NE || SignPred == ICmpInst::ICMP_SGT) &&
4801 (Op0->hasOneUse() || Op1->hasOneUse()))
4802 return replaceInstUsesWith(
4803 I, V: Builder.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::scmp,
4804 Args: {X, Constant::getNullValue(Ty)}));
4805 }
4806
4807 return nullptr;
4808}
4809
4810/// A ^ B can be specified using other logic ops in a variety of patterns. We
4811/// can fold these early and efficiently by morphing an existing instruction.
4812static Instruction *foldXorToXor(BinaryOperator &I,
4813 InstCombiner::BuilderTy &Builder) {
4814 assert(I.getOpcode() == Instruction::Xor);
4815 Value *Op0 = I.getOperand(i_nocapture: 0);
4816 Value *Op1 = I.getOperand(i_nocapture: 1);
4817 Value *A, *B;
4818
4819 // There are 4 commuted variants for each of the basic patterns.
4820
4821 // (A & B) ^ (A | B) -> A ^ B
4822 // (A & B) ^ (B | A) -> A ^ B
4823 // (A | B) ^ (A & B) -> A ^ B
4824 // (A | B) ^ (B & A) -> A ^ B
4825 if (match(V: &I, P: m_c_Xor(L: m_And(L: m_Value(V&: A), R: m_Value(V&: B)),
4826 R: m_c_Or(L: m_Deferred(V: A), R: m_Deferred(V: B)))))
4827 return BinaryOperator::CreateXor(V1: A, V2: B);
4828
4829 // (A | ~B) ^ (~A | B) -> A ^ B
4830 // (~B | A) ^ (~A | B) -> A ^ B
4831 // (~A | B) ^ (A | ~B) -> A ^ B
4832 // (B | ~A) ^ (A | ~B) -> A ^ B
4833 if (match(V: &I, P: m_Xor(L: m_c_Or(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B))),
4834 R: m_c_Or(L: m_Not(V: m_Deferred(V: A)), R: m_Deferred(V: B)))))
4835 return BinaryOperator::CreateXor(V1: A, V2: B);
4836
4837 // (A & ~B) ^ (~A & B) -> A ^ B
4838 // (~B & A) ^ (~A & B) -> A ^ B
4839 // (~A & B) ^ (A & ~B) -> A ^ B
4840 // (B & ~A) ^ (A & ~B) -> A ^ B
4841 if (match(V: &I, P: m_Xor(L: m_c_And(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B))),
4842 R: m_c_And(L: m_Not(V: m_Deferred(V: A)), R: m_Deferred(V: B)))))
4843 return BinaryOperator::CreateXor(V1: A, V2: B);
4844
4845 // For the remaining cases we need to get rid of one of the operands.
4846 if (!Op0->hasOneUse() && !Op1->hasOneUse())
4847 return nullptr;
4848
4849 // (A | B) ^ ~(A & B) -> ~(A ^ B)
4850 // (A | B) ^ ~(B & A) -> ~(A ^ B)
4851 // (A & B) ^ ~(A | B) -> ~(A ^ B)
4852 // (A & B) ^ ~(B | A) -> ~(A ^ B)
4853 // Complexity sorting ensures the not will be on the right side.
4854 if ((match(V: Op0, P: m_Or(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4855 match(V: Op1, P: m_Not(V: m_c_And(L: m_Specific(V: A), R: m_Specific(V: B))))) ||
4856 (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
4857 match(V: Op1, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))))
4858 return BinaryOperator::CreateNot(Op: Builder.CreateXor(LHS: A, RHS: B));
4859
4860 return nullptr;
4861}
4862
4863Value *InstCombinerImpl::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS,
4864 BinaryOperator &I) {
4865 assert(I.getOpcode() == Instruction::Xor && I.getOperand(0) == LHS &&
4866 I.getOperand(1) == RHS && "Should be 'xor' with these operands");
4867
4868 ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
4869 Value *LHS0 = LHS->getOperand(i_nocapture: 0), *LHS1 = LHS->getOperand(i_nocapture: 1);
4870 Value *RHS0 = RHS->getOperand(i_nocapture: 0), *RHS1 = RHS->getOperand(i_nocapture: 1);
4871
4872 if (predicatesFoldable(P1: PredL, P2: PredR)) {
4873 if (LHS0 == RHS1 && LHS1 == RHS0) {
4874 std::swap(a&: LHS0, b&: LHS1);
4875 PredL = ICmpInst::getSwappedPredicate(pred: PredL);
4876 }
4877 if (LHS0 == RHS0 && LHS1 == RHS1) {
4878 // (icmp1 A, B) ^ (icmp2 A, B) --> (icmp3 A, B)
4879 unsigned Code = getICmpCode(Pred: PredL) ^ getICmpCode(Pred: PredR);
4880 bool IsSigned = LHS->isSigned() || RHS->isSigned();
4881 return getNewICmpValue(Code, Sign: IsSigned, LHS: LHS0, RHS: LHS1, Builder);
4882 }
4883 }
4884
4885 const APInt *LC, *RC;
4886 if (match(V: LHS1, P: m_APInt(Res&: LC)) && match(V: RHS1, P: m_APInt(Res&: RC)) &&
4887 LHS0->getType() == RHS0->getType() &&
4888 LHS0->getType()->isIntOrIntVectorTy()) {
4889 // Convert xor of signbit tests to signbit test of xor'd values:
4890 // (X > -1) ^ (Y > -1) --> (X ^ Y) < 0
4891 // (X < 0) ^ (Y < 0) --> (X ^ Y) < 0
4892 // (X > -1) ^ (Y < 0) --> (X ^ Y) > -1
4893 // (X < 0) ^ (Y > -1) --> (X ^ Y) > -1
4894 bool TrueIfSignedL, TrueIfSignedR;
4895 if ((LHS->hasOneUse() || RHS->hasOneUse()) &&
4896 isSignBitCheck(Pred: PredL, RHS: *LC, TrueIfSigned&: TrueIfSignedL) &&
4897 isSignBitCheck(Pred: PredR, RHS: *RC, TrueIfSigned&: TrueIfSignedR)) {
4898 Value *XorLR = Builder.CreateXor(LHS: LHS0, RHS: RHS0);
4899 return TrueIfSignedL == TrueIfSignedR ? Builder.CreateIsNeg(Arg: XorLR) :
4900 Builder.CreateIsNotNeg(Arg: XorLR);
4901 }
4902
4903 // Fold (icmp pred1 X, C1) ^ (icmp pred2 X, C2)
4904 // into a single comparison using range-based reasoning.
4905 if (LHS0 == RHS0) {
4906 ConstantRange CR1 = ConstantRange::makeExactICmpRegion(Pred: PredL, Other: *LC);
4907 ConstantRange CR2 = ConstantRange::makeExactICmpRegion(Pred: PredR, Other: *RC);
4908 auto CRUnion = CR1.exactUnionWith(CR: CR2);
4909 auto CRIntersect = CR1.exactIntersectWith(CR: CR2);
4910 if (CRUnion && CRIntersect)
4911 if (auto CR = CRUnion->exactIntersectWith(CR: CRIntersect->inverse())) {
4912 if (CR->isFullSet())
4913 return ConstantInt::getTrue(Ty: I.getType());
4914 if (CR->isEmptySet())
4915 return ConstantInt::getFalse(Ty: I.getType());
4916
4917 CmpInst::Predicate NewPred;
4918 APInt NewC, Offset;
4919 CR->getEquivalentICmp(Pred&: NewPred, RHS&: NewC, Offset);
4920
4921 if ((Offset.isZero() && (LHS->hasOneUse() || RHS->hasOneUse())) ||
4922 (LHS->hasOneUse() && RHS->hasOneUse())) {
4923 Value *NewV = LHS0;
4924 Type *Ty = LHS0->getType();
4925 if (!Offset.isZero())
4926 NewV = Builder.CreateAdd(LHS: NewV, RHS: ConstantInt::get(Ty, V: Offset));
4927 return Builder.CreateICmp(P: NewPred, LHS: NewV,
4928 RHS: ConstantInt::get(Ty, V: NewC));
4929 }
4930 }
4931 }
4932
4933 // Fold (icmp eq/ne (X & Pow2), 0) ^ (icmp eq/ne (Y & Pow2), 0) into
4934 // (icmp eq/ne ((X ^ Y) & Pow2), 0)
4935 Value *X, *Y, *Pow2;
4936 if (ICmpInst::isEquality(P: PredL) && ICmpInst::isEquality(P: PredR) &&
4937 LC->isZero() && RC->isZero() && LHS->hasOneUse() && RHS->hasOneUse() &&
4938 match(V: LHS0, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Pow2))) &&
4939 match(V: RHS0, P: m_And(L: m_Value(V&: Y), R: m_Specific(V: Pow2))) &&
4940 isKnownToBeAPowerOfTwo(V: Pow2, /*OrZero=*/true, CtxI: &I)) {
4941 Value *Xor = Builder.CreateXor(LHS: X, RHS: Y);
4942 Value *And = Builder.CreateAnd(LHS: Xor, RHS: Pow2);
4943 return Builder.CreateICmp(P: PredL == PredR ? ICmpInst::ICMP_NE
4944 : ICmpInst::ICMP_EQ,
4945 LHS: And, RHS: ConstantInt::getNullValue(Ty: Xor->getType()));
4946 }
4947 }
4948
4949 // Instead of trying to imitate the folds for and/or, decompose this 'xor'
4950 // into those logic ops. That is, try to turn this into an and-of-icmps
4951 // because we have many folds for that pattern.
4952 //
4953 // This is based on a truth table definition of xor:
4954 // X ^ Y --> (X | Y) & !(X & Y)
4955 if (Value *OrICmp = simplifyBinOp(Opcode: Instruction::Or, LHS, RHS, Q: SQ)) {
4956 // TODO: If OrICmp is true, then the definition of xor simplifies to !(X&Y).
4957 // TODO: If OrICmp is false, the whole thing is false (InstSimplify?).
4958 if (Value *AndICmp = simplifyBinOp(Opcode: Instruction::And, LHS, RHS, Q: SQ)) {
4959 // TODO: Independently handle cases where the 'and' side is a constant.
4960 ICmpInst *X = nullptr, *Y = nullptr;
4961 if (OrICmp == LHS && AndICmp == RHS) {
4962 // (LHS | RHS) & !(LHS & RHS) --> LHS & !RHS --> X & !Y
4963 X = LHS;
4964 Y = RHS;
4965 }
4966 if (OrICmp == RHS && AndICmp == LHS) {
4967 // !(LHS & RHS) & (LHS | RHS) --> !LHS & RHS --> !Y & X
4968 X = RHS;
4969 Y = LHS;
4970 }
4971 if (X && Y && (Y->hasOneUse() || canFreelyInvertAllUsersOf(V: Y, IgnoredUser: &I))) {
4972 // Invert the predicate of 'Y', thus inverting its output.
4973 Y->setPredicate(Y->getInversePredicate());
4974 // So, are there other uses of Y?
4975 if (!Y->hasOneUse()) {
4976 // We need to adapt other uses of Y though. Get a value that matches
4977 // the original value of Y before inversion. While this increases
4978 // immediate instruction count, we have just ensured that all the
4979 // users are freely-invertible, so that 'not' *will* get folded away.
4980 BuilderTy::InsertPointGuard Guard(Builder);
4981 // Set insertion point to right after the Y.
4982 Builder.SetInsertPoint(++(Y->getIterator()));
4983 Value *NotY = Builder.CreateNot(V: Y, Name: Y->getName() + ".not");
4984 // Replace all uses of Y (excluding the one in NotY!) with NotY.
4985 Worklist.pushUsersToWorkList(I&: *Y);
4986 Y->replaceUsesWithIf(New: NotY,
4987 ShouldReplace: [NotY](Use &U) { return U.getUser() != NotY; });
4988 }
4989 // All done.
4990 return Builder.CreateAnd(LHS, RHS);
4991 }
4992 }
4993 }
4994
4995 return nullptr;
4996}
4997
4998/// If we have a masked merge, in the canonical form of:
4999/// (assuming that A only has one use.)
5000/// | A | |B|
5001/// ((x ^ y) & M) ^ y
5002/// | D |
5003/// * If M is inverted:
5004/// | D |
5005/// ((x ^ y) & ~M) ^ y
5006/// We can canonicalize by swapping the final xor operand
5007/// to eliminate the 'not' of the mask.
5008/// ((x ^ y) & M) ^ x
5009/// * If M is a constant, and D has one use, we transform to 'and' / 'or' ops
5010/// because that shortens the dependency chain and improves analysis:
5011/// (x & M) | (y & ~M)
5012static Instruction *visitMaskedMerge(BinaryOperator &I,
5013 InstCombiner::BuilderTy &Builder) {
5014 Value *B, *X, *D;
5015 Value *M;
5016 if (!match(V: &I, P: m_c_Xor(L: m_Value(V&: B),
5017 R: m_OneUse(SubPattern: m_c_And(
5018 L: m_Value(V&: D, P: m_c_Xor(L: m_Deferred(V: B), R: m_Value(V&: X))),
5019 R: m_Value(V&: M))))))
5020 return nullptr;
5021
5022 Value *NotM;
5023 if (match(V: M, P: m_Not(V: m_Value(V&: NotM)))) {
5024 // De-invert the mask and swap the value in B part.
5025 Value *NewA = Builder.CreateAnd(LHS: D, RHS: NotM);
5026 return BinaryOperator::CreateXor(V1: NewA, V2: X);
5027 }
5028
5029 Constant *C;
5030 if (D->hasOneUse() && match(V: M, P: m_Constant(C))) {
5031 // Propagating undef is unsafe. Clamp undef elements to -1.
5032 Type *EltTy = C->getType()->getScalarType();
5033 C = Constant::replaceUndefsWith(C, Replacement: ConstantInt::getAllOnesValue(Ty: EltTy));
5034 // Unfold.
5035 Value *LHS = Builder.CreateAnd(LHS: X, RHS: C);
5036 Value *NotC = Builder.CreateNot(V: C);
5037 Value *RHS = Builder.CreateAnd(LHS: B, RHS: NotC);
5038 return BinaryOperator::CreateOr(V1: LHS, V2: RHS);
5039 }
5040
5041 return nullptr;
5042}
5043
5044static Instruction *foldNotXor(BinaryOperator &I,
5045 InstCombiner::BuilderTy &Builder) {
5046 Value *X, *Y;
5047 // FIXME: one-use check is not needed in general, but currently we are unable
5048 // to fold 'not' into 'icmp', if that 'icmp' has multiple uses. (D35182)
5049 if (!match(V: &I, P: m_Not(V: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: X), R: m_Value(V&: Y))))))
5050 return nullptr;
5051
5052 auto hasCommonOperand = [](Value *A, Value *B, Value *C, Value *D) {
5053 return A == C || A == D || B == C || B == D;
5054 };
5055
5056 Value *A, *B, *C, *D;
5057 // Canonicalize ~((A & B) ^ (A | ?)) -> (A & B) | ~(A | ?)
5058 // 4 commuted variants
5059 if (match(V: X, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
5060 match(V: Y, P: m_Or(L: m_Value(V&: C), R: m_Value(V&: D))) && hasCommonOperand(A, B, C, D)) {
5061 Value *NotY = Builder.CreateNot(V: Y);
5062 return BinaryOperator::CreateOr(V1: X, V2: NotY);
5063 };
5064
5065 // Canonicalize ~((A | ?) ^ (A & B)) -> (A & B) | ~(A | ?)
5066 // 4 commuted variants
5067 if (match(V: Y, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
5068 match(V: X, P: m_Or(L: m_Value(V&: C), R: m_Value(V&: D))) && hasCommonOperand(A, B, C, D)) {
5069 Value *NotX = Builder.CreateNot(V: X);
5070 return BinaryOperator::CreateOr(V1: Y, V2: NotX);
5071 };
5072
5073 return nullptr;
5074}
5075
5076/// Canonicalize a shifty way to code absolute value to the more common pattern
5077/// that uses negation and select.
5078static Instruction *canonicalizeAbs(BinaryOperator &Xor,
5079 InstCombiner::BuilderTy &Builder) {
5080 assert(Xor.getOpcode() == Instruction::Xor && "Expected an xor instruction.");
5081
5082 // There are 4 potential commuted variants. Move the 'ashr' candidate to Op1.
5083 // We're relying on the fact that we only do this transform when the shift has
5084 // exactly 2 uses and the add has exactly 1 use (otherwise, we might increase
5085 // instructions).
5086 Value *Op0 = Xor.getOperand(i_nocapture: 0), *Op1 = Xor.getOperand(i_nocapture: 1);
5087 if (Op0->hasNUses(N: 2))
5088 std::swap(a&: Op0, b&: Op1);
5089
5090 Type *Ty = Xor.getType();
5091 Value *A;
5092 const APInt *ShAmt;
5093 if (match(V: Op1, P: m_AShr(L: m_Value(V&: A), R: m_APInt(Res&: ShAmt))) &&
5094 Op1->hasNUses(N: 2) && *ShAmt == Ty->getScalarSizeInBits() - 1 &&
5095 match(V: Op0, P: m_OneUse(SubPattern: m_c_Add(L: m_Specific(V: A), R: m_Specific(V: Op1))))) {
5096 // Op1 = ashr i32 A, 31 ; smear the sign bit
5097 // xor (add A, Op1), Op1 ; add -1 and flip bits if negative
5098 // --> (A < 0) ? -A : A
5099 Value *IsNeg = Builder.CreateIsNeg(Arg: A);
5100 // Copy the nsw flags from the add to the negate.
5101 auto *Add = cast<BinaryOperator>(Val: Op0);
5102 Value *NegA = Add->hasNoUnsignedWrap()
5103 ? Constant::getNullValue(Ty: A->getType())
5104 : Builder.CreateNeg(V: A, Name: "", HasNSW: Add->hasNoSignedWrap());
5105 return SelectInst::Create(C: IsNeg, S1: NegA, S2: A);
5106 }
5107 return nullptr;
5108}
5109
5110static bool canFreelyInvert(InstCombiner &IC, Value *Op,
5111 Instruction *IgnoredUser) {
5112 auto *I = dyn_cast<Instruction>(Val: Op);
5113 return I && I->getInsertionPointAfterDef() &&
5114 IC.isFreeToInvert(V: I, /*WillInvertAllUses=*/true) &&
5115 IC.canFreelyInvertAllUsersOf(V: I, IgnoredUser);
5116}
5117
5118static Value *freelyInvert(InstCombinerImpl &IC, Value *Op,
5119 Instruction *IgnoredUser) {
5120 auto *I = cast<Instruction>(Val: Op);
5121 auto InsertPt = I->getInsertionPointAfterDef();
5122 assert(InsertPt &&
5123 "freelyInvert requires an instruction with a valid insertion point");
5124 IC.Builder.SetInsertPoint(*InsertPt);
5125 Value *NotOp = IC.Builder.CreateNot(V: Op, Name: Op->getName() + ".not");
5126 Op->replaceUsesWithIf(New: NotOp,
5127 ShouldReplace: [NotOp](Use &U) { return U.getUser() != NotOp; });
5128 IC.freelyInvertAllUsersOf(V: NotOp, IgnoredUser);
5129 return NotOp;
5130}
5131
5132// Transform
5133// z = ~(x &/| y)
5134// into:
5135// z = ((~x) |/& (~y))
5136// iff both x and y are free to invert and all uses of z can be freely updated.
5137bool InstCombinerImpl::sinkNotIntoLogicalOp(Instruction &I) {
5138 Value *Op0, *Op1;
5139 if (!match(V: &I, P: m_LogicalOp(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
5140 return false;
5141
5142 // If this logic op has not been simplified yet, just bail out and let that
5143 // happen first. Otherwise, the code below may wrongly invert.
5144 if (Op0 == Op1)
5145 return false;
5146
5147 // If one of the operands is a user of the other,
5148 // freelyInvert->freelyInvertAllUsersOf will change the operands of I, which
5149 // may cause miscompilation.
5150 if (match(V: Op0, P: m_Not(V: m_Specific(V: Op1))) || match(V: Op1, P: m_Not(V: m_Specific(V: Op0))))
5151 return false;
5152
5153 Instruction::BinaryOps NewOpc =
5154 match(V: &I, P: m_LogicalAnd()) ? Instruction::Or : Instruction::And;
5155 bool IsBinaryOp = isa<BinaryOperator>(Val: I);
5156
5157 // Can our users be adapted?
5158 if (!InstCombiner::canFreelyInvertAllUsersOf(V: &I, /*IgnoredUser=*/nullptr))
5159 return false;
5160
5161 // And can the operands be adapted?
5162 if (!canFreelyInvert(IC&: *this, Op: Op0, IgnoredUser: &I) || !canFreelyInvert(IC&: *this, Op: Op1, IgnoredUser: &I))
5163 return false;
5164
5165 Op0 = freelyInvert(IC&: *this, Op: Op0, IgnoredUser: &I);
5166 Op1 = freelyInvert(IC&: *this, Op: Op1, IgnoredUser: &I);
5167
5168 auto InsertPt = I.getInsertionPointAfterDef();
5169 assert(InsertPt && "sinkNotIntoLogicalOp requires an instruction with a "
5170 "valid insertion point");
5171 Builder.SetInsertPoint(*InsertPt);
5172 Value *NewLogicOp;
5173 if (IsBinaryOp) {
5174 NewLogicOp = Builder.CreateBinOp(Opc: NewOpc, LHS: Op0, RHS: Op1, Name: I.getName() + ".not");
5175 } else {
5176 NewLogicOp =
5177 Builder.CreateLogicalOp(Opc: NewOpc, Cond1: Op0, Cond2: Op1, Name: I.getName() + ".not", MDFrom: &I);
5178 if (SelectInst *SI = dyn_cast<SelectInst>(Val: NewLogicOp))
5179 SI->swapProfMetadata();
5180 }
5181
5182 replaceInstUsesWith(I, V: NewLogicOp);
5183 // We can not just create an outer `not`, it will most likely be immediately
5184 // folded back, reconstructing our initial pattern, and causing an
5185 // infinite combine loop, so immediately manually fold it away.
5186 freelyInvertAllUsersOf(V: NewLogicOp);
5187 return true;
5188}
5189
5190// Transform
5191// z = (~x) &/| y
5192// into:
5193// z = ~(x |/& (~y))
5194// iff y is free to invert and all uses of z can be freely updated.
5195bool InstCombinerImpl::sinkNotIntoOtherHandOfLogicalOp(Instruction &I) {
5196 Value *Op0, *Op1;
5197 if (!match(V: &I, P: m_LogicalOp(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
5198 return false;
5199 Instruction::BinaryOps NewOpc =
5200 match(V: &I, P: m_LogicalAnd()) ? Instruction::Or : Instruction::And;
5201 bool IsBinaryOp = isa<BinaryOperator>(Val: I);
5202
5203 Value *NotOp0 = nullptr;
5204 Value *NotOp1 = nullptr;
5205 Value **OpToInvert = nullptr;
5206 if (match(V: Op0, P: m_Not(V: m_Value(V&: NotOp0))) && canFreelyInvert(IC&: *this, Op: Op1, IgnoredUser: &I)) {
5207 Op0 = NotOp0;
5208 OpToInvert = &Op1;
5209 } else if (match(V: Op1, P: m_Not(V: m_Value(V&: NotOp1))) &&
5210 canFreelyInvert(IC&: *this, Op: Op0, IgnoredUser: &I)) {
5211 Op1 = NotOp1;
5212 OpToInvert = &Op0;
5213 } else
5214 return false;
5215
5216 // If the kept operand is defined as NOT(OpToInvert), freelyInvert(OpToInvert)
5217 // will also flip the kept operand as a side effect of updating its uses,
5218 // invalidating the assumption that it stays fixed while I is rewriten.
5219 if (match(V: *OpToInvert == Op1 ? Op0 : Op1, P: m_Not(V: m_Specific(V: *OpToInvert))))
5220 return false;
5221
5222 // And can our users be adapted?
5223 if (!InstCombiner::canFreelyInvertAllUsersOf(V: &I, /*IgnoredUser=*/nullptr))
5224 return false;
5225
5226 *OpToInvert = freelyInvert(IC&: *this, Op: *OpToInvert, IgnoredUser: &I);
5227
5228 Builder.SetInsertPoint(*I.getInsertionPointAfterDef());
5229 Value *NewBinOp;
5230 if (IsBinaryOp) {
5231 NewBinOp = Builder.CreateBinOp(Opc: NewOpc, LHS: Op0, RHS: Op1, Name: I.getName() + ".not");
5232 } else {
5233 NewBinOp =
5234 Builder.CreateLogicalOp(Opc: NewOpc, Cond1: Op0, Cond2: Op1, Name: I.getName() + ".not",
5235 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &I);
5236 if (auto *NewSI = dyn_cast<SelectInst>(Val: NewBinOp))
5237 NewSI->swapProfMetadata();
5238 }
5239 replaceInstUsesWith(I, V: NewBinOp);
5240 // We can not just create an outer `not`, it will most likely be immediately
5241 // folded back, reconstructing our initial pattern, and causing an
5242 // infinite combine loop, so immediately manually fold it away.
5243 freelyInvertAllUsersOf(V: NewBinOp);
5244 return true;
5245}
5246
5247Instruction *InstCombinerImpl::foldNot(BinaryOperator &I) {
5248 Value *NotOp;
5249 if (!match(V: &I, P: m_Not(V: m_Value(V&: NotOp))))
5250 return nullptr;
5251
5252 // Apply DeMorgan's Law for 'nand' / 'nor' logic with an inverted operand.
5253 // We must eliminate the and/or (one-use) for these transforms to not increase
5254 // the instruction count.
5255 //
5256 // ~(~X & Y) --> (X | ~Y)
5257 // ~(Y & ~X) --> (X | ~Y)
5258 //
5259 // Note: The logical matches do not check for the commuted patterns because
5260 // those are handled via SimplifySelectsFeedingBinaryOp().
5261 Type *Ty = I.getType();
5262 Value *X, *Y;
5263 if (match(V: NotOp, P: m_OneUse(SubPattern: m_c_And(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))) {
5264 Value *NotY = Builder.CreateNot(V: Y, Name: Y->getName() + ".not");
5265 return BinaryOperator::CreateOr(V1: X, V2: NotY);
5266 }
5267 if (match(V: NotOp, P: m_OneUse(SubPattern: m_LogicalAnd(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))) {
5268 Value *NotY = Builder.CreateNot(V: Y, Name: Y->getName() + ".not");
5269 SelectInst *SI = SelectInst::Create(C: X, S1: ConstantInt::getTrue(Ty), S2: NotY, NameStr: "",
5270 InsertBefore: nullptr, MDFrom: cast<Instruction>(Val: NotOp));
5271 SI->swapProfMetadata();
5272 return SI;
5273 }
5274
5275 // ~(~X | Y) --> (X & ~Y)
5276 // ~(Y | ~X) --> (X & ~Y)
5277 if (match(V: NotOp, P: m_OneUse(SubPattern: m_c_Or(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))) {
5278 Value *NotY = Builder.CreateNot(V: Y, Name: Y->getName() + ".not");
5279 return BinaryOperator::CreateAnd(V1: X, V2: NotY);
5280 }
5281 if (match(V: NotOp, P: m_OneUse(SubPattern: m_LogicalOr(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))) {
5282 Value *NotY = Builder.CreateNot(V: Y, Name: Y->getName() + ".not");
5283 SelectInst *SI = SelectInst::Create(C: X, S1: NotY, S2: ConstantInt::getFalse(Ty), NameStr: "",
5284 InsertBefore: nullptr, MDFrom: cast<Instruction>(Val: NotOp));
5285 SI->swapProfMetadata();
5286 return SI;
5287 }
5288
5289 // Is this a 'not' (~) fed by a binary operator?
5290 BinaryOperator *NotVal;
5291 if (match(V: NotOp, P: m_BinOp(I&: NotVal))) {
5292 // ~((-X) | Y) --> (X - 1) & (~Y)
5293 if (match(V: NotVal,
5294 P: m_OneUse(SubPattern: m_c_Or(L: m_OneUse(SubPattern: m_Neg(V: m_Value(V&: X))), R: m_Value(V&: Y))))) {
5295 Value *DecX = Builder.CreateAdd(LHS: X, RHS: ConstantInt::getAllOnesValue(Ty));
5296 Value *NotY = Builder.CreateNot(V: Y);
5297 return BinaryOperator::CreateAnd(V1: DecX, V2: NotY);
5298 }
5299
5300 // ~(~X >>s Y) --> (X >>s Y)
5301 if (match(V: NotVal, P: m_AShr(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))
5302 return BinaryOperator::CreateAShr(V1: X, V2: Y);
5303
5304 // Treat lshr with non-negative operand as ashr.
5305 // ~(~X >>u Y) --> (X >>s Y) iff X is known negative
5306 if (match(V: NotVal, P: m_LShr(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))) &&
5307 isKnownNegative(V: X, SQ: SQ.getWithInstruction(I: NotVal)))
5308 return BinaryOperator::CreateAShr(V1: X, V2: Y);
5309
5310 // Bit-hack form of a signbit test for iN type:
5311 // ~(X >>s (N - 1)) --> sext i1 (X > -1) to iN
5312 unsigned FullShift = Ty->getScalarSizeInBits() - 1;
5313 if (match(V: NotVal, P: m_OneUse(SubPattern: m_AShr(L: m_Value(V&: X), R: m_SpecificInt(V: FullShift))))) {
5314 Value *IsNotNeg = Builder.CreateIsNotNeg(Arg: X, Name: "isnotneg");
5315 return new SExtInst(IsNotNeg, Ty);
5316 }
5317
5318 // If we are inverting a right-shifted constant, we may be able to eliminate
5319 // the 'not' by inverting the constant and using the opposite shift type.
5320 // Canonicalization rules ensure that only a negative constant uses 'ashr',
5321 // but we must check that in case that transform has not fired yet.
5322
5323 // ~(C >>s Y) --> ~C >>u Y (when inverting the replicated sign bits)
5324 Constant *C;
5325 if (match(V: NotVal, P: m_AShr(L: m_Constant(C), R: m_Value(V&: Y))) &&
5326 match(V: C, P: m_Negative()))
5327 return BinaryOperator::CreateLShr(V1: ConstantExpr::getNot(C), V2: Y);
5328
5329 // ~(C >>u Y) --> ~C >>s Y (when inverting the replicated sign bits)
5330 if (match(V: NotVal, P: m_LShr(L: m_Constant(C), R: m_Value(V&: Y))) &&
5331 match(V: C, P: m_NonNegative()))
5332 return BinaryOperator::CreateAShr(V1: ConstantExpr::getNot(C), V2: Y);
5333
5334 // ~(X + C) --> ~C - X
5335 if (match(V: NotVal, P: m_Add(L: m_Value(V&: X), R: m_ImmConstant(C))))
5336 return BinaryOperator::CreateSub(V1: ConstantExpr::getNot(C), V2: X);
5337
5338 // ~(X - Y) --> ~X + Y
5339 // FIXME: is it really beneficial to sink the `not` here?
5340 if (match(V: NotVal, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y))))
5341 if (isa<Constant>(Val: X) || NotVal->hasOneUse())
5342 return BinaryOperator::CreateAdd(V1: Builder.CreateNot(V: X), V2: Y);
5343
5344 // ~(~X + Y) --> X - Y
5345 if (match(V: NotVal, P: m_c_Add(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y))))
5346 return BinaryOperator::CreateWithCopiedFlags(Opc: Instruction::Sub, V1: X, V2: Y,
5347 CopyO: NotVal);
5348 }
5349
5350 // not (cmp A, B) = !cmp A, B
5351 CmpPredicate Pred;
5352 if (match(V: NotOp, P: m_Cmp(Pred, L: m_Value(), R: m_Value())) &&
5353 (NotOp->hasOneUse() ||
5354 InstCombiner::canFreelyInvertAllUsersOf(V: cast<Instruction>(Val: NotOp),
5355 /*IgnoredUser=*/nullptr))) {
5356 cast<CmpInst>(Val: NotOp)->setPredicate(CmpInst::getInversePredicate(pred: Pred));
5357 freelyInvertAllUsersOf(V: NotOp);
5358 return &I;
5359 }
5360
5361 // not (bitcast (cmp A, B) --> bitcast (!cmp A, B)
5362 if (match(V: NotOp, P: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: X)))) &&
5363 match(V: X, P: m_OneUse(SubPattern: m_Cmp(Pred, L: m_Value(), R: m_Value())))) {
5364 cast<CmpInst>(Val: X)->setPredicate(CmpInst::getInversePredicate(pred: Pred));
5365 return new BitCastInst(X, Ty);
5366 }
5367
5368 // Move a 'not' ahead of casts of a bool to enable logic reduction:
5369 // not (bitcast (sext i1 X)) --> bitcast (sext (not i1 X))
5370 if (match(V: NotOp, P: m_OneUse(SubPattern: m_BitCast(Op: m_OneUse(SubPattern: m_SExt(Op: m_Value(V&: X)))))) &&
5371 X->getType()->isIntOrIntVectorTy(BitWidth: 1)) {
5372 Type *SextTy = cast<BitCastOperator>(Val: NotOp)->getSrcTy();
5373 Value *NotX = Builder.CreateNot(V: X);
5374 Value *Sext = Builder.CreateSExt(V: NotX, DestTy: SextTy);
5375 return new BitCastInst(Sext, Ty);
5376 }
5377
5378 if (auto *NotOpI = dyn_cast<Instruction>(Val: NotOp))
5379 if (sinkNotIntoLogicalOp(I&: *NotOpI))
5380 return &I;
5381
5382 // Eliminate a bitwise 'not' op of 'not' min/max by inverting the min/max:
5383 // ~min(~X, ~Y) --> max(X, Y)
5384 // ~max(~X, Y) --> min(X, ~Y)
5385 auto *II = dyn_cast<IntrinsicInst>(Val: NotOp);
5386 if (II && II->hasOneUse()) {
5387 if (match(V: NotOp, P: m_c_MaxOrMin(L: m_Not(V: m_Value(V&: X)), R: m_Value(V&: Y)))) {
5388 Intrinsic::ID InvID = getInverseMinMaxIntrinsic(MinMaxID: II->getIntrinsicID());
5389 Value *NotY = Builder.CreateNot(V: Y);
5390 Value *InvMaxMin = Builder.CreateBinaryIntrinsic(ID: InvID, LHS: X, RHS: NotY);
5391 return replaceInstUsesWith(I, V: InvMaxMin);
5392 }
5393
5394 if (II->getIntrinsicID() == Intrinsic::is_fpclass) {
5395 ConstantInt *ClassMask = cast<ConstantInt>(Val: II->getArgOperand(i: 1));
5396 II->setArgOperand(
5397 i: 1, v: ConstantInt::get(Ty: ClassMask->getType(),
5398 V: ~ClassMask->getZExtValue() & fcAllFlags));
5399 return replaceInstUsesWith(I, V: II);
5400 }
5401 }
5402
5403 if (NotOp->hasOneUse()) {
5404 // Pull 'not' into operands of select if both operands are one-use compares
5405 // or one is one-use compare and the other one is a constant.
5406 // Inverting the predicates eliminates the 'not' operation.
5407 // Example:
5408 // not (select ?, (cmp TPred, ?, ?), (cmp FPred, ?, ?) -->
5409 // select ?, (cmp InvTPred, ?, ?), (cmp InvFPred, ?, ?)
5410 // not (select ?, (cmp TPred, ?, ?), true -->
5411 // select ?, (cmp InvTPred, ?, ?), false
5412 if (auto *Sel = dyn_cast<SelectInst>(Val: NotOp)) {
5413 Value *TV = Sel->getTrueValue();
5414 Value *FV = Sel->getFalseValue();
5415 auto *CmpT = dyn_cast<CmpInst>(Val: TV);
5416 auto *CmpF = dyn_cast<CmpInst>(Val: FV);
5417 bool InvertibleT = (CmpT && CmpT->hasOneUse()) || isa<Constant>(Val: TV);
5418 bool InvertibleF = (CmpF && CmpF->hasOneUse()) || isa<Constant>(Val: FV);
5419 if (InvertibleT && InvertibleF) {
5420 if (CmpT)
5421 CmpT->setPredicate(CmpT->getInversePredicate());
5422 else
5423 Sel->setTrueValue(ConstantExpr::getNot(C: cast<Constant>(Val: TV)));
5424 if (CmpF)
5425 CmpF->setPredicate(CmpF->getInversePredicate());
5426 else
5427 Sel->setFalseValue(ConstantExpr::getNot(C: cast<Constant>(Val: FV)));
5428 return replaceInstUsesWith(I, V: Sel);
5429 }
5430 }
5431 }
5432
5433 if (Instruction *NewXor = foldNotXor(I, Builder))
5434 return NewXor;
5435
5436 // TODO: Could handle multi-use better by checking if all uses of NotOp (other
5437 // than I) can be inverted.
5438 if (Value *R = getFreelyInverted(V: NotOp, WillInvertAllUses: NotOp->hasOneUse(), Builder: &Builder))
5439 return replaceInstUsesWith(I, V: R);
5440
5441 return nullptr;
5442}
5443
5444// ((X + C) & M) ^ M --> (~C − X) & M
5445static Instruction *foldMaskedAddXorPattern(BinaryOperator &I,
5446 InstCombiner::BuilderTy &Builder) {
5447 Value *X, *Mask;
5448 Constant *AddC;
5449 BinaryOperator *AddInst;
5450 if (match(V: &I,
5451 P: m_Xor(L: m_OneUse(SubPattern: m_And(L: m_OneUse(SubPattern: m_CombineAnd(
5452 Ps: m_BinOp(I&: AddInst),
5453 Ps: m_Add(L: m_Value(V&: X), R: m_ImmConstant(C&: AddC)))),
5454 R: m_Value(V&: Mask))),
5455 R: m_Deferred(V: Mask)))) {
5456 Value *NotC = Builder.CreateNot(V: AddC);
5457 Value *NewSub = Builder.CreateSub(LHS: NotC, RHS: X, Name: "", HasNUW: AddInst->hasNoUnsignedWrap(),
5458 HasNSW: AddInst->hasNoSignedWrap());
5459 return BinaryOperator::CreateAnd(V1: NewSub, V2: Mask);
5460 }
5461
5462 return nullptr;
5463}
5464
5465// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
5466// here. We should standardize that construct where it is needed or choose some
5467// other way to ensure that commutated variants of patterns are not missed.
5468Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
5469 if (Value *V = simplifyXorInst(LHS: I.getOperand(i_nocapture: 0), RHS: I.getOperand(i_nocapture: 1),
5470 Q: SQ.getWithInstruction(I: &I)))
5471 return replaceInstUsesWith(I, V);
5472
5473 if (SimplifyAssociativeOrCommutative(I))
5474 return &I;
5475
5476 if (Instruction *X = foldVectorBinop(Inst&: I))
5477 return X;
5478
5479 if (Instruction *Phi = foldBinopWithPhiOperands(BO&: I))
5480 return Phi;
5481
5482 if (Instruction *NewXor = foldXorToXor(I, Builder))
5483 return NewXor;
5484
5485 // (A&B)^(A&C) -> A&(B^C) etc
5486 if (Value *V = foldUsingDistributiveLaws(I))
5487 return replaceInstUsesWith(I, V);
5488
5489 // See if we can simplify any instructions used by the instruction whose sole
5490 // purpose is to compute bits we don't care about.
5491 if (SimplifyDemandedInstructionBits(Inst&: I))
5492 return &I;
5493
5494 if (Instruction *R = foldNot(I))
5495 return R;
5496
5497 if (Instruction *R = foldBinOpShiftWithShift(I))
5498 return R;
5499
5500 Value *Op0 = I.getOperand(i_nocapture: 0), *Op1 = I.getOperand(i_nocapture: 1);
5501 Value *X, *Y, *M;
5502
5503 // (X | Y) ^ M -> (X ^ M) ^ Y
5504 // (X | Y) ^ M -> (Y ^ M) ^ X
5505 if (match(V: &I, P: m_c_Xor(L: m_OneUse(SubPattern: m_DisjointOr(L: m_Value(V&: X), R: m_Value(V&: Y))),
5506 R: m_Value(V&: M)))) {
5507 if (Value *XorAC = simplifyXorInst(LHS: X, RHS: M, Q: SQ.getWithInstruction(I: &I)))
5508 return BinaryOperator::CreateXor(V1: XorAC, V2: Y);
5509
5510 if (Value *XorBC = simplifyXorInst(LHS: Y, RHS: M, Q: SQ.getWithInstruction(I: &I)))
5511 return BinaryOperator::CreateXor(V1: XorBC, V2: X);
5512 }
5513
5514 // Fold (X & M) ^ (Y & ~M) -> (X & M) | (Y & ~M)
5515 // This it a special case in haveNoCommonBitsSet, but the computeKnownBits
5516 // calls in there are unnecessary as SimplifyDemandedInstructionBits should
5517 // have already taken care of those cases.
5518 if (match(V: &I, P: m_c_Xor(L: m_c_And(L: m_Not(V: m_Value(V&: M)), R: m_Value()),
5519 R: m_c_And(L: m_Deferred(V: M), R: m_Value())))) {
5520 if (isGuaranteedNotToBeUndef(V: M))
5521 return BinaryOperator::CreateDisjointOr(V1: Op0, V2: Op1);
5522 else
5523 return BinaryOperator::CreateOr(V1: Op0, V2: Op1);
5524 }
5525
5526 if (Instruction *Xor = visitMaskedMerge(I, Builder))
5527 return Xor;
5528
5529 Constant *C1;
5530 if (match(V: Op1, P: m_Constant(C&: C1))) {
5531 Constant *C2;
5532
5533 if (match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: X), R: m_ImmConstant(C&: C2)))) &&
5534 match(V: C1, P: m_ImmConstant())) {
5535 // (X | C2) ^ C1 --> (X & ~C2) ^ (C1^C2)
5536 C2 = Constant::replaceUndefsWith(
5537 C: C2, Replacement: Constant::getAllOnesValue(Ty: C2->getType()->getScalarType()));
5538 Value *And = Builder.CreateAnd(
5539 LHS: X, RHS: Constant::mergeUndefsWith(C: ConstantExpr::getNot(C: C2), Other: C1));
5540 return BinaryOperator::CreateXor(
5541 V1: And, V2: Constant::mergeUndefsWith(C: ConstantExpr::getXor(C1, C2), Other: C1));
5542 }
5543
5544 // Use DeMorgan and reassociation to eliminate a 'not' op.
5545 if (match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Not(V: m_Value(V&: X)), R: m_Constant(C&: C2))))) {
5546 // (~X | C2) ^ C1 --> ((X & ~C2) ^ -1) ^ C1 --> (X & ~C2) ^ ~C1
5547 Value *And = Builder.CreateAnd(LHS: X, RHS: ConstantExpr::getNot(C: C2));
5548 return BinaryOperator::CreateXor(V1: And, V2: ConstantExpr::getNot(C: C1));
5549 }
5550 if (match(V: Op0, P: m_OneUse(SubPattern: m_And(L: m_Not(V: m_Value(V&: X)), R: m_Constant(C&: C2))))) {
5551 // (~X & C2) ^ C1 --> ((X | ~C2) ^ -1) ^ C1 --> (X | ~C2) ^ ~C1
5552 Value *Or = Builder.CreateOr(LHS: X, RHS: ConstantExpr::getNot(C: C2));
5553 return BinaryOperator::CreateXor(V1: Or, V2: ConstantExpr::getNot(C: C1));
5554 }
5555
5556 // Convert xor ([trunc] (ashr X, BW-1)), C =>
5557 // select(X >s -1, C, ~C)
5558 // The ashr creates "AllZeroOrAllOne's", which then optionally inverses the
5559 // constant depending on whether this input is less than 0.
5560 const APInt *CA;
5561 if (match(V: Op0, P: m_OneUse(SubPattern: m_TruncOrSelf(
5562 Op: m_AShr(L: m_Value(V&: X), R: m_APIntAllowPoison(Res&: CA))))) &&
5563 *CA == X->getType()->getScalarSizeInBits() - 1 &&
5564 !match(V: C1, P: m_AllOnes())) {
5565 assert(!C1->isNullValue() && "Unexpected xor with 0");
5566 Value *IsNotNeg = Builder.CreateIsNotNeg(Arg: X);
5567 return createSelectInstWithUnknownProfile(C: IsNotNeg, S1: Op1,
5568 S2: Builder.CreateNot(V: Op1));
5569 }
5570 }
5571
5572 Type *Ty = I.getType();
5573 {
5574 const APInt *RHSC;
5575 if (match(V: Op1, P: m_APInt(Res&: RHSC))) {
5576 Value *X;
5577 const APInt *C;
5578 // (C - X) ^ signmaskC --> (C + signmaskC) - X
5579 if (RHSC->isSignMask() && match(V: Op0, P: m_Sub(L: m_APInt(Res&: C), R: m_Value(V&: X))))
5580 return BinaryOperator::CreateSub(V1: ConstantInt::get(Ty, V: *C + *RHSC), V2: X);
5581
5582 // (X + C) ^ signmaskC --> X + (C + signmaskC)
5583 if (RHSC->isSignMask() && match(V: Op0, P: m_Add(L: m_Value(V&: X), R: m_APInt(Res&: C))))
5584 return BinaryOperator::CreateAdd(V1: X, V2: ConstantInt::get(Ty, V: *C + *RHSC));
5585
5586 // (X | C) ^ RHSC --> X ^ (C ^ RHSC) iff X & C == 0
5587 if (match(V: Op0, P: m_Or(L: m_Value(V&: X), R: m_APInt(Res&: C))) &&
5588 MaskedValueIsZero(V: X, Mask: *C, CtxI: &I))
5589 return BinaryOperator::CreateXor(V1: X, V2: ConstantInt::get(Ty, V: *C ^ *RHSC));
5590
5591 // When X is a power-of-two or zero and zero input is poison:
5592 // ctlz(i32 X) ^ 31 --> cttz(X)
5593 // cttz(i32 X) ^ 31 --> ctlz(X)
5594 // The bit width must be a power of two so BW - 1 is an all-ones mask.
5595 auto *II = dyn_cast<IntrinsicInst>(Val: Op0);
5596 if (II && II->hasOneUse() && isPowerOf2_32(Value: Ty->getScalarSizeInBits()) &&
5597 *RHSC == Ty->getScalarSizeInBits() - 1) {
5598 Intrinsic::ID IID = II->getIntrinsicID();
5599 if ((IID == Intrinsic::ctlz || IID == Intrinsic::cttz) &&
5600 match(V: II->getArgOperand(i: 1), P: m_One()) &&
5601 isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), /*OrZero */ true)) {
5602 IID = (IID == Intrinsic::ctlz) ? Intrinsic::cttz : Intrinsic::ctlz;
5603 Function *F =
5604 Intrinsic::getOrInsertDeclaration(M: II->getModule(), id: IID, OverloadTys: Ty);
5605 return CallInst::Create(Func: F, Args: {II->getArgOperand(i: 0), Builder.getTrue()});
5606 }
5607 }
5608
5609 // If RHSC is inverting the remaining bits of shifted X,
5610 // canonicalize to a 'not' before the shift to help SCEV and codegen:
5611 // (X << C) ^ RHSC --> ~X << C
5612 if (match(V: Op0, P: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: X), R: m_APInt(Res&: C)))) &&
5613 *RHSC == APInt::getAllOnes(numBits: Ty->getScalarSizeInBits()).shl(ShiftAmt: *C)) {
5614 Value *NotX = Builder.CreateNot(V: X);
5615 return BinaryOperator::CreateShl(V1: NotX, V2: ConstantInt::get(Ty, V: *C));
5616 }
5617 // (X >>u C) ^ RHSC --> ~X >>u C
5618 if (match(V: Op0, P: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: C)))) &&
5619 *RHSC == APInt::getAllOnes(numBits: Ty->getScalarSizeInBits()).lshr(ShiftAmt: *C)) {
5620 Value *NotX = Builder.CreateNot(V: X);
5621 return BinaryOperator::CreateLShr(V1: NotX, V2: ConstantInt::get(Ty, V: *C));
5622 }
5623 // TODO: We could handle 'ashr' here as well. That would be matching
5624 // a 'not' op and moving it before the shift. Doing that requires
5625 // preventing the inverse fold in canShiftBinOpWithConstantRHS().
5626 }
5627
5628 // If we are XORing the sign bit of a floating-point value, convert
5629 // this to fneg, then cast back to integer.
5630 //
5631 // This is generous interpretation of noimplicitfloat, this is not a true
5632 // floating-point operation.
5633 //
5634 // Assumes any IEEE-represented type has the sign bit in the high bit.
5635 // TODO: Unify with APInt matcher. This version allows undef unlike m_APInt
5636 Value *CastOp;
5637 if (match(V: Op0, P: m_ElementWiseBitCast(Op: m_Value(V&: CastOp))) &&
5638 match(V: Op1, P: m_SignMask()) &&
5639 !Builder.GetInsertBlock()->getParent()->hasFnAttribute(
5640 Kind: Attribute::NoImplicitFloat)) {
5641 Type *EltTy = CastOp->getType()->getScalarType();
5642 if (EltTy->isFloatingPointTy() &&
5643 APFloat::hasSignBitInMSB(EltTy->getFltSemantics())) {
5644 Value *FNeg = Builder.CreateFNeg(V: CastOp);
5645 return new BitCastInst(FNeg, I.getType());
5646 }
5647 }
5648 }
5649
5650 // FIXME: This should not be limited to scalar (pull into APInt match above).
5651 {
5652 Value *X;
5653 ConstantInt *C1, *C2, *C3;
5654 // ((X^C1) >> C2) ^ C3 -> (X>>C2) ^ ((C1>>C2)^C3)
5655 if (match(V: Op1, P: m_ConstantInt(CI&: C3)) &&
5656 match(V: Op0, P: m_LShr(L: m_Xor(L: m_Value(V&: X), R: m_ConstantInt(CI&: C1)),
5657 R: m_ConstantInt(CI&: C2))) &&
5658 Op0->hasOneUse()) {
5659 // fold (C1 >> C2) ^ C3
5660 APInt FoldConst = C1->getValue().lshr(ShiftAmt: C2->getValue());
5661 FoldConst ^= C3->getValue();
5662 // Prepare the two operands.
5663 auto *Opnd0 = Builder.CreateLShr(LHS: X, RHS: C2);
5664 Opnd0->takeName(V: Op0);
5665 return BinaryOperator::CreateXor(V1: Opnd0, V2: ConstantInt::get(Ty, V: FoldConst));
5666 }
5667 }
5668
5669 if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
5670 return FoldedLogic;
5671
5672 if (Instruction *FoldedLogic = foldBinOpSelectBinOp(Op&: I))
5673 return FoldedLogic;
5674
5675 // Y ^ (X | Y) --> X & ~Y
5676 // Y ^ (Y | X) --> X & ~Y
5677 if (match(V: Op1, P: m_OneUse(SubPattern: m_c_Or(L: m_Value(V&: X), R: m_Specific(V: Op0)))))
5678 return BinaryOperator::CreateAnd(V1: X, V2: Builder.CreateNot(V: Op0));
5679 // (X | Y) ^ Y --> X & ~Y
5680 // (Y | X) ^ Y --> X & ~Y
5681 if (match(V: Op0, P: m_OneUse(SubPattern: m_c_Or(L: m_Value(V&: X), R: m_Specific(V: Op1)))))
5682 return BinaryOperator::CreateAnd(V1: X, V2: Builder.CreateNot(V: Op1));
5683
5684 // Y ^ (X & Y) --> ~X & Y
5685 // Y ^ (Y & X) --> ~X & Y
5686 if (match(V: Op1, P: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: X), R: m_Specific(V: Op0)))))
5687 return BinaryOperator::CreateAnd(V1: Op0, V2: Builder.CreateNot(V: X));
5688 // (X & Y) ^ Y --> ~X & Y
5689 // (Y & X) ^ Y --> ~X & Y
5690 // Canonical form is (X & C) ^ C; don't touch that.
5691 // TODO: A 'not' op is better for analysis and codegen, but demanded bits must
5692 // be fixed to prefer that (otherwise we get infinite looping).
5693 if (!match(V: Op1, P: m_Constant()) &&
5694 match(V: Op0, P: m_OneUse(SubPattern: m_c_And(L: m_Value(V&: X), R: m_Specific(V: Op1)))))
5695 return BinaryOperator::CreateAnd(V1: Op1, V2: Builder.CreateNot(V: X));
5696
5697 Value *A, *B, *C;
5698 // (A ^ B) ^ (A | C) --> (~A & C) ^ B -- There are 4 commuted variants.
5699 if (match(V: &I, P: m_c_Xor(L: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))),
5700 R: m_OneUse(SubPattern: m_c_Or(L: m_Deferred(V: A), R: m_Value(V&: C))))))
5701 return BinaryOperator::CreateXor(
5702 V1: Builder.CreateAnd(LHS: Builder.CreateNot(V: A), RHS: C), V2: B);
5703
5704 // (A ^ B) ^ (B | C) --> (~B & C) ^ A -- There are 4 commuted variants.
5705 if (match(V: &I, P: m_c_Xor(L: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))),
5706 R: m_OneUse(SubPattern: m_c_Or(L: m_Deferred(V: B), R: m_Value(V&: C))))))
5707 return BinaryOperator::CreateXor(
5708 V1: Builder.CreateAnd(LHS: Builder.CreateNot(V: B), RHS: C), V2: A);
5709
5710 // (A & B) ^ (A ^ B) -> (A | B)
5711 if (match(V: Op0, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
5712 match(V: Op1, P: m_c_Xor(L: m_Specific(V: A), R: m_Specific(V: B))))
5713 return BinaryOperator::CreateOr(V1: A, V2: B);
5714 // (A ^ B) ^ (A & B) -> (A | B)
5715 if (match(V: Op0, P: m_Xor(L: m_Value(V&: A), R: m_Value(V&: B))) &&
5716 match(V: Op1, P: m_c_And(L: m_Specific(V: A), R: m_Specific(V: B))))
5717 return BinaryOperator::CreateOr(V1: A, V2: B);
5718
5719 // (A & ~B) ^ ~A -> ~(A & B)
5720 // (~B & A) ^ ~A -> ~(A & B)
5721 if (match(V: Op0, P: m_c_And(L: m_Value(V&: A), R: m_Not(V: m_Value(V&: B)))) &&
5722 match(V: Op1, P: m_Not(V: m_Specific(V: A))))
5723 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: A, RHS: B));
5724
5725 // (~A & B) ^ A --> A | B -- There are 4 commuted variants.
5726 if (match(V: &I, P: m_c_Xor(L: m_c_And(L: m_Not(V: m_Value(V&: A)), R: m_Value(V&: B)), R: m_Deferred(V: A))))
5727 return BinaryOperator::CreateOr(V1: A, V2: B);
5728
5729 // (~A | B) ^ A --> ~(A & B)
5730 if (match(V: Op0, P: m_OneUse(SubPattern: m_c_Or(L: m_Not(V: m_Specific(V: Op1)), R: m_Value(V&: B)))))
5731 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: Op1, RHS: B));
5732
5733 // A ^ (~A | B) --> ~(A & B)
5734 if (match(V: Op1, P: m_OneUse(SubPattern: m_c_Or(L: m_Not(V: m_Specific(V: Op0)), R: m_Value(V&: B)))))
5735 return BinaryOperator::CreateNot(Op: Builder.CreateAnd(LHS: Op0, RHS: B));
5736
5737 // (A | B) ^ (A | C) --> (B ^ C) & ~A -- There are 4 commuted variants.
5738 // TODO: Loosen one-use restriction if common operand is a constant.
5739 Value *D;
5740 if (match(V: Op0, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: A), R: m_Value(V&: B)))) &&
5741 match(V: Op1, P: m_OneUse(SubPattern: m_Or(L: m_Value(V&: C), R: m_Value(V&: D))))) {
5742 if (B == C || B == D)
5743 std::swap(a&: A, b&: B);
5744 if (A == C)
5745 std::swap(a&: C, b&: D);
5746 if (A == D) {
5747 Value *NotA = Builder.CreateNot(V: A);
5748 return BinaryOperator::CreateAnd(V1: Builder.CreateXor(LHS: B, RHS: C), V2: NotA);
5749 }
5750 }
5751
5752 // (A & B) ^ (A | C) --> A ? ~B : C -- There are 4 commuted variants.
5753 if (I.getType()->isIntOrIntVectorTy(BitWidth: 1) &&
5754 match(V: &I, P: m_c_Xor(L: m_OneUse(SubPattern: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B))),
5755 R: m_OneUse(SubPattern: m_LogicalOr(L: m_Value(V&: C), R: m_Value(V&: D)))))) {
5756 bool NeedFreeze = isa<SelectInst>(Val: Op0) && isa<SelectInst>(Val: Op1) && B == D;
5757 if (B == C || B == D) {
5758 std::swap(a&: A, b&: B);
5759 }
5760 if (A == C)
5761 std::swap(a&: C, b&: D);
5762 if (A == D) {
5763 if (NeedFreeze)
5764 A = Builder.CreateFreeze(V: A);
5765 Value *NotB = Builder.CreateNot(V: B);
5766 Instruction *MDFrom = nullptr;
5767 // If one of the operands has the same condition as we will use for the
5768 // select we are going to create, pull the metadata from it (primarily the
5769 // profile info).
5770 if (auto *Op0SI = dyn_cast<SelectInst>(Val: Op0)) {
5771 if (Op0SI->getCondition() == A)
5772 MDFrom = Op0SI;
5773 }
5774 if (auto *Op1SI = dyn_cast<SelectInst>(Val: Op1)) {
5775 if (Op1SI->getCondition() == A)
5776 MDFrom = Op1SI;
5777 }
5778 return (MDFrom == nullptr || ProfcheckDisableMetadataFixes)
5779 ? createSelectInstWithUnknownProfile(C: A, S1: NotB, S2: C)
5780 : SelectInst::Create(C: A, S1: NotB, S2: C, NameStr: "", InsertBefore: nullptr, MDFrom);
5781 }
5782 }
5783
5784 if (auto *LHS = dyn_cast<ICmpInst>(Val: I.getOperand(i_nocapture: 0)))
5785 if (auto *RHS = dyn_cast<ICmpInst>(Val: I.getOperand(i_nocapture: 1)))
5786 if (Value *V = foldXorOfICmps(LHS, RHS, I))
5787 return replaceInstUsesWith(I, V);
5788
5789 if (Instruction *CastedXor = foldCastedBitwiseLogic(I))
5790 return CastedXor;
5791
5792 if (Instruction *Abs = canonicalizeAbs(Xor&: I, Builder))
5793 return Abs;
5794
5795 // Otherwise, if all else failed, try to hoist the xor-by-constant:
5796 // (X ^ C) ^ Y --> (X ^ Y) ^ C
5797 // Just like we do in other places, we completely avoid the fold
5798 // for constantexprs, at least to avoid endless combine loop.
5799 if (match(V: &I, P: m_c_Xor(L: m_OneUse(SubPattern: m_Xor(L: m_Value(V&: X, P: m_Unless(P: m_ConstantExpr())),
5800 R: m_ImmConstant(C&: C1))),
5801 R: m_Value(V&: Y))))
5802 return BinaryOperator::CreateXor(V1: Builder.CreateXor(LHS: X, RHS: Y), V2: C1);
5803
5804 if (Instruction *R = reassociateForUses(BO&: I, Builder))
5805 return R;
5806
5807 if (Instruction *Canonicalized = canonicalizeLogicFirst(I, Builder))
5808 return Canonicalized;
5809
5810 if (Instruction *Folded = foldLogicOfIsFPClass(BO&: I, Op0, Op1))
5811 return Folded;
5812
5813 if (Instruction *Folded = canonicalizeConditionalNegationViaMathToSelect(I))
5814 return Folded;
5815
5816 if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
5817 return Res;
5818
5819 if (Instruction *Res = foldBitwiseLogicWithIntrinsics(I, Builder))
5820 return Res;
5821
5822 if (Instruction *Res = foldMaskedAddXorPattern(I, Builder))
5823 return Res;
5824
5825 return nullptr;
5826}
5827