1//===- ValueTracking.cpp - Walk computations to compute properties --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains routines that help analyze properties that chains of
10// computations have.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ValueTracking.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/FloatingPointMode.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/ScopeExit.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/ADT/iterator_range.h"
25#include "llvm/Analysis/AliasAnalysis.h"
26#include "llvm/Analysis/AssumeBundleQueries.h"
27#include "llvm/Analysis/AssumptionCache.h"
28#include "llvm/Analysis/ConstantFolding.h"
29#include "llvm/Analysis/DomConditionCache.h"
30#include "llvm/Analysis/FloatingPointPredicateUtils.h"
31#include "llvm/Analysis/GuardUtils.h"
32#include "llvm/Analysis/InstructionSimplify.h"
33#include "llvm/Analysis/Loads.h"
34#include "llvm/Analysis/LoopInfo.h"
35#include "llvm/Analysis/TargetLibraryInfo.h"
36#include "llvm/Analysis/VectorUtils.h"
37#include "llvm/Analysis/WithCache.h"
38#include "llvm/IR/Argument.h"
39#include "llvm/IR/Attributes.h"
40#include "llvm/IR/BasicBlock.h"
41#include "llvm/IR/Constant.h"
42#include "llvm/IR/ConstantFPRange.h"
43#include "llvm/IR/ConstantRange.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DerivedTypes.h"
46#include "llvm/IR/DiagnosticInfo.h"
47#include "llvm/IR/Dominators.h"
48#include "llvm/IR/EHPersonalities.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GetElementPtrTypeIterator.h"
51#include "llvm/IR/GlobalAlias.h"
52#include "llvm/IR/GlobalValue.h"
53#include "llvm/IR/GlobalVariable.h"
54#include "llvm/IR/InstrTypes.h"
55#include "llvm/IR/Instruction.h"
56#include "llvm/IR/Instructions.h"
57#include "llvm/IR/IntrinsicInst.h"
58#include "llvm/IR/Intrinsics.h"
59#include "llvm/IR/IntrinsicsAArch64.h"
60#include "llvm/IR/IntrinsicsAMDGPU.h"
61#include "llvm/IR/IntrinsicsRISCV.h"
62#include "llvm/IR/IntrinsicsX86.h"
63#include "llvm/IR/LLVMContext.h"
64#include "llvm/IR/Metadata.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/Operator.h"
67#include "llvm/IR/PatternMatch.h"
68#include "llvm/IR/Type.h"
69#include "llvm/IR/User.h"
70#include "llvm/IR/Value.h"
71#include "llvm/Support/Casting.h"
72#include "llvm/Support/CommandLine.h"
73#include "llvm/Support/Compiler.h"
74#include "llvm/Support/ErrorHandling.h"
75#include "llvm/Support/KnownBits.h"
76#include "llvm/Support/KnownFPClass.h"
77#include "llvm/Support/MathExtras.h"
78#include "llvm/TargetParser/RISCVTargetParser.h"
79#include <algorithm>
80#include <cassert>
81#include <cstdint>
82#include <optional>
83#include <utility>
84
85using namespace llvm;
86using namespace llvm::PatternMatch;
87
88// Controls the number of uses of the value searched for possible
89// dominating comparisons.
90static cl::opt<unsigned> DomConditionsMaxUses("dom-conditions-max-uses",
91 cl::Hidden, cl::init(Val: 20));
92
93/// Maximum number of instructions to check between assume and context
94/// instruction.
95static constexpr unsigned MaxInstrsToCheckForFree = 16;
96
97/// Returns the bitwidth of the given scalar or pointer type. For vector types,
98/// returns the element type's bitwidth.
99static unsigned getBitWidth(Type *Ty, const DataLayout &DL) {
100 if (unsigned BitWidth = Ty->getScalarSizeInBits())
101 return BitWidth;
102
103 return DL.getPointerTypeSizeInBits(Ty);
104}
105
106// Given the provided Value and, potentially, a context instruction, return
107// the preferred context instruction (if any).
108static const Instruction *safeCxtI(const Value *V, const Instruction *CxtI) {
109 // If we've been provided with a context instruction, then use that (provided
110 // it has been inserted).
111 if (CxtI && CxtI->getParent())
112 return CxtI;
113
114 // If the value is really an already-inserted instruction, then use that.
115 CxtI = dyn_cast<Instruction>(Val: V);
116 if (CxtI && CxtI->getParent())
117 return CxtI;
118
119 return nullptr;
120}
121
122static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
123 const APInt &DemandedElts,
124 APInt &DemandedLHS, APInt &DemandedRHS) {
125 if (isa<ScalableVectorType>(Val: Shuf->getType())) {
126 assert(DemandedElts == APInt(1,1));
127 DemandedLHS = DemandedRHS = DemandedElts;
128 return true;
129 }
130
131 int NumElts =
132 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
133 return llvm::getShuffleDemandedElts(SrcWidth: NumElts, Mask: Shuf->getShuffleMask(),
134 DemandedElts, DemandedLHS, DemandedRHS);
135}
136
137static void computeKnownBits(const Value *V, const APInt &DemandedElts,
138 KnownBits &Known, const SimplifyQuery &Q,
139 unsigned Depth);
140
141void llvm::computeKnownBits(const Value *V, KnownBits &Known,
142 const SimplifyQuery &Q, unsigned Depth) {
143 // Since the number of lanes in a scalable vector is unknown at compile time,
144 // we track one bit which is implicitly broadcast to all lanes. This means
145 // that all lanes in a scalable vector are considered demanded.
146 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
147 APInt DemandedElts =
148 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
149 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
150}
151
152void llvm::computeKnownBits(const Value *V, KnownBits &Known,
153 const DataLayout &DL, AssumptionCache *AC,
154 const Instruction *CxtI, const DominatorTree *DT,
155 bool UseInstrInfo, unsigned Depth) {
156 computeKnownBits(V, Known,
157 Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo),
158 Depth);
159}
160
161KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
162 AssumptionCache *AC, const Instruction *CxtI,
163 const DominatorTree *DT, bool UseInstrInfo,
164 unsigned Depth) {
165 return computeKnownBits(
166 V, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
167}
168
169KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
170 const DataLayout &DL, AssumptionCache *AC,
171 const Instruction *CxtI,
172 const DominatorTree *DT, bool UseInstrInfo,
173 unsigned Depth) {
174 return computeKnownBits(
175 V, DemandedElts,
176 Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
177}
178
179static bool haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
180 const SimplifyQuery &SQ) {
181 // Look for an inverted mask: (X & ~M) op (Y & M).
182 {
183 Value *M;
184 if (match(V: LHS, P: m_c_And(L: m_Not(V: m_Value(V&: M)), R: m_Value())) &&
185 match(V: RHS, P: m_c_And(L: m_Specific(V: M), R: m_Value())) &&
186 isGuaranteedNotToBeUndef(V: M, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
187 return true;
188 }
189
190 // X op (Y & ~X)
191 if (match(V: RHS, P: m_c_And(L: m_Not(V: m_Specific(V: LHS)), R: m_Value())) &&
192 isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
193 return true;
194
195 // X op ((X & Y) ^ Y) -- this is the canonical form of the previous pattern
196 // for constant Y.
197 Value *Y;
198 if (match(V: RHS,
199 P: m_c_Xor(L: m_c_And(L: m_Specific(V: LHS), R: m_Value(V&: Y)), R: m_Deferred(V: Y))) &&
200 isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT) &&
201 isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
202 return true;
203
204 // Peek through extends to find a 'not' of the other side:
205 // (ext Y) op ext(~Y)
206 if (match(V: LHS, P: m_ZExtOrSExt(Op: m_Value(V&: Y))) &&
207 match(V: RHS, P: m_ZExtOrSExt(Op: m_Not(V: m_Specific(V: Y)))) &&
208 isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
209 return true;
210
211 // Look for: (A & B) op ~(A | B)
212 {
213 Value *A, *B;
214 if (match(V: LHS, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
215 match(V: RHS, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B)))) &&
216 isGuaranteedNotToBeUndef(V: A, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT) &&
217 isGuaranteedNotToBeUndef(V: B, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
218 return true;
219 }
220
221 // Look for: (X << V) op (Y >> (BitWidth - V))
222 // or (X >> V) op (Y << (BitWidth - V))
223 {
224 const Value *V;
225 const APInt *R;
226 if (((match(V: RHS, P: m_Shl(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
227 match(V: LHS, P: m_LShr(L: m_Value(), R: m_Specific(V)))) ||
228 (match(V: RHS, P: m_LShr(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
229 match(V: LHS, P: m_Shl(L: m_Value(), R: m_Specific(V))))) &&
230 R->uge(RHS: LHS->getType()->getScalarSizeInBits()))
231 return true;
232 }
233
234 return false;
235}
236
237bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
238 const WithCache<const Value *> &RHSCache,
239 const SimplifyQuery &SQ) {
240 const Value *LHS = LHSCache.getValue();
241 const Value *RHS = RHSCache.getValue();
242
243 assert(LHS->getType() == RHS->getType() &&
244 "LHS and RHS should have the same type");
245 assert(LHS->getType()->isIntOrIntVectorTy() &&
246 "LHS and RHS should be integers");
247
248 if (haveNoCommonBitsSetSpecialCases(LHS, RHS, SQ) ||
249 haveNoCommonBitsSetSpecialCases(LHS: RHS, RHS: LHS, SQ))
250 return true;
251
252 return KnownBits::haveNoCommonBitsSet(LHS: LHSCache.getKnownBits(Q: SQ),
253 RHS: RHSCache.getKnownBits(Q: SQ));
254}
255
256bool llvm::isOnlyUsedInZeroComparison(const Instruction *I) {
257 return !I->user_empty() &&
258 all_of(Range: I->users(), P: match_fn(P: m_ICmp(L: m_Value(), R: m_Zero())));
259}
260
261bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) {
262 return !I->user_empty() && all_of(Range: I->users(), P: [](const User *U) {
263 CmpPredicate P;
264 return match(V: U, P: m_ICmp(Pred&: P, L: m_Value(), R: m_Zero())) && ICmpInst::isEquality(P);
265 });
266}
267
268bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
269 bool OrZero, AssumptionCache *AC,
270 const Instruction *CxtI,
271 const DominatorTree *DT, bool UseInstrInfo,
272 unsigned Depth) {
273 return ::isKnownToBeAPowerOfTwo(
274 V, OrZero, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo),
275 Depth);
276}
277
278static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
279 const SimplifyQuery &Q, unsigned Depth);
280
281bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
282 unsigned Depth) {
283 return computeKnownBits(V, Q: SQ, Depth).isNonNegative();
284}
285
286bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
287 unsigned Depth) {
288 if (auto *CI = dyn_cast<ConstantInt>(Val: V))
289 return CI->getValue().isStrictlyPositive();
290
291 // If `isKnownNonNegative` ever becomes more sophisticated, make sure to keep
292 // this updated.
293 KnownBits Known = computeKnownBits(V, Q: SQ, Depth);
294 return Known.isNonNegative() &&
295 (Known.isNonZero() || isKnownNonZero(V, Q: SQ, Depth));
296}
297
298bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ,
299 unsigned Depth) {
300 return computeKnownBits(V, Q: SQ, Depth).isNegative();
301}
302
303static bool isKnownNonEqual(const Value *V1, const Value *V2,
304 const APInt &DemandedElts, const SimplifyQuery &Q,
305 unsigned Depth);
306
307bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
308 const SimplifyQuery &Q, unsigned Depth) {
309 // We don't support looking through casts.
310 if (V1 == V2 || V1->getType() != V2->getType())
311 return false;
312 auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType());
313 APInt DemandedElts =
314 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
315 return ::isKnownNonEqual(V1, V2, DemandedElts, Q, Depth);
316}
317
318bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
319 const SimplifyQuery &SQ, unsigned Depth) {
320 KnownBits Known(Mask.getBitWidth());
321 computeKnownBits(V, Known, Q: SQ, Depth);
322 return Mask.isSubsetOf(RHS: Known.Zero);
323}
324
325static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
326 const SimplifyQuery &Q, unsigned Depth);
327
328static unsigned ComputeNumSignBits(const Value *V, const SimplifyQuery &Q,
329 unsigned Depth = 0) {
330 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
331 APInt DemandedElts =
332 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
333 return ComputeNumSignBits(V, DemandedElts, Q, Depth);
334}
335
336unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL,
337 AssumptionCache *AC, const Instruction *CxtI,
338 const DominatorTree *DT, bool UseInstrInfo,
339 unsigned Depth) {
340 return ::ComputeNumSignBits(
341 V, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
342}
343
344unsigned llvm::ComputeMaxSignificantBits(const Value *V, const DataLayout &DL,
345 AssumptionCache *AC,
346 const Instruction *CxtI,
347 const DominatorTree *DT,
348 unsigned Depth) {
349 unsigned SignBits = ComputeNumSignBits(V, DL, AC, CxtI, DT, UseInstrInfo: Depth);
350 return V->getType()->getScalarSizeInBits() - SignBits + 1;
351}
352
353/// Try to detect the lerp pattern: a * (b - c) + c * d
354/// where a >= 0, b >= 0, c >= 0, d >= 0, and b >= c.
355///
356/// In that particular case, we can use the following chain of reasoning:
357///
358/// a * (b - c) + c * d <= a' * (b - c) + a' * c = a' * b where a' = max(a, d)
359///
360/// Since that is true for arbitrary a, b, c and d within our constraints, we
361/// can conclude that:
362///
363/// max(a * (b - c) + c * d) <= max(max(a), max(d)) * max(b) = U
364///
365/// Considering that any result of the lerp would be less or equal to U, it
366/// would have at least the number of leading 0s as in U.
367///
368/// While being quite a specific situation, it is fairly common in computer
369/// graphics in the shape of alpha blending.
370///
371/// Modifies given KnownOut in-place with the inferred information.
372static void computeKnownBitsFromLerpPattern(const Value *Op0, const Value *Op1,
373 const APInt &DemandedElts,
374 KnownBits &KnownOut,
375 const SimplifyQuery &Q,
376 unsigned Depth) {
377
378 Type *Ty = Op0->getType();
379 const unsigned BitWidth = Ty->getScalarSizeInBits();
380
381 // Only handle scalar types for now
382 if (Ty->isVectorTy())
383 return;
384
385 // Try to match: a * (b - c) + c * d.
386 // When a == 1 => A == nullptr, the same applies to d/D as well.
387 const Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
388 const Instruction *SubBC = nullptr;
389
390 const auto MatchSubBC = [&]() {
391 // (b - c) can have two forms that interest us:
392 //
393 // 1. sub nuw %b, %c
394 // 2. xor %c, %b
395 //
396 // For the first case, nuw flag guarantees our requirement b >= c.
397 //
398 // The second case might happen when the analysis can infer that b is a mask
399 // for c and we can transform sub operation into xor (that is usually true
400 // for constant b's). Even though xor is symmetrical, canonicalization
401 // ensures that the constant will be the RHS. We have additional checks
402 // later on to ensure that this xor operation is equivalent to subtraction.
403 return m_Instruction(I&: SubBC, Match: m_CombineOr(L: m_NUWSub(L: m_Value(V&: B), R: m_Value(V&: C)),
404 R: m_Xor(L: m_Value(V&: C), R: m_Value(V&: B))));
405 };
406
407 const auto MatchASubBC = [&]() {
408 // Cases:
409 // - a * (b - c)
410 // - (b - c) * a
411 // - (b - c) <- a implicitly equals 1
412 return m_CombineOr(L: m_c_Mul(L: m_Value(V&: A), R: MatchSubBC()), R: MatchSubBC());
413 };
414
415 const auto MatchCD = [&]() {
416 // Cases:
417 // - d * c
418 // - c * d
419 // - c <- d implicitly equals 1
420 return m_CombineOr(L: m_c_Mul(L: m_Value(V&: D), R: m_Specific(V: C)), R: m_Specific(V: C));
421 };
422
423 const auto Match = [&](const Value *LHS, const Value *RHS) {
424 // We do use m_Specific(C) in MatchCD, so we have to make sure that
425 // it's bound to anything and match(LHS, MatchASubBC()) absolutely
426 // has to evaluate first and return true.
427 //
428 // If Match returns true, it is guaranteed that B != nullptr, C != nullptr.
429 return match(V: LHS, P: MatchASubBC()) && match(V: RHS, P: MatchCD());
430 };
431
432 if (!Match(Op0, Op1) && !Match(Op1, Op0))
433 return;
434
435 const auto ComputeKnownBitsOrOne = [&](const Value *V) {
436 // For some of the values we use the convention of leaving
437 // it nullptr to signify an implicit constant 1.
438 return V ? computeKnownBits(V, DemandedElts, Q, Depth: Depth + 1)
439 : KnownBits::makeConstant(C: APInt(BitWidth, 1));
440 };
441
442 // Check that all operands are non-negative
443 const KnownBits KnownA = ComputeKnownBitsOrOne(A);
444 if (!KnownA.isNonNegative())
445 return;
446
447 const KnownBits KnownD = ComputeKnownBitsOrOne(D);
448 if (!KnownD.isNonNegative())
449 return;
450
451 const KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
452 if (!KnownB.isNonNegative())
453 return;
454
455 const KnownBits KnownC = computeKnownBits(V: C, DemandedElts, Q, Depth: Depth + 1);
456 if (!KnownC.isNonNegative())
457 return;
458
459 // If we matched subtraction as xor, we need to actually check that xor
460 // is semantically equivalent to subtraction.
461 //
462 // For that to be true, b has to be a mask for c or that b's known
463 // ones cover all known and possible ones of c.
464 if (SubBC->getOpcode() == Instruction::Xor &&
465 !KnownC.getMaxValue().isSubsetOf(RHS: KnownB.getMinValue()))
466 return;
467
468 const APInt MaxA = KnownA.getMaxValue();
469 const APInt MaxD = KnownD.getMaxValue();
470 const APInt MaxAD = APIntOps::umax(A: MaxA, B: MaxD);
471 const APInt MaxB = KnownB.getMaxValue();
472
473 // We can't infer leading zeros info if the upper-bound estimate wraps.
474 bool Overflow;
475 const APInt UpperBound = MaxAD.umul_ov(RHS: MaxB, Overflow);
476
477 if (Overflow)
478 return;
479
480 // If we know that x <= y and both are positive than x has at least the same
481 // number of leading zeros as y.
482 const unsigned MinimumNumberOfLeadingZeros = UpperBound.countl_zero();
483 KnownOut.Zero.setHighBits(MinimumNumberOfLeadingZeros);
484}
485
486static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
487 bool NSW, bool NUW,
488 const APInt &DemandedElts,
489 KnownBits &KnownOut, KnownBits &Known2,
490 const SimplifyQuery &Q, unsigned Depth) {
491 computeKnownBits(V: Op1, DemandedElts, Known&: KnownOut, Q, Depth: Depth + 1);
492
493 // If one operand is unknown and we have no nowrap information,
494 // the result will be unknown independently of the second operand.
495 if (KnownOut.isUnknown() && !NSW && !NUW)
496 return;
497
498 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
499 KnownOut = KnownBits::computeForAddSub(Add, NSW, NUW, LHS: Known2, RHS: KnownOut);
500
501 if (!Add && NSW && !KnownOut.isNonNegative() &&
502 (isImpliedByDomCondition(Pred: ICmpInst::ICMP_SLE, LHS: Op1, RHS: Op0, ContextI: Q.CxtI, DL: Q.DL)
503 .value_or(u: false) ||
504 match(V: Op1, P: m_c_SMin(L: m_Specific(V: Op0), R: m_Value()))))
505 KnownOut.makeNonNegative();
506
507 if (Add)
508 // Try to match lerp pattern and combine results
509 computeKnownBitsFromLerpPattern(Op0, Op1, DemandedElts, KnownOut, Q, Depth);
510}
511
512static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
513 bool NUW, const APInt &DemandedElts,
514 KnownBits &Known, KnownBits &Known2,
515 const SimplifyQuery &Q, unsigned Depth) {
516 computeKnownBits(V: Op1, DemandedElts, Known, Q, Depth: Depth + 1);
517 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
518
519 bool isKnownNegative = false;
520 bool isKnownNonNegative = false;
521 // If the multiplication is known not to overflow, compute the sign bit.
522 if (NSW) {
523 if (Op0 == Op1) {
524 // The product of a number with itself is non-negative.
525 isKnownNonNegative = true;
526 } else {
527 bool isKnownNonNegativeOp1 = Known.isNonNegative();
528 bool isKnownNonNegativeOp0 = Known2.isNonNegative();
529 bool isKnownNegativeOp1 = Known.isNegative();
530 bool isKnownNegativeOp0 = Known2.isNegative();
531 // The product of two numbers with the same sign is non-negative.
532 isKnownNonNegative = (isKnownNegativeOp1 && isKnownNegativeOp0) ||
533 (isKnownNonNegativeOp1 && isKnownNonNegativeOp0);
534 if (!isKnownNonNegative && NUW) {
535 // mul nuw nsw with a factor > 1 is non-negative.
536 KnownBits One = KnownBits::makeConstant(C: APInt(Known.getBitWidth(), 1));
537 isKnownNonNegative = KnownBits::sgt(LHS: Known, RHS: One).value_or(u: false) ||
538 KnownBits::sgt(LHS: Known2, RHS: One).value_or(u: false);
539 }
540
541 // The product of a negative number and a non-negative number is either
542 // negative or zero.
543 if (!isKnownNonNegative)
544 isKnownNegative =
545 (isKnownNegativeOp1 && isKnownNonNegativeOp0 &&
546 Known2.isNonZero()) ||
547 (isKnownNegativeOp0 && isKnownNonNegativeOp1 && Known.isNonZero());
548 }
549 }
550
551 bool SelfMultiply = Op0 == Op1;
552 if (SelfMultiply)
553 SelfMultiply &=
554 isGuaranteedNotToBeUndef(V: Op0, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1);
555 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
556
557 if (SelfMultiply) {
558 unsigned SignBits = ComputeNumSignBits(V: Op0, DemandedElts, Q, Depth: Depth + 1);
559 unsigned TyBits = Op0->getType()->getScalarSizeInBits();
560 unsigned OutValidBits = 2 * (TyBits - SignBits + 1);
561
562 if (OutValidBits < TyBits) {
563 APInt KnownZeroMask =
564 APInt::getHighBitsSet(numBits: TyBits, hiBitsSet: TyBits - OutValidBits + 1);
565 Known.Zero |= KnownZeroMask;
566 }
567 }
568
569 // Only make use of no-wrap flags if we failed to compute the sign bit
570 // directly. This matters if the multiplication always overflows, in
571 // which case we prefer to follow the result of the direct computation,
572 // though as the program is invoking undefined behaviour we can choose
573 // whatever we like here.
574 if (isKnownNonNegative && !Known.isNegative())
575 Known.makeNonNegative();
576 else if (isKnownNegative && !Known.isNonNegative())
577 Known.makeNegative();
578}
579
580void llvm::computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
581 KnownBits &Known) {
582 unsigned BitWidth = Known.getBitWidth();
583 unsigned NumRanges = Ranges.getNumOperands() / 2;
584 assert(NumRanges >= 1);
585
586 Known.setAllConflict();
587
588 for (unsigned i = 0; i < NumRanges; ++i) {
589 ConstantInt *Lower =
590 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 0));
591 ConstantInt *Upper =
592 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 1));
593 ConstantRange Range(Lower->getValue(), Upper->getValue());
594 // BitWidth must equal the Ranges BitWidth for the correct number of high
595 // bits to be set.
596 assert(BitWidth == Range.getBitWidth() &&
597 "Known bit width must match range bit width!");
598
599 // The first CommonPrefixBits of all values in Range are equal.
600 unsigned CommonPrefixBits =
601 (Range.getUnsignedMax() ^ Range.getUnsignedMin()).countl_zero();
602 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: CommonPrefixBits);
603 APInt UnsignedMax = Range.getUnsignedMax().zextOrTrunc(width: BitWidth);
604 Known.One &= UnsignedMax & Mask;
605 Known.Zero &= ~UnsignedMax & Mask;
606 }
607}
608
609static bool isEphemeralValueOf(const Instruction *I, const Value *E) {
610 SmallVector<const Instruction *, 16> WorkSet(1, I);
611 SmallPtrSet<const Instruction *, 32> Visited;
612 SmallPtrSet<const Instruction *, 16> EphValues;
613
614 // The instruction defining an assumption's condition itself is always
615 // considered ephemeral to that assumption (even if it has other
616 // non-ephemeral users). See r246696's test case for an example.
617 if (is_contained(Range: I->operands(), Element: E))
618 return true;
619
620 while (!WorkSet.empty()) {
621 const Instruction *V = WorkSet.pop_back_val();
622 if (!Visited.insert(Ptr: V).second)
623 continue;
624
625 // If all uses of this value are ephemeral, then so is this value.
626 if (all_of(Range: V->users(), P: [&](const User *U) {
627 return EphValues.count(Ptr: cast<Instruction>(Val: U));
628 })) {
629 if (V == E)
630 return true;
631
632 if (V == I || (!V->mayHaveSideEffects() && !V->isTerminator())) {
633 EphValues.insert(Ptr: V);
634
635 if (const User *U = dyn_cast<User>(Val: V)) {
636 for (const Use &U : U->operands()) {
637 if (const auto *I = dyn_cast<Instruction>(Val: U.get()))
638 WorkSet.push_back(Elt: I);
639 }
640 }
641 }
642 }
643 }
644
645 return false;
646}
647
648// Is this an intrinsic that cannot be speculated but also cannot trap?
649bool llvm::isAssumeLikeIntrinsic(const Instruction *I) {
650 if (const IntrinsicInst *CI = dyn_cast<IntrinsicInst>(Val: I))
651 return CI->isAssumeLikeIntrinsic();
652
653 return false;
654}
655
656bool llvm::isValidAssumeForContext(const Instruction *Inv,
657 const Instruction *CxtI,
658 const DominatorTree *DT,
659 bool AllowEphemerals) {
660 // There are two restrictions on the use of an assume:
661 // 1. The assume must dominate the context (or the control flow must
662 // reach the assume whenever it reaches the context).
663 // 2. The context must not be in the assume's set of ephemeral values
664 // (otherwise we will use the assume to prove that the condition
665 // feeding the assume is trivially true, thus causing the removal of
666 // the assume).
667
668 if (Inv->getParent() == CxtI->getParent()) {
669 // If Inv and CtxI are in the same block, check if the assume (Inv) is first
670 // in the BB.
671 if (Inv->comesBefore(Other: CxtI))
672 return true;
673
674 // Don't let an assume affect itself - this would cause the problems
675 // `isEphemeralValueOf` is trying to prevent, and it would also make
676 // the loop below go out of bounds.
677 if (!AllowEphemerals && Inv == CxtI)
678 return false;
679
680 // The context comes first, but they're both in the same block.
681 // Make sure there is nothing in between that might interrupt
682 // the control flow, not even CxtI itself.
683 // We limit the scan distance between the assume and its context instruction
684 // to avoid a compile-time explosion. This limit is chosen arbitrarily, so
685 // it can be adjusted if needed (could be turned into a cl::opt).
686 auto Range = make_range(x: CxtI->getIterator(), y: Inv->getIterator());
687 if (!isGuaranteedToTransferExecutionToSuccessor(Range, ScanLimit: 15))
688 return false;
689
690 return AllowEphemerals || !isEphemeralValueOf(I: Inv, E: CxtI);
691 }
692
693 // Inv and CxtI are in different blocks.
694 if (DT) {
695 if (DT->dominates(Def: Inv, User: CxtI))
696 return true;
697 } else if (Inv->getParent() == CxtI->getParent()->getSinglePredecessor() ||
698 Inv->getParent()->isEntryBlock()) {
699 // We don't have a DT, but this trivially dominates.
700 return true;
701 }
702
703 return false;
704}
705
706bool llvm::willNotFreeBetween(const Instruction *Assume,
707 const Instruction *CtxI) {
708 // Helper to check if there are any calls in the range that may free memory.
709 auto hasNoFreeCalls = [](auto Range) {
710 for (const auto &[Idx, I] : enumerate(Range)) {
711 if (Idx > MaxInstrsToCheckForFree)
712 return false;
713 if (const auto *CB = dyn_cast<CallBase>(&I))
714 if (!CB->hasFnAttr(Attribute::NoFree))
715 return false;
716 }
717 return true;
718 };
719
720 // Make sure the current function cannot arrange for another thread to free on
721 // its behalf.
722 if (!CtxI->getFunction()->hasNoSync())
723 return false;
724
725 // Handle cross-block case: CtxI in a successor of Assume's block.
726 const BasicBlock *CtxBB = CtxI->getParent();
727 const BasicBlock *AssumeBB = Assume->getParent();
728 BasicBlock::const_iterator CtxIter = CtxI->getIterator();
729 if (CtxBB != AssumeBB) {
730 if (CtxBB->getSinglePredecessor() != AssumeBB)
731 return false;
732
733 if (!hasNoFreeCalls(make_range(x: CtxBB->begin(), y: CtxIter)))
734 return false;
735
736 CtxIter = AssumeBB->end();
737 } else {
738 // Same block case: check that Assume comes before CtxI.
739 if (!Assume->comesBefore(Other: CtxI))
740 return false;
741 }
742
743 // Check if there are any calls between Assume and CtxIter that may free
744 // memory.
745 return hasNoFreeCalls(make_range(x: Assume->getIterator(), y: CtxIter));
746}
747
748// TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but
749// we still have enough information about `RHS` to conclude non-zero. For
750// example Pred=EQ, RHS=isKnownNonZero. cmpExcludesZero is called in loops
751// so the extra compile time may not be worth it, but possibly a second API
752// should be created for use outside of loops.
753static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) {
754 // v u> y implies v != 0.
755 if (Pred == ICmpInst::ICMP_UGT)
756 return true;
757
758 // Special-case v != 0 to also handle v != null.
759 if (Pred == ICmpInst::ICMP_NE)
760 return match(V: RHS, P: m_Zero());
761
762 // All other predicates - rely on generic ConstantRange handling.
763 const APInt *C;
764 auto Zero = APInt::getZero(numBits: RHS->getType()->getScalarSizeInBits());
765 if (match(V: RHS, P: m_APInt(Res&: C))) {
766 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
767 return !TrueValues.contains(Val: Zero);
768 }
769
770 auto *VC = dyn_cast<ConstantDataVector>(Val: RHS);
771 if (VC == nullptr)
772 return false;
773
774 for (unsigned ElemIdx = 0, NElem = VC->getNumElements(); ElemIdx < NElem;
775 ++ElemIdx) {
776 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(
777 Pred, Other: VC->getElementAsAPInt(i: ElemIdx));
778 if (TrueValues.contains(Val: Zero))
779 return false;
780 }
781 return true;
782}
783
784static void breakSelfRecursivePHI(const Use *U, const PHINode *PHI,
785 Value *&ValOut, Instruction *&CtxIOut,
786 const PHINode **PhiOut = nullptr) {
787 ValOut = U->get();
788 if (ValOut == PHI)
789 return;
790 CtxIOut = PHI->getIncomingBlock(U: *U)->getTerminator();
791 if (PhiOut)
792 *PhiOut = PHI;
793 Value *V;
794 // If the Use is a select of this phi, compute analysis on other arm to break
795 // recursion.
796 // TODO: Min/Max
797 if (match(V: ValOut, P: m_Select(C: m_Value(), L: m_Specific(V: PHI), R: m_Value(V))) ||
798 match(V: ValOut, P: m_Select(C: m_Value(), L: m_Value(V), R: m_Specific(V: PHI))))
799 ValOut = V;
800
801 // Same for select, if this phi is 2-operand phi, compute analysis on other
802 // incoming value to break recursion.
803 // TODO: We could handle any number of incoming edges as long as we only have
804 // two unique values.
805 if (auto *IncPhi = dyn_cast<PHINode>(Val: ValOut);
806 IncPhi && IncPhi->getNumIncomingValues() == 2) {
807 for (int Idx = 0; Idx < 2; ++Idx) {
808 if (IncPhi->getIncomingValue(i: Idx) == PHI) {
809 ValOut = IncPhi->getIncomingValue(i: 1 - Idx);
810 if (PhiOut)
811 *PhiOut = IncPhi;
812 CtxIOut = IncPhi->getIncomingBlock(i: 1 - Idx)->getTerminator();
813 break;
814 }
815 }
816 }
817}
818
819static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) {
820 // Use of assumptions is context-sensitive. If we don't have a context, we
821 // cannot use them!
822 if (!Q.AC || !Q.CxtI)
823 return false;
824
825 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
826 if (!Elem.Assume)
827 continue;
828
829 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
830 assert(I->getFunction() == Q.CxtI->getFunction() &&
831 "Got assumption for the wrong function!");
832
833 if (Elem.Index != AssumptionCache::ExprResultIdx) {
834 if (!V->getType()->isPointerTy())
835 continue;
836 if (RetainedKnowledge RK = getKnowledgeFromBundle(
837 Assume&: *I, BOI: I->bundle_op_info_begin()[Elem.Index])) {
838 if (RK.WasOn != V)
839 continue;
840 bool AssumeImpliesNonNull = [&]() {
841 if (RK.AttrKind == Attribute::NonNull)
842 return true;
843
844 if (RK.AttrKind == Attribute::Dereferenceable) {
845 if (NullPointerIsDefined(F: Q.CxtI->getFunction(),
846 AS: V->getType()->getPointerAddressSpace()))
847 return false;
848 assert(RK.IRArgValue &&
849 "Dereferenceable attribute without IR argument?");
850
851 auto *CI = dyn_cast<ConstantInt>(Val: RK.IRArgValue);
852 return CI && !CI->isZero();
853 }
854
855 return false;
856 }();
857 if (AssumeImpliesNonNull && isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
858 return true;
859 }
860 continue;
861 }
862
863 // Warning: This loop can end up being somewhat performance sensitive.
864 // We're running this loop for once for each value queried resulting in a
865 // runtime of ~O(#assumes * #values).
866
867 Value *RHS;
868 CmpPredicate Pred;
869 auto m_V = m_CombineOr(L: m_Specific(V), R: m_PtrToInt(Op: m_Specific(V)));
870 if (!match(V: I->getArgOperand(i: 0), P: m_c_ICmp(Pred, L: m_V, R: m_Value(V&: RHS))))
871 continue;
872
873 if (cmpExcludesZero(Pred, RHS) && isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
874 return true;
875 }
876
877 return false;
878}
879
880static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
881 Value *LHS, Value *RHS, KnownBits &Known,
882 const SimplifyQuery &Q) {
883 if (RHS->getType()->isPointerTy()) {
884 // Handle comparison of pointer to null explicitly, as it will not be
885 // covered by the m_APInt() logic below.
886 if (LHS == V && match(V: RHS, P: m_Zero())) {
887 switch (Pred) {
888 case ICmpInst::ICMP_EQ:
889 Known.setAllZero();
890 break;
891 case ICmpInst::ICMP_SGE:
892 case ICmpInst::ICMP_SGT:
893 Known.makeNonNegative();
894 break;
895 case ICmpInst::ICMP_SLT:
896 Known.makeNegative();
897 break;
898 default:
899 break;
900 }
901 }
902 return;
903 }
904
905 unsigned BitWidth = Known.getBitWidth();
906 auto m_V =
907 m_CombineOr(L: m_Specific(V), R: m_PtrToIntSameSize(DL: Q.DL, Op: m_Specific(V)));
908
909 Value *Y;
910 const APInt *Mask, *C;
911 if (!match(V: RHS, P: m_APInt(Res&: C)))
912 return;
913
914 uint64_t ShAmt;
915 switch (Pred) {
916 case ICmpInst::ICMP_EQ:
917 // assume(V = C)
918 if (match(V: LHS, P: m_V)) {
919 Known = Known.unionWith(RHS: KnownBits::makeConstant(C: *C));
920 // assume(V & Mask = C)
921 } else if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value(V&: Y)))) {
922 // For one bits in Mask, we can propagate bits from C to V.
923 Known.One |= *C;
924 if (match(V: Y, P: m_APInt(Res&: Mask)))
925 Known.Zero |= ~*C & *Mask;
926 // assume(V | Mask = C)
927 } else if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value(V&: Y)))) {
928 // For zero bits in Mask, we can propagate bits from C to V.
929 Known.Zero |= ~*C;
930 if (match(V: Y, P: m_APInt(Res&: Mask)))
931 Known.One |= *C & ~*Mask;
932 // assume(V << ShAmt = C)
933 } else if (match(V: LHS, P: m_Shl(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
934 ShAmt < BitWidth) {
935 // For those bits in C that are known, we can propagate them to known
936 // bits in V shifted to the right by ShAmt.
937 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
938 RHSKnown >>= ShAmt;
939 Known = Known.unionWith(RHS: RHSKnown);
940 // assume(V >> ShAmt = C)
941 } else if (match(V: LHS, P: m_Shr(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
942 ShAmt < BitWidth) {
943 // For those bits in RHS that are known, we can propagate them to known
944 // bits in V shifted to the right by C.
945 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
946 RHSKnown <<= ShAmt;
947 Known = Known.unionWith(RHS: RHSKnown);
948 }
949 break;
950 case ICmpInst::ICMP_NE: {
951 // assume (V & B != 0) where B is a power of 2
952 const APInt *BPow2;
953 if (C->isZero() && match(V: LHS, P: m_And(L: m_V, R: m_Power2(V&: BPow2))))
954 Known.One |= *BPow2;
955 break;
956 }
957 default: {
958 const APInt *Offset = nullptr;
959 if (match(V: LHS, P: m_CombineOr(L: m_V, R: m_AddLike(L: m_V, R: m_APInt(Res&: Offset))))) {
960 ConstantRange LHSRange = ConstantRange::makeAllowedICmpRegion(Pred, Other: *C);
961 if (Offset)
962 LHSRange = LHSRange.sub(Other: *Offset);
963 Known = Known.unionWith(RHS: LHSRange.toKnownBits());
964 }
965 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
966 // X & Y u> C -> X u> C && Y u> C
967 // X nuw- Y u> C -> X u> C
968 if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value())) ||
969 match(V: LHS, P: m_NUWSub(L: m_V, R: m_Value())))
970 Known.One.setHighBits(
971 (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
972 }
973 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
974 // X | Y u< C -> X u< C && Y u< C
975 // X nuw+ Y u< C -> X u< C && Y u< C
976 if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value())) ||
977 match(V: LHS, P: m_c_NUWAdd(L: m_V, R: m_Value()))) {
978 Known.Zero.setHighBits(
979 (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
980 }
981 }
982 } break;
983 }
984}
985
986static void computeKnownBitsFromICmpCond(const Value *V, ICmpInst *Cmp,
987 KnownBits &Known,
988 const SimplifyQuery &SQ, bool Invert) {
989 ICmpInst::Predicate Pred =
990 Invert ? Cmp->getInversePredicate() : Cmp->getPredicate();
991 Value *LHS = Cmp->getOperand(i_nocapture: 0);
992 Value *RHS = Cmp->getOperand(i_nocapture: 1);
993
994 // Handle icmp pred (trunc V), C
995 if (match(V: LHS, P: m_Trunc(Op: m_Specific(V)))) {
996 KnownBits DstKnown(LHS->getType()->getScalarSizeInBits());
997 computeKnownBitsFromCmp(V: LHS, Pred, LHS, RHS, Known&: DstKnown, Q: SQ);
998 if (cast<TruncInst>(Val: LHS)->hasNoUnsignedWrap())
999 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1000 else
1001 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1002 return;
1003 }
1004
1005 computeKnownBitsFromCmp(V, Pred, LHS, RHS, Known, Q: SQ);
1006}
1007
1008static void computeKnownBitsFromCond(const Value *V, Value *Cond,
1009 KnownBits &Known, const SimplifyQuery &SQ,
1010 bool Invert, unsigned Depth) {
1011 Value *A, *B;
1012 if (Depth < MaxAnalysisRecursionDepth &&
1013 match(V: Cond, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
1014 KnownBits Known2(Known.getBitWidth());
1015 KnownBits Known3(Known.getBitWidth());
1016 computeKnownBitsFromCond(V, Cond: A, Known&: Known2, SQ, Invert, Depth: Depth + 1);
1017 computeKnownBitsFromCond(V, Cond: B, Known&: Known3, SQ, Invert, Depth: Depth + 1);
1018 if (Invert ? match(V: Cond, P: m_LogicalOr(L: m_Value(), R: m_Value()))
1019 : match(V: Cond, P: m_LogicalAnd(L: m_Value(), R: m_Value())))
1020 Known2 = Known2.unionWith(RHS: Known3);
1021 else
1022 Known2 = Known2.intersectWith(RHS: Known3);
1023 Known = Known.unionWith(RHS: Known2);
1024 return;
1025 }
1026
1027 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
1028 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ, Invert);
1029 return;
1030 }
1031
1032 if (match(V: Cond, P: m_Trunc(Op: m_Specific(V)))) {
1033 KnownBits DstKnown(1);
1034 if (Invert) {
1035 DstKnown.setAllZero();
1036 } else {
1037 DstKnown.setAllOnes();
1038 }
1039 if (cast<TruncInst>(Val: Cond)->hasNoUnsignedWrap()) {
1040 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1041 return;
1042 }
1043 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1044 return;
1045 }
1046
1047 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A))))
1048 computeKnownBitsFromCond(V, Cond: A, Known, SQ, Invert: !Invert, Depth: Depth + 1);
1049}
1050
1051void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
1052 const SimplifyQuery &Q, unsigned Depth) {
1053 // Handle injected condition.
1054 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
1055 computeKnownBitsFromCond(V, Cond: Q.CC->Cond, Known, SQ: Q, Invert: Q.CC->Invert, Depth);
1056
1057 if (!Q.CxtI)
1058 return;
1059
1060 if (Q.DC && Q.DT) {
1061 // Handle dominating conditions.
1062 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
1063 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
1064 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
1065 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1066 /*Invert*/ false, Depth);
1067
1068 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
1069 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
1070 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1071 /*Invert*/ true, Depth);
1072 }
1073
1074 if (Known.hasConflict())
1075 Known.resetAll();
1076 }
1077
1078 if (!Q.AC)
1079 return;
1080
1081 unsigned BitWidth = Known.getBitWidth();
1082
1083 // Note that the patterns below need to be kept in sync with the code
1084 // in AssumptionCache::updateAffectedValues.
1085
1086 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
1087 if (!Elem.Assume)
1088 continue;
1089
1090 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
1091 assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() &&
1092 "Got assumption for the wrong function!");
1093
1094 if (Elem.Index != AssumptionCache::ExprResultIdx) {
1095 if (!V->getType()->isPointerTy())
1096 continue;
1097 if (RetainedKnowledge RK = getKnowledgeFromBundle(
1098 Assume&: *I, BOI: I->bundle_op_info_begin()[Elem.Index])) {
1099 // Allow AllowEphemerals in isValidAssumeForContext, as the CxtI might
1100 // be the producer of the pointer in the bundle. At the moment, align
1101 // assumptions aren't optimized away.
1102 if (RK.WasOn == V && RK.AttrKind == Attribute::Alignment &&
1103 isPowerOf2_64(Value: RK.ArgValue) &&
1104 isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT, /*AllowEphemerals*/ true))
1105 Known.Zero.setLowBits(Log2_64(Value: RK.ArgValue));
1106 }
1107 continue;
1108 }
1109
1110 // Warning: This loop can end up being somewhat performance sensitive.
1111 // We're running this loop for once for each value queried resulting in a
1112 // runtime of ~O(#assumes * #values).
1113
1114 Value *Arg = I->getArgOperand(i: 0);
1115
1116 if (Arg == V && isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT)) {
1117 assert(BitWidth == 1 && "assume operand is not i1?");
1118 (void)BitWidth;
1119 Known.setAllOnes();
1120 return;
1121 }
1122 if (match(V: Arg, P: m_Not(V: m_Specific(V))) &&
1123 isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT)) {
1124 assert(BitWidth == 1 && "assume operand is not i1?");
1125 (void)BitWidth;
1126 Known.setAllZero();
1127 return;
1128 }
1129 auto *Trunc = dyn_cast<TruncInst>(Val: Arg);
1130 if (Trunc && Trunc->getOperand(i_nocapture: 0) == V &&
1131 isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT)) {
1132 if (Trunc->hasNoUnsignedWrap()) {
1133 Known = KnownBits::makeConstant(C: APInt(BitWidth, 1));
1134 return;
1135 }
1136 Known.One.setBit(0);
1137 return;
1138 }
1139
1140 // The remaining tests are all recursive, so bail out if we hit the limit.
1141 if (Depth == MaxAnalysisRecursionDepth)
1142 continue;
1143
1144 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
1145 if (!Cmp)
1146 continue;
1147
1148 if (!isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
1149 continue;
1150
1151 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ: Q, /*Invert=*/false);
1152 }
1153
1154 // Conflicting assumption: Undefined behavior will occur on this execution
1155 // path.
1156 if (Known.hasConflict())
1157 Known.resetAll();
1158}
1159
1160/// Compute known bits from a shift operator, including those with a
1161/// non-constant shift amount. Known is the output of this function. Known2 is a
1162/// pre-allocated temporary with the same bit width as Known and on return
1163/// contains the known bit of the shift value source. KF is an
1164/// operator-specific function that, given the known-bits and a shift amount,
1165/// compute the implied known-bits of the shift operator's result respectively
1166/// for that shift amount. The results from calling KF are conservatively
1167/// combined for all permitted shift amounts.
1168static void computeKnownBitsFromShiftOperator(
1169 const Operator *I, const APInt &DemandedElts, KnownBits &Known,
1170 KnownBits &Known2, const SimplifyQuery &Q, unsigned Depth,
1171 function_ref<KnownBits(const KnownBits &, const KnownBits &, bool)> KF) {
1172 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1173 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1174 // To limit compile-time impact, only query isKnownNonZero() if we know at
1175 // least something about the shift amount.
1176 bool ShAmtNonZero =
1177 Known.isNonZero() ||
1178 (Known.getMaxValue().ult(RHS: Known.getBitWidth()) &&
1179 isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1));
1180 Known = KF(Known2, Known, ShAmtNonZero);
1181}
1182
1183static KnownBits
1184getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
1185 const KnownBits &KnownLHS, const KnownBits &KnownRHS,
1186 const SimplifyQuery &Q, unsigned Depth) {
1187 unsigned BitWidth = KnownLHS.getBitWidth();
1188 KnownBits KnownOut(BitWidth);
1189 bool IsAnd = false;
1190 bool HasKnownOne = !KnownLHS.One.isZero() || !KnownRHS.One.isZero();
1191 Value *X = nullptr, *Y = nullptr;
1192
1193 switch (I->getOpcode()) {
1194 case Instruction::And:
1195 KnownOut = KnownLHS & KnownRHS;
1196 IsAnd = true;
1197 // and(x, -x) is common idioms that will clear all but lowest set
1198 // bit. If we have a single known bit in x, we can clear all bits
1199 // above it.
1200 // TODO: instcombine often reassociates independent `and` which can hide
1201 // this pattern. Try to match and(x, and(-x, y)) / and(and(x, y), -x).
1202 if (HasKnownOne && match(V: I, P: m_c_And(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X))))) {
1203 // -(-x) == x so using whichever (LHS/RHS) gets us a better result.
1204 if (KnownLHS.countMaxTrailingZeros() <= KnownRHS.countMaxTrailingZeros())
1205 KnownOut = KnownLHS.blsi();
1206 else
1207 KnownOut = KnownRHS.blsi();
1208 }
1209 break;
1210 case Instruction::Or:
1211 KnownOut = KnownLHS | KnownRHS;
1212 break;
1213 case Instruction::Xor:
1214 KnownOut = KnownLHS ^ KnownRHS;
1215 // xor(x, x-1) is common idioms that will clear all but lowest set
1216 // bit. If we have a single known bit in x, we can clear all bits
1217 // above it.
1218 // TODO: xor(x, x-1) is often rewritting as xor(x, x-C) where C !=
1219 // -1 but for the purpose of demanded bits (xor(x, x-C) &
1220 // Demanded) == (xor(x, x-1) & Demanded). Extend the xor pattern
1221 // to use arbitrary C if xor(x, x-C) as the same as xor(x, x-1).
1222 if (HasKnownOne &&
1223 match(V: I, P: m_c_Xor(L: m_Value(V&: X), R: m_Add(L: m_Deferred(V: X), R: m_AllOnes())))) {
1224 const KnownBits &XBits = I->getOperand(i: 0) == X ? KnownLHS : KnownRHS;
1225 KnownOut = XBits.blsmsk();
1226 }
1227 break;
1228 default:
1229 llvm_unreachable("Invalid Op used in 'analyzeKnownBitsFromAndXorOr'");
1230 }
1231
1232 // and(x, add (x, -1)) is a common idiom that always clears the low bit;
1233 // xor/or(x, add (x, -1)) is an idiom that will always set the low bit.
1234 // here we handle the more general case of adding any odd number by
1235 // matching the form and/xor/or(x, add(x, y)) where y is odd.
1236 // TODO: This could be generalized to clearing any bit set in y where the
1237 // following bit is known to be unset in y.
1238 if (!KnownOut.Zero[0] && !KnownOut.One[0] &&
1239 (match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_c_Add(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1240 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1241 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Value(V&: Y), R: m_Deferred(V: X)))))) {
1242 KnownBits KnownY(BitWidth);
1243 computeKnownBits(V: Y, DemandedElts, Known&: KnownY, Q, Depth: Depth + 1);
1244 if (KnownY.countMinTrailingOnes() > 0) {
1245 if (IsAnd)
1246 KnownOut.Zero.setBit(0);
1247 else
1248 KnownOut.One.setBit(0);
1249 }
1250 }
1251 return KnownOut;
1252}
1253
1254static KnownBits computeKnownBitsForHorizontalOperation(
1255 const Operator *I, const APInt &DemandedElts, const SimplifyQuery &Q,
1256 unsigned Depth,
1257 const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
1258 KnownBitsFunc) {
1259 APInt DemandedEltsLHS, DemandedEltsRHS;
1260 getHorizDemandedEltsForFirstOperand(VectorBitWidth: Q.DL.getTypeSizeInBits(Ty: I->getType()),
1261 DemandedElts, DemandedLHS&: DemandedEltsLHS,
1262 DemandedRHS&: DemandedEltsRHS);
1263
1264 const auto ComputeForSingleOpFunc =
1265 [Depth, &Q, KnownBitsFunc](const Value *Op, APInt &DemandedEltsOp) {
1266 return KnownBitsFunc(
1267 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp, Q, Depth: Depth + 1),
1268 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp << 1, Q, Depth: Depth + 1));
1269 };
1270
1271 if (DemandedEltsRHS.isZero())
1272 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS);
1273 if (DemandedEltsLHS.isZero())
1274 return ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS);
1275
1276 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS)
1277 .intersectWith(RHS: ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS));
1278}
1279
1280// Public so this can be used in `SimplifyDemandedUseBits`.
1281KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
1282 const KnownBits &KnownLHS,
1283 const KnownBits &KnownRHS,
1284 const SimplifyQuery &SQ,
1285 unsigned Depth) {
1286 auto *FVTy = dyn_cast<FixedVectorType>(Val: I->getType());
1287 APInt DemandedElts =
1288 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
1289
1290 return getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS, KnownRHS, Q: SQ,
1291 Depth);
1292}
1293
1294ConstantRange llvm::getVScaleRange(const Function *F, unsigned BitWidth) {
1295 Attribute Attr = F->getFnAttribute(Kind: Attribute::VScaleRange);
1296 // Without vscale_range, we only know that vscale is non-zero.
1297 if (!Attr.isValid())
1298 return ConstantRange(APInt(BitWidth, 1), APInt::getZero(numBits: BitWidth));
1299
1300 unsigned AttrMin = Attr.getVScaleRangeMin();
1301 // Minimum is larger than vscale width, result is always poison.
1302 if ((unsigned)llvm::bit_width(Value: AttrMin) > BitWidth)
1303 return ConstantRange::getEmpty(BitWidth);
1304
1305 APInt Min(BitWidth, AttrMin);
1306 std::optional<unsigned> AttrMax = Attr.getVScaleRangeMax();
1307 if (!AttrMax || (unsigned)llvm::bit_width(Value: *AttrMax) > BitWidth)
1308 return ConstantRange(Min, APInt::getZero(numBits: BitWidth));
1309
1310 return ConstantRange(Min, APInt(BitWidth, *AttrMax) + 1);
1311}
1312
1313void llvm::adjustKnownBitsForSelectArm(KnownBits &Known, Value *Cond,
1314 Value *Arm, bool Invert,
1315 const SimplifyQuery &Q, unsigned Depth) {
1316 // If we have a constant arm, we are done.
1317 if (Known.isConstant())
1318 return;
1319
1320 // See what condition implies about the bits of the select arm.
1321 KnownBits CondRes(Known.getBitWidth());
1322 computeKnownBitsFromCond(V: Arm, Cond, Known&: CondRes, SQ: Q, Invert, Depth: Depth + 1);
1323 // If we don't get any information from the condition, no reason to
1324 // proceed.
1325 if (CondRes.isUnknown())
1326 return;
1327
1328 // We can have conflict if the condition is dead. I.e if we have
1329 // (x | 64) < 32 ? (x | 64) : y
1330 // we will have conflict at bit 6 from the condition/the `or`.
1331 // In that case just return. Its not particularly important
1332 // what we do, as this select is going to be simplified soon.
1333 CondRes = CondRes.unionWith(RHS: Known);
1334 if (CondRes.hasConflict())
1335 return;
1336
1337 // Finally make sure the information we found is valid. This is relatively
1338 // expensive so it's left for the very end.
1339 if (!isGuaranteedNotToBeUndef(V: Arm, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1))
1340 return;
1341
1342 // Finally, we know we get information from the condition and its valid,
1343 // so return it.
1344 Known = std::move(CondRes);
1345}
1346
1347// Match a signed min+max clamp pattern like smax(smin(In, CHigh), CLow).
1348// Returns the input and lower/upper bounds.
1349static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
1350 const APInt *&CLow, const APInt *&CHigh) {
1351 assert(isa<Operator>(Select) &&
1352 cast<Operator>(Select)->getOpcode() == Instruction::Select &&
1353 "Input should be a Select!");
1354
1355 const Value *LHS = nullptr, *RHS = nullptr;
1356 SelectPatternFlavor SPF = matchSelectPattern(V: Select, LHS, RHS).Flavor;
1357 if (SPF != SPF_SMAX && SPF != SPF_SMIN)
1358 return false;
1359
1360 if (!match(V: RHS, P: m_APInt(Res&: CLow)))
1361 return false;
1362
1363 const Value *LHS2 = nullptr, *RHS2 = nullptr;
1364 SelectPatternFlavor SPF2 = matchSelectPattern(V: LHS, LHS&: LHS2, RHS&: RHS2).Flavor;
1365 if (getInverseMinMaxFlavor(SPF) != SPF2)
1366 return false;
1367
1368 if (!match(V: RHS2, P: m_APInt(Res&: CHigh)))
1369 return false;
1370
1371 if (SPF == SPF_SMIN)
1372 std::swap(a&: CLow, b&: CHigh);
1373
1374 In = LHS2;
1375 return CLow->sle(RHS: *CHigh);
1376}
1377
1378static bool isSignedMinMaxIntrinsicClamp(const IntrinsicInst *II,
1379 const APInt *&CLow,
1380 const APInt *&CHigh) {
1381 assert((II->getIntrinsicID() == Intrinsic::smin ||
1382 II->getIntrinsicID() == Intrinsic::smax) &&
1383 "Must be smin/smax");
1384
1385 Intrinsic::ID InverseID = getInverseMinMaxIntrinsic(MinMaxID: II->getIntrinsicID());
1386 auto *InnerII = dyn_cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
1387 if (!InnerII || InnerII->getIntrinsicID() != InverseID ||
1388 !match(V: II->getArgOperand(i: 1), P: m_APInt(Res&: CLow)) ||
1389 !match(V: InnerII->getArgOperand(i: 1), P: m_APInt(Res&: CHigh)))
1390 return false;
1391
1392 if (II->getIntrinsicID() == Intrinsic::smin)
1393 std::swap(a&: CLow, b&: CHigh);
1394 return CLow->sle(RHS: *CHigh);
1395}
1396
1397static void unionWithMinMaxIntrinsicClamp(const IntrinsicInst *II,
1398 KnownBits &Known) {
1399 const APInt *CLow, *CHigh;
1400 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
1401 Known = Known.unionWith(
1402 RHS: ConstantRange::getNonEmpty(Lower: *CLow, Upper: *CHigh + 1).toKnownBits());
1403}
1404
1405static void computeKnownBitsFromOperator(const Operator *I,
1406 const APInt &DemandedElts,
1407 KnownBits &Known,
1408 const SimplifyQuery &Q,
1409 unsigned Depth) {
1410 unsigned BitWidth = Known.getBitWidth();
1411
1412 KnownBits Known2(BitWidth);
1413 switch (I->getOpcode()) {
1414 default: break;
1415 case Instruction::Load:
1416 if (MDNode *MD =
1417 Q.IIQ.getMetadata(I: cast<LoadInst>(Val: I), KindID: LLVMContext::MD_range))
1418 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
1419 break;
1420 case Instruction::And:
1421 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1422 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1423
1424 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1425 break;
1426 case Instruction::Or:
1427 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1428 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1429
1430 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1431 break;
1432 case Instruction::Xor:
1433 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1434 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1435
1436 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1437 break;
1438 case Instruction::Mul: {
1439 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1440 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1441 computeKnownBitsMul(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1442 DemandedElts, Known, Known2, Q, Depth);
1443 break;
1444 }
1445 case Instruction::UDiv: {
1446 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1447 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1448 Known =
1449 KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1450 break;
1451 }
1452 case Instruction::SDiv: {
1453 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1454 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1455 Known =
1456 KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1457 break;
1458 }
1459 case Instruction::Select: {
1460 auto ComputeForArm = [&](Value *Arm, bool Invert) {
1461 KnownBits Res(Known.getBitWidth());
1462 computeKnownBits(V: Arm, DemandedElts, Known&: Res, Q, Depth: Depth + 1);
1463 adjustKnownBitsForSelectArm(Known&: Res, Cond: I->getOperand(i: 0), Arm, Invert, Q, Depth);
1464 return Res;
1465 };
1466 // Only known if known in both the LHS and RHS.
1467 Known =
1468 ComputeForArm(I->getOperand(i: 1), /*Invert=*/false)
1469 .intersectWith(RHS: ComputeForArm(I->getOperand(i: 2), /*Invert=*/true));
1470 break;
1471 }
1472 case Instruction::FPTrunc:
1473 case Instruction::FPExt:
1474 case Instruction::FPToUI:
1475 case Instruction::FPToSI:
1476 case Instruction::SIToFP:
1477 case Instruction::UIToFP:
1478 break; // Can't work with floating point.
1479 case Instruction::PtrToInt:
1480 case Instruction::PtrToAddr:
1481 case Instruction::IntToPtr:
1482 // Fall through and handle them the same as zext/trunc.
1483 [[fallthrough]];
1484 case Instruction::ZExt:
1485 case Instruction::Trunc: {
1486 Type *SrcTy = I->getOperand(i: 0)->getType();
1487
1488 unsigned SrcBitWidth;
1489 // Note that we handle pointer operands here because of inttoptr/ptrtoint
1490 // which fall through here.
1491 Type *ScalarTy = SrcTy->getScalarType();
1492 SrcBitWidth = ScalarTy->isPointerTy() ?
1493 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
1494 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
1495
1496 assert(SrcBitWidth && "SrcBitWidth can't be zero");
1497 Known = Known.anyextOrTrunc(BitWidth: SrcBitWidth);
1498 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1499 if (auto *Inst = dyn_cast<PossiblyNonNegInst>(Val: I);
1500 Inst && Inst->hasNonNeg() && !Known.isNegative())
1501 Known.makeNonNegative();
1502 Known = Known.zextOrTrunc(BitWidth);
1503 break;
1504 }
1505 case Instruction::BitCast: {
1506 Type *SrcTy = I->getOperand(i: 0)->getType();
1507 if (SrcTy->isIntOrPtrTy() &&
1508 // TODO: For now, not handling conversions like:
1509 // (bitcast i64 %x to <2 x i32>)
1510 !I->getType()->isVectorTy()) {
1511 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1512 break;
1513 }
1514
1515 const Value *V;
1516 // Handle bitcast from floating point to integer.
1517 if (match(V: I, P: m_ElementWiseBitCast(Op: m_Value(V))) &&
1518 V->getType()->isFPOrFPVectorTy()) {
1519 Type *FPType = V->getType()->getScalarType();
1520 KnownFPClass Result =
1521 computeKnownFPClass(V, DemandedElts, InterestedClasses: fcAllFlags, SQ: Q, Depth: Depth + 1);
1522 FPClassTest FPClasses = Result.KnownFPClasses;
1523
1524 // TODO: Treat it as zero/poison if the use of I is unreachable.
1525 if (FPClasses == fcNone)
1526 break;
1527
1528 if (Result.isKnownNever(Mask: fcNormal | fcSubnormal | fcNan)) {
1529 Known.setAllConflict();
1530
1531 if (FPClasses & fcInf)
1532 Known = Known.intersectWith(RHS: KnownBits::makeConstant(
1533 C: APFloat::getInf(Sem: FPType->getFltSemantics()).bitcastToAPInt()));
1534
1535 if (FPClasses & fcZero)
1536 Known = Known.intersectWith(RHS: KnownBits::makeConstant(
1537 C: APInt::getZero(numBits: FPType->getScalarSizeInBits())));
1538
1539 Known.Zero.clearSignBit();
1540 Known.One.clearSignBit();
1541 }
1542
1543 if (Result.SignBit) {
1544 if (*Result.SignBit)
1545 Known.makeNegative();
1546 else
1547 Known.makeNonNegative();
1548 }
1549
1550 break;
1551 }
1552
1553 // Handle cast from vector integer type to scalar or vector integer.
1554 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy);
1555 if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
1556 !I->getType()->isIntOrIntVectorTy() ||
1557 isa<ScalableVectorType>(Val: I->getType()))
1558 break;
1559
1560 unsigned NumElts = DemandedElts.getBitWidth();
1561 bool IsLE = Q.DL.isLittleEndian();
1562 // Look through a cast from narrow vector elements to wider type.
1563 // Examples: v4i32 -> v2i64, v3i8 -> v24
1564 unsigned SubBitWidth = SrcVecTy->getScalarSizeInBits();
1565 if (BitWidth % SubBitWidth == 0) {
1566 // Known bits are automatically intersected across demanded elements of a
1567 // vector. So for example, if a bit is computed as known zero, it must be
1568 // zero across all demanded elements of the vector.
1569 //
1570 // For this bitcast, each demanded element of the output is sub-divided
1571 // across a set of smaller vector elements in the source vector. To get
1572 // the known bits for an entire element of the output, compute the known
1573 // bits for each sub-element sequentially. This is done by shifting the
1574 // one-set-bit demanded elements parameter across the sub-elements for
1575 // consecutive calls to computeKnownBits. We are using the demanded
1576 // elements parameter as a mask operator.
1577 //
1578 // The known bits of each sub-element are then inserted into place
1579 // (dependent on endian) to form the full result of known bits.
1580 unsigned SubScale = BitWidth / SubBitWidth;
1581 APInt SubDemandedElts = APInt::getZero(numBits: NumElts * SubScale);
1582 for (unsigned i = 0; i != NumElts; ++i) {
1583 if (DemandedElts[i])
1584 SubDemandedElts.setBit(i * SubScale);
1585 }
1586
1587 KnownBits KnownSrc(SubBitWidth);
1588 for (unsigned i = 0; i != SubScale; ++i) {
1589 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts.shl(shiftAmt: i), Known&: KnownSrc, Q,
1590 Depth: Depth + 1);
1591 unsigned ShiftElt = IsLE ? i : SubScale - 1 - i;
1592 Known.insertBits(SubBits: KnownSrc, BitPosition: ShiftElt * SubBitWidth);
1593 }
1594 }
1595 // Look through a cast from wider vector elements to narrow type.
1596 // Examples: v2i64 -> v4i32
1597 if (SubBitWidth % BitWidth == 0) {
1598 unsigned SubScale = SubBitWidth / BitWidth;
1599 KnownBits KnownSrc(SubBitWidth);
1600 APInt SubDemandedElts =
1601 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
1602 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts, Known&: KnownSrc, Q,
1603 Depth: Depth + 1);
1604
1605 Known.setAllConflict();
1606 for (unsigned i = 0; i != NumElts; ++i) {
1607 if (DemandedElts[i]) {
1608 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
1609 unsigned Offset = (Shifts % SubScale) * BitWidth;
1610 Known = Known.intersectWith(RHS: KnownSrc.extractBits(NumBits: BitWidth, BitPosition: Offset));
1611 if (Known.isUnknown())
1612 break;
1613 }
1614 }
1615 }
1616 break;
1617 }
1618 case Instruction::SExt: {
1619 // Compute the bits in the result that are not present in the input.
1620 unsigned SrcBitWidth = I->getOperand(i: 0)->getType()->getScalarSizeInBits();
1621
1622 Known = Known.trunc(BitWidth: SrcBitWidth);
1623 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1624 // If the sign bit of the input is known set or clear, then we know the
1625 // top bits of the result.
1626 Known = Known.sext(BitWidth);
1627 break;
1628 }
1629 case Instruction::Shl: {
1630 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1631 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1632 auto KF = [NUW, NSW](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1633 bool ShAmtNonZero) {
1634 return KnownBits::shl(LHS: KnownVal, RHS: KnownAmt, NUW, NSW, ShAmtNonZero);
1635 };
1636 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1637 KF);
1638 // Trailing zeros of a right-shifted constant never decrease.
1639 const APInt *C;
1640 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1641 Known.Zero.setLowBits(C->countr_zero());
1642 break;
1643 }
1644 case Instruction::LShr: {
1645 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1646 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1647 bool ShAmtNonZero) {
1648 return KnownBits::lshr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1649 };
1650 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1651 KF);
1652 // Leading zeros of a left-shifted constant never decrease.
1653 const APInt *C;
1654 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1655 Known.Zero.setHighBits(C->countl_zero());
1656 break;
1657 }
1658 case Instruction::AShr: {
1659 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1660 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1661 bool ShAmtNonZero) {
1662 return KnownBits::ashr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1663 };
1664 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1665 KF);
1666 break;
1667 }
1668 case Instruction::Sub: {
1669 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1670 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1671 computeKnownBitsAddSub(Add: false, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1672 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1673 break;
1674 }
1675 case Instruction::Add: {
1676 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1677 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1678 computeKnownBitsAddSub(Add: true, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1679 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1680 break;
1681 }
1682 case Instruction::SRem:
1683 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1684 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1685 Known = KnownBits::srem(LHS: Known, RHS: Known2);
1686 break;
1687
1688 case Instruction::URem:
1689 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1690 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1691 Known = KnownBits::urem(LHS: Known, RHS: Known2);
1692 break;
1693 case Instruction::Alloca:
1694 Known.Zero.setLowBits(Log2(A: cast<AllocaInst>(Val: I)->getAlign()));
1695 break;
1696 case Instruction::GetElementPtr: {
1697 // Analyze all of the subscripts of this getelementptr instruction
1698 // to determine if we can prove known low zero bits.
1699 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1700 // Accumulate the constant indices in a separate variable
1701 // to minimize the number of calls to computeForAddSub.
1702 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: I->getType());
1703 APInt AccConstIndices(IndexWidth, 0);
1704
1705 auto AddIndexToKnown = [&](KnownBits IndexBits) {
1706 if (IndexWidth == BitWidth) {
1707 // Note that inbounds does *not* guarantee nsw for the addition, as only
1708 // the offset is signed, while the base address is unsigned.
1709 Known = KnownBits::add(LHS: Known, RHS: IndexBits);
1710 } else {
1711 // If the index width is smaller than the pointer width, only add the
1712 // value to the low bits.
1713 assert(IndexWidth < BitWidth &&
1714 "Index width can't be larger than pointer width");
1715 Known.insertBits(SubBits: KnownBits::add(LHS: Known.trunc(BitWidth: IndexWidth), RHS: IndexBits), BitPosition: 0);
1716 }
1717 };
1718
1719 gep_type_iterator GTI = gep_type_begin(GEP: I);
1720 for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
1721 // TrailZ can only become smaller, short-circuit if we hit zero.
1722 if (Known.isUnknown())
1723 break;
1724
1725 Value *Index = I->getOperand(i);
1726
1727 // Handle case when index is zero.
1728 Constant *CIndex = dyn_cast<Constant>(Val: Index);
1729 if (CIndex && CIndex->isNullValue())
1730 continue;
1731
1732 if (StructType *STy = GTI.getStructTypeOrNull()) {
1733 // Handle struct member offset arithmetic.
1734
1735 assert(CIndex &&
1736 "Access to structure field must be known at compile time");
1737
1738 if (CIndex->getType()->isVectorTy())
1739 Index = CIndex->getSplatValue();
1740
1741 unsigned Idx = cast<ConstantInt>(Val: Index)->getZExtValue();
1742 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
1743 uint64_t Offset = SL->getElementOffset(Idx);
1744 AccConstIndices += Offset;
1745 continue;
1746 }
1747
1748 // Handle array index arithmetic.
1749 Type *IndexedTy = GTI.getIndexedType();
1750 if (!IndexedTy->isSized()) {
1751 Known.resetAll();
1752 break;
1753 }
1754
1755 TypeSize Stride = GTI.getSequentialElementStride(DL: Q.DL);
1756 uint64_t StrideInBytes = Stride.getKnownMinValue();
1757 if (!Stride.isScalable()) {
1758 // Fast path for constant offset.
1759 if (auto *CI = dyn_cast<ConstantInt>(Val: Index)) {
1760 AccConstIndices +=
1761 CI->getValue().sextOrTrunc(width: IndexWidth) * StrideInBytes;
1762 continue;
1763 }
1764 }
1765
1766 KnownBits IndexBits =
1767 computeKnownBits(V: Index, Q, Depth: Depth + 1).sextOrTrunc(BitWidth: IndexWidth);
1768 KnownBits ScalingFactor(IndexWidth);
1769 // Multiply by current sizeof type.
1770 // &A[i] == A + i * sizeof(*A[i]).
1771 if (Stride.isScalable()) {
1772 // For scalable types the only thing we know about sizeof is
1773 // that this is a multiple of the minimum size.
1774 ScalingFactor.Zero.setLowBits(llvm::countr_zero(Val: StrideInBytes));
1775 } else {
1776 ScalingFactor =
1777 KnownBits::makeConstant(C: APInt(IndexWidth, StrideInBytes));
1778 }
1779 AddIndexToKnown(KnownBits::mul(LHS: IndexBits, RHS: ScalingFactor));
1780 }
1781 if (!Known.isUnknown() && !AccConstIndices.isZero())
1782 AddIndexToKnown(KnownBits::makeConstant(C: AccConstIndices));
1783 break;
1784 }
1785 case Instruction::PHI: {
1786 const PHINode *P = cast<PHINode>(Val: I);
1787 BinaryOperator *BO = nullptr;
1788 Value *R = nullptr, *L = nullptr;
1789 if (matchSimpleRecurrence(P, BO, Start&: R, Step&: L)) {
1790 // Handle the case of a simple two-predecessor recurrence PHI.
1791 // There's a lot more that could theoretically be done here, but
1792 // this is sufficient to catch some interesting cases.
1793 unsigned Opcode = BO->getOpcode();
1794
1795 switch (Opcode) {
1796 // If this is a shift recurrence, we know the bits being shifted in. We
1797 // can combine that with information about the start value of the
1798 // recurrence to conclude facts about the result. If this is a udiv
1799 // recurrence, we know that the result can never exceed either the
1800 // numerator or the start value, whichever is greater.
1801 case Instruction::LShr:
1802 case Instruction::AShr:
1803 case Instruction::Shl:
1804 case Instruction::UDiv:
1805 if (BO->getOperand(i_nocapture: 0) != I)
1806 break;
1807 [[fallthrough]];
1808
1809 // For a urem recurrence, the result can never exceed the start value. The
1810 // phi could either be the numerator or the denominator.
1811 case Instruction::URem: {
1812 // We have matched a recurrence of the form:
1813 // %iv = [R, %entry], [%iv.next, %backedge]
1814 // %iv.next = shift_op %iv, L
1815
1816 // Recurse with the phi context to avoid concern about whether facts
1817 // inferred hold at original context instruction. TODO: It may be
1818 // correct to use the original context. IF warranted, explore and
1819 // add sufficient tests to cover.
1820 SimplifyQuery RecQ = Q.getWithoutCondContext();
1821 RecQ.CxtI = P;
1822 computeKnownBits(V: R, DemandedElts, Known&: Known2, Q: RecQ, Depth: Depth + 1);
1823 switch (Opcode) {
1824 case Instruction::Shl:
1825 // A shl recurrence will only increase the tailing zeros
1826 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
1827 break;
1828 case Instruction::LShr:
1829 case Instruction::UDiv:
1830 case Instruction::URem:
1831 // lshr, udiv, and urem recurrences will preserve the leading zeros of
1832 // the start value.
1833 Known.Zero.setHighBits(Known2.countMinLeadingZeros());
1834 break;
1835 case Instruction::AShr:
1836 // An ashr recurrence will extend the initial sign bit
1837 Known.Zero.setHighBits(Known2.countMinLeadingZeros());
1838 Known.One.setHighBits(Known2.countMinLeadingOnes());
1839 break;
1840 }
1841 break;
1842 }
1843
1844 // Check for operations that have the property that if
1845 // both their operands have low zero bits, the result
1846 // will have low zero bits.
1847 case Instruction::Add:
1848 case Instruction::Sub:
1849 case Instruction::And:
1850 case Instruction::Or:
1851 case Instruction::Mul: {
1852 // Change the context instruction to the "edge" that flows into the
1853 // phi. This is important because that is where the value is actually
1854 // "evaluated" even though it is used later somewhere else. (see also
1855 // D69571).
1856 SimplifyQuery RecQ = Q.getWithoutCondContext();
1857
1858 unsigned OpNum = P->getOperand(i_nocapture: 0) == R ? 0 : 1;
1859 Instruction *RInst = P->getIncomingBlock(i: OpNum)->getTerminator();
1860 Instruction *LInst = P->getIncomingBlock(i: 1 - OpNum)->getTerminator();
1861
1862 // Ok, we have a PHI of the form L op= R. Check for low
1863 // zero bits.
1864 RecQ.CxtI = RInst;
1865 computeKnownBits(V: R, DemandedElts, Known&: Known2, Q: RecQ, Depth: Depth + 1);
1866
1867 // We need to take the minimum number of known bits
1868 KnownBits Known3(BitWidth);
1869 RecQ.CxtI = LInst;
1870 computeKnownBits(V: L, DemandedElts, Known&: Known3, Q: RecQ, Depth: Depth + 1);
1871
1872 Known.Zero.setLowBits(std::min(a: Known2.countMinTrailingZeros(),
1873 b: Known3.countMinTrailingZeros()));
1874
1875 auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(Val: BO);
1876 if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(Op: OverflowOp))
1877 break;
1878
1879 switch (Opcode) {
1880 // If initial value of recurrence is nonnegative, and we are adding
1881 // a nonnegative number with nsw, the result can only be nonnegative
1882 // or poison value regardless of the number of times we execute the
1883 // add in phi recurrence. If initial value is negative and we are
1884 // adding a negative number with nsw, the result can only be
1885 // negative or poison value. Similar arguments apply to sub and mul.
1886 //
1887 // (add non-negative, non-negative) --> non-negative
1888 // (add negative, negative) --> negative
1889 case Instruction::Add: {
1890 if (Known2.isNonNegative() && Known3.isNonNegative())
1891 Known.makeNonNegative();
1892 else if (Known2.isNegative() && Known3.isNegative())
1893 Known.makeNegative();
1894 break;
1895 }
1896
1897 // (sub nsw non-negative, negative) --> non-negative
1898 // (sub nsw negative, non-negative) --> negative
1899 case Instruction::Sub: {
1900 if (BO->getOperand(i_nocapture: 0) != I)
1901 break;
1902 if (Known2.isNonNegative() && Known3.isNegative())
1903 Known.makeNonNegative();
1904 else if (Known2.isNegative() && Known3.isNonNegative())
1905 Known.makeNegative();
1906 break;
1907 }
1908
1909 // (mul nsw non-negative, non-negative) --> non-negative
1910 case Instruction::Mul:
1911 if (Known2.isNonNegative() && Known3.isNonNegative())
1912 Known.makeNonNegative();
1913 break;
1914
1915 default:
1916 break;
1917 }
1918 break;
1919 }
1920
1921 default:
1922 break;
1923 }
1924 }
1925
1926 // Unreachable blocks may have zero-operand PHI nodes.
1927 if (P->getNumIncomingValues() == 0)
1928 break;
1929
1930 // Otherwise take the unions of the known bit sets of the operands,
1931 // taking conservative care to avoid excessive recursion.
1932 if (Depth < MaxAnalysisRecursionDepth - 1 && Known.isUnknown()) {
1933 // Skip if every incoming value references to ourself.
1934 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
1935 break;
1936
1937 Known.setAllConflict();
1938 for (const Use &U : P->operands()) {
1939 Value *IncValue;
1940 const PHINode *CxtPhi;
1941 Instruction *CxtI;
1942 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CxtI, PhiOut: &CxtPhi);
1943 // Skip direct self references.
1944 if (IncValue == P)
1945 continue;
1946
1947 // Change the context instruction to the "edge" that flows into the
1948 // phi. This is important because that is where the value is actually
1949 // "evaluated" even though it is used later somewhere else. (see also
1950 // D69571).
1951 SimplifyQuery RecQ = Q.getWithoutCondContext().getWithInstruction(I: CxtI);
1952
1953 Known2 = KnownBits(BitWidth);
1954
1955 // Recurse, but cap the recursion to one level, because we don't
1956 // want to waste time spinning around in loops.
1957 // TODO: See if we can base recursion limiter on number of incoming phi
1958 // edges so we don't overly clamp analysis.
1959 computeKnownBits(V: IncValue, DemandedElts, Known&: Known2, Q: RecQ,
1960 Depth: MaxAnalysisRecursionDepth - 1);
1961
1962 // See if we can further use a conditional branch into the phi
1963 // to help us determine the range of the value.
1964 if (!Known2.isConstant()) {
1965 CmpPredicate Pred;
1966 const APInt *RHSC;
1967 BasicBlock *TrueSucc, *FalseSucc;
1968 // TODO: Use RHS Value and compute range from its known bits.
1969 if (match(V: RecQ.CxtI,
1970 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: IncValue), R: m_APInt(Res&: RHSC)),
1971 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
1972 // Check for cases of duplicate successors.
1973 if ((TrueSucc == CxtPhi->getParent()) !=
1974 (FalseSucc == CxtPhi->getParent())) {
1975 // If we're using the false successor, invert the predicate.
1976 if (FalseSucc == CxtPhi->getParent())
1977 Pred = CmpInst::getInversePredicate(pred: Pred);
1978 // Get the knownbits implied by the incoming phi condition.
1979 auto CR = ConstantRange::makeExactICmpRegion(Pred, Other: *RHSC);
1980 KnownBits KnownUnion = Known2.unionWith(RHS: CR.toKnownBits());
1981 // We can have conflicts here if we are analyzing deadcode (its
1982 // impossible for us reach this BB based the icmp).
1983 if (KnownUnion.hasConflict()) {
1984 // No reason to continue analyzing in a known dead region, so
1985 // just resetAll and break. This will cause us to also exit the
1986 // outer loop.
1987 Known.resetAll();
1988 break;
1989 }
1990 Known2 = KnownUnion;
1991 }
1992 }
1993 }
1994
1995 Known = Known.intersectWith(RHS: Known2);
1996 // If all bits have been ruled out, there's no need to check
1997 // more operands.
1998 if (Known.isUnknown())
1999 break;
2000 }
2001 }
2002 break;
2003 }
2004 case Instruction::Call:
2005 case Instruction::Invoke: {
2006 // If range metadata is attached to this call, set known bits from that,
2007 // and then intersect with known bits based on other properties of the
2008 // function.
2009 if (MDNode *MD =
2010 Q.IIQ.getMetadata(I: cast<Instruction>(Val: I), KindID: LLVMContext::MD_range))
2011 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
2012
2013 const auto *CB = cast<CallBase>(Val: I);
2014
2015 if (std::optional<ConstantRange> Range = CB->getRange())
2016 Known = Known.unionWith(RHS: Range->toKnownBits());
2017
2018 if (const Value *RV = CB->getReturnedArgOperand()) {
2019 if (RV->getType() == I->getType()) {
2020 computeKnownBits(V: RV, Known&: Known2, Q, Depth: Depth + 1);
2021 Known = Known.unionWith(RHS: Known2);
2022 // If the function doesn't return properly for all input values
2023 // (e.g. unreachable exits) then there might be conflicts between the
2024 // argument value and the range metadata. Simply discard the known bits
2025 // in case of conflicts.
2026 if (Known.hasConflict())
2027 Known.resetAll();
2028 }
2029 }
2030 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
2031 switch (II->getIntrinsicID()) {
2032 default:
2033 break;
2034 case Intrinsic::abs: {
2035 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2036 bool IntMinIsPoison = match(V: II->getArgOperand(i: 1), P: m_One());
2037 Known = Known.unionWith(RHS: Known2.abs(IntMinIsPoison));
2038 break;
2039 }
2040 case Intrinsic::bitreverse:
2041 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2042 Known = Known.unionWith(RHS: Known2.reverseBits());
2043 break;
2044 case Intrinsic::bswap:
2045 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2046 Known = Known.unionWith(RHS: Known2.byteSwap());
2047 break;
2048 case Intrinsic::ctlz: {
2049 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2050 // If we have a known 1, its position is our upper bound.
2051 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2052 // If this call is poison for 0 input, the result will be less than 2^n.
2053 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2054 PossibleLZ = std::min(a: PossibleLZ, b: BitWidth - 1);
2055 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
2056 Known.Zero.setBitsFrom(LowBits);
2057 break;
2058 }
2059 case Intrinsic::cttz: {
2060 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2061 // If we have a known 1, its position is our upper bound.
2062 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2063 // If this call is poison for 0 input, the result will be less than 2^n.
2064 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2065 PossibleTZ = std::min(a: PossibleTZ, b: BitWidth - 1);
2066 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
2067 Known.Zero.setBitsFrom(LowBits);
2068 break;
2069 }
2070 case Intrinsic::ctpop: {
2071 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2072 // We can bound the space the count needs. Also, bits known to be zero
2073 // can't contribute to the population.
2074 unsigned BitsPossiblySet = Known2.countMaxPopulation();
2075 unsigned LowBits = llvm::bit_width(Value: BitsPossiblySet);
2076 Known.Zero.setBitsFrom(LowBits);
2077 // TODO: we could bound KnownOne using the lower bound on the number
2078 // of bits which might be set provided by popcnt KnownOne2.
2079 break;
2080 }
2081 case Intrinsic::fshr:
2082 case Intrinsic::fshl: {
2083 const APInt *SA;
2084 if (!match(V: I->getOperand(i: 2), P: m_APInt(Res&: SA)))
2085 break;
2086
2087 // Normalize to funnel shift left.
2088 uint64_t ShiftAmt = SA->urem(RHS: BitWidth);
2089 if (II->getIntrinsicID() == Intrinsic::fshr)
2090 ShiftAmt = BitWidth - ShiftAmt;
2091
2092 KnownBits Known3(BitWidth);
2093 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2094 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known3, Q, Depth: Depth + 1);
2095
2096 Known2 <<= ShiftAmt;
2097 Known3 >>= BitWidth - ShiftAmt;
2098 Known = Known2.unionWith(RHS: Known3);
2099 break;
2100 }
2101 case Intrinsic::clmul:
2102 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2103 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2104 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
2105 break;
2106 case Intrinsic::uadd_sat:
2107 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2108 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2109 Known = KnownBits::uadd_sat(LHS: Known, RHS: Known2);
2110 break;
2111 case Intrinsic::usub_sat:
2112 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2113 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2114 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
2115 break;
2116 case Intrinsic::sadd_sat:
2117 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2118 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2119 Known = KnownBits::sadd_sat(LHS: Known, RHS: Known2);
2120 break;
2121 case Intrinsic::ssub_sat:
2122 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2123 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2124 Known = KnownBits::ssub_sat(LHS: Known, RHS: Known2);
2125 break;
2126 // Vec reverse preserves bits from input vec.
2127 case Intrinsic::vector_reverse:
2128 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: DemandedElts.reverseBits(), Known, Q,
2129 Depth: Depth + 1);
2130 break;
2131 // for min/max/and/or reduce, any bit common to each element in the
2132 // input vec is set in the output.
2133 case Intrinsic::vector_reduce_and:
2134 case Intrinsic::vector_reduce_or:
2135 case Intrinsic::vector_reduce_umax:
2136 case Intrinsic::vector_reduce_umin:
2137 case Intrinsic::vector_reduce_smax:
2138 case Intrinsic::vector_reduce_smin:
2139 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2140 break;
2141 case Intrinsic::vector_reduce_xor: {
2142 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2143 // The zeros common to all vecs are zero in the output.
2144 // If the number of elements is odd, then the common ones remain. If the
2145 // number of elements is even, then the common ones becomes zeros.
2146 auto *VecTy = cast<VectorType>(Val: I->getOperand(i: 0)->getType());
2147 // Even, so the ones become zeros.
2148 bool EvenCnt = VecTy->getElementCount().isKnownEven();
2149 if (EvenCnt)
2150 Known.Zero |= Known.One;
2151 // Maybe even element count so need to clear ones.
2152 if (VecTy->isScalableTy() || EvenCnt)
2153 Known.One.clearAllBits();
2154 break;
2155 }
2156 case Intrinsic::vector_reduce_add: {
2157 auto *VecTy = dyn_cast<FixedVectorType>(Val: I->getOperand(i: 0)->getType());
2158 if (!VecTy)
2159 break;
2160 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2161 Known = Known.reduceAdd(NumElts: VecTy->getNumElements());
2162 break;
2163 }
2164 case Intrinsic::umin:
2165 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2166 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2167 Known = KnownBits::umin(LHS: Known, RHS: Known2);
2168 break;
2169 case Intrinsic::umax:
2170 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2171 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2172 Known = KnownBits::umax(LHS: Known, RHS: Known2);
2173 break;
2174 case Intrinsic::smin:
2175 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2176 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2177 Known = KnownBits::smin(LHS: Known, RHS: Known2);
2178 unionWithMinMaxIntrinsicClamp(II, Known);
2179 break;
2180 case Intrinsic::smax:
2181 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2182 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2183 Known = KnownBits::smax(LHS: Known, RHS: Known2);
2184 unionWithMinMaxIntrinsicClamp(II, Known);
2185 break;
2186 case Intrinsic::ptrmask: {
2187 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2188
2189 const Value *Mask = I->getOperand(i: 1);
2190 Known2 = KnownBits(Mask->getType()->getScalarSizeInBits());
2191 computeKnownBits(V: Mask, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2192 // TODO: 1-extend would be more precise.
2193 Known &= Known2.anyextOrTrunc(BitWidth);
2194 break;
2195 }
2196 case Intrinsic::x86_sse2_pmulh_w:
2197 case Intrinsic::x86_avx2_pmulh_w:
2198 case Intrinsic::x86_avx512_pmulh_w_512:
2199 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2200 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2201 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
2202 break;
2203 case Intrinsic::x86_sse2_pmulhu_w:
2204 case Intrinsic::x86_avx2_pmulhu_w:
2205 case Intrinsic::x86_avx512_pmulhu_w_512:
2206 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2207 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2208 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
2209 break;
2210 case Intrinsic::x86_sse42_crc32_64_64:
2211 Known.Zero.setBitsFrom(32);
2212 break;
2213 case Intrinsic::x86_ssse3_phadd_d_128:
2214 case Intrinsic::x86_ssse3_phadd_w_128:
2215 case Intrinsic::x86_avx2_phadd_d:
2216 case Intrinsic::x86_avx2_phadd_w: {
2217 Known = computeKnownBitsForHorizontalOperation(
2218 I, DemandedElts, Q, Depth,
2219 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2220 return KnownBits::add(LHS: KnownLHS, RHS: KnownRHS);
2221 });
2222 break;
2223 }
2224 case Intrinsic::x86_ssse3_phadd_sw_128:
2225 case Intrinsic::x86_avx2_phadd_sw: {
2226 Known = computeKnownBitsForHorizontalOperation(
2227 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::sadd_sat);
2228 break;
2229 }
2230 case Intrinsic::x86_ssse3_phsub_d_128:
2231 case Intrinsic::x86_ssse3_phsub_w_128:
2232 case Intrinsic::x86_avx2_phsub_d:
2233 case Intrinsic::x86_avx2_phsub_w: {
2234 Known = computeKnownBitsForHorizontalOperation(
2235 I, DemandedElts, Q, Depth,
2236 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2237 return KnownBits::sub(LHS: KnownLHS, RHS: KnownRHS);
2238 });
2239 break;
2240 }
2241 case Intrinsic::x86_ssse3_phsub_sw_128:
2242 case Intrinsic::x86_avx2_phsub_sw: {
2243 Known = computeKnownBitsForHorizontalOperation(
2244 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::ssub_sat);
2245 break;
2246 }
2247 case Intrinsic::riscv_vsetvli:
2248 case Intrinsic::riscv_vsetvlimax: {
2249 bool HasAVL = II->getIntrinsicID() == Intrinsic::riscv_vsetvli;
2250 const ConstantRange Range = getVScaleRange(F: II->getFunction(), BitWidth);
2251 uint64_t SEW = RISCVVType::decodeVSEW(
2252 VSEW: cast<ConstantInt>(Val: II->getArgOperand(i: HasAVL))->getZExtValue());
2253 RISCVVType::VLMUL VLMUL = static_cast<RISCVVType::VLMUL>(
2254 cast<ConstantInt>(Val: II->getArgOperand(i: 1 + HasAVL))->getZExtValue());
2255 uint64_t MaxVLEN =
2256 Range.getUnsignedMax().getZExtValue() * RISCV::RVVBitsPerBlock;
2257 uint64_t MaxVL = MaxVLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
2258
2259 // Result of vsetvli must be not larger than AVL.
2260 if (HasAVL)
2261 if (auto *CI = dyn_cast<ConstantInt>(Val: II->getArgOperand(i: 0)))
2262 MaxVL = std::min(a: MaxVL, b: CI->getZExtValue());
2263
2264 unsigned KnownZeroFirstBit = Log2_32(Value: MaxVL) + 1;
2265 if (BitWidth > KnownZeroFirstBit)
2266 Known.Zero.setBitsFrom(KnownZeroFirstBit);
2267 break;
2268 }
2269 case Intrinsic::amdgcn_mbcnt_hi:
2270 case Intrinsic::amdgcn_mbcnt_lo: {
2271 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
2272 // most 31 + src1.
2273 Known.Zero.setBitsFrom(
2274 II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo ? 6 : 5);
2275 computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Q, Depth: Depth + 1);
2276 Known = KnownBits::add(LHS: Known, RHS: Known2);
2277 break;
2278 }
2279 case Intrinsic::vscale: {
2280 if (!II->getParent() || !II->getFunction())
2281 break;
2282
2283 Known = getVScaleRange(F: II->getFunction(), BitWidth).toKnownBits();
2284 break;
2285 }
2286 }
2287 }
2288 break;
2289 }
2290 case Instruction::ShuffleVector: {
2291 if (auto *Splat = getSplatValue(V: I)) {
2292 computeKnownBits(V: Splat, Known, Q, Depth: Depth + 1);
2293 break;
2294 }
2295
2296 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
2297 // FIXME: Do we need to handle ConstantExpr involving shufflevectors?
2298 if (!Shuf) {
2299 Known.resetAll();
2300 return;
2301 }
2302 // For undef elements, we don't know anything about the common state of
2303 // the shuffle result.
2304 APInt DemandedLHS, DemandedRHS;
2305 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS)) {
2306 Known.resetAll();
2307 return;
2308 }
2309 Known.setAllConflict();
2310 if (!!DemandedLHS) {
2311 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
2312 computeKnownBits(V: LHS, DemandedElts: DemandedLHS, Known, Q, Depth: Depth + 1);
2313 // If we don't know any bits, early out.
2314 if (Known.isUnknown())
2315 break;
2316 }
2317 if (!!DemandedRHS) {
2318 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
2319 computeKnownBits(V: RHS, DemandedElts: DemandedRHS, Known&: Known2, Q, Depth: Depth + 1);
2320 Known = Known.intersectWith(RHS: Known2);
2321 }
2322 break;
2323 }
2324 case Instruction::InsertElement: {
2325 if (isa<ScalableVectorType>(Val: I->getType())) {
2326 Known.resetAll();
2327 return;
2328 }
2329 const Value *Vec = I->getOperand(i: 0);
2330 const Value *Elt = I->getOperand(i: 1);
2331 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
2332 unsigned NumElts = DemandedElts.getBitWidth();
2333 APInt DemandedVecElts = DemandedElts;
2334 bool NeedsElt = true;
2335 // If we know the index we are inserting too, clear it from Vec check.
2336 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
2337 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
2338 NeedsElt = DemandedElts[CIdx->getZExtValue()];
2339 }
2340
2341 Known.setAllConflict();
2342 if (NeedsElt) {
2343 computeKnownBits(V: Elt, Known, Q, Depth: Depth + 1);
2344 // If we don't know any bits, early out.
2345 if (Known.isUnknown())
2346 break;
2347 }
2348
2349 if (!DemandedVecElts.isZero()) {
2350 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known&: Known2, Q, Depth: Depth + 1);
2351 Known = Known.intersectWith(RHS: Known2);
2352 }
2353 break;
2354 }
2355 case Instruction::ExtractElement: {
2356 // Look through extract element. If the index is non-constant or
2357 // out-of-range demand all elements, otherwise just the extracted element.
2358 const Value *Vec = I->getOperand(i: 0);
2359 const Value *Idx = I->getOperand(i: 1);
2360 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
2361 if (isa<ScalableVectorType>(Val: Vec->getType())) {
2362 // FIXME: there's probably *something* we can do with scalable vectors
2363 Known.resetAll();
2364 break;
2365 }
2366 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2367 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
2368 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
2369 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
2370 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known, Q, Depth: Depth + 1);
2371 break;
2372 }
2373 case Instruction::ExtractValue:
2374 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I->getOperand(i: 0))) {
2375 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: I);
2376 if (EVI->getNumIndices() != 1) break;
2377 if (EVI->getIndices()[0] == 0) {
2378 switch (II->getIntrinsicID()) {
2379 default: break;
2380 case Intrinsic::uadd_with_overflow:
2381 case Intrinsic::sadd_with_overflow:
2382 computeKnownBitsAddSub(
2383 Add: true, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2384 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2385 break;
2386 case Intrinsic::usub_with_overflow:
2387 case Intrinsic::ssub_with_overflow:
2388 computeKnownBitsAddSub(
2389 Add: false, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2390 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2391 break;
2392 case Intrinsic::umul_with_overflow:
2393 case Intrinsic::smul_with_overflow:
2394 computeKnownBitsMul(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), NSW: false,
2395 NUW: false, DemandedElts, Known, Known2, Q, Depth);
2396 break;
2397 }
2398 }
2399 }
2400 break;
2401 case Instruction::Freeze:
2402 if (isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
2403 Depth: Depth + 1))
2404 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2405 break;
2406 }
2407}
2408
2409/// Determine which bits of V are known to be either zero or one and return
2410/// them.
2411KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
2412 const SimplifyQuery &Q, unsigned Depth) {
2413 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2414 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
2415 return Known;
2416}
2417
2418/// Determine which bits of V are known to be either zero or one and return
2419/// them.
2420KnownBits llvm::computeKnownBits(const Value *V, const SimplifyQuery &Q,
2421 unsigned Depth) {
2422 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2423 computeKnownBits(V, Known, Q, Depth);
2424 return Known;
2425}
2426
2427/// Determine which bits of V are known to be either zero or one and return
2428/// them in the Known bit set.
2429///
2430/// NOTE: we cannot consider 'undef' to be "IsZero" here. The problem is that
2431/// we cannot optimize based on the assumption that it is zero without changing
2432/// it to be an explicit zero. If we don't change it to zero, other code could
2433/// optimized based on the contradictory assumption that it is non-zero.
2434/// Because instcombine aggressively folds operations with undef args anyway,
2435/// this won't lose us code quality.
2436///
2437/// This function is defined on values with integer type, values with pointer
2438/// type, and vectors of integers. In the case
2439/// where V is a vector, known zero, and known one values are the
2440/// same width as the vector element, and the bit is set only if it is true
2441/// for all of the demanded elements in the vector specified by DemandedElts.
2442void computeKnownBits(const Value *V, const APInt &DemandedElts,
2443 KnownBits &Known, const SimplifyQuery &Q,
2444 unsigned Depth) {
2445 if (!DemandedElts) {
2446 // No demanded elts, better to assume we don't know anything.
2447 Known.resetAll();
2448 return;
2449 }
2450
2451 assert(V && "No Value?");
2452 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2453
2454#ifndef NDEBUG
2455 Type *Ty = V->getType();
2456 unsigned BitWidth = Known.getBitWidth();
2457
2458 assert((Ty->isIntOrIntVectorTy(BitWidth) || Ty->isPtrOrPtrVectorTy()) &&
2459 "Not integer or pointer type!");
2460
2461 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
2462 assert(
2463 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
2464 "DemandedElt width should equal the fixed vector number of elements");
2465 } else {
2466 assert(DemandedElts == APInt(1, 1) &&
2467 "DemandedElt width should be 1 for scalars or scalable vectors");
2468 }
2469
2470 Type *ScalarTy = Ty->getScalarType();
2471 if (ScalarTy->isPointerTy()) {
2472 assert(BitWidth == Q.DL.getPointerTypeSizeInBits(ScalarTy) &&
2473 "V and Known should have same BitWidth");
2474 } else {
2475 assert(BitWidth == Q.DL.getTypeSizeInBits(ScalarTy) &&
2476 "V and Known should have same BitWidth");
2477 }
2478#endif
2479
2480 const APInt *C;
2481 if (match(V, P: m_APInt(Res&: C))) {
2482 // We know all of the bits for a scalar constant or a splat vector constant!
2483 Known = KnownBits::makeConstant(C: *C);
2484 return;
2485 }
2486 // Null and aggregate-zero are all-zeros.
2487 if (isa<ConstantPointerNull>(Val: V) || isa<ConstantAggregateZero>(Val: V)) {
2488 Known.setAllZero();
2489 return;
2490 }
2491 // Handle a constant vector by taking the intersection of the known bits of
2492 // each element.
2493 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: V)) {
2494 assert(!isa<ScalableVectorType>(V->getType()));
2495 // We know that CDV must be a vector of integers. Take the intersection of
2496 // each element.
2497 Known.setAllConflict();
2498 for (unsigned i = 0, e = CDV->getNumElements(); i != e; ++i) {
2499 if (!DemandedElts[i])
2500 continue;
2501 APInt Elt = CDV->getElementAsAPInt(i);
2502 Known.Zero &= ~Elt;
2503 Known.One &= Elt;
2504 }
2505 if (Known.hasConflict())
2506 Known.resetAll();
2507 return;
2508 }
2509
2510 if (const auto *CV = dyn_cast<ConstantVector>(Val: V)) {
2511 assert(!isa<ScalableVectorType>(V->getType()));
2512 // We know that CV must be a vector of integers. Take the intersection of
2513 // each element.
2514 Known.setAllConflict();
2515 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
2516 if (!DemandedElts[i])
2517 continue;
2518 Constant *Element = CV->getAggregateElement(Elt: i);
2519 if (isa<PoisonValue>(Val: Element))
2520 continue;
2521 auto *ElementCI = dyn_cast_or_null<ConstantInt>(Val: Element);
2522 if (!ElementCI) {
2523 Known.resetAll();
2524 return;
2525 }
2526 const APInt &Elt = ElementCI->getValue();
2527 Known.Zero &= ~Elt;
2528 Known.One &= Elt;
2529 }
2530 if (Known.hasConflict())
2531 Known.resetAll();
2532 return;
2533 }
2534
2535 // Start out not knowing anything.
2536 Known.resetAll();
2537
2538 // We can't imply anything about undefs.
2539 if (isa<UndefValue>(Val: V))
2540 return;
2541
2542 // There's no point in looking through other users of ConstantData for
2543 // assumptions. Confirm that we've handled them all.
2544 assert(!isa<ConstantData>(V) && "Unhandled constant data!");
2545
2546 if (const auto *A = dyn_cast<Argument>(Val: V))
2547 if (std::optional<ConstantRange> Range = A->getRange())
2548 Known = Range->toKnownBits();
2549
2550 // All recursive calls that increase depth must come after this.
2551 if (Depth == MaxAnalysisRecursionDepth)
2552 return;
2553
2554 // A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
2555 // the bits of its aliasee.
2556 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: V)) {
2557 if (!GA->isInterposable())
2558 computeKnownBits(V: GA->getAliasee(), Known, Q, Depth: Depth + 1);
2559 return;
2560 }
2561
2562 if (const Operator *I = dyn_cast<Operator>(Val: V))
2563 computeKnownBitsFromOperator(I, DemandedElts, Known, Q, Depth);
2564 else if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
2565 if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
2566 Known = CR->toKnownBits();
2567 }
2568
2569 // Aligned pointers have trailing zeros - refine Known.Zero set
2570 if (isa<PointerType>(Val: V->getType())) {
2571 Align Alignment = V->getPointerAlignment(DL: Q.DL);
2572 Known.Zero.setLowBits(Log2(A: Alignment));
2573 }
2574
2575 // computeKnownBitsFromContext strictly refines Known.
2576 // Therefore, we run them after computeKnownBitsFromOperator.
2577
2578 // Check whether we can determine known bits from context such as assumes.
2579 computeKnownBitsFromContext(V, Known, Q, Depth);
2580}
2581
2582/// Try to detect a recurrence that the value of the induction variable is
2583/// always a power of two (or zero).
2584static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
2585 SimplifyQuery &Q, unsigned Depth) {
2586 BinaryOperator *BO = nullptr;
2587 Value *Start = nullptr, *Step = nullptr;
2588 if (!matchSimpleRecurrence(P: PN, BO, Start, Step))
2589 return false;
2590
2591 // Initial value must be a power of two.
2592 for (const Use &U : PN->operands()) {
2593 if (U.get() == Start) {
2594 // Initial value comes from a different BB, need to adjust context
2595 // instruction for analysis.
2596 Q.CxtI = PN->getIncomingBlock(U)->getTerminator();
2597 if (!isKnownToBeAPowerOfTwo(V: Start, OrZero, Q, Depth))
2598 return false;
2599 }
2600 }
2601
2602 // Except for Mul, the induction variable must be on the left side of the
2603 // increment expression, otherwise its value can be arbitrary.
2604 if (BO->getOpcode() != Instruction::Mul && BO->getOperand(i_nocapture: 1) != Step)
2605 return false;
2606
2607 Q.CxtI = BO->getParent()->getTerminator();
2608 switch (BO->getOpcode()) {
2609 case Instruction::Mul:
2610 // Power of two is closed under multiplication.
2611 return (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) ||
2612 Q.IIQ.hasNoSignedWrap(Op: BO)) &&
2613 isKnownToBeAPowerOfTwo(V: Step, OrZero, Q, Depth);
2614 case Instruction::SDiv:
2615 // Start value must not be signmask for signed division, so simply being a
2616 // power of two is not sufficient, and it has to be a constant.
2617 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2618 return false;
2619 [[fallthrough]];
2620 case Instruction::UDiv:
2621 // Divisor must be a power of two.
2622 // If OrZero is false, cannot guarantee induction variable is non-zero after
2623 // division, same for Shr, unless it is exact division.
2624 return (OrZero || Q.IIQ.isExact(Op: BO)) &&
2625 isKnownToBeAPowerOfTwo(V: Step, OrZero: false, Q, Depth);
2626 case Instruction::Shl:
2627 return OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO);
2628 case Instruction::AShr:
2629 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2630 return false;
2631 [[fallthrough]];
2632 case Instruction::LShr:
2633 return OrZero || Q.IIQ.isExact(Op: BO);
2634 default:
2635 return false;
2636 }
2637}
2638
2639/// Return true if we can infer that \p V is known to be a power of 2 from
2640/// dominating condition \p Cond (e.g., ctpop(V) == 1).
2641static bool isImpliedToBeAPowerOfTwoFromCond(const Value *V, bool OrZero,
2642 const Value *Cond,
2643 bool CondIsTrue) {
2644 CmpPredicate Pred;
2645 const APInt *RHSC;
2646 if (!match(V: Cond, P: m_ICmp(Pred, L: m_Intrinsic<Intrinsic::ctpop>(Op0: m_Specific(V)),
2647 R: m_APInt(Res&: RHSC))))
2648 return false;
2649 if (!CondIsTrue)
2650 Pred = ICmpInst::getInversePredicate(pred: Pred);
2651 // ctpop(V) u< 2
2652 if (OrZero && Pred == ICmpInst::ICMP_ULT && *RHSC == 2)
2653 return true;
2654 // ctpop(V) == 1
2655 return Pred == ICmpInst::ICMP_EQ && *RHSC == 1;
2656}
2657
2658/// Return true if the given value is known to have exactly one
2659/// bit set when defined. For vectors return true if every element is known to
2660/// be a power of two when defined. Supports values with integer or pointer
2661/// types and vectors of integers.
2662bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero,
2663 const SimplifyQuery &Q, unsigned Depth) {
2664 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2665
2666 if (isa<Constant>(Val: V))
2667 return OrZero ? match(V, P: m_Power2OrZero()) : match(V, P: m_Power2());
2668
2669 // i1 is by definition a power of 2 or zero.
2670 if (OrZero && V->getType()->getScalarSizeInBits() == 1)
2671 return true;
2672
2673 // Try to infer from assumptions.
2674 if (Q.AC && Q.CxtI) {
2675 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
2676 if (!AssumeVH)
2677 continue;
2678 CallInst *I = cast<CallInst>(Val&: AssumeVH);
2679 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond: I->getArgOperand(i: 0),
2680 /*CondIsTrue=*/true) &&
2681 isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
2682 return true;
2683 }
2684 }
2685
2686 // Handle dominating conditions.
2687 if (Q.DC && Q.CxtI && Q.DT) {
2688 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
2689 Value *Cond = BI->getCondition();
2690
2691 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
2692 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2693 /*CondIsTrue=*/true) &&
2694 Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
2695 return true;
2696
2697 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
2698 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2699 /*CondIsTrue=*/false) &&
2700 Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
2701 return true;
2702 }
2703 }
2704
2705 auto *I = dyn_cast<Instruction>(Val: V);
2706 if (!I)
2707 return false;
2708
2709 if (Q.CxtI && match(V, P: m_VScale())) {
2710 const Function *F = Q.CxtI->getFunction();
2711 // The vscale_range indicates vscale is a power-of-two.
2712 return F->hasFnAttribute(Kind: Attribute::VScaleRange);
2713 }
2714
2715 // 1 << X is clearly a power of two if the one is not shifted off the end. If
2716 // it is shifted off the end then the result is undefined.
2717 if (match(V: I, P: m_Shl(L: m_One(), R: m_Value())))
2718 return true;
2719
2720 // (signmask) >>l X is clearly a power of two if the one is not shifted off
2721 // the bottom. If it is shifted off the bottom then the result is undefined.
2722 if (match(V: I, P: m_LShr(L: m_SignMask(), R: m_Value())))
2723 return true;
2724
2725 // The remaining tests are all recursive, so bail out if we hit the limit.
2726 if (Depth++ == MaxAnalysisRecursionDepth)
2727 return false;
2728
2729 switch (I->getOpcode()) {
2730 case Instruction::ZExt:
2731 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2732 case Instruction::Trunc:
2733 return OrZero && isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2734 case Instruction::Shl:
2735 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: I) || Q.IIQ.hasNoSignedWrap(Op: I))
2736 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2737 return false;
2738 case Instruction::LShr:
2739 if (OrZero || Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2740 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2741 return false;
2742 case Instruction::UDiv:
2743 if (Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2744 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2745 return false;
2746 case Instruction::Mul:
2747 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2748 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth) &&
2749 (OrZero || isKnownNonZero(V: I, Q, Depth));
2750 case Instruction::And:
2751 // A power of two and'd with anything is a power of two or zero.
2752 if (OrZero &&
2753 (isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), /*OrZero*/ true, Q, Depth) ||
2754 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), /*OrZero*/ true, Q, Depth)))
2755 return true;
2756 // X & (-X) is always a power of two or zero.
2757 if (match(V: I->getOperand(i: 0), P: m_Neg(V: m_Specific(V: I->getOperand(i: 1)))) ||
2758 match(V: I->getOperand(i: 1), P: m_Neg(V: m_Specific(V: I->getOperand(i: 0)))))
2759 return OrZero || isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
2760 return false;
2761 case Instruction::Add: {
2762 // Adding a power-of-two or zero to the same power-of-two or zero yields
2763 // either the original power-of-two, a larger power-of-two or zero.
2764 const OverflowingBinaryOperator *VOBO = cast<OverflowingBinaryOperator>(Val: V);
2765 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO) ||
2766 Q.IIQ.hasNoSignedWrap(Op: VOBO)) {
2767 if (match(V: I->getOperand(i: 0),
2768 P: m_c_And(L: m_Specific(V: I->getOperand(i: 1)), R: m_Value())) &&
2769 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth))
2770 return true;
2771 if (match(V: I->getOperand(i: 1),
2772 P: m_c_And(L: m_Specific(V: I->getOperand(i: 0)), R: m_Value())) &&
2773 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth))
2774 return true;
2775
2776 unsigned BitWidth = V->getType()->getScalarSizeInBits();
2777 KnownBits LHSBits(BitWidth);
2778 computeKnownBits(V: I->getOperand(i: 0), Known&: LHSBits, Q, Depth);
2779
2780 KnownBits RHSBits(BitWidth);
2781 computeKnownBits(V: I->getOperand(i: 1), Known&: RHSBits, Q, Depth);
2782 // If i8 V is a power of two or zero:
2783 // ZeroBits: 1 1 1 0 1 1 1 1
2784 // ~ZeroBits: 0 0 0 1 0 0 0 0
2785 if ((~(LHSBits.Zero & RHSBits.Zero)).isPowerOf2())
2786 // If OrZero isn't set, we cannot give back a zero result.
2787 // Make sure either the LHS or RHS has a bit set.
2788 if (OrZero || RHSBits.One.getBoolValue() || LHSBits.One.getBoolValue())
2789 return true;
2790 }
2791
2792 // LShr(UINT_MAX, Y) + 1 is a power of two (if add is nuw) or zero.
2793 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO))
2794 if (match(V: I, P: m_Add(L: m_LShr(L: m_AllOnes(), R: m_Value()), R: m_One())))
2795 return true;
2796 return false;
2797 }
2798 case Instruction::Select:
2799 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2800 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 2), OrZero, Q, Depth);
2801 case Instruction::PHI: {
2802 // A PHI node is power of two if all incoming values are power of two, or if
2803 // it is an induction variable where in each step its value is a power of
2804 // two.
2805 auto *PN = cast<PHINode>(Val: I);
2806 SimplifyQuery RecQ = Q.getWithoutCondContext();
2807
2808 // Check if it is an induction variable and always power of two.
2809 if (isPowerOfTwoRecurrence(PN, OrZero, Q&: RecQ, Depth))
2810 return true;
2811
2812 // Recursively check all incoming values. Limit recursion to 2 levels, so
2813 // that search complexity is limited to number of operands^2.
2814 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
2815 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
2816 // Value is power of 2 if it is coming from PHI node itself by induction.
2817 if (U.get() == PN)
2818 return true;
2819
2820 // Change the context instruction to the incoming block where it is
2821 // evaluated.
2822 RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
2823 return isKnownToBeAPowerOfTwo(V: U.get(), OrZero, Q: RecQ, Depth: NewDepth);
2824 });
2825 }
2826 case Instruction::Invoke:
2827 case Instruction::Call: {
2828 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2829 switch (II->getIntrinsicID()) {
2830 case Intrinsic::umax:
2831 case Intrinsic::smax:
2832 case Intrinsic::umin:
2833 case Intrinsic::smin:
2834 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 1), OrZero, Q, Depth) &&
2835 isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2836 // bswap/bitreverse just move around bits, but don't change any 1s/0s
2837 // thus dont change pow2/non-pow2 status.
2838 case Intrinsic::bitreverse:
2839 case Intrinsic::bswap:
2840 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2841 case Intrinsic::fshr:
2842 case Intrinsic::fshl:
2843 // If Op0 == Op1, this is a rotate. is_pow2(rotate(x, y)) == is_pow2(x)
2844 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
2845 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2846 break;
2847 default:
2848 break;
2849 }
2850 }
2851 return false;
2852 }
2853 default:
2854 return false;
2855 }
2856}
2857
2858/// Test whether a GEP's result is known to be non-null.
2859///
2860/// Uses properties inherent in a GEP to try to determine whether it is known
2861/// to be non-null.
2862///
2863/// Currently this routine does not support vector GEPs.
2864static bool isGEPKnownNonNull(const GEPOperator *GEP, const SimplifyQuery &Q,
2865 unsigned Depth) {
2866 const Function *F = nullptr;
2867 if (const Instruction *I = dyn_cast<Instruction>(Val: GEP))
2868 F = I->getFunction();
2869
2870 // If the gep is nuw or inbounds with invalid null pointer, then the GEP
2871 // may be null iff the base pointer is null and the offset is zero.
2872 if (!GEP->hasNoUnsignedWrap() &&
2873 !(GEP->isInBounds() &&
2874 !NullPointerIsDefined(F, AS: GEP->getPointerAddressSpace())))
2875 return false;
2876
2877 // FIXME: Support vector-GEPs.
2878 assert(GEP->getType()->isPointerTy() && "We only support plain pointer GEP");
2879
2880 // If the base pointer is non-null, we cannot walk to a null address with an
2881 // inbounds GEP in address space zero.
2882 if (isKnownNonZero(V: GEP->getPointerOperand(), Q, Depth))
2883 return true;
2884
2885 // Walk the GEP operands and see if any operand introduces a non-zero offset.
2886 // If so, then the GEP cannot produce a null pointer, as doing so would
2887 // inherently violate the inbounds contract within address space zero.
2888 for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
2889 GTI != GTE; ++GTI) {
2890 // Struct types are easy -- they must always be indexed by a constant.
2891 if (StructType *STy = GTI.getStructTypeOrNull()) {
2892 ConstantInt *OpC = cast<ConstantInt>(Val: GTI.getOperand());
2893 unsigned ElementIdx = OpC->getZExtValue();
2894 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
2895 uint64_t ElementOffset = SL->getElementOffset(Idx: ElementIdx);
2896 if (ElementOffset > 0)
2897 return true;
2898 continue;
2899 }
2900
2901 // If we have a zero-sized type, the index doesn't matter. Keep looping.
2902 if (GTI.getSequentialElementStride(DL: Q.DL).isZero())
2903 continue;
2904
2905 // Fast path the constant operand case both for efficiency and so we don't
2906 // increment Depth when just zipping down an all-constant GEP.
2907 if (ConstantInt *OpC = dyn_cast<ConstantInt>(Val: GTI.getOperand())) {
2908 if (!OpC->isZero())
2909 return true;
2910 continue;
2911 }
2912
2913 // We post-increment Depth here because while isKnownNonZero increments it
2914 // as well, when we pop back up that increment won't persist. We don't want
2915 // to recurse 10k times just because we have 10k GEP operands. We don't
2916 // bail completely out because we want to handle constant GEPs regardless
2917 // of depth.
2918 if (Depth++ >= MaxAnalysisRecursionDepth)
2919 continue;
2920
2921 if (isKnownNonZero(V: GTI.getOperand(), Q, Depth))
2922 return true;
2923 }
2924
2925 return false;
2926}
2927
2928static bool isKnownNonNullFromDominatingCondition(const Value *V,
2929 const Instruction *CtxI,
2930 const DominatorTree *DT) {
2931 assert(!isa<Constant>(V) && "Called for constant?");
2932
2933 if (!CtxI || !DT)
2934 return false;
2935
2936 unsigned NumUsesExplored = 0;
2937 for (auto &U : V->uses()) {
2938 // Avoid massive lists
2939 if (NumUsesExplored >= DomConditionsMaxUses)
2940 break;
2941 NumUsesExplored++;
2942
2943 const Instruction *UI = cast<Instruction>(Val: U.getUser());
2944 // If the value is used as an argument to a call or invoke, then argument
2945 // attributes may provide an answer about null-ness.
2946 if (V->getType()->isPointerTy()) {
2947 if (const auto *CB = dyn_cast<CallBase>(Val: UI)) {
2948 if (CB->isArgOperand(U: &U) &&
2949 CB->paramHasNonNullAttr(ArgNo: CB->getArgOperandNo(U: &U),
2950 /*AllowUndefOrPoison=*/false) &&
2951 DT->dominates(Def: CB, User: CtxI))
2952 return true;
2953 }
2954 }
2955
2956 // If the value is used as a load/store, then the pointer must be non null.
2957 if (V == getLoadStorePointerOperand(V: UI)) {
2958 if (!NullPointerIsDefined(F: UI->getFunction(),
2959 AS: V->getType()->getPointerAddressSpace()) &&
2960 DT->dominates(Def: UI, User: CtxI))
2961 return true;
2962 }
2963
2964 if ((match(V: UI, P: m_IDiv(L: m_Value(), R: m_Specific(V))) ||
2965 match(V: UI, P: m_IRem(L: m_Value(), R: m_Specific(V)))) &&
2966 isValidAssumeForContext(Inv: UI, CxtI: CtxI, DT))
2967 return true;
2968
2969 // Consider only compare instructions uniquely controlling a branch
2970 Value *RHS;
2971 CmpPredicate Pred;
2972 if (!match(V: UI, P: m_c_ICmp(Pred, L: m_Specific(V), R: m_Value(V&: RHS))))
2973 continue;
2974
2975 bool NonNullIfTrue;
2976 if (cmpExcludesZero(Pred, RHS))
2977 NonNullIfTrue = true;
2978 else if (cmpExcludesZero(Pred: CmpInst::getInversePredicate(pred: Pred), RHS))
2979 NonNullIfTrue = false;
2980 else
2981 continue;
2982
2983 SmallVector<const User *, 4> WorkList;
2984 SmallPtrSet<const User *, 4> Visited;
2985 for (const auto *CmpU : UI->users()) {
2986 assert(WorkList.empty() && "Should be!");
2987 if (Visited.insert(Ptr: CmpU).second)
2988 WorkList.push_back(Elt: CmpU);
2989
2990 while (!WorkList.empty()) {
2991 auto *Curr = WorkList.pop_back_val();
2992
2993 // If a user is an AND, add all its users to the work list. We only
2994 // propagate "pred != null" condition through AND because it is only
2995 // correct to assume that all conditions of AND are met in true branch.
2996 // TODO: Support similar logic of OR and EQ predicate?
2997 if (NonNullIfTrue)
2998 if (match(V: Curr, P: m_LogicalAnd(L: m_Value(), R: m_Value()))) {
2999 for (const auto *CurrU : Curr->users())
3000 if (Visited.insert(Ptr: CurrU).second)
3001 WorkList.push_back(Elt: CurrU);
3002 continue;
3003 }
3004
3005 if (const CondBrInst *BI = dyn_cast<CondBrInst>(Val: Curr)) {
3006 BasicBlock *NonNullSuccessor =
3007 BI->getSuccessor(i: NonNullIfTrue ? 0 : 1);
3008 BasicBlockEdge Edge(BI->getParent(), NonNullSuccessor);
3009 if (DT->dominates(BBE: Edge, BB: CtxI->getParent()))
3010 return true;
3011 } else if (NonNullIfTrue && isGuard(U: Curr) &&
3012 DT->dominates(Def: cast<Instruction>(Val: Curr), User: CtxI)) {
3013 return true;
3014 }
3015 }
3016 }
3017 }
3018
3019 return false;
3020}
3021
3022/// Does the 'Range' metadata (which must be a valid MD_range operand list)
3023/// ensure that the value it's attached to is never Value? 'RangeType' is
3024/// is the type of the value described by the range.
3025static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value) {
3026 const unsigned NumRanges = Ranges->getNumOperands() / 2;
3027 assert(NumRanges >= 1);
3028 for (unsigned i = 0; i < NumRanges; ++i) {
3029 ConstantInt *Lower =
3030 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 0));
3031 ConstantInt *Upper =
3032 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 1));
3033 ConstantRange Range(Lower->getValue(), Upper->getValue());
3034 if (Range.contains(Val: Value))
3035 return false;
3036 }
3037 return true;
3038}
3039
3040/// Try to detect a recurrence that monotonically increases/decreases from a
3041/// non-zero starting value. These are common as induction variables.
3042static bool isNonZeroRecurrence(const PHINode *PN) {
3043 BinaryOperator *BO = nullptr;
3044 Value *Start = nullptr, *Step = nullptr;
3045 const APInt *StartC, *StepC;
3046 if (!matchSimpleRecurrence(P: PN, BO, Start, Step) ||
3047 !match(V: Start, P: m_APInt(Res&: StartC)) || StartC->isZero())
3048 return false;
3049
3050 switch (BO->getOpcode()) {
3051 case Instruction::Add:
3052 // Starting from non-zero and stepping away from zero can never wrap back
3053 // to zero.
3054 return BO->hasNoUnsignedWrap() ||
3055 (BO->hasNoSignedWrap() && match(V: Step, P: m_APInt(Res&: StepC)) &&
3056 StartC->isNegative() == StepC->isNegative());
3057 case Instruction::Mul:
3058 return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) &&
3059 match(V: Step, P: m_APInt(Res&: StepC)) && !StepC->isZero();
3060 case Instruction::Shl:
3061 return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap();
3062 case Instruction::AShr:
3063 case Instruction::LShr:
3064 return BO->isExact();
3065 default:
3066 return false;
3067 }
3068}
3069
3070static bool matchOpWithOpEqZero(Value *Op0, Value *Op1) {
3071 return match(V: Op0, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3072 L: m_Specific(V: Op1), R: m_Zero()))) ||
3073 match(V: Op1, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3074 L: m_Specific(V: Op0), R: m_Zero())));
3075}
3076
3077static bool isNonZeroAdd(const APInt &DemandedElts, const SimplifyQuery &Q,
3078 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3079 bool NUW, unsigned Depth) {
3080 // (X + (X != 0)) is non zero
3081 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3082 return true;
3083
3084 if (NUW)
3085 return isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3086 isKnownNonZero(V: X, DemandedElts, Q, Depth);
3087
3088 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3089 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3090
3091 // If X and Y are both non-negative (as signed values) then their sum is not
3092 // zero unless both X and Y are zero.
3093 if (XKnown.isNonNegative() && YKnown.isNonNegative())
3094 if (isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3095 isKnownNonZero(V: X, DemandedElts, Q, Depth))
3096 return true;
3097
3098 // If X and Y are both negative (as signed values) then their sum is not
3099 // zero unless both X and Y equal INT_MIN.
3100 if (XKnown.isNegative() && YKnown.isNegative()) {
3101 APInt Mask = APInt::getSignedMaxValue(numBits: BitWidth);
3102 // The sign bit of X is set. If some other bit is set then X is not equal
3103 // to INT_MIN.
3104 if (XKnown.One.intersects(RHS: Mask))
3105 return true;
3106 // The sign bit of Y is set. If some other bit is set then Y is not equal
3107 // to INT_MIN.
3108 if (YKnown.One.intersects(RHS: Mask))
3109 return true;
3110 }
3111
3112 // The sum of a non-negative number and a power of two is not zero.
3113 if (XKnown.isNonNegative() &&
3114 isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ false, Q, Depth))
3115 return true;
3116 if (YKnown.isNonNegative() &&
3117 isKnownToBeAPowerOfTwo(V: X, /*OrZero*/ false, Q, Depth))
3118 return true;
3119
3120 return KnownBits::add(LHS: XKnown, RHS: YKnown, NSW, NUW).isNonZero();
3121}
3122
3123static bool isNonZeroSub(const APInt &DemandedElts, const SimplifyQuery &Q,
3124 unsigned BitWidth, Value *X, Value *Y,
3125 unsigned Depth) {
3126 // (X - (X != 0)) is non zero
3127 // ((X != 0) - X) is non zero
3128 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3129 return true;
3130
3131 // TODO: Move this case into isKnownNonEqual().
3132 if (auto *C = dyn_cast<Constant>(Val: X))
3133 if (C->isNullValue() && isKnownNonZero(V: Y, DemandedElts, Q, Depth))
3134 return true;
3135
3136 return ::isKnownNonEqual(V1: X, V2: Y, DemandedElts, Q, Depth);
3137}
3138
3139static bool isNonZeroMul(const APInt &DemandedElts, const SimplifyQuery &Q,
3140 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3141 bool NUW, unsigned Depth) {
3142 // If X and Y are non-zero then so is X * Y as long as the multiplication
3143 // does not overflow.
3144 if (NSW || NUW)
3145 return isKnownNonZero(V: X, DemandedElts, Q, Depth) &&
3146 isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3147
3148 // If either X or Y is odd, then if the other is non-zero the result can't
3149 // be zero.
3150 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3151 if (XKnown.One[0])
3152 return isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3153
3154 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3155 if (YKnown.One[0])
3156 return XKnown.isNonZero() || isKnownNonZero(V: X, DemandedElts, Q, Depth);
3157
3158 // If there exists any subset of X (sX) and subset of Y (sY) s.t sX * sY is
3159 // non-zero, then X * Y is non-zero. We can find sX and sY by just taking
3160 // the lowest known One of X and Y. If they are non-zero, the result
3161 // must be non-zero. We can check if LSB(X) * LSB(Y) != 0 by doing
3162 // X.CountLeadingZeros + Y.CountLeadingZeros < BitWidth.
3163 return (XKnown.countMaxTrailingZeros() + YKnown.countMaxTrailingZeros()) <
3164 BitWidth;
3165}
3166
3167static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
3168 const SimplifyQuery &Q, const KnownBits &KnownVal,
3169 unsigned Depth) {
3170 auto ShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3171 switch (I->getOpcode()) {
3172 case Instruction::Shl:
3173 return Lhs.shl(ShiftAmt: Rhs);
3174 case Instruction::LShr:
3175 return Lhs.lshr(ShiftAmt: Rhs);
3176 case Instruction::AShr:
3177 return Lhs.ashr(ShiftAmt: Rhs);
3178 default:
3179 llvm_unreachable("Unknown Shift Opcode");
3180 }
3181 };
3182
3183 auto InvShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3184 switch (I->getOpcode()) {
3185 case Instruction::Shl:
3186 return Lhs.lshr(ShiftAmt: Rhs);
3187 case Instruction::LShr:
3188 case Instruction::AShr:
3189 return Lhs.shl(ShiftAmt: Rhs);
3190 default:
3191 llvm_unreachable("Unknown Shift Opcode");
3192 }
3193 };
3194
3195 if (KnownVal.isUnknown())
3196 return false;
3197
3198 KnownBits KnownCnt =
3199 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3200 APInt MaxShift = KnownCnt.getMaxValue();
3201 unsigned NumBits = KnownVal.getBitWidth();
3202 if (MaxShift.uge(RHS: NumBits))
3203 return false;
3204
3205 if (!ShiftOp(KnownVal.One, MaxShift).isZero())
3206 return true;
3207
3208 // If all of the bits shifted out are known to be zero, and Val is known
3209 // non-zero then at least one non-zero bit must remain.
3210 if (InvShiftOp(KnownVal.Zero, NumBits - MaxShift)
3211 .eq(RHS: InvShiftOp(APInt::getAllOnes(numBits: NumBits), NumBits - MaxShift)) &&
3212 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth))
3213 return true;
3214
3215 return false;
3216}
3217
3218static bool isKnownNonZeroFromOperator(const Operator *I,
3219 const APInt &DemandedElts,
3220 const SimplifyQuery &Q, unsigned Depth) {
3221 unsigned BitWidth = getBitWidth(Ty: I->getType()->getScalarType(), DL: Q.DL);
3222 switch (I->getOpcode()) {
3223 case Instruction::Alloca:
3224 // Alloca never returns null, malloc might.
3225 return I->getType()->getPointerAddressSpace() == 0;
3226 case Instruction::GetElementPtr:
3227 if (I->getType()->isPointerTy())
3228 return isGEPKnownNonNull(GEP: cast<GEPOperator>(Val: I), Q, Depth);
3229 break;
3230 case Instruction::BitCast: {
3231 // We need to be a bit careful here. We can only peek through the bitcast
3232 // if the scalar size of elements in the operand are smaller than and a
3233 // multiple of the size they are casting too. Take three cases:
3234 //
3235 // 1) Unsafe:
3236 // bitcast <2 x i16> %NonZero to <4 x i8>
3237 //
3238 // %NonZero can have 2 non-zero i16 elements, but isKnownNonZero on a
3239 // <4 x i8> requires that all 4 i8 elements be non-zero which isn't
3240 // guranteed (imagine just sign bit set in the 2 i16 elements).
3241 //
3242 // 2) Unsafe:
3243 // bitcast <4 x i3> %NonZero to <3 x i4>
3244 //
3245 // Even though the scalar size of the src (`i3`) is smaller than the
3246 // scalar size of the dst `i4`, because `i3` is not a multiple of `i4`
3247 // its possible for the `3 x i4` elements to be zero because there are
3248 // some elements in the destination that don't contain any full src
3249 // element.
3250 //
3251 // 3) Safe:
3252 // bitcast <4 x i8> %NonZero to <2 x i16>
3253 //
3254 // This is always safe as non-zero in the 4 i8 elements implies
3255 // non-zero in the combination of any two adjacent ones. Since i8 is a
3256 // multiple of i16, each i16 is guranteed to have 2 full i8 elements.
3257 // This all implies the 2 i16 elements are non-zero.
3258 Type *FromTy = I->getOperand(i: 0)->getType();
3259 if ((FromTy->isIntOrIntVectorTy() || FromTy->isPtrOrPtrVectorTy()) &&
3260 (BitWidth % getBitWidth(Ty: FromTy->getScalarType(), DL: Q.DL)) == 0)
3261 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3262 } break;
3263 case Instruction::IntToPtr:
3264 // Note that we have to take special care to avoid looking through
3265 // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
3266 // as casts that can alter the value, e.g., AddrSpaceCasts.
3267 if (!isa<ScalableVectorType>(Val: I->getType()) &&
3268 Q.DL.getTypeSizeInBits(Ty: I->getOperand(i: 0)->getType()).getFixedValue() <=
3269 Q.DL.getTypeSizeInBits(Ty: I->getType()).getFixedValue())
3270 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3271 break;
3272 case Instruction::PtrToAddr:
3273 // isKnownNonZero() for pointers refers to the address bits being non-zero,
3274 // so we can directly forward.
3275 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3276 case Instruction::PtrToInt:
3277 // For inttoptr, make sure the result size is >= the address size. If the
3278 // address is non-zero, any larger value is also non-zero.
3279 if (Q.DL.getAddressSizeInBits(Ty: I->getOperand(i: 0)->getType()) <=
3280 I->getType()->getScalarSizeInBits())
3281 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3282 break;
3283 case Instruction::Trunc:
3284 // nuw/nsw trunc preserves zero/non-zero status of input.
3285 if (auto *TI = dyn_cast<TruncInst>(Val: I))
3286 if (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap())
3287 return isKnownNonZero(V: TI->getOperand(i_nocapture: 0), DemandedElts, Q, Depth);
3288 break;
3289
3290 // Iff x - y != 0, then x ^ y != 0
3291 // Therefore we can do the same exact checks
3292 case Instruction::Xor:
3293 case Instruction::Sub:
3294 return isNonZeroSub(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3295 Y: I->getOperand(i: 1), Depth);
3296 case Instruction::Or:
3297 // (X | (X != 0)) is non zero
3298 if (matchOpWithOpEqZero(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1)))
3299 return true;
3300 // X | Y != 0 if X != Y.
3301 if (isKnownNonEqual(V1: I->getOperand(i: 0), V2: I->getOperand(i: 1), DemandedElts, Q,
3302 Depth))
3303 return true;
3304 // X | Y != 0 if X != 0 or Y != 0.
3305 return isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth) ||
3306 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3307 case Instruction::SExt:
3308 case Instruction::ZExt:
3309 // ext X != 0 if X != 0.
3310 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3311
3312 case Instruction::Shl: {
3313 // shl nsw/nuw can't remove any non-zero bits.
3314 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3315 if (Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO))
3316 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3317
3318 // shl X, Y != 0 if X is odd. Note that the value of the shift is undefined
3319 // if the lowest bit is shifted off the end.
3320 KnownBits Known(BitWidth);
3321 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth);
3322 if (Known.One[0])
3323 return true;
3324
3325 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3326 }
3327 case Instruction::LShr:
3328 case Instruction::AShr: {
3329 // shr exact can only shift out zero bits.
3330 const PossiblyExactOperator *BO = cast<PossiblyExactOperator>(Val: I);
3331 if (BO->isExact())
3332 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3333
3334 // shr X, Y != 0 if X is negative. Note that the value of the shift is not
3335 // defined if the sign bit is shifted off the end.
3336 KnownBits Known =
3337 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3338 if (Known.isNegative())
3339 return true;
3340
3341 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3342 }
3343 case Instruction::UDiv:
3344 case Instruction::SDiv: {
3345 // X / Y
3346 // div exact can only produce a zero if the dividend is zero.
3347 if (cast<PossiblyExactOperator>(Val: I)->isExact())
3348 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3349
3350 KnownBits XKnown =
3351 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3352 // If X is fully unknown we won't be able to figure anything out so don't
3353 // both computing knownbits for Y.
3354 if (XKnown.isUnknown())
3355 return false;
3356
3357 KnownBits YKnown =
3358 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3359 if (I->getOpcode() == Instruction::SDiv) {
3360 // For signed division need to compare abs value of the operands.
3361 XKnown = XKnown.abs(/*IntMinIsPoison*/ false);
3362 YKnown = YKnown.abs(/*IntMinIsPoison*/ false);
3363 }
3364 // If X u>= Y then div is non zero (0/0 is UB).
3365 std::optional<bool> XUgeY = KnownBits::uge(LHS: XKnown, RHS: YKnown);
3366 // If X is total unknown or X u< Y we won't be able to prove non-zero
3367 // with compute known bits so just return early.
3368 return XUgeY && *XUgeY;
3369 }
3370 case Instruction::Add: {
3371 // X + Y.
3372
3373 // If Add has nuw wrap flag, then if either X or Y is non-zero the result is
3374 // non-zero.
3375 auto *BO = cast<OverflowingBinaryOperator>(Val: I);
3376 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3377 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3378 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3379 }
3380 case Instruction::Mul: {
3381 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3382 return isNonZeroMul(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3383 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3384 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3385 }
3386 case Instruction::Select: {
3387 // (C ? X : Y) != 0 if X != 0 and Y != 0.
3388
3389 // First check if the arm is non-zero using `isKnownNonZero`. If that fails,
3390 // then see if the select condition implies the arm is non-zero. For example
3391 // (X != 0 ? X : Y), we know the true arm is non-zero as the `X` "return" is
3392 // dominated by `X != 0`.
3393 auto SelectArmIsNonZero = [&](bool IsTrueArm) {
3394 Value *Op;
3395 Op = IsTrueArm ? I->getOperand(i: 1) : I->getOperand(i: 2);
3396 // Op is trivially non-zero.
3397 if (isKnownNonZero(V: Op, DemandedElts, Q, Depth))
3398 return true;
3399
3400 // The condition of the select dominates the true/false arm. Check if the
3401 // condition implies that a given arm is non-zero.
3402 Value *X;
3403 CmpPredicate Pred;
3404 if (!match(V: I->getOperand(i: 0), P: m_c_ICmp(Pred, L: m_Specific(V: Op), R: m_Value(V&: X))))
3405 return false;
3406
3407 if (!IsTrueArm)
3408 Pred = ICmpInst::getInversePredicate(pred: Pred);
3409
3410 return cmpExcludesZero(Pred, RHS: X);
3411 };
3412
3413 if (SelectArmIsNonZero(/* IsTrueArm */ true) &&
3414 SelectArmIsNonZero(/* IsTrueArm */ false))
3415 return true;
3416 break;
3417 }
3418 case Instruction::PHI: {
3419 auto *PN = cast<PHINode>(Val: I);
3420 if (Q.IIQ.UseInstrInfo && isNonZeroRecurrence(PN))
3421 return true;
3422
3423 // Check if all incoming values are non-zero using recursion.
3424 SimplifyQuery RecQ = Q.getWithoutCondContext();
3425 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
3426 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
3427 if (U.get() == PN)
3428 return true;
3429 RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
3430 // Check if the branch on the phi excludes zero.
3431 CmpPredicate Pred;
3432 Value *X;
3433 BasicBlock *TrueSucc, *FalseSucc;
3434 if (match(V: RecQ.CxtI,
3435 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: U.get()), R: m_Value(V&: X)),
3436 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
3437 // Check for cases of duplicate successors.
3438 if ((TrueSucc == PN->getParent()) != (FalseSucc == PN->getParent())) {
3439 // If we're using the false successor, invert the predicate.
3440 if (FalseSucc == PN->getParent())
3441 Pred = CmpInst::getInversePredicate(pred: Pred);
3442 if (cmpExcludesZero(Pred, RHS: X))
3443 return true;
3444 }
3445 }
3446 // Finally recurse on the edge and check it directly.
3447 return isKnownNonZero(V: U.get(), DemandedElts, Q: RecQ, Depth: NewDepth);
3448 });
3449 }
3450 case Instruction::InsertElement: {
3451 if (isa<ScalableVectorType>(Val: I->getType()))
3452 break;
3453
3454 const Value *Vec = I->getOperand(i: 0);
3455 const Value *Elt = I->getOperand(i: 1);
3456 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
3457
3458 unsigned NumElts = DemandedElts.getBitWidth();
3459 APInt DemandedVecElts = DemandedElts;
3460 bool SkipElt = false;
3461 // If we know the index we are inserting too, clear it from Vec check.
3462 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
3463 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
3464 SkipElt = !DemandedElts[CIdx->getZExtValue()];
3465 }
3466
3467 // Result is zero if Elt is non-zero and rest of the demanded elts in Vec
3468 // are non-zero.
3469 return (SkipElt || isKnownNonZero(V: Elt, Q, Depth)) &&
3470 (DemandedVecElts.isZero() ||
3471 isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth));
3472 }
3473 case Instruction::ExtractElement:
3474 if (const auto *EEI = dyn_cast<ExtractElementInst>(Val: I)) {
3475 const Value *Vec = EEI->getVectorOperand();
3476 const Value *Idx = EEI->getIndexOperand();
3477 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
3478 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
3479 unsigned NumElts = VecTy->getNumElements();
3480 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
3481 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
3482 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
3483 return isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth);
3484 }
3485 }
3486 break;
3487 case Instruction::ShuffleVector: {
3488 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
3489 if (!Shuf)
3490 break;
3491 APInt DemandedLHS, DemandedRHS;
3492 // For undef elements, we don't know anything about the common state of
3493 // the shuffle result.
3494 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
3495 break;
3496 // If demanded elements for both vecs are non-zero, the shuffle is non-zero.
3497 return (DemandedRHS.isZero() ||
3498 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 1), DemandedElts: DemandedRHS, Q, Depth)) &&
3499 (DemandedLHS.isZero() ||
3500 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 0), DemandedElts: DemandedLHS, Q, Depth));
3501 }
3502 case Instruction::Freeze:
3503 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth) &&
3504 isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
3505 Depth);
3506 case Instruction::Load: {
3507 auto *LI = cast<LoadInst>(Val: I);
3508 // A Load tagged with nonnull or dereferenceable with null pointer undefined
3509 // is never null.
3510 if (auto *PtrT = dyn_cast<PointerType>(Val: I->getType())) {
3511 if (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_nonnull) ||
3512 (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_dereferenceable) &&
3513 !NullPointerIsDefined(F: LI->getFunction(), AS: PtrT->getAddressSpace())))
3514 return true;
3515 } else if (MDNode *Ranges = Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_range)) {
3516 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3517 }
3518
3519 // No need to fall through to computeKnownBits as range metadata is already
3520 // handled in isKnownNonZero.
3521 return false;
3522 }
3523 case Instruction::ExtractValue: {
3524 const WithOverflowInst *WO;
3525 if (match(V: I, P: m_ExtractValue<0>(V: m_WithOverflowInst(I&: WO)))) {
3526 switch (WO->getBinaryOp()) {
3527 default:
3528 break;
3529 case Instruction::Add:
3530 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3531 Y: WO->getArgOperand(i: 1),
3532 /*NSW=*/false,
3533 /*NUW=*/false, Depth);
3534 case Instruction::Sub:
3535 return isNonZeroSub(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3536 Y: WO->getArgOperand(i: 1), Depth);
3537 case Instruction::Mul:
3538 return isNonZeroMul(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3539 Y: WO->getArgOperand(i: 1),
3540 /*NSW=*/false, /*NUW=*/false, Depth);
3541 break;
3542 }
3543 }
3544 break;
3545 }
3546 case Instruction::Call:
3547 case Instruction::Invoke: {
3548 const auto *Call = cast<CallBase>(Val: I);
3549 if (I->getType()->isPointerTy()) {
3550 if (Call->isReturnNonNull())
3551 return true;
3552 if (const auto *RP = getArgumentAliasingToReturnedPointer(Call, MustPreserveNullness: true))
3553 return isKnownNonZero(V: RP, Q, Depth);
3554 } else {
3555 if (MDNode *Ranges = Q.IIQ.getMetadata(I: Call, KindID: LLVMContext::MD_range))
3556 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3557 if (std::optional<ConstantRange> Range = Call->getRange()) {
3558 const APInt ZeroValue(Range->getBitWidth(), 0);
3559 if (!Range->contains(Val: ZeroValue))
3560 return true;
3561 }
3562 if (const Value *RV = Call->getReturnedArgOperand())
3563 if (RV->getType() == I->getType() && isKnownNonZero(V: RV, Q, Depth))
3564 return true;
3565 }
3566
3567 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3568 switch (II->getIntrinsicID()) {
3569 case Intrinsic::sshl_sat:
3570 case Intrinsic::ushl_sat:
3571 case Intrinsic::abs:
3572 case Intrinsic::bitreverse:
3573 case Intrinsic::bswap:
3574 case Intrinsic::ctpop:
3575 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3576 // NB: We don't do usub_sat here as in any case we can prove its
3577 // non-zero, we will fold it to `sub nuw` in InstCombine.
3578 case Intrinsic::ssub_sat:
3579 // For most types, if x != y then ssub.sat x, y != 0. But
3580 // ssub.sat.i1 0, -1 = 0, because 1 saturates to 0. This means
3581 // isNonZeroSub will do the wrong thing for ssub.sat.i1.
3582 if (BitWidth == 1)
3583 return false;
3584 return isNonZeroSub(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3585 Y: II->getArgOperand(i: 1), Depth);
3586 case Intrinsic::sadd_sat:
3587 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3588 Y: II->getArgOperand(i: 1),
3589 /*NSW=*/true, /* NUW=*/false, Depth);
3590 // Vec reverse preserves zero/non-zero status from input vec.
3591 case Intrinsic::vector_reverse:
3592 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
3593 Q, Depth);
3594 // umin/smin/smax/smin/or of all non-zero elements is always non-zero.
3595 case Intrinsic::vector_reduce_or:
3596 case Intrinsic::vector_reduce_umax:
3597 case Intrinsic::vector_reduce_umin:
3598 case Intrinsic::vector_reduce_smax:
3599 case Intrinsic::vector_reduce_smin:
3600 return isKnownNonZero(V: II->getArgOperand(i: 0), Q, Depth);
3601 case Intrinsic::umax:
3602 case Intrinsic::uadd_sat:
3603 // umax(X, (X != 0)) is non zero
3604 // X +usat (X != 0) is non zero
3605 if (matchOpWithOpEqZero(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1)))
3606 return true;
3607
3608 return isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth) ||
3609 isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3610 case Intrinsic::smax: {
3611 // If either arg is strictly positive the result is non-zero. Otherwise
3612 // the result is non-zero if both ops are non-zero.
3613 auto IsNonZero = [&](Value *Op, std::optional<bool> &OpNonZero,
3614 const KnownBits &OpKnown) {
3615 if (!OpNonZero.has_value())
3616 OpNonZero = OpKnown.isNonZero() ||
3617 isKnownNonZero(V: Op, DemandedElts, Q, Depth);
3618 return *OpNonZero;
3619 };
3620 // Avoid re-computing isKnownNonZero.
3621 std::optional<bool> Op0NonZero, Op1NonZero;
3622 KnownBits Op1Known =
3623 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3624 if (Op1Known.isNonNegative() &&
3625 IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known))
3626 return true;
3627 KnownBits Op0Known =
3628 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3629 if (Op0Known.isNonNegative() &&
3630 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known))
3631 return true;
3632 return IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known) &&
3633 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known);
3634 }
3635 case Intrinsic::smin: {
3636 // If either arg is negative the result is non-zero. Otherwise
3637 // the result is non-zero if both ops are non-zero.
3638 KnownBits Op1Known =
3639 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3640 if (Op1Known.isNegative())
3641 return true;
3642 KnownBits Op0Known =
3643 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3644 if (Op0Known.isNegative())
3645 return true;
3646
3647 if (Op1Known.isNonZero() && Op0Known.isNonZero())
3648 return true;
3649 }
3650 [[fallthrough]];
3651 case Intrinsic::umin:
3652 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth) &&
3653 isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3654 case Intrinsic::cttz:
3655 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3656 .Zero[0];
3657 case Intrinsic::ctlz:
3658 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3659 .isNonNegative();
3660 case Intrinsic::fshr:
3661 case Intrinsic::fshl:
3662 // If Op0 == Op1, this is a rotate. rotate(x, y) != 0 iff x != 0.
3663 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
3664 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3665 break;
3666 case Intrinsic::vscale:
3667 return true;
3668 case Intrinsic::experimental_get_vector_length:
3669 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3670 default:
3671 break;
3672 }
3673 break;
3674 }
3675
3676 return false;
3677 }
3678 }
3679
3680 KnownBits Known(BitWidth);
3681 computeKnownBits(V: I, DemandedElts, Known, Q, Depth);
3682 return Known.One != 0;
3683}
3684
3685/// Return true if the given value is known to be non-zero when defined. For
3686/// vectors, return true if every demanded element is known to be non-zero when
3687/// defined. For pointers, if the context instruction and dominator tree are
3688/// specified, perform context-sensitive analysis and return true if the
3689/// pointer couldn't possibly be null at the specified instruction.
3690/// Supports values with integer or pointer type and vectors of integers.
3691bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
3692 const SimplifyQuery &Q, unsigned Depth) {
3693 Type *Ty = V->getType();
3694
3695#ifndef NDEBUG
3696 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
3697
3698 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
3699 assert(
3700 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
3701 "DemandedElt width should equal the fixed vector number of elements");
3702 } else {
3703 assert(DemandedElts == APInt(1, 1) &&
3704 "DemandedElt width should be 1 for scalars");
3705 }
3706#endif
3707
3708 if (auto *C = dyn_cast<Constant>(Val: V)) {
3709 if (C->isNullValue())
3710 return false;
3711 if (isa<ConstantInt>(Val: C))
3712 // Must be non-zero due to null test above.
3713 return true;
3714
3715 // For constant vectors, check that all elements are poison or known
3716 // non-zero to determine that the whole vector is known non-zero.
3717 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Ty)) {
3718 for (unsigned i = 0, e = VecTy->getNumElements(); i != e; ++i) {
3719 if (!DemandedElts[i])
3720 continue;
3721 Constant *Elt = C->getAggregateElement(Elt: i);
3722 if (!Elt || Elt->isNullValue())
3723 return false;
3724 if (!isa<PoisonValue>(Val: Elt) && !isa<ConstantInt>(Val: Elt))
3725 return false;
3726 }
3727 return true;
3728 }
3729
3730 // Constant ptrauth can be null, iff the base pointer can be.
3731 if (auto *CPA = dyn_cast<ConstantPtrAuth>(Val: V))
3732 return isKnownNonZero(V: CPA->getPointer(), DemandedElts, Q, Depth);
3733
3734 // A global variable in address space 0 is non null unless extern weak
3735 // or an absolute symbol reference. Other address spaces may have null as a
3736 // valid address for a global, so we can't assume anything.
3737 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
3738 if (!GV->isAbsoluteSymbolRef() && !GV->hasExternalWeakLinkage() &&
3739 GV->getType()->getAddressSpace() == 0)
3740 return true;
3741 }
3742
3743 // For constant expressions, fall through to the Operator code below.
3744 if (!isa<ConstantExpr>(Val: V))
3745 return false;
3746 }
3747
3748 if (const auto *A = dyn_cast<Argument>(Val: V))
3749 if (std::optional<ConstantRange> Range = A->getRange()) {
3750 const APInt ZeroValue(Range->getBitWidth(), 0);
3751 if (!Range->contains(Val: ZeroValue))
3752 return true;
3753 }
3754
3755 if (!isa<Constant>(Val: V) && isKnownNonZeroFromAssume(V, Q))
3756 return true;
3757
3758 // Some of the tests below are recursive, so bail out if we hit the limit.
3759 if (Depth++ >= MaxAnalysisRecursionDepth)
3760 return false;
3761
3762 // Check for pointer simplifications.
3763
3764 if (PointerType *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
3765 // A byval, inalloca may not be null in a non-default addres space. A
3766 // nonnull argument is assumed never 0.
3767 if (const Argument *A = dyn_cast<Argument>(Val: V)) {
3768 if (((A->hasPassPointeeByValueCopyAttr() &&
3769 !NullPointerIsDefined(F: A->getParent(), AS: PtrTy->getAddressSpace())) ||
3770 A->hasNonNullAttr()))
3771 return true;
3772 }
3773 }
3774
3775 if (const auto *I = dyn_cast<Operator>(Val: V))
3776 if (isKnownNonZeroFromOperator(I, DemandedElts, Q, Depth))
3777 return true;
3778
3779 if (!isa<Constant>(Val: V) &&
3780 isKnownNonNullFromDominatingCondition(V, CtxI: Q.CxtI, DT: Q.DT))
3781 return true;
3782
3783 if (const Value *Stripped = stripNullTest(V))
3784 return isKnownNonZero(V: Stripped, DemandedElts, Q, Depth);
3785
3786 return false;
3787}
3788
3789bool llvm::isKnownNonZero(const Value *V, const SimplifyQuery &Q,
3790 unsigned Depth) {
3791 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
3792 APInt DemandedElts =
3793 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
3794 return ::isKnownNonZero(V, DemandedElts, Q, Depth);
3795}
3796
3797/// If the pair of operators are the same invertible function, return the
3798/// the operands of the function corresponding to each input. Otherwise,
3799/// return std::nullopt. An invertible function is one that is 1-to-1 and maps
3800/// every input value to exactly one output value. This is equivalent to
3801/// saying that Op1 and Op2 are equal exactly when the specified pair of
3802/// operands are equal, (except that Op1 and Op2 may be poison more often.)
3803static std::optional<std::pair<Value*, Value*>>
3804getInvertibleOperands(const Operator *Op1,
3805 const Operator *Op2) {
3806 if (Op1->getOpcode() != Op2->getOpcode())
3807 return std::nullopt;
3808
3809 auto getOperands = [&](unsigned OpNum) -> auto {
3810 return std::make_pair(x: Op1->getOperand(i: OpNum), y: Op2->getOperand(i: OpNum));
3811 };
3812
3813 switch (Op1->getOpcode()) {
3814 default:
3815 break;
3816 case Instruction::Or:
3817 if (!cast<PossiblyDisjointInst>(Val: Op1)->isDisjoint() ||
3818 !cast<PossiblyDisjointInst>(Val: Op2)->isDisjoint())
3819 break;
3820 [[fallthrough]];
3821 case Instruction::Xor:
3822 case Instruction::Add: {
3823 Value *Other;
3824 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 0)), R: m_Value(V&: Other))))
3825 return std::make_pair(x: Op1->getOperand(i: 1), y&: Other);
3826 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 1)), R: m_Value(V&: Other))))
3827 return std::make_pair(x: Op1->getOperand(i: 0), y&: Other);
3828 break;
3829 }
3830 case Instruction::Sub:
3831 if (Op1->getOperand(i: 0) == Op2->getOperand(i: 0))
3832 return getOperands(1);
3833 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3834 return getOperands(0);
3835 break;
3836 case Instruction::Mul: {
3837 // invertible if A * B == (A * B) mod 2^N where A, and B are integers
3838 // and N is the bitwdith. The nsw case is non-obvious, but proven by
3839 // alive2: https://alive2.llvm.org/ce/z/Z6D5qK
3840 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
3841 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
3842 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
3843 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
3844 break;
3845
3846 // Assume operand order has been canonicalized
3847 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1) &&
3848 isa<ConstantInt>(Val: Op1->getOperand(i: 1)) &&
3849 !cast<ConstantInt>(Val: Op1->getOperand(i: 1))->isZero())
3850 return getOperands(0);
3851 break;
3852 }
3853 case Instruction::Shl: {
3854 // Same as multiplies, with the difference that we don't need to check
3855 // for a non-zero multiply. Shifts always multiply by non-zero.
3856 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
3857 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
3858 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
3859 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
3860 break;
3861
3862 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3863 return getOperands(0);
3864 break;
3865 }
3866 case Instruction::AShr:
3867 case Instruction::LShr: {
3868 auto *PEO1 = cast<PossiblyExactOperator>(Val: Op1);
3869 auto *PEO2 = cast<PossiblyExactOperator>(Val: Op2);
3870 if (!PEO1->isExact() || !PEO2->isExact())
3871 break;
3872
3873 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3874 return getOperands(0);
3875 break;
3876 }
3877 case Instruction::SExt:
3878 case Instruction::ZExt:
3879 if (Op1->getOperand(i: 0)->getType() == Op2->getOperand(i: 0)->getType())
3880 return getOperands(0);
3881 break;
3882 case Instruction::PHI: {
3883 const PHINode *PN1 = cast<PHINode>(Val: Op1);
3884 const PHINode *PN2 = cast<PHINode>(Val: Op2);
3885
3886 // If PN1 and PN2 are both recurrences, can we prove the entire recurrences
3887 // are a single invertible function of the start values? Note that repeated
3888 // application of an invertible function is also invertible
3889 BinaryOperator *BO1 = nullptr;
3890 Value *Start1 = nullptr, *Step1 = nullptr;
3891 BinaryOperator *BO2 = nullptr;
3892 Value *Start2 = nullptr, *Step2 = nullptr;
3893 if (PN1->getParent() != PN2->getParent() ||
3894 !matchSimpleRecurrence(P: PN1, BO&: BO1, Start&: Start1, Step&: Step1) ||
3895 !matchSimpleRecurrence(P: PN2, BO&: BO2, Start&: Start2, Step&: Step2))
3896 break;
3897
3898 auto Values = getInvertibleOperands(Op1: cast<Operator>(Val: BO1),
3899 Op2: cast<Operator>(Val: BO2));
3900 if (!Values)
3901 break;
3902
3903 // We have to be careful of mutually defined recurrences here. Ex:
3904 // * X_i = X_(i-1) OP Y_(i-1), and Y_i = X_(i-1) OP V
3905 // * X_i = Y_i = X_(i-1) OP Y_(i-1)
3906 // The invertibility of these is complicated, and not worth reasoning
3907 // about (yet?).
3908 if (Values->first != PN1 || Values->second != PN2)
3909 break;
3910
3911 return std::make_pair(x&: Start1, y&: Start2);
3912 }
3913 }
3914 return std::nullopt;
3915}
3916
3917/// Return true if V1 == (binop V2, X), where X is known non-zero.
3918/// Only handle a small subset of binops where (binop V2, X) with non-zero X
3919/// implies V2 != V1.
3920static bool isModifyingBinopOfNonZero(const Value *V1, const Value *V2,
3921 const APInt &DemandedElts,
3922 const SimplifyQuery &Q, unsigned Depth) {
3923 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V1);
3924 if (!BO)
3925 return false;
3926 switch (BO->getOpcode()) {
3927 default:
3928 break;
3929 case Instruction::Or:
3930 if (!cast<PossiblyDisjointInst>(Val: V1)->isDisjoint())
3931 break;
3932 [[fallthrough]];
3933 case Instruction::Xor:
3934 case Instruction::Add:
3935 Value *Op = nullptr;
3936 if (V2 == BO->getOperand(i_nocapture: 0))
3937 Op = BO->getOperand(i_nocapture: 1);
3938 else if (V2 == BO->getOperand(i_nocapture: 1))
3939 Op = BO->getOperand(i_nocapture: 0);
3940 else
3941 return false;
3942 return isKnownNonZero(V: Op, DemandedElts, Q, Depth: Depth + 1);
3943 }
3944 return false;
3945}
3946
3947/// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
3948/// the multiplication is nuw or nsw.
3949static bool isNonEqualMul(const Value *V1, const Value *V2,
3950 const APInt &DemandedElts, const SimplifyQuery &Q,
3951 unsigned Depth) {
3952 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
3953 const APInt *C;
3954 return match(V: OBO, P: m_Mul(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
3955 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
3956 !C->isZero() && !C->isOne() &&
3957 isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
3958 }
3959 return false;
3960}
3961
3962/// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
3963/// the shift is nuw or nsw.
3964static bool isNonEqualShl(const Value *V1, const Value *V2,
3965 const APInt &DemandedElts, const SimplifyQuery &Q,
3966 unsigned Depth) {
3967 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
3968 const APInt *C;
3969 return match(V: OBO, P: m_Shl(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
3970 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
3971 !C->isZero() && isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
3972 }
3973 return false;
3974}
3975
3976static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
3977 const APInt &DemandedElts, const SimplifyQuery &Q,
3978 unsigned Depth) {
3979 // Check two PHIs are in same block.
3980 if (PN1->getParent() != PN2->getParent())
3981 return false;
3982
3983 SmallPtrSet<const BasicBlock *, 8> VisitedBBs;
3984 bool UsedFullRecursion = false;
3985 for (const BasicBlock *IncomBB : PN1->blocks()) {
3986 if (!VisitedBBs.insert(Ptr: IncomBB).second)
3987 continue; // Don't reprocess blocks that we have dealt with already.
3988 const Value *IV1 = PN1->getIncomingValueForBlock(BB: IncomBB);
3989 const Value *IV2 = PN2->getIncomingValueForBlock(BB: IncomBB);
3990 const APInt *C1, *C2;
3991 if (match(V: IV1, P: m_APInt(Res&: C1)) && match(V: IV2, P: m_APInt(Res&: C2)) && *C1 != *C2)
3992 continue;
3993
3994 // Only one pair of phi operands is allowed for full recursion.
3995 if (UsedFullRecursion)
3996 return false;
3997
3998 SimplifyQuery RecQ = Q.getWithoutCondContext();
3999 RecQ.CxtI = IncomBB->getTerminator();
4000 if (!isKnownNonEqual(V1: IV1, V2: IV2, DemandedElts, Q: RecQ, Depth: Depth + 1))
4001 return false;
4002 UsedFullRecursion = true;
4003 }
4004 return true;
4005}
4006
4007static bool isNonEqualSelect(const Value *V1, const Value *V2,
4008 const APInt &DemandedElts, const SimplifyQuery &Q,
4009 unsigned Depth) {
4010 const SelectInst *SI1 = dyn_cast<SelectInst>(Val: V1);
4011 if (!SI1)
4012 return false;
4013
4014 if (const SelectInst *SI2 = dyn_cast<SelectInst>(Val: V2)) {
4015 const Value *Cond1 = SI1->getCondition();
4016 const Value *Cond2 = SI2->getCondition();
4017 if (Cond1 == Cond2)
4018 return isKnownNonEqual(V1: SI1->getTrueValue(), V2: SI2->getTrueValue(),
4019 DemandedElts, Q, Depth: Depth + 1) &&
4020 isKnownNonEqual(V1: SI1->getFalseValue(), V2: SI2->getFalseValue(),
4021 DemandedElts, Q, Depth: Depth + 1);
4022 }
4023 return isKnownNonEqual(V1: SI1->getTrueValue(), V2, DemandedElts, Q, Depth: Depth + 1) &&
4024 isKnownNonEqual(V1: SI1->getFalseValue(), V2, DemandedElts, Q, Depth: Depth + 1);
4025}
4026
4027// Check to see if A is both a GEP and is the incoming value for a PHI in the
4028// loop, and B is either a ptr or another GEP. If the PHI has 2 incoming values,
4029// one of them being the recursive GEP A and the other a ptr at same base and at
4030// the same/higher offset than B we are only incrementing the pointer further in
4031// loop if offset of recursive GEP is greater than 0.
4032static bool isNonEqualPointersWithRecursiveGEP(const Value *A, const Value *B,
4033 const SimplifyQuery &Q) {
4034 if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
4035 return false;
4036
4037 auto *GEPA = dyn_cast<GEPOperator>(Val: A);
4038 if (!GEPA || GEPA->getNumIndices() != 1 || !isa<Constant>(Val: GEPA->idx_begin()))
4039 return false;
4040
4041 // Handle 2 incoming PHI values with one being a recursive GEP.
4042 auto *PN = dyn_cast<PHINode>(Val: GEPA->getPointerOperand());
4043 if (!PN || PN->getNumIncomingValues() != 2)
4044 return false;
4045
4046 // Search for the recursive GEP as an incoming operand, and record that as
4047 // Step.
4048 Value *Start = nullptr;
4049 Value *Step = const_cast<Value *>(A);
4050 if (PN->getIncomingValue(i: 0) == Step)
4051 Start = PN->getIncomingValue(i: 1);
4052 else if (PN->getIncomingValue(i: 1) == Step)
4053 Start = PN->getIncomingValue(i: 0);
4054 else
4055 return false;
4056
4057 // Other incoming node base should match the B base.
4058 // StartOffset >= OffsetB && StepOffset > 0?
4059 // StartOffset <= OffsetB && StepOffset < 0?
4060 // Is non-equal if above are true.
4061 // We use stripAndAccumulateInBoundsConstantOffsets to restrict the
4062 // optimisation to inbounds GEPs only.
4063 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: Start->getType());
4064 APInt StartOffset(IndexWidth, 0);
4065 Start = Start->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StartOffset);
4066 APInt StepOffset(IndexWidth, 0);
4067 Step = Step->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StepOffset);
4068
4069 // Check if Base Pointer of Step matches the PHI.
4070 if (Step != PN)
4071 return false;
4072 APInt OffsetB(IndexWidth, 0);
4073 B = B->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: OffsetB);
4074 return Start == B &&
4075 ((StartOffset.sge(RHS: OffsetB) && StepOffset.isStrictlyPositive()) ||
4076 (StartOffset.sle(RHS: OffsetB) && StepOffset.isNegative()));
4077}
4078
4079static bool isKnownNonEqualFromContext(const Value *V1, const Value *V2,
4080 const SimplifyQuery &Q, unsigned Depth) {
4081 if (!Q.CxtI)
4082 return false;
4083
4084 // Try to infer NonEqual based on information from dominating conditions.
4085 if (Q.DC && Q.DT) {
4086 auto IsKnownNonEqualFromDominatingCondition = [&](const Value *V) {
4087 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4088 Value *Cond = BI->getCondition();
4089 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4090 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()) &&
4091 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4092 /*LHSIsTrue=*/true, Depth)
4093 .value_or(u: false))
4094 return true;
4095
4096 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4097 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()) &&
4098 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4099 /*LHSIsTrue=*/false, Depth)
4100 .value_or(u: false))
4101 return true;
4102 }
4103
4104 return false;
4105 };
4106
4107 if (IsKnownNonEqualFromDominatingCondition(V1) ||
4108 IsKnownNonEqualFromDominatingCondition(V2))
4109 return true;
4110 }
4111
4112 if (!Q.AC)
4113 return false;
4114
4115 // Try to infer NonEqual based on information from assumptions.
4116 for (auto &AssumeVH : Q.AC->assumptionsFor(V: V1)) {
4117 if (!AssumeVH)
4118 continue;
4119 CallInst *I = cast<CallInst>(Val&: AssumeVH);
4120
4121 assert(I->getFunction() == Q.CxtI->getFunction() &&
4122 "Got assumption for the wrong function!");
4123 assert(I->getIntrinsicID() == Intrinsic::assume &&
4124 "must be an assume intrinsic");
4125
4126 if (isImpliedCondition(LHS: I->getArgOperand(i: 0), RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4127 /*LHSIsTrue=*/true, Depth)
4128 .value_or(u: false) &&
4129 isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
4130 return true;
4131 }
4132
4133 return false;
4134}
4135
4136/// Return true if it is known that V1 != V2.
4137static bool isKnownNonEqual(const Value *V1, const Value *V2,
4138 const APInt &DemandedElts, const SimplifyQuery &Q,
4139 unsigned Depth) {
4140 if (V1 == V2)
4141 return false;
4142 if (V1->getType() != V2->getType())
4143 // We can't look through casts yet.
4144 return false;
4145
4146 if (Depth >= MaxAnalysisRecursionDepth)
4147 return false;
4148
4149 // See if we can recurse through (exactly one of) our operands. This
4150 // requires our operation be 1-to-1 and map every input value to exactly
4151 // one output value. Such an operation is invertible.
4152 auto *O1 = dyn_cast<Operator>(Val: V1);
4153 auto *O2 = dyn_cast<Operator>(Val: V2);
4154 if (O1 && O2 && O1->getOpcode() == O2->getOpcode()) {
4155 if (auto Values = getInvertibleOperands(Op1: O1, Op2: O2))
4156 return isKnownNonEqual(V1: Values->first, V2: Values->second, DemandedElts, Q,
4157 Depth: Depth + 1);
4158
4159 if (const PHINode *PN1 = dyn_cast<PHINode>(Val: V1)) {
4160 const PHINode *PN2 = cast<PHINode>(Val: V2);
4161 // FIXME: This is missing a generalization to handle the case where one is
4162 // a PHI and another one isn't.
4163 if (isNonEqualPHIs(PN1, PN2, DemandedElts, Q, Depth))
4164 return true;
4165 };
4166 }
4167
4168 if (isModifyingBinopOfNonZero(V1, V2, DemandedElts, Q, Depth) ||
4169 isModifyingBinopOfNonZero(V1: V2, V2: V1, DemandedElts, Q, Depth))
4170 return true;
4171
4172 if (isNonEqualMul(V1, V2, DemandedElts, Q, Depth) ||
4173 isNonEqualMul(V1: V2, V2: V1, DemandedElts, Q, Depth))
4174 return true;
4175
4176 if (isNonEqualShl(V1, V2, DemandedElts, Q, Depth) ||
4177 isNonEqualShl(V1: V2, V2: V1, DemandedElts, Q, Depth))
4178 return true;
4179
4180 if (V1->getType()->isIntOrIntVectorTy()) {
4181 // Are any known bits in V1 contradictory to known bits in V2? If V1
4182 // has a known zero where V2 has a known one, they must not be equal.
4183 KnownBits Known1 = computeKnownBits(V: V1, DemandedElts, Q, Depth);
4184 if (!Known1.isUnknown()) {
4185 KnownBits Known2 = computeKnownBits(V: V2, DemandedElts, Q, Depth);
4186 if (Known1.Zero.intersects(RHS: Known2.One) ||
4187 Known2.Zero.intersects(RHS: Known1.One))
4188 return true;
4189 }
4190 }
4191
4192 if (isNonEqualSelect(V1, V2, DemandedElts, Q, Depth) ||
4193 isNonEqualSelect(V1: V2, V2: V1, DemandedElts, Q, Depth))
4194 return true;
4195
4196 if (isNonEqualPointersWithRecursiveGEP(A: V1, B: V2, Q) ||
4197 isNonEqualPointersWithRecursiveGEP(A: V2, B: V1, Q))
4198 return true;
4199
4200 Value *A, *B;
4201 // PtrToInts are NonEqual if their Ptrs are NonEqual.
4202 // Check PtrToInt type matches the pointer size.
4203 if (match(V: V1, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: A))) &&
4204 match(V: V2, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: B))))
4205 return isKnownNonEqual(V1: A, V2: B, DemandedElts, Q, Depth: Depth + 1);
4206
4207 if (isKnownNonEqualFromContext(V1, V2, Q, Depth))
4208 return true;
4209
4210 return false;
4211}
4212
4213/// For vector constants, loop over the elements and find the constant with the
4214/// minimum number of sign bits. Return 0 if the value is not a vector constant
4215/// or if any element was not analyzed; otherwise, return the count for the
4216/// element with the minimum number of sign bits.
4217static unsigned computeNumSignBitsVectorConstant(const Value *V,
4218 const APInt &DemandedElts,
4219 unsigned TyBits) {
4220 const auto *CV = dyn_cast<Constant>(Val: V);
4221 if (!CV || !isa<FixedVectorType>(Val: CV->getType()))
4222 return 0;
4223
4224 unsigned MinSignBits = TyBits;
4225 unsigned NumElts = cast<FixedVectorType>(Val: CV->getType())->getNumElements();
4226 for (unsigned i = 0; i != NumElts; ++i) {
4227 if (!DemandedElts[i])
4228 continue;
4229 // If we find a non-ConstantInt, bail out.
4230 auto *Elt = dyn_cast_or_null<ConstantInt>(Val: CV->getAggregateElement(Elt: i));
4231 if (!Elt)
4232 return 0;
4233
4234 MinSignBits = std::min(a: MinSignBits, b: Elt->getValue().getNumSignBits());
4235 }
4236
4237 return MinSignBits;
4238}
4239
4240static unsigned ComputeNumSignBitsImpl(const Value *V,
4241 const APInt &DemandedElts,
4242 const SimplifyQuery &Q, unsigned Depth);
4243
4244static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
4245 const SimplifyQuery &Q, unsigned Depth) {
4246 unsigned Result = ComputeNumSignBitsImpl(V, DemandedElts, Q, Depth);
4247 assert(Result > 0 && "At least one sign bit needs to be present!");
4248 return Result;
4249}
4250
4251/// Return the number of times the sign bit of the register is replicated into
4252/// the other bits. We know that at least 1 bit is always equal to the sign bit
4253/// (itself), but other cases can give us information. For example, immediately
4254/// after an "ashr X, 2", we know that the top 3 bits are all equal to each
4255/// other, so we return 3. For vectors, return the number of sign bits for the
4256/// vector element with the minimum number of known sign bits of the demanded
4257/// elements in the vector specified by DemandedElts.
4258static unsigned ComputeNumSignBitsImpl(const Value *V,
4259 const APInt &DemandedElts,
4260 const SimplifyQuery &Q, unsigned Depth) {
4261 Type *Ty = V->getType();
4262#ifndef NDEBUG
4263 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
4264
4265 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
4266 assert(
4267 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
4268 "DemandedElt width should equal the fixed vector number of elements");
4269 } else {
4270 assert(DemandedElts == APInt(1, 1) &&
4271 "DemandedElt width should be 1 for scalars");
4272 }
4273#endif
4274
4275 // We return the minimum number of sign bits that are guaranteed to be present
4276 // in V, so for undef we have to conservatively return 1. We don't have the
4277 // same behavior for poison though -- that's a FIXME today.
4278
4279 Type *ScalarTy = Ty->getScalarType();
4280 unsigned TyBits = ScalarTy->isPointerTy() ?
4281 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
4282 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
4283
4284 unsigned Tmp, Tmp2;
4285 unsigned FirstAnswer = 1;
4286
4287 // Note that ConstantInt is handled by the general computeKnownBits case
4288 // below.
4289
4290 if (Depth == MaxAnalysisRecursionDepth)
4291 return 1;
4292
4293 if (auto *U = dyn_cast<Operator>(Val: V)) {
4294 switch (Operator::getOpcode(V)) {
4295 default: break;
4296 case Instruction::BitCast: {
4297 Value *Src = U->getOperand(i: 0);
4298 Type *SrcTy = Src->getType();
4299
4300 // Skip if the source type is not an integer or integer vector type
4301 // This ensures we only process integer-like types
4302 if (!SrcTy->isIntOrIntVectorTy())
4303 break;
4304
4305 unsigned SrcBits = SrcTy->getScalarSizeInBits();
4306
4307 // Bitcast 'large element' scalar/vector to 'small element' vector.
4308 if ((SrcBits % TyBits) != 0)
4309 break;
4310
4311 // Only proceed if the destination type is a fixed-size vector
4312 if (isa<FixedVectorType>(Val: Ty)) {
4313 // Fast case - sign splat can be simply split across the small elements.
4314 // This works for both vector and scalar sources
4315 Tmp = ComputeNumSignBits(V: Src, Q, Depth: Depth + 1);
4316 if (Tmp == SrcBits)
4317 return TyBits;
4318 }
4319 break;
4320 }
4321 case Instruction::SExt:
4322 Tmp = TyBits - U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4323 return ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1) +
4324 Tmp;
4325
4326 case Instruction::SDiv: {
4327 const APInt *Denominator;
4328 // sdiv X, C -> adds log(C) sign bits.
4329 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4330
4331 // Ignore non-positive denominator.
4332 if (!Denominator->isStrictlyPositive())
4333 break;
4334
4335 // Calculate the incoming numerator bits.
4336 unsigned NumBits =
4337 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4338
4339 // Add floor(log(C)) bits to the numerator bits.
4340 return std::min(a: TyBits, b: NumBits + Denominator->logBase2());
4341 }
4342 break;
4343 }
4344
4345 case Instruction::SRem: {
4346 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4347
4348 const APInt *Denominator;
4349 // srem X, C -> we know that the result is within [-C+1,C) when C is a
4350 // positive constant. This let us put a lower bound on the number of sign
4351 // bits.
4352 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4353
4354 // Ignore non-positive denominator.
4355 if (Denominator->isStrictlyPositive()) {
4356 // Calculate the leading sign bit constraints by examining the
4357 // denominator. Given that the denominator is positive, there are two
4358 // cases:
4359 //
4360 // 1. The numerator is positive. The result range is [0,C) and
4361 // [0,C) u< (1 << ceilLogBase2(C)).
4362 //
4363 // 2. The numerator is negative. Then the result range is (-C,0] and
4364 // integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
4365 //
4366 // Thus a lower bound on the number of sign bits is `TyBits -
4367 // ceilLogBase2(C)`.
4368
4369 unsigned ResBits = TyBits - Denominator->ceilLogBase2();
4370 Tmp = std::max(a: Tmp, b: ResBits);
4371 }
4372 }
4373 return Tmp;
4374 }
4375
4376 case Instruction::AShr: {
4377 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4378 // ashr X, C -> adds C sign bits. Vectors too.
4379 const APInt *ShAmt;
4380 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4381 if (ShAmt->uge(RHS: TyBits))
4382 break; // Bad shift.
4383 unsigned ShAmtLimited = ShAmt->getZExtValue();
4384 Tmp += ShAmtLimited;
4385 if (Tmp > TyBits) Tmp = TyBits;
4386 }
4387 return Tmp;
4388 }
4389 case Instruction::Shl: {
4390 const APInt *ShAmt;
4391 Value *X = nullptr;
4392 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4393 // shl destroys sign bits.
4394 if (ShAmt->uge(RHS: TyBits))
4395 break; // Bad shift.
4396 // We can look through a zext (more or less treating it as a sext) if
4397 // all extended bits are shifted out.
4398 if (match(V: U->getOperand(i: 0), P: m_ZExt(Op: m_Value(V&: X))) &&
4399 ShAmt->uge(RHS: TyBits - X->getType()->getScalarSizeInBits())) {
4400 Tmp = ComputeNumSignBits(V: X, DemandedElts, Q, Depth: Depth + 1);
4401 Tmp += TyBits - X->getType()->getScalarSizeInBits();
4402 } else
4403 Tmp =
4404 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4405 if (ShAmt->uge(RHS: Tmp))
4406 break; // Shifted all sign bits out.
4407 Tmp2 = ShAmt->getZExtValue();
4408 return Tmp - Tmp2;
4409 }
4410 break;
4411 }
4412 case Instruction::And:
4413 case Instruction::Or:
4414 case Instruction::Xor: // NOT is handled here.
4415 // Logical binary ops preserve the number of sign bits at the worst.
4416 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4417 if (Tmp != 1) {
4418 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4419 FirstAnswer = std::min(a: Tmp, b: Tmp2);
4420 // We computed what we know about the sign bits as our first
4421 // answer. Now proceed to the generic code that uses
4422 // computeKnownBits, and pick whichever answer is better.
4423 }
4424 break;
4425
4426 case Instruction::Select: {
4427 // If we have a clamp pattern, we know that the number of sign bits will
4428 // be the minimum of the clamp min/max range.
4429 const Value *X;
4430 const APInt *CLow, *CHigh;
4431 if (isSignedMinMaxClamp(Select: U, In&: X, CLow, CHigh))
4432 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4433
4434 Tmp = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4435 if (Tmp == 1)
4436 break;
4437 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 2), DemandedElts, Q, Depth: Depth + 1);
4438 return std::min(a: Tmp, b: Tmp2);
4439 }
4440
4441 case Instruction::Add:
4442 // Add can have at most one carry bit. Thus we know that the output
4443 // is, at worst, one more bit than the inputs.
4444 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4445 if (Tmp == 1) break;
4446
4447 // Special case decrementing a value (ADD X, -1):
4448 if (const auto *CRHS = dyn_cast<Constant>(Val: U->getOperand(i: 1)))
4449 if (CRHS->isAllOnesValue()) {
4450 KnownBits Known(TyBits);
4451 computeKnownBits(V: U->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
4452
4453 // If the input is known to be 0 or 1, the output is 0/-1, which is
4454 // all sign bits set.
4455 if ((Known.Zero | 1).isAllOnes())
4456 return TyBits;
4457
4458 // If we are subtracting one from a positive number, there is no carry
4459 // out of the result.
4460 if (Known.isNonNegative())
4461 return Tmp;
4462 }
4463
4464 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4465 if (Tmp2 == 1)
4466 break;
4467 return std::min(a: Tmp, b: Tmp2) - 1;
4468
4469 case Instruction::Sub:
4470 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4471 if (Tmp2 == 1)
4472 break;
4473
4474 // Handle NEG.
4475 if (const auto *CLHS = dyn_cast<Constant>(Val: U->getOperand(i: 0)))
4476 if (CLHS->isNullValue()) {
4477 KnownBits Known(TyBits);
4478 computeKnownBits(V: U->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
4479 // If the input is known to be 0 or 1, the output is 0/-1, which is
4480 // all sign bits set.
4481 if ((Known.Zero | 1).isAllOnes())
4482 return TyBits;
4483
4484 // If the input is known to be positive (the sign bit is known clear),
4485 // the output of the NEG has the same number of sign bits as the
4486 // input.
4487 if (Known.isNonNegative())
4488 return Tmp2;
4489
4490 // Otherwise, we treat this like a SUB.
4491 }
4492
4493 // Sub can have at most one carry bit. Thus we know that the output
4494 // is, at worst, one more bit than the inputs.
4495 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4496 if (Tmp == 1)
4497 break;
4498 return std::min(a: Tmp, b: Tmp2) - 1;
4499
4500 case Instruction::Mul: {
4501 // The output of the Mul can be at most twice the valid bits in the
4502 // inputs.
4503 unsigned SignBitsOp0 =
4504 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4505 if (SignBitsOp0 == 1)
4506 break;
4507 unsigned SignBitsOp1 =
4508 ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4509 if (SignBitsOp1 == 1)
4510 break;
4511 unsigned OutValidBits =
4512 (TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
4513 return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
4514 }
4515
4516 case Instruction::PHI: {
4517 const PHINode *PN = cast<PHINode>(Val: U);
4518 unsigned NumIncomingValues = PN->getNumIncomingValues();
4519 // Don't analyze large in-degree PHIs.
4520 if (NumIncomingValues > 4) break;
4521 // Unreachable blocks may have zero-operand PHI nodes.
4522 if (NumIncomingValues == 0) break;
4523
4524 // Take the minimum of all incoming values. This can't infinitely loop
4525 // because of our depth threshold.
4526 SimplifyQuery RecQ = Q.getWithoutCondContext();
4527 Tmp = TyBits;
4528 for (unsigned i = 0, e = NumIncomingValues; i != e; ++i) {
4529 if (Tmp == 1) return Tmp;
4530 RecQ.CxtI = PN->getIncomingBlock(i)->getTerminator();
4531 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(V: PN->getIncomingValue(i),
4532 DemandedElts, Q: RecQ, Depth: Depth + 1));
4533 }
4534 return Tmp;
4535 }
4536
4537 case Instruction::Trunc: {
4538 // If the input contained enough sign bits that some remain after the
4539 // truncation, then we can make use of that. Otherwise we don't know
4540 // anything.
4541 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4542 unsigned OperandTyBits = U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4543 if (Tmp > (OperandTyBits - TyBits))
4544 return Tmp - (OperandTyBits - TyBits);
4545
4546 return 1;
4547 }
4548
4549 case Instruction::ExtractElement:
4550 // Look through extract element. At the moment we keep this simple and
4551 // skip tracking the specific element. But at least we might find
4552 // information valid for all elements of the vector (for example if vector
4553 // is sign extended, shifted, etc).
4554 return ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4555
4556 case Instruction::ShuffleVector: {
4557 // Collect the minimum number of sign bits that are shared by every vector
4558 // element referenced by the shuffle.
4559 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: U);
4560 if (!Shuf) {
4561 // FIXME: Add support for shufflevector constant expressions.
4562 return 1;
4563 }
4564 APInt DemandedLHS, DemandedRHS;
4565 // For undef elements, we don't know anything about the common state of
4566 // the shuffle result.
4567 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
4568 return 1;
4569 Tmp = std::numeric_limits<unsigned>::max();
4570 if (!!DemandedLHS) {
4571 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
4572 Tmp = ComputeNumSignBits(V: LHS, DemandedElts: DemandedLHS, Q, Depth: Depth + 1);
4573 }
4574 // If we don't know anything, early out and try computeKnownBits
4575 // fall-back.
4576 if (Tmp == 1)
4577 break;
4578 if (!!DemandedRHS) {
4579 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
4580 Tmp2 = ComputeNumSignBits(V: RHS, DemandedElts: DemandedRHS, Q, Depth: Depth + 1);
4581 Tmp = std::min(a: Tmp, b: Tmp2);
4582 }
4583 // If we don't know anything, early out and try computeKnownBits
4584 // fall-back.
4585 if (Tmp == 1)
4586 break;
4587 assert(Tmp <= TyBits && "Failed to determine minimum sign bits");
4588 return Tmp;
4589 }
4590 case Instruction::Call: {
4591 if (const auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
4592 switch (II->getIntrinsicID()) {
4593 default:
4594 break;
4595 case Intrinsic::abs:
4596 Tmp =
4597 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4598 if (Tmp == 1)
4599 break;
4600
4601 // Absolute value reduces number of sign bits by at most 1.
4602 return Tmp - 1;
4603 case Intrinsic::smin:
4604 case Intrinsic::smax: {
4605 const APInt *CLow, *CHigh;
4606 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
4607 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4608 }
4609 }
4610 }
4611 }
4612 }
4613 }
4614
4615 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4616 // use this information.
4617
4618 // If we can examine all elements of a vector constant successfully, we're
4619 // done (we can't do any better than that). If not, keep trying.
4620 if (unsigned VecSignBits =
4621 computeNumSignBitsVectorConstant(V, DemandedElts, TyBits))
4622 return VecSignBits;
4623
4624 KnownBits Known(TyBits);
4625 computeKnownBits(V, DemandedElts, Known, Q, Depth);
4626
4627 // If we know that the sign bit is either zero or one, determine the number of
4628 // identical bits in the top of the input value.
4629 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
4630}
4631
4632Intrinsic::ID llvm::getIntrinsicForCallSite(const CallBase &CB,
4633 const TargetLibraryInfo *TLI) {
4634 const Function *F = CB.getCalledFunction();
4635 if (!F)
4636 return Intrinsic::not_intrinsic;
4637
4638 if (F->isIntrinsic())
4639 return F->getIntrinsicID();
4640
4641 // We are going to infer semantics of a library function based on mapping it
4642 // to an LLVM intrinsic. Check that the library function is available from
4643 // this callbase and in this environment.
4644 LibFunc Func;
4645 if (F->hasLocalLinkage() || !TLI || !TLI->getLibFunc(CB, F&: Func) ||
4646 !CB.onlyReadsMemory())
4647 return Intrinsic::not_intrinsic;
4648
4649 switch (Func) {
4650 default:
4651 break;
4652 case LibFunc_sin:
4653 case LibFunc_sinf:
4654 case LibFunc_sinl:
4655 return Intrinsic::sin;
4656 case LibFunc_cos:
4657 case LibFunc_cosf:
4658 case LibFunc_cosl:
4659 return Intrinsic::cos;
4660 case LibFunc_tan:
4661 case LibFunc_tanf:
4662 case LibFunc_tanl:
4663 return Intrinsic::tan;
4664 case LibFunc_asin:
4665 case LibFunc_asinf:
4666 case LibFunc_asinl:
4667 return Intrinsic::asin;
4668 case LibFunc_acos:
4669 case LibFunc_acosf:
4670 case LibFunc_acosl:
4671 return Intrinsic::acos;
4672 case LibFunc_atan:
4673 case LibFunc_atanf:
4674 case LibFunc_atanl:
4675 return Intrinsic::atan;
4676 case LibFunc_atan2:
4677 case LibFunc_atan2f:
4678 case LibFunc_atan2l:
4679 return Intrinsic::atan2;
4680 case LibFunc_sinh:
4681 case LibFunc_sinhf:
4682 case LibFunc_sinhl:
4683 return Intrinsic::sinh;
4684 case LibFunc_cosh:
4685 case LibFunc_coshf:
4686 case LibFunc_coshl:
4687 return Intrinsic::cosh;
4688 case LibFunc_tanh:
4689 case LibFunc_tanhf:
4690 case LibFunc_tanhl:
4691 return Intrinsic::tanh;
4692 case LibFunc_exp:
4693 case LibFunc_expf:
4694 case LibFunc_expl:
4695 return Intrinsic::exp;
4696 case LibFunc_exp2:
4697 case LibFunc_exp2f:
4698 case LibFunc_exp2l:
4699 return Intrinsic::exp2;
4700 case LibFunc_exp10:
4701 case LibFunc_exp10f:
4702 case LibFunc_exp10l:
4703 return Intrinsic::exp10;
4704 case LibFunc_log:
4705 case LibFunc_logf:
4706 case LibFunc_logl:
4707 return Intrinsic::log;
4708 case LibFunc_log10:
4709 case LibFunc_log10f:
4710 case LibFunc_log10l:
4711 return Intrinsic::log10;
4712 case LibFunc_log2:
4713 case LibFunc_log2f:
4714 case LibFunc_log2l:
4715 return Intrinsic::log2;
4716 case LibFunc_fabs:
4717 case LibFunc_fabsf:
4718 case LibFunc_fabsl:
4719 return Intrinsic::fabs;
4720 case LibFunc_fmin:
4721 case LibFunc_fminf:
4722 case LibFunc_fminl:
4723 return Intrinsic::minnum;
4724 case LibFunc_fmax:
4725 case LibFunc_fmaxf:
4726 case LibFunc_fmaxl:
4727 return Intrinsic::maxnum;
4728 case LibFunc_copysign:
4729 case LibFunc_copysignf:
4730 case LibFunc_copysignl:
4731 return Intrinsic::copysign;
4732 case LibFunc_floor:
4733 case LibFunc_floorf:
4734 case LibFunc_floorl:
4735 return Intrinsic::floor;
4736 case LibFunc_ceil:
4737 case LibFunc_ceilf:
4738 case LibFunc_ceill:
4739 return Intrinsic::ceil;
4740 case LibFunc_trunc:
4741 case LibFunc_truncf:
4742 case LibFunc_truncl:
4743 return Intrinsic::trunc;
4744 case LibFunc_rint:
4745 case LibFunc_rintf:
4746 case LibFunc_rintl:
4747 return Intrinsic::rint;
4748 case LibFunc_nearbyint:
4749 case LibFunc_nearbyintf:
4750 case LibFunc_nearbyintl:
4751 return Intrinsic::nearbyint;
4752 case LibFunc_round:
4753 case LibFunc_roundf:
4754 case LibFunc_roundl:
4755 return Intrinsic::round;
4756 case LibFunc_roundeven:
4757 case LibFunc_roundevenf:
4758 case LibFunc_roundevenl:
4759 return Intrinsic::roundeven;
4760 case LibFunc_pow:
4761 case LibFunc_powf:
4762 case LibFunc_powl:
4763 return Intrinsic::pow;
4764 case LibFunc_sqrt:
4765 case LibFunc_sqrtf:
4766 case LibFunc_sqrtl:
4767 return Intrinsic::sqrt;
4768 }
4769
4770 return Intrinsic::not_intrinsic;
4771}
4772
4773/// Given an exploded icmp instruction, return true if the comparison only
4774/// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if
4775/// the result of the comparison is true when the input value is signed.
4776bool llvm::isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
4777 bool &TrueIfSigned) {
4778 switch (Pred) {
4779 case ICmpInst::ICMP_SLT: // True if LHS s< 0
4780 TrueIfSigned = true;
4781 return RHS.isZero();
4782 case ICmpInst::ICMP_SLE: // True if LHS s<= -1
4783 TrueIfSigned = true;
4784 return RHS.isAllOnes();
4785 case ICmpInst::ICMP_SGT: // True if LHS s> -1
4786 TrueIfSigned = false;
4787 return RHS.isAllOnes();
4788 case ICmpInst::ICMP_SGE: // True if LHS s>= 0
4789 TrueIfSigned = false;
4790 return RHS.isZero();
4791 case ICmpInst::ICMP_UGT:
4792 // True if LHS u> RHS and RHS == sign-bit-mask - 1
4793 TrueIfSigned = true;
4794 return RHS.isMaxSignedValue();
4795 case ICmpInst::ICMP_UGE:
4796 // True if LHS u>= RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
4797 TrueIfSigned = true;
4798 return RHS.isMinSignedValue();
4799 case ICmpInst::ICMP_ULT:
4800 // True if LHS u< RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
4801 TrueIfSigned = false;
4802 return RHS.isMinSignedValue();
4803 case ICmpInst::ICMP_ULE:
4804 // True if LHS u<= RHS and RHS == sign-bit-mask - 1
4805 TrueIfSigned = false;
4806 return RHS.isMaxSignedValue();
4807 default:
4808 return false;
4809 }
4810}
4811
4812static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
4813 bool CondIsTrue,
4814 const Instruction *CxtI,
4815 KnownFPClass &KnownFromContext,
4816 unsigned Depth = 0) {
4817 Value *A, *B;
4818 if (Depth < MaxAnalysisRecursionDepth &&
4819 (CondIsTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B)))
4820 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))))) {
4821 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue, CxtI, KnownFromContext,
4822 Depth: Depth + 1);
4823 computeKnownFPClassFromCond(V, Cond: B, CondIsTrue, CxtI, KnownFromContext,
4824 Depth: Depth + 1);
4825 return;
4826 }
4827 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A)))) {
4828 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue: !CondIsTrue, CxtI, KnownFromContext,
4829 Depth: Depth + 1);
4830 return;
4831 }
4832 CmpPredicate Pred;
4833 Value *LHS;
4834 uint64_t ClassVal = 0;
4835 const APFloat *CRHS;
4836 const APInt *RHS;
4837 if (match(V: Cond, P: m_FCmp(Pred, L: m_Value(V&: LHS), R: m_APFloat(Res&: CRHS)))) {
4838 auto [CmpVal, MaskIfTrue, MaskIfFalse] = fcmpImpliesClass(
4839 Pred, F: *cast<Instruction>(Val: Cond)->getParent()->getParent(), LHS, ConstRHS: *CRHS,
4840 LookThroughSrc: LHS != V);
4841 if (CmpVal == V)
4842 KnownFromContext.knownNot(RuleOut: ~(CondIsTrue ? MaskIfTrue : MaskIfFalse));
4843 } else if (match(V: Cond, P: m_Intrinsic<Intrinsic::is_fpclass>(
4844 Op0: m_Specific(V), Op1: m_ConstantInt(V&: ClassVal)))) {
4845 FPClassTest Mask = static_cast<FPClassTest>(ClassVal);
4846 KnownFromContext.knownNot(RuleOut: CondIsTrue ? ~Mask : Mask);
4847 } else if (match(V: Cond, P: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Specific(V)),
4848 R: m_APInt(Res&: RHS)))) {
4849 bool TrueIfSigned;
4850 if (!isSignBitCheck(Pred, RHS: *RHS, TrueIfSigned))
4851 return;
4852 if (TrueIfSigned == CondIsTrue)
4853 KnownFromContext.signBitMustBeOne();
4854 else
4855 KnownFromContext.signBitMustBeZero();
4856 }
4857}
4858
4859static KnownFPClass computeKnownFPClassFromContext(const Value *V,
4860 const SimplifyQuery &Q) {
4861 KnownFPClass KnownFromContext;
4862
4863 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
4864 computeKnownFPClassFromCond(V, Cond: Q.CC->Cond, CondIsTrue: !Q.CC->Invert, CxtI: Q.CxtI,
4865 KnownFromContext);
4866
4867 if (!Q.CxtI)
4868 return KnownFromContext;
4869
4870 if (Q.DC && Q.DT) {
4871 // Handle dominating conditions.
4872 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4873 Value *Cond = BI->getCondition();
4874
4875 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4876 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
4877 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, CxtI: Q.CxtI,
4878 KnownFromContext);
4879
4880 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4881 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
4882 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, CxtI: Q.CxtI,
4883 KnownFromContext);
4884 }
4885 }
4886
4887 if (!Q.AC)
4888 return KnownFromContext;
4889
4890 // Try to restrict the floating-point classes based on information from
4891 // assumptions.
4892 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
4893 if (!AssumeVH)
4894 continue;
4895 CallInst *I = cast<CallInst>(Val&: AssumeVH);
4896
4897 assert(I->getFunction() == Q.CxtI->getParent()->getParent() &&
4898 "Got assumption for the wrong function!");
4899 assert(I->getIntrinsicID() == Intrinsic::assume &&
4900 "must be an assume intrinsic");
4901
4902 if (!isValidAssumeForContext(Inv: I, CxtI: Q.CxtI, DT: Q.DT))
4903 continue;
4904
4905 computeKnownFPClassFromCond(V, Cond: I->getArgOperand(i: 0),
4906 /*CondIsTrue=*/true, CxtI: Q.CxtI, KnownFromContext);
4907 }
4908
4909 return KnownFromContext;
4910}
4911
4912void llvm::adjustKnownFPClassForSelectArm(KnownFPClass &Known, Value *Cond,
4913 Value *Arm, bool Invert,
4914 const SimplifyQuery &SQ,
4915 unsigned Depth) {
4916
4917 KnownFPClass KnownSrc;
4918 computeKnownFPClassFromCond(V: Arm, Cond,
4919 /*CondIsTrue=*/!Invert, CxtI: SQ.CxtI, KnownFromContext&: KnownSrc,
4920 Depth: Depth + 1);
4921 KnownSrc = KnownSrc.unionWith(RHS: Known);
4922 if (KnownSrc.isUnknown())
4923 return;
4924
4925 if (isGuaranteedNotToBeUndef(V: Arm, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT, Depth: Depth + 1))
4926 Known = KnownSrc;
4927}
4928
4929void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
4930 FPClassTest InterestedClasses, KnownFPClass &Known,
4931 const SimplifyQuery &Q, unsigned Depth);
4932
4933static void computeKnownFPClass(const Value *V, KnownFPClass &Known,
4934 FPClassTest InterestedClasses,
4935 const SimplifyQuery &Q, unsigned Depth) {
4936 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
4937 APInt DemandedElts =
4938 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
4939 computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Q, Depth);
4940}
4941
4942static void computeKnownFPClassForFPTrunc(const Operator *Op,
4943 const APInt &DemandedElts,
4944 FPClassTest InterestedClasses,
4945 KnownFPClass &Known,
4946 const SimplifyQuery &Q,
4947 unsigned Depth) {
4948 if ((InterestedClasses &
4949 (KnownFPClass::OrderedLessThanZeroMask | fcNan)) == fcNone)
4950 return;
4951
4952 KnownFPClass KnownSrc;
4953 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
4954 Known&: KnownSrc, Q, Depth: Depth + 1);
4955 Known = KnownFPClass::fptrunc(KnownSrc);
4956}
4957
4958static constexpr KnownFPClass::MinMaxKind getMinMaxKind(Intrinsic::ID IID) {
4959 switch (IID) {
4960 case Intrinsic::minimum:
4961 return KnownFPClass::MinMaxKind::minimum;
4962 case Intrinsic::maximum:
4963 return KnownFPClass::MinMaxKind::maximum;
4964 case Intrinsic::minimumnum:
4965 return KnownFPClass::MinMaxKind::minimumnum;
4966 case Intrinsic::maximumnum:
4967 return KnownFPClass::MinMaxKind::maximumnum;
4968 case Intrinsic::minnum:
4969 return KnownFPClass::MinMaxKind::minnum;
4970 case Intrinsic::maxnum:
4971 return KnownFPClass::MinMaxKind::maxnum;
4972 default:
4973 llvm_unreachable("not a floating-point min-max intrinsic");
4974 }
4975}
4976
4977/// \return true if this is a floating point value that is known to have a
4978/// magnitude smaller than 1. i.e., fabs(X) <= 1.0
4979static bool isAbsoluteValueLessEqualOne(const Value *V) {
4980 // TODO: Handle frexp and x - floor(x)?
4981 return match(V, P: m_Intrinsic<Intrinsic::amdgcn_trig_preop>(Op0: m_Value()));
4982}
4983
4984void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
4985 FPClassTest InterestedClasses, KnownFPClass &Known,
4986 const SimplifyQuery &Q, unsigned Depth) {
4987 assert(Known.isUnknown() && "should not be called with known information");
4988
4989 if (!DemandedElts) {
4990 // No demanded elts, better to assume we don't know anything.
4991 Known.resetAll();
4992 return;
4993 }
4994
4995 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
4996
4997 if (auto *CFP = dyn_cast<ConstantFP>(Val: V)) {
4998 Known = KnownFPClass(CFP->getValueAPF());
4999 return;
5000 }
5001
5002 if (isa<ConstantAggregateZero>(Val: V)) {
5003 Known.KnownFPClasses = fcPosZero;
5004 Known.SignBit = false;
5005 return;
5006 }
5007
5008 if (isa<PoisonValue>(Val: V)) {
5009 Known.KnownFPClasses = fcNone;
5010 Known.SignBit = false;
5011 return;
5012 }
5013
5014 // Try to handle fixed width vector constants
5015 auto *VFVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5016 const Constant *CV = dyn_cast<Constant>(Val: V);
5017 if (VFVTy && CV) {
5018 Known.KnownFPClasses = fcNone;
5019 bool SignBitAllZero = true;
5020 bool SignBitAllOne = true;
5021
5022 // For vectors, verify that each element is not NaN.
5023 unsigned NumElts = VFVTy->getNumElements();
5024 for (unsigned i = 0; i != NumElts; ++i) {
5025 if (!DemandedElts[i])
5026 continue;
5027
5028 Constant *Elt = CV->getAggregateElement(Elt: i);
5029 if (!Elt) {
5030 Known = KnownFPClass();
5031 return;
5032 }
5033 if (isa<PoisonValue>(Val: Elt))
5034 continue;
5035 auto *CElt = dyn_cast<ConstantFP>(Val: Elt);
5036 if (!CElt) {
5037 Known = KnownFPClass();
5038 return;
5039 }
5040
5041 const APFloat &C = CElt->getValueAPF();
5042 Known.KnownFPClasses |= C.classify();
5043 if (C.isNegative())
5044 SignBitAllZero = false;
5045 else
5046 SignBitAllOne = false;
5047 }
5048 if (SignBitAllOne != SignBitAllZero)
5049 Known.SignBit = SignBitAllOne;
5050 return;
5051 }
5052
5053 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) {
5054 Known.KnownFPClasses = fcNone;
5055 for (size_t I = 0, E = CDS->getNumElements(); I != E; ++I)
5056 Known |= CDS->getElementAsAPFloat(i: I).classify();
5057 return;
5058 }
5059
5060 if (const auto *CA = dyn_cast<ConstantAggregate>(Val: V)) {
5061 // TODO: Handle complex aggregates
5062 Known.KnownFPClasses = fcNone;
5063 for (const Use &Op : CA->operands()) {
5064 auto *CFP = dyn_cast<ConstantFP>(Val: Op.get());
5065 if (!CFP) {
5066 Known = KnownFPClass();
5067 return;
5068 }
5069
5070 Known |= CFP->getValueAPF().classify();
5071 }
5072
5073 return;
5074 }
5075
5076 FPClassTest KnownNotFromFlags = fcNone;
5077 if (const auto *CB = dyn_cast<CallBase>(Val: V))
5078 KnownNotFromFlags |= CB->getRetNoFPClass();
5079 else if (const auto *Arg = dyn_cast<Argument>(Val: V))
5080 KnownNotFromFlags |= Arg->getNoFPClass();
5081
5082 const Operator *Op = dyn_cast<Operator>(Val: V);
5083 if (const FPMathOperator *FPOp = dyn_cast_or_null<FPMathOperator>(Val: Op)) {
5084 if (FPOp->hasNoNaNs())
5085 KnownNotFromFlags |= fcNan;
5086 if (FPOp->hasNoInfs())
5087 KnownNotFromFlags |= fcInf;
5088 }
5089
5090 KnownFPClass AssumedClasses = computeKnownFPClassFromContext(V, Q);
5091 KnownNotFromFlags |= ~AssumedClasses.KnownFPClasses;
5092
5093 // We no longer need to find out about these bits from inputs if we can
5094 // assume this from flags/attributes.
5095 InterestedClasses &= ~KnownNotFromFlags;
5096
5097 llvm::scope_exit ClearClassesFromFlags([=, &Known] {
5098 Known.knownNot(RuleOut: KnownNotFromFlags);
5099 if (!Known.SignBit && AssumedClasses.SignBit) {
5100 if (*AssumedClasses.SignBit)
5101 Known.signBitMustBeOne();
5102 else
5103 Known.signBitMustBeZero();
5104 }
5105 });
5106
5107 if (!Op)
5108 return;
5109
5110 // All recursive calls that increase depth must come after this.
5111 if (Depth == MaxAnalysisRecursionDepth)
5112 return;
5113
5114 const unsigned Opc = Op->getOpcode();
5115 switch (Opc) {
5116 case Instruction::FNeg: {
5117 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5118 Known, Q, Depth: Depth + 1);
5119 Known.fneg();
5120 break;
5121 }
5122 case Instruction::Select: {
5123 auto ComputeForArm = [&](Value *Arm, bool Invert) {
5124 KnownFPClass Res;
5125 computeKnownFPClass(V: Arm, DemandedElts, InterestedClasses, Known&: Res, Q,
5126 Depth: Depth + 1);
5127 adjustKnownFPClassForSelectArm(Known&: Res, Cond: Op->getOperand(i: 0), Arm, Invert, SQ: Q,
5128 Depth);
5129 return Res;
5130 };
5131 // Only known if known in both the LHS and RHS.
5132 Known =
5133 ComputeForArm(Op->getOperand(i: 1), /*Invert=*/false)
5134 .intersectWith(RHS: ComputeForArm(Op->getOperand(i: 2), /*Invert=*/true));
5135 break;
5136 }
5137 case Instruction::Load: {
5138 const MDNode *NoFPClass =
5139 cast<LoadInst>(Val: Op)->getMetadata(KindID: LLVMContext::MD_nofpclass);
5140 if (!NoFPClass)
5141 break;
5142
5143 ConstantInt *MaskVal =
5144 mdconst::extract<ConstantInt>(MD: NoFPClass->getOperand(I: 0));
5145 Known.knownNot(RuleOut: static_cast<FPClassTest>(MaskVal->getZExtValue()));
5146 break;
5147 }
5148 case Instruction::Call: {
5149 const CallInst *II = cast<CallInst>(Val: Op);
5150 const Intrinsic::ID IID = II->getIntrinsicID();
5151 switch (IID) {
5152 case Intrinsic::fabs: {
5153 if ((InterestedClasses & (fcNan | fcPositive)) != fcNone) {
5154 // If we only care about the sign bit we don't need to inspect the
5155 // operand.
5156 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5157 InterestedClasses, Known, Q, Depth: Depth + 1);
5158 }
5159
5160 Known.fabs();
5161 break;
5162 }
5163 case Intrinsic::copysign: {
5164 KnownFPClass KnownSign;
5165
5166 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5167 Known, Q, Depth: Depth + 1);
5168 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5169 Known&: KnownSign, Q, Depth: Depth + 1);
5170 Known.copysign(Sign: KnownSign);
5171 break;
5172 }
5173 case Intrinsic::fma:
5174 case Intrinsic::fmuladd: {
5175 if ((InterestedClasses & fcNegative) == fcNone)
5176 break;
5177
5178 // FIXME: This should check isGuaranteedNotToBeUndef
5179 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) {
5180 KnownFPClass KnownSrc, KnownAddend;
5181 computeKnownFPClass(V: II->getArgOperand(i: 2), DemandedElts,
5182 InterestedClasses, Known&: KnownAddend, Q, Depth: Depth + 1);
5183 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5184 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5185
5186 const Function *F = II->getFunction();
5187 const fltSemantics &FltSem =
5188 II->getType()->getScalarType()->getFltSemantics();
5189 DenormalMode Mode =
5190 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5191
5192 if (KnownNotFromFlags & fcNan) {
5193 KnownSrc.knownNot(RuleOut: fcNan);
5194 KnownAddend.knownNot(RuleOut: fcNan);
5195 }
5196
5197 if (KnownNotFromFlags & fcInf) {
5198 KnownSrc.knownNot(RuleOut: fcInf);
5199 KnownAddend.knownNot(RuleOut: fcInf);
5200 }
5201
5202 Known = KnownFPClass::fma_square(Squared: KnownSrc, Addend: KnownAddend, Mode);
5203 break;
5204 }
5205
5206 KnownFPClass KnownSrc[3];
5207 for (int I = 0; I != 3; ++I) {
5208 computeKnownFPClass(V: II->getArgOperand(i: I), DemandedElts,
5209 InterestedClasses, Known&: KnownSrc[I], Q, Depth: Depth + 1);
5210 if (KnownSrc[I].isUnknown())
5211 return;
5212
5213 if (KnownNotFromFlags & fcNan)
5214 KnownSrc[I].knownNot(RuleOut: fcNan);
5215 if (KnownNotFromFlags & fcInf)
5216 KnownSrc[I].knownNot(RuleOut: fcInf);
5217 }
5218
5219 const Function *F = II->getFunction();
5220 const fltSemantics &FltSem =
5221 II->getType()->getScalarType()->getFltSemantics();
5222 DenormalMode Mode =
5223 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5224 Known = KnownFPClass::fma(LHS: KnownSrc[0], RHS: KnownSrc[1], Addend: KnownSrc[2], Mode);
5225 break;
5226 }
5227 case Intrinsic::sqrt:
5228 case Intrinsic::experimental_constrained_sqrt: {
5229 KnownFPClass KnownSrc;
5230 FPClassTest InterestedSrcs = InterestedClasses;
5231 if (InterestedClasses & fcNan)
5232 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5233
5234 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5235 Known&: KnownSrc, Q, Depth: Depth + 1);
5236
5237 DenormalMode Mode = DenormalMode::getDynamic();
5238
5239 bool HasNSZ = Q.IIQ.hasNoSignedZeros(Op: II);
5240 if (!HasNSZ) {
5241 const Function *F = II->getFunction();
5242 const fltSemantics &FltSem =
5243 II->getType()->getScalarType()->getFltSemantics();
5244 Mode = F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5245 }
5246
5247 Known = KnownFPClass::sqrt(Src: KnownSrc, Mode);
5248 if (HasNSZ)
5249 Known.knownNot(RuleOut: fcNegZero);
5250
5251 break;
5252 }
5253 case Intrinsic::sin:
5254 case Intrinsic::cos: {
5255 // Return NaN on infinite inputs.
5256 KnownFPClass KnownSrc;
5257 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5258 Known&: KnownSrc, Q, Depth: Depth + 1);
5259 Known = IID == Intrinsic::sin ? KnownFPClass::sin(Src: KnownSrc)
5260 : KnownFPClass::cos(Src: KnownSrc);
5261 break;
5262 }
5263 case Intrinsic::maxnum:
5264 case Intrinsic::minnum:
5265 case Intrinsic::minimum:
5266 case Intrinsic::maximum:
5267 case Intrinsic::minimumnum:
5268 case Intrinsic::maximumnum: {
5269 KnownFPClass KnownLHS, KnownRHS;
5270 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5271 Known&: KnownLHS, Q, Depth: Depth + 1);
5272 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5273 Known&: KnownRHS, Q, Depth: Depth + 1);
5274
5275 const Function *F = II->getFunction();
5276
5277 DenormalMode Mode =
5278 F ? F->getDenormalMode(
5279 FPType: II->getType()->getScalarType()->getFltSemantics())
5280 : DenormalMode::getDynamic();
5281
5282 Known = KnownFPClass::minMaxLike(LHS: KnownLHS, RHS: KnownRHS, Kind: getMinMaxKind(IID),
5283 DenormMode: Mode);
5284 break;
5285 }
5286 case Intrinsic::canonicalize: {
5287 KnownFPClass KnownSrc;
5288 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5289 Known&: KnownSrc, Q, Depth: Depth + 1);
5290
5291 const Function *F = II->getFunction();
5292 DenormalMode DenormMode =
5293 F ? F->getDenormalMode(
5294 FPType: II->getType()->getScalarType()->getFltSemantics())
5295 : DenormalMode::getDynamic();
5296 Known = KnownFPClass::canonicalize(Src: KnownSrc, DenormMode);
5297 break;
5298 }
5299 case Intrinsic::vector_reduce_fmax:
5300 case Intrinsic::vector_reduce_fmin:
5301 case Intrinsic::vector_reduce_fmaximum:
5302 case Intrinsic::vector_reduce_fminimum: {
5303 // reduce min/max will choose an element from one of the vector elements,
5304 // so we can infer and class information that is common to all elements.
5305 Known = computeKnownFPClass(V: II->getArgOperand(i: 0), FMF: II->getFastMathFlags(),
5306 InterestedClasses, SQ: Q, Depth: Depth + 1);
5307 // Can only propagate sign if output is never NaN.
5308 if (!Known.isKnownNeverNaN())
5309 Known.SignBit.reset();
5310 break;
5311 }
5312 // reverse preserves all characteristics of the input vec's element.
5313 case Intrinsic::vector_reverse:
5314 Known = computeKnownFPClass(
5315 V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
5316 FMF: II->getFastMathFlags(), InterestedClasses, SQ: Q, Depth: Depth + 1);
5317 break;
5318 case Intrinsic::trunc:
5319 case Intrinsic::floor:
5320 case Intrinsic::ceil:
5321 case Intrinsic::rint:
5322 case Intrinsic::nearbyint:
5323 case Intrinsic::round:
5324 case Intrinsic::roundeven: {
5325 KnownFPClass KnownSrc;
5326 FPClassTest InterestedSrcs = InterestedClasses;
5327 if (InterestedSrcs & fcPosFinite)
5328 InterestedSrcs |= fcPosFinite;
5329 if (InterestedSrcs & fcNegFinite)
5330 InterestedSrcs |= fcNegFinite;
5331 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5332 Known&: KnownSrc, Q, Depth: Depth + 1);
5333
5334 Known = KnownFPClass::roundToIntegral(
5335 Src: KnownSrc, IsTrunc: IID == Intrinsic::trunc,
5336 IsMultiUnitFPType: V->getType()->getScalarType()->isMultiUnitFPType());
5337 break;
5338 }
5339 case Intrinsic::exp:
5340 case Intrinsic::exp2:
5341 case Intrinsic::exp10:
5342 case Intrinsic::amdgcn_exp2: {
5343 KnownFPClass KnownSrc;
5344 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5345 Known&: KnownSrc, Q, Depth: Depth + 1);
5346
5347 Known = KnownFPClass::exp(Src: KnownSrc);
5348
5349 Type *EltTy = II->getType()->getScalarType();
5350 if (IID == Intrinsic::amdgcn_exp2 && EltTy->isFloatTy())
5351 Known.knownNot(RuleOut: fcSubnormal);
5352
5353 break;
5354 }
5355 case Intrinsic::fptrunc_round: {
5356 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
5357 Q, Depth);
5358 break;
5359 }
5360 case Intrinsic::log:
5361 case Intrinsic::log10:
5362 case Intrinsic::log2:
5363 case Intrinsic::experimental_constrained_log:
5364 case Intrinsic::experimental_constrained_log10:
5365 case Intrinsic::experimental_constrained_log2:
5366 case Intrinsic::amdgcn_log: {
5367 Type *EltTy = II->getType()->getScalarType();
5368
5369 // log(+inf) -> +inf
5370 // log([+-]0.0) -> -inf
5371 // log(-inf) -> nan
5372 // log(-x) -> nan
5373 if ((InterestedClasses & (fcNan | fcInf)) != fcNone) {
5374 FPClassTest InterestedSrcs = InterestedClasses;
5375 if ((InterestedClasses & fcNegInf) != fcNone)
5376 InterestedSrcs |= fcZero | fcSubnormal;
5377 if ((InterestedClasses & fcNan) != fcNone)
5378 InterestedSrcs |= fcNan | fcNegative;
5379
5380 KnownFPClass KnownSrc;
5381 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5382 Known&: KnownSrc, Q, Depth: Depth + 1);
5383
5384 const Function *F = II->getFunction();
5385 DenormalMode Mode = F ? F->getDenormalMode(FPType: EltTy->getFltSemantics())
5386 : DenormalMode::getDynamic();
5387 Known = KnownFPClass::log(Src: KnownSrc, Mode);
5388 }
5389
5390 break;
5391 }
5392 case Intrinsic::powi: {
5393 if ((InterestedClasses & fcNegative) == fcNone)
5394 break;
5395
5396 const Value *Exp = II->getArgOperand(i: 1);
5397 Type *ExpTy = Exp->getType();
5398 unsigned BitWidth = ExpTy->getScalarType()->getIntegerBitWidth();
5399 KnownBits ExponentKnownBits(BitWidth);
5400 computeKnownBits(V: Exp, DemandedElts: isa<VectorType>(Val: ExpTy) ? DemandedElts : APInt(1, 1),
5401 Known&: ExponentKnownBits, Q, Depth: Depth + 1);
5402
5403 KnownFPClass KnownSrc;
5404 if (ExponentKnownBits.isZero() || !ExponentKnownBits.isEven()) {
5405 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: fcNegative,
5406 Known&: KnownSrc, Q, Depth: Depth + 1);
5407 }
5408
5409 Known = KnownFPClass::powi(Src: KnownSrc, N: ExponentKnownBits);
5410 break;
5411 }
5412 case Intrinsic::ldexp: {
5413 KnownFPClass KnownSrc;
5414 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5415 Known&: KnownSrc, Q, Depth: Depth + 1);
5416 // Can refine inf/zero handling based on the exponent operand.
5417 const FPClassTest ExpInfoMask = fcZero | fcSubnormal | fcInf;
5418
5419 KnownBits ExpBits;
5420 if ((KnownSrc.KnownFPClasses & ExpInfoMask) != fcNone) {
5421 const Value *ExpArg = II->getArgOperand(i: 1);
5422 ExpBits = computeKnownBits(V: ExpArg, DemandedElts, Q, Depth: Depth + 1);
5423 }
5424
5425 const fltSemantics &Flt =
5426 II->getType()->getScalarType()->getFltSemantics();
5427
5428 const Function *F = II->getFunction();
5429 DenormalMode Mode =
5430 F ? F->getDenormalMode(FPType: Flt) : DenormalMode::getDynamic();
5431
5432 Known = KnownFPClass::ldexp(Src: KnownSrc, N: ExpBits, Flt, Mode);
5433 break;
5434 }
5435 case Intrinsic::arithmetic_fence: {
5436 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5437 Known, Q, Depth: Depth + 1);
5438 break;
5439 }
5440 case Intrinsic::experimental_constrained_sitofp:
5441 case Intrinsic::experimental_constrained_uitofp:
5442 // Cannot produce nan
5443 Known.knownNot(RuleOut: fcNan);
5444
5445 // sitofp and uitofp turn into +0.0 for zero.
5446 Known.knownNot(RuleOut: fcNegZero);
5447
5448 // Integers cannot be subnormal
5449 Known.knownNot(RuleOut: fcSubnormal);
5450
5451 if (IID == Intrinsic::experimental_constrained_uitofp)
5452 Known.signBitMustBeZero();
5453
5454 // TODO: Copy inf handling from instructions
5455 break;
5456
5457 case Intrinsic::amdgcn_fract: {
5458 Known.knownNot(RuleOut: fcInf);
5459
5460 if (InterestedClasses & fcNan) {
5461 KnownFPClass KnownSrc;
5462 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5463 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5464
5465 if (KnownSrc.isKnownNeverInfOrNaN())
5466 Known.knownNot(RuleOut: fcNan);
5467 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5468 Known.knownNot(RuleOut: fcSNan);
5469 }
5470
5471 break;
5472 }
5473 case Intrinsic::amdgcn_rcp: {
5474 KnownFPClass KnownSrc;
5475 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5476 Known&: KnownSrc, Q, Depth: Depth + 1);
5477
5478 Known.propagateNaN(Src: KnownSrc);
5479
5480 Type *EltTy = II->getType()->getScalarType();
5481
5482 // f32 denormal always flushed.
5483 if (EltTy->isFloatTy()) {
5484 Known.knownNot(RuleOut: fcSubnormal);
5485 KnownSrc.knownNot(RuleOut: fcSubnormal);
5486 }
5487
5488 if (KnownSrc.isKnownNever(Mask: fcNegative))
5489 Known.knownNot(RuleOut: fcNegative);
5490 if (KnownSrc.isKnownNever(Mask: fcPositive))
5491 Known.knownNot(RuleOut: fcPositive);
5492
5493 if (const Function *F = II->getFunction()) {
5494 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5495 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5496 Known.knownNot(RuleOut: fcPosInf);
5497 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5498 Known.knownNot(RuleOut: fcNegInf);
5499 }
5500
5501 break;
5502 }
5503 case Intrinsic::amdgcn_rsq: {
5504 KnownFPClass KnownSrc;
5505 // The only negative value that can be returned is -inf for -0 inputs.
5506 Known.knownNot(RuleOut: fcNegZero | fcNegSubnormal | fcNegNormal);
5507
5508 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5509 Known&: KnownSrc, Q, Depth: Depth + 1);
5510
5511 // Negative -> nan
5512 if (KnownSrc.isKnownNeverNaN() && KnownSrc.cannotBeOrderedLessThanZero())
5513 Known.knownNot(RuleOut: fcNan);
5514 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5515 Known.knownNot(RuleOut: fcSNan);
5516
5517 // +inf -> +0
5518 if (KnownSrc.isKnownNeverPosInfinity())
5519 Known.knownNot(RuleOut: fcPosZero);
5520
5521 Type *EltTy = II->getType()->getScalarType();
5522
5523 // f32 denormal always flushed.
5524 if (EltTy->isFloatTy())
5525 Known.knownNot(RuleOut: fcPosSubnormal);
5526
5527 if (const Function *F = II->getFunction()) {
5528 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5529
5530 // -0 -> -inf
5531 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5532 Known.knownNot(RuleOut: fcNegInf);
5533
5534 // +0 -> +inf
5535 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5536 Known.knownNot(RuleOut: fcPosInf);
5537 }
5538
5539 break;
5540 }
5541 case Intrinsic::amdgcn_trig_preop: {
5542 // Always returns a value [0, 1)
5543 Known.knownNot(RuleOut: fcNan | fcInf | fcNegative);
5544 break;
5545 }
5546 default:
5547 break;
5548 }
5549
5550 break;
5551 }
5552 case Instruction::FAdd:
5553 case Instruction::FSub: {
5554 KnownFPClass KnownLHS, KnownRHS;
5555 bool WantNegative =
5556 Op->getOpcode() == Instruction::FAdd &&
5557 (InterestedClasses & KnownFPClass::OrderedLessThanZeroMask) != fcNone;
5558 bool WantNaN = (InterestedClasses & fcNan) != fcNone;
5559 bool WantNegZero = (InterestedClasses & fcNegZero) != fcNone;
5560
5561 if (!WantNaN && !WantNegative && !WantNegZero)
5562 break;
5563
5564 FPClassTest InterestedSrcs = InterestedClasses;
5565 if (WantNegative)
5566 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5567 if (InterestedClasses & fcNan)
5568 InterestedSrcs |= fcInf;
5569 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: InterestedSrcs,
5570 Known&: KnownRHS, Q, Depth: Depth + 1);
5571
5572 // Special case fadd x, x, which is the canonical form of fmul x, 2.
5573 bool Self = Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
5574 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
5575 Depth: Depth + 1);
5576 if (Self)
5577 KnownLHS = KnownRHS;
5578
5579 if ((WantNaN && KnownRHS.isKnownNeverNaN()) ||
5580 (WantNegative && KnownRHS.cannotBeOrderedLessThanZero()) ||
5581 WantNegZero || Opc == Instruction::FSub) {
5582
5583 // FIXME: Context function should always be passed in separately
5584 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5585 const fltSemantics &FltSem =
5586 Op->getType()->getScalarType()->getFltSemantics();
5587 DenormalMode Mode =
5588 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5589
5590 if (Self && Opc == Instruction::FAdd) {
5591 Known = KnownFPClass::fadd_self(Src: KnownLHS, Mode);
5592 } else {
5593 // RHS is canonically cheaper to compute. Skip inspecting the LHS if
5594 // there's no point.
5595
5596 if (!Self) {
5597 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5598 Known&: KnownLHS, Q, Depth: Depth + 1);
5599 }
5600
5601 Known = Opc == Instruction::FAdd
5602 ? KnownFPClass::fadd(LHS: KnownLHS, RHS: KnownRHS, Mode)
5603 : KnownFPClass::fsub(LHS: KnownLHS, RHS: KnownRHS, Mode);
5604 }
5605 }
5606
5607 break;
5608 }
5609 case Instruction::FMul: {
5610 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5611 DenormalMode Mode =
5612 F ? F->getDenormalMode(
5613 FPType: Op->getType()->getScalarType()->getFltSemantics())
5614 : DenormalMode::getDynamic();
5615
5616 Value *LHS = Op->getOperand(i: 0);
5617 Value *RHS = Op->getOperand(i: 1);
5618 // X * X is always non-negative or a NaN.
5619 // FIXME: Should check isGuaranteedNotToBeUndef
5620 if (LHS == RHS) {
5621 KnownFPClass KnownSrc;
5622 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownSrc, Q,
5623 Depth: Depth + 1);
5624 Known = KnownFPClass::square(Src: KnownSrc, Mode);
5625 break;
5626 }
5627
5628 KnownFPClass KnownLHS, KnownRHS;
5629
5630 const APFloat *CRHS;
5631 if (match(V: RHS, P: m_APFloat(Res&: CRHS))) {
5632 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
5633 Depth: Depth + 1);
5634 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: *CRHS, Mode);
5635 } else {
5636 computeKnownFPClass(V: RHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS, Q,
5637 Depth: Depth + 1);
5638 // TODO: Improve accuracy in unfused FMA pattern. We can prove an
5639 // additional not-nan if the addend is known-not negative infinity if the
5640 // multiply is known-not infinity.
5641
5642 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
5643 Depth: Depth + 1);
5644 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: KnownRHS, Mode);
5645 }
5646
5647 /// Propgate no-infs if the other source is known smaller than one, such
5648 /// that this cannot introduce overflow.
5649 if (KnownLHS.isKnownNever(Mask: fcInf) && isAbsoluteValueLessEqualOne(V: RHS))
5650 Known.knownNot(RuleOut: fcInf);
5651 else if (KnownRHS.isKnownNever(Mask: fcInf) && isAbsoluteValueLessEqualOne(V: LHS))
5652 Known.knownNot(RuleOut: fcInf);
5653
5654 break;
5655 }
5656 case Instruction::FDiv:
5657 case Instruction::FRem: {
5658 const bool WantNan = (InterestedClasses & fcNan) != fcNone;
5659
5660 if (Op->getOpcode() == Instruction::FRem)
5661 Known.knownNot(RuleOut: fcInf);
5662
5663 if (Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
5664 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT)) {
5665 if (Op->getOpcode() == Instruction::FDiv) {
5666 // X / X is always exactly 1.0 or a NaN.
5667 Known.KnownFPClasses = fcNan | fcPosNormal;
5668 } else {
5669 // X % X is always exactly [+-]0.0 or a NaN.
5670 Known.KnownFPClasses = fcNan | fcZero;
5671 }
5672
5673 if (!WantNan)
5674 break;
5675
5676 KnownFPClass KnownSrc;
5677 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts,
5678 InterestedClasses: fcNan | fcInf | fcZero | fcSubnormal, Known&: KnownSrc, Q,
5679 Depth: Depth + 1);
5680 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5681 const fltSemantics &FltSem =
5682 Op->getType()->getScalarType()->getFltSemantics();
5683
5684 DenormalMode Mode =
5685 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5686
5687 Known = Op->getOpcode() == Instruction::FDiv
5688 ? KnownFPClass::fdiv_self(Src: KnownSrc, Mode)
5689 : KnownFPClass::frem_self(Src: KnownSrc, Mode);
5690 break;
5691 }
5692
5693 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
5694 const bool WantPositive =
5695 Opc == Instruction::FRem && (InterestedClasses & fcPositive) != fcNone;
5696 if (!WantNan && !WantNegative && !WantPositive)
5697 break;
5698
5699 KnownFPClass KnownLHS, KnownRHS;
5700
5701 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts,
5702 InterestedClasses: fcNan | fcInf | fcZero | fcNegative, Known&: KnownRHS, Q,
5703 Depth: Depth + 1);
5704
5705 bool KnowSomethingUseful = KnownRHS.isKnownNeverNaN() ||
5706 KnownRHS.isKnownNever(Mask: fcNegative) ||
5707 KnownRHS.isKnownNever(Mask: fcPositive);
5708
5709 if (KnowSomethingUseful || WantPositive) {
5710 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS,
5711 Q, Depth: Depth + 1);
5712 }
5713
5714 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5715 const fltSemantics &FltSem =
5716 Op->getType()->getScalarType()->getFltSemantics();
5717
5718 if (Op->getOpcode() == Instruction::FDiv) {
5719 DenormalMode Mode =
5720 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5721 Known = KnownFPClass::fdiv(LHS: KnownLHS, RHS: KnownRHS, Mode);
5722 } else {
5723 // Inf REM x and x REM 0 produce NaN.
5724 if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
5725 KnownLHS.isKnownNeverInfinity() && F &&
5726 KnownRHS.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem))) {
5727 Known.knownNot(RuleOut: fcNan);
5728 }
5729
5730 // The sign for frem is the same as the first operand.
5731 if (KnownLHS.cannotBeOrderedLessThanZero())
5732 Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
5733 if (KnownLHS.cannotBeOrderedGreaterThanZero())
5734 Known.knownNot(RuleOut: KnownFPClass::OrderedGreaterThanZeroMask);
5735
5736 // See if we can be more aggressive about the sign of 0.
5737 if (KnownLHS.isKnownNever(Mask: fcNegative))
5738 Known.knownNot(RuleOut: fcNegative);
5739 if (KnownLHS.isKnownNever(Mask: fcPositive))
5740 Known.knownNot(RuleOut: fcPositive);
5741 }
5742
5743 break;
5744 }
5745 case Instruction::FPExt: {
5746 KnownFPClass KnownSrc;
5747 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5748 Known&: KnownSrc, Q, Depth: Depth + 1);
5749
5750 const fltSemantics &DstTy =
5751 Op->getType()->getScalarType()->getFltSemantics();
5752 const fltSemantics &SrcTy =
5753 Op->getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
5754
5755 Known = KnownFPClass::fpext(KnownSrc, DstTy, SrcTy);
5756 break;
5757 }
5758 case Instruction::FPTrunc: {
5759 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known, Q,
5760 Depth);
5761 break;
5762 }
5763 case Instruction::SIToFP:
5764 case Instruction::UIToFP: {
5765 // Cannot produce nan
5766 Known.knownNot(RuleOut: fcNan);
5767
5768 // Integers cannot be subnormal
5769 Known.knownNot(RuleOut: fcSubnormal);
5770
5771 // sitofp and uitofp turn into +0.0 for zero.
5772 Known.knownNot(RuleOut: fcNegZero);
5773 if (Op->getOpcode() == Instruction::UIToFP)
5774 Known.signBitMustBeZero();
5775
5776 if (InterestedClasses & fcInf) {
5777 // Get width of largest magnitude integer (remove a bit if signed).
5778 // This still works for a signed minimum value because the largest FP
5779 // value is scaled by some fraction close to 2.0 (1.0 + 0.xxxx).
5780 int IntSize = Op->getOperand(i: 0)->getType()->getScalarSizeInBits();
5781 if (Op->getOpcode() == Instruction::SIToFP)
5782 --IntSize;
5783
5784 // If the exponent of the largest finite FP value can hold the largest
5785 // integer, the result of the cast must be finite.
5786 Type *FPTy = Op->getType()->getScalarType();
5787 if (ilogb(Arg: APFloat::getLargest(Sem: FPTy->getFltSemantics())) >= IntSize)
5788 Known.knownNot(RuleOut: fcInf);
5789 }
5790
5791 break;
5792 }
5793 case Instruction::ExtractElement: {
5794 // Look through extract element. If the index is non-constant or
5795 // out-of-range demand all elements, otherwise just the extracted element.
5796 const Value *Vec = Op->getOperand(i: 0);
5797
5798 APInt DemandedVecElts;
5799 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
5800 unsigned NumElts = VecTy->getNumElements();
5801 DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
5802 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1));
5803 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
5804 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
5805 } else {
5806 DemandedVecElts = APInt(1, 1);
5807 }
5808
5809 return computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known,
5810 Q, Depth: Depth + 1);
5811 }
5812 case Instruction::InsertElement: {
5813 if (isa<ScalableVectorType>(Val: Op->getType()))
5814 return;
5815
5816 const Value *Vec = Op->getOperand(i: 0);
5817 const Value *Elt = Op->getOperand(i: 1);
5818 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 2));
5819 unsigned NumElts = DemandedElts.getBitWidth();
5820 APInt DemandedVecElts = DemandedElts;
5821 bool NeedsElt = true;
5822 // If we know the index we are inserting to, clear it from Vec check.
5823 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
5824 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
5825 NeedsElt = DemandedElts[CIdx->getZExtValue()];
5826 }
5827
5828 // Do we demand the inserted element?
5829 if (NeedsElt) {
5830 computeKnownFPClass(V: Elt, Known, InterestedClasses, Q, Depth: Depth + 1);
5831 // If we don't know any bits, early out.
5832 if (Known.isUnknown())
5833 break;
5834 } else {
5835 Known.KnownFPClasses = fcNone;
5836 }
5837
5838 // Do we need anymore elements from Vec?
5839 if (!DemandedVecElts.isZero()) {
5840 KnownFPClass Known2;
5841 computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known&: Known2, Q,
5842 Depth: Depth + 1);
5843 Known |= Known2;
5844 }
5845
5846 break;
5847 }
5848 case Instruction::ShuffleVector: {
5849 // Handle vector splat idiom
5850 if (Value *Splat = getSplatValue(V)) {
5851 computeKnownFPClass(V: Splat, Known, InterestedClasses, Q, Depth: Depth + 1);
5852 break;
5853 }
5854
5855 // For undef elements, we don't know anything about the common state of
5856 // the shuffle result.
5857 APInt DemandedLHS, DemandedRHS;
5858 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: Op);
5859 if (!Shuf || !getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
5860 return;
5861
5862 if (!!DemandedLHS) {
5863 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
5864 computeKnownFPClass(V: LHS, DemandedElts: DemandedLHS, InterestedClasses, Known, Q,
5865 Depth: Depth + 1);
5866
5867 // If we don't know any bits, early out.
5868 if (Known.isUnknown())
5869 break;
5870 } else {
5871 Known.KnownFPClasses = fcNone;
5872 }
5873
5874 if (!!DemandedRHS) {
5875 KnownFPClass Known2;
5876 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
5877 computeKnownFPClass(V: RHS, DemandedElts: DemandedRHS, InterestedClasses, Known&: Known2, Q,
5878 Depth: Depth + 1);
5879 Known |= Known2;
5880 }
5881
5882 break;
5883 }
5884 case Instruction::ExtractValue: {
5885 const ExtractValueInst *Extract = cast<ExtractValueInst>(Val: Op);
5886 ArrayRef<unsigned> Indices = Extract->getIndices();
5887 const Value *Src = Extract->getAggregateOperand();
5888 if (isa<StructType>(Val: Src->getType()) && Indices.size() == 1 &&
5889 Indices[0] == 0) {
5890 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Src)) {
5891 switch (II->getIntrinsicID()) {
5892 case Intrinsic::frexp: {
5893 Known.knownNot(RuleOut: fcSubnormal);
5894
5895 KnownFPClass KnownSrc;
5896 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5897 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5898
5899 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5900 const fltSemantics &FltSem =
5901 Op->getType()->getScalarType()->getFltSemantics();
5902
5903 DenormalMode Mode =
5904 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5905 Known = KnownFPClass::frexp_mant(Src: KnownSrc, Mode);
5906 return;
5907 }
5908 default:
5909 break;
5910 }
5911 }
5912 }
5913
5914 computeKnownFPClass(V: Src, DemandedElts, InterestedClasses, Known, Q,
5915 Depth: Depth + 1);
5916 break;
5917 }
5918 case Instruction::PHI: {
5919 const PHINode *P = cast<PHINode>(Val: Op);
5920 // Unreachable blocks may have zero-operand PHI nodes.
5921 if (P->getNumIncomingValues() == 0)
5922 break;
5923
5924 // Otherwise take the unions of the known bit sets of the operands,
5925 // taking conservative care to avoid excessive recursion.
5926 const unsigned PhiRecursionLimit = MaxAnalysisRecursionDepth - 2;
5927
5928 if (Depth < PhiRecursionLimit) {
5929 // Skip if every incoming value references to ourself.
5930 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
5931 break;
5932
5933 bool First = true;
5934
5935 for (const Use &U : P->operands()) {
5936 Value *IncValue;
5937 Instruction *CxtI;
5938 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CxtI);
5939 // Skip direct self references.
5940 if (IncValue == P)
5941 continue;
5942
5943 KnownFPClass KnownSrc;
5944 // Recurse, but cap the recursion to two levels, because we don't want
5945 // to waste time spinning around in loops. We need at least depth 2 to
5946 // detect known sign bits.
5947 computeKnownFPClass(V: IncValue, DemandedElts, InterestedClasses, Known&: KnownSrc,
5948 Q: Q.getWithoutCondContext().getWithInstruction(I: CxtI),
5949 Depth: PhiRecursionLimit);
5950
5951 if (First) {
5952 Known = KnownSrc;
5953 First = false;
5954 } else {
5955 Known |= KnownSrc;
5956 }
5957
5958 if (Known.KnownFPClasses == fcAllFlags)
5959 break;
5960 }
5961 }
5962
5963 // Look for the case of a for loop which has a positive
5964 // initial value and is incremented by a squared value.
5965 // This will propagate sign information out of such loops.
5966 if (P->getNumIncomingValues() != 2 || Known.cannotBeOrderedLessThanZero())
5967 break;
5968 for (unsigned I = 0; I < 2; I++) {
5969 Value *RecurValue = P->getIncomingValue(i: 1 - I);
5970 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: RecurValue);
5971 if (!II)
5972 continue;
5973 Value *R, *L, *Init;
5974 PHINode *PN;
5975 if (matchSimpleTernaryIntrinsicRecurrence(I: II, P&: PN, Init, OtherOp0&: L, OtherOp1&: R) &&
5976 PN == P) {
5977 switch (II->getIntrinsicID()) {
5978 case Intrinsic::fma:
5979 case Intrinsic::fmuladd: {
5980 KnownFPClass KnownStart;
5981 computeKnownFPClass(V: Init, DemandedElts, InterestedClasses, Known&: KnownStart,
5982 Q, Depth: Depth + 1);
5983 if (KnownStart.cannotBeOrderedLessThanZero() && L == R &&
5984 isGuaranteedNotToBeUndef(V: L, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1))
5985 Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
5986 break;
5987 }
5988 }
5989 }
5990 }
5991 break;
5992 }
5993 case Instruction::BitCast: {
5994 const Value *Src;
5995 if (!match(V: Op, P: m_ElementWiseBitCast(Op: m_Value(V&: Src))) ||
5996 !Src->getType()->isIntOrIntVectorTy())
5997 break;
5998
5999 const Type *Ty = Op->getType();
6000
6001 Value *CastLHS, *CastRHS;
6002
6003 // Match bitcast(umax(bitcast(a), bitcast(b)))
6004 if (match(V: Src, P: m_c_MaxOrMin(L: m_BitCast(Op: m_Value(V&: CastLHS)),
6005 R: m_BitCast(Op: m_Value(V&: CastRHS)))) &&
6006 CastLHS->getType() == Ty && CastRHS->getType() == Ty) {
6007 KnownFPClass KnownLHS, KnownRHS;
6008 computeKnownFPClass(V: CastRHS, DemandedElts, InterestedClasses, Known&: KnownRHS, Q,
6009 Depth: Depth + 1);
6010 if (!KnownRHS.isUnknown()) {
6011 computeKnownFPClass(V: CastLHS, DemandedElts, InterestedClasses, Known&: KnownLHS,
6012 Q, Depth: Depth + 1);
6013 Known = KnownLHS | KnownRHS;
6014 }
6015
6016 return;
6017 }
6018
6019 const Type *EltTy = Ty->getScalarType();
6020 KnownBits Bits(EltTy->getPrimitiveSizeInBits());
6021 computeKnownBits(V: Src, DemandedElts, Known&: Bits, Q, Depth: Depth + 1);
6022
6023 // Transfer information from the sign bit.
6024 if (Bits.isNonNegative())
6025 Known.signBitMustBeZero();
6026 else if (Bits.isNegative())
6027 Known.signBitMustBeOne();
6028
6029 if (EltTy->isIEEELikeFPTy()) {
6030 // IEEE floats are NaN when all bits of the exponent plus at least one of
6031 // the fraction bits are 1. This means:
6032 // - If we assume unknown bits are 0 and the value is NaN, it will
6033 // always be NaN
6034 // - If we assume unknown bits are 1 and the value is not NaN, it can
6035 // never be NaN
6036 // Note: They do not hold for x86_fp80 format.
6037 if (APFloat(EltTy->getFltSemantics(), Bits.One).isNaN())
6038 Known.KnownFPClasses = fcNan;
6039 else if (!APFloat(EltTy->getFltSemantics(), ~Bits.Zero).isNaN())
6040 Known.knownNot(RuleOut: fcNan);
6041
6042 // Build KnownBits representing Inf and check if it must be equal or
6043 // unequal to this value.
6044 auto InfKB = KnownBits::makeConstant(
6045 C: APFloat::getInf(Sem: EltTy->getFltSemantics()).bitcastToAPInt());
6046 InfKB.Zero.clearSignBit();
6047 if (const auto InfResult = KnownBits::eq(LHS: Bits, RHS: InfKB)) {
6048 assert(!InfResult.value());
6049 Known.knownNot(RuleOut: fcInf);
6050 } else if (Bits == InfKB) {
6051 Known.KnownFPClasses = fcInf;
6052 }
6053
6054 // Build KnownBits representing Zero and check if it must be equal or
6055 // unequal to this value.
6056 auto ZeroKB = KnownBits::makeConstant(
6057 C: APFloat::getZero(Sem: EltTy->getFltSemantics()).bitcastToAPInt());
6058 ZeroKB.Zero.clearSignBit();
6059 if (const auto ZeroResult = KnownBits::eq(LHS: Bits, RHS: ZeroKB)) {
6060 assert(!ZeroResult.value());
6061 Known.knownNot(RuleOut: fcZero);
6062 } else if (Bits == ZeroKB) {
6063 Known.KnownFPClasses = fcZero;
6064 }
6065 }
6066
6067 break;
6068 }
6069 default:
6070 break;
6071 }
6072}
6073
6074KnownFPClass llvm::computeKnownFPClass(const Value *V,
6075 const APInt &DemandedElts,
6076 FPClassTest InterestedClasses,
6077 const SimplifyQuery &SQ,
6078 unsigned Depth) {
6079 KnownFPClass KnownClasses;
6080 ::computeKnownFPClass(V, DemandedElts, InterestedClasses, Known&: KnownClasses, Q: SQ,
6081 Depth);
6082 return KnownClasses;
6083}
6084
6085KnownFPClass llvm::computeKnownFPClass(const Value *V,
6086 FPClassTest InterestedClasses,
6087 const SimplifyQuery &SQ,
6088 unsigned Depth) {
6089 KnownFPClass Known;
6090 ::computeKnownFPClass(V, Known, InterestedClasses, Q: SQ, Depth);
6091 return Known;
6092}
6093
6094KnownFPClass llvm::computeKnownFPClass(
6095 const Value *V, const DataLayout &DL, FPClassTest InterestedClasses,
6096 const TargetLibraryInfo *TLI, AssumptionCache *AC, const Instruction *CxtI,
6097 const DominatorTree *DT, bool UseInstrInfo, unsigned Depth) {
6098 return computeKnownFPClass(V, InterestedClasses,
6099 SQ: SimplifyQuery(DL, TLI, DT, AC, CxtI, UseInstrInfo),
6100 Depth);
6101}
6102
6103KnownFPClass
6104llvm::computeKnownFPClass(const Value *V, const APInt &DemandedElts,
6105 FastMathFlags FMF, FPClassTest InterestedClasses,
6106 const SimplifyQuery &SQ, unsigned Depth) {
6107 if (FMF.noNaNs())
6108 InterestedClasses &= ~fcNan;
6109 if (FMF.noInfs())
6110 InterestedClasses &= ~fcInf;
6111
6112 KnownFPClass Result =
6113 computeKnownFPClass(V, DemandedElts, InterestedClasses, SQ, Depth);
6114
6115 if (FMF.noNaNs())
6116 Result.KnownFPClasses &= ~fcNan;
6117 if (FMF.noInfs())
6118 Result.KnownFPClasses &= ~fcInf;
6119 return Result;
6120}
6121
6122KnownFPClass llvm::computeKnownFPClass(const Value *V, FastMathFlags FMF,
6123 FPClassTest InterestedClasses,
6124 const SimplifyQuery &SQ,
6125 unsigned Depth) {
6126 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
6127 APInt DemandedElts =
6128 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
6129 return computeKnownFPClass(V, DemandedElts, FMF, InterestedClasses, SQ,
6130 Depth);
6131}
6132
6133bool llvm::cannotBeNegativeZero(const Value *V, const SimplifyQuery &SQ,
6134 unsigned Depth) {
6135 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNegZero, SQ, Depth);
6136 return Known.isKnownNeverNegZero();
6137}
6138
6139bool llvm::cannotBeOrderedLessThanZero(const Value *V, const SimplifyQuery &SQ,
6140 unsigned Depth) {
6141 KnownFPClass Known =
6142 computeKnownFPClass(V, InterestedClasses: KnownFPClass::OrderedLessThanZeroMask, SQ, Depth);
6143 return Known.cannotBeOrderedLessThanZero();
6144}
6145
6146bool llvm::isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ,
6147 unsigned Depth) {
6148 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf, SQ, Depth);
6149 return Known.isKnownNeverInfinity();
6150}
6151
6152/// Return true if the floating-point value can never contain a NaN or infinity.
6153bool llvm::isKnownNeverInfOrNaN(const Value *V, const SimplifyQuery &SQ,
6154 unsigned Depth) {
6155 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf | fcNan, SQ, Depth);
6156 return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity();
6157}
6158
6159/// Return true if the floating-point scalar value is not a NaN or if the
6160/// floating-point vector value has no NaN elements. Return false if a value
6161/// could ever be NaN.
6162bool llvm::isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ,
6163 unsigned Depth) {
6164 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNan, SQ, Depth);
6165 return Known.isKnownNeverNaN();
6166}
6167
6168/// Return false if we can prove that the specified FP value's sign bit is 0.
6169/// Return true if we can prove that the specified FP value's sign bit is 1.
6170/// Otherwise return std::nullopt.
6171std::optional<bool> llvm::computeKnownFPSignBit(const Value *V,
6172 const SimplifyQuery &SQ,
6173 unsigned Depth) {
6174 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcAllFlags, SQ, Depth);
6175 return Known.SignBit;
6176}
6177
6178bool llvm::canIgnoreSignBitOfZero(const Use &U) {
6179 auto *User = cast<Instruction>(Val: U.getUser());
6180 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6181 if (FPOp->hasNoSignedZeros())
6182 return true;
6183 }
6184
6185 switch (User->getOpcode()) {
6186 case Instruction::FPToSI:
6187 case Instruction::FPToUI:
6188 return true;
6189 case Instruction::FCmp:
6190 // fcmp treats both positive and negative zero as equal.
6191 return true;
6192 case Instruction::Call:
6193 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6194 switch (II->getIntrinsicID()) {
6195 case Intrinsic::fabs:
6196 return true;
6197 case Intrinsic::copysign:
6198 return U.getOperandNo() == 0;
6199 case Intrinsic::is_fpclass:
6200 case Intrinsic::vp_is_fpclass: {
6201 auto Test =
6202 static_cast<FPClassTest>(
6203 cast<ConstantInt>(Val: II->getArgOperand(i: 1))->getZExtValue()) &
6204 FPClassTest::fcZero;
6205 return Test == FPClassTest::fcZero || Test == FPClassTest::fcNone;
6206 }
6207 default:
6208 return false;
6209 }
6210 }
6211 return false;
6212 default:
6213 return false;
6214 }
6215}
6216
6217bool llvm::canIgnoreSignBitOfNaN(const Use &U) {
6218 auto *User = cast<Instruction>(Val: U.getUser());
6219 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6220 if (FPOp->hasNoNaNs())
6221 return true;
6222 }
6223
6224 switch (User->getOpcode()) {
6225 case Instruction::FPToSI:
6226 case Instruction::FPToUI:
6227 return true;
6228 // Proper FP math operations ignore the sign bit of NaN.
6229 case Instruction::FAdd:
6230 case Instruction::FSub:
6231 case Instruction::FMul:
6232 case Instruction::FDiv:
6233 case Instruction::FRem:
6234 case Instruction::FPTrunc:
6235 case Instruction::FPExt:
6236 case Instruction::FCmp:
6237 return true;
6238 // Bitwise FP operations should preserve the sign bit of NaN.
6239 case Instruction::FNeg:
6240 case Instruction::Select:
6241 case Instruction::PHI:
6242 return false;
6243 case Instruction::Ret:
6244 return User->getFunction()->getAttributes().getRetNoFPClass() &
6245 FPClassTest::fcNan;
6246 case Instruction::Call:
6247 case Instruction::Invoke: {
6248 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6249 switch (II->getIntrinsicID()) {
6250 case Intrinsic::fabs:
6251 return true;
6252 case Intrinsic::copysign:
6253 return U.getOperandNo() == 0;
6254 // Other proper FP math intrinsics ignore the sign bit of NaN.
6255 case Intrinsic::maxnum:
6256 case Intrinsic::minnum:
6257 case Intrinsic::maximum:
6258 case Intrinsic::minimum:
6259 case Intrinsic::maximumnum:
6260 case Intrinsic::minimumnum:
6261 case Intrinsic::canonicalize:
6262 case Intrinsic::fma:
6263 case Intrinsic::fmuladd:
6264 case Intrinsic::sqrt:
6265 case Intrinsic::pow:
6266 case Intrinsic::powi:
6267 case Intrinsic::fptoui_sat:
6268 case Intrinsic::fptosi_sat:
6269 case Intrinsic::is_fpclass:
6270 case Intrinsic::vp_is_fpclass:
6271 return true;
6272 default:
6273 return false;
6274 }
6275 }
6276
6277 FPClassTest NoFPClass =
6278 cast<CallBase>(Val: User)->getParamNoFPClass(i: U.getOperandNo());
6279 return NoFPClass & FPClassTest::fcNan;
6280 }
6281 default:
6282 return false;
6283 }
6284}
6285
6286bool llvm::isKnownIntegral(const Value *V, const SimplifyQuery &SQ,
6287 FastMathFlags FMF) {
6288 if (isa<PoisonValue>(Val: V))
6289 return true;
6290 if (isa<UndefValue>(Val: V))
6291 return false;
6292
6293 if (match(V, P: m_CheckedFp(CheckFn: [](const APFloat &Val) { return Val.isInteger(); })))
6294 return true;
6295
6296 const Instruction *I = dyn_cast<Instruction>(Val: V);
6297 if (!I)
6298 return false;
6299
6300 switch (I->getOpcode()) {
6301 case Instruction::SIToFP:
6302 case Instruction::UIToFP:
6303 // TODO: Could check nofpclass(inf) on incoming argument
6304 if (FMF.noInfs())
6305 return true;
6306
6307 // Need to check int size cannot produce infinity, which computeKnownFPClass
6308 // knows how to do already.
6309 return isKnownNeverInfinity(V: I, SQ);
6310 case Instruction::Call: {
6311 const CallInst *CI = cast<CallInst>(Val: I);
6312 switch (CI->getIntrinsicID()) {
6313 case Intrinsic::trunc:
6314 case Intrinsic::floor:
6315 case Intrinsic::ceil:
6316 case Intrinsic::rint:
6317 case Intrinsic::nearbyint:
6318 case Intrinsic::round:
6319 case Intrinsic::roundeven:
6320 return (FMF.noInfs() && FMF.noNaNs()) || isKnownNeverInfOrNaN(V: I, SQ);
6321 default:
6322 break;
6323 }
6324
6325 break;
6326 }
6327 default:
6328 break;
6329 }
6330
6331 return false;
6332}
6333
6334Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
6335
6336 // All byte-wide stores are splatable, even of arbitrary variables.
6337 if (V->getType()->isIntegerTy(Bitwidth: 8))
6338 return V;
6339
6340 LLVMContext &Ctx = V->getContext();
6341
6342 // Undef don't care.
6343 auto *UndefInt8 = UndefValue::get(T: Type::getInt8Ty(C&: Ctx));
6344 if (isa<UndefValue>(Val: V))
6345 return UndefInt8;
6346
6347 // Return poison for zero-sized type.
6348 if (DL.getTypeStoreSize(Ty: V->getType()).isZero())
6349 return PoisonValue::get(T: Type::getInt8Ty(C&: Ctx));
6350
6351 Constant *C = dyn_cast<Constant>(Val: V);
6352 if (!C) {
6353 // Conceptually, we could handle things like:
6354 // %a = zext i8 %X to i16
6355 // %b = shl i16 %a, 8
6356 // %c = or i16 %a, %b
6357 // but until there is an example that actually needs this, it doesn't seem
6358 // worth worrying about.
6359 return nullptr;
6360 }
6361
6362 // Handle 'null' ConstantArrayZero etc.
6363 if (C->isNullValue())
6364 return Constant::getNullValue(Ty: Type::getInt8Ty(C&: Ctx));
6365
6366 // Constant floating-point values can be handled as integer values if the
6367 // corresponding integer value is "byteable". An important case is 0.0.
6368 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
6369 Type *ScalarTy = CFP->getType()->getScalarType();
6370 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
6371 return isBytewiseValue(
6372 V: ConstantInt::get(Context&: Ctx, V: CFP->getValue().bitcastToAPInt()), DL);
6373
6374 // Don't handle long double formats, which have strange constraints.
6375 return nullptr;
6376 }
6377
6378 // We can handle constant integers that are multiple of 8 bits.
6379 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
6380 if (CI->getBitWidth() % 8 == 0) {
6381 if (!CI->getValue().isSplat(SplatSizeInBits: 8))
6382 return nullptr;
6383 return ConstantInt::get(Context&: Ctx, V: CI->getValue().trunc(width: 8));
6384 }
6385 }
6386
6387 if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
6388 if (CE->getOpcode() == Instruction::IntToPtr) {
6389 if (auto *PtrTy = dyn_cast<PointerType>(Val: CE->getType())) {
6390 unsigned BitWidth = DL.getPointerSizeInBits(AS: PtrTy->getAddressSpace());
6391 if (Constant *Op = ConstantFoldIntegerCast(
6392 C: CE->getOperand(i_nocapture: 0), DestTy: Type::getIntNTy(C&: Ctx, N: BitWidth), IsSigned: false, DL))
6393 return isBytewiseValue(V: Op, DL);
6394 }
6395 }
6396 }
6397
6398 auto Merge = [&](Value *LHS, Value *RHS) -> Value * {
6399 if (LHS == RHS)
6400 return LHS;
6401 if (!LHS || !RHS)
6402 return nullptr;
6403 if (LHS == UndefInt8)
6404 return RHS;
6405 if (RHS == UndefInt8)
6406 return LHS;
6407 return nullptr;
6408 };
6409
6410 if (ConstantDataSequential *CA = dyn_cast<ConstantDataSequential>(Val: C)) {
6411 Value *Val = UndefInt8;
6412 for (uint64_t I = 0, E = CA->getNumElements(); I != E; ++I)
6413 if (!(Val = Merge(Val, isBytewiseValue(V: CA->getElementAsConstant(i: I), DL))))
6414 return nullptr;
6415 return Val;
6416 }
6417
6418 if (isa<ConstantAggregate>(Val: C)) {
6419 Value *Val = UndefInt8;
6420 for (Value *Op : C->operands())
6421 if (!(Val = Merge(Val, isBytewiseValue(V: Op, DL))))
6422 return nullptr;
6423 return Val;
6424 }
6425
6426 // Don't try to handle the handful of other constants.
6427 return nullptr;
6428}
6429
6430// This is the recursive version of BuildSubAggregate. It takes a few different
6431// arguments. Idxs is the index within the nested struct From that we are
6432// looking at now (which is of type IndexedType). IdxSkip is the number of
6433// indices from Idxs that should be left out when inserting into the resulting
6434// struct. To is the result struct built so far, new insertvalue instructions
6435// build on that.
6436static Value *BuildSubAggregate(Value *From, Value *To, Type *IndexedType,
6437 SmallVectorImpl<unsigned> &Idxs,
6438 unsigned IdxSkip,
6439 BasicBlock::iterator InsertBefore) {
6440 StructType *STy = dyn_cast<StructType>(Val: IndexedType);
6441 if (STy) {
6442 // Save the original To argument so we can modify it
6443 Value *OrigTo = To;
6444 // General case, the type indexed by Idxs is a struct
6445 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6446 // Process each struct element recursively
6447 Idxs.push_back(Elt: i);
6448 Value *PrevTo = To;
6449 To = BuildSubAggregate(From, To, IndexedType: STy->getElementType(N: i), Idxs, IdxSkip,
6450 InsertBefore);
6451 Idxs.pop_back();
6452 if (!To) {
6453 // Couldn't find any inserted value for this index? Cleanup
6454 while (PrevTo != OrigTo) {
6455 InsertValueInst* Del = cast<InsertValueInst>(Val: PrevTo);
6456 PrevTo = Del->getAggregateOperand();
6457 Del->eraseFromParent();
6458 }
6459 // Stop processing elements
6460 break;
6461 }
6462 }
6463 // If we successfully found a value for each of our subaggregates
6464 if (To)
6465 return To;
6466 }
6467 // Base case, the type indexed by SourceIdxs is not a struct, or not all of
6468 // the struct's elements had a value that was inserted directly. In the latter
6469 // case, perhaps we can't determine each of the subelements individually, but
6470 // we might be able to find the complete struct somewhere.
6471
6472 // Find the value that is at that particular spot
6473 Value *V = FindInsertedValue(V: From, idx_range: Idxs);
6474
6475 if (!V)
6476 return nullptr;
6477
6478 // Insert the value in the new (sub) aggregate
6479 return InsertValueInst::Create(Agg: To, Val: V, Idxs: ArrayRef(Idxs).slice(N: IdxSkip), NameStr: "tmp",
6480 InsertBefore);
6481}
6482
6483// This helper takes a nested struct and extracts a part of it (which is again a
6484// struct) into a new value. For example, given the struct:
6485// { a, { b, { c, d }, e } }
6486// and the indices "1, 1" this returns
6487// { c, d }.
6488//
6489// It does this by inserting an insertvalue for each element in the resulting
6490// struct, as opposed to just inserting a single struct. This will only work if
6491// each of the elements of the substruct are known (ie, inserted into From by an
6492// insertvalue instruction somewhere).
6493//
6494// All inserted insertvalue instructions are inserted before InsertBefore
6495static Value *BuildSubAggregate(Value *From, ArrayRef<unsigned> idx_range,
6496 BasicBlock::iterator InsertBefore) {
6497 Type *IndexedType = ExtractValueInst::getIndexedType(Agg: From->getType(),
6498 Idxs: idx_range);
6499 Value *To = PoisonValue::get(T: IndexedType);
6500 SmallVector<unsigned, 10> Idxs(idx_range);
6501 unsigned IdxSkip = Idxs.size();
6502
6503 return BuildSubAggregate(From, To, IndexedType, Idxs, IdxSkip, InsertBefore);
6504}
6505
6506/// Given an aggregate and a sequence of indices, see if the scalar value
6507/// indexed is already around as a register, for example if it was inserted
6508/// directly into the aggregate.
6509///
6510/// If InsertBefore is not null, this function will duplicate (modified)
6511/// insertvalues when a part of a nested struct is extracted.
6512Value *
6513llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
6514 std::optional<BasicBlock::iterator> InsertBefore) {
6515 // Nothing to index? Just return V then (this is useful at the end of our
6516 // recursion).
6517 if (idx_range.empty())
6518 return V;
6519 // We have indices, so V should have an indexable type.
6520 assert((V->getType()->isStructTy() || V->getType()->isArrayTy()) &&
6521 "Not looking at a struct or array?");
6522 assert(ExtractValueInst::getIndexedType(V->getType(), idx_range) &&
6523 "Invalid indices for type?");
6524
6525 if (Constant *C = dyn_cast<Constant>(Val: V)) {
6526 C = C->getAggregateElement(Elt: idx_range[0]);
6527 if (!C) return nullptr;
6528 return FindInsertedValue(V: C, idx_range: idx_range.slice(N: 1), InsertBefore);
6529 }
6530
6531 if (InsertValueInst *I = dyn_cast<InsertValueInst>(Val: V)) {
6532 // Loop the indices for the insertvalue instruction in parallel with the
6533 // requested indices
6534 const unsigned *req_idx = idx_range.begin();
6535 for (const unsigned *i = I->idx_begin(), *e = I->idx_end();
6536 i != e; ++i, ++req_idx) {
6537 if (req_idx == idx_range.end()) {
6538 // We can't handle this without inserting insertvalues
6539 if (!InsertBefore)
6540 return nullptr;
6541
6542 // The requested index identifies a part of a nested aggregate. Handle
6543 // this specially. For example,
6544 // %A = insertvalue { i32, {i32, i32 } } undef, i32 10, 1, 0
6545 // %B = insertvalue { i32, {i32, i32 } } %A, i32 11, 1, 1
6546 // %C = extractvalue {i32, { i32, i32 } } %B, 1
6547 // This can be changed into
6548 // %A = insertvalue {i32, i32 } undef, i32 10, 0
6549 // %C = insertvalue {i32, i32 } %A, i32 11, 1
6550 // which allows the unused 0,0 element from the nested struct to be
6551 // removed.
6552 return BuildSubAggregate(From: V, idx_range: ArrayRef(idx_range.begin(), req_idx),
6553 InsertBefore: *InsertBefore);
6554 }
6555
6556 // This insert value inserts something else than what we are looking for.
6557 // See if the (aggregate) value inserted into has the value we are
6558 // looking for, then.
6559 if (*req_idx != *i)
6560 return FindInsertedValue(V: I->getAggregateOperand(), idx_range,
6561 InsertBefore);
6562 }
6563 // If we end up here, the indices of the insertvalue match with those
6564 // requested (though possibly only partially). Now we recursively look at
6565 // the inserted value, passing any remaining indices.
6566 return FindInsertedValue(V: I->getInsertedValueOperand(),
6567 idx_range: ArrayRef(req_idx, idx_range.end()), InsertBefore);
6568 }
6569
6570 if (ExtractValueInst *I = dyn_cast<ExtractValueInst>(Val: V)) {
6571 // If we're extracting a value from an aggregate that was extracted from
6572 // something else, we can extract from that something else directly instead.
6573 // However, we will need to chain I's indices with the requested indices.
6574
6575 // Calculate the number of indices required
6576 unsigned size = I->getNumIndices() + idx_range.size();
6577 // Allocate some space to put the new indices in
6578 SmallVector<unsigned, 5> Idxs;
6579 Idxs.reserve(N: size);
6580 // Add indices from the extract value instruction
6581 Idxs.append(in_start: I->idx_begin(), in_end: I->idx_end());
6582
6583 // Add requested indices
6584 Idxs.append(in_start: idx_range.begin(), in_end: idx_range.end());
6585
6586 assert(Idxs.size() == size
6587 && "Number of indices added not correct?");
6588
6589 return FindInsertedValue(V: I->getAggregateOperand(), idx_range: Idxs, InsertBefore);
6590 }
6591 // Otherwise, we don't know (such as, extracting from a function return value
6592 // or load instruction)
6593 return nullptr;
6594}
6595
6596// If V refers to an initialized global constant, set Slice either to
6597// its initializer if the size of its elements equals ElementSize, or,
6598// for ElementSize == 8, to its representation as an array of unsiged
6599// char. Return true on success.
6600// Offset is in the unit "nr of ElementSize sized elements".
6601bool llvm::getConstantDataArrayInfo(const Value *V,
6602 ConstantDataArraySlice &Slice,
6603 unsigned ElementSize, uint64_t Offset) {
6604 assert(V && "V should not be null.");
6605 assert((ElementSize % 8) == 0 &&
6606 "ElementSize expected to be a multiple of the size of a byte.");
6607 unsigned ElementSizeInBytes = ElementSize / 8;
6608
6609 // Drill down into the pointer expression V, ignoring any intervening
6610 // casts, and determine the identity of the object it references along
6611 // with the cumulative byte offset into it.
6612 const GlobalVariable *GV =
6613 dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V));
6614 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
6615 // Fail if V is not based on constant global object.
6616 return false;
6617
6618 const DataLayout &DL = GV->getDataLayout();
6619 APInt Off(DL.getIndexTypeSizeInBits(Ty: V->getType()), 0);
6620
6621 if (GV != V->stripAndAccumulateConstantOffsets(DL, Offset&: Off,
6622 /*AllowNonInbounds*/ true))
6623 // Fail if a constant offset could not be determined.
6624 return false;
6625
6626 uint64_t StartIdx = Off.getLimitedValue();
6627 if (StartIdx == UINT64_MAX)
6628 // Fail if the constant offset is excessive.
6629 return false;
6630
6631 // Off/StartIdx is in the unit of bytes. So we need to convert to number of
6632 // elements. Simply bail out if that isn't possible.
6633 if ((StartIdx % ElementSizeInBytes) != 0)
6634 return false;
6635
6636 Offset += StartIdx / ElementSizeInBytes;
6637 ConstantDataArray *Array = nullptr;
6638 ArrayType *ArrayTy = nullptr;
6639
6640 if (GV->getInitializer()->isNullValue()) {
6641 Type *GVTy = GV->getValueType();
6642 uint64_t SizeInBytes = DL.getTypeStoreSize(Ty: GVTy).getFixedValue();
6643 uint64_t Length = SizeInBytes / ElementSizeInBytes;
6644
6645 Slice.Array = nullptr;
6646 Slice.Offset = 0;
6647 // Return an empty Slice for undersized constants to let callers
6648 // transform even undefined library calls into simpler, well-defined
6649 // expressions. This is preferable to making the calls although it
6650 // prevents sanitizers from detecting such calls.
6651 Slice.Length = Length < Offset ? 0 : Length - Offset;
6652 return true;
6653 }
6654
6655 auto *Init = const_cast<Constant *>(GV->getInitializer());
6656 if (auto *ArrayInit = dyn_cast<ConstantDataArray>(Val: Init)) {
6657 Type *InitElTy = ArrayInit->getElementType();
6658 if (InitElTy->isIntegerTy(Bitwidth: ElementSize)) {
6659 // If Init is an initializer for an array of the expected type
6660 // and size, use it as is.
6661 Array = ArrayInit;
6662 ArrayTy = ArrayInit->getType();
6663 }
6664 }
6665
6666 if (!Array) {
6667 if (ElementSize != 8)
6668 // TODO: Handle conversions to larger integral types.
6669 return false;
6670
6671 // Otherwise extract the portion of the initializer starting
6672 // at Offset as an array of bytes, and reset Offset.
6673 Init = ReadByteArrayFromGlobal(GV, Offset);
6674 if (!Init)
6675 return false;
6676
6677 Offset = 0;
6678 Array = dyn_cast<ConstantDataArray>(Val: Init);
6679 ArrayTy = dyn_cast<ArrayType>(Val: Init->getType());
6680 }
6681
6682 uint64_t NumElts = ArrayTy->getArrayNumElements();
6683 if (Offset > NumElts)
6684 return false;
6685
6686 Slice.Array = Array;
6687 Slice.Offset = Offset;
6688 Slice.Length = NumElts - Offset;
6689 return true;
6690}
6691
6692/// Extract bytes from the initializer of the constant array V, which need
6693/// not be a nul-terminated string. On success, store the bytes in Str and
6694/// return true. When TrimAtNul is set, Str will contain only the bytes up
6695/// to but not including the first nul. Return false on failure.
6696bool llvm::getConstantStringInfo(const Value *V, StringRef &Str,
6697 bool TrimAtNul) {
6698 ConstantDataArraySlice Slice;
6699 if (!getConstantDataArrayInfo(V, Slice, ElementSize: 8))
6700 return false;
6701
6702 if (Slice.Array == nullptr) {
6703 if (TrimAtNul) {
6704 // Return a nul-terminated string even for an empty Slice. This is
6705 // safe because all existing SimplifyLibcalls callers require string
6706 // arguments and the behavior of the functions they fold is undefined
6707 // otherwise. Folding the calls this way is preferable to making
6708 // the undefined library calls, even though it prevents sanitizers
6709 // from reporting such calls.
6710 Str = StringRef();
6711 return true;
6712 }
6713 if (Slice.Length == 1) {
6714 Str = StringRef("", 1);
6715 return true;
6716 }
6717 // We cannot instantiate a StringRef as we do not have an appropriate string
6718 // of 0s at hand.
6719 return false;
6720 }
6721
6722 // Start out with the entire array in the StringRef.
6723 Str = Slice.Array->getAsString();
6724 // Skip over 'offset' bytes.
6725 Str = Str.substr(Start: Slice.Offset);
6726
6727 if (TrimAtNul) {
6728 // Trim off the \0 and anything after it. If the array is not nul
6729 // terminated, we just return the whole end of string. The client may know
6730 // some other way that the string is length-bound.
6731 Str = Str.substr(Start: 0, N: Str.find(C: '\0'));
6732 }
6733 return true;
6734}
6735
6736// These next two are very similar to the above, but also look through PHI
6737// nodes.
6738// TODO: See if we can integrate these two together.
6739
6740/// If we can compute the length of the string pointed to by
6741/// the specified pointer, return 'len+1'. If we can't, return 0.
6742static uint64_t GetStringLengthH(const Value *V,
6743 SmallPtrSetImpl<const PHINode*> &PHIs,
6744 unsigned CharSize) {
6745 // Look through noop bitcast instructions.
6746 V = V->stripPointerCasts();
6747
6748 // If this is a PHI node, there are two cases: either we have already seen it
6749 // or we haven't.
6750 if (const PHINode *PN = dyn_cast<PHINode>(Val: V)) {
6751 if (!PHIs.insert(Ptr: PN).second)
6752 return ~0ULL; // already in the set.
6753
6754 // If it was new, see if all the input strings are the same length.
6755 uint64_t LenSoFar = ~0ULL;
6756 for (Value *IncValue : PN->incoming_values()) {
6757 uint64_t Len = GetStringLengthH(V: IncValue, PHIs, CharSize);
6758 if (Len == 0) return 0; // Unknown length -> unknown.
6759
6760 if (Len == ~0ULL) continue;
6761
6762 if (Len != LenSoFar && LenSoFar != ~0ULL)
6763 return 0; // Disagree -> unknown.
6764 LenSoFar = Len;
6765 }
6766
6767 // Success, all agree.
6768 return LenSoFar;
6769 }
6770
6771 // strlen(select(c,x,y)) -> strlen(x) ^ strlen(y)
6772 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: V)) {
6773 uint64_t Len1 = GetStringLengthH(V: SI->getTrueValue(), PHIs, CharSize);
6774 if (Len1 == 0) return 0;
6775 uint64_t Len2 = GetStringLengthH(V: SI->getFalseValue(), PHIs, CharSize);
6776 if (Len2 == 0) return 0;
6777 if (Len1 == ~0ULL) return Len2;
6778 if (Len2 == ~0ULL) return Len1;
6779 if (Len1 != Len2) return 0;
6780 return Len1;
6781 }
6782
6783 // Otherwise, see if we can read the string.
6784 ConstantDataArraySlice Slice;
6785 if (!getConstantDataArrayInfo(V, Slice, ElementSize: CharSize))
6786 return 0;
6787
6788 if (Slice.Array == nullptr)
6789 // Zeroinitializer (including an empty one).
6790 return 1;
6791
6792 // Search for the first nul character. Return a conservative result even
6793 // when there is no nul. This is safe since otherwise the string function
6794 // being folded such as strlen is undefined, and can be preferable to
6795 // making the undefined library call.
6796 unsigned NullIndex = 0;
6797 for (unsigned E = Slice.Length; NullIndex < E; ++NullIndex) {
6798 if (Slice.Array->getElementAsInteger(i: Slice.Offset + NullIndex) == 0)
6799 break;
6800 }
6801
6802 return NullIndex + 1;
6803}
6804
6805/// If we can compute the length of the string pointed to by
6806/// the specified pointer, return 'len+1'. If we can't, return 0.
6807uint64_t llvm::GetStringLength(const Value *V, unsigned CharSize) {
6808 if (!V->getType()->isPointerTy())
6809 return 0;
6810
6811 SmallPtrSet<const PHINode*, 32> PHIs;
6812 uint64_t Len = GetStringLengthH(V, PHIs, CharSize);
6813 // If Len is ~0ULL, we had an infinite phi cycle: this is dead code, so return
6814 // an empty string as a length.
6815 return Len == ~0ULL ? 1 : Len;
6816}
6817
6818const Value *
6819llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call,
6820 bool MustPreserveNullness) {
6821 assert(Call &&
6822 "getArgumentAliasingToReturnedPointer only works on nonnull calls");
6823 if (const Value *RV = Call->getReturnedArgOperand())
6824 return RV;
6825 // This can be used only as a aliasing property.
6826 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
6827 Call, MustPreserveNullness))
6828 return Call->getArgOperand(i: 0);
6829 return nullptr;
6830}
6831
6832bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
6833 const CallBase *Call, bool MustPreserveNullness) {
6834 switch (Call->getIntrinsicID()) {
6835 case Intrinsic::launder_invariant_group:
6836 case Intrinsic::strip_invariant_group:
6837 case Intrinsic::aarch64_irg:
6838 case Intrinsic::aarch64_tagp:
6839 // The amdgcn_make_buffer_rsrc function does not alter the address of the
6840 // input pointer (and thus preserve null-ness for the purposes of escape
6841 // analysis, which is where the MustPreserveNullness flag comes in to play).
6842 // However, it will not necessarily map ptr addrspace(N) null to ptr
6843 // addrspace(8) null, aka the "null descriptor", which has "all loads return
6844 // 0, all stores are dropped" semantics. Given the context of this intrinsic
6845 // list, no one should be relying on such a strict interpretation of
6846 // MustPreserveNullness (and, at time of writing, they are not), but we
6847 // document this fact out of an abundance of caution.
6848 case Intrinsic::amdgcn_make_buffer_rsrc:
6849 return true;
6850 case Intrinsic::ptrmask:
6851 return !MustPreserveNullness;
6852 case Intrinsic::threadlocal_address:
6853 // The underlying variable changes with thread ID. The Thread ID may change
6854 // at coroutine suspend points.
6855 return !Call->getParent()->getParent()->isPresplitCoroutine();
6856 default:
6857 return false;
6858 }
6859}
6860
6861/// \p PN defines a loop-variant pointer to an object. Check if the
6862/// previous iteration of the loop was referring to the same object as \p PN.
6863static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
6864 const LoopInfo *LI) {
6865 // Find the loop-defined value.
6866 Loop *L = LI->getLoopFor(BB: PN->getParent());
6867 if (PN->getNumIncomingValues() != 2)
6868 return true;
6869
6870 // Find the value from previous iteration.
6871 auto *PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 0));
6872 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
6873 PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 1));
6874 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
6875 return true;
6876
6877 // If a new pointer is loaded in the loop, the pointer references a different
6878 // object in every iteration. E.g.:
6879 // for (i)
6880 // int *p = a[i];
6881 // ...
6882 if (auto *Load = dyn_cast<LoadInst>(Val: PrevValue))
6883 if (!L->isLoopInvariant(V: Load->getPointerOperand()))
6884 return false;
6885 return true;
6886}
6887
6888const Value *llvm::getUnderlyingObject(const Value *V, unsigned MaxLookup) {
6889 for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
6890 if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
6891 const Value *PtrOp = GEP->getPointerOperand();
6892 if (!PtrOp->getType()->isPointerTy()) // Only handle scalar pointer base.
6893 return V;
6894 V = PtrOp;
6895 } else if (Operator::getOpcode(V) == Instruction::BitCast ||
6896 Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
6897 Value *NewV = cast<Operator>(Val: V)->getOperand(i: 0);
6898 if (!NewV->getType()->isPointerTy())
6899 return V;
6900 V = NewV;
6901 } else if (auto *GA = dyn_cast<GlobalAlias>(Val: V)) {
6902 if (GA->isInterposable())
6903 return V;
6904 V = GA->getAliasee();
6905 } else {
6906 if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
6907 // Look through single-arg phi nodes created by LCSSA.
6908 if (PHI->getNumIncomingValues() == 1) {
6909 V = PHI->getIncomingValue(i: 0);
6910 continue;
6911 }
6912 } else if (auto *Call = dyn_cast<CallBase>(Val: V)) {
6913 // CaptureTracking can know about special capturing properties of some
6914 // intrinsics like launder.invariant.group, that can't be expressed with
6915 // the attributes, but have properties like returning aliasing pointer.
6916 // Because some analysis may assume that nocaptured pointer is not
6917 // returned from some special intrinsic (because function would have to
6918 // be marked with returns attribute), it is crucial to use this function
6919 // because it should be in sync with CaptureTracking. Not using it may
6920 // cause weird miscompilations where 2 aliasing pointers are assumed to
6921 // noalias.
6922 if (auto *RP = getArgumentAliasingToReturnedPointer(Call, MustPreserveNullness: false)) {
6923 V = RP;
6924 continue;
6925 }
6926 }
6927
6928 return V;
6929 }
6930 assert(V->getType()->isPointerTy() && "Unexpected operand type!");
6931 }
6932 return V;
6933}
6934
6935void llvm::getUnderlyingObjects(const Value *V,
6936 SmallVectorImpl<const Value *> &Objects,
6937 const LoopInfo *LI, unsigned MaxLookup) {
6938 SmallPtrSet<const Value *, 4> Visited;
6939 SmallVector<const Value *, 4> Worklist;
6940 Worklist.push_back(Elt: V);
6941 do {
6942 const Value *P = Worklist.pop_back_val();
6943 P = getUnderlyingObject(V: P, MaxLookup);
6944
6945 if (!Visited.insert(Ptr: P).second)
6946 continue;
6947
6948 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
6949 Worklist.push_back(Elt: SI->getTrueValue());
6950 Worklist.push_back(Elt: SI->getFalseValue());
6951 continue;
6952 }
6953
6954 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
6955 // If this PHI changes the underlying object in every iteration of the
6956 // loop, don't look through it. Consider:
6957 // int **A;
6958 // for (i) {
6959 // Prev = Curr; // Prev = PHI (Prev_0, Curr)
6960 // Curr = A[i];
6961 // *Prev, *Curr;
6962 //
6963 // Prev is tracking Curr one iteration behind so they refer to different
6964 // underlying objects.
6965 if (!LI || !LI->isLoopHeader(BB: PN->getParent()) ||
6966 isSameUnderlyingObjectInLoop(PN, LI))
6967 append_range(C&: Worklist, R: PN->incoming_values());
6968 else
6969 Objects.push_back(Elt: P);
6970 continue;
6971 }
6972
6973 Objects.push_back(Elt: P);
6974 } while (!Worklist.empty());
6975}
6976
6977const Value *llvm::getUnderlyingObjectAggressive(const Value *V) {
6978 const unsigned MaxVisited = 8;
6979
6980 SmallPtrSet<const Value *, 8> Visited;
6981 SmallVector<const Value *, 8> Worklist;
6982 Worklist.push_back(Elt: V);
6983 const Value *Object = nullptr;
6984 // Used as fallback if we can't find a common underlying object through
6985 // recursion.
6986 bool First = true;
6987 const Value *FirstObject = getUnderlyingObject(V);
6988 do {
6989 const Value *P = Worklist.pop_back_val();
6990 P = First ? FirstObject : getUnderlyingObject(V: P);
6991 First = false;
6992
6993 if (!Visited.insert(Ptr: P).second)
6994 continue;
6995
6996 if (Visited.size() == MaxVisited)
6997 return FirstObject;
6998
6999 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7000 Worklist.push_back(Elt: SI->getTrueValue());
7001 Worklist.push_back(Elt: SI->getFalseValue());
7002 continue;
7003 }
7004
7005 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7006 append_range(C&: Worklist, R: PN->incoming_values());
7007 continue;
7008 }
7009
7010 if (!Object)
7011 Object = P;
7012 else if (Object != P)
7013 return FirstObject;
7014 } while (!Worklist.empty());
7015
7016 return Object ? Object : FirstObject;
7017}
7018
7019/// This is the function that does the work of looking through basic
7020/// ptrtoint+arithmetic+inttoptr sequences.
7021static const Value *getUnderlyingObjectFromInt(const Value *V) {
7022 do {
7023 if (const Operator *U = dyn_cast<Operator>(Val: V)) {
7024 // If we find a ptrtoint, we can transfer control back to the
7025 // regular getUnderlyingObjectFromInt.
7026 if (U->getOpcode() == Instruction::PtrToInt)
7027 return U->getOperand(i: 0);
7028 // If we find an add of a constant, a multiplied value, or a phi, it's
7029 // likely that the other operand will lead us to the base
7030 // object. We don't have to worry about the case where the
7031 // object address is somehow being computed by the multiply,
7032 // because our callers only care when the result is an
7033 // identifiable object.
7034 if (U->getOpcode() != Instruction::Add ||
7035 (!isa<ConstantInt>(Val: U->getOperand(i: 1)) &&
7036 Operator::getOpcode(V: U->getOperand(i: 1)) != Instruction::Mul &&
7037 !isa<PHINode>(Val: U->getOperand(i: 1))))
7038 return V;
7039 V = U->getOperand(i: 0);
7040 } else {
7041 return V;
7042 }
7043 assert(V->getType()->isIntegerTy() && "Unexpected operand type!");
7044 } while (true);
7045}
7046
7047/// This is a wrapper around getUnderlyingObjects and adds support for basic
7048/// ptrtoint+arithmetic+inttoptr sequences.
7049/// It returns false if unidentified object is found in getUnderlyingObjects.
7050bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
7051 SmallVectorImpl<Value *> &Objects) {
7052 SmallPtrSet<const Value *, 16> Visited;
7053 SmallVector<const Value *, 4> Working(1, V);
7054 do {
7055 V = Working.pop_back_val();
7056
7057 SmallVector<const Value *, 4> Objs;
7058 getUnderlyingObjects(V, Objects&: Objs);
7059
7060 for (const Value *V : Objs) {
7061 if (!Visited.insert(Ptr: V).second)
7062 continue;
7063 if (Operator::getOpcode(V) == Instruction::IntToPtr) {
7064 const Value *O =
7065 getUnderlyingObjectFromInt(V: cast<User>(Val: V)->getOperand(i: 0));
7066 if (O->getType()->isPointerTy()) {
7067 Working.push_back(Elt: O);
7068 continue;
7069 }
7070 }
7071 // If getUnderlyingObjects fails to find an identifiable object,
7072 // getUnderlyingObjectsForCodeGen also fails for safety.
7073 if (!isIdentifiedObject(V)) {
7074 Objects.clear();
7075 return false;
7076 }
7077 Objects.push_back(Elt: const_cast<Value *>(V));
7078 }
7079 } while (!Working.empty());
7080 return true;
7081}
7082
7083AllocaInst *llvm::findAllocaForValue(Value *V, bool OffsetZero) {
7084 AllocaInst *Result = nullptr;
7085 SmallPtrSet<Value *, 4> Visited;
7086 SmallVector<Value *, 4> Worklist;
7087
7088 auto AddWork = [&](Value *V) {
7089 if (Visited.insert(Ptr: V).second)
7090 Worklist.push_back(Elt: V);
7091 };
7092
7093 AddWork(V);
7094 do {
7095 V = Worklist.pop_back_val();
7096 assert(Visited.count(V));
7097
7098 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
7099 if (Result && Result != AI)
7100 return nullptr;
7101 Result = AI;
7102 } else if (CastInst *CI = dyn_cast<CastInst>(Val: V)) {
7103 AddWork(CI->getOperand(i_nocapture: 0));
7104 } else if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7105 for (Value *IncValue : PN->incoming_values())
7106 AddWork(IncValue);
7107 } else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
7108 AddWork(SI->getTrueValue());
7109 AddWork(SI->getFalseValue());
7110 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: V)) {
7111 if (OffsetZero && !GEP->hasAllZeroIndices())
7112 return nullptr;
7113 AddWork(GEP->getPointerOperand());
7114 } else if (CallBase *CB = dyn_cast<CallBase>(Val: V)) {
7115 Value *Returned = CB->getReturnedArgOperand();
7116 if (Returned)
7117 AddWork(Returned);
7118 else
7119 return nullptr;
7120 } else {
7121 return nullptr;
7122 }
7123 } while (!Worklist.empty());
7124
7125 return Result;
7126}
7127
7128static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7129 const Value *V, bool AllowLifetime, bool AllowDroppable) {
7130 for (const User *U : V->users()) {
7131 const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
7132 if (!II)
7133 return false;
7134
7135 if (AllowLifetime && II->isLifetimeStartOrEnd())
7136 continue;
7137
7138 if (AllowDroppable && II->isDroppable())
7139 continue;
7140
7141 return false;
7142 }
7143 return true;
7144}
7145
7146bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
7147 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7148 V, /* AllowLifetime */ true, /* AllowDroppable */ false);
7149}
7150bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) {
7151 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7152 V, /* AllowLifetime */ true, /* AllowDroppable */ true);
7153}
7154
7155bool llvm::isNotCrossLaneOperation(const Instruction *I) {
7156 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
7157 return isTriviallyVectorizable(ID: II->getIntrinsicID());
7158 auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: I);
7159 return (!Shuffle || Shuffle->isSelect()) &&
7160 !isa<CallBase, BitCastInst, ExtractElementInst>(Val: I);
7161}
7162
7163bool llvm::isSafeToSpeculativelyExecute(
7164 const Instruction *Inst, const Instruction *CtxI, AssumptionCache *AC,
7165 const DominatorTree *DT, const TargetLibraryInfo *TLI, bool UseVariableInfo,
7166 bool IgnoreUBImplyingAttrs) {
7167 return isSafeToSpeculativelyExecuteWithOpcode(Opcode: Inst->getOpcode(), Inst, CtxI,
7168 AC, DT, TLI, UseVariableInfo,
7169 IgnoreUBImplyingAttrs);
7170}
7171
7172bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
7173 unsigned Opcode, const Instruction *Inst, const Instruction *CtxI,
7174 AssumptionCache *AC, const DominatorTree *DT, const TargetLibraryInfo *TLI,
7175 bool UseVariableInfo, bool IgnoreUBImplyingAttrs) {
7176#ifndef NDEBUG
7177 if (Inst->getOpcode() != Opcode) {
7178 // Check that the operands are actually compatible with the Opcode override.
7179 auto hasEqualReturnAndLeadingOperandTypes =
7180 [](const Instruction *Inst, unsigned NumLeadingOperands) {
7181 if (Inst->getNumOperands() < NumLeadingOperands)
7182 return false;
7183 const Type *ExpectedType = Inst->getType();
7184 for (unsigned ItOp = 0; ItOp < NumLeadingOperands; ++ItOp)
7185 if (Inst->getOperand(ItOp)->getType() != ExpectedType)
7186 return false;
7187 return true;
7188 };
7189 assert(!Instruction::isBinaryOp(Opcode) ||
7190 hasEqualReturnAndLeadingOperandTypes(Inst, 2));
7191 assert(!Instruction::isUnaryOp(Opcode) ||
7192 hasEqualReturnAndLeadingOperandTypes(Inst, 1));
7193 }
7194#endif
7195
7196 switch (Opcode) {
7197 default:
7198 return true;
7199 case Instruction::UDiv:
7200 case Instruction::URem: {
7201 // x / y is undefined if y == 0.
7202 const APInt *V;
7203 if (match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: V)))
7204 return *V != 0;
7205 return false;
7206 }
7207 case Instruction::SDiv:
7208 case Instruction::SRem: {
7209 // x / y is undefined if y == 0 or x == INT_MIN and y == -1
7210 const APInt *Numerator, *Denominator;
7211 if (!match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: Denominator)))
7212 return false;
7213 // We cannot hoist this division if the denominator is 0.
7214 if (*Denominator == 0)
7215 return false;
7216 // It's safe to hoist if the denominator is not 0 or -1.
7217 if (!Denominator->isAllOnes())
7218 return true;
7219 // At this point we know that the denominator is -1. It is safe to hoist as
7220 // long we know that the numerator is not INT_MIN.
7221 if (match(V: Inst->getOperand(i: 0), P: m_APInt(Res&: Numerator)))
7222 return !Numerator->isMinSignedValue();
7223 // The numerator *might* be MinSignedValue.
7224 return false;
7225 }
7226 case Instruction::Load: {
7227 if (!UseVariableInfo)
7228 return false;
7229
7230 const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst);
7231 if (!LI)
7232 return false;
7233 if (mustSuppressSpeculation(LI: *LI))
7234 return false;
7235 const DataLayout &DL = LI->getDataLayout();
7236 return isDereferenceableAndAlignedPointer(V: LI->getPointerOperand(),
7237 Ty: LI->getType(), Alignment: LI->getAlign(), DL,
7238 CtxI, AC, DT, TLI);
7239 }
7240 case Instruction::Call: {
7241 auto *CI = dyn_cast<const CallInst>(Val: Inst);
7242 if (!CI)
7243 return false;
7244 const Function *Callee = CI->getCalledFunction();
7245
7246 // The called function could have undefined behavior or side-effects, even
7247 // if marked readnone nounwind.
7248 if (!Callee || !Callee->isSpeculatable())
7249 return false;
7250 // Since the operands may be changed after hoisting, undefined behavior may
7251 // be triggered by some UB-implying attributes.
7252 return IgnoreUBImplyingAttrs || !CI->hasUBImplyingAttrs();
7253 }
7254 case Instruction::VAArg:
7255 case Instruction::Alloca:
7256 case Instruction::Invoke:
7257 case Instruction::CallBr:
7258 case Instruction::PHI:
7259 case Instruction::Store:
7260 case Instruction::Ret:
7261 case Instruction::UncondBr:
7262 case Instruction::CondBr:
7263 case Instruction::IndirectBr:
7264 case Instruction::Switch:
7265 case Instruction::Unreachable:
7266 case Instruction::Fence:
7267 case Instruction::AtomicRMW:
7268 case Instruction::AtomicCmpXchg:
7269 case Instruction::LandingPad:
7270 case Instruction::Resume:
7271 case Instruction::CatchSwitch:
7272 case Instruction::CatchPad:
7273 case Instruction::CatchRet:
7274 case Instruction::CleanupPad:
7275 case Instruction::CleanupRet:
7276 return false; // Misc instructions which have effects
7277 }
7278}
7279
7280bool llvm::mayHaveNonDefUseDependency(const Instruction &I) {
7281 if (I.mayReadOrWriteMemory())
7282 // Memory dependency possible
7283 return true;
7284 if (!isSafeToSpeculativelyExecute(Inst: &I))
7285 // Can't move above a maythrow call or infinite loop. Or if an
7286 // inalloca alloca, above a stacksave call.
7287 return true;
7288 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7289 // 1) Can't reorder two inf-loop calls, even if readonly
7290 // 2) Also can't reorder an inf-loop call below a instruction which isn't
7291 // safe to speculative execute. (Inverse of above)
7292 return true;
7293 return false;
7294}
7295
7296/// Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
7297static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) {
7298 switch (OR) {
7299 case ConstantRange::OverflowResult::MayOverflow:
7300 return OverflowResult::MayOverflow;
7301 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
7302 return OverflowResult::AlwaysOverflowsLow;
7303 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
7304 return OverflowResult::AlwaysOverflowsHigh;
7305 case ConstantRange::OverflowResult::NeverOverflows:
7306 return OverflowResult::NeverOverflows;
7307 }
7308 llvm_unreachable("Unknown OverflowResult");
7309}
7310
7311/// Combine constant ranges from computeConstantRange() and computeKnownBits().
7312ConstantRange
7313llvm::computeConstantRangeIncludingKnownBits(const WithCache<const Value *> &V,
7314 bool ForSigned,
7315 const SimplifyQuery &SQ) {
7316 ConstantRange CR1 =
7317 ConstantRange::fromKnownBits(Known: V.getKnownBits(Q: SQ), IsSigned: ForSigned);
7318 ConstantRange CR2 = computeConstantRange(V, ForSigned, UseInstrInfo: SQ.IIQ.UseInstrInfo);
7319 ConstantRange::PreferredRangeType RangeType =
7320 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
7321 return CR1.intersectWith(CR: CR2, Type: RangeType);
7322}
7323
7324OverflowResult llvm::computeOverflowForUnsignedMul(const Value *LHS,
7325 const Value *RHS,
7326 const SimplifyQuery &SQ,
7327 bool IsNSW) {
7328 ConstantRange LHSRange =
7329 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7330 ConstantRange RHSRange =
7331 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7332
7333 // mul nsw of two non-negative numbers is also nuw.
7334 if (IsNSW && LHSRange.isAllNonNegative() && RHSRange.isAllNonNegative())
7335 return OverflowResult::NeverOverflows;
7336
7337 return mapOverflowResult(OR: LHSRange.unsignedMulMayOverflow(Other: RHSRange));
7338}
7339
7340OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
7341 const Value *RHS,
7342 const SimplifyQuery &SQ) {
7343 // Multiplying n * m significant bits yields a result of n + m significant
7344 // bits. If the total number of significant bits does not exceed the
7345 // result bit width (minus 1), there is no overflow.
7346 // This means if we have enough leading sign bits in the operands
7347 // we can guarantee that the result does not overflow.
7348 // Ref: "Hacker's Delight" by Henry Warren
7349 unsigned BitWidth = LHS->getType()->getScalarSizeInBits();
7350
7351 // Note that underestimating the number of sign bits gives a more
7352 // conservative answer.
7353 unsigned SignBits =
7354 ::ComputeNumSignBits(V: LHS, Q: SQ) + ::ComputeNumSignBits(V: RHS, Q: SQ);
7355
7356 // First handle the easy case: if we have enough sign bits there's
7357 // definitely no overflow.
7358 if (SignBits > BitWidth + 1)
7359 return OverflowResult::NeverOverflows;
7360
7361 // There are two ambiguous cases where there can be no overflow:
7362 // SignBits == BitWidth + 1 and
7363 // SignBits == BitWidth
7364 // The second case is difficult to check, therefore we only handle the
7365 // first case.
7366 if (SignBits == BitWidth + 1) {
7367 // It overflows only when both arguments are negative and the true
7368 // product is exactly the minimum negative number.
7369 // E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
7370 // For simplicity we just check if at least one side is not negative.
7371 KnownBits LHSKnown = computeKnownBits(V: LHS, Q: SQ);
7372 KnownBits RHSKnown = computeKnownBits(V: RHS, Q: SQ);
7373 if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
7374 return OverflowResult::NeverOverflows;
7375 }
7376 return OverflowResult::MayOverflow;
7377}
7378
7379OverflowResult
7380llvm::computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
7381 const WithCache<const Value *> &RHS,
7382 const SimplifyQuery &SQ) {
7383 ConstantRange LHSRange =
7384 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7385 ConstantRange RHSRange =
7386 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7387 return mapOverflowResult(OR: LHSRange.unsignedAddMayOverflow(Other: RHSRange));
7388}
7389
7390static OverflowResult
7391computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
7392 const WithCache<const Value *> &RHS,
7393 const AddOperator *Add, const SimplifyQuery &SQ) {
7394 if (Add && Add->hasNoSignedWrap()) {
7395 return OverflowResult::NeverOverflows;
7396 }
7397
7398 // If LHS and RHS each have at least two sign bits, the addition will look
7399 // like
7400 //
7401 // XX..... +
7402 // YY.....
7403 //
7404 // If the carry into the most significant position is 0, X and Y can't both
7405 // be 1 and therefore the carry out of the addition is also 0.
7406 //
7407 // If the carry into the most significant position is 1, X and Y can't both
7408 // be 0 and therefore the carry out of the addition is also 1.
7409 //
7410 // Since the carry into the most significant position is always equal to
7411 // the carry out of the addition, there is no signed overflow.
7412 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7413 return OverflowResult::NeverOverflows;
7414
7415 ConstantRange LHSRange =
7416 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7417 ConstantRange RHSRange =
7418 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7419 OverflowResult OR =
7420 mapOverflowResult(OR: LHSRange.signedAddMayOverflow(Other: RHSRange));
7421 if (OR != OverflowResult::MayOverflow)
7422 return OR;
7423
7424 // The remaining code needs Add to be available. Early returns if not so.
7425 if (!Add)
7426 return OverflowResult::MayOverflow;
7427
7428 // If the sign of Add is the same as at least one of the operands, this add
7429 // CANNOT overflow. If this can be determined from the known bits of the
7430 // operands the above signedAddMayOverflow() check will have already done so.
7431 // The only other way to improve on the known bits is from an assumption, so
7432 // call computeKnownBitsFromContext() directly.
7433 bool LHSOrRHSKnownNonNegative =
7434 (LHSRange.isAllNonNegative() || RHSRange.isAllNonNegative());
7435 bool LHSOrRHSKnownNegative =
7436 (LHSRange.isAllNegative() || RHSRange.isAllNegative());
7437 if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) {
7438 KnownBits AddKnown(LHSRange.getBitWidth());
7439 computeKnownBitsFromContext(V: Add, Known&: AddKnown, Q: SQ);
7440 if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) ||
7441 (AddKnown.isNegative() && LHSOrRHSKnownNegative))
7442 return OverflowResult::NeverOverflows;
7443 }
7444
7445 return OverflowResult::MayOverflow;
7446}
7447
7448OverflowResult llvm::computeOverflowForUnsignedSub(const Value *LHS,
7449 const Value *RHS,
7450 const SimplifyQuery &SQ) {
7451 // X - (X % ?)
7452 // The remainder of a value can't have greater magnitude than itself,
7453 // so the subtraction can't overflow.
7454
7455 // X - (X -nuw ?)
7456 // In the minimal case, this would simplify to "?", so there's no subtract
7457 // at all. But if this analysis is used to peek through casts, for example,
7458 // then determining no-overflow may allow other transforms.
7459
7460 // TODO: There are other patterns like this.
7461 // See simplifyICmpWithBinOpOnLHS() for candidates.
7462 if (match(V: RHS, P: m_URem(L: m_Specific(V: LHS), R: m_Value())) ||
7463 match(V: RHS, P: m_NUWSub(L: m_Specific(V: LHS), R: m_Value())))
7464 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
7465 return OverflowResult::NeverOverflows;
7466
7467 if (auto C = isImpliedByDomCondition(Pred: CmpInst::ICMP_UGE, LHS, RHS, ContextI: SQ.CxtI,
7468 DL: SQ.DL)) {
7469 if (*C)
7470 return OverflowResult::NeverOverflows;
7471 return OverflowResult::AlwaysOverflowsLow;
7472 }
7473
7474 ConstantRange LHSRange =
7475 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7476 ConstantRange RHSRange =
7477 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7478 return mapOverflowResult(OR: LHSRange.unsignedSubMayOverflow(Other: RHSRange));
7479}
7480
7481OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS,
7482 const Value *RHS,
7483 const SimplifyQuery &SQ) {
7484 // X - (X % ?)
7485 // The remainder of a value can't have greater magnitude than itself,
7486 // so the subtraction can't overflow.
7487
7488 // X - (X -nsw ?)
7489 // In the minimal case, this would simplify to "?", so there's no subtract
7490 // at all. But if this analysis is used to peek through casts, for example,
7491 // then determining no-overflow may allow other transforms.
7492 if (match(V: RHS, P: m_SRem(L: m_Specific(V: LHS), R: m_Value())) ||
7493 match(V: RHS, P: m_NSWSub(L: m_Specific(V: LHS), R: m_Value())))
7494 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
7495 return OverflowResult::NeverOverflows;
7496
7497 // If LHS and RHS each have at least two sign bits, the subtraction
7498 // cannot overflow.
7499 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7500 return OverflowResult::NeverOverflows;
7501
7502 ConstantRange LHSRange =
7503 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7504 ConstantRange RHSRange =
7505 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7506 return mapOverflowResult(OR: LHSRange.signedSubMayOverflow(Other: RHSRange));
7507}
7508
7509bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO,
7510 const DominatorTree &DT) {
7511 SmallVector<const CondBrInst *, 2> GuardingBranches;
7512 SmallVector<const ExtractValueInst *, 2> Results;
7513
7514 for (const User *U : WO->users()) {
7515 if (const auto *EVI = dyn_cast<ExtractValueInst>(Val: U)) {
7516 assert(EVI->getNumIndices() == 1 && "Obvious from CI's type");
7517
7518 if (EVI->getIndices()[0] == 0)
7519 Results.push_back(Elt: EVI);
7520 else {
7521 assert(EVI->getIndices()[0] == 1 && "Obvious from CI's type");
7522
7523 for (const auto *U : EVI->users())
7524 if (const auto *B = dyn_cast<CondBrInst>(Val: U))
7525 GuardingBranches.push_back(Elt: B);
7526 }
7527 } else {
7528 // We are using the aggregate directly in a way we don't want to analyze
7529 // here (storing it to a global, say).
7530 return false;
7531 }
7532 }
7533
7534 auto AllUsesGuardedByBranch = [&](const CondBrInst *BI) {
7535 BasicBlockEdge NoWrapEdge(BI->getParent(), BI->getSuccessor(i: 1));
7536
7537 // Check if all users of the add are provably no-wrap.
7538 for (const auto *Result : Results) {
7539 // If the extractvalue itself is not executed on overflow, the we don't
7540 // need to check each use separately, since domination is transitive.
7541 if (DT.dominates(BBE: NoWrapEdge, BB: Result->getParent()))
7542 continue;
7543
7544 for (const auto &RU : Result->uses())
7545 if (!DT.dominates(BBE: NoWrapEdge, U: RU))
7546 return false;
7547 }
7548
7549 return true;
7550 };
7551
7552 return llvm::any_of(Range&: GuardingBranches, P: AllUsesGuardedByBranch);
7553}
7554
7555/// Shifts return poison if shiftwidth is larger than the bitwidth.
7556static bool shiftAmountKnownInRange(const Value *ShiftAmount) {
7557 auto *C = dyn_cast<Constant>(Val: ShiftAmount);
7558 if (!C)
7559 return false;
7560
7561 // Shifts return poison if shiftwidth is larger than the bitwidth.
7562 SmallVector<const Constant *, 4> ShiftAmounts;
7563 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: C->getType())) {
7564 unsigned NumElts = FVTy->getNumElements();
7565 for (unsigned i = 0; i < NumElts; ++i)
7566 ShiftAmounts.push_back(Elt: C->getAggregateElement(Elt: i));
7567 } else if (isa<ScalableVectorType>(Val: C->getType()))
7568 return false; // Can't tell, just return false to be safe
7569 else
7570 ShiftAmounts.push_back(Elt: C);
7571
7572 bool Safe = llvm::all_of(Range&: ShiftAmounts, P: [](const Constant *C) {
7573 auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
7574 return CI && CI->getValue().ult(RHS: C->getType()->getIntegerBitWidth());
7575 });
7576
7577 return Safe;
7578}
7579
7580enum class UndefPoisonKind {
7581 PoisonOnly = (1 << 0),
7582 UndefOnly = (1 << 1),
7583 UndefOrPoison = PoisonOnly | UndefOnly,
7584};
7585
7586static bool includesPoison(UndefPoisonKind Kind) {
7587 return (unsigned(Kind) & unsigned(UndefPoisonKind::PoisonOnly)) != 0;
7588}
7589
7590static bool includesUndef(UndefPoisonKind Kind) {
7591 return (unsigned(Kind) & unsigned(UndefPoisonKind::UndefOnly)) != 0;
7592}
7593
7594static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
7595 bool ConsiderFlagsAndMetadata) {
7596
7597 if (ConsiderFlagsAndMetadata && includesPoison(Kind) &&
7598 Op->hasPoisonGeneratingAnnotations())
7599 return true;
7600
7601 unsigned Opcode = Op->getOpcode();
7602
7603 // Check whether opcode is a poison/undef-generating operation
7604 switch (Opcode) {
7605 case Instruction::Shl:
7606 case Instruction::AShr:
7607 case Instruction::LShr:
7608 return includesPoison(Kind) && !shiftAmountKnownInRange(ShiftAmount: Op->getOperand(i: 1));
7609 case Instruction::FPToSI:
7610 case Instruction::FPToUI:
7611 // fptosi/ui yields poison if the resulting value does not fit in the
7612 // destination type.
7613 return true;
7614 case Instruction::Call:
7615 if (auto *II = dyn_cast<IntrinsicInst>(Val: Op)) {
7616 switch (II->getIntrinsicID()) {
7617 // NOTE: Use IntrNoCreateUndefOrPoison when possible.
7618 case Intrinsic::ctlz:
7619 case Intrinsic::cttz:
7620 case Intrinsic::abs:
7621 // We're not considering flags so it is safe to just return false.
7622 return false;
7623 case Intrinsic::sshl_sat:
7624 case Intrinsic::ushl_sat:
7625 if (!includesPoison(Kind) ||
7626 shiftAmountKnownInRange(ShiftAmount: II->getArgOperand(i: 1)))
7627 return false;
7628 break;
7629 }
7630 }
7631 [[fallthrough]];
7632 case Instruction::CallBr:
7633 case Instruction::Invoke: {
7634 const auto *CB = cast<CallBase>(Val: Op);
7635 return !CB->hasRetAttr(Kind: Attribute::NoUndef) &&
7636 !CB->hasFnAttr(Kind: Attribute::NoCreateUndefOrPoison);
7637 }
7638 case Instruction::InsertElement:
7639 case Instruction::ExtractElement: {
7640 // If index exceeds the length of the vector, it returns poison
7641 auto *VTy = cast<VectorType>(Val: Op->getOperand(i: 0)->getType());
7642 unsigned IdxOp = Op->getOpcode() == Instruction::InsertElement ? 2 : 1;
7643 auto *Idx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: IdxOp));
7644 if (includesPoison(Kind))
7645 return !Idx ||
7646 Idx->getValue().uge(RHS: VTy->getElementCount().getKnownMinValue());
7647 return false;
7648 }
7649 case Instruction::ShuffleVector: {
7650 ArrayRef<int> Mask = isa<ConstantExpr>(Val: Op)
7651 ? cast<ConstantExpr>(Val: Op)->getShuffleMask()
7652 : cast<ShuffleVectorInst>(Val: Op)->getShuffleMask();
7653 return includesPoison(Kind) && is_contained(Range&: Mask, Element: PoisonMaskElem);
7654 }
7655 case Instruction::FNeg:
7656 case Instruction::PHI:
7657 case Instruction::Select:
7658 case Instruction::ExtractValue:
7659 case Instruction::InsertValue:
7660 case Instruction::Freeze:
7661 case Instruction::ICmp:
7662 case Instruction::FCmp:
7663 case Instruction::GetElementPtr:
7664 return false;
7665 case Instruction::AddrSpaceCast:
7666 return true;
7667 default: {
7668 const auto *CE = dyn_cast<ConstantExpr>(Val: Op);
7669 if (isa<CastInst>(Val: Op) || (CE && CE->isCast()))
7670 return false;
7671 else if (Instruction::isBinaryOp(Opcode))
7672 return false;
7673 // Be conservative and return true.
7674 return true;
7675 }
7676 }
7677}
7678
7679bool llvm::canCreateUndefOrPoison(const Operator *Op,
7680 bool ConsiderFlagsAndMetadata) {
7681 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::UndefOrPoison,
7682 ConsiderFlagsAndMetadata);
7683}
7684
7685bool llvm::canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata) {
7686 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::PoisonOnly,
7687 ConsiderFlagsAndMetadata);
7688}
7689
7690static bool directlyImpliesPoison(const Value *ValAssumedPoison, const Value *V,
7691 unsigned Depth) {
7692 if (ValAssumedPoison == V)
7693 return true;
7694
7695 const unsigned MaxDepth = 2;
7696 if (Depth >= MaxDepth)
7697 return false;
7698
7699 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
7700 if (any_of(Range: I->operands(), P: [=](const Use &Op) {
7701 return propagatesPoison(PoisonOp: Op) &&
7702 directlyImpliesPoison(ValAssumedPoison, V: Op, Depth: Depth + 1);
7703 }))
7704 return true;
7705
7706 // V = extractvalue V0, idx
7707 // V2 = extractvalue V0, idx2
7708 // V0's elements are all poison or not. (e.g., add_with_overflow)
7709 const WithOverflowInst *II;
7710 if (match(V: I, P: m_ExtractValue(V: m_WithOverflowInst(I&: II))) &&
7711 (match(V: ValAssumedPoison, P: m_ExtractValue(V: m_Specific(V: II))) ||
7712 llvm::is_contained(Range: II->args(), Element: ValAssumedPoison)))
7713 return true;
7714 }
7715 return false;
7716}
7717
7718static bool impliesPoison(const Value *ValAssumedPoison, const Value *V,
7719 unsigned Depth) {
7720 if (isGuaranteedNotToBePoison(V: ValAssumedPoison))
7721 return true;
7722
7723 if (directlyImpliesPoison(ValAssumedPoison, V, /* Depth */ 0))
7724 return true;
7725
7726 const unsigned MaxDepth = 2;
7727 if (Depth >= MaxDepth)
7728 return false;
7729
7730 const auto *I = dyn_cast<Instruction>(Val: ValAssumedPoison);
7731 if (I && !canCreatePoison(Op: cast<Operator>(Val: I))) {
7732 return all_of(Range: I->operands(), P: [=](const Value *Op) {
7733 return impliesPoison(ValAssumedPoison: Op, V, Depth: Depth + 1);
7734 });
7735 }
7736 return false;
7737}
7738
7739bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
7740 return ::impliesPoison(ValAssumedPoison, V, /* Depth */ 0);
7741}
7742
7743static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
7744
7745static bool isGuaranteedNotToBeUndefOrPoison(
7746 const Value *V, AssumptionCache *AC, const Instruction *CtxI,
7747 const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
7748 if (Depth >= MaxAnalysisRecursionDepth)
7749 return false;
7750
7751 if (isa<MetadataAsValue>(Val: V))
7752 return false;
7753
7754 if (const auto *A = dyn_cast<Argument>(Val: V)) {
7755 if (A->hasAttribute(Kind: Attribute::NoUndef) ||
7756 A->hasAttribute(Kind: Attribute::Dereferenceable) ||
7757 A->hasAttribute(Kind: Attribute::DereferenceableOrNull))
7758 return true;
7759 }
7760
7761 if (auto *C = dyn_cast<Constant>(Val: V)) {
7762 if (isa<PoisonValue>(Val: C))
7763 return !includesPoison(Kind);
7764
7765 if (isa<UndefValue>(Val: C))
7766 return !includesUndef(Kind);
7767
7768 if (isa<ConstantInt>(Val: C) || isa<GlobalVariable>(Val: C) || isa<ConstantFP>(Val: C) ||
7769 isa<ConstantPointerNull>(Val: C) || isa<Function>(Val: C))
7770 return true;
7771
7772 if (C->getType()->isVectorTy()) {
7773 if (isa<ConstantExpr>(Val: C)) {
7774 // Scalable vectors can use a ConstantExpr to build a splat.
7775 if (Constant *SplatC = C->getSplatValue())
7776 if (isa<ConstantInt>(Val: SplatC) || isa<ConstantFP>(Val: SplatC))
7777 return true;
7778 } else {
7779 if (includesUndef(Kind) && C->containsUndefElement())
7780 return false;
7781 if (includesPoison(Kind) && C->containsPoisonElement())
7782 return false;
7783 return !C->containsConstantExpression();
7784 }
7785 }
7786 }
7787
7788 // Strip cast operations from a pointer value.
7789 // Note that stripPointerCastsSameRepresentation can strip off getelementptr
7790 // inbounds with zero offset. To guarantee that the result isn't poison, the
7791 // stripped pointer is checked as it has to be pointing into an allocated
7792 // object or be null `null` to ensure `inbounds` getelement pointers with a
7793 // zero offset could not produce poison.
7794 // It can strip off addrspacecast that do not change bit representation as
7795 // well. We believe that such addrspacecast is equivalent to no-op.
7796 auto *StrippedV = V->stripPointerCastsSameRepresentation();
7797 if (isa<AllocaInst>(Val: StrippedV) || isa<GlobalVariable>(Val: StrippedV) ||
7798 isa<Function>(Val: StrippedV) || isa<ConstantPointerNull>(Val: StrippedV))
7799 return true;
7800
7801 auto OpCheck = [&](const Value *V) {
7802 return isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth: Depth + 1, Kind);
7803 };
7804
7805 if (auto *Opr = dyn_cast<Operator>(Val: V)) {
7806 // If the value is a freeze instruction, then it can never
7807 // be undef or poison.
7808 if (isa<FreezeInst>(Val: V))
7809 return true;
7810
7811 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
7812 if (CB->hasRetAttr(Kind: Attribute::NoUndef) ||
7813 CB->hasRetAttr(Kind: Attribute::Dereferenceable) ||
7814 CB->hasRetAttr(Kind: Attribute::DereferenceableOrNull))
7815 return true;
7816 }
7817
7818 if (!::canCreateUndefOrPoison(Op: Opr, Kind,
7819 /*ConsiderFlagsAndMetadata=*/true)) {
7820 if (const auto *PN = dyn_cast<PHINode>(Val: V)) {
7821 unsigned Num = PN->getNumIncomingValues();
7822 bool IsWellDefined = true;
7823 for (unsigned i = 0; i < Num; ++i) {
7824 if (PN == PN->getIncomingValue(i))
7825 continue;
7826 auto *TI = PN->getIncomingBlock(i)->getTerminator();
7827 if (!isGuaranteedNotToBeUndefOrPoison(V: PN->getIncomingValue(i), AC, CtxI: TI,
7828 DT, Depth: Depth + 1, Kind)) {
7829 IsWellDefined = false;
7830 break;
7831 }
7832 }
7833 if (IsWellDefined)
7834 return true;
7835 } else if (auto *Splat = isa<ShuffleVectorInst>(Val: Opr) ? getSplatValue(V: Opr)
7836 : nullptr) {
7837 // For splats we only need to check the value being splatted.
7838 if (OpCheck(Splat))
7839 return true;
7840 } else if (all_of(Range: Opr->operands(), P: OpCheck))
7841 return true;
7842 }
7843 }
7844
7845 if (auto *I = dyn_cast<LoadInst>(Val: V))
7846 if (I->hasMetadata(KindID: LLVMContext::MD_noundef) ||
7847 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable) ||
7848 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable_or_null))
7849 return true;
7850
7851 if (programUndefinedIfUndefOrPoison(V, PoisonOnly: !includesUndef(Kind)))
7852 return true;
7853
7854 // CxtI may be null or a cloned instruction.
7855 if (!CtxI || !CtxI->getParent() || !DT)
7856 return false;
7857
7858 auto *DNode = DT->getNode(BB: CtxI->getParent());
7859 if (!DNode)
7860 // Unreachable block
7861 return false;
7862
7863 // If V is used as a branch condition before reaching CtxI, V cannot be
7864 // undef or poison.
7865 // br V, BB1, BB2
7866 // BB1:
7867 // CtxI ; V cannot be undef or poison here
7868 auto *Dominator = DNode->getIDom();
7869 // This check is purely for compile time reasons: we can skip the IDom walk
7870 // if what we are checking for includes undef and the value is not an integer.
7871 if (!includesUndef(Kind) || V->getType()->isIntegerTy())
7872 while (Dominator) {
7873 auto *TI = Dominator->getBlock()->getTerminator();
7874
7875 Value *Cond = nullptr;
7876 if (auto BI = dyn_cast_or_null<CondBrInst>(Val: TI)) {
7877 Cond = BI->getCondition();
7878 } else if (auto SI = dyn_cast_or_null<SwitchInst>(Val: TI)) {
7879 Cond = SI->getCondition();
7880 }
7881
7882 if (Cond) {
7883 if (Cond == V)
7884 return true;
7885 else if (!includesUndef(Kind) && isa<Operator>(Val: Cond)) {
7886 // For poison, we can analyze further
7887 auto *Opr = cast<Operator>(Val: Cond);
7888 if (any_of(Range: Opr->operands(), P: [V](const Use &U) {
7889 return V == U && propagatesPoison(PoisonOp: U);
7890 }))
7891 return true;
7892 }
7893 }
7894
7895 Dominator = Dominator->getIDom();
7896 }
7897
7898 if (AC && getKnowledgeValidInContext(V, AttrKinds: {Attribute::NoUndef}, AC&: *AC, CtxI, DT))
7899 return true;
7900
7901 return false;
7902}
7903
7904bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC,
7905 const Instruction *CtxI,
7906 const DominatorTree *DT,
7907 unsigned Depth) {
7908 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
7909 Kind: UndefPoisonKind::UndefOrPoison);
7910}
7911
7912bool llvm::isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC,
7913 const Instruction *CtxI,
7914 const DominatorTree *DT, unsigned Depth) {
7915 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
7916 Kind: UndefPoisonKind::PoisonOnly);
7917}
7918
7919bool llvm::isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC,
7920 const Instruction *CtxI,
7921 const DominatorTree *DT, unsigned Depth) {
7922 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
7923 Kind: UndefPoisonKind::UndefOnly);
7924}
7925
7926/// Return true if undefined behavior would provably be executed on the path to
7927/// OnPathTo if Root produced a posion result. Note that this doesn't say
7928/// anything about whether OnPathTo is actually executed or whether Root is
7929/// actually poison. This can be used to assess whether a new use of Root can
7930/// be added at a location which is control equivalent with OnPathTo (such as
7931/// immediately before it) without introducing UB which didn't previously
7932/// exist. Note that a false result conveys no information.
7933bool llvm::mustExecuteUBIfPoisonOnPathTo(Instruction *Root,
7934 Instruction *OnPathTo,
7935 DominatorTree *DT) {
7936 // Basic approach is to assume Root is poison, propagate poison forward
7937 // through all users we can easily track, and then check whether any of those
7938 // users are provable UB and must execute before out exiting block might
7939 // exit.
7940
7941 // The set of all recursive users we've visited (which are assumed to all be
7942 // poison because of said visit)
7943 SmallPtrSet<const Value *, 16> KnownPoison;
7944 SmallVector<const Instruction*, 16> Worklist;
7945 Worklist.push_back(Elt: Root);
7946 while (!Worklist.empty()) {
7947 const Instruction *I = Worklist.pop_back_val();
7948
7949 // If we know this must trigger UB on a path leading our target.
7950 if (mustTriggerUB(I, KnownPoison) && DT->dominates(Def: I, User: OnPathTo))
7951 return true;
7952
7953 // If we can't analyze propagation through this instruction, just skip it
7954 // and transitive users. Safe as false is a conservative result.
7955 if (I != Root && !any_of(Range: I->operands(), P: [&KnownPoison](const Use &U) {
7956 return KnownPoison.contains(Ptr: U) && propagatesPoison(PoisonOp: U);
7957 }))
7958 continue;
7959
7960 if (KnownPoison.insert(Ptr: I).second)
7961 for (const User *User : I->users())
7962 Worklist.push_back(Elt: cast<Instruction>(Val: User));
7963 }
7964
7965 // Might be non-UB, or might have a path we couldn't prove must execute on
7966 // way to exiting bb.
7967 return false;
7968}
7969
7970OverflowResult llvm::computeOverflowForSignedAdd(const AddOperator *Add,
7971 const SimplifyQuery &SQ) {
7972 return ::computeOverflowForSignedAdd(LHS: Add->getOperand(i_nocapture: 0), RHS: Add->getOperand(i_nocapture: 1),
7973 Add, SQ);
7974}
7975
7976OverflowResult
7977llvm::computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
7978 const WithCache<const Value *> &RHS,
7979 const SimplifyQuery &SQ) {
7980 return ::computeOverflowForSignedAdd(LHS, RHS, Add: nullptr, SQ);
7981}
7982
7983bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
7984 // Note: An atomic operation isn't guaranteed to return in a reasonable amount
7985 // of time because it's possible for another thread to interfere with it for an
7986 // arbitrary length of time, but programs aren't allowed to rely on that.
7987
7988 // If there is no successor, then execution can't transfer to it.
7989 if (isa<ReturnInst>(Val: I))
7990 return false;
7991 if (isa<UnreachableInst>(Val: I))
7992 return false;
7993
7994 // Note: Do not add new checks here; instead, change Instruction::mayThrow or
7995 // Instruction::willReturn.
7996 //
7997 // FIXME: Move this check into Instruction::willReturn.
7998 if (isa<CatchPadInst>(Val: I)) {
7999 switch (classifyEHPersonality(Pers: I->getFunction()->getPersonalityFn())) {
8000 default:
8001 // A catchpad may invoke exception object constructors and such, which
8002 // in some languages can be arbitrary code, so be conservative by default.
8003 return false;
8004 case EHPersonality::CoreCLR:
8005 // For CoreCLR, it just involves a type test.
8006 return true;
8007 }
8008 }
8009
8010 // An instruction that returns without throwing must transfer control flow
8011 // to a successor.
8012 return !I->mayThrow() && I->willReturn();
8013}
8014
8015bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) {
8016 // TODO: This is slightly conservative for invoke instruction since exiting
8017 // via an exception *is* normal control for them.
8018 for (const Instruction &I : *BB)
8019 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8020 return false;
8021 return true;
8022}
8023
8024bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8025 BasicBlock::const_iterator Begin, BasicBlock::const_iterator End,
8026 unsigned ScanLimit) {
8027 return isGuaranteedToTransferExecutionToSuccessor(Range: make_range(x: Begin, y: End),
8028 ScanLimit);
8029}
8030
8031bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8032 iterator_range<BasicBlock::const_iterator> Range, unsigned ScanLimit) {
8033 assert(ScanLimit && "scan limit must be non-zero");
8034 for (const Instruction &I : Range) {
8035 if (--ScanLimit == 0)
8036 return false;
8037 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8038 return false;
8039 }
8040 return true;
8041}
8042
8043bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
8044 const Loop *L) {
8045 // The loop header is guaranteed to be executed for every iteration.
8046 //
8047 // FIXME: Relax this constraint to cover all basic blocks that are
8048 // guaranteed to be executed at every iteration.
8049 if (I->getParent() != L->getHeader()) return false;
8050
8051 for (const Instruction &LI : *L->getHeader()) {
8052 if (&LI == I) return true;
8053 if (!isGuaranteedToTransferExecutionToSuccessor(I: &LI)) return false;
8054 }
8055 llvm_unreachable("Instruction not contained in its own parent basic block.");
8056}
8057
8058bool llvm::intrinsicPropagatesPoison(Intrinsic::ID IID) {
8059 switch (IID) {
8060 // TODO: Add more intrinsics.
8061 case Intrinsic::sadd_with_overflow:
8062 case Intrinsic::ssub_with_overflow:
8063 case Intrinsic::smul_with_overflow:
8064 case Intrinsic::uadd_with_overflow:
8065 case Intrinsic::usub_with_overflow:
8066 case Intrinsic::umul_with_overflow:
8067 // If an input is a vector containing a poison element, the
8068 // two output vectors (calculated results, overflow bits)'
8069 // corresponding lanes are poison.
8070 return true;
8071 case Intrinsic::ctpop:
8072 case Intrinsic::ctlz:
8073 case Intrinsic::cttz:
8074 case Intrinsic::abs:
8075 case Intrinsic::smax:
8076 case Intrinsic::smin:
8077 case Intrinsic::umax:
8078 case Intrinsic::umin:
8079 case Intrinsic::scmp:
8080 case Intrinsic::is_fpclass:
8081 case Intrinsic::ptrmask:
8082 case Intrinsic::ucmp:
8083 case Intrinsic::bitreverse:
8084 case Intrinsic::bswap:
8085 case Intrinsic::sadd_sat:
8086 case Intrinsic::ssub_sat:
8087 case Intrinsic::sshl_sat:
8088 case Intrinsic::uadd_sat:
8089 case Intrinsic::usub_sat:
8090 case Intrinsic::ushl_sat:
8091 case Intrinsic::smul_fix:
8092 case Intrinsic::smul_fix_sat:
8093 case Intrinsic::umul_fix:
8094 case Intrinsic::umul_fix_sat:
8095 case Intrinsic::pow:
8096 case Intrinsic::powi:
8097 case Intrinsic::sin:
8098 case Intrinsic::sinh:
8099 case Intrinsic::cos:
8100 case Intrinsic::cosh:
8101 case Intrinsic::sincos:
8102 case Intrinsic::sincospi:
8103 case Intrinsic::tan:
8104 case Intrinsic::tanh:
8105 case Intrinsic::asin:
8106 case Intrinsic::acos:
8107 case Intrinsic::atan:
8108 case Intrinsic::atan2:
8109 case Intrinsic::canonicalize:
8110 case Intrinsic::sqrt:
8111 case Intrinsic::exp:
8112 case Intrinsic::exp2:
8113 case Intrinsic::exp10:
8114 case Intrinsic::log:
8115 case Intrinsic::log2:
8116 case Intrinsic::log10:
8117 case Intrinsic::modf:
8118 case Intrinsic::floor:
8119 case Intrinsic::ceil:
8120 case Intrinsic::trunc:
8121 case Intrinsic::rint:
8122 case Intrinsic::nearbyint:
8123 case Intrinsic::round:
8124 case Intrinsic::roundeven:
8125 case Intrinsic::lrint:
8126 case Intrinsic::llrint:
8127 case Intrinsic::fshl:
8128 case Intrinsic::fshr:
8129 return true;
8130 default:
8131 return false;
8132 }
8133}
8134
8135bool llvm::propagatesPoison(const Use &PoisonOp) {
8136 const Operator *I = cast<Operator>(Val: PoisonOp.getUser());
8137 switch (I->getOpcode()) {
8138 case Instruction::Freeze:
8139 case Instruction::PHI:
8140 case Instruction::Invoke:
8141 return false;
8142 case Instruction::Select:
8143 return PoisonOp.getOperandNo() == 0;
8144 case Instruction::Call:
8145 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
8146 return intrinsicPropagatesPoison(IID: II->getIntrinsicID());
8147 return false;
8148 case Instruction::ICmp:
8149 case Instruction::FCmp:
8150 case Instruction::GetElementPtr:
8151 return true;
8152 default:
8153 if (isa<BinaryOperator>(Val: I) || isa<UnaryOperator>(Val: I) || isa<CastInst>(Val: I))
8154 return true;
8155
8156 // Be conservative and return false.
8157 return false;
8158 }
8159}
8160
8161/// Enumerates all operands of \p I that are guaranteed to not be undef or
8162/// poison. If the callback \p Handle returns true, stop processing and return
8163/// true. Otherwise, return false.
8164template <typename CallableT>
8165static bool handleGuaranteedWellDefinedOps(const Instruction *I,
8166 const CallableT &Handle) {
8167 switch (I->getOpcode()) {
8168 case Instruction::Store:
8169 if (Handle(cast<StoreInst>(Val: I)->getPointerOperand()))
8170 return true;
8171 break;
8172
8173 case Instruction::Load:
8174 if (Handle(cast<LoadInst>(Val: I)->getPointerOperand()))
8175 return true;
8176 break;
8177
8178 // Since dereferenceable attribute imply noundef, atomic operations
8179 // also implicitly have noundef pointers too
8180 case Instruction::AtomicCmpXchg:
8181 if (Handle(cast<AtomicCmpXchgInst>(Val: I)->getPointerOperand()))
8182 return true;
8183 break;
8184
8185 case Instruction::AtomicRMW:
8186 if (Handle(cast<AtomicRMWInst>(Val: I)->getPointerOperand()))
8187 return true;
8188 break;
8189
8190 case Instruction::Call:
8191 case Instruction::Invoke: {
8192 const CallBase *CB = cast<CallBase>(Val: I);
8193 if (CB->isIndirectCall() && Handle(CB->getCalledOperand()))
8194 return true;
8195 for (unsigned i = 0; i < CB->arg_size(); ++i)
8196 if ((CB->paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef) ||
8197 CB->paramHasAttr(ArgNo: i, Kind: Attribute::Dereferenceable) ||
8198 CB->paramHasAttr(ArgNo: i, Kind: Attribute::DereferenceableOrNull)) &&
8199 Handle(CB->getArgOperand(i)))
8200 return true;
8201 break;
8202 }
8203 case Instruction::Ret:
8204 if (I->getFunction()->hasRetAttribute(Kind: Attribute::NoUndef) &&
8205 Handle(I->getOperand(i: 0)))
8206 return true;
8207 break;
8208 case Instruction::Switch:
8209 if (Handle(cast<SwitchInst>(Val: I)->getCondition()))
8210 return true;
8211 break;
8212 case Instruction::CondBr:
8213 if (Handle(cast<CondBrInst>(Val: I)->getCondition()))
8214 return true;
8215 break;
8216 default:
8217 break;
8218 }
8219
8220 return false;
8221}
8222
8223/// Enumerates all operands of \p I that are guaranteed to not be poison.
8224template <typename CallableT>
8225static bool handleGuaranteedNonPoisonOps(const Instruction *I,
8226 const CallableT &Handle) {
8227 if (handleGuaranteedWellDefinedOps(I, Handle))
8228 return true;
8229 switch (I->getOpcode()) {
8230 // Divisors of these operations are allowed to be partially undef.
8231 case Instruction::UDiv:
8232 case Instruction::SDiv:
8233 case Instruction::URem:
8234 case Instruction::SRem:
8235 return Handle(I->getOperand(i: 1));
8236 default:
8237 return false;
8238 }
8239}
8240
8241bool llvm::mustTriggerUB(const Instruction *I,
8242 const SmallPtrSetImpl<const Value *> &KnownPoison) {
8243 return handleGuaranteedNonPoisonOps(
8244 I, Handle: [&](const Value *V) { return KnownPoison.count(Ptr: V); });
8245}
8246
8247static bool programUndefinedIfUndefOrPoison(const Value *V,
8248 bool PoisonOnly) {
8249 // We currently only look for uses of values within the same basic
8250 // block, as that makes it easier to guarantee that the uses will be
8251 // executed given that Inst is executed.
8252 //
8253 // FIXME: Expand this to consider uses beyond the same basic block. To do
8254 // this, look out for the distinction between post-dominance and strong
8255 // post-dominance.
8256 const BasicBlock *BB = nullptr;
8257 BasicBlock::const_iterator Begin;
8258 if (const auto *Inst = dyn_cast<Instruction>(Val: V)) {
8259 BB = Inst->getParent();
8260 Begin = Inst->getIterator();
8261 Begin++;
8262 } else if (const auto *Arg = dyn_cast<Argument>(Val: V)) {
8263 if (Arg->getParent()->isDeclaration())
8264 return false;
8265 BB = &Arg->getParent()->getEntryBlock();
8266 Begin = BB->begin();
8267 } else {
8268 return false;
8269 }
8270
8271 // Limit number of instructions we look at, to avoid scanning through large
8272 // blocks. The current limit is chosen arbitrarily.
8273 unsigned ScanLimit = 32;
8274 BasicBlock::const_iterator End = BB->end();
8275
8276 if (!PoisonOnly) {
8277 // Since undef does not propagate eagerly, be conservative & just check
8278 // whether a value is directly passed to an instruction that must take
8279 // well-defined operands.
8280
8281 for (const auto &I : make_range(x: Begin, y: End)) {
8282 if (--ScanLimit == 0)
8283 break;
8284
8285 if (handleGuaranteedWellDefinedOps(I: &I, Handle: [V](const Value *WellDefinedOp) {
8286 return WellDefinedOp == V;
8287 }))
8288 return true;
8289
8290 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8291 break;
8292 }
8293 return false;
8294 }
8295
8296 // Set of instructions that we have proved will yield poison if Inst
8297 // does.
8298 SmallPtrSet<const Value *, 16> YieldsPoison;
8299 SmallPtrSet<const BasicBlock *, 4> Visited;
8300
8301 YieldsPoison.insert(Ptr: V);
8302 Visited.insert(Ptr: BB);
8303
8304 while (true) {
8305 for (const auto &I : make_range(x: Begin, y: End)) {
8306 if (--ScanLimit == 0)
8307 return false;
8308 if (mustTriggerUB(I: &I, KnownPoison: YieldsPoison))
8309 return true;
8310 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8311 return false;
8312
8313 // If an operand is poison and propagates it, mark I as yielding poison.
8314 for (const Use &Op : I.operands()) {
8315 if (YieldsPoison.count(Ptr: Op) && propagatesPoison(PoisonOp: Op)) {
8316 YieldsPoison.insert(Ptr: &I);
8317 break;
8318 }
8319 }
8320
8321 // Special handling for select, which returns poison if its operand 0 is
8322 // poison (handled in the loop above) *or* if both its true/false operands
8323 // are poison (handled here).
8324 if (I.getOpcode() == Instruction::Select &&
8325 YieldsPoison.count(Ptr: I.getOperand(i: 1)) &&
8326 YieldsPoison.count(Ptr: I.getOperand(i: 2))) {
8327 YieldsPoison.insert(Ptr: &I);
8328 }
8329 }
8330
8331 BB = BB->getSingleSuccessor();
8332 if (!BB || !Visited.insert(Ptr: BB).second)
8333 break;
8334
8335 Begin = BB->getFirstNonPHIIt();
8336 End = BB->end();
8337 }
8338 return false;
8339}
8340
8341bool llvm::programUndefinedIfUndefOrPoison(const Instruction *Inst) {
8342 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: false);
8343}
8344
8345bool llvm::programUndefinedIfPoison(const Instruction *Inst) {
8346 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: true);
8347}
8348
8349static bool isKnownNonNaN(const Value *V, FastMathFlags FMF) {
8350 if (FMF.noNaNs())
8351 return true;
8352
8353 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8354 return !C->isNaN();
8355
8356 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8357 if (!C->getElementType()->isFloatingPointTy())
8358 return false;
8359 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8360 if (C->getElementAsAPFloat(i: I).isNaN())
8361 return false;
8362 }
8363 return true;
8364 }
8365
8366 if (isa<ConstantAggregateZero>(Val: V))
8367 return true;
8368
8369 return false;
8370}
8371
8372static bool isKnownNonZero(const Value *V) {
8373 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8374 return !C->isZero();
8375
8376 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8377 if (!C->getElementType()->isFloatingPointTy())
8378 return false;
8379 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8380 if (C->getElementAsAPFloat(i: I).isZero())
8381 return false;
8382 }
8383 return true;
8384 }
8385
8386 return false;
8387}
8388
8389/// Match clamp pattern for float types without care about NaNs or signed zeros.
8390/// Given non-min/max outer cmp/select from the clamp pattern this
8391/// function recognizes if it can be substitued by a "canonical" min/max
8392/// pattern.
8393static SelectPatternResult matchFastFloatClamp(CmpInst::Predicate Pred,
8394 Value *CmpLHS, Value *CmpRHS,
8395 Value *TrueVal, Value *FalseVal,
8396 Value *&LHS, Value *&RHS) {
8397 // Try to match
8398 // X < C1 ? C1 : Min(X, C2) --> Max(C1, Min(X, C2))
8399 // X > C1 ? C1 : Max(X, C2) --> Min(C1, Max(X, C2))
8400 // and return description of the outer Max/Min.
8401
8402 // First, check if select has inverse order:
8403 if (CmpRHS == FalseVal) {
8404 std::swap(a&: TrueVal, b&: FalseVal);
8405 Pred = CmpInst::getInversePredicate(pred: Pred);
8406 }
8407
8408 // Assume success now. If there's no match, callers should not use these anyway.
8409 LHS = TrueVal;
8410 RHS = FalseVal;
8411
8412 const APFloat *FC1;
8413 if (CmpRHS != TrueVal || !match(V: CmpRHS, P: m_APFloat(Res&: FC1)) || !FC1->isFinite())
8414 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8415
8416 const APFloat *FC2;
8417 switch (Pred) {
8418 case CmpInst::FCMP_OLT:
8419 case CmpInst::FCMP_OLE:
8420 case CmpInst::FCMP_ULT:
8421 case CmpInst::FCMP_ULE:
8422 if (match(V: FalseVal, P: m_OrdOrUnordFMin(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8423 *FC1 < *FC2)
8424 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8425 break;
8426 case CmpInst::FCMP_OGT:
8427 case CmpInst::FCMP_OGE:
8428 case CmpInst::FCMP_UGT:
8429 case CmpInst::FCMP_UGE:
8430 if (match(V: FalseVal, P: m_OrdOrUnordFMax(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8431 *FC1 > *FC2)
8432 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8433 break;
8434 default:
8435 break;
8436 }
8437
8438 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8439}
8440
8441/// Recognize variations of:
8442/// CLAMP(v,l,h) ==> ((v) < (l) ? (l) : ((v) > (h) ? (h) : (v)))
8443static SelectPatternResult matchClamp(CmpInst::Predicate Pred,
8444 Value *CmpLHS, Value *CmpRHS,
8445 Value *TrueVal, Value *FalseVal) {
8446 // Swap the select operands and predicate to match the patterns below.
8447 if (CmpRHS != TrueVal) {
8448 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8449 std::swap(a&: TrueVal, b&: FalseVal);
8450 }
8451 const APInt *C1;
8452 if (CmpRHS == TrueVal && match(V: CmpRHS, P: m_APInt(Res&: C1))) {
8453 const APInt *C2;
8454 // (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
8455 if (match(V: FalseVal, P: m_SMin(L: m_Specific(V: CmpLHS), R: m_APInt(Res&: C2))) &&
8456 C1->slt(RHS: *C2) && Pred == CmpInst::ICMP_SLT)
8457 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8458
8459 // (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
8460 if (match(V: FalseVal, P: m_SMax(L: m_Specific(V: CmpLHS), R: m_APInt(Res&: C2))) &&
8461 C1->sgt(RHS: *C2) && Pred == CmpInst::ICMP_SGT)
8462 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8463
8464 // (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
8465 if (match(V: FalseVal, P: m_UMin(L: m_Specific(V: CmpLHS), R: m_APInt(Res&: C2))) &&
8466 C1->ult(RHS: *C2) && Pred == CmpInst::ICMP_ULT)
8467 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8468
8469 // (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
8470 if (match(V: FalseVal, P: m_UMax(L: m_Specific(V: CmpLHS), R: m_APInt(Res&: C2))) &&
8471 C1->ugt(RHS: *C2) && Pred == CmpInst::ICMP_UGT)
8472 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8473 }
8474 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8475}
8476
8477/// Recognize variations of:
8478/// a < c ? min(a,b) : min(b,c) ==> min(min(a,b),min(b,c))
8479static SelectPatternResult matchMinMaxOfMinMax(CmpInst::Predicate Pred,
8480 Value *CmpLHS, Value *CmpRHS,
8481 Value *TVal, Value *FVal,
8482 unsigned Depth) {
8483 // TODO: Allow FP min/max with nnan/nsz.
8484 assert(CmpInst::isIntPredicate(Pred) && "Expected integer comparison");
8485
8486 Value *A = nullptr, *B = nullptr;
8487 SelectPatternResult L = matchSelectPattern(V: TVal, LHS&: A, RHS&: B, CastOp: nullptr, Depth: Depth + 1);
8488 if (!SelectPatternResult::isMinOrMax(SPF: L.Flavor))
8489 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8490
8491 Value *C = nullptr, *D = nullptr;
8492 SelectPatternResult R = matchSelectPattern(V: FVal, LHS&: C, RHS&: D, CastOp: nullptr, Depth: Depth + 1);
8493 if (L.Flavor != R.Flavor)
8494 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8495
8496 // We have something like: x Pred y ? min(a, b) : min(c, d).
8497 // Try to match the compare to the min/max operations of the select operands.
8498 // First, make sure we have the right compare predicate.
8499 switch (L.Flavor) {
8500 case SPF_SMIN:
8501 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) {
8502 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8503 std::swap(a&: CmpLHS, b&: CmpRHS);
8504 }
8505 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
8506 break;
8507 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8508 case SPF_SMAX:
8509 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) {
8510 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8511 std::swap(a&: CmpLHS, b&: CmpRHS);
8512 }
8513 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)
8514 break;
8515 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8516 case SPF_UMIN:
8517 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
8518 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8519 std::swap(a&: CmpLHS, b&: CmpRHS);
8520 }
8521 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
8522 break;
8523 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8524 case SPF_UMAX:
8525 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
8526 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8527 std::swap(a&: CmpLHS, b&: CmpRHS);
8528 }
8529 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
8530 break;
8531 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8532 default:
8533 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8534 }
8535
8536 // If there is a common operand in the already matched min/max and the other
8537 // min/max operands match the compare operands (either directly or inverted),
8538 // then this is min/max of the same flavor.
8539
8540 // a pred c ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
8541 // ~c pred ~a ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
8542 if (D == B) {
8543 if ((CmpLHS == A && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8544 match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
8545 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8546 }
8547 // a pred d ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
8548 // ~d pred ~a ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
8549 if (C == B) {
8550 if ((CmpLHS == A && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8551 match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
8552 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8553 }
8554 // b pred c ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
8555 // ~c pred ~b ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
8556 if (D == A) {
8557 if ((CmpLHS == B && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8558 match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
8559 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8560 }
8561 // b pred d ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
8562 // ~d pred ~b ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
8563 if (C == A) {
8564 if ((CmpLHS == B && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8565 match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
8566 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8567 }
8568
8569 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8570}
8571
8572/// If the input value is the result of a 'not' op, constant integer, or vector
8573/// splat of a constant integer, return the bitwise-not source value.
8574/// TODO: This could be extended to handle non-splat vector integer constants.
8575static Value *getNotValue(Value *V) {
8576 Value *NotV;
8577 if (match(V, P: m_Not(V: m_Value(V&: NotV))))
8578 return NotV;
8579
8580 const APInt *C;
8581 if (match(V, P: m_APInt(Res&: C)))
8582 return ConstantInt::get(Ty: V->getType(), V: ~(*C));
8583
8584 return nullptr;
8585}
8586
8587/// Match non-obvious integer minimum and maximum sequences.
8588static SelectPatternResult matchMinMax(CmpInst::Predicate Pred,
8589 Value *CmpLHS, Value *CmpRHS,
8590 Value *TrueVal, Value *FalseVal,
8591 Value *&LHS, Value *&RHS,
8592 unsigned Depth) {
8593 // Assume success. If there's no match, callers should not use these anyway.
8594 LHS = TrueVal;
8595 RHS = FalseVal;
8596
8597 SelectPatternResult SPR = matchClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal);
8598 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
8599 return SPR;
8600
8601 SPR = matchMinMaxOfMinMax(Pred, CmpLHS, CmpRHS, TVal: TrueVal, FVal: FalseVal, Depth);
8602 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
8603 return SPR;
8604
8605 // Look through 'not' ops to find disguised min/max.
8606 // (X > Y) ? ~X : ~Y ==> (~X < ~Y) ? ~X : ~Y ==> MIN(~X, ~Y)
8607 // (X < Y) ? ~X : ~Y ==> (~X > ~Y) ? ~X : ~Y ==> MAX(~X, ~Y)
8608 if (CmpLHS == getNotValue(V: TrueVal) && CmpRHS == getNotValue(V: FalseVal)) {
8609 switch (Pred) {
8610 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8611 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8612 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8613 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8614 default: break;
8615 }
8616 }
8617
8618 // (X > Y) ? ~Y : ~X ==> (~X < ~Y) ? ~Y : ~X ==> MAX(~Y, ~X)
8619 // (X < Y) ? ~Y : ~X ==> (~X > ~Y) ? ~Y : ~X ==> MIN(~Y, ~X)
8620 if (CmpLHS == getNotValue(V: FalseVal) && CmpRHS == getNotValue(V: TrueVal)) {
8621 switch (Pred) {
8622 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8623 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8624 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8625 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8626 default: break;
8627 }
8628 }
8629
8630 if (Pred != CmpInst::ICMP_SGT && Pred != CmpInst::ICMP_SLT)
8631 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8632
8633 const APInt *C1;
8634 if (!match(V: CmpRHS, P: m_APInt(Res&: C1)))
8635 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8636
8637 // An unsigned min/max can be written with a signed compare.
8638 const APInt *C2;
8639 if ((CmpLHS == TrueVal && match(V: FalseVal, P: m_APInt(Res&: C2))) ||
8640 (CmpLHS == FalseVal && match(V: TrueVal, P: m_APInt(Res&: C2)))) {
8641 // Is the sign bit set?
8642 // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
8643 // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
8644 if (Pred == CmpInst::ICMP_SLT && C1->isZero() && C2->isMaxSignedValue())
8645 return {.Flavor: CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8646
8647 // Is the sign bit clear?
8648 // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
8649 // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
8650 if (Pred == CmpInst::ICMP_SGT && C1->isAllOnes() && C2->isMinSignedValue())
8651 return {.Flavor: CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8652 }
8653
8654 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8655}
8656
8657bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW,
8658 bool AllowPoison) {
8659 assert(X && Y && "Invalid operand");
8660
8661 auto IsNegationOf = [&](const Value *X, const Value *Y) {
8662 if (!match(V: X, P: m_Neg(V: m_Specific(V: Y))))
8663 return false;
8664
8665 auto *BO = cast<BinaryOperator>(Val: X);
8666 if (NeedNSW && !BO->hasNoSignedWrap())
8667 return false;
8668
8669 auto *Zero = cast<Constant>(Val: BO->getOperand(i_nocapture: 0));
8670 if (!AllowPoison && !Zero->isNullValue())
8671 return false;
8672
8673 return true;
8674 };
8675
8676 // X = -Y or Y = -X
8677 if (IsNegationOf(X, Y) || IsNegationOf(Y, X))
8678 return true;
8679
8680 // X = sub (A, B), Y = sub (B, A) || X = sub nsw (A, B), Y = sub nsw (B, A)
8681 Value *A, *B;
8682 return (!NeedNSW && (match(V: X, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
8683 match(V: Y, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))))) ||
8684 (NeedNSW && (match(V: X, P: m_NSWSub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
8685 match(V: Y, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A)))));
8686}
8687
8688bool llvm::isKnownInversion(const Value *X, const Value *Y) {
8689 // Handle X = icmp pred A, B, Y = icmp pred A, C.
8690 Value *A, *B, *C;
8691 CmpPredicate Pred1, Pred2;
8692 if (!match(V: X, P: m_ICmp(Pred&: Pred1, L: m_Value(V&: A), R: m_Value(V&: B))) ||
8693 !match(V: Y, P: m_c_ICmp(Pred&: Pred2, L: m_Specific(V: A), R: m_Value(V&: C))))
8694 return false;
8695
8696 // They must both have samesign flag or not.
8697 if (Pred1.hasSameSign() != Pred2.hasSameSign())
8698 return false;
8699
8700 if (B == C)
8701 return Pred1 == ICmpInst::getInversePredicate(pred: Pred2);
8702
8703 // Try to infer the relationship from constant ranges.
8704 const APInt *RHSC1, *RHSC2;
8705 if (!match(V: B, P: m_APInt(Res&: RHSC1)) || !match(V: C, P: m_APInt(Res&: RHSC2)))
8706 return false;
8707
8708 // Sign bits of two RHSCs should match.
8709 if (Pred1.hasSameSign() && RHSC1->isNonNegative() != RHSC2->isNonNegative())
8710 return false;
8711
8712 const auto CR1 = ConstantRange::makeExactICmpRegion(Pred: Pred1, Other: *RHSC1);
8713 const auto CR2 = ConstantRange::makeExactICmpRegion(Pred: Pred2, Other: *RHSC2);
8714
8715 return CR1.inverse() == CR2;
8716}
8717
8718SelectPatternResult llvm::getSelectPattern(CmpInst::Predicate Pred,
8719 SelectPatternNaNBehavior NaNBehavior,
8720 bool Ordered) {
8721 switch (Pred) {
8722 default:
8723 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false}; // Equality.
8724 case ICmpInst::ICMP_UGT:
8725 case ICmpInst::ICMP_UGE:
8726 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8727 case ICmpInst::ICMP_SGT:
8728 case ICmpInst::ICMP_SGE:
8729 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8730 case ICmpInst::ICMP_ULT:
8731 case ICmpInst::ICMP_ULE:
8732 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8733 case ICmpInst::ICMP_SLT:
8734 case ICmpInst::ICMP_SLE:
8735 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8736 case FCmpInst::FCMP_UGT:
8737 case FCmpInst::FCMP_UGE:
8738 case FCmpInst::FCMP_OGT:
8739 case FCmpInst::FCMP_OGE:
8740 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
8741 case FCmpInst::FCMP_ULT:
8742 case FCmpInst::FCMP_ULE:
8743 case FCmpInst::FCMP_OLT:
8744 case FCmpInst::FCMP_OLE:
8745 return {.Flavor: SPF_FMINNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
8746 }
8747}
8748
8749std::optional<std::pair<CmpPredicate, Constant *>>
8750llvm::getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred, Constant *C) {
8751 assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
8752 "Only for relational integer predicates.");
8753 if (isa<UndefValue>(Val: C))
8754 return std::nullopt;
8755
8756 Type *Type = C->getType();
8757 bool IsSigned = ICmpInst::isSigned(Pred);
8758
8759 CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred);
8760 bool WillIncrement =
8761 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
8762
8763 // Check if the constant operand can be safely incremented/decremented
8764 // without overflowing/underflowing.
8765 auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) {
8766 return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned);
8767 };
8768
8769 Constant *SafeReplacementConstant = nullptr;
8770 if (auto *CI = dyn_cast<ConstantInt>(Val: C)) {
8771 // Bail out if the constant can't be safely incremented/decremented.
8772 if (!ConstantIsOk(CI))
8773 return std::nullopt;
8774 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Type)) {
8775 unsigned NumElts = FVTy->getNumElements();
8776 for (unsigned i = 0; i != NumElts; ++i) {
8777 Constant *Elt = C->getAggregateElement(Elt: i);
8778 if (!Elt)
8779 return std::nullopt;
8780
8781 if (isa<UndefValue>(Val: Elt))
8782 continue;
8783
8784 // Bail out if we can't determine if this constant is min/max or if we
8785 // know that this constant is min/max.
8786 auto *CI = dyn_cast<ConstantInt>(Val: Elt);
8787 if (!CI || !ConstantIsOk(CI))
8788 return std::nullopt;
8789
8790 if (!SafeReplacementConstant)
8791 SafeReplacementConstant = CI;
8792 }
8793 } else if (isa<VectorType>(Val: C->getType())) {
8794 // Handle scalable splat
8795 Value *SplatC = C->getSplatValue();
8796 auto *CI = dyn_cast_or_null<ConstantInt>(Val: SplatC);
8797 // Bail out if the constant can't be safely incremented/decremented.
8798 if (!CI || !ConstantIsOk(CI))
8799 return std::nullopt;
8800 } else {
8801 // ConstantExpr?
8802 return std::nullopt;
8803 }
8804
8805 // It may not be safe to change a compare predicate in the presence of
8806 // undefined elements, so replace those elements with the first safe constant
8807 // that we found.
8808 // TODO: in case of poison, it is safe; let's replace undefs only.
8809 if (C->containsUndefOrPoisonElement()) {
8810 assert(SafeReplacementConstant && "Replacement constant not set");
8811 C = Constant::replaceUndefsWith(C, Replacement: SafeReplacementConstant);
8812 }
8813
8814 CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(pred: Pred);
8815
8816 // Increment or decrement the constant.
8817 Constant *OneOrNegOne = ConstantInt::get(Ty: Type, V: WillIncrement ? 1 : -1, IsSigned: true);
8818 Constant *NewC = ConstantExpr::getAdd(C1: C, C2: OneOrNegOne);
8819
8820 return std::make_pair(x&: NewPred, y&: NewC);
8821}
8822
8823static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
8824 FastMathFlags FMF,
8825 Value *CmpLHS, Value *CmpRHS,
8826 Value *TrueVal, Value *FalseVal,
8827 Value *&LHS, Value *&RHS,
8828 unsigned Depth) {
8829 bool HasMismatchedZeros = false;
8830 if (CmpInst::isFPPredicate(P: Pred)) {
8831 // IEEE-754 ignores the sign of 0.0 in comparisons. So if the select has one
8832 // 0.0 operand, set the compare's 0.0 operands to that same value for the
8833 // purpose of identifying min/max. Disregard vector constants with undefined
8834 // elements because those can not be back-propagated for analysis.
8835 Value *OutputZeroVal = nullptr;
8836 if (match(V: TrueVal, P: m_AnyZeroFP()) && !match(V: FalseVal, P: m_AnyZeroFP()) &&
8837 !cast<Constant>(Val: TrueVal)->containsUndefOrPoisonElement())
8838 OutputZeroVal = TrueVal;
8839 else if (match(V: FalseVal, P: m_AnyZeroFP()) && !match(V: TrueVal, P: m_AnyZeroFP()) &&
8840 !cast<Constant>(Val: FalseVal)->containsUndefOrPoisonElement())
8841 OutputZeroVal = FalseVal;
8842
8843 if (OutputZeroVal) {
8844 if (match(V: CmpLHS, P: m_AnyZeroFP()) && CmpLHS != OutputZeroVal) {
8845 HasMismatchedZeros = true;
8846 CmpLHS = OutputZeroVal;
8847 }
8848 if (match(V: CmpRHS, P: m_AnyZeroFP()) && CmpRHS != OutputZeroVal) {
8849 HasMismatchedZeros = true;
8850 CmpRHS = OutputZeroVal;
8851 }
8852 }
8853 }
8854
8855 LHS = CmpLHS;
8856 RHS = CmpRHS;
8857
8858 // Signed zero may return inconsistent results between implementations.
8859 // (0.0 <= -0.0) ? 0.0 : -0.0 // Returns 0.0
8860 // minNum(0.0, -0.0) // May return -0.0 or 0.0 (IEEE 754-2008 5.3.1)
8861 // Therefore, we behave conservatively and only proceed if at least one of the
8862 // operands is known to not be zero or if we don't care about signed zero.
8863 switch (Pred) {
8864 default: break;
8865 case CmpInst::FCMP_OGT: case CmpInst::FCMP_OLT:
8866 case CmpInst::FCMP_UGT: case CmpInst::FCMP_ULT:
8867 if (!HasMismatchedZeros)
8868 break;
8869 [[fallthrough]];
8870 case CmpInst::FCMP_OGE: case CmpInst::FCMP_OLE:
8871 case CmpInst::FCMP_UGE: case CmpInst::FCMP_ULE:
8872 if (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
8873 !isKnownNonZero(V: CmpRHS))
8874 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8875 }
8876
8877 SelectPatternNaNBehavior NaNBehavior = SPNB_NA;
8878 bool Ordered = false;
8879
8880 // When given one NaN and one non-NaN input:
8881 // - maxnum/minnum (C99 fmaxf()/fminf()) return the non-NaN input.
8882 // - A simple C99 (a < b ? a : b) construction will return 'b' (as the
8883 // ordered comparison fails), which could be NaN or non-NaN.
8884 // so here we discover exactly what NaN behavior is required/accepted.
8885 if (CmpInst::isFPPredicate(P: Pred)) {
8886 bool LHSSafe = isKnownNonNaN(V: CmpLHS, FMF);
8887 bool RHSSafe = isKnownNonNaN(V: CmpRHS, FMF);
8888
8889 if (LHSSafe && RHSSafe) {
8890 // Both operands are known non-NaN.
8891 NaNBehavior = SPNB_RETURNS_ANY;
8892 Ordered = CmpInst::isOrdered(predicate: Pred);
8893 } else if (CmpInst::isOrdered(predicate: Pred)) {
8894 // An ordered comparison will return false when given a NaN, so it
8895 // returns the RHS.
8896 Ordered = true;
8897 if (LHSSafe)
8898 // LHS is non-NaN, so if RHS is NaN then NaN will be returned.
8899 NaNBehavior = SPNB_RETURNS_NAN;
8900 else if (RHSSafe)
8901 NaNBehavior = SPNB_RETURNS_OTHER;
8902 else
8903 // Completely unsafe.
8904 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8905 } else {
8906 Ordered = false;
8907 // An unordered comparison will return true when given a NaN, so it
8908 // returns the LHS.
8909 if (LHSSafe)
8910 // LHS is non-NaN, so if RHS is NaN then non-NaN will be returned.
8911 NaNBehavior = SPNB_RETURNS_OTHER;
8912 else if (RHSSafe)
8913 NaNBehavior = SPNB_RETURNS_NAN;
8914 else
8915 // Completely unsafe.
8916 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8917 }
8918 }
8919
8920 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
8921 std::swap(a&: CmpLHS, b&: CmpRHS);
8922 Pred = CmpInst::getSwappedPredicate(pred: Pred);
8923 if (NaNBehavior == SPNB_RETURNS_NAN)
8924 NaNBehavior = SPNB_RETURNS_OTHER;
8925 else if (NaNBehavior == SPNB_RETURNS_OTHER)
8926 NaNBehavior = SPNB_RETURNS_NAN;
8927 Ordered = !Ordered;
8928 }
8929
8930 // ([if]cmp X, Y) ? X : Y
8931 if (TrueVal == CmpLHS && FalseVal == CmpRHS)
8932 return getSelectPattern(Pred, NaNBehavior, Ordered);
8933
8934 if (isKnownNegation(X: TrueVal, Y: FalseVal)) {
8935 // Sign-extending LHS does not change its sign, so TrueVal/FalseVal can
8936 // match against either LHS or sext(LHS).
8937 auto MaybeSExtCmpLHS =
8938 m_CombineOr(L: m_Specific(V: CmpLHS), R: m_SExt(Op: m_Specific(V: CmpLHS)));
8939 auto ZeroOrAllOnes = m_CombineOr(L: m_ZeroInt(), R: m_AllOnes());
8940 auto ZeroOrOne = m_CombineOr(L: m_ZeroInt(), R: m_One());
8941 if (match(V: TrueVal, P: MaybeSExtCmpLHS)) {
8942 // Set the return values. If the compare uses the negated value (-X >s 0),
8943 // swap the return values because the negated value is always 'RHS'.
8944 LHS = TrueVal;
8945 RHS = FalseVal;
8946 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: FalseVal))))
8947 std::swap(a&: LHS, b&: RHS);
8948
8949 // (X >s 0) ? X : -X or (X >s -1) ? X : -X --> ABS(X)
8950 // (-X >s 0) ? -X : X or (-X >s -1) ? -X : X --> ABS(X)
8951 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
8952 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
8953
8954 // (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X)
8955 if (Pred == ICmpInst::ICMP_SGE && match(V: CmpRHS, P: ZeroOrOne))
8956 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
8957
8958 // (X <s 0) ? X : -X or (X <s 1) ? X : -X --> NABS(X)
8959 // (-X <s 0) ? -X : X or (-X <s 1) ? -X : X --> NABS(X)
8960 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
8961 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
8962 }
8963 else if (match(V: FalseVal, P: MaybeSExtCmpLHS)) {
8964 // Set the return values. If the compare uses the negated value (-X >s 0),
8965 // swap the return values because the negated value is always 'RHS'.
8966 LHS = FalseVal;
8967 RHS = TrueVal;
8968 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: TrueVal))))
8969 std::swap(a&: LHS, b&: RHS);
8970
8971 // (X >s 0) ? -X : X or (X >s -1) ? -X : X --> NABS(X)
8972 // (-X >s 0) ? X : -X or (-X >s -1) ? X : -X --> NABS(X)
8973 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
8974 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
8975
8976 // (X <s 0) ? -X : X or (X <s 1) ? -X : X --> ABS(X)
8977 // (-X <s 0) ? X : -X or (-X <s 1) ? X : -X --> ABS(X)
8978 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
8979 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
8980 }
8981 }
8982
8983 if (CmpInst::isIntPredicate(P: Pred))
8984 return matchMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS, Depth);
8985
8986 // According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar
8987 // may return either -0.0 or 0.0, so fcmp/select pair has stricter
8988 // semantics than minNum. Be conservative in such case.
8989 if (NaNBehavior != SPNB_RETURNS_ANY ||
8990 (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
8991 !isKnownNonZero(V: CmpRHS)))
8992 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8993
8994 return matchFastFloatClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
8995}
8996
8997static Value *lookThroughCastConst(CmpInst *CmpI, Type *SrcTy, Constant *C,
8998 Instruction::CastOps *CastOp) {
8999 const DataLayout &DL = CmpI->getDataLayout();
9000
9001 Constant *CastedTo = nullptr;
9002 switch (*CastOp) {
9003 case Instruction::ZExt:
9004 if (CmpI->isUnsigned())
9005 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy);
9006 break;
9007 case Instruction::SExt:
9008 if (CmpI->isSigned())
9009 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy, OnlyIfReduced: true);
9010 break;
9011 case Instruction::Trunc:
9012 Constant *CmpConst;
9013 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_Constant(C&: CmpConst)) &&
9014 CmpConst->getType() == SrcTy) {
9015 // Here we have the following case:
9016 //
9017 // %cond = cmp iN %x, CmpConst
9018 // %tr = trunc iN %x to iK
9019 // %narrowsel = select i1 %cond, iK %t, iK C
9020 //
9021 // We can always move trunc after select operation:
9022 //
9023 // %cond = cmp iN %x, CmpConst
9024 // %widesel = select i1 %cond, iN %x, iN CmpConst
9025 // %tr = trunc iN %widesel to iK
9026 //
9027 // Note that C could be extended in any way because we don't care about
9028 // upper bits after truncation. It can't be abs pattern, because it would
9029 // look like:
9030 //
9031 // select i1 %cond, x, -x.
9032 //
9033 // So only min/max pattern could be matched. Such match requires widened C
9034 // == CmpConst. That is why set widened C = CmpConst, condition trunc
9035 // CmpConst == C is checked below.
9036 CastedTo = CmpConst;
9037 } else {
9038 unsigned ExtOp = CmpI->isSigned() ? Instruction::SExt : Instruction::ZExt;
9039 CastedTo = ConstantFoldCastOperand(Opcode: ExtOp, C, DestTy: SrcTy, DL);
9040 }
9041 break;
9042 case Instruction::FPTrunc:
9043 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPExt, C, DestTy: SrcTy, DL);
9044 break;
9045 case Instruction::FPExt:
9046 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPTrunc, C, DestTy: SrcTy, DL);
9047 break;
9048 case Instruction::FPToUI:
9049 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::UIToFP, C, DestTy: SrcTy, DL);
9050 break;
9051 case Instruction::FPToSI:
9052 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::SIToFP, C, DestTy: SrcTy, DL);
9053 break;
9054 case Instruction::UIToFP:
9055 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToUI, C, DestTy: SrcTy, DL);
9056 break;
9057 case Instruction::SIToFP:
9058 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToSI, C, DestTy: SrcTy, DL);
9059 break;
9060 default:
9061 break;
9062 }
9063
9064 if (!CastedTo)
9065 return nullptr;
9066
9067 // Make sure the cast doesn't lose any information.
9068 Constant *CastedBack =
9069 ConstantFoldCastOperand(Opcode: *CastOp, C: CastedTo, DestTy: C->getType(), DL);
9070 if (CastedBack && CastedBack != C)
9071 return nullptr;
9072
9073 return CastedTo;
9074}
9075
9076/// Helps to match a select pattern in case of a type mismatch.
9077///
9078/// The function processes the case when type of true and false values of a
9079/// select instruction differs from type of the cmp instruction operands because
9080/// of a cast instruction. The function checks if it is legal to move the cast
9081/// operation after "select". If yes, it returns the new second value of
9082/// "select" (with the assumption that cast is moved):
9083/// 1. As operand of cast instruction when both values of "select" are same cast
9084/// instructions.
9085/// 2. As restored constant (by applying reverse cast operation) when the first
9086/// value of the "select" is a cast operation and the second value is a
9087/// constant. It is implemented in lookThroughCastConst().
9088/// 3. As one operand is cast instruction and the other is not. The operands in
9089/// sel(cmp) are in different type integer.
9090/// NOTE: We return only the new second value because the first value could be
9091/// accessed as operand of cast instruction.
9092static Value *lookThroughCast(CmpInst *CmpI, Value *V1, Value *V2,
9093 Instruction::CastOps *CastOp) {
9094 auto *Cast1 = dyn_cast<CastInst>(Val: V1);
9095 if (!Cast1)
9096 return nullptr;
9097
9098 *CastOp = Cast1->getOpcode();
9099 Type *SrcTy = Cast1->getSrcTy();
9100 if (auto *Cast2 = dyn_cast<CastInst>(Val: V2)) {
9101 // If V1 and V2 are both the same cast from the same type, look through V1.
9102 if (*CastOp == Cast2->getOpcode() && SrcTy == Cast2->getSrcTy())
9103 return Cast2->getOperand(i_nocapture: 0);
9104 return nullptr;
9105 }
9106
9107 auto *C = dyn_cast<Constant>(Val: V2);
9108 if (C)
9109 return lookThroughCastConst(CmpI, SrcTy, C, CastOp);
9110
9111 Value *CastedTo = nullptr;
9112 if (*CastOp == Instruction::Trunc) {
9113 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_ZExtOrSExt(Op: m_Specific(V: V2)))) {
9114 // Here we have the following case:
9115 // %y_ext = sext iK %y to iN
9116 // %cond = cmp iN %x, %y_ext
9117 // %tr = trunc iN %x to iK
9118 // %narrowsel = select i1 %cond, iK %tr, iK %y
9119 //
9120 // We can always move trunc after select operation:
9121 // %y_ext = sext iK %y to iN
9122 // %cond = cmp iN %x, %y_ext
9123 // %widesel = select i1 %cond, iN %x, iN %y_ext
9124 // %tr = trunc iN %widesel to iK
9125 assert(V2->getType() == Cast1->getType() &&
9126 "V2 and Cast1 should be the same type.");
9127 CastedTo = CmpI->getOperand(i_nocapture: 1);
9128 }
9129 }
9130
9131 return CastedTo;
9132}
9133SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
9134 Instruction::CastOps *CastOp,
9135 unsigned Depth) {
9136 if (Depth >= MaxAnalysisRecursionDepth)
9137 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9138
9139 SelectInst *SI = dyn_cast<SelectInst>(Val: V);
9140 if (!SI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9141
9142 CmpInst *CmpI = dyn_cast<CmpInst>(Val: SI->getCondition());
9143 if (!CmpI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9144
9145 Value *TrueVal = SI->getTrueValue();
9146 Value *FalseVal = SI->getFalseValue();
9147
9148 return llvm::matchDecomposedSelectPattern(
9149 CmpI, TrueVal, FalseVal, LHS, RHS,
9150 FMF: isa<FPMathOperator>(Val: SI) ? SI->getFastMathFlags() : FastMathFlags(),
9151 CastOp, Depth);
9152}
9153
9154SelectPatternResult llvm::matchDecomposedSelectPattern(
9155 CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
9156 FastMathFlags FMF, Instruction::CastOps *CastOp, unsigned Depth) {
9157 CmpInst::Predicate Pred = CmpI->getPredicate();
9158 Value *CmpLHS = CmpI->getOperand(i_nocapture: 0);
9159 Value *CmpRHS = CmpI->getOperand(i_nocapture: 1);
9160 if (isa<FPMathOperator>(Val: CmpI) && CmpI->hasNoNaNs())
9161 FMF.setNoNaNs();
9162
9163 // Bail out early.
9164 if (CmpI->isEquality())
9165 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9166
9167 // Deal with type mismatches.
9168 if (CastOp && CmpLHS->getType() != TrueVal->getType()) {
9169 if (Value *C = lookThroughCast(CmpI, V1: TrueVal, V2: FalseVal, CastOp)) {
9170 // If this is a potential fmin/fmax with a cast to integer, then ignore
9171 // -0.0 because there is no corresponding integer value.
9172 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9173 FMF.setNoSignedZeros();
9174 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9175 TrueVal: cast<CastInst>(Val: TrueVal)->getOperand(i_nocapture: 0), FalseVal: C,
9176 LHS, RHS, Depth);
9177 }
9178 if (Value *C = lookThroughCast(CmpI, V1: FalseVal, V2: TrueVal, CastOp)) {
9179 // If this is a potential fmin/fmax with a cast to integer, then ignore
9180 // -0.0 because there is no corresponding integer value.
9181 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9182 FMF.setNoSignedZeros();
9183 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9184 TrueVal: C, FalseVal: cast<CastInst>(Val: FalseVal)->getOperand(i_nocapture: 0),
9185 LHS, RHS, Depth);
9186 }
9187 }
9188 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal, FalseVal,
9189 LHS, RHS, Depth);
9190}
9191
9192CmpInst::Predicate llvm::getMinMaxPred(SelectPatternFlavor SPF, bool Ordered) {
9193 if (SPF == SPF_SMIN) return ICmpInst::ICMP_SLT;
9194 if (SPF == SPF_UMIN) return ICmpInst::ICMP_ULT;
9195 if (SPF == SPF_SMAX) return ICmpInst::ICMP_SGT;
9196 if (SPF == SPF_UMAX) return ICmpInst::ICMP_UGT;
9197 if (SPF == SPF_FMINNUM)
9198 return Ordered ? FCmpInst::FCMP_OLT : FCmpInst::FCMP_ULT;
9199 if (SPF == SPF_FMAXNUM)
9200 return Ordered ? FCmpInst::FCMP_OGT : FCmpInst::FCMP_UGT;
9201 llvm_unreachable("unhandled!");
9202}
9203
9204Intrinsic::ID llvm::getMinMaxIntrinsic(SelectPatternFlavor SPF) {
9205 switch (SPF) {
9206 case SelectPatternFlavor::SPF_UMIN:
9207 return Intrinsic::umin;
9208 case SelectPatternFlavor::SPF_UMAX:
9209 return Intrinsic::umax;
9210 case SelectPatternFlavor::SPF_SMIN:
9211 return Intrinsic::smin;
9212 case SelectPatternFlavor::SPF_SMAX:
9213 return Intrinsic::smax;
9214 default:
9215 llvm_unreachable("Unexpected SPF");
9216 }
9217}
9218
9219SelectPatternFlavor llvm::getInverseMinMaxFlavor(SelectPatternFlavor SPF) {
9220 if (SPF == SPF_SMIN) return SPF_SMAX;
9221 if (SPF == SPF_UMIN) return SPF_UMAX;
9222 if (SPF == SPF_SMAX) return SPF_SMIN;
9223 if (SPF == SPF_UMAX) return SPF_UMIN;
9224 llvm_unreachable("unhandled!");
9225}
9226
9227Intrinsic::ID llvm::getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID) {
9228 switch (MinMaxID) {
9229 case Intrinsic::smax: return Intrinsic::smin;
9230 case Intrinsic::smin: return Intrinsic::smax;
9231 case Intrinsic::umax: return Intrinsic::umin;
9232 case Intrinsic::umin: return Intrinsic::umax;
9233 // Please note that next four intrinsics may produce the same result for
9234 // original and inverted case even if X != Y due to NaN is handled specially.
9235 case Intrinsic::maximum: return Intrinsic::minimum;
9236 case Intrinsic::minimum: return Intrinsic::maximum;
9237 case Intrinsic::maxnum: return Intrinsic::minnum;
9238 case Intrinsic::minnum: return Intrinsic::maxnum;
9239 case Intrinsic::maximumnum:
9240 return Intrinsic::minimumnum;
9241 case Intrinsic::minimumnum:
9242 return Intrinsic::maximumnum;
9243 default: llvm_unreachable("Unexpected intrinsic");
9244 }
9245}
9246
9247APInt llvm::getMinMaxLimit(SelectPatternFlavor SPF, unsigned BitWidth) {
9248 switch (SPF) {
9249 case SPF_SMAX: return APInt::getSignedMaxValue(numBits: BitWidth);
9250 case SPF_SMIN: return APInt::getSignedMinValue(numBits: BitWidth);
9251 case SPF_UMAX: return APInt::getMaxValue(numBits: BitWidth);
9252 case SPF_UMIN: return APInt::getMinValue(numBits: BitWidth);
9253 default: llvm_unreachable("Unexpected flavor");
9254 }
9255}
9256
9257std::pair<Intrinsic::ID, bool>
9258llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
9259 // Check if VL contains select instructions that can be folded into a min/max
9260 // vector intrinsic and return the intrinsic if it is possible.
9261 // TODO: Support floating point min/max.
9262 bool AllCmpSingleUse = true;
9263 SelectPatternResult SelectPattern;
9264 SelectPattern.Flavor = SPF_UNKNOWN;
9265 if (all_of(Range&: VL, P: [&SelectPattern, &AllCmpSingleUse](Value *I) {
9266 Value *LHS, *RHS;
9267 auto CurrentPattern = matchSelectPattern(V: I, LHS, RHS);
9268 if (!SelectPatternResult::isMinOrMax(SPF: CurrentPattern.Flavor))
9269 return false;
9270 if (SelectPattern.Flavor != SPF_UNKNOWN &&
9271 SelectPattern.Flavor != CurrentPattern.Flavor)
9272 return false;
9273 SelectPattern = CurrentPattern;
9274 AllCmpSingleUse &=
9275 match(V: I, P: m_Select(C: m_OneUse(SubPattern: m_Value()), L: m_Value(), R: m_Value()));
9276 return true;
9277 })) {
9278 switch (SelectPattern.Flavor) {
9279 case SPF_SMIN:
9280 return {Intrinsic::smin, AllCmpSingleUse};
9281 case SPF_UMIN:
9282 return {Intrinsic::umin, AllCmpSingleUse};
9283 case SPF_SMAX:
9284 return {Intrinsic::smax, AllCmpSingleUse};
9285 case SPF_UMAX:
9286 return {Intrinsic::umax, AllCmpSingleUse};
9287 case SPF_FMAXNUM:
9288 return {Intrinsic::maxnum, AllCmpSingleUse};
9289 case SPF_FMINNUM:
9290 return {Intrinsic::minnum, AllCmpSingleUse};
9291 default:
9292 llvm_unreachable("unexpected select pattern flavor");
9293 }
9294 }
9295 return {Intrinsic::not_intrinsic, false};
9296}
9297
9298template <typename InstTy>
9299static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
9300 Value *&Init, Value *&OtherOp) {
9301 // Handle the case of a simple two-predecessor recurrence PHI.
9302 // There's a lot more that could theoretically be done here, but
9303 // this is sufficient to catch some interesting cases.
9304 // TODO: Expand list -- gep, uadd.sat etc.
9305 if (PN->getNumIncomingValues() != 2)
9306 return false;
9307
9308 for (unsigned I = 0; I != 2; ++I) {
9309 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9310 Operation && Operation->getNumOperands() >= 2) {
9311 Value *LHS = Operation->getOperand(0);
9312 Value *RHS = Operation->getOperand(1);
9313 if (LHS != PN && RHS != PN)
9314 continue;
9315
9316 Inst = Operation;
9317 Init = PN->getIncomingValue(i: !I);
9318 OtherOp = (LHS == PN) ? RHS : LHS;
9319 return true;
9320 }
9321 }
9322 return false;
9323}
9324
9325template <typename InstTy>
9326static bool matchThreeInputRecurrence(const PHINode *PN, InstTy *&Inst,
9327 Value *&Init, Value *&OtherOp0,
9328 Value *&OtherOp1) {
9329 if (PN->getNumIncomingValues() != 2)
9330 return false;
9331
9332 for (unsigned I = 0; I != 2; ++I) {
9333 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9334 Operation && Operation->getNumOperands() >= 3) {
9335 Value *Op0 = Operation->getOperand(0);
9336 Value *Op1 = Operation->getOperand(1);
9337 Value *Op2 = Operation->getOperand(2);
9338
9339 if (Op0 != PN && Op1 != PN && Op2 != PN)
9340 continue;
9341
9342 Inst = Operation;
9343 Init = PN->getIncomingValue(i: !I);
9344 if (Op0 == PN) {
9345 OtherOp0 = Op1;
9346 OtherOp1 = Op2;
9347 } else if (Op1 == PN) {
9348 OtherOp0 = Op0;
9349 OtherOp1 = Op2;
9350 } else {
9351 OtherOp0 = Op0;
9352 OtherOp1 = Op1;
9353 }
9354 return true;
9355 }
9356 }
9357 return false;
9358}
9359bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
9360 Value *&Start, Value *&Step) {
9361 // We try to match a recurrence of the form:
9362 // %iv = [Start, %entry], [%iv.next, %backedge]
9363 // %iv.next = binop %iv, Step
9364 // Or:
9365 // %iv = [Start, %entry], [%iv.next, %backedge]
9366 // %iv.next = binop Step, %iv
9367 return matchTwoInputRecurrence(PN: P, Inst&: BO, Init&: Start, OtherOp&: Step);
9368}
9369
9370bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
9371 Value *&Start, Value *&Step) {
9372 BinaryOperator *BO = nullptr;
9373 P = dyn_cast<PHINode>(Val: I->getOperand(i_nocapture: 0));
9374 if (!P)
9375 P = dyn_cast<PHINode>(Val: I->getOperand(i_nocapture: 1));
9376 return P && matchSimpleRecurrence(P, BO, Start, Step) && BO == I;
9377}
9378
9379bool llvm::matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I,
9380 PHINode *&P, Value *&Init,
9381 Value *&OtherOp) {
9382 // Binary intrinsics only supported for now.
9383 if (I->arg_size() != 2 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9384 I->getType() != I->getArgOperand(i: 1)->getType())
9385 return false;
9386
9387 IntrinsicInst *II = nullptr;
9388 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9389 if (!P)
9390 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9391
9392 return P && matchTwoInputRecurrence(PN: P, Inst&: II, Init, OtherOp) && II == I;
9393}
9394
9395bool llvm::matchSimpleTernaryIntrinsicRecurrence(const IntrinsicInst *I,
9396 PHINode *&P, Value *&Init,
9397 Value *&OtherOp0,
9398 Value *&OtherOp1) {
9399 if (I->arg_size() != 3 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9400 I->getType() != I->getArgOperand(i: 1)->getType() ||
9401 I->getType() != I->getArgOperand(i: 2)->getType())
9402 return false;
9403 IntrinsicInst *II = nullptr;
9404 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9405 if (!P) {
9406 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9407 if (!P)
9408 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 2));
9409 }
9410 return P && matchThreeInputRecurrence(PN: P, Inst&: II, Init, OtherOp0, OtherOp1) &&
9411 II == I;
9412}
9413
9414/// Return true if "icmp Pred LHS RHS" is always true.
9415static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
9416 const Value *RHS) {
9417 if (ICmpInst::isTrueWhenEqual(predicate: Pred) && LHS == RHS)
9418 return true;
9419
9420 switch (Pred) {
9421 default:
9422 return false;
9423
9424 case CmpInst::ICMP_SLE: {
9425 const APInt *C;
9426
9427 // LHS s<= LHS +_{nsw} C if C >= 0
9428 // LHS s<= LHS | C if C >= 0
9429 if (match(V: RHS, P: m_NSWAdd(L: m_Specific(V: LHS), R: m_APInt(Res&: C))) ||
9430 match(V: RHS, P: m_Or(L: m_Specific(V: LHS), R: m_APInt(Res&: C))))
9431 return !C->isNegative();
9432
9433 // LHS s<= smax(LHS, V) for any V
9434 if (match(V: RHS, P: m_c_SMax(L: m_Specific(V: LHS), R: m_Value())))
9435 return true;
9436
9437 // smin(RHS, V) s<= RHS for any V
9438 if (match(V: LHS, P: m_c_SMin(L: m_Specific(V: RHS), R: m_Value())))
9439 return true;
9440
9441 // Match A to (X +_{nsw} CA) and B to (X +_{nsw} CB)
9442 const Value *X;
9443 const APInt *CLHS, *CRHS;
9444 if (match(V: LHS, P: m_NSWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9445 match(V: RHS, P: m_NSWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9446 return CLHS->sle(RHS: *CRHS);
9447
9448 return false;
9449 }
9450
9451 case CmpInst::ICMP_ULE: {
9452 // LHS u<= LHS +_{nuw} V for any V
9453 if (match(V: RHS, P: m_c_Add(L: m_Specific(V: LHS), R: m_Value())) &&
9454 cast<OverflowingBinaryOperator>(Val: RHS)->hasNoUnsignedWrap())
9455 return true;
9456
9457 // LHS u<= LHS | V for any V
9458 if (match(V: RHS, P: m_c_Or(L: m_Specific(V: LHS), R: m_Value())))
9459 return true;
9460
9461 // LHS u<= umax(LHS, V) for any V
9462 if (match(V: RHS, P: m_c_UMax(L: m_Specific(V: LHS), R: m_Value())))
9463 return true;
9464
9465 // RHS >> V u<= RHS for any V
9466 if (match(V: LHS, P: m_LShr(L: m_Specific(V: RHS), R: m_Value())))
9467 return true;
9468
9469 // RHS u/ C_ugt_1 u<= RHS
9470 const APInt *C;
9471 if (match(V: LHS, P: m_UDiv(L: m_Specific(V: RHS), R: m_APInt(Res&: C))) && C->ugt(RHS: 1))
9472 return true;
9473
9474 // RHS & V u<= RHS for any V
9475 if (match(V: LHS, P: m_c_And(L: m_Specific(V: RHS), R: m_Value())))
9476 return true;
9477
9478 // umin(RHS, V) u<= RHS for any V
9479 if (match(V: LHS, P: m_c_UMin(L: m_Specific(V: RHS), R: m_Value())))
9480 return true;
9481
9482 // Match A to (X +_{nuw} CA) and B to (X +_{nuw} CB)
9483 const Value *X;
9484 const APInt *CLHS, *CRHS;
9485 if (match(V: LHS, P: m_NUWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9486 match(V: RHS, P: m_NUWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9487 return CLHS->ule(RHS: *CRHS);
9488
9489 return false;
9490 }
9491 }
9492}
9493
9494/// Return true if "icmp Pred BLHS BRHS" is true whenever "icmp Pred
9495/// ALHS ARHS" is true. Otherwise, return std::nullopt.
9496static std::optional<bool>
9497isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS,
9498 const Value *ARHS, const Value *BLHS, const Value *BRHS) {
9499 switch (Pred) {
9500 default:
9501 return std::nullopt;
9502
9503 case CmpInst::ICMP_SLT:
9504 case CmpInst::ICMP_SLE:
9505 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BLHS, RHS: ALHS) &&
9506 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ARHS, RHS: BRHS))
9507 return true;
9508 return std::nullopt;
9509
9510 case CmpInst::ICMP_SGT:
9511 case CmpInst::ICMP_SGE:
9512 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ALHS, RHS: BLHS) &&
9513 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BRHS, RHS: ARHS))
9514 return true;
9515 return std::nullopt;
9516
9517 case CmpInst::ICMP_ULT:
9518 case CmpInst::ICMP_ULE:
9519 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BLHS, RHS: ALHS) &&
9520 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ARHS, RHS: BRHS))
9521 return true;
9522 return std::nullopt;
9523
9524 case CmpInst::ICMP_UGT:
9525 case CmpInst::ICMP_UGE:
9526 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ALHS, RHS: BLHS) &&
9527 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BRHS, RHS: ARHS))
9528 return true;
9529 return std::nullopt;
9530 }
9531}
9532
9533/// Return true if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is true.
9534/// Return false if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is false.
9535/// Otherwise, return std::nullopt if we can't infer anything.
9536static std::optional<bool>
9537isImpliedCondCommonOperandWithCR(CmpPredicate LPred, const ConstantRange &LCR,
9538 CmpPredicate RPred, const ConstantRange &RCR) {
9539 auto CRImpliesPred = [&](ConstantRange CR,
9540 CmpInst::Predicate Pred) -> std::optional<bool> {
9541 // If all true values for lhs and true for rhs, lhs implies rhs
9542 if (CR.icmp(Pred, Other: RCR))
9543 return true;
9544
9545 // If there is no overlap, lhs implies not rhs
9546 if (CR.icmp(Pred: CmpInst::getInversePredicate(pred: Pred), Other: RCR))
9547 return false;
9548
9549 return std::nullopt;
9550 };
9551 if (auto Res = CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
9552 RPred))
9553 return Res;
9554 if (LPred.hasSameSign() ^ RPred.hasSameSign()) {
9555 LPred = LPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: LPred)
9556 : LPred.dropSameSign();
9557 RPred = RPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: RPred)
9558 : RPred.dropSameSign();
9559 return CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
9560 RPred);
9561 }
9562 return std::nullopt;
9563}
9564
9565/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
9566/// is true. Return false if LHS implies RHS is false. Otherwise, return
9567/// std::nullopt if we can't infer anything.
9568static std::optional<bool>
9569isImpliedCondICmps(CmpPredicate LPred, const Value *L0, const Value *L1,
9570 CmpPredicate RPred, const Value *R0, const Value *R1,
9571 const DataLayout &DL, bool LHSIsTrue) {
9572 // The rest of the logic assumes the LHS condition is true. If that's not the
9573 // case, invert the predicate to make it so.
9574 if (!LHSIsTrue)
9575 LPred = ICmpInst::getInverseCmpPredicate(Pred: LPred);
9576
9577 // We can have non-canonical operands, so try to normalize any common operand
9578 // to L0/R0.
9579 if (L0 == R1) {
9580 std::swap(a&: R0, b&: R1);
9581 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
9582 }
9583 if (R0 == L1) {
9584 std::swap(a&: L0, b&: L1);
9585 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9586 }
9587 if (L1 == R1) {
9588 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
9589 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
9590 std::swap(a&: L0, b&: L1);
9591 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9592 std::swap(a&: R0, b&: R1);
9593 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
9594 }
9595 }
9596
9597 // See if we can infer anything if operand-0 matches and we have at least one
9598 // constant.
9599 const APInt *Unused;
9600 if (L0 == R0 && (match(V: L1, P: m_APInt(Res&: Unused)) || match(V: R1, P: m_APInt(Res&: Unused)))) {
9601 // Potential TODO: We could also further use the constant range of L0/R0 to
9602 // further constraint the constant ranges. At the moment this leads to
9603 // several regressions related to not transforming `multi_use(A + C0) eq/ne
9604 // C1` (see discussion: D58633).
9605 ConstantRange LCR = computeConstantRange(
9606 V: L1, ForSigned: ICmpInst::isSigned(Pred: LPred), /* UseInstrInfo=*/true, /*AC=*/nullptr,
9607 /*CxtI=*/CtxI: nullptr, /*DT=*/nullptr, Depth: MaxAnalysisRecursionDepth - 1);
9608 ConstantRange RCR = computeConstantRange(
9609 V: R1, ForSigned: ICmpInst::isSigned(Pred: RPred), /* UseInstrInfo=*/true, /*AC=*/nullptr,
9610 /*CxtI=*/CtxI: nullptr, /*DT=*/nullptr, Depth: MaxAnalysisRecursionDepth - 1);
9611 // Even if L1/R1 are not both constant, we can still sometimes deduce
9612 // relationship from a single constant. For example X u> Y implies X != 0.
9613 if (auto R = isImpliedCondCommonOperandWithCR(LPred, LCR, RPred, RCR))
9614 return R;
9615 // If both L1/R1 were exact constant ranges and we didn't get anything
9616 // here, we won't be able to deduce this.
9617 if (match(V: L1, P: m_APInt(Res&: Unused)) && match(V: R1, P: m_APInt(Res&: Unused)))
9618 return std::nullopt;
9619 }
9620
9621 // Can we infer anything when the two compares have matching operands?
9622 if (L0 == R0 && L1 == R1)
9623 return ICmpInst::isImpliedByMatchingCmp(Pred1: LPred, Pred2: RPred);
9624
9625 // It only really makes sense in the context of signed comparison for "X - Y
9626 // must be positive if X >= Y and no overflow".
9627 // Take SGT as an example: L0:x > L1:y and C >= 0
9628 // ==> R0:(x -nsw y) < R1:(-C) is false
9629 CmpInst::Predicate SignedLPred = LPred.getPreferredSignedPredicate();
9630 if ((SignedLPred == ICmpInst::ICMP_SGT ||
9631 SignedLPred == ICmpInst::ICMP_SGE) &&
9632 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
9633 if (match(V: R1, P: m_NonPositive()) &&
9634 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == false)
9635 return false;
9636 }
9637
9638 // Take SLT as an example: L0:x < L1:y and C <= 0
9639 // ==> R0:(x -nsw y) < R1:(-C) is true
9640 if ((SignedLPred == ICmpInst::ICMP_SLT ||
9641 SignedLPred == ICmpInst::ICMP_SLE) &&
9642 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
9643 if (match(V: R1, P: m_NonNegative()) &&
9644 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == true)
9645 return true;
9646 }
9647
9648 // a - b == NonZero -> a != b
9649 // ptrtoint(a) - ptrtoint(b) == NonZero -> a != b
9650 const APInt *L1C;
9651 Value *A, *B;
9652 if (LPred == ICmpInst::ICMP_EQ && ICmpInst::isEquality(P: RPred) &&
9653 match(V: L1, P: m_APInt(Res&: L1C)) && !L1C->isZero() &&
9654 match(V: L0, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9655 ((A == R0 && B == R1) || (A == R1 && B == R0) ||
9656 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0))) &&
9657 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1)))) ||
9658 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1))) &&
9659 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0)))))) {
9660 return RPred.dropSameSign() == ICmpInst::ICMP_NE;
9661 }
9662
9663 // L0 = R0 = L1 + R1, L0 >=u L1 implies R0 >=u R1, L0 <u L1 implies R0 <u R1
9664 if (L0 == R0 &&
9665 (LPred == ICmpInst::ICMP_ULT || LPred == ICmpInst::ICMP_UGE) &&
9666 (RPred == ICmpInst::ICMP_ULT || RPred == ICmpInst::ICMP_UGE) &&
9667 match(V: L0, P: m_c_Add(L: m_Specific(V: L1), R: m_Specific(V: R1))))
9668 return CmpPredicate::getMatching(A: LPred, B: RPred).has_value();
9669
9670 if (auto P = CmpPredicate::getMatching(A: LPred, B: RPred))
9671 return isImpliedCondOperands(Pred: *P, ALHS: L0, ARHS: L1, BLHS: R0, BRHS: R1);
9672
9673 return std::nullopt;
9674}
9675
9676/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
9677/// is true. Return false if LHS implies RHS is false. Otherwise, return
9678/// std::nullopt if we can't infer anything.
9679static std::optional<bool>
9680isImpliedCondFCmps(FCmpInst::Predicate LPred, const Value *L0, const Value *L1,
9681 FCmpInst::Predicate RPred, const Value *R0, const Value *R1,
9682 const DataLayout &DL, bool LHSIsTrue) {
9683 // The rest of the logic assumes the LHS condition is true. If that's not the
9684 // case, invert the predicate to make it so.
9685 if (!LHSIsTrue)
9686 LPred = FCmpInst::getInversePredicate(pred: LPred);
9687
9688 // We can have non-canonical operands, so try to normalize any common operand
9689 // to L0/R0.
9690 if (L0 == R1) {
9691 std::swap(a&: R0, b&: R1);
9692 RPred = FCmpInst::getSwappedPredicate(pred: RPred);
9693 }
9694 if (R0 == L1) {
9695 std::swap(a&: L0, b&: L1);
9696 LPred = FCmpInst::getSwappedPredicate(pred: LPred);
9697 }
9698 if (L1 == R1) {
9699 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
9700 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
9701 std::swap(a&: L0, b&: L1);
9702 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9703 std::swap(a&: R0, b&: R1);
9704 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
9705 }
9706 }
9707
9708 // Can we infer anything when the two compares have matching operands?
9709 if (L0 == R0 && L1 == R1) {
9710 if ((LPred & RPred) == LPred)
9711 return true;
9712 if ((LPred & ~RPred) == LPred)
9713 return false;
9714 }
9715
9716 // See if we can infer anything if operand-0 matches and we have at least one
9717 // constant.
9718 const APFloat *L1C, *R1C;
9719 if (L0 == R0 && match(V: L1, P: m_APFloat(Res&: L1C)) && match(V: R1, P: m_APFloat(Res&: R1C))) {
9720 if (std::optional<ConstantFPRange> DomCR =
9721 ConstantFPRange::makeExactFCmpRegion(Pred: LPred, Other: *L1C)) {
9722 if (std::optional<ConstantFPRange> ImpliedCR =
9723 ConstantFPRange::makeExactFCmpRegion(Pred: RPred, Other: *R1C)) {
9724 if (ImpliedCR->contains(CR: *DomCR))
9725 return true;
9726 }
9727 if (std::optional<ConstantFPRange> ImpliedCR =
9728 ConstantFPRange::makeExactFCmpRegion(
9729 Pred: FCmpInst::getInversePredicate(pred: RPred), Other: *R1C)) {
9730 if (ImpliedCR->contains(CR: *DomCR))
9731 return false;
9732 }
9733 }
9734 }
9735
9736 return std::nullopt;
9737}
9738
9739/// Return true if LHS implies RHS is true. Return false if LHS implies RHS is
9740/// false. Otherwise, return std::nullopt if we can't infer anything. We
9741/// expect the RHS to be an icmp and the LHS to be an 'and', 'or', or a 'select'
9742/// instruction.
9743static std::optional<bool>
9744isImpliedCondAndOr(const Instruction *LHS, CmpPredicate RHSPred,
9745 const Value *RHSOp0, const Value *RHSOp1,
9746 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
9747 // The LHS must be an 'or', 'and', or a 'select' instruction.
9748 assert((LHS->getOpcode() == Instruction::And ||
9749 LHS->getOpcode() == Instruction::Or ||
9750 LHS->getOpcode() == Instruction::Select) &&
9751 "Expected LHS to be 'and', 'or', or 'select'.");
9752
9753 assert(Depth <= MaxAnalysisRecursionDepth && "Hit recursion limit");
9754
9755 // If the result of an 'or' is false, then we know both legs of the 'or' are
9756 // false. Similarly, if the result of an 'and' is true, then we know both
9757 // legs of the 'and' are true.
9758 const Value *ALHS, *ARHS;
9759 if ((!LHSIsTrue && match(V: LHS, P: m_LogicalOr(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS)))) ||
9760 (LHSIsTrue && match(V: LHS, P: m_LogicalAnd(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS))))) {
9761 // FIXME: Make this non-recursion.
9762 if (std::optional<bool> Implication = isImpliedCondition(
9763 LHS: ALHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
9764 return Implication;
9765 if (std::optional<bool> Implication = isImpliedCondition(
9766 LHS: ARHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
9767 return Implication;
9768 return std::nullopt;
9769 }
9770 return std::nullopt;
9771}
9772
9773std::optional<bool>
9774llvm::isImpliedCondition(const Value *LHS, CmpPredicate RHSPred,
9775 const Value *RHSOp0, const Value *RHSOp1,
9776 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
9777 // Bail out when we hit the limit.
9778 if (Depth == MaxAnalysisRecursionDepth)
9779 return std::nullopt;
9780
9781 // A mismatch occurs when we compare a scalar cmp to a vector cmp, for
9782 // example.
9783 if (RHSOp0->getType()->isVectorTy() != LHS->getType()->isVectorTy())
9784 return std::nullopt;
9785
9786 assert(LHS->getType()->isIntOrIntVectorTy(1) &&
9787 "Expected integer type only!");
9788
9789 // Match not
9790 if (match(V: LHS, P: m_Not(V: m_Value(V&: LHS))))
9791 LHSIsTrue = !LHSIsTrue;
9792
9793 // Both LHS and RHS are icmps.
9794 if (RHSOp0->getType()->getScalarType()->isIntOrPtrTy()) {
9795 if (const auto *LHSCmp = dyn_cast<ICmpInst>(Val: LHS))
9796 return isImpliedCondICmps(LPred: LHSCmp->getCmpPredicate(),
9797 L0: LHSCmp->getOperand(i_nocapture: 0), L1: LHSCmp->getOperand(i_nocapture: 1),
9798 RPred: RHSPred, R0: RHSOp0, R1: RHSOp1, DL, LHSIsTrue);
9799 const Value *V;
9800 if (match(V: LHS, P: m_NUWTrunc(Op: m_Value(V))))
9801 return isImpliedCondICmps(LPred: CmpInst::ICMP_NE, L0: V,
9802 L1: ConstantInt::get(Ty: V->getType(), V: 0), RPred: RHSPred,
9803 R0: RHSOp0, R1: RHSOp1, DL, LHSIsTrue);
9804 } else {
9805 assert(RHSOp0->getType()->isFPOrFPVectorTy() &&
9806 "Expected floating point type only!");
9807 if (const auto *LHSCmp = dyn_cast<FCmpInst>(Val: LHS))
9808 return isImpliedCondFCmps(LPred: LHSCmp->getPredicate(), L0: LHSCmp->getOperand(i_nocapture: 0),
9809 L1: LHSCmp->getOperand(i_nocapture: 1), RPred: RHSPred, R0: RHSOp0, R1: RHSOp1,
9810 DL, LHSIsTrue);
9811 }
9812
9813 /// The LHS should be an 'or', 'and', or a 'select' instruction. We expect
9814 /// the RHS to be an icmp.
9815 /// FIXME: Add support for and/or/select on the RHS.
9816 if (const Instruction *LHSI = dyn_cast<Instruction>(Val: LHS)) {
9817 if ((LHSI->getOpcode() == Instruction::And ||
9818 LHSI->getOpcode() == Instruction::Or ||
9819 LHSI->getOpcode() == Instruction::Select))
9820 return isImpliedCondAndOr(LHS: LHSI, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue,
9821 Depth);
9822 }
9823 return std::nullopt;
9824}
9825
9826std::optional<bool> llvm::isImpliedCondition(const Value *LHS, const Value *RHS,
9827 const DataLayout &DL,
9828 bool LHSIsTrue, unsigned Depth) {
9829 // LHS ==> RHS by definition
9830 if (LHS == RHS)
9831 return LHSIsTrue;
9832
9833 // Match not
9834 bool InvertRHS = false;
9835 if (match(V: RHS, P: m_Not(V: m_Value(V&: RHS)))) {
9836 if (LHS == RHS)
9837 return !LHSIsTrue;
9838 InvertRHS = true;
9839 }
9840
9841 if (const ICmpInst *RHSCmp = dyn_cast<ICmpInst>(Val: RHS)) {
9842 if (auto Implied = isImpliedCondition(
9843 LHS, RHSPred: RHSCmp->getCmpPredicate(), RHSOp0: RHSCmp->getOperand(i_nocapture: 0),
9844 RHSOp1: RHSCmp->getOperand(i_nocapture: 1), DL, LHSIsTrue, Depth))
9845 return InvertRHS ? !*Implied : *Implied;
9846 return std::nullopt;
9847 }
9848 if (const FCmpInst *RHSCmp = dyn_cast<FCmpInst>(Val: RHS)) {
9849 if (auto Implied = isImpliedCondition(
9850 LHS, RHSPred: RHSCmp->getPredicate(), RHSOp0: RHSCmp->getOperand(i_nocapture: 0),
9851 RHSOp1: RHSCmp->getOperand(i_nocapture: 1), DL, LHSIsTrue, Depth))
9852 return InvertRHS ? !*Implied : *Implied;
9853 return std::nullopt;
9854 }
9855
9856 const Value *V;
9857 if (match(V: RHS, P: m_NUWTrunc(Op: m_Value(V)))) {
9858 if (auto Implied = isImpliedCondition(LHS, RHSPred: CmpInst::ICMP_NE, RHSOp0: V,
9859 RHSOp1: ConstantInt::get(Ty: V->getType(), V: 0), DL,
9860 LHSIsTrue, Depth))
9861 return InvertRHS ? !*Implied : *Implied;
9862 return std::nullopt;
9863 }
9864
9865 if (Depth == MaxAnalysisRecursionDepth)
9866 return std::nullopt;
9867
9868 // LHS ==> (RHS1 || RHS2) if LHS ==> RHS1 or LHS ==> RHS2
9869 // LHS ==> !(RHS1 && RHS2) if LHS ==> !RHS1 or LHS ==> !RHS2
9870 const Value *RHS1, *RHS2;
9871 if (match(V: RHS, P: m_LogicalOr(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
9872 if (std::optional<bool> Imp =
9873 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
9874 if (*Imp == true)
9875 return !InvertRHS;
9876 if (std::optional<bool> Imp =
9877 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
9878 if (*Imp == true)
9879 return !InvertRHS;
9880 }
9881 if (match(V: RHS, P: m_LogicalAnd(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
9882 if (std::optional<bool> Imp =
9883 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
9884 if (*Imp == false)
9885 return InvertRHS;
9886 if (std::optional<bool> Imp =
9887 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
9888 if (*Imp == false)
9889 return InvertRHS;
9890 }
9891
9892 return std::nullopt;
9893}
9894
9895// Returns a pair (Condition, ConditionIsTrue), where Condition is a branch
9896// condition dominating ContextI or nullptr, if no condition is found.
9897static std::pair<Value *, bool>
9898getDomPredecessorCondition(const Instruction *ContextI) {
9899 if (!ContextI || !ContextI->getParent())
9900 return {nullptr, false};
9901
9902 // TODO: This is a poor/cheap way to determine dominance. Should we use a
9903 // dominator tree (eg, from a SimplifyQuery) instead?
9904 const BasicBlock *ContextBB = ContextI->getParent();
9905 const BasicBlock *PredBB = ContextBB->getSinglePredecessor();
9906 if (!PredBB)
9907 return {nullptr, false};
9908
9909 // We need a conditional branch in the predecessor.
9910 Value *PredCond;
9911 BasicBlock *TrueBB, *FalseBB;
9912 if (!match(V: PredBB->getTerminator(), P: m_Br(C: m_Value(V&: PredCond), T&: TrueBB, F&: FalseBB)))
9913 return {nullptr, false};
9914
9915 // The branch should get simplified. Don't bother simplifying this condition.
9916 if (TrueBB == FalseBB)
9917 return {nullptr, false};
9918
9919 assert((TrueBB == ContextBB || FalseBB == ContextBB) &&
9920 "Predecessor block does not point to successor?");
9921
9922 // Is this condition implied by the predecessor condition?
9923 return {PredCond, TrueBB == ContextBB};
9924}
9925
9926std::optional<bool> llvm::isImpliedByDomCondition(const Value *Cond,
9927 const Instruction *ContextI,
9928 const DataLayout &DL) {
9929 assert(Cond->getType()->isIntOrIntVectorTy(1) && "Condition must be bool");
9930 auto PredCond = getDomPredecessorCondition(ContextI);
9931 if (PredCond.first)
9932 return isImpliedCondition(LHS: PredCond.first, RHS: Cond, DL, LHSIsTrue: PredCond.second);
9933 return std::nullopt;
9934}
9935
9936std::optional<bool> llvm::isImpliedByDomCondition(CmpPredicate Pred,
9937 const Value *LHS,
9938 const Value *RHS,
9939 const Instruction *ContextI,
9940 const DataLayout &DL) {
9941 auto PredCond = getDomPredecessorCondition(ContextI);
9942 if (PredCond.first)
9943 return isImpliedCondition(LHS: PredCond.first, RHSPred: Pred, RHSOp0: LHS, RHSOp1: RHS, DL,
9944 LHSIsTrue: PredCond.second);
9945 return std::nullopt;
9946}
9947
9948static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
9949 APInt &Upper, const InstrInfoQuery &IIQ,
9950 bool PreferSignedRange) {
9951 unsigned Width = Lower.getBitWidth();
9952 const APInt *C;
9953 switch (BO.getOpcode()) {
9954 case Instruction::Sub:
9955 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
9956 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
9957 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
9958
9959 // If the caller expects a signed compare, then try to use a signed range.
9960 // Otherwise if both no-wraps are set, use the unsigned range because it
9961 // is never larger than the signed range. Example:
9962 // "sub nuw nsw i8 -2, x" is unsigned [0, 254] vs. signed [-128, 126].
9963 // "sub nuw nsw i8 2, x" is unsigned [0, 2] vs. signed [-125, 127].
9964 if (PreferSignedRange && HasNSW && HasNUW)
9965 HasNUW = false;
9966
9967 if (HasNUW) {
9968 // 'sub nuw c, x' produces [0, C].
9969 Upper = *C + 1;
9970 } else if (HasNSW) {
9971 if (C->isNegative()) {
9972 // 'sub nsw -C, x' produces [SINT_MIN, -C - SINT_MIN].
9973 Lower = APInt::getSignedMinValue(numBits: Width);
9974 Upper = *C - APInt::getSignedMaxValue(numBits: Width);
9975 } else {
9976 // Note that sub 0, INT_MIN is not NSW. It techically is a signed wrap
9977 // 'sub nsw C, x' produces [C - SINT_MAX, SINT_MAX].
9978 Lower = *C - APInt::getSignedMaxValue(numBits: Width);
9979 Upper = APInt::getSignedMinValue(numBits: Width);
9980 }
9981 }
9982 }
9983 break;
9984 case Instruction::Add:
9985 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
9986 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
9987 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
9988
9989 // If the caller expects a signed compare, then try to use a signed
9990 // range. Otherwise if both no-wraps are set, use the unsigned range
9991 // because it is never larger than the signed range. Example: "add nuw
9992 // nsw i8 X, -2" is unsigned [254,255] vs. signed [-128, 125].
9993 if (PreferSignedRange && HasNSW && HasNUW)
9994 HasNUW = false;
9995
9996 if (HasNUW) {
9997 // 'add nuw x, C' produces [C, UINT_MAX].
9998 Lower = *C;
9999 } else if (HasNSW) {
10000 if (C->isNegative()) {
10001 // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C].
10002 Lower = APInt::getSignedMinValue(numBits: Width);
10003 Upper = APInt::getSignedMaxValue(numBits: Width) + *C + 1;
10004 } else {
10005 // 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX].
10006 Lower = APInt::getSignedMinValue(numBits: Width) + *C;
10007 Upper = APInt::getSignedMaxValue(numBits: Width) + 1;
10008 }
10009 }
10010 }
10011 break;
10012
10013 case Instruction::And:
10014 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10015 // 'and x, C' produces [0, C].
10016 Upper = *C + 1;
10017 // X & -X is a power of two or zero. So we can cap the value at max power of
10018 // two.
10019 if (match(V: BO.getOperand(i_nocapture: 0), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 1)))) ||
10020 match(V: BO.getOperand(i_nocapture: 1), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 0)))))
10021 Upper = APInt::getSignedMinValue(numBits: Width) + 1;
10022 break;
10023
10024 case Instruction::Or:
10025 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10026 // 'or x, C' produces [C, UINT_MAX].
10027 Lower = *C;
10028 break;
10029
10030 case Instruction::AShr:
10031 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10032 // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C].
10033 Lower = APInt::getSignedMinValue(numBits: Width).ashr(ShiftAmt: *C);
10034 Upper = APInt::getSignedMaxValue(numBits: Width).ashr(ShiftAmt: *C) + 1;
10035 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10036 unsigned ShiftAmount = Width - 1;
10037 if (!C->isZero() && IIQ.isExact(Op: &BO))
10038 ShiftAmount = C->countr_zero();
10039 if (C->isNegative()) {
10040 // 'ashr C, x' produces [C, C >> (Width-1)]
10041 Lower = *C;
10042 Upper = C->ashr(ShiftAmt: ShiftAmount) + 1;
10043 } else {
10044 // 'ashr C, x' produces [C >> (Width-1), C]
10045 Lower = C->ashr(ShiftAmt: ShiftAmount);
10046 Upper = *C + 1;
10047 }
10048 }
10049 break;
10050
10051 case Instruction::LShr:
10052 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10053 // 'lshr x, C' produces [0, UINT_MAX >> C].
10054 Upper = APInt::getAllOnes(numBits: Width).lshr(ShiftAmt: *C) + 1;
10055 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10056 // 'lshr C, x' produces [C >> (Width-1), C].
10057 unsigned ShiftAmount = Width - 1;
10058 if (!C->isZero() && IIQ.isExact(Op: &BO))
10059 ShiftAmount = C->countr_zero();
10060 Lower = C->lshr(shiftAmt: ShiftAmount);
10061 Upper = *C + 1;
10062 }
10063 break;
10064
10065 case Instruction::Shl:
10066 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10067 if (IIQ.hasNoUnsignedWrap(Op: &BO)) {
10068 // 'shl nuw C, x' produces [C, C << CLZ(C)]
10069 Lower = *C;
10070 Upper = Lower.shl(shiftAmt: Lower.countl_zero()) + 1;
10071 } else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw?
10072 if (C->isNegative()) {
10073 // 'shl nsw C, x' produces [C << CLO(C)-1, C]
10074 unsigned ShiftAmount = C->countl_one() - 1;
10075 Lower = C->shl(shiftAmt: ShiftAmount);
10076 Upper = *C + 1;
10077 } else {
10078 // 'shl nsw C, x' produces [C, C << CLZ(C)-1]
10079 unsigned ShiftAmount = C->countl_zero() - 1;
10080 Lower = *C;
10081 Upper = C->shl(shiftAmt: ShiftAmount) + 1;
10082 }
10083 } else {
10084 // If lowbit is set, value can never be zero.
10085 if ((*C)[0])
10086 Lower = APInt::getOneBitSet(numBits: Width, BitNo: 0);
10087 // If we are shifting a constant the largest it can be is if the longest
10088 // sequence of consecutive ones is shifted to the highbits (breaking
10089 // ties for which sequence is higher). At the moment we take a liberal
10090 // upper bound on this by just popcounting the constant.
10091 // TODO: There may be a bitwise trick for it longest/highest
10092 // consecutative sequence of ones (naive method is O(Width) loop).
10093 Upper = APInt::getHighBitsSet(numBits: Width, hiBitsSet: C->popcount()) + 1;
10094 }
10095 } else if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10096 Upper = APInt::getBitsSetFrom(numBits: Width, loBit: C->getZExtValue()) + 1;
10097 }
10098 break;
10099
10100 case Instruction::SDiv:
10101 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10102 APInt IntMin = APInt::getSignedMinValue(numBits: Width);
10103 APInt IntMax = APInt::getSignedMaxValue(numBits: Width);
10104 if (C->isAllOnes()) {
10105 // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX]
10106 // where C != -1 and C != 0 and C != 1
10107 Lower = IntMin + 1;
10108 Upper = IntMax + 1;
10109 } else if (C->countl_zero() < Width - 1) {
10110 // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C]
10111 // where C != -1 and C != 0 and C != 1
10112 Lower = IntMin.sdiv(RHS: *C);
10113 Upper = IntMax.sdiv(RHS: *C);
10114 if (Lower.sgt(RHS: Upper))
10115 std::swap(a&: Lower, b&: Upper);
10116 Upper = Upper + 1;
10117 assert(Upper != Lower && "Upper part of range has wrapped!");
10118 }
10119 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10120 if (C->isMinSignedValue()) {
10121 // 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2].
10122 Lower = *C;
10123 Upper = Lower.lshr(shiftAmt: 1) + 1;
10124 } else {
10125 // 'sdiv C, x' produces [-|C|, |C|].
10126 Upper = C->abs() + 1;
10127 Lower = (-Upper) + 1;
10128 }
10129 }
10130 break;
10131
10132 case Instruction::UDiv:
10133 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10134 // 'udiv x, C' produces [0, UINT_MAX / C].
10135 Upper = APInt::getMaxValue(numBits: Width).udiv(RHS: *C) + 1;
10136 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10137 // 'udiv C, x' produces [0, C].
10138 Upper = *C + 1;
10139 }
10140 break;
10141
10142 case Instruction::SRem:
10143 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10144 // 'srem x, C' produces (-|C|, |C|).
10145 Upper = C->abs();
10146 Lower = (-Upper) + 1;
10147 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10148 if (C->isNegative()) {
10149 // 'srem -|C|, x' produces [-|C|, 0].
10150 Upper = 1;
10151 Lower = *C;
10152 } else {
10153 // 'srem |C|, x' produces [0, |C|].
10154 Upper = *C + 1;
10155 }
10156 }
10157 break;
10158
10159 case Instruction::URem:
10160 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10161 // 'urem x, C' produces [0, C).
10162 Upper = *C;
10163 else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10164 // 'urem C, x' produces [0, C].
10165 Upper = *C + 1;
10166 break;
10167
10168 default:
10169 break;
10170 }
10171}
10172
10173static ConstantRange getRangeForIntrinsic(const IntrinsicInst &II,
10174 bool UseInstrInfo) {
10175 unsigned Width = II.getType()->getScalarSizeInBits();
10176 const APInt *C;
10177 switch (II.getIntrinsicID()) {
10178 case Intrinsic::ctlz:
10179 case Intrinsic::cttz: {
10180 APInt Upper(Width, Width);
10181 if (!UseInstrInfo || !match(V: II.getArgOperand(i: 1), P: m_One()))
10182 Upper += 1;
10183 // Maximum of set/clear bits is the bit width.
10184 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper);
10185 }
10186 case Intrinsic::ctpop:
10187 // Maximum of set/clear bits is the bit width.
10188 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10189 Upper: APInt(Width, Width) + 1);
10190 case Intrinsic::uadd_sat:
10191 // uadd.sat(x, C) produces [C, UINT_MAX].
10192 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10193 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10194 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10195 break;
10196 case Intrinsic::sadd_sat:
10197 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10198 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10199 if (C->isNegative())
10200 // sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)].
10201 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10202 Upper: APInt::getSignedMaxValue(numBits: Width) + *C +
10203 1);
10204
10205 // sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX].
10206 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) + *C,
10207 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10208 }
10209 break;
10210 case Intrinsic::usub_sat:
10211 // usub.sat(C, x) produces [0, C].
10212 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10213 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10214
10215 // usub.sat(x, C) produces [0, UINT_MAX - C].
10216 if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10217 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10218 Upper: APInt::getMaxValue(numBits: Width) - *C + 1);
10219 break;
10220 case Intrinsic::ssub_sat:
10221 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10222 if (C->isNegative())
10223 // ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)].
10224 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10225 Upper: *C - APInt::getSignedMinValue(numBits: Width) +
10226 1);
10227
10228 // ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX].
10229 return ConstantRange::getNonEmpty(Lower: *C - APInt::getSignedMaxValue(numBits: Width),
10230 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10231 } else if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10232 if (C->isNegative())
10233 // ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]:
10234 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) - *C,
10235 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10236
10237 // ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C].
10238 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10239 Upper: APInt::getSignedMaxValue(numBits: Width) - *C +
10240 1);
10241 }
10242 break;
10243 case Intrinsic::umin:
10244 case Intrinsic::umax:
10245 case Intrinsic::smin:
10246 case Intrinsic::smax:
10247 if (!match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) &&
10248 !match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10249 break;
10250
10251 switch (II.getIntrinsicID()) {
10252 case Intrinsic::umin:
10253 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10254 case Intrinsic::umax:
10255 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10256 case Intrinsic::smin:
10257 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10258 Upper: *C + 1);
10259 case Intrinsic::smax:
10260 return ConstantRange::getNonEmpty(Lower: *C,
10261 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10262 default:
10263 llvm_unreachable("Must be min/max intrinsic");
10264 }
10265 break;
10266 case Intrinsic::abs:
10267 // If abs of SIGNED_MIN is poison, then the result is [0..SIGNED_MAX],
10268 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10269 if (match(V: II.getOperand(i_nocapture: 1), P: m_One()))
10270 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10271 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10272
10273 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10274 Upper: APInt::getSignedMinValue(numBits: Width) + 1);
10275 case Intrinsic::vscale:
10276 if (!II.getParent() || !II.getFunction())
10277 break;
10278 return getVScaleRange(F: II.getFunction(), BitWidth: Width);
10279 default:
10280 break;
10281 }
10282
10283 return ConstantRange::getFull(BitWidth: Width);
10284}
10285
10286static ConstantRange getRangeForSelectPattern(const SelectInst &SI,
10287 const InstrInfoQuery &IIQ) {
10288 unsigned BitWidth = SI.getType()->getScalarSizeInBits();
10289 const Value *LHS = nullptr, *RHS = nullptr;
10290 SelectPatternResult R = matchSelectPattern(V: &SI, LHS, RHS);
10291 if (R.Flavor == SPF_UNKNOWN)
10292 return ConstantRange::getFull(BitWidth);
10293
10294 if (R.Flavor == SelectPatternFlavor::SPF_ABS) {
10295 // If the negation part of the abs (in RHS) has the NSW flag,
10296 // then the result of abs(X) is [0..SIGNED_MAX],
10297 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10298 if (match(V: RHS, P: m_Neg(V: m_Specific(V: LHS))) &&
10299 IIQ.hasNoSignedWrap(Op: cast<Instruction>(Val: RHS)))
10300 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10301 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10302
10303 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10304 Upper: APInt::getSignedMinValue(numBits: BitWidth) + 1);
10305 }
10306
10307 if (R.Flavor == SelectPatternFlavor::SPF_NABS) {
10308 // The result of -abs(X) is <= 0.
10309 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10310 Upper: APInt(BitWidth, 1));
10311 }
10312
10313 const APInt *C;
10314 if (!match(V: LHS, P: m_APInt(Res&: C)) && !match(V: RHS, P: m_APInt(Res&: C)))
10315 return ConstantRange::getFull(BitWidth);
10316
10317 switch (R.Flavor) {
10318 case SPF_UMIN:
10319 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth), Upper: *C + 1);
10320 case SPF_UMAX:
10321 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: BitWidth));
10322 case SPF_SMIN:
10323 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10324 Upper: *C + 1);
10325 case SPF_SMAX:
10326 return ConstantRange::getNonEmpty(Lower: *C,
10327 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10328 default:
10329 return ConstantRange::getFull(BitWidth);
10330 }
10331}
10332
10333static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) {
10334 // The maximum representable value of a half is 65504. For floats the maximum
10335 // value is 3.4e38 which requires roughly 129 bits.
10336 unsigned BitWidth = I->getType()->getScalarSizeInBits();
10337 if (!I->getOperand(i: 0)->getType()->getScalarType()->isHalfTy())
10338 return;
10339 if (isa<FPToSIInst>(Val: I) && BitWidth >= 17) {
10340 Lower = APInt(BitWidth, -65504, true);
10341 Upper = APInt(BitWidth, 65505);
10342 }
10343
10344 if (isa<FPToUIInst>(Val: I) && BitWidth >= 16) {
10345 // For a fptoui the lower limit is left as 0.
10346 Upper = APInt(BitWidth, 65505);
10347 }
10348}
10349
10350ConstantRange llvm::computeConstantRange(const Value *V, bool ForSigned,
10351 bool UseInstrInfo, AssumptionCache *AC,
10352 const Instruction *CtxI,
10353 const DominatorTree *DT,
10354 unsigned Depth) {
10355 assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction");
10356
10357 if (Depth == MaxAnalysisRecursionDepth)
10358 return ConstantRange::getFull(BitWidth: V->getType()->getScalarSizeInBits());
10359
10360 if (auto *C = dyn_cast<Constant>(Val: V))
10361 return C->toConstantRange();
10362
10363 unsigned BitWidth = V->getType()->getScalarSizeInBits();
10364 InstrInfoQuery IIQ(UseInstrInfo);
10365 ConstantRange CR = ConstantRange::getFull(BitWidth);
10366 if (auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
10367 APInt Lower = APInt(BitWidth, 0);
10368 APInt Upper = APInt(BitWidth, 0);
10369 // TODO: Return ConstantRange.
10370 setLimitsForBinOp(BO: *BO, Lower, Upper, IIQ, PreferSignedRange: ForSigned);
10371 CR = ConstantRange::getNonEmpty(Lower, Upper);
10372 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
10373 CR = getRangeForIntrinsic(II: *II, UseInstrInfo);
10374 else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
10375 ConstantRange CRTrue = computeConstantRange(
10376 V: SI->getTrueValue(), ForSigned, UseInstrInfo, AC, CtxI, DT, Depth: Depth + 1);
10377 ConstantRange CRFalse = computeConstantRange(
10378 V: SI->getFalseValue(), ForSigned, UseInstrInfo, AC, CtxI, DT, Depth: Depth + 1);
10379 CR = CRTrue.unionWith(CR: CRFalse);
10380 CR = CR.intersectWith(CR: getRangeForSelectPattern(SI: *SI, IIQ));
10381 } else if (isa<FPToUIInst>(Val: V) || isa<FPToSIInst>(Val: V)) {
10382 APInt Lower = APInt(BitWidth, 0);
10383 APInt Upper = APInt(BitWidth, 0);
10384 // TODO: Return ConstantRange.
10385 setLimitForFPToI(I: cast<Instruction>(Val: V), Lower, Upper);
10386 CR = ConstantRange::getNonEmpty(Lower, Upper);
10387 } else if (const auto *A = dyn_cast<Argument>(Val: V))
10388 if (std::optional<ConstantRange> Range = A->getRange())
10389 CR = *Range;
10390
10391 if (auto *I = dyn_cast<Instruction>(Val: V)) {
10392 if (auto *Range = IIQ.getMetadata(I, KindID: LLVMContext::MD_range))
10393 CR = CR.intersectWith(CR: getConstantRangeFromMetadata(RangeMD: *Range));
10394
10395 if (const auto *CB = dyn_cast<CallBase>(Val: V))
10396 if (std::optional<ConstantRange> Range = CB->getRange())
10397 CR = CR.intersectWith(CR: *Range);
10398 }
10399
10400 if (CtxI && AC) {
10401 // Try to restrict the range based on information from assumptions.
10402 for (auto &AssumeVH : AC->assumptionsFor(V)) {
10403 if (!AssumeVH)
10404 continue;
10405 CallInst *I = cast<CallInst>(Val&: AssumeVH);
10406 assert(I->getParent()->getParent() == CtxI->getParent()->getParent() &&
10407 "Got assumption for the wrong function!");
10408 assert(I->getIntrinsicID() == Intrinsic::assume &&
10409 "must be an assume intrinsic");
10410
10411 if (!isValidAssumeForContext(Inv: I, CxtI: CtxI, DT))
10412 continue;
10413 Value *Arg = I->getArgOperand(i: 0);
10414 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
10415 // Currently we just use information from comparisons.
10416 if (!Cmp || Cmp->getOperand(i_nocapture: 0) != V)
10417 continue;
10418 // TODO: Set "ForSigned" parameter via Cmp->isSigned()?
10419 ConstantRange RHS =
10420 computeConstantRange(V: Cmp->getOperand(i_nocapture: 1), /* ForSigned */ false,
10421 UseInstrInfo, AC, CtxI: I, DT, Depth: Depth + 1);
10422 CR = CR.intersectWith(
10423 CR: ConstantRange::makeAllowedICmpRegion(Pred: Cmp->getPredicate(), Other: RHS));
10424 }
10425 }
10426
10427 return CR;
10428}
10429
10430static void
10431addValueAffectedByCondition(Value *V,
10432 function_ref<void(Value *)> InsertAffected) {
10433 assert(V != nullptr);
10434 if (isa<Argument>(Val: V) || isa<GlobalValue>(Val: V)) {
10435 InsertAffected(V);
10436 } else if (auto *I = dyn_cast<Instruction>(Val: V)) {
10437 InsertAffected(V);
10438
10439 // Peek through unary operators to find the source of the condition.
10440 Value *Op;
10441 if (match(V: I, P: m_CombineOr(L: m_PtrToIntOrAddr(Op: m_Value(V&: Op)),
10442 R: m_Trunc(Op: m_Value(V&: Op))))) {
10443 if (isa<Instruction>(Val: Op) || isa<Argument>(Val: Op))
10444 InsertAffected(Op);
10445 }
10446 }
10447}
10448
10449void llvm::findValuesAffectedByCondition(
10450 Value *Cond, bool IsAssume, function_ref<void(Value *)> InsertAffected) {
10451 auto AddAffected = [&InsertAffected](Value *V) {
10452 addValueAffectedByCondition(V, InsertAffected);
10453 };
10454
10455 auto AddCmpOperands = [&AddAffected, IsAssume](Value *LHS, Value *RHS) {
10456 if (IsAssume) {
10457 AddAffected(LHS);
10458 AddAffected(RHS);
10459 } else if (match(V: RHS, P: m_Constant()))
10460 AddAffected(LHS);
10461 };
10462
10463 SmallVector<Value *, 8> Worklist;
10464 SmallPtrSet<Value *, 8> Visited;
10465 Worklist.push_back(Elt: Cond);
10466 while (!Worklist.empty()) {
10467 Value *V = Worklist.pop_back_val();
10468 if (!Visited.insert(Ptr: V).second)
10469 continue;
10470
10471 CmpPredicate Pred;
10472 Value *A, *B, *X;
10473
10474 if (IsAssume) {
10475 AddAffected(V);
10476 if (match(V, P: m_Not(V: m_Value(V&: X))))
10477 AddAffected(X);
10478 }
10479
10480 if (match(V, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
10481 // assume(A && B) is split to -> assume(A); assume(B);
10482 // assume(!(A || B)) is split to -> assume(!A); assume(!B);
10483 // Finally, assume(A || B) / assume(!(A && B)) generally don't provide
10484 // enough information to be worth handling (intersection of information as
10485 // opposed to union).
10486 if (!IsAssume) {
10487 Worklist.push_back(Elt: A);
10488 Worklist.push_back(Elt: B);
10489 }
10490 } else if (match(V, P: m_ICmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
10491 bool HasRHSC = match(V: B, P: m_ConstantInt());
10492 if (ICmpInst::isEquality(P: Pred)) {
10493 AddAffected(A);
10494 if (IsAssume)
10495 AddAffected(B);
10496 if (HasRHSC) {
10497 Value *Y;
10498 // (X << C) or (X >>_s C) or (X >>_u C).
10499 if (match(V: A, P: m_Shift(L: m_Value(V&: X), R: m_ConstantInt())))
10500 AddAffected(X);
10501 // (X & C) or (X | C).
10502 else if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10503 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10504 AddAffected(X);
10505 AddAffected(Y);
10506 }
10507 // X - Y
10508 else if (match(V: A, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10509 AddAffected(X);
10510 AddAffected(Y);
10511 }
10512 }
10513 } else {
10514 AddCmpOperands(A, B);
10515 if (HasRHSC) {
10516 // Handle (A + C1) u< C2, which is the canonical form of
10517 // A > C3 && A < C4.
10518 if (match(V: A, P: m_AddLike(L: m_Value(V&: X), R: m_ConstantInt())))
10519 AddAffected(X);
10520
10521 if (ICmpInst::isUnsigned(Pred)) {
10522 Value *Y;
10523 // X & Y u> C -> X >u C && Y >u C
10524 // X | Y u< C -> X u< C && Y u< C
10525 // X nuw+ Y u< C -> X u< C && Y u< C
10526 if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10527 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10528 match(V: A, P: m_NUWAdd(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10529 AddAffected(X);
10530 AddAffected(Y);
10531 }
10532 // X nuw- Y u> C -> X u> C
10533 if (match(V: A, P: m_NUWSub(L: m_Value(V&: X), R: m_Value())))
10534 AddAffected(X);
10535 }
10536 }
10537
10538 // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported
10539 // by computeKnownFPClass().
10540 if (match(V: A, P: m_ElementWiseBitCast(Op: m_Value(V&: X)))) {
10541 if (Pred == ICmpInst::ICMP_SLT && match(V: B, P: m_Zero()))
10542 InsertAffected(X);
10543 else if (Pred == ICmpInst::ICMP_SGT && match(V: B, P: m_AllOnes()))
10544 InsertAffected(X);
10545 }
10546 }
10547
10548 if (HasRHSC && match(V: A, P: m_Intrinsic<Intrinsic::ctpop>(Op0: m_Value(V&: X))))
10549 AddAffected(X);
10550 } else if (match(V, P: m_FCmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
10551 AddCmpOperands(A, B);
10552
10553 // fcmp fneg(x), y
10554 // fcmp fabs(x), y
10555 // fcmp fneg(fabs(x)), y
10556 if (match(V: A, P: m_FNeg(X: m_Value(V&: A))))
10557 AddAffected(A);
10558 if (match(V: A, P: m_FAbs(Op0: m_Value(V&: A))))
10559 AddAffected(A);
10560
10561 } else if (match(V, P: m_Intrinsic<Intrinsic::is_fpclass>(Op0: m_Value(V&: A),
10562 Op1: m_Value()))) {
10563 // Handle patterns that computeKnownFPClass() support.
10564 AddAffected(A);
10565 } else if (!IsAssume && match(V, P: m_Trunc(Op: m_Value(V&: X)))) {
10566 // Assume is checked here as X is already added above for assumes in
10567 // addValueAffectedByCondition
10568 AddAffected(X);
10569 } else if (!IsAssume && match(V, P: m_Not(V: m_Value(V&: X)))) {
10570 // Assume is checked here to avoid issues with ephemeral values
10571 Worklist.push_back(Elt: X);
10572 }
10573 }
10574}
10575
10576const Value *llvm::stripNullTest(const Value *V) {
10577 // (X >> C) or/add (X & mask(C) != 0)
10578 if (const auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
10579 if (BO->getOpcode() == Instruction::Add ||
10580 BO->getOpcode() == Instruction::Or) {
10581 const Value *X;
10582 const APInt *C1, *C2;
10583 if (match(V: BO, P: m_c_BinOp(L: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: C1)),
10584 R: m_ZExt(Op: m_SpecificICmp(
10585 MatchPred: ICmpInst::ICMP_NE,
10586 L: m_And(L: m_Deferred(V: X), R: m_LowBitMask(V&: C2)),
10587 R: m_Zero())))) &&
10588 C2->popcount() == C1->getZExtValue())
10589 return X;
10590 }
10591 }
10592 return nullptr;
10593}
10594
10595Value *llvm::stripNullTest(Value *V) {
10596 return const_cast<Value *>(stripNullTest(V: const_cast<const Value *>(V)));
10597}
10598
10599bool llvm::collectPossibleValues(const Value *V,
10600 SmallPtrSetImpl<const Constant *> &Constants,
10601 unsigned MaxCount, bool AllowUndefOrPoison) {
10602 SmallPtrSet<const Instruction *, 8> Visited;
10603 SmallVector<const Instruction *, 8> Worklist;
10604 auto Push = [&](const Value *V) -> bool {
10605 Constant *C;
10606 if (match(V: const_cast<Value *>(V), P: m_ImmConstant(C))) {
10607 if (!AllowUndefOrPoison && !isGuaranteedNotToBeUndefOrPoison(V: C))
10608 return false;
10609 // Check existence first to avoid unnecessary allocations.
10610 if (Constants.contains(Ptr: C))
10611 return true;
10612 if (Constants.size() == MaxCount)
10613 return false;
10614 Constants.insert(Ptr: C);
10615 return true;
10616 }
10617
10618 if (auto *Inst = dyn_cast<Instruction>(Val: V)) {
10619 if (Visited.insert(Ptr: Inst).second)
10620 Worklist.push_back(Elt: Inst);
10621 return true;
10622 }
10623 return false;
10624 };
10625 if (!Push(V))
10626 return false;
10627 while (!Worklist.empty()) {
10628 const Instruction *CurInst = Worklist.pop_back_val();
10629 switch (CurInst->getOpcode()) {
10630 case Instruction::Select:
10631 if (!Push(CurInst->getOperand(i: 1)))
10632 return false;
10633 if (!Push(CurInst->getOperand(i: 2)))
10634 return false;
10635 break;
10636 case Instruction::PHI:
10637 for (Value *IncomingValue : cast<PHINode>(Val: CurInst)->incoming_values()) {
10638 // Fast path for recurrence PHI.
10639 if (IncomingValue == CurInst)
10640 continue;
10641 if (!Push(IncomingValue))
10642 return false;
10643 }
10644 break;
10645 default:
10646 return false;
10647 }
10648 }
10649 return true;
10650}
10651