1//===- ValueTracking.cpp - Walk computations to compute properties --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains routines that help analyze properties that chains of
10// computations have.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ValueTracking.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/FloatingPointMode.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/ScopeExit.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/ADT/iterator_range.h"
25#include "llvm/Analysis/AliasAnalysis.h"
26#include "llvm/Analysis/AssumeBundleQueries.h"
27#include "llvm/Analysis/AssumptionCache.h"
28#include "llvm/Analysis/ConstantFolding.h"
29#include "llvm/Analysis/DomConditionCache.h"
30#include "llvm/Analysis/FloatingPointPredicateUtils.h"
31#include "llvm/Analysis/GuardUtils.h"
32#include "llvm/Analysis/InstructionSimplify.h"
33#include "llvm/Analysis/Loads.h"
34#include "llvm/Analysis/LoopInfo.h"
35#include "llvm/Analysis/TargetLibraryInfo.h"
36#include "llvm/Analysis/VectorUtils.h"
37#include "llvm/Analysis/WithCache.h"
38#include "llvm/IR/Argument.h"
39#include "llvm/IR/Attributes.h"
40#include "llvm/IR/BasicBlock.h"
41#include "llvm/IR/BundleAttributes.h"
42#include "llvm/IR/Constant.h"
43#include "llvm/IR/ConstantFPRange.h"
44#include "llvm/IR/ConstantRange.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/DiagnosticInfo.h"
48#include "llvm/IR/Dominators.h"
49#include "llvm/IR/EHPersonalities.h"
50#include "llvm/IR/Function.h"
51#include "llvm/IR/GetElementPtrTypeIterator.h"
52#include "llvm/IR/GlobalAlias.h"
53#include "llvm/IR/GlobalValue.h"
54#include "llvm/IR/GlobalVariable.h"
55#include "llvm/IR/InstrTypes.h"
56#include "llvm/IR/Instruction.h"
57#include "llvm/IR/Instructions.h"
58#include "llvm/IR/IntrinsicInst.h"
59#include "llvm/IR/Intrinsics.h"
60#include "llvm/IR/IntrinsicsAArch64.h"
61#include "llvm/IR/IntrinsicsAMDGPU.h"
62#include "llvm/IR/IntrinsicsRISCV.h"
63#include "llvm/IR/IntrinsicsX86.h"
64#include "llvm/IR/LLVMContext.h"
65#include "llvm/IR/Metadata.h"
66#include "llvm/IR/Module.h"
67#include "llvm/IR/Operator.h"
68#include "llvm/IR/PatternMatch.h"
69#include "llvm/IR/Type.h"
70#include "llvm/IR/User.h"
71#include "llvm/IR/Value.h"
72#include "llvm/Support/Casting.h"
73#include "llvm/Support/CommandLine.h"
74#include "llvm/Support/Compiler.h"
75#include "llvm/Support/ErrorHandling.h"
76#include "llvm/Support/KnownBits.h"
77#include "llvm/Support/KnownFPClass.h"
78#include "llvm/Support/MathExtras.h"
79#include "llvm/Support/UndefPoison.h"
80#include "llvm/TargetParser/RISCVTargetParser.h"
81#include <algorithm>
82#include <cassert>
83#include <cstdint>
84#include <optional>
85#include <utility>
86
87using namespace llvm;
88using namespace llvm::PatternMatch;
89
90// Controls the number of uses of the value searched for possible
91// dominating comparisons.
92static cl::opt<unsigned> DomConditionsMaxUses("dom-conditions-max-uses",
93 cl::Hidden, cl::init(Val: 20));
94
95/// Maximum number of instructions to check between assume and context
96/// instruction.
97static constexpr unsigned MaxInstrsToCheckForFree = 32;
98
99/// Returns the bitwidth of the given scalar or pointer type. For vector types,
100/// returns the element type's bitwidth.
101static unsigned getBitWidth(Type *Ty, const DataLayout &DL) {
102 if (unsigned BitWidth = Ty->getScalarSizeInBits())
103 return BitWidth;
104
105 return DL.getPointerTypeSizeInBits(Ty);
106}
107
108// Given the provided Value and, potentially, a context instruction, return
109// the preferred context instruction (if any).
110static const Instruction *safeCxtI(const Value *V, const Instruction *CxtI) {
111 // If we've been provided with a context instruction, then use that (provided
112 // it has been inserted).
113 if (CxtI && CxtI->getParent())
114 return CxtI;
115
116 // If the value is really an already-inserted instruction, then use that.
117 CxtI = dyn_cast<Instruction>(Val: V);
118 if (CxtI && CxtI->getParent())
119 return CxtI;
120
121 return nullptr;
122}
123
124static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
125 const APInt &DemandedElts,
126 APInt &DemandedLHS, APInt &DemandedRHS) {
127 if (isa<ScalableVectorType>(Val: Shuf->getType())) {
128 assert(DemandedElts == APInt(1,1));
129 DemandedLHS = DemandedRHS = DemandedElts;
130 return true;
131 }
132
133 int NumElts =
134 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
135 return llvm::getShuffleDemandedElts(SrcWidth: NumElts, Mask: Shuf->getShuffleMask(),
136 DemandedElts, DemandedLHS, DemandedRHS);
137}
138
139static void computeKnownBits(const Value *V, const APInt &DemandedElts,
140 KnownBits &Known, const SimplifyQuery &Q,
141 unsigned Depth);
142
143void llvm::computeKnownBits(const Value *V, KnownBits &Known,
144 const SimplifyQuery &Q, unsigned Depth) {
145 // Since the number of lanes in a scalable vector is unknown at compile time,
146 // we track one bit which is implicitly broadcast to all lanes. This means
147 // that all lanes in a scalable vector are considered demanded.
148 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
149 APInt DemandedElts =
150 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
151 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
152}
153
154void llvm::computeKnownBits(const Value *V, KnownBits &Known,
155 const DataLayout &DL, AssumptionCache *AC,
156 const Instruction *CxtI, const DominatorTree *DT,
157 bool UseInstrInfo, unsigned Depth) {
158 computeKnownBits(V, Known,
159 Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo),
160 Depth);
161}
162
163KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
164 AssumptionCache *AC, const Instruction *CxtI,
165 const DominatorTree *DT, bool UseInstrInfo,
166 unsigned Depth) {
167 return computeKnownBits(
168 V, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
169}
170
171KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
172 const DataLayout &DL, AssumptionCache *AC,
173 const Instruction *CxtI,
174 const DominatorTree *DT, bool UseInstrInfo,
175 unsigned Depth) {
176 return computeKnownBits(
177 V, DemandedElts,
178 Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
179}
180
181static bool haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
182 const SimplifyQuery &SQ) {
183 // Look for an inverted mask: (X & ~M) op (Y & M).
184 {
185 Value *M;
186 if (match(V: LHS, P: m_c_And(L: m_Not(V: m_Value(V&: M)), R: m_Value())) &&
187 match(V: RHS, P: m_c_And(L: m_Specific(V: M), R: m_Value())) &&
188 isGuaranteedNotToBeUndef(V: M, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
189 return true;
190 }
191
192 // X op (Y & ~X)
193 if (match(V: RHS, P: m_c_And(L: m_Not(V: m_Specific(V: LHS)), R: m_Value())) &&
194 isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
195 return true;
196
197 // X op ((X & Y) ^ Y) -- this is the canonical form of the previous pattern
198 // for constant Y.
199 Value *Y;
200 if (match(V: RHS,
201 P: m_c_Xor(L: m_c_And(L: m_Specific(V: LHS), R: m_Value(V&: Y)), R: m_Deferred(V: Y))) &&
202 isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT) &&
203 isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
204 return true;
205
206 // Peek through extends to find a 'not' of the other side:
207 // (ext Y) op ext(~Y)
208 if (match(V: LHS, P: m_ZExtOrSExt(Op: m_Value(V&: Y))) &&
209 match(V: RHS, P: m_ZExtOrSExt(Op: m_Not(V: m_Specific(V: Y)))) &&
210 isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
211 return true;
212
213 // Look for: (A & B) op ~(A | B)
214 {
215 Value *A, *B;
216 if (match(V: LHS, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
217 match(V: RHS, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B)))) &&
218 isGuaranteedNotToBeUndef(V: A, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT) &&
219 isGuaranteedNotToBeUndef(V: B, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
220 return true;
221 }
222
223 // Look for: (X << V) op (Y >> (BitWidth - V))
224 // or (X >> V) op (Y << (BitWidth - V))
225 {
226 const Value *V;
227 const APInt *R;
228 if (((match(V: RHS, P: m_Shl(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
229 match(V: LHS, P: m_LShr(L: m_Value(), R: m_Specific(V)))) ||
230 (match(V: RHS, P: m_LShr(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
231 match(V: LHS, P: m_Shl(L: m_Value(), R: m_Specific(V))))) &&
232 R->uge(RHS: LHS->getType()->getScalarSizeInBits()))
233 return true;
234 }
235
236 return false;
237}
238
239bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
240 const WithCache<const Value *> &RHSCache,
241 const SimplifyQuery &SQ) {
242 const Value *LHS = LHSCache.getValue();
243 const Value *RHS = RHSCache.getValue();
244
245 assert(LHS->getType() == RHS->getType() &&
246 "LHS and RHS should have the same type");
247 assert(LHS->getType()->isIntOrIntVectorTy() &&
248 "LHS and RHS should be integers");
249
250 if (haveNoCommonBitsSetSpecialCases(LHS, RHS, SQ) ||
251 haveNoCommonBitsSetSpecialCases(LHS: RHS, RHS: LHS, SQ))
252 return true;
253
254 return KnownBits::haveNoCommonBitsSet(LHS: LHSCache.getKnownBits(Q: SQ),
255 RHS: RHSCache.getKnownBits(Q: SQ));
256}
257
258bool llvm::isOnlyUsedInZeroComparison(const Instruction *I) {
259 return !I->user_empty() &&
260 all_of(Range: I->users(), P: match_fn(P: m_ICmp(L: m_Value(), R: m_Zero())));
261}
262
263bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) {
264 return !I->user_empty() && all_of(Range: I->users(), P: [](const User *U) {
265 CmpPredicate P;
266 return match(V: U, P: m_ICmp(Pred&: P, L: m_Value(), R: m_Zero())) && ICmpInst::isEquality(P);
267 });
268}
269
270bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
271 bool OrZero, AssumptionCache *AC,
272 const Instruction *CxtI,
273 const DominatorTree *DT, bool UseInstrInfo,
274 unsigned Depth) {
275 return ::isKnownToBeAPowerOfTwo(
276 V, OrZero, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo),
277 Depth);
278}
279
280static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
281 const SimplifyQuery &Q, unsigned Depth);
282
283bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
284 unsigned Depth) {
285 return computeKnownBits(V, Q: SQ, Depth).isNonNegative();
286}
287
288bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
289 unsigned Depth) {
290 if (auto *CI = dyn_cast<ConstantInt>(Val: V))
291 return CI->getValue().isStrictlyPositive();
292
293 // If `isKnownNonNegative` ever becomes more sophisticated, make sure to keep
294 // this updated.
295 KnownBits Known = computeKnownBits(V, Q: SQ, Depth);
296 return Known.isNonNegative() &&
297 (Known.isNonZero() || isKnownNonZero(V, Q: SQ, Depth));
298}
299
300bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ,
301 unsigned Depth) {
302 return computeKnownBits(V, Q: SQ, Depth).isNegative();
303}
304
305static bool isKnownNonEqual(const Value *V1, const Value *V2,
306 const APInt &DemandedElts, const SimplifyQuery &Q,
307 unsigned Depth);
308
309bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
310 const SimplifyQuery &Q, unsigned Depth) {
311 // We don't support looking through casts.
312 if (V1 == V2 || V1->getType() != V2->getType())
313 return false;
314 auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType());
315 APInt DemandedElts =
316 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
317 return ::isKnownNonEqual(V1, V2, DemandedElts, Q, Depth);
318}
319
320bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
321 const SimplifyQuery &SQ, unsigned Depth) {
322 KnownBits Known(Mask.getBitWidth());
323 computeKnownBits(V, Known, Q: SQ, Depth);
324 return Mask.isSubsetOf(RHS: Known.Zero);
325}
326
327static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
328 const SimplifyQuery &Q, unsigned Depth);
329
330static unsigned ComputeNumSignBits(const Value *V, const SimplifyQuery &Q,
331 unsigned Depth = 0) {
332 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
333 APInt DemandedElts =
334 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
335 return ComputeNumSignBits(V, DemandedElts, Q, Depth);
336}
337
338unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL,
339 AssumptionCache *AC, const Instruction *CxtI,
340 const DominatorTree *DT, bool UseInstrInfo,
341 unsigned Depth) {
342 return ::ComputeNumSignBits(
343 V, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
344}
345
346unsigned llvm::ComputeMaxSignificantBits(const Value *V, const DataLayout &DL,
347 AssumptionCache *AC,
348 const Instruction *CxtI,
349 const DominatorTree *DT,
350 unsigned Depth) {
351 unsigned SignBits = ComputeNumSignBits(V, DL, AC, CxtI, DT, UseInstrInfo: Depth);
352 return V->getType()->getScalarSizeInBits() - SignBits + 1;
353}
354
355/// Try to detect the lerp pattern: a * (b - c) + c * d
356/// where a >= 0, b >= 0, c >= 0, d >= 0, and b >= c.
357///
358/// In that particular case, we can use the following chain of reasoning:
359///
360/// a * (b - c) + c * d <= a' * (b - c) + a' * c = a' * b where a' = max(a, d)
361///
362/// Since that is true for arbitrary a, b, c and d within our constraints, we
363/// can conclude that:
364///
365/// max(a * (b - c) + c * d) <= max(max(a), max(d)) * max(b) = U
366///
367/// Considering that any result of the lerp would be less or equal to U, it
368/// would have at least the number of leading 0s as in U.
369///
370/// While being quite a specific situation, it is fairly common in computer
371/// graphics in the shape of alpha blending.
372///
373/// Modifies given KnownOut in-place with the inferred information.
374static void computeKnownBitsFromLerpPattern(const Value *Op0, const Value *Op1,
375 const APInt &DemandedElts,
376 KnownBits &KnownOut,
377 const SimplifyQuery &Q,
378 unsigned Depth) {
379
380 Type *Ty = Op0->getType();
381 const unsigned BitWidth = Ty->getScalarSizeInBits();
382
383 // Only handle scalar types for now
384 if (Ty->isVectorTy())
385 return;
386
387 // Try to match: a * (b - c) + c * d.
388 // When a == 1 => A == nullptr, the same applies to d/D as well.
389 const Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
390 const Instruction *SubBC = nullptr;
391
392 const auto MatchSubBC = [&]() {
393 // (b - c) can have two forms that interest us:
394 //
395 // 1. sub nuw %b, %c
396 // 2. xor %c, %b
397 //
398 // For the first case, nuw flag guarantees our requirement b >= c.
399 //
400 // The second case might happen when the analysis can infer that b is a mask
401 // for c and we can transform sub operation into xor (that is usually true
402 // for constant b's). Even though xor is symmetrical, canonicalization
403 // ensures that the constant will be the RHS. We have additional checks
404 // later on to ensure that this xor operation is equivalent to subtraction.
405 return m_Instruction(I&: SubBC, P: m_CombineOr(Ps: m_NUWSub(L: m_Value(V&: B), R: m_Value(V&: C)),
406 Ps: m_Xor(L: m_Value(V&: C), R: m_Value(V&: B))));
407 };
408
409 const auto MatchASubBC = [&]() {
410 // Cases:
411 // - a * (b - c)
412 // - (b - c) * a
413 // - (b - c) <- a implicitly equals 1
414 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: A), R: MatchSubBC()), Ps: MatchSubBC());
415 };
416
417 const auto MatchCD = [&]() {
418 // Cases:
419 // - d * c
420 // - c * d
421 // - c <- d implicitly equals 1
422 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: D), R: m_Specific(V: C)), Ps: m_Specific(V: C));
423 };
424
425 const auto Match = [&](const Value *LHS, const Value *RHS) {
426 // We do use m_Specific(C) in MatchCD, so we have to make sure that
427 // it's bound to anything and match(LHS, MatchASubBC()) absolutely
428 // has to evaluate first and return true.
429 //
430 // If Match returns true, it is guaranteed that B != nullptr, C != nullptr.
431 return match(V: LHS, P: MatchASubBC()) && match(V: RHS, P: MatchCD());
432 };
433
434 if (!Match(Op0, Op1) && !Match(Op1, Op0))
435 return;
436
437 const auto ComputeKnownBitsOrOne = [&](const Value *V) {
438 // For some of the values we use the convention of leaving
439 // it nullptr to signify an implicit constant 1.
440 return V ? computeKnownBits(V, DemandedElts, Q, Depth: Depth + 1)
441 : KnownBits::makeConstant(C: APInt(BitWidth, 1));
442 };
443
444 // Check that all operands are non-negative
445 const KnownBits KnownA = ComputeKnownBitsOrOne(A);
446 if (!KnownA.isNonNegative())
447 return;
448
449 const KnownBits KnownD = ComputeKnownBitsOrOne(D);
450 if (!KnownD.isNonNegative())
451 return;
452
453 const KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
454 if (!KnownB.isNonNegative())
455 return;
456
457 const KnownBits KnownC = computeKnownBits(V: C, DemandedElts, Q, Depth: Depth + 1);
458 if (!KnownC.isNonNegative())
459 return;
460
461 // If we matched subtraction as xor, we need to actually check that xor
462 // is semantically equivalent to subtraction.
463 //
464 // For that to be true, b has to be a mask for c or that b's known
465 // ones cover all known and possible ones of c.
466 if (SubBC->getOpcode() == Instruction::Xor &&
467 !KnownC.getMaxValue().isSubsetOf(RHS: KnownB.getMinValue()))
468 return;
469
470 const APInt MaxA = KnownA.getMaxValue();
471 const APInt MaxD = KnownD.getMaxValue();
472 const APInt MaxAD = APIntOps::umax(A: MaxA, B: MaxD);
473 const APInt MaxB = KnownB.getMaxValue();
474
475 // We can't infer leading zeros info if the upper-bound estimate wraps.
476 bool Overflow;
477 const APInt UpperBound = MaxAD.umul_ov(RHS: MaxB, Overflow);
478
479 if (Overflow)
480 return;
481
482 // If we know that x <= y and both are positive than x has at least the same
483 // number of leading zeros as y.
484 const unsigned MinimumNumberOfLeadingZeros = UpperBound.countl_zero();
485 KnownOut.Zero.setHighBits(MinimumNumberOfLeadingZeros);
486}
487
488static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
489 bool NSW, bool NUW,
490 const APInt &DemandedElts,
491 KnownBits &KnownOut, KnownBits &Known2,
492 const SimplifyQuery &Q, unsigned Depth) {
493 computeKnownBits(V: Op1, DemandedElts, Known&: KnownOut, Q, Depth: Depth + 1);
494
495 // If one operand is unknown and we have no nowrap information,
496 // the result will be unknown independently of the second operand.
497 if (KnownOut.isUnknown() && !NSW && !NUW)
498 return;
499
500 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
501 KnownOut = KnownBits::computeForAddSub(Add, NSW, NUW, LHS: Known2, RHS: KnownOut);
502
503 if (!Add && NSW && !KnownOut.isNonNegative() &&
504 (isImpliedByDomCondition(Pred: ICmpInst::ICMP_SLE, LHS: Op1, RHS: Op0, ContextI: Q.CxtI, DL: Q.DL)
505 .value_or(u: false) ||
506 match(V: Op1, P: m_c_SMin(L: m_Specific(V: Op0), R: m_Value()))))
507 KnownOut.makeNonNegative();
508
509 if (Add)
510 // Try to match lerp pattern and combine results
511 computeKnownBitsFromLerpPattern(Op0, Op1, DemandedElts, KnownOut, Q, Depth);
512}
513
514static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
515 bool NUW, const APInt &DemandedElts,
516 KnownBits &Known, KnownBits &Known2,
517 const SimplifyQuery &Q, unsigned Depth) {
518 computeKnownBits(V: Op1, DemandedElts, Known, Q, Depth: Depth + 1);
519 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
520
521 bool isKnownNegative = false;
522 bool isKnownNonNegative = false;
523 // If the multiplication is known not to overflow, compute the sign bit.
524 if (NSW) {
525 if (Op0 == Op1) {
526 // The product of a number with itself is non-negative.
527 isKnownNonNegative = true;
528 } else {
529 bool isKnownNonNegativeOp1 = Known.isNonNegative();
530 bool isKnownNonNegativeOp0 = Known2.isNonNegative();
531 bool isKnownNegativeOp1 = Known.isNegative();
532 bool isKnownNegativeOp0 = Known2.isNegative();
533 // The product of two numbers with the same sign is non-negative.
534 isKnownNonNegative = (isKnownNegativeOp1 && isKnownNegativeOp0) ||
535 (isKnownNonNegativeOp1 && isKnownNonNegativeOp0);
536 if (!isKnownNonNegative && NUW) {
537 // mul nuw nsw with a factor > 1 is non-negative.
538 KnownBits One = KnownBits::makeConstant(C: APInt(Known.getBitWidth(), 1));
539 isKnownNonNegative = KnownBits::sgt(LHS: Known, RHS: One).value_or(u: false) ||
540 KnownBits::sgt(LHS: Known2, RHS: One).value_or(u: false);
541 }
542
543 // The product of a negative number and a non-negative number is either
544 // negative or zero.
545 if (!isKnownNonNegative)
546 isKnownNegative =
547 (isKnownNegativeOp1 && isKnownNonNegativeOp0 &&
548 Known2.isNonZero()) ||
549 (isKnownNegativeOp0 && isKnownNonNegativeOp1 && Known.isNonZero());
550 }
551 }
552
553 bool SelfMultiply = Op0 == Op1;
554 if (SelfMultiply)
555 SelfMultiply &=
556 isGuaranteedNotToBeUndef(V: Op0, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1);
557 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
558
559 if (SelfMultiply) {
560 unsigned SignBits = ComputeNumSignBits(V: Op0, DemandedElts, Q, Depth: Depth + 1);
561 unsigned TyBits = Op0->getType()->getScalarSizeInBits();
562 unsigned OutValidBits = 2 * (TyBits - SignBits + 1);
563
564 if (OutValidBits < TyBits) {
565 APInt KnownZeroMask =
566 APInt::getHighBitsSet(numBits: TyBits, hiBitsSet: TyBits - OutValidBits + 1);
567 Known.Zero |= KnownZeroMask;
568 }
569 }
570
571 // Only make use of no-wrap flags if we failed to compute the sign bit
572 // directly. This matters if the multiplication always overflows, in
573 // which case we prefer to follow the result of the direct computation,
574 // though as the program is invoking undefined behaviour we can choose
575 // whatever we like here.
576 if (isKnownNonNegative && !Known.isNegative())
577 Known.makeNonNegative();
578 else if (isKnownNegative && !Known.isNonNegative())
579 Known.makeNegative();
580}
581
582void llvm::computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
583 KnownBits &Known) {
584 unsigned BitWidth = Known.getBitWidth();
585 unsigned NumRanges = Ranges.getNumOperands() / 2;
586 assert(NumRanges >= 1);
587
588 Known.setAllConflict();
589
590 for (unsigned i = 0; i < NumRanges; ++i) {
591 ConstantInt *Lower =
592 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 0));
593 ConstantInt *Upper =
594 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 1));
595 ConstantRange Range(Lower->getValue(), Upper->getValue());
596 // BitWidth must equal the Ranges BitWidth for the correct number of high
597 // bits to be set.
598 assert(BitWidth == Range.getBitWidth() &&
599 "Known bit width must match range bit width!");
600
601 // The first CommonPrefixBits of all values in Range are equal.
602 unsigned CommonPrefixBits =
603 (Range.getUnsignedMax() ^ Range.getUnsignedMin()).countl_zero();
604 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: CommonPrefixBits);
605 APInt UnsignedMax = Range.getUnsignedMax().zextOrTrunc(width: BitWidth);
606 Known.One &= UnsignedMax & Mask;
607 Known.Zero &= ~UnsignedMax & Mask;
608 }
609}
610
611static bool isEphemeralValueOf(const Instruction *I, const Value *E) {
612 // The instruction defining an assumption's condition itself is always
613 // considered ephemeral to that assumption (even if it has other
614 // non-ephemeral users). See r246696's test case for an example.
615 if (is_contained(Range: I->operands(), Element: E))
616 return true;
617
618 const auto *EI = dyn_cast<Instruction>(Val: E);
619 if (!EI)
620 return false;
621
622 if (EI == I)
623 return true;
624
625 SmallPtrSet<const Instruction *, 16> Visited;
626 SmallVector<const Instruction *, 16> WorkList;
627 Visited.insert(Ptr: EI);
628 WorkList.push_back(Elt: EI);
629 bool ReachesI = false;
630 while (!WorkList.empty()) {
631 const Instruction *V = WorkList.pop_back_val();
632 for (const User *U : V->users()) {
633 const auto *UI = cast<Instruction>(Val: U);
634 if (UI == I) {
635 ReachesI = true;
636 continue;
637 }
638 if (UI->mayHaveSideEffects() || UI->isTerminator())
639 return false;
640 if (Visited.insert(Ptr: UI).second)
641 WorkList.push_back(Elt: UI);
642 }
643 }
644 return ReachesI;
645}
646
647// Is this an intrinsic that cannot be speculated but also cannot trap?
648bool llvm::isAssumeLikeIntrinsic(const Instruction *I) {
649 if (const IntrinsicInst *CI = dyn_cast<IntrinsicInst>(Val: I))
650 return CI->isAssumeLikeIntrinsic();
651
652 return false;
653}
654
655bool llvm::isValidAssumeForContext(const Instruction *Inv,
656 const Instruction *CxtI,
657 const DominatorTree *DT,
658 bool AllowEphemerals) {
659 // There are two restrictions on the use of an assume:
660 // 1. The assume must dominate the context (or the control flow must
661 // reach the assume whenever it reaches the context).
662 // 2. The context must not be in the assume's set of ephemeral values
663 // (otherwise we will use the assume to prove that the condition
664 // feeding the assume is trivially true, thus causing the removal of
665 // the assume).
666
667 if (Inv->getParent() == CxtI->getParent()) {
668 // If Inv and CtxI are in the same block, check if the assume (Inv) is first
669 // in the BB.
670 if (Inv->comesBefore(Other: CxtI))
671 return true;
672
673 // Don't let an assume affect itself - this would cause the problems
674 // `isEphemeralValueOf` is trying to prevent, and it would also make
675 // the loop below go out of bounds.
676 if (!AllowEphemerals && Inv == CxtI)
677 return false;
678
679 // The context comes first, but they're both in the same block.
680 // Make sure there is nothing in between that might interrupt
681 // the control flow, not even CxtI itself.
682 // We limit the scan distance between the assume and its context instruction
683 // to avoid a compile-time explosion. This limit is chosen arbitrarily, so
684 // it can be adjusted if needed (could be turned into a cl::opt).
685 auto Range = make_range(x: CxtI->getIterator(), y: Inv->getIterator());
686 if (!isGuaranteedToTransferExecutionToSuccessor(Range, ScanLimit: 15))
687 return false;
688
689 return AllowEphemerals || !isEphemeralValueOf(I: Inv, E: CxtI);
690 }
691
692 // Inv and CxtI are in different blocks.
693 if (DT) {
694 if (DT->dominates(Def: Inv, User: CxtI))
695 return true;
696 } else if (Inv->getParent() == CxtI->getParent()->getSinglePredecessor() ||
697 Inv->getParent()->isEntryBlock()) {
698 // We don't have a DT, but this trivially dominates.
699 return true;
700 }
701
702 return false;
703}
704
705bool llvm::willNotFreeBetween(const Instruction *Assume,
706 const Instruction *CtxI) {
707 // Helper to check if there are any calls in the range that may free memory.
708 unsigned NumChecked = 0;
709 auto hasNoFreeInRange = [&NumChecked](auto Range) {
710 for (const Instruction &I : Range) {
711 if (NumChecked++ > MaxInstrsToCheckForFree)
712 return false;
713
714 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
715 if (!CB->hasFnAttr(Kind: Attribute::NoFree))
716 return false;
717 } else if (I.maySynchronize())
718 return false;
719 }
720 return true;
721 };
722
723 const BasicBlock *CtxBB = CtxI->getParent();
724 const BasicBlock *AssumeBB = Assume->getParent();
725 BasicBlock::const_iterator CtxIter = CtxI->getIterator();
726 if (CtxBB == AssumeBB) {
727 // Same block case: check that Assume comes before CtxI.
728 if (Assume != CtxI && !Assume->comesBefore(Other: CtxI))
729 return false;
730 return hasNoFreeInRange(make_range(x: Assume->getIterator(), y: CtxIter));
731 }
732
733 // Handle chain of single-predecessor blocks.
734 const BasicBlock *CurBB = CtxBB;
735 while (true) {
736 if (CurBB == AssumeBB)
737 return hasNoFreeInRange(
738 make_range(x: Assume->getIterator(), y: AssumeBB->end()));
739
740 const BasicBlock *PredBB = CurBB->getSinglePredecessor();
741 if (!PredBB)
742 return false;
743
744 if (!hasNoFreeInRange(make_range(x: CurBB->begin(),
745 y: CurBB == CtxBB ? CtxIter : CurBB->end())))
746 return false;
747 CurBB = PredBB;
748 }
749}
750
751// TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but
752// we still have enough information about `RHS` to conclude non-zero. For
753// example Pred=EQ, RHS=isKnownNonZero. cmpExcludesZero is called in loops
754// so the extra compile time may not be worth it, but possibly a second API
755// should be created for use outside of loops.
756static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) {
757 // v u> y implies v != 0.
758 if (Pred == ICmpInst::ICMP_UGT)
759 return true;
760
761 // Special-case v != 0 to also handle v != null.
762 if (Pred == ICmpInst::ICMP_NE)
763 return match(V: RHS, P: m_Zero());
764
765 // All other predicates - rely on generic ConstantRange handling.
766 const APInt *C;
767 auto Zero = APInt::getZero(numBits: RHS->getType()->getScalarSizeInBits());
768 if (match(V: RHS, P: m_APInt(Res&: C))) {
769 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
770 return !TrueValues.contains(Val: Zero);
771 }
772
773 auto *VC = dyn_cast<ConstantDataVector>(Val: RHS);
774 if (VC == nullptr)
775 return false;
776
777 for (unsigned ElemIdx = 0, NElem = VC->getNumElements(); ElemIdx < NElem;
778 ++ElemIdx) {
779 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(
780 Pred, Other: VC->getElementAsAPInt(i: ElemIdx));
781 if (TrueValues.contains(Val: Zero))
782 return false;
783 }
784 return true;
785}
786
787static void breakSelfRecursivePHI(const Use *U, const PHINode *PHI,
788 Value *&ValOut, Instruction *&CtxIOut,
789 const PHINode **PhiOut = nullptr) {
790 ValOut = U->get();
791 if (ValOut == PHI)
792 return;
793 CtxIOut = PHI->getIncomingBlock(U: *U)->getTerminator();
794 if (PhiOut)
795 *PhiOut = PHI;
796 Value *V;
797 // If the Use is a select of this phi, compute analysis on other arm to break
798 // recursion.
799 // TODO: Min/Max
800 if (match(V: ValOut, P: m_Select(C: m_Value(), L: m_Specific(V: PHI), R: m_Value(V))) ||
801 match(V: ValOut, P: m_Select(C: m_Value(), L: m_Value(V), R: m_Specific(V: PHI))))
802 ValOut = V;
803
804 // Same for select, if this phi is 2-operand phi, compute analysis on other
805 // incoming value to break recursion.
806 // TODO: We could handle any number of incoming edges as long as we only have
807 // two unique values.
808 if (auto *IncPhi = dyn_cast<PHINode>(Val: ValOut);
809 IncPhi && IncPhi->getNumIncomingValues() == 2) {
810 for (int Idx = 0; Idx < 2; ++Idx) {
811 if (IncPhi->getIncomingValue(i: Idx) == PHI) {
812 ValOut = IncPhi->getIncomingValue(i: 1 - Idx);
813 if (PhiOut)
814 *PhiOut = IncPhi;
815 CtxIOut = IncPhi->getIncomingBlock(i: 1 - Idx)->getTerminator();
816 break;
817 }
818 }
819 }
820}
821
822static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) {
823 // Use of assumptions is context-sensitive. If we don't have a context, we
824 // cannot use them!
825 if (!Q.AC || !Q.CxtI)
826 return false;
827
828 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
829 if (!Elem.Assume)
830 continue;
831
832 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
833 assert(I->getFunction() == Q.CxtI->getFunction() &&
834 "Got assumption for the wrong function!");
835
836 if (Elem.Index != AssumptionCache::ExprResultIdx) {
837 if (assumeBundleImpliesNonNull(Val: V, Context: Q.CxtI->getFunction(),
838 OBU: I->getOperandBundleAt(Index: Elem.Index)) &&
839 isValidAssumeForContext(I, Q))
840 return true;
841 continue;
842 }
843
844 // Warning: This loop can end up being somewhat performance sensitive.
845 // We're running this loop for once for each value queried resulting in a
846 // runtime of ~O(#assumes * #values).
847
848 Value *RHS;
849 CmpPredicate Pred;
850 auto m_V = m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToInt(Op: m_Specific(V)));
851 if (!match(V: I->getArgOperand(i: 0), P: m_c_ICmp(Pred, L: m_V, R: m_Value(V&: RHS))))
852 continue;
853
854 if (cmpExcludesZero(Pred, RHS) && isValidAssumeForContext(I, Q))
855 return true;
856 }
857
858 return false;
859}
860
861static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
862 Value *LHS, Value *RHS, KnownBits &Known,
863 const SimplifyQuery &Q) {
864 if (RHS->getType()->isPointerTy()) {
865 // Handle comparison of pointer to null explicitly, as it will not be
866 // covered by the m_APInt() logic below.
867 if (LHS == V && match(V: RHS, P: m_Zero())) {
868 switch (Pred) {
869 case ICmpInst::ICMP_EQ:
870 Known.setAllZero();
871 break;
872 case ICmpInst::ICMP_SGE:
873 case ICmpInst::ICMP_SGT:
874 Known.makeNonNegative();
875 break;
876 case ICmpInst::ICMP_SLT:
877 Known.makeNegative();
878 break;
879 default:
880 break;
881 }
882 }
883 return;
884 }
885
886 unsigned BitWidth = Known.getBitWidth();
887 auto m_V =
888 m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToIntSameSize(DL: Q.DL, Op: m_Specific(V)));
889
890 Value *Y;
891 const APInt *Mask, *C;
892 if (!match(V: RHS, P: m_APInt(Res&: C)))
893 return;
894
895 uint64_t ShAmt;
896 switch (Pred) {
897 case ICmpInst::ICMP_EQ:
898 // assume(V = C)
899 if (match(V: LHS, P: m_V)) {
900 Known = Known.unionWith(RHS: KnownBits::makeConstant(C: *C));
901 // assume(V & Mask = C)
902 } else if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value(V&: Y)))) {
903 // For one bits in Mask, we can propagate bits from C to V.
904 Known.One |= *C;
905 if (match(V: Y, P: m_APInt(Res&: Mask)))
906 Known.Zero |= ~*C & *Mask;
907 // assume(V | Mask = C)
908 } else if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value(V&: Y)))) {
909 // For zero bits in Mask, we can propagate bits from C to V.
910 Known.Zero |= ~*C;
911 if (match(V: Y, P: m_APInt(Res&: Mask)))
912 Known.One |= *C & ~*Mask;
913 // assume(V << ShAmt = C)
914 } else if (match(V: LHS, P: m_Shl(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
915 ShAmt < BitWidth) {
916 // For those bits in C that are known, we can propagate them to known
917 // bits in V shifted to the right by ShAmt.
918 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
919 RHSKnown >>= ShAmt;
920 Known = Known.unionWith(RHS: RHSKnown);
921 // assume(V >> ShAmt = C)
922 } else if (match(V: LHS, P: m_Shr(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
923 ShAmt < BitWidth) {
924 // For those bits in RHS that are known, we can propagate them to known
925 // bits in V shifted to the right by C.
926 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
927 RHSKnown <<= ShAmt;
928 Known = Known.unionWith(RHS: RHSKnown);
929 }
930 break;
931 case ICmpInst::ICMP_NE: {
932 // assume (V & B != 0) where B is a power of 2
933 const APInt *BPow2;
934 if (C->isZero() && match(V: LHS, P: m_And(L: m_V, R: m_Power2(V&: BPow2))))
935 Known.One |= *BPow2;
936 break;
937 }
938 default: {
939 const APInt *Offset = nullptr;
940 if (match(V: LHS, P: m_CombineOr(Ps: m_V, Ps: m_AddLike(L: m_V, R: m_APInt(Res&: Offset))))) {
941 ConstantRange LHSRange = ConstantRange::makeAllowedICmpRegion(Pred, Other: *C);
942 if (Offset)
943 LHSRange = LHSRange.sub(Other: *Offset);
944 Known = Known.unionWith(RHS: LHSRange.toKnownBits());
945 }
946 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
947 // X & Y u> C -> X u> C && Y u> C
948 // X nuw- Y u> C -> X u> C
949 if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value())) ||
950 match(V: LHS, P: m_NUWSub(L: m_V, R: m_Value())))
951 Known.One.setHighBits(
952 (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
953 }
954 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
955 // X | Y u< C -> X u< C && Y u< C
956 // X nuw+ Y u< C -> X u< C && Y u< C
957 if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value())) ||
958 match(V: LHS, P: m_c_NUWAdd(L: m_V, R: m_Value()))) {
959 Known.Zero.setHighBits(
960 (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
961 }
962 }
963 } break;
964 }
965}
966
967static void computeKnownBitsFromICmpCond(const Value *V, ICmpInst *Cmp,
968 KnownBits &Known,
969 const SimplifyQuery &SQ, bool Invert) {
970 ICmpInst::Predicate Pred =
971 Invert ? Cmp->getInversePredicate() : Cmp->getPredicate();
972 Value *LHS = Cmp->getOperand(i_nocapture: 0);
973 Value *RHS = Cmp->getOperand(i_nocapture: 1);
974
975 // Handle icmp pred (trunc V), C
976 if (match(V: LHS, P: m_Trunc(Op: m_Specific(V)))) {
977 KnownBits DstKnown(LHS->getType()->getScalarSizeInBits());
978 computeKnownBitsFromCmp(V: LHS, Pred, LHS, RHS, Known&: DstKnown, Q: SQ);
979 if (cast<TruncInst>(Val: LHS)->hasNoUnsignedWrap())
980 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
981 else
982 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
983 return;
984 }
985
986 computeKnownBitsFromCmp(V, Pred, LHS, RHS, Known, Q: SQ);
987}
988
989static void computeKnownBitsFromCond(const Value *V, Value *Cond,
990 KnownBits &Known, const SimplifyQuery &SQ,
991 bool Invert, unsigned Depth) {
992 Value *A, *B;
993 if (Depth < MaxAnalysisRecursionDepth &&
994 match(V: Cond, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
995 KnownBits Known2(Known.getBitWidth());
996 KnownBits Known3(Known.getBitWidth());
997 computeKnownBitsFromCond(V, Cond: A, Known&: Known2, SQ, Invert, Depth: Depth + 1);
998 computeKnownBitsFromCond(V, Cond: B, Known&: Known3, SQ, Invert, Depth: Depth + 1);
999 if (Invert ? match(V: Cond, P: m_LogicalOr(L: m_Value(), R: m_Value()))
1000 : match(V: Cond, P: m_LogicalAnd(L: m_Value(), R: m_Value())))
1001 Known2 = Known2.unionWith(RHS: Known3);
1002 else
1003 Known2 = Known2.intersectWith(RHS: Known3);
1004 Known = Known.unionWith(RHS: Known2);
1005 return;
1006 }
1007
1008 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
1009 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ, Invert);
1010 return;
1011 }
1012
1013 if (match(V: Cond, P: m_Trunc(Op: m_Specific(V)))) {
1014 KnownBits DstKnown(1);
1015 if (Invert) {
1016 DstKnown.setAllZero();
1017 } else {
1018 DstKnown.setAllOnes();
1019 }
1020 if (cast<TruncInst>(Val: Cond)->hasNoUnsignedWrap()) {
1021 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1022 return;
1023 }
1024 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1025 return;
1026 }
1027
1028 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A))))
1029 computeKnownBitsFromCond(V, Cond: A, Known, SQ, Invert: !Invert, Depth: Depth + 1);
1030}
1031
1032void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
1033 const SimplifyQuery &Q, unsigned Depth) {
1034 // Handle injected condition.
1035 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
1036 computeKnownBitsFromCond(V, Cond: Q.CC->Cond, Known, SQ: Q, Invert: Q.CC->Invert, Depth);
1037
1038 if (!Q.CxtI)
1039 return;
1040
1041 if (Q.DC && Q.DT) {
1042 // Handle dominating conditions.
1043 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
1044 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
1045 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
1046 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1047 /*Invert*/ false, Depth);
1048
1049 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
1050 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
1051 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1052 /*Invert*/ true, Depth);
1053 }
1054
1055 if (Known.hasConflict())
1056 Known.resetAll();
1057 }
1058
1059 if (!Q.AC)
1060 return;
1061
1062 unsigned BitWidth = Known.getBitWidth();
1063
1064 // Note that the patterns below need to be kept in sync with the code
1065 // in AssumptionCache::updateAffectedValues.
1066
1067 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
1068 if (!Elem.Assume)
1069 continue;
1070
1071 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
1072 assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() &&
1073 "Got assumption for the wrong function!");
1074
1075 if (Elem.Index != AssumptionCache::ExprResultIdx) {
1076 if (auto OBU = I->getOperandBundleAt(Index: Elem.Index);
1077 getBundleAttrFromOBU(OBU) == BundleAttr::Align) {
1078 auto [Ptr, _, _2, Alignment, Offset] = getAssumeAlignInfo(OBU);
1079 if (Ptr == V && Alignment && Offset && isPowerOf2_64(Value: *Alignment) &&
1080 isValidAssumeForContext(I, Q)) {
1081 Known.Zero |= (*Alignment - 1) & ~*Offset;
1082 Known.One |= (*Alignment - 1) & *Offset;
1083 }
1084 }
1085 continue;
1086 }
1087
1088 // Warning: This loop can end up being somewhat performance sensitive.
1089 // We're running this loop for once for each value queried resulting in a
1090 // runtime of ~O(#assumes * #values).
1091
1092 Value *Arg = I->getArgOperand(i: 0);
1093
1094 if (Arg == V && isValidAssumeForContext(I, Q)) {
1095 assert(BitWidth == 1 && "assume operand is not i1?");
1096 (void)BitWidth;
1097 Known.setAllOnes();
1098 return;
1099 }
1100 if (match(V: Arg, P: m_Not(V: m_Specific(V))) &&
1101 isValidAssumeForContext(I, Q)) {
1102 assert(BitWidth == 1 && "assume operand is not i1?");
1103 (void)BitWidth;
1104 Known.setAllZero();
1105 return;
1106 }
1107 auto *Trunc = dyn_cast<TruncInst>(Val: Arg);
1108 if (Trunc && Trunc->getOperand(i_nocapture: 0) == V &&
1109 isValidAssumeForContext(I, Q)) {
1110 if (Trunc->hasNoUnsignedWrap()) {
1111 Known = KnownBits::makeConstant(C: APInt(BitWidth, 1));
1112 return;
1113 }
1114 Known.One.setBit(0);
1115 return;
1116 }
1117
1118 // The remaining tests are all recursive, so bail out if we hit the limit.
1119 if (Depth == MaxAnalysisRecursionDepth)
1120 continue;
1121
1122 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
1123 if (!Cmp)
1124 continue;
1125
1126 if (!isValidAssumeForContext(I, Q))
1127 continue;
1128
1129 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ: Q, /*Invert=*/false);
1130 }
1131
1132 // Conflicting assumption: Undefined behavior will occur on this execution
1133 // path.
1134 if (Known.hasConflict())
1135 Known.resetAll();
1136}
1137
1138/// Compute known bits from a shift operator, including those with a
1139/// non-constant shift amount. Known is the output of this function. Known2 is a
1140/// pre-allocated temporary with the same bit width as Known and on return
1141/// contains the known bit of the shift value source. KF is an
1142/// operator-specific function that, given the known-bits and a shift amount,
1143/// compute the implied known-bits of the shift operator's result respectively
1144/// for that shift amount. The results from calling KF are conservatively
1145/// combined for all permitted shift amounts.
1146static void computeKnownBitsFromShiftOperator(
1147 const Operator *I, const APInt &DemandedElts, KnownBits &Known,
1148 KnownBits &Known2, const SimplifyQuery &Q, unsigned Depth,
1149 function_ref<KnownBits(const KnownBits &, const KnownBits &, bool)> KF) {
1150 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1151 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1152 // To limit compile-time impact, only query isKnownNonZero() if we know at
1153 // least something about the shift amount.
1154 bool ShAmtNonZero =
1155 Known.isNonZero() ||
1156 (Known.getMaxValue().ult(RHS: Known.getBitWidth()) &&
1157 isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1));
1158 Known = KF(Known2, Known, ShAmtNonZero);
1159}
1160
1161static KnownBits
1162getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
1163 const KnownBits &KnownLHS, const KnownBits &KnownRHS,
1164 const SimplifyQuery &Q, unsigned Depth) {
1165 unsigned BitWidth = KnownLHS.getBitWidth();
1166 KnownBits KnownOut(BitWidth);
1167 bool IsAnd = false;
1168 bool HasKnownOne = !KnownLHS.One.isZero() || !KnownRHS.One.isZero();
1169 Value *X = nullptr, *Y = nullptr;
1170
1171 switch (I->getOpcode()) {
1172 case Instruction::And:
1173 KnownOut = KnownLHS & KnownRHS;
1174 IsAnd = true;
1175 // and(x, -x) is common idioms that will clear all but lowest set
1176 // bit. If we have a single known bit in x, we can clear all bits
1177 // above it.
1178 // TODO: instcombine often reassociates independent `and` which can hide
1179 // this pattern. Try to match and(x, and(-x, y)) / and(and(x, y), -x).
1180 if (HasKnownOne && match(V: I, P: m_c_And(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X))))) {
1181 // -(-x) == x so using whichever (LHS/RHS) gets us a better result.
1182 if (KnownLHS.countMaxTrailingZeros() <= KnownRHS.countMaxTrailingZeros())
1183 KnownOut = KnownLHS.blsi();
1184 else
1185 KnownOut = KnownRHS.blsi();
1186 }
1187 break;
1188 case Instruction::Or:
1189 KnownOut = KnownLHS | KnownRHS;
1190 break;
1191 case Instruction::Xor:
1192 KnownOut = KnownLHS ^ KnownRHS;
1193 // xor(x, x-1) is common idioms that will clear all but lowest set
1194 // bit. If we have a single known bit in x, we can clear all bits
1195 // above it.
1196 // TODO: xor(x, x-1) is often rewritting as xor(x, x-C) where C !=
1197 // -1 but for the purpose of demanded bits (xor(x, x-C) &
1198 // Demanded) == (xor(x, x-1) & Demanded). Extend the xor pattern
1199 // to use arbitrary C if xor(x, x-C) as the same as xor(x, x-1).
1200 if (HasKnownOne &&
1201 match(V: I, P: m_c_Xor(L: m_Value(V&: X), R: m_Add(L: m_Deferred(V: X), R: m_AllOnes())))) {
1202 const KnownBits &XBits = I->getOperand(i: 0) == X ? KnownLHS : KnownRHS;
1203 KnownOut = XBits.blsmsk();
1204 }
1205 break;
1206 default:
1207 llvm_unreachable("Invalid Op used in 'analyzeKnownBitsFromAndXorOr'");
1208 }
1209
1210 // and(x, add (x, -1)) is a common idiom that always clears the low bit;
1211 // xor/or(x, add (x, -1)) is an idiom that will always set the low bit.
1212 // here we handle the more general case of adding any odd number by
1213 // matching the form and/xor/or(x, add(x, y)) where y is odd.
1214 // TODO: This could be generalized to clearing any bit set in y where the
1215 // following bit is known to be unset in y.
1216 if (!KnownOut.Zero[0] && !KnownOut.One[0] &&
1217 (match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_c_Add(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1218 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1219 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Value(V&: Y), R: m_Deferred(V: X)))))) {
1220 KnownBits KnownY(BitWidth);
1221 computeKnownBits(V: Y, DemandedElts, Known&: KnownY, Q, Depth: Depth + 1);
1222 if (KnownY.countMinTrailingOnes() > 0) {
1223 if (IsAnd)
1224 KnownOut.Zero.setBit(0);
1225 else
1226 KnownOut.One.setBit(0);
1227 }
1228 }
1229 return KnownOut;
1230}
1231
1232static KnownBits computeKnownBitsForHorizontalOperation(
1233 const Operator *I, const APInt &DemandedElts, const SimplifyQuery &Q,
1234 unsigned Depth,
1235 const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
1236 KnownBitsFunc) {
1237 APInt DemandedEltsLHS, DemandedEltsRHS;
1238 getHorizDemandedEltsForFirstOperand(VectorBitWidth: Q.DL.getTypeSizeInBits(Ty: I->getType()),
1239 DemandedElts, DemandedLHS&: DemandedEltsLHS,
1240 DemandedRHS&: DemandedEltsRHS);
1241
1242 const auto ComputeForSingleOpFunc =
1243 [Depth, &Q, KnownBitsFunc](const Value *Op, APInt &DemandedEltsOp) {
1244 return KnownBitsFunc(
1245 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp, Q, Depth: Depth + 1),
1246 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp << 1, Q, Depth: Depth + 1));
1247 };
1248
1249 if (DemandedEltsRHS.isZero())
1250 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS);
1251 if (DemandedEltsLHS.isZero())
1252 return ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS);
1253
1254 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS)
1255 .intersectWith(RHS: ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS));
1256}
1257
1258// Public so this can be used in `SimplifyDemandedUseBits`.
1259KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
1260 const KnownBits &KnownLHS,
1261 const KnownBits &KnownRHS,
1262 const SimplifyQuery &SQ,
1263 unsigned Depth) {
1264 auto *FVTy = dyn_cast<FixedVectorType>(Val: I->getType());
1265 APInt DemandedElts =
1266 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
1267
1268 return getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS, KnownRHS, Q: SQ,
1269 Depth);
1270}
1271
1272ConstantRange llvm::getVScaleRange(const Function *F, unsigned BitWidth) {
1273 Attribute Attr = F->getFnAttribute(Kind: Attribute::VScaleRange);
1274 // Without vscale_range, we only know that vscale is non-zero.
1275 if (!Attr.isValid())
1276 return ConstantRange(APInt(BitWidth, 1), APInt::getZero(numBits: BitWidth));
1277
1278 unsigned AttrMin = Attr.getVScaleRangeMin();
1279 // Minimum is larger than vscale width, result is always poison.
1280 if ((unsigned)llvm::bit_width(Value: AttrMin) > BitWidth)
1281 return ConstantRange::getEmpty(BitWidth);
1282
1283 APInt Min(BitWidth, AttrMin);
1284 std::optional<unsigned> AttrMax = Attr.getVScaleRangeMax();
1285 if (!AttrMax || (unsigned)llvm::bit_width(Value: *AttrMax) > BitWidth)
1286 return ConstantRange(Min, APInt::getZero(numBits: BitWidth));
1287
1288 return ConstantRange(Min, APInt(BitWidth, *AttrMax) + 1);
1289}
1290
1291void llvm::adjustKnownBitsForSelectArm(KnownBits &Known, Value *Cond,
1292 Value *Arm, bool Invert,
1293 const SimplifyQuery &Q, unsigned Depth) {
1294 // If we have a constant arm, we are done.
1295 if (Known.isConstant())
1296 return;
1297
1298 // See what condition implies about the bits of the select arm.
1299 KnownBits CondRes(Known.getBitWidth());
1300 computeKnownBitsFromCond(V: Arm, Cond, Known&: CondRes, SQ: Q, Invert, Depth: Depth + 1);
1301 // If we don't get any information from the condition, no reason to
1302 // proceed.
1303 if (CondRes.isUnknown())
1304 return;
1305
1306 // We can have conflict if the condition is dead. I.e if we have
1307 // (x | 64) < 32 ? (x | 64) : y
1308 // we will have conflict at bit 6 from the condition/the `or`.
1309 // In that case just return. Its not particularly important
1310 // what we do, as this select is going to be simplified soon.
1311 CondRes = CondRes.unionWith(RHS: Known);
1312 if (CondRes.hasConflict())
1313 return;
1314
1315 // Finally make sure the information we found is valid. This is relatively
1316 // expensive so it's left for the very end.
1317 if (!isGuaranteedNotToBeUndef(V: Arm, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1))
1318 return;
1319
1320 // Finally, we know we get information from the condition and its valid,
1321 // so return it.
1322 Known = std::move(CondRes);
1323}
1324
1325// Match a signed min+max clamp pattern like smax(smin(In, CHigh), CLow).
1326// Returns the input and lower/upper bounds.
1327static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
1328 const APInt *&CLow, const APInt *&CHigh) {
1329 assert(isa<Operator>(Select) &&
1330 cast<Operator>(Select)->getOpcode() == Instruction::Select &&
1331 "Input should be a Select!");
1332
1333 const Value *LHS = nullptr, *RHS = nullptr;
1334 SelectPatternFlavor SPF = matchSelectPattern(V: Select, LHS, RHS).Flavor;
1335 if (SPF != SPF_SMAX && SPF != SPF_SMIN)
1336 return false;
1337
1338 if (!match(V: RHS, P: m_APInt(Res&: CLow)))
1339 return false;
1340
1341 const Value *LHS2 = nullptr, *RHS2 = nullptr;
1342 SelectPatternFlavor SPF2 = matchSelectPattern(V: LHS, LHS&: LHS2, RHS&: RHS2).Flavor;
1343 if (getInverseMinMaxFlavor(SPF) != SPF2)
1344 return false;
1345
1346 if (!match(V: RHS2, P: m_APInt(Res&: CHigh)))
1347 return false;
1348
1349 if (SPF == SPF_SMIN)
1350 std::swap(a&: CLow, b&: CHigh);
1351
1352 In = LHS2;
1353 return CLow->sle(RHS: *CHigh);
1354}
1355
1356static bool isSignedMinMaxIntrinsicClamp(const IntrinsicInst *II,
1357 const APInt *&CLow,
1358 const APInt *&CHigh) {
1359 assert((II->getIntrinsicID() == Intrinsic::smin ||
1360 II->getIntrinsicID() == Intrinsic::smax) &&
1361 "Must be smin/smax");
1362
1363 Intrinsic::ID InverseID = getInverseMinMaxIntrinsic(MinMaxID: II->getIntrinsicID());
1364 auto *InnerII = dyn_cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
1365 if (!InnerII || InnerII->getIntrinsicID() != InverseID ||
1366 !match(V: II->getArgOperand(i: 1), P: m_APInt(Res&: CLow)) ||
1367 !match(V: InnerII->getArgOperand(i: 1), P: m_APInt(Res&: CHigh)))
1368 return false;
1369
1370 if (II->getIntrinsicID() == Intrinsic::smin)
1371 std::swap(a&: CLow, b&: CHigh);
1372 return CLow->sle(RHS: *CHigh);
1373}
1374
1375static void unionWithMinMaxIntrinsicClamp(const IntrinsicInst *II,
1376 KnownBits &Known) {
1377 const APInt *CLow, *CHigh;
1378 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
1379 Known = Known.unionWith(
1380 RHS: ConstantRange::getNonEmpty(Lower: *CLow, Upper: *CHigh + 1).toKnownBits());
1381}
1382
1383static void computeKnownBitsFromOperator(const Operator *I,
1384 const APInt &DemandedElts,
1385 KnownBits &Known,
1386 const SimplifyQuery &Q,
1387 unsigned Depth) {
1388 unsigned BitWidth = Known.getBitWidth();
1389
1390 KnownBits Known2(BitWidth);
1391 switch (I->getOpcode()) {
1392 default: break;
1393 case Instruction::Load:
1394 if (MDNode *MD =
1395 Q.IIQ.getMetadata(I: cast<LoadInst>(Val: I), KindID: LLVMContext::MD_range))
1396 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
1397 break;
1398 case Instruction::And:
1399 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1400 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1401
1402 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1403 break;
1404 case Instruction::Or:
1405 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1406 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1407
1408 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1409 break;
1410 case Instruction::Xor:
1411 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1412 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1413
1414 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1415 break;
1416 case Instruction::Mul: {
1417 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1418 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1419 computeKnownBitsMul(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1420 DemandedElts, Known, Known2, Q, Depth);
1421 break;
1422 }
1423 case Instruction::UDiv: {
1424 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1425 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1426 Known =
1427 KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1428 break;
1429 }
1430 case Instruction::SDiv: {
1431 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1432 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1433 Known =
1434 KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1435 break;
1436 }
1437 case Instruction::Select: {
1438 auto ComputeForArm = [&](Value *Arm, bool Invert) {
1439 KnownBits Res(Known.getBitWidth());
1440 computeKnownBits(V: Arm, DemandedElts, Known&: Res, Q, Depth: Depth + 1);
1441 adjustKnownBitsForSelectArm(Known&: Res, Cond: I->getOperand(i: 0), Arm, Invert, Q, Depth);
1442 return Res;
1443 };
1444 // Only known if known in both the LHS and RHS.
1445 Known =
1446 ComputeForArm(I->getOperand(i: 1), /*Invert=*/false)
1447 .intersectWith(RHS: ComputeForArm(I->getOperand(i: 2), /*Invert=*/true));
1448 break;
1449 }
1450 case Instruction::FPTrunc:
1451 case Instruction::FPExt:
1452 case Instruction::FPToUI:
1453 case Instruction::FPToSI:
1454 case Instruction::SIToFP:
1455 case Instruction::UIToFP:
1456 break; // Can't work with floating point.
1457 case Instruction::PtrToInt:
1458 case Instruction::PtrToAddr:
1459 case Instruction::IntToPtr:
1460 // Fall through and handle them the same as zext/trunc.
1461 [[fallthrough]];
1462 case Instruction::ZExt:
1463 case Instruction::Trunc: {
1464 Type *SrcTy = I->getOperand(i: 0)->getType();
1465
1466 unsigned SrcBitWidth;
1467 // Note that we handle pointer operands here because of inttoptr/ptrtoint
1468 // which fall through here.
1469 Type *ScalarTy = SrcTy->getScalarType();
1470 SrcBitWidth = ScalarTy->isPointerTy() ?
1471 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
1472 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
1473
1474 assert(SrcBitWidth && "SrcBitWidth can't be zero");
1475 Known = Known.anyextOrTrunc(BitWidth: SrcBitWidth);
1476 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1477 if (auto *Inst = dyn_cast<PossiblyNonNegInst>(Val: I);
1478 Inst && Inst->hasNonNeg() && !Known.isNegative())
1479 Known.makeNonNegative();
1480 Known = Known.zextOrTrunc(BitWidth);
1481 break;
1482 }
1483 case Instruction::BitCast: {
1484 Type *SrcTy = I->getOperand(i: 0)->getType();
1485 if (SrcTy->isIntOrPtrTy() &&
1486 // TODO: For now, not handling conversions like:
1487 // (bitcast i64 %x to <2 x i32>)
1488 !I->getType()->isVectorTy()) {
1489 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1490 break;
1491 }
1492
1493 const Value *V;
1494 // Handle bitcast from floating point to integer.
1495 if (match(V: I, P: m_ElementWiseBitCast(Op: m_Value(V))) &&
1496 V->getType()->isFPOrFPVectorTy()) {
1497 Type *FPType = V->getType()->getScalarType();
1498 KnownFPClass Result =
1499 computeKnownFPClass(V, DemandedElts, InterestedClasses: fcAllFlags, SQ: Q, Depth: Depth + 1);
1500 FPClassTest FPClasses = Result.KnownFPClasses;
1501
1502 // TODO: Treat it as zero/poison if the use of I is unreachable.
1503 if (FPClasses == fcNone)
1504 break;
1505
1506 if (Result.isKnownNever(Mask: fcNormal | fcSubnormal | fcNan)) {
1507 Known.setAllConflict();
1508
1509 if (FPClasses & fcInf)
1510 Known = Known.intersectWith(RHS: KnownBits::makeConstant(
1511 C: APFloat::getInf(Sem: FPType->getFltSemantics()).bitcastToAPInt()));
1512
1513 if (FPClasses & fcZero)
1514 Known = Known.intersectWith(RHS: KnownBits::makeConstant(
1515 C: APInt::getZero(numBits: FPType->getScalarSizeInBits())));
1516
1517 Known.Zero.clearSignBit();
1518 Known.One.clearSignBit();
1519 }
1520
1521 if (Result.SignBit) {
1522 if (*Result.SignBit)
1523 Known.makeNegative();
1524 else
1525 Known.makeNonNegative();
1526 }
1527
1528 break;
1529 }
1530
1531 // Handle cast from vector integer type to scalar or vector integer.
1532 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy);
1533 if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
1534 !I->getType()->isIntOrIntVectorTy() ||
1535 isa<ScalableVectorType>(Val: I->getType()))
1536 break;
1537
1538 unsigned NumElts = DemandedElts.getBitWidth();
1539 bool IsLE = Q.DL.isLittleEndian();
1540 // Look through a cast from narrow vector elements to wider type.
1541 // Examples: v4i32 -> v2i64, v3i8 -> v24
1542 unsigned SubBitWidth = SrcVecTy->getScalarSizeInBits();
1543 if (BitWidth % SubBitWidth == 0) {
1544 // Known bits are automatically intersected across demanded elements of a
1545 // vector. So for example, if a bit is computed as known zero, it must be
1546 // zero across all demanded elements of the vector.
1547 //
1548 // For this bitcast, each demanded element of the output is sub-divided
1549 // across a set of smaller vector elements in the source vector. To get
1550 // the known bits for an entire element of the output, compute the known
1551 // bits for each sub-element sequentially. This is done by shifting the
1552 // one-set-bit demanded elements parameter across the sub-elements for
1553 // consecutive calls to computeKnownBits. We are using the demanded
1554 // elements parameter as a mask operator.
1555 //
1556 // The known bits of each sub-element are then inserted into place
1557 // (dependent on endian) to form the full result of known bits.
1558 unsigned SubScale = BitWidth / SubBitWidth;
1559 APInt SubDemandedElts = APInt::getZero(numBits: NumElts * SubScale);
1560 for (unsigned i = 0; i != NumElts; ++i) {
1561 if (DemandedElts[i])
1562 SubDemandedElts.setBit(i * SubScale);
1563 }
1564
1565 KnownBits KnownSrc(SubBitWidth);
1566 for (unsigned i = 0; i != SubScale; ++i) {
1567 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts.shl(shiftAmt: i), Known&: KnownSrc, Q,
1568 Depth: Depth + 1);
1569 unsigned ShiftElt = IsLE ? i : SubScale - 1 - i;
1570 Known.insertBits(SubBits: KnownSrc, BitPosition: ShiftElt * SubBitWidth);
1571 }
1572 }
1573 // Look through a cast from wider vector elements to narrow type.
1574 // Examples: v2i64 -> v4i32
1575 if (SubBitWidth % BitWidth == 0) {
1576 unsigned SubScale = SubBitWidth / BitWidth;
1577 KnownBits KnownSrc(SubBitWidth);
1578 APInt SubDemandedElts =
1579 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
1580 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts, Known&: KnownSrc, Q,
1581 Depth: Depth + 1);
1582
1583 Known.setAllConflict();
1584 for (unsigned i = 0; i != NumElts; ++i) {
1585 if (DemandedElts[i]) {
1586 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
1587 unsigned Offset = (Shifts % SubScale) * BitWidth;
1588 Known = Known.intersectWith(RHS: KnownSrc.extractBits(NumBits: BitWidth, BitPosition: Offset));
1589 if (Known.isUnknown())
1590 break;
1591 }
1592 }
1593 }
1594 break;
1595 }
1596 case Instruction::SExt: {
1597 // Compute the bits in the result that are not present in the input.
1598 unsigned SrcBitWidth = I->getOperand(i: 0)->getType()->getScalarSizeInBits();
1599
1600 Known = Known.trunc(BitWidth: SrcBitWidth);
1601 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1602 // If the sign bit of the input is known set or clear, then we know the
1603 // top bits of the result.
1604 Known = Known.sext(BitWidth);
1605 break;
1606 }
1607 case Instruction::Shl: {
1608 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1609 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1610 auto KF = [NUW, NSW](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1611 bool ShAmtNonZero) {
1612 return KnownBits::shl(LHS: KnownVal, RHS: KnownAmt, NUW, NSW, ShAmtNonZero);
1613 };
1614 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1615 KF);
1616 // Trailing zeros of a right-shifted constant never decrease.
1617 const APInt *C;
1618 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1619 Known.Zero.setLowBits(C->countr_zero());
1620
1621 // shl X, sub(Y, xor(ctlz(X, true), BitWidth-1)) shifts X so that its MSB
1622 // lands at bit Y, when BitWidth is a power of 2.
1623 const APInt *YC;
1624 Value *X = I->getOperand(i: 0);
1625 if (isPowerOf2_32(Value: BitWidth) &&
1626 match(V: I->getOperand(i: 1),
1627 P: m_Sub(L: m_APInt(Res&: YC), R: m_Xor(L: m_Ctlz(Op0: m_Specific(V: X), Op1: m_One()),
1628 R: m_SpecificInt(V: BitWidth - 1)))) &&
1629 YC->ult(RHS: BitWidth - 1)) {
1630 unsigned Y = YC->getZExtValue();
1631 Known.One.setBit(Y);
1632 Known.Zero.setBitsFrom(Y + 1);
1633 }
1634 break;
1635 }
1636 case Instruction::LShr: {
1637 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1638 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1639 bool ShAmtNonZero) {
1640 return KnownBits::lshr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1641 };
1642 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1643 KF);
1644 // Leading zeros of a left-shifted constant never decrease.
1645 const APInt *C;
1646 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1647 Known.Zero.setHighBits(C->countl_zero());
1648 break;
1649 }
1650 case Instruction::AShr: {
1651 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1652 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1653 bool ShAmtNonZero) {
1654 return KnownBits::ashr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1655 };
1656 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1657 KF);
1658 break;
1659 }
1660 case Instruction::Sub: {
1661 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1662 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1663 computeKnownBitsAddSub(Add: false, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1664 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1665 break;
1666 }
1667 case Instruction::Add: {
1668 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1669 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1670 computeKnownBitsAddSub(Add: true, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1671 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1672 break;
1673 }
1674 case Instruction::SRem:
1675 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1676 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1677 Known = KnownBits::srem(LHS: Known, RHS: Known2);
1678 break;
1679
1680 case Instruction::URem:
1681 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1682 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1683 Known = KnownBits::urem(LHS: Known, RHS: Known2);
1684 break;
1685 case Instruction::Alloca:
1686 Known.Zero.setLowBits(Log2(A: cast<AllocaInst>(Val: I)->getAlign()));
1687 break;
1688 case Instruction::GetElementPtr: {
1689 // Analyze all of the subscripts of this getelementptr instruction
1690 // to determine if we can prove known low zero bits.
1691 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1692 // Accumulate the constant indices in a separate variable
1693 // to minimize the number of calls to computeForAddSub.
1694 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: I->getType());
1695 APInt AccConstIndices(IndexWidth, 0);
1696
1697 auto AddIndexToKnown = [&](KnownBits IndexBits) {
1698 if (IndexWidth == BitWidth) {
1699 // Note that inbounds does *not* guarantee nsw for the addition, as only
1700 // the offset is signed, while the base address is unsigned.
1701 Known = KnownBits::add(LHS: Known, RHS: IndexBits);
1702 } else {
1703 // If the index width is smaller than the pointer width, only add the
1704 // value to the low bits.
1705 assert(IndexWidth < BitWidth &&
1706 "Index width can't be larger than pointer width");
1707 Known.insertBits(SubBits: KnownBits::add(LHS: Known.trunc(BitWidth: IndexWidth), RHS: IndexBits), BitPosition: 0);
1708 }
1709 };
1710
1711 gep_type_iterator GTI = gep_type_begin(GEP: I);
1712 for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
1713 // TrailZ can only become smaller, short-circuit if we hit zero.
1714 if (Known.isUnknown())
1715 break;
1716
1717 Value *Index = I->getOperand(i);
1718
1719 // Handle case when index is zero.
1720 Constant *CIndex = dyn_cast<Constant>(Val: Index);
1721 if (CIndex && CIndex->isNullValue())
1722 continue;
1723
1724 if (StructType *STy = GTI.getStructTypeOrNull()) {
1725 // Handle struct member offset arithmetic.
1726
1727 assert(CIndex &&
1728 "Access to structure field must be known at compile time");
1729
1730 if (CIndex->getType()->isVectorTy())
1731 Index = CIndex->getSplatValue();
1732
1733 unsigned Idx = cast<ConstantInt>(Val: Index)->getZExtValue();
1734 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
1735 uint64_t Offset = SL->getElementOffset(Idx);
1736 AccConstIndices += Offset;
1737 continue;
1738 }
1739
1740 // Handle array index arithmetic.
1741 Type *IndexedTy = GTI.getIndexedType();
1742 if (!IndexedTy->isSized()) {
1743 Known.resetAll();
1744 break;
1745 }
1746
1747 TypeSize Stride = GTI.getSequentialElementStride(DL: Q.DL);
1748 uint64_t StrideInBytes = Stride.getKnownMinValue();
1749 if (!Stride.isScalable()) {
1750 // Fast path for constant offset.
1751 if (auto *CI = dyn_cast<ConstantInt>(Val: Index)) {
1752 AccConstIndices +=
1753 CI->getValue().sextOrTrunc(width: IndexWidth) * StrideInBytes;
1754 continue;
1755 }
1756 }
1757
1758 KnownBits IndexBits =
1759 computeKnownBits(V: Index, Q, Depth: Depth + 1).sextOrTrunc(BitWidth: IndexWidth);
1760 KnownBits ScalingFactor(IndexWidth);
1761 // Multiply by current sizeof type.
1762 // &A[i] == A + i * sizeof(*A[i]).
1763 if (Stride.isScalable()) {
1764 // For scalable types the only thing we know about sizeof is
1765 // that this is a multiple of the minimum size.
1766 ScalingFactor.Zero.setLowBits(llvm::countr_zero(Val: StrideInBytes));
1767 } else {
1768 ScalingFactor =
1769 KnownBits::makeConstant(C: APInt(IndexWidth, StrideInBytes));
1770 }
1771 AddIndexToKnown(KnownBits::mul(LHS: IndexBits, RHS: ScalingFactor));
1772 }
1773 if (!Known.isUnknown() && !AccConstIndices.isZero())
1774 AddIndexToKnown(KnownBits::makeConstant(C: AccConstIndices));
1775 break;
1776 }
1777 case Instruction::PHI: {
1778 const PHINode *P = cast<PHINode>(Val: I);
1779 BinaryOperator *BO = nullptr;
1780 Value *R = nullptr, *L = nullptr;
1781 if (matchSimpleRecurrence(P, BO, Start&: R, Step&: L)) {
1782 // Handle the case of a simple two-predecessor recurrence PHI.
1783 // There's a lot more that could theoretically be done here, but
1784 // this is sufficient to catch some interesting cases.
1785 unsigned Opcode = BO->getOpcode();
1786
1787 switch (Opcode) {
1788 // If this is a shift recurrence, we know the bits being shifted in. We
1789 // can combine that with information about the start value of the
1790 // recurrence to conclude facts about the result. If this is a udiv
1791 // recurrence, we know that the result can never exceed either the
1792 // numerator or the start value, whichever is greater.
1793 case Instruction::LShr:
1794 case Instruction::AShr:
1795 case Instruction::Shl:
1796 case Instruction::UDiv:
1797 if (BO->getOperand(i_nocapture: 0) != I)
1798 break;
1799 [[fallthrough]];
1800
1801 // For a urem recurrence, the result can never exceed the start value. The
1802 // phi could either be the numerator or the denominator.
1803 case Instruction::URem: {
1804 // We have matched a recurrence of the form:
1805 // %iv = [R, %entry], [%iv.next, %backedge]
1806 // %iv.next = shift_op %iv, L
1807
1808 // Recurse with the phi context to avoid concern about whether facts
1809 // inferred hold at original context instruction. TODO: It may be
1810 // correct to use the original context. IF warranted, explore and
1811 // add sufficient tests to cover.
1812 SimplifyQuery RecQ = Q.getWithoutCondContext();
1813 RecQ.CxtI = P;
1814 computeKnownBits(V: R, DemandedElts, Known&: Known2, Q: RecQ, Depth: Depth + 1);
1815 switch (Opcode) {
1816 case Instruction::Shl:
1817 // A shl recurrence will only increase the tailing zeros
1818 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
1819 break;
1820 case Instruction::LShr:
1821 case Instruction::UDiv:
1822 case Instruction::URem:
1823 // lshr, udiv, and urem recurrences will preserve the leading zeros of
1824 // the start value.
1825 Known.Zero.setHighBits(Known2.countMinLeadingZeros());
1826 break;
1827 case Instruction::AShr:
1828 // An ashr recurrence will extend the initial sign bit
1829 Known.Zero.setHighBits(Known2.countMinLeadingZeros());
1830 Known.One.setHighBits(Known2.countMinLeadingOnes());
1831 break;
1832 }
1833 break;
1834 }
1835
1836 // Check for operations that have the property that if
1837 // both their operands have low zero bits, the result
1838 // will have low zero bits.
1839 case Instruction::Add:
1840 case Instruction::Sub:
1841 case Instruction::And:
1842 case Instruction::Or:
1843 case Instruction::Mul: {
1844 // Change the context instruction to the "edge" that flows into the
1845 // phi. This is important because that is where the value is actually
1846 // "evaluated" even though it is used later somewhere else. (see also
1847 // D69571).
1848 SimplifyQuery RecQ = Q.getWithoutCondContext();
1849
1850 unsigned OpNum = P->getOperand(i_nocapture: 0) == R ? 0 : 1;
1851 Instruction *RInst = P->getIncomingBlock(i: OpNum)->getTerminator();
1852 Instruction *LInst = P->getIncomingBlock(i: 1 - OpNum)->getTerminator();
1853
1854 // Ok, we have a PHI of the form L op= R. Check for low
1855 // zero bits.
1856 RecQ.CxtI = RInst;
1857 computeKnownBits(V: R, DemandedElts, Known&: Known2, Q: RecQ, Depth: Depth + 1);
1858
1859 // We need to take the minimum number of known bits
1860 KnownBits Known3(BitWidth);
1861 RecQ.CxtI = LInst;
1862 computeKnownBits(V: L, DemandedElts, Known&: Known3, Q: RecQ, Depth: Depth + 1);
1863
1864 Known.Zero.setLowBits(std::min(a: Known2.countMinTrailingZeros(),
1865 b: Known3.countMinTrailingZeros()));
1866
1867 auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(Val: BO);
1868 if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(Op: OverflowOp))
1869 break;
1870
1871 switch (Opcode) {
1872 // If initial value of recurrence is nonnegative, and we are adding
1873 // a nonnegative number with nsw, the result can only be nonnegative
1874 // or poison value regardless of the number of times we execute the
1875 // add in phi recurrence. If initial value is negative and we are
1876 // adding a negative number with nsw, the result can only be
1877 // negative or poison value. Similar arguments apply to sub and mul.
1878 //
1879 // (add non-negative, non-negative) --> non-negative
1880 // (add negative, negative) --> negative
1881 case Instruction::Add: {
1882 if (Known2.isNonNegative() && Known3.isNonNegative())
1883 Known.makeNonNegative();
1884 else if (Known2.isNegative() && Known3.isNegative())
1885 Known.makeNegative();
1886 break;
1887 }
1888
1889 // (sub nsw non-negative, negative) --> non-negative
1890 // (sub nsw negative, non-negative) --> negative
1891 case Instruction::Sub: {
1892 if (BO->getOperand(i_nocapture: 0) != I)
1893 break;
1894 if (Known2.isNonNegative() && Known3.isNegative())
1895 Known.makeNonNegative();
1896 else if (Known2.isNegative() && Known3.isNonNegative())
1897 Known.makeNegative();
1898 break;
1899 }
1900
1901 // (mul nsw non-negative, non-negative) --> non-negative
1902 case Instruction::Mul:
1903 if (Known2.isNonNegative() && Known3.isNonNegative())
1904 Known.makeNonNegative();
1905 break;
1906
1907 default:
1908 break;
1909 }
1910 break;
1911 }
1912
1913 default:
1914 break;
1915 }
1916 }
1917
1918 // Unreachable blocks may have zero-operand PHI nodes.
1919 if (P->getNumIncomingValues() == 0)
1920 break;
1921
1922 // Otherwise take the unions of the known bit sets of the operands,
1923 // taking conservative care to avoid excessive recursion.
1924 if (Depth < MaxAnalysisRecursionDepth - 1 && Known.isUnknown()) {
1925 // Skip if every incoming value references to ourself.
1926 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
1927 break;
1928
1929 Known.setAllConflict();
1930 for (const Use &U : P->operands()) {
1931 Value *IncValue;
1932 const PHINode *CxtPhi;
1933 Instruction *CxtI;
1934 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CxtI, PhiOut: &CxtPhi);
1935 // Skip direct self references.
1936 if (IncValue == P)
1937 continue;
1938
1939 // Change the context instruction to the "edge" that flows into the
1940 // phi. This is important because that is where the value is actually
1941 // "evaluated" even though it is used later somewhere else. (see also
1942 // D69571).
1943 SimplifyQuery RecQ = Q.getWithoutCondContext().getWithInstruction(I: CxtI);
1944
1945 Known2 = KnownBits(BitWidth);
1946
1947 // Recurse, but cap the recursion to one level, because we don't
1948 // want to waste time spinning around in loops.
1949 // TODO: See if we can base recursion limiter on number of incoming phi
1950 // edges so we don't overly clamp analysis.
1951 computeKnownBits(V: IncValue, DemandedElts, Known&: Known2, Q: RecQ,
1952 Depth: MaxAnalysisRecursionDepth - 1);
1953
1954 // See if we can further use a conditional branch into the phi
1955 // to help us determine the range of the value.
1956 if (!Known2.isConstant()) {
1957 CmpPredicate Pred;
1958 const APInt *RHSC;
1959 BasicBlock *TrueSucc, *FalseSucc;
1960 // TODO: Use RHS Value and compute range from its known bits.
1961 if (match(V: RecQ.CxtI,
1962 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: IncValue), R: m_APInt(Res&: RHSC)),
1963 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
1964 // Check for cases of duplicate successors.
1965 if ((TrueSucc == CxtPhi->getParent()) !=
1966 (FalseSucc == CxtPhi->getParent())) {
1967 // If we're using the false successor, invert the predicate.
1968 if (FalseSucc == CxtPhi->getParent())
1969 Pred = CmpInst::getInversePredicate(pred: Pred);
1970 // Get the knownbits implied by the incoming phi condition.
1971 auto CR = ConstantRange::makeExactICmpRegion(Pred, Other: *RHSC);
1972 KnownBits KnownUnion = Known2.unionWith(RHS: CR.toKnownBits());
1973 // We can have conflicts here if we are analyzing deadcode (its
1974 // impossible for us reach this BB based the icmp).
1975 if (KnownUnion.hasConflict()) {
1976 // No reason to continue analyzing in a known dead region, so
1977 // just resetAll and break. This will cause us to also exit the
1978 // outer loop.
1979 Known.resetAll();
1980 break;
1981 }
1982 Known2 = KnownUnion;
1983 }
1984 }
1985 }
1986
1987 Known = Known.intersectWith(RHS: Known2);
1988 // If all bits have been ruled out, there's no need to check
1989 // more operands.
1990 if (Known.isUnknown())
1991 break;
1992 }
1993 }
1994 break;
1995 }
1996 case Instruction::Call:
1997 case Instruction::Invoke: {
1998 // If range metadata is attached to this call, set known bits from that,
1999 // and then intersect with known bits based on other properties of the
2000 // function.
2001 if (MDNode *MD =
2002 Q.IIQ.getMetadata(I: cast<Instruction>(Val: I), KindID: LLVMContext::MD_range))
2003 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
2004
2005 const auto *CB = cast<CallBase>(Val: I);
2006
2007 if (std::optional<ConstantRange> Range = CB->getRange())
2008 Known = Known.unionWith(RHS: Range->toKnownBits());
2009
2010 if (const Value *RV = CB->getReturnedArgOperand()) {
2011 if (RV->getType() == I->getType()) {
2012 computeKnownBits(V: RV, Known&: Known2, Q, Depth: Depth + 1);
2013 Known = Known.unionWith(RHS: Known2);
2014 // If the function doesn't return properly for all input values
2015 // (e.g. unreachable exits) then there might be conflicts between the
2016 // argument value and the range metadata. Simply discard the known bits
2017 // in case of conflicts.
2018 if (Known.hasConflict())
2019 Known.resetAll();
2020 }
2021 }
2022 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
2023 switch (II->getIntrinsicID()) {
2024 default:
2025 break;
2026 case Intrinsic::abs: {
2027 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2028 bool IntMinIsPoison = match(V: II->getArgOperand(i: 1), P: m_One());
2029 Known = Known.unionWith(RHS: Known2.abs(IntMinIsPoison));
2030 break;
2031 }
2032 case Intrinsic::bitreverse:
2033 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2034 Known = Known.unionWith(RHS: Known2.reverseBits());
2035 break;
2036 case Intrinsic::bswap:
2037 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2038 Known = Known.unionWith(RHS: Known2.byteSwap());
2039 break;
2040 case Intrinsic::ctlz: {
2041 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2042 // If we have a known 1, its position is our upper bound.
2043 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2044 // If this call is poison for 0 input, the result will be less than 2^n.
2045 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2046 PossibleLZ = std::min(a: PossibleLZ, b: BitWidth - 1);
2047 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
2048 Known.Zero.setBitsFrom(LowBits);
2049 break;
2050 }
2051 case Intrinsic::cttz: {
2052 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2053 // If we have a known 1, its position is our upper bound.
2054 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2055 // If this call is poison for 0 input, the result will be less than 2^n.
2056 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2057 PossibleTZ = std::min(a: PossibleTZ, b: BitWidth - 1);
2058 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
2059 Known.Zero.setBitsFrom(LowBits);
2060 break;
2061 }
2062 case Intrinsic::ctpop: {
2063 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2064 // We can bound the space the count needs. Also, bits known to be zero
2065 // can't contribute to the population.
2066 unsigned BitsPossiblySet = Known2.countMaxPopulation();
2067 unsigned LowBits = llvm::bit_width(Value: BitsPossiblySet);
2068 Known.Zero.setBitsFrom(LowBits);
2069 // TODO: we could bound KnownOne using the lower bound on the number
2070 // of bits which might be set provided by popcnt KnownOne2.
2071 break;
2072 }
2073 case Intrinsic::fshr:
2074 case Intrinsic::fshl: {
2075 const APInt *SA;
2076 if (!match(V: I->getOperand(i: 2), P: m_APInt(Res&: SA)))
2077 break;
2078
2079 KnownBits Known3(BitWidth);
2080 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2081 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known3, Q, Depth: Depth + 1);
2082 Known = II->getIntrinsicID() == Intrinsic::fshl
2083 ? KnownBits::fshl(LHS: Known2, RHS: Known3, Amt: *SA)
2084 : KnownBits::fshr(LHS: Known2, RHS: Known3, Amt: *SA);
2085 break;
2086 }
2087 case Intrinsic::clmul:
2088 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2089 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2090 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
2091 break;
2092 case Intrinsic::pext:
2093 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2094 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2095 Known = KnownBits::pext(Val: Known, Mask: Known2);
2096 break;
2097 case Intrinsic::pdep:
2098 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2099 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2100 Known = KnownBits::pdep(Val: Known, Mask: Known2);
2101 break;
2102 case Intrinsic::uadd_sat:
2103 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2104 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2105 Known = KnownBits::uadd_sat(LHS: Known, RHS: Known2);
2106 break;
2107 case Intrinsic::usub_sat:
2108 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2109 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2110 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
2111 break;
2112 case Intrinsic::sadd_sat:
2113 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2114 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2115 Known = KnownBits::sadd_sat(LHS: Known, RHS: Known2);
2116 break;
2117 case Intrinsic::ssub_sat:
2118 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2119 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2120 Known = KnownBits::ssub_sat(LHS: Known, RHS: Known2);
2121 break;
2122 // Vec reverse preserves bits from input vec.
2123 case Intrinsic::vector_reverse:
2124 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: DemandedElts.reverseBits(), Known, Q,
2125 Depth: Depth + 1);
2126 break;
2127 // for min/max/and/or reduce, any bit common to each element in the
2128 // input vec is set in the output.
2129 case Intrinsic::vector_reduce_and:
2130 case Intrinsic::vector_reduce_or:
2131 case Intrinsic::vector_reduce_umax:
2132 case Intrinsic::vector_reduce_umin:
2133 case Intrinsic::vector_reduce_smax:
2134 case Intrinsic::vector_reduce_smin:
2135 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2136 break;
2137 case Intrinsic::vector_reduce_xor: {
2138 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2139 // The zeros common to all vecs are zero in the output.
2140 // If the number of elements is odd, then the common ones remain. If the
2141 // number of elements is even, then the common ones becomes zeros.
2142 auto *VecTy = cast<VectorType>(Val: I->getOperand(i: 0)->getType());
2143 // Even, so the ones become zeros.
2144 bool EvenCnt = VecTy->getElementCount().isKnownEven();
2145 if (EvenCnt)
2146 Known.Zero |= Known.One;
2147 // Maybe even element count so need to clear ones.
2148 if (VecTy->isScalableTy() || EvenCnt)
2149 Known.One.clearAllBits();
2150 break;
2151 }
2152 case Intrinsic::vector_reduce_add: {
2153 auto *VecTy = dyn_cast<FixedVectorType>(Val: I->getOperand(i: 0)->getType());
2154 if (!VecTy)
2155 break;
2156 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2157 Known = Known.reduceAdd(NumElts: VecTy->getNumElements());
2158 break;
2159 }
2160 case Intrinsic::umin:
2161 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2162 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2163 Known = KnownBits::umin(LHS: Known, RHS: Known2);
2164 break;
2165 case Intrinsic::umax:
2166 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2167 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2168 Known = KnownBits::umax(LHS: Known, RHS: Known2);
2169 break;
2170 case Intrinsic::smin:
2171 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2172 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2173 Known = KnownBits::smin(LHS: Known, RHS: Known2);
2174 unionWithMinMaxIntrinsicClamp(II, Known);
2175 break;
2176 case Intrinsic::smax:
2177 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2178 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2179 Known = KnownBits::smax(LHS: Known, RHS: Known2);
2180 unionWithMinMaxIntrinsicClamp(II, Known);
2181 break;
2182 case Intrinsic::ptrmask: {
2183 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2184
2185 const Value *Mask = I->getOperand(i: 1);
2186 Known2 = KnownBits(Mask->getType()->getScalarSizeInBits());
2187 computeKnownBits(V: Mask, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2188 // TODO: 1-extend would be more precise.
2189 Known &= Known2.anyextOrTrunc(BitWidth);
2190 break;
2191 }
2192 case Intrinsic::x86_sse2_pmulh_w:
2193 case Intrinsic::x86_avx2_pmulh_w:
2194 case Intrinsic::x86_avx512_pmulh_w_512:
2195 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2196 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2197 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
2198 break;
2199 case Intrinsic::x86_sse2_pmulhu_w:
2200 case Intrinsic::x86_avx2_pmulhu_w:
2201 case Intrinsic::x86_avx512_pmulhu_w_512:
2202 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2203 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2204 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
2205 break;
2206 case Intrinsic::x86_sse42_crc32_64_64:
2207 Known.Zero.setBitsFrom(32);
2208 break;
2209 case Intrinsic::x86_ssse3_phadd_d_128:
2210 case Intrinsic::x86_ssse3_phadd_w_128:
2211 case Intrinsic::x86_avx2_phadd_d:
2212 case Intrinsic::x86_avx2_phadd_w: {
2213 Known = computeKnownBitsForHorizontalOperation(
2214 I, DemandedElts, Q, Depth,
2215 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2216 return KnownBits::add(LHS: KnownLHS, RHS: KnownRHS);
2217 });
2218 break;
2219 }
2220 case Intrinsic::x86_ssse3_phadd_sw_128:
2221 case Intrinsic::x86_avx2_phadd_sw: {
2222 Known = computeKnownBitsForHorizontalOperation(
2223 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::sadd_sat);
2224 break;
2225 }
2226 case Intrinsic::x86_ssse3_phsub_d_128:
2227 case Intrinsic::x86_ssse3_phsub_w_128:
2228 case Intrinsic::x86_avx2_phsub_d:
2229 case Intrinsic::x86_avx2_phsub_w: {
2230 Known = computeKnownBitsForHorizontalOperation(
2231 I, DemandedElts, Q, Depth,
2232 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2233 return KnownBits::sub(LHS: KnownLHS, RHS: KnownRHS);
2234 });
2235 break;
2236 }
2237 case Intrinsic::x86_ssse3_phsub_sw_128:
2238 case Intrinsic::x86_avx2_phsub_sw: {
2239 Known = computeKnownBitsForHorizontalOperation(
2240 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::ssub_sat);
2241 break;
2242 }
2243 case Intrinsic::riscv_vsetvli:
2244 case Intrinsic::riscv_vsetvlimax: {
2245 bool HasAVL = II->getIntrinsicID() == Intrinsic::riscv_vsetvli;
2246 const ConstantRange Range = getVScaleRange(F: II->getFunction(), BitWidth);
2247 uint64_t SEW = RISCVVType::decodeVSEW(
2248 VSEW: cast<ConstantInt>(Val: II->getArgOperand(i: HasAVL))->getZExtValue());
2249 RISCVVType::VLMUL VLMUL = static_cast<RISCVVType::VLMUL>(
2250 cast<ConstantInt>(Val: II->getArgOperand(i: 1 + HasAVL))->getZExtValue());
2251 uint64_t MaxVLEN =
2252 Range.getUnsignedMax().getZExtValue() * RISCV::RVVBitsPerBlock;
2253 uint64_t MaxVL = MaxVLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
2254
2255 // Result of vsetvli must be not larger than AVL.
2256 if (HasAVL)
2257 if (auto *CI = dyn_cast<ConstantInt>(Val: II->getArgOperand(i: 0)))
2258 MaxVL = std::min(a: MaxVL, b: CI->getZExtValue());
2259
2260 unsigned KnownZeroFirstBit = Log2_32(Value: MaxVL) + 1;
2261 if (BitWidth > KnownZeroFirstBit)
2262 Known.Zero.setBitsFrom(KnownZeroFirstBit);
2263 break;
2264 }
2265 case Intrinsic::amdgcn_mbcnt_hi:
2266 case Intrinsic::amdgcn_mbcnt_lo: {
2267 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
2268 // most 31 + src1.
2269 Known.Zero.setBitsFrom(
2270 II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo ? 6 : 5);
2271 computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Q, Depth: Depth + 1);
2272 Known = KnownBits::add(LHS: Known, RHS: Known2);
2273 break;
2274 }
2275 case Intrinsic::vscale: {
2276 if (!II->getParent() || !II->getFunction())
2277 break;
2278
2279 Known = getVScaleRange(F: II->getFunction(), BitWidth).toKnownBits();
2280 break;
2281 }
2282 }
2283 }
2284 break;
2285 }
2286 case Instruction::ShuffleVector: {
2287 if (auto *Splat = getSplatValue(V: I)) {
2288 computeKnownBits(V: Splat, Known, Q, Depth: Depth + 1);
2289 break;
2290 }
2291
2292 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
2293 // FIXME: Do we need to handle ConstantExpr involving shufflevectors?
2294 if (!Shuf) {
2295 Known.resetAll();
2296 return;
2297 }
2298 // For undef elements, we don't know anything about the common state of
2299 // the shuffle result.
2300 APInt DemandedLHS, DemandedRHS;
2301 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS)) {
2302 Known.resetAll();
2303 return;
2304 }
2305 Known.setAllConflict();
2306 if (!!DemandedLHS) {
2307 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
2308 computeKnownBits(V: LHS, DemandedElts: DemandedLHS, Known, Q, Depth: Depth + 1);
2309 // If we don't know any bits, early out.
2310 if (Known.isUnknown())
2311 break;
2312 }
2313 if (!!DemandedRHS) {
2314 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
2315 computeKnownBits(V: RHS, DemandedElts: DemandedRHS, Known&: Known2, Q, Depth: Depth + 1);
2316 Known = Known.intersectWith(RHS: Known2);
2317 }
2318 break;
2319 }
2320 case Instruction::InsertElement: {
2321 if (isa<ScalableVectorType>(Val: I->getType())) {
2322 Known.resetAll();
2323 return;
2324 }
2325 const Value *Vec = I->getOperand(i: 0);
2326 const Value *Elt = I->getOperand(i: 1);
2327 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
2328 unsigned NumElts = DemandedElts.getBitWidth();
2329 APInt DemandedVecElts = DemandedElts;
2330 bool NeedsElt = true;
2331 // If we know the index we are inserting too, clear it from Vec check.
2332 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
2333 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
2334 NeedsElt = DemandedElts[CIdx->getZExtValue()];
2335 }
2336
2337 Known.setAllConflict();
2338 if (NeedsElt) {
2339 computeKnownBits(V: Elt, Known, Q, Depth: Depth + 1);
2340 // If we don't know any bits, early out.
2341 if (Known.isUnknown())
2342 break;
2343 }
2344
2345 if (!DemandedVecElts.isZero()) {
2346 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known&: Known2, Q, Depth: Depth + 1);
2347 Known = Known.intersectWith(RHS: Known2);
2348 }
2349 break;
2350 }
2351 case Instruction::ExtractElement: {
2352 // Look through extract element. If the index is non-constant or
2353 // out-of-range demand all elements, otherwise just the extracted element.
2354 const Value *Vec = I->getOperand(i: 0);
2355 const Value *Idx = I->getOperand(i: 1);
2356 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
2357 if (isa<ScalableVectorType>(Val: Vec->getType())) {
2358 // FIXME: there's probably *something* we can do with scalable vectors
2359 Known.resetAll();
2360 break;
2361 }
2362 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2363 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
2364 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
2365 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
2366 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known, Q, Depth: Depth + 1);
2367 break;
2368 }
2369 case Instruction::ExtractValue:
2370 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I->getOperand(i: 0))) {
2371 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: I);
2372 if (EVI->getNumIndices() != 1) break;
2373 if (EVI->getIndices()[0] == 0) {
2374 switch (II->getIntrinsicID()) {
2375 default: break;
2376 case Intrinsic::uadd_with_overflow:
2377 case Intrinsic::sadd_with_overflow:
2378 computeKnownBitsAddSub(
2379 Add: true, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2380 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2381 break;
2382 case Intrinsic::usub_with_overflow:
2383 case Intrinsic::ssub_with_overflow:
2384 computeKnownBitsAddSub(
2385 Add: false, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2386 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2387 break;
2388 case Intrinsic::umul_with_overflow:
2389 case Intrinsic::smul_with_overflow:
2390 computeKnownBitsMul(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), NSW: false,
2391 NUW: false, DemandedElts, Known, Known2, Q, Depth);
2392 break;
2393 }
2394 }
2395 }
2396 break;
2397 case Instruction::Freeze:
2398 if (isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
2399 Depth: Depth + 1))
2400 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2401 break;
2402 }
2403}
2404
2405/// Determine which bits of V are known to be either zero or one and return
2406/// them.
2407KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
2408 const SimplifyQuery &Q, unsigned Depth) {
2409 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2410 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
2411 return Known;
2412}
2413
2414/// Determine which bits of V are known to be either zero or one and return
2415/// them.
2416KnownBits llvm::computeKnownBits(const Value *V, const SimplifyQuery &Q,
2417 unsigned Depth) {
2418 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2419 computeKnownBits(V, Known, Q, Depth);
2420 return Known;
2421}
2422
2423/// Determine which bits of V are known to be either zero or one and return
2424/// them in the Known bit set.
2425///
2426/// NOTE: we cannot consider 'undef' to be "IsZero" here. The problem is that
2427/// we cannot optimize based on the assumption that it is zero without changing
2428/// it to be an explicit zero. If we don't change it to zero, other code could
2429/// optimized based on the contradictory assumption that it is non-zero.
2430/// Because instcombine aggressively folds operations with undef args anyway,
2431/// this won't lose us code quality.
2432///
2433/// This function is defined on values with integer type, values with pointer
2434/// type, and vectors of integers. In the case
2435/// where V is a vector, known zero, and known one values are the
2436/// same width as the vector element, and the bit is set only if it is true
2437/// for all of the demanded elements in the vector specified by DemandedElts.
2438void computeKnownBits(const Value *V, const APInt &DemandedElts,
2439 KnownBits &Known, const SimplifyQuery &Q,
2440 unsigned Depth) {
2441 if (!DemandedElts) {
2442 // No demanded elts, better to assume we don't know anything.
2443 Known.resetAll();
2444 return;
2445 }
2446
2447 assert(V && "No Value?");
2448 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2449
2450#ifndef NDEBUG
2451 Type *Ty = V->getType();
2452 unsigned BitWidth = Known.getBitWidth();
2453
2454 assert((Ty->isIntOrIntVectorTy(BitWidth) || Ty->isPtrOrPtrVectorTy()) &&
2455 "Not integer or pointer type!");
2456
2457 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
2458 assert(
2459 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
2460 "DemandedElt width should equal the fixed vector number of elements");
2461 } else {
2462 assert(DemandedElts == APInt(1, 1) &&
2463 "DemandedElt width should be 1 for scalars or scalable vectors");
2464 }
2465
2466 Type *ScalarTy = Ty->getScalarType();
2467 if (ScalarTy->isPointerTy()) {
2468 assert(BitWidth == Q.DL.getPointerTypeSizeInBits(ScalarTy) &&
2469 "V and Known should have same BitWidth");
2470 } else {
2471 assert(BitWidth == Q.DL.getTypeSizeInBits(ScalarTy) &&
2472 "V and Known should have same BitWidth");
2473 }
2474#endif
2475
2476 const APInt *C;
2477 if (match(V, P: m_APInt(Res&: C))) {
2478 // We know all of the bits for a scalar constant or a splat vector constant!
2479 Known = KnownBits::makeConstant(C: *C);
2480 return;
2481 }
2482 // Null and aggregate-zero are all-zeros.
2483 if (isa<ConstantPointerNull>(Val: V) || isa<ConstantAggregateZero>(Val: V)) {
2484 Known.setAllZero();
2485 return;
2486 }
2487 // Handle a constant vector by taking the intersection of the known bits of
2488 // each element.
2489 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: V)) {
2490 assert(!isa<ScalableVectorType>(V->getType()));
2491 // We know that CDV must be a vector of integers. Take the intersection of
2492 // each element.
2493 Known.setAllConflict();
2494 for (unsigned i = 0, e = CDV->getNumElements(); i != e; ++i) {
2495 if (!DemandedElts[i])
2496 continue;
2497 APInt Elt = CDV->getElementAsAPInt(i);
2498 Known.Zero &= ~Elt;
2499 Known.One &= Elt;
2500 }
2501 if (Known.hasConflict())
2502 Known.resetAll();
2503 return;
2504 }
2505
2506 if (const auto *CV = dyn_cast<ConstantVector>(Val: V)) {
2507 assert(!isa<ScalableVectorType>(V->getType()));
2508 // We know that CV must be a vector of integers. Take the intersection of
2509 // each element.
2510 Known.setAllConflict();
2511 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
2512 if (!DemandedElts[i])
2513 continue;
2514 Constant *Element = CV->getAggregateElement(Elt: i);
2515 if (isa<PoisonValue>(Val: Element))
2516 continue;
2517 auto *ElementCI = dyn_cast_or_null<ConstantInt>(Val: Element);
2518 if (!ElementCI) {
2519 Known.resetAll();
2520 return;
2521 }
2522 const APInt &Elt = ElementCI->getValue();
2523 Known.Zero &= ~Elt;
2524 Known.One &= Elt;
2525 }
2526 if (Known.hasConflict())
2527 Known.resetAll();
2528 return;
2529 }
2530
2531 // Start out not knowing anything.
2532 Known.resetAll();
2533
2534 // We can't imply anything about undefs.
2535 if (isa<UndefValue>(Val: V))
2536 return;
2537
2538 // There's no point in looking through other users of ConstantData for
2539 // assumptions. Confirm that we've handled them all.
2540 assert(!isa<ConstantData>(V) && "Unhandled constant data!");
2541
2542 if (const auto *A = dyn_cast<Argument>(Val: V))
2543 if (std::optional<ConstantRange> Range = A->getRange())
2544 Known = Range->toKnownBits();
2545
2546 // All recursive calls that increase depth must come after this.
2547 if (Depth == MaxAnalysisRecursionDepth)
2548 return;
2549
2550 // A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
2551 // the bits of its aliasee.
2552 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: V)) {
2553 if (!GA->isInterposable())
2554 computeKnownBits(V: GA->getAliasee(), Known, Q, Depth: Depth + 1);
2555 return;
2556 }
2557
2558 if (const Operator *I = dyn_cast<Operator>(Val: V))
2559 computeKnownBitsFromOperator(I, DemandedElts, Known, Q, Depth);
2560 else if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
2561 if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
2562 Known = CR->toKnownBits();
2563 }
2564
2565 // Aligned pointers have trailing zeros - refine Known.Zero set
2566 if (isa<PointerType>(Val: V->getType())) {
2567 Align Alignment = V->getPointerAlignment(DL: Q.DL);
2568 Known.Zero.setLowBits(Log2(A: Alignment));
2569 }
2570
2571 // computeKnownBitsFromContext strictly refines Known.
2572 // Therefore, we run them after computeKnownBitsFromOperator.
2573
2574 // Check whether we can determine known bits from context such as assumes.
2575 computeKnownBitsFromContext(V, Known, Q, Depth);
2576}
2577
2578/// Try to detect a recurrence that the value of the induction variable is
2579/// always a power of two (or zero).
2580static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
2581 SimplifyQuery &Q, unsigned Depth) {
2582 BinaryOperator *BO = nullptr;
2583 Value *Start = nullptr, *Step = nullptr;
2584 if (!matchSimpleRecurrence(P: PN, BO, Start, Step))
2585 return false;
2586
2587 // Initial value must be a power of two.
2588 for (const Use &U : PN->operands()) {
2589 if (U.get() == Start) {
2590 // Initial value comes from a different BB, need to adjust context
2591 // instruction for analysis.
2592 Q.CxtI = PN->getIncomingBlock(U)->getTerminator();
2593 if (!isKnownToBeAPowerOfTwo(V: Start, OrZero, Q, Depth))
2594 return false;
2595 }
2596 }
2597
2598 // Except for Mul, the induction variable must be on the left side of the
2599 // increment expression, otherwise its value can be arbitrary.
2600 if (BO->getOpcode() != Instruction::Mul && BO->getOperand(i_nocapture: 1) != Step)
2601 return false;
2602
2603 Q.CxtI = BO->getParent()->getTerminator();
2604 switch (BO->getOpcode()) {
2605 case Instruction::Mul:
2606 // Power of two is closed under multiplication.
2607 return (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) ||
2608 Q.IIQ.hasNoSignedWrap(Op: BO)) &&
2609 isKnownToBeAPowerOfTwo(V: Step, OrZero, Q, Depth);
2610 case Instruction::SDiv:
2611 // Start value must not be signmask for signed division, so simply being a
2612 // power of two is not sufficient, and it has to be a constant.
2613 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2614 return false;
2615 [[fallthrough]];
2616 case Instruction::UDiv:
2617 // Divisor must be a power of two.
2618 // If OrZero is false, cannot guarantee induction variable is non-zero after
2619 // division, same for Shr, unless it is exact division.
2620 return (OrZero || Q.IIQ.isExact(Op: BO)) &&
2621 isKnownToBeAPowerOfTwo(V: Step, OrZero: false, Q, Depth);
2622 case Instruction::Shl:
2623 return OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO);
2624 case Instruction::AShr:
2625 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2626 return false;
2627 [[fallthrough]];
2628 case Instruction::LShr:
2629 return OrZero || Q.IIQ.isExact(Op: BO);
2630 default:
2631 return false;
2632 }
2633}
2634
2635/// Return true if we can infer that \p V is known to be a power of 2 from
2636/// dominating condition \p Cond (e.g., ctpop(V) == 1).
2637static bool isImpliedToBeAPowerOfTwoFromCond(const Value *V, bool OrZero,
2638 const Value *Cond,
2639 bool CondIsTrue) {
2640 CmpPredicate Pred;
2641 const APInt *RHSC;
2642 if (!match(V: Cond, P: m_ICmp(Pred, L: m_Ctpop(Op0: m_Specific(V)), R: m_APInt(Res&: RHSC))))
2643 return false;
2644 if (!CondIsTrue)
2645 Pred = ICmpInst::getInversePredicate(pred: Pred);
2646 // ctpop(V) u< 2
2647 if (OrZero && Pred == ICmpInst::ICMP_ULT && *RHSC == 2)
2648 return true;
2649 // ctpop(V) == 1
2650 return Pred == ICmpInst::ICMP_EQ && *RHSC == 1;
2651}
2652
2653/// Return true if the given value is known to have exactly one
2654/// bit set when defined. For vectors return true if every element is known to
2655/// be a power of two when defined. Supports values with integer or pointer
2656/// types and vectors of integers.
2657bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero,
2658 const SimplifyQuery &Q, unsigned Depth) {
2659 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2660
2661 if (isa<Constant>(Val: V))
2662 return OrZero ? match(V, P: m_Power2OrZero()) : match(V, P: m_Power2());
2663
2664 // i1 is by definition a power of 2 or zero.
2665 if (OrZero && V->getType()->getScalarSizeInBits() == 1)
2666 return true;
2667
2668 // Try to infer from assumptions.
2669 if (Q.AC && Q.CxtI) {
2670 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
2671 if (!AssumeVH)
2672 continue;
2673 CallInst *I = cast<CallInst>(Val&: AssumeVH);
2674 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond: I->getArgOperand(i: 0),
2675 /*CondIsTrue=*/true) &&
2676 isValidAssumeForContext(I, Q))
2677 return true;
2678 }
2679 }
2680
2681 // Handle dominating conditions.
2682 if (Q.DC && Q.CxtI && Q.DT) {
2683 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
2684 Value *Cond = BI->getCondition();
2685
2686 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
2687 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2688 /*CondIsTrue=*/true) &&
2689 Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
2690 return true;
2691
2692 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
2693 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2694 /*CondIsTrue=*/false) &&
2695 Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
2696 return true;
2697 }
2698 }
2699
2700 auto *I = dyn_cast<Instruction>(Val: V);
2701 if (!I)
2702 return false;
2703
2704 if (Q.CxtI && match(V, P: m_VScale())) {
2705 const Function *F = Q.CxtI->getFunction();
2706 // The vscale_range indicates vscale is a power-of-two.
2707 return F->hasFnAttribute(Kind: Attribute::VScaleRange);
2708 }
2709
2710 // 1 << X is clearly a power of two if the one is not shifted off the end. If
2711 // it is shifted off the end then the result is undefined.
2712 if (match(V: I, P: m_Shl(L: m_One(), R: m_Value())))
2713 return true;
2714
2715 // (signmask) >>l X is clearly a power of two if the one is not shifted off
2716 // the bottom. If it is shifted off the bottom then the result is undefined.
2717 if (match(V: I, P: m_LShr(L: m_SignMask(), R: m_Value())))
2718 return true;
2719
2720 // The remaining tests are all recursive, so bail out if we hit the limit.
2721 if (Depth++ == MaxAnalysisRecursionDepth)
2722 return false;
2723
2724 switch (I->getOpcode()) {
2725 case Instruction::ZExt:
2726 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2727 case Instruction::Trunc:
2728 return OrZero && isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2729 case Instruction::Shl:
2730 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: I) || Q.IIQ.hasNoSignedWrap(Op: I))
2731 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2732 return false;
2733 case Instruction::LShr:
2734 if (OrZero || Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2735 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2736 return false;
2737 case Instruction::UDiv:
2738 if (Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2739 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2740 return false;
2741 case Instruction::Mul:
2742 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2743 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth) &&
2744 (OrZero || isKnownNonZero(V: I, Q, Depth));
2745 case Instruction::And:
2746 // A power of two and'd with anything is a power of two or zero.
2747 if (OrZero &&
2748 (isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), /*OrZero*/ true, Q, Depth) ||
2749 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), /*OrZero*/ true, Q, Depth)))
2750 return true;
2751 // X & (-X) is always a power of two or zero.
2752 if (match(V: I->getOperand(i: 0), P: m_Neg(V: m_Specific(V: I->getOperand(i: 1)))) ||
2753 match(V: I->getOperand(i: 1), P: m_Neg(V: m_Specific(V: I->getOperand(i: 0)))))
2754 return OrZero || isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
2755 return false;
2756 case Instruction::Add: {
2757 // Adding a power-of-two or zero to the same power-of-two or zero yields
2758 // either the original power-of-two, a larger power-of-two or zero.
2759 const OverflowingBinaryOperator *VOBO = cast<OverflowingBinaryOperator>(Val: V);
2760 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO) ||
2761 Q.IIQ.hasNoSignedWrap(Op: VOBO)) {
2762 if (match(V: I->getOperand(i: 0),
2763 P: m_c_And(L: m_Specific(V: I->getOperand(i: 1)), R: m_Value())) &&
2764 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth))
2765 return true;
2766 if (match(V: I->getOperand(i: 1),
2767 P: m_c_And(L: m_Specific(V: I->getOperand(i: 0)), R: m_Value())) &&
2768 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth))
2769 return true;
2770
2771 unsigned BitWidth = V->getType()->getScalarSizeInBits();
2772 KnownBits LHSBits(BitWidth);
2773 computeKnownBits(V: I->getOperand(i: 0), Known&: LHSBits, Q, Depth);
2774
2775 KnownBits RHSBits(BitWidth);
2776 computeKnownBits(V: I->getOperand(i: 1), Known&: RHSBits, Q, Depth);
2777 // If i8 V is a power of two or zero:
2778 // ZeroBits: 1 1 1 0 1 1 1 1
2779 // ~ZeroBits: 0 0 0 1 0 0 0 0
2780 if ((~(LHSBits.Zero & RHSBits.Zero)).isPowerOf2())
2781 // If OrZero isn't set, we cannot give back a zero result.
2782 // Make sure either the LHS or RHS has a bit set.
2783 if (OrZero || RHSBits.One.getBoolValue() || LHSBits.One.getBoolValue())
2784 return true;
2785 }
2786
2787 // LShr(UINT_MAX, Y) + 1 is a power of two (if add is nuw) or zero.
2788 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO))
2789 if (match(V: I, P: m_Add(L: m_LShr(L: m_AllOnes(), R: m_Value()), R: m_One())))
2790 return true;
2791 return false;
2792 }
2793 case Instruction::Select:
2794 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2795 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 2), OrZero, Q, Depth);
2796 case Instruction::PHI: {
2797 // A PHI node is power of two if all incoming values are power of two, or if
2798 // it is an induction variable where in each step its value is a power of
2799 // two.
2800 auto *PN = cast<PHINode>(Val: I);
2801 SimplifyQuery RecQ = Q.getWithoutCondContext();
2802
2803 // Check if it is an induction variable and always power of two.
2804 if (isPowerOfTwoRecurrence(PN, OrZero, Q&: RecQ, Depth))
2805 return true;
2806
2807 // Recursively check all incoming values. Limit recursion to 2 levels, so
2808 // that search complexity is limited to number of operands^2.
2809 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
2810 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
2811 // Value is power of 2 if it is coming from PHI node itself by induction.
2812 if (U.get() == PN)
2813 return true;
2814
2815 // Change the context instruction to the incoming block where it is
2816 // evaluated.
2817 RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
2818 return isKnownToBeAPowerOfTwo(V: U.get(), OrZero, Q: RecQ, Depth: NewDepth);
2819 });
2820 }
2821 case Instruction::Invoke:
2822 case Instruction::Call: {
2823 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2824 switch (II->getIntrinsicID()) {
2825 case Intrinsic::umax:
2826 case Intrinsic::smax:
2827 case Intrinsic::umin:
2828 case Intrinsic::smin:
2829 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 1), OrZero, Q, Depth) &&
2830 isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2831 // bswap/bitreverse just move around bits, but don't change any 1s/0s
2832 // thus dont change pow2/non-pow2 status.
2833 case Intrinsic::bitreverse:
2834 case Intrinsic::bswap:
2835 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2836 case Intrinsic::fshr:
2837 case Intrinsic::fshl:
2838 // If Op0 == Op1, this is a rotate. is_pow2(rotate(x, y)) == is_pow2(x)
2839 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
2840 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2841 break;
2842 default:
2843 break;
2844 }
2845 }
2846 return false;
2847 }
2848 default:
2849 return false;
2850 }
2851}
2852
2853/// Test whether a GEP's result is known to be non-null.
2854///
2855/// Uses properties inherent in a GEP to try to determine whether it is known
2856/// to be non-null.
2857///
2858/// Currently this routine does not support vector GEPs.
2859static bool isGEPKnownNonNull(const GEPOperator *GEP, const SimplifyQuery &Q,
2860 unsigned Depth) {
2861 const Function *F = nullptr;
2862 if (const Instruction *I = dyn_cast<Instruction>(Val: GEP))
2863 F = I->getFunction();
2864
2865 // If the gep is nuw or inbounds with invalid null pointer, then the GEP
2866 // may be null iff the base pointer is null and the offset is zero.
2867 if (!GEP->hasNoUnsignedWrap() &&
2868 !(GEP->isInBounds() &&
2869 !NullPointerIsDefined(F, AS: GEP->getPointerAddressSpace())))
2870 return false;
2871
2872 // FIXME: Support vector-GEPs.
2873 assert(GEP->getType()->isPointerTy() && "We only support plain pointer GEP");
2874
2875 // If the base pointer is non-null, we cannot walk to a null address with an
2876 // inbounds GEP in address space zero.
2877 if (isKnownNonZero(V: GEP->getPointerOperand(), Q, Depth))
2878 return true;
2879
2880 // Walk the GEP operands and see if any operand introduces a non-zero offset.
2881 // If so, then the GEP cannot produce a null pointer, as doing so would
2882 // inherently violate the inbounds contract within address space zero.
2883 for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
2884 GTI != GTE; ++GTI) {
2885 // Struct types are easy -- they must always be indexed by a constant.
2886 if (StructType *STy = GTI.getStructTypeOrNull()) {
2887 ConstantInt *OpC = cast<ConstantInt>(Val: GTI.getOperand());
2888 unsigned ElementIdx = OpC->getZExtValue();
2889 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
2890 uint64_t ElementOffset = SL->getElementOffset(Idx: ElementIdx);
2891 if (ElementOffset > 0)
2892 return true;
2893 continue;
2894 }
2895
2896 // If we have a zero-sized type, the index doesn't matter. Keep looping.
2897 if (GTI.getSequentialElementStride(DL: Q.DL).isZero())
2898 continue;
2899
2900 // Fast path the constant operand case both for efficiency and so we don't
2901 // increment Depth when just zipping down an all-constant GEP.
2902 if (ConstantInt *OpC = dyn_cast<ConstantInt>(Val: GTI.getOperand())) {
2903 if (!OpC->isZero())
2904 return true;
2905 continue;
2906 }
2907
2908 // We post-increment Depth here because while isKnownNonZero increments it
2909 // as well, when we pop back up that increment won't persist. We don't want
2910 // to recurse 10k times just because we have 10k GEP operands. We don't
2911 // bail completely out because we want to handle constant GEPs regardless
2912 // of depth.
2913 if (Depth++ >= MaxAnalysisRecursionDepth)
2914 continue;
2915
2916 if (isKnownNonZero(V: GTI.getOperand(), Q, Depth))
2917 return true;
2918 }
2919
2920 return false;
2921}
2922
2923static bool isKnownNonNullFromDominatingCondition(const Value *V,
2924 const Instruction *CtxI,
2925 const DominatorTree *DT) {
2926 assert(!isa<Constant>(V) && "Called for constant?");
2927
2928 if (!CtxI || !DT)
2929 return false;
2930
2931 unsigned NumUsesExplored = 0;
2932 for (auto &U : V->uses()) {
2933 // Avoid massive lists
2934 if (NumUsesExplored >= DomConditionsMaxUses)
2935 break;
2936 NumUsesExplored++;
2937
2938 const Instruction *UI = cast<Instruction>(Val: U.getUser());
2939 // If the value is used as an argument to a call or invoke, then argument
2940 // attributes may provide an answer about null-ness.
2941 if (V->getType()->isPointerTy()) {
2942 if (const auto *CB = dyn_cast<CallBase>(Val: UI)) {
2943 if (CB->isArgOperand(U: &U) &&
2944 CB->paramHasNonNullAttr(ArgNo: CB->getArgOperandNo(U: &U),
2945 /*AllowUndefOrPoison=*/false) &&
2946 DT->dominates(Def: CB, User: CtxI))
2947 return true;
2948 }
2949 }
2950
2951 // If the value is used as a load/store, then the pointer must be non null.
2952 if (V == getLoadStorePointerOperand(V: UI)) {
2953 if (!NullPointerIsDefined(F: UI->getFunction(),
2954 AS: V->getType()->getPointerAddressSpace()) &&
2955 DT->dominates(Def: UI, User: CtxI))
2956 return true;
2957 }
2958
2959 if ((match(V: UI, P: m_IDiv(L: m_Value(), R: m_Specific(V))) ||
2960 match(V: UI, P: m_IRem(L: m_Value(), R: m_Specific(V)))) &&
2961 isValidAssumeForContext(Inv: UI, CxtI: CtxI, DT))
2962 return true;
2963
2964 // Consider only compare instructions uniquely controlling a branch
2965 Value *RHS;
2966 CmpPredicate Pred;
2967 if (!match(V: UI, P: m_c_ICmp(Pred, L: m_Specific(V), R: m_Value(V&: RHS))))
2968 continue;
2969
2970 bool NonNullIfTrue;
2971 if (cmpExcludesZero(Pred, RHS))
2972 NonNullIfTrue = true;
2973 else if (cmpExcludesZero(Pred: CmpInst::getInversePredicate(pred: Pred), RHS))
2974 NonNullIfTrue = false;
2975 else
2976 continue;
2977
2978 SmallVector<const User *, 4> WorkList;
2979 SmallPtrSet<const User *, 4> Visited;
2980 for (const auto *CmpU : UI->users()) {
2981 assert(WorkList.empty() && "Should be!");
2982 if (Visited.insert(Ptr: CmpU).second)
2983 WorkList.push_back(Elt: CmpU);
2984
2985 while (!WorkList.empty()) {
2986 auto *Curr = WorkList.pop_back_val();
2987
2988 // If a user is an AND, add all its users to the work list. We only
2989 // propagate "pred != null" condition through AND because it is only
2990 // correct to assume that all conditions of AND are met in true branch.
2991 // TODO: Support similar logic of OR and EQ predicate?
2992 if (NonNullIfTrue)
2993 if (match(V: Curr, P: m_LogicalAnd(L: m_Value(), R: m_Value()))) {
2994 for (const auto *CurrU : Curr->users())
2995 if (Visited.insert(Ptr: CurrU).second)
2996 WorkList.push_back(Elt: CurrU);
2997 continue;
2998 }
2999
3000 if (const CondBrInst *BI = dyn_cast<CondBrInst>(Val: Curr)) {
3001 BasicBlock *NonNullSuccessor =
3002 BI->getSuccessor(i: NonNullIfTrue ? 0 : 1);
3003 BasicBlockEdge Edge(BI->getParent(), NonNullSuccessor);
3004 if (DT->dominates(BBE: Edge, BB: CtxI->getParent()))
3005 return true;
3006 } else if (NonNullIfTrue && isGuard(U: Curr) &&
3007 DT->dominates(Def: cast<Instruction>(Val: Curr), User: CtxI)) {
3008 return true;
3009 }
3010 }
3011 }
3012 }
3013
3014 return false;
3015}
3016
3017/// Does the 'Range' metadata (which must be a valid MD_range operand list)
3018/// ensure that the value it's attached to is never Value? 'RangeType' is
3019/// is the type of the value described by the range.
3020static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value) {
3021 const unsigned NumRanges = Ranges->getNumOperands() / 2;
3022 assert(NumRanges >= 1);
3023 for (unsigned i = 0; i < NumRanges; ++i) {
3024 ConstantInt *Lower =
3025 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 0));
3026 ConstantInt *Upper =
3027 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 1));
3028 ConstantRange Range(Lower->getValue(), Upper->getValue());
3029 if (Range.contains(Val: Value))
3030 return false;
3031 }
3032 return true;
3033}
3034
3035/// Try to detect a recurrence that monotonically increases/decreases from a
3036/// non-zero starting value. These are common as induction variables.
3037static bool isNonZeroRecurrence(const PHINode *PN) {
3038 BinaryOperator *BO = nullptr;
3039 Value *Start = nullptr, *Step = nullptr;
3040 const APInt *StartC, *StepC;
3041 if (!matchSimpleRecurrence(P: PN, BO, Start, Step) ||
3042 !match(V: Start, P: m_APInt(Res&: StartC)) || StartC->isZero())
3043 return false;
3044
3045 switch (BO->getOpcode()) {
3046 case Instruction::Add:
3047 // Starting from non-zero and stepping away from zero can never wrap back
3048 // to zero.
3049 return BO->hasNoUnsignedWrap() ||
3050 (BO->hasNoSignedWrap() && match(V: Step, P: m_APInt(Res&: StepC)) &&
3051 StartC->isNegative() == StepC->isNegative());
3052 case Instruction::Mul:
3053 return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) &&
3054 match(V: Step, P: m_APInt(Res&: StepC)) && !StepC->isZero();
3055 case Instruction::Shl:
3056 return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap();
3057 case Instruction::AShr:
3058 case Instruction::LShr:
3059 return BO->isExact();
3060 default:
3061 return false;
3062 }
3063}
3064
3065static bool matchOpWithOpEqZero(Value *Op0, Value *Op1) {
3066 return match(V: Op0, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3067 L: m_Specific(V: Op1), R: m_Zero()))) ||
3068 match(V: Op1, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3069 L: m_Specific(V: Op0), R: m_Zero())));
3070}
3071
3072static bool isNonZeroAdd(const APInt &DemandedElts, const SimplifyQuery &Q,
3073 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3074 bool NUW, unsigned Depth) {
3075 // (X + (X != 0)) is non zero
3076 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3077 return true;
3078
3079 if (NUW)
3080 return isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3081 isKnownNonZero(V: X, DemandedElts, Q, Depth);
3082
3083 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3084 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3085
3086 // If X and Y are both non-negative (as signed values) then their sum is not
3087 // zero unless both X and Y are zero.
3088 if (XKnown.isNonNegative() && YKnown.isNonNegative())
3089 if (isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3090 isKnownNonZero(V: X, DemandedElts, Q, Depth))
3091 return true;
3092
3093 // If X and Y are both negative (as signed values) then their sum is not
3094 // zero unless both X and Y equal INT_MIN.
3095 if (XKnown.isNegative() && YKnown.isNegative()) {
3096 APInt Mask = APInt::getSignedMaxValue(numBits: BitWidth);
3097 // The sign bit of X is set. If some other bit is set then X is not equal
3098 // to INT_MIN.
3099 if (XKnown.One.intersects(RHS: Mask))
3100 return true;
3101 // The sign bit of Y is set. If some other bit is set then Y is not equal
3102 // to INT_MIN.
3103 if (YKnown.One.intersects(RHS: Mask))
3104 return true;
3105 }
3106
3107 // The sum of a non-negative number and a power of two is not zero.
3108 if (XKnown.isNonNegative() &&
3109 isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ false, Q, Depth))
3110 return true;
3111 if (YKnown.isNonNegative() &&
3112 isKnownToBeAPowerOfTwo(V: X, /*OrZero*/ false, Q, Depth))
3113 return true;
3114
3115 return KnownBits::add(LHS: XKnown, RHS: YKnown, NSW, NUW).isNonZero();
3116}
3117
3118static bool isNonZeroSub(const APInt &DemandedElts, const SimplifyQuery &Q,
3119 unsigned BitWidth, Value *X, Value *Y,
3120 unsigned Depth) {
3121 // (X - (X != 0)) is non zero
3122 // ((X != 0) - X) is non zero
3123 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3124 return true;
3125
3126 // TODO: Move this case into isKnownNonEqual().
3127 if (auto *C = dyn_cast<Constant>(Val: X))
3128 if (C->isNullValue() && isKnownNonZero(V: Y, DemandedElts, Q, Depth))
3129 return true;
3130
3131 return ::isKnownNonEqual(V1: X, V2: Y, DemandedElts, Q, Depth);
3132}
3133
3134static bool isNonZeroMul(const APInt &DemandedElts, const SimplifyQuery &Q,
3135 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3136 bool NUW, unsigned Depth) {
3137 // If X and Y are non-zero then so is X * Y as long as the multiplication
3138 // does not overflow.
3139 if (NSW || NUW)
3140 return isKnownNonZero(V: X, DemandedElts, Q, Depth) &&
3141 isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3142
3143 // If either X or Y is odd, then if the other is non-zero the result can't
3144 // be zero.
3145 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3146 if (XKnown.One[0])
3147 return isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3148
3149 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3150 if (YKnown.One[0])
3151 return XKnown.isNonZero() || isKnownNonZero(V: X, DemandedElts, Q, Depth);
3152
3153 // If there exists any subset of X (sX) and subset of Y (sY) s.t sX * sY is
3154 // non-zero, then X * Y is non-zero. We can find sX and sY by just taking
3155 // the lowest known One of X and Y. If they are non-zero, the result
3156 // must be non-zero. We can check if LSB(X) * LSB(Y) != 0 by doing
3157 // X.CountLeadingZeros + Y.CountLeadingZeros < BitWidth.
3158 return (XKnown.countMaxTrailingZeros() + YKnown.countMaxTrailingZeros()) <
3159 BitWidth;
3160}
3161
3162static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
3163 const SimplifyQuery &Q, const KnownBits &KnownVal,
3164 unsigned Depth) {
3165 auto ShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3166 switch (I->getOpcode()) {
3167 case Instruction::Shl:
3168 return Lhs.shl(ShiftAmt: Rhs);
3169 case Instruction::LShr:
3170 return Lhs.lshr(ShiftAmt: Rhs);
3171 case Instruction::AShr:
3172 return Lhs.ashr(ShiftAmt: Rhs);
3173 default:
3174 llvm_unreachable("Unknown Shift Opcode");
3175 }
3176 };
3177
3178 auto InvShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3179 switch (I->getOpcode()) {
3180 case Instruction::Shl:
3181 return Lhs.lshr(ShiftAmt: Rhs);
3182 case Instruction::LShr:
3183 case Instruction::AShr:
3184 return Lhs.shl(ShiftAmt: Rhs);
3185 default:
3186 llvm_unreachable("Unknown Shift Opcode");
3187 }
3188 };
3189
3190 if (KnownVal.isUnknown())
3191 return false;
3192
3193 KnownBits KnownCnt =
3194 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3195 APInt MaxShift = KnownCnt.getMaxValue();
3196 unsigned NumBits = KnownVal.getBitWidth();
3197 if (MaxShift.uge(RHS: NumBits))
3198 return false;
3199
3200 if (!ShiftOp(KnownVal.One, MaxShift).isZero())
3201 return true;
3202
3203 // If all of the bits shifted out are known to be zero, and Val is known
3204 // non-zero then at least one non-zero bit must remain.
3205 if (InvShiftOp(KnownVal.Zero, NumBits - MaxShift)
3206 .eq(RHS: InvShiftOp(APInt::getAllOnes(numBits: NumBits), NumBits - MaxShift)) &&
3207 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth))
3208 return true;
3209
3210 return false;
3211}
3212
3213static bool isKnownNonZeroFromOperator(const Operator *I,
3214 const APInt &DemandedElts,
3215 const SimplifyQuery &Q, unsigned Depth) {
3216 unsigned BitWidth = getBitWidth(Ty: I->getType()->getScalarType(), DL: Q.DL);
3217 switch (I->getOpcode()) {
3218 case Instruction::Alloca:
3219 // Alloca never returns null, malloc might.
3220 return I->getType()->getPointerAddressSpace() == 0;
3221 case Instruction::GetElementPtr:
3222 if (I->getType()->isPointerTy())
3223 return isGEPKnownNonNull(GEP: cast<GEPOperator>(Val: I), Q, Depth);
3224 break;
3225 case Instruction::BitCast: {
3226 // We need to be a bit careful here. We can only peek through the bitcast
3227 // if the scalar size of elements in the operand are smaller than and a
3228 // multiple of the size they are casting too. Take three cases:
3229 //
3230 // 1) Unsafe:
3231 // bitcast <2 x i16> %NonZero to <4 x i8>
3232 //
3233 // %NonZero can have 2 non-zero i16 elements, but isKnownNonZero on a
3234 // <4 x i8> requires that all 4 i8 elements be non-zero which isn't
3235 // guranteed (imagine just sign bit set in the 2 i16 elements).
3236 //
3237 // 2) Unsafe:
3238 // bitcast <4 x i3> %NonZero to <3 x i4>
3239 //
3240 // Even though the scalar size of the src (`i3`) is smaller than the
3241 // scalar size of the dst `i4`, because `i3` is not a multiple of `i4`
3242 // its possible for the `3 x i4` elements to be zero because there are
3243 // some elements in the destination that don't contain any full src
3244 // element.
3245 //
3246 // 3) Safe:
3247 // bitcast <4 x i8> %NonZero to <2 x i16>
3248 //
3249 // This is always safe as non-zero in the 4 i8 elements implies
3250 // non-zero in the combination of any two adjacent ones. Since i8 is a
3251 // multiple of i16, each i16 is guranteed to have 2 full i8 elements.
3252 // This all implies the 2 i16 elements are non-zero.
3253 Type *FromTy = I->getOperand(i: 0)->getType();
3254 if ((FromTy->isIntOrIntVectorTy() || FromTy->isPtrOrPtrVectorTy()) &&
3255 (BitWidth % getBitWidth(Ty: FromTy->getScalarType(), DL: Q.DL)) == 0)
3256 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3257 } break;
3258 case Instruction::IntToPtr:
3259 // Note that we have to take special care to avoid looking through
3260 // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
3261 // as casts that can alter the value, e.g., AddrSpaceCasts.
3262 if (!isa<ScalableVectorType>(Val: I->getType()) &&
3263 Q.DL.getTypeSizeInBits(Ty: I->getOperand(i: 0)->getType()).getFixedValue() <=
3264 Q.DL.getTypeSizeInBits(Ty: I->getType()).getFixedValue())
3265 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3266 break;
3267 case Instruction::PtrToAddr:
3268 // isKnownNonZero() for pointers refers to the address bits being non-zero,
3269 // so we can directly forward.
3270 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3271 case Instruction::PtrToInt:
3272 // For inttoptr, make sure the result size is >= the address size. If the
3273 // address is non-zero, any larger value is also non-zero.
3274 if (Q.DL.getAddressSizeInBits(Ty: I->getOperand(i: 0)->getType()) <=
3275 I->getType()->getScalarSizeInBits())
3276 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3277 break;
3278 case Instruction::Trunc:
3279 // nuw/nsw trunc preserves zero/non-zero status of input.
3280 if (auto *TI = dyn_cast<TruncInst>(Val: I))
3281 if (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap())
3282 return isKnownNonZero(V: TI->getOperand(i_nocapture: 0), DemandedElts, Q, Depth);
3283 break;
3284
3285 // Iff x - y != 0, then x ^ y != 0
3286 // Therefore we can do the same exact checks
3287 case Instruction::Xor:
3288 case Instruction::Sub:
3289 return isNonZeroSub(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3290 Y: I->getOperand(i: 1), Depth);
3291 case Instruction::Or:
3292 // (X | (X != 0)) is non zero
3293 if (matchOpWithOpEqZero(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1)))
3294 return true;
3295 // X | Y != 0 if X != Y.
3296 if (isKnownNonEqual(V1: I->getOperand(i: 0), V2: I->getOperand(i: 1), DemandedElts, Q,
3297 Depth))
3298 return true;
3299 // X | Y != 0 if X != 0 or Y != 0.
3300 return isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth) ||
3301 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3302 case Instruction::SExt:
3303 case Instruction::ZExt:
3304 // ext X != 0 if X != 0.
3305 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3306
3307 case Instruction::Shl: {
3308 // shl nsw/nuw can't remove any non-zero bits.
3309 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3310 if (Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO))
3311 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3312
3313 // shl X, Y != 0 if X is odd. Note that the value of the shift is undefined
3314 // if the lowest bit is shifted off the end.
3315 KnownBits Known(BitWidth);
3316 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth);
3317 if (Known.One[0])
3318 return true;
3319
3320 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3321 }
3322 case Instruction::LShr:
3323 case Instruction::AShr: {
3324 // shr exact can only shift out zero bits.
3325 const PossiblyExactOperator *BO = cast<PossiblyExactOperator>(Val: I);
3326 if (BO->isExact())
3327 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3328
3329 // shr X, Y != 0 if X is negative. Note that the value of the shift is not
3330 // defined if the sign bit is shifted off the end.
3331 KnownBits Known =
3332 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3333 if (Known.isNegative())
3334 return true;
3335
3336 // shr (add nuw A, B), C is non-zero if A or B has a known-one bit at
3337 // position >= C, because the sum >= max(A, B).
3338 Value *A, *B;
3339 const APInt *C;
3340 if (Depth + 1 < MaxAnalysisRecursionDepth &&
3341 match(V: I->getOperand(i: 0), P: m_NUWAdd(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3342 match(V: I->getOperand(i: 1), P: m_APInt(Res&: C)) && C->ult(RHS: BitWidth)) {
3343 KnownBits KnownA = computeKnownBits(V: A, DemandedElts, Q, Depth: Depth + 1);
3344 if (!KnownA.One.lshr(ShiftAmt: *C).isZero())
3345 return true;
3346 KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
3347 if (!KnownB.One.lshr(ShiftAmt: *C).isZero())
3348 return true;
3349 }
3350
3351 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3352 }
3353 case Instruction::UDiv:
3354 case Instruction::SDiv: {
3355 // X / Y
3356 // div exact can only produce a zero if the dividend is zero.
3357 if (cast<PossiblyExactOperator>(Val: I)->isExact())
3358 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3359
3360 KnownBits XKnown =
3361 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3362 // If X is fully unknown we won't be able to figure anything out so don't
3363 // both computing knownbits for Y.
3364 if (XKnown.isUnknown())
3365 return false;
3366
3367 KnownBits YKnown =
3368 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3369 if (I->getOpcode() == Instruction::SDiv) {
3370 // For signed division need to compare abs value of the operands.
3371 XKnown = XKnown.abs(/*IntMinIsPoison*/ false);
3372 YKnown = YKnown.abs(/*IntMinIsPoison*/ false);
3373 }
3374 // If X u>= Y then div is non zero (0/0 is UB).
3375 std::optional<bool> XUgeY = KnownBits::uge(LHS: XKnown, RHS: YKnown);
3376 // If X is total unknown or X u< Y we won't be able to prove non-zero
3377 // with compute known bits so just return early.
3378 return XUgeY && *XUgeY;
3379 }
3380 case Instruction::Add: {
3381 // X + Y.
3382
3383 // If Add has nuw wrap flag, then if either X or Y is non-zero the result is
3384 // non-zero.
3385 auto *BO = cast<OverflowingBinaryOperator>(Val: I);
3386 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3387 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3388 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3389 }
3390 case Instruction::Mul: {
3391 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3392 return isNonZeroMul(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3393 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3394 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3395 }
3396 case Instruction::Select: {
3397 // (C ? X : Y) != 0 if X != 0 and Y != 0.
3398
3399 // First check if the arm is non-zero using `isKnownNonZero`. If that fails,
3400 // then see if the select condition implies the arm is non-zero. For example
3401 // (X != 0 ? X : Y), we know the true arm is non-zero as the `X` "return" is
3402 // dominated by `X != 0`.
3403 auto SelectArmIsNonZero = [&](bool IsTrueArm) {
3404 Value *Op;
3405 Op = IsTrueArm ? I->getOperand(i: 1) : I->getOperand(i: 2);
3406 // Op is trivially non-zero.
3407 if (isKnownNonZero(V: Op, DemandedElts, Q, Depth))
3408 return true;
3409
3410 // The condition of the select dominates the true/false arm. Check if the
3411 // condition implies that a given arm is non-zero.
3412 Value *X;
3413 CmpPredicate Pred;
3414 if (!match(V: I->getOperand(i: 0), P: m_c_ICmp(Pred, L: m_Specific(V: Op), R: m_Value(V&: X))))
3415 return false;
3416
3417 if (!IsTrueArm)
3418 Pred = ICmpInst::getInversePredicate(pred: Pred);
3419
3420 return cmpExcludesZero(Pred, RHS: X);
3421 };
3422
3423 if (SelectArmIsNonZero(/* IsTrueArm */ true) &&
3424 SelectArmIsNonZero(/* IsTrueArm */ false))
3425 return true;
3426 break;
3427 }
3428 case Instruction::PHI: {
3429 auto *PN = cast<PHINode>(Val: I);
3430 if (Q.IIQ.UseInstrInfo && isNonZeroRecurrence(PN))
3431 return true;
3432
3433 // Check if all incoming values are non-zero using recursion.
3434 SimplifyQuery RecQ = Q.getWithoutCondContext();
3435 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
3436 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
3437 if (U.get() == PN)
3438 return true;
3439 RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
3440 // Check if the branch on the phi excludes zero.
3441 CmpPredicate Pred;
3442 Value *X;
3443 BasicBlock *TrueSucc, *FalseSucc;
3444 if (match(V: RecQ.CxtI,
3445 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: U.get()), R: m_Value(V&: X)),
3446 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
3447 // Check for cases of duplicate successors.
3448 if ((TrueSucc == PN->getParent()) != (FalseSucc == PN->getParent())) {
3449 // If we're using the false successor, invert the predicate.
3450 if (FalseSucc == PN->getParent())
3451 Pred = CmpInst::getInversePredicate(pred: Pred);
3452 if (cmpExcludesZero(Pred, RHS: X))
3453 return true;
3454 }
3455 }
3456 // Finally recurse on the edge and check it directly.
3457 return isKnownNonZero(V: U.get(), DemandedElts, Q: RecQ, Depth: NewDepth);
3458 });
3459 }
3460 case Instruction::InsertElement: {
3461 if (isa<ScalableVectorType>(Val: I->getType()))
3462 break;
3463
3464 const Value *Vec = I->getOperand(i: 0);
3465 const Value *Elt = I->getOperand(i: 1);
3466 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
3467
3468 unsigned NumElts = DemandedElts.getBitWidth();
3469 APInt DemandedVecElts = DemandedElts;
3470 bool SkipElt = false;
3471 // If we know the index we are inserting too, clear it from Vec check.
3472 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
3473 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
3474 SkipElt = !DemandedElts[CIdx->getZExtValue()];
3475 }
3476
3477 // Result is zero if Elt is non-zero and rest of the demanded elts in Vec
3478 // are non-zero.
3479 return (SkipElt || isKnownNonZero(V: Elt, Q, Depth)) &&
3480 (DemandedVecElts.isZero() ||
3481 isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth));
3482 }
3483 case Instruction::ExtractElement:
3484 if (const auto *EEI = dyn_cast<ExtractElementInst>(Val: I)) {
3485 const Value *Vec = EEI->getVectorOperand();
3486 const Value *Idx = EEI->getIndexOperand();
3487 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
3488 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
3489 unsigned NumElts = VecTy->getNumElements();
3490 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
3491 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
3492 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
3493 return isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth);
3494 }
3495 }
3496 break;
3497 case Instruction::ShuffleVector: {
3498 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
3499 if (!Shuf)
3500 break;
3501 APInt DemandedLHS, DemandedRHS;
3502 // For undef elements, we don't know anything about the common state of
3503 // the shuffle result.
3504 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
3505 break;
3506 // If demanded elements for both vecs are non-zero, the shuffle is non-zero.
3507 return (DemandedRHS.isZero() ||
3508 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 1), DemandedElts: DemandedRHS, Q, Depth)) &&
3509 (DemandedLHS.isZero() ||
3510 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 0), DemandedElts: DemandedLHS, Q, Depth));
3511 }
3512 case Instruction::Freeze:
3513 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth) &&
3514 isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
3515 Depth);
3516 case Instruction::Load: {
3517 auto *LI = cast<LoadInst>(Val: I);
3518 // A Load tagged with nonnull or dereferenceable with null pointer undefined
3519 // is never null.
3520 if (auto *PtrT = dyn_cast<PointerType>(Val: I->getType())) {
3521 if (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_nonnull) ||
3522 (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_dereferenceable) &&
3523 !NullPointerIsDefined(F: LI->getFunction(), AS: PtrT->getAddressSpace())))
3524 return true;
3525 } else if (MDNode *Ranges = Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_range)) {
3526 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3527 }
3528
3529 // No need to fall through to computeKnownBits as range metadata is already
3530 // handled in isKnownNonZero.
3531 return false;
3532 }
3533 case Instruction::ExtractValue: {
3534 const WithOverflowInst *WO;
3535 if (match(V: I, P: m_ExtractValue<0>(V: m_WithOverflowInst(I&: WO)))) {
3536 switch (WO->getBinaryOp()) {
3537 default:
3538 break;
3539 case Instruction::Add:
3540 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3541 Y: WO->getArgOperand(i: 1),
3542 /*NSW=*/false,
3543 /*NUW=*/false, Depth);
3544 case Instruction::Sub:
3545 return isNonZeroSub(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3546 Y: WO->getArgOperand(i: 1), Depth);
3547 case Instruction::Mul:
3548 return isNonZeroMul(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3549 Y: WO->getArgOperand(i: 1),
3550 /*NSW=*/false, /*NUW=*/false, Depth);
3551 break;
3552 }
3553 }
3554 break;
3555 }
3556 case Instruction::Call:
3557 case Instruction::Invoke: {
3558 const auto *Call = cast<CallBase>(Val: I);
3559 if (I->getType()->isPointerTy()) {
3560 if (Call->isReturnNonNull())
3561 return true;
3562 if (const auto *RP = getArgumentAliasingToReturnedPointer(
3563 Call, /*MustPreserveOffset=*/true))
3564 return isKnownNonZero(V: RP, Q, Depth);
3565 } else {
3566 if (MDNode *Ranges = Q.IIQ.getMetadata(I: Call, KindID: LLVMContext::MD_range))
3567 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3568 if (std::optional<ConstantRange> Range = Call->getRange()) {
3569 const APInt ZeroValue(Range->getBitWidth(), 0);
3570 if (!Range->contains(Val: ZeroValue))
3571 return true;
3572 }
3573 if (const Value *RV = Call->getReturnedArgOperand())
3574 if (RV->getType() == I->getType() && isKnownNonZero(V: RV, Q, Depth))
3575 return true;
3576 }
3577
3578 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3579 switch (II->getIntrinsicID()) {
3580 case Intrinsic::sshl_sat:
3581 case Intrinsic::ushl_sat:
3582 case Intrinsic::abs:
3583 case Intrinsic::bitreverse:
3584 case Intrinsic::bswap:
3585 case Intrinsic::ctpop:
3586 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3587 // NB: We don't do usub_sat here as in any case we can prove its
3588 // non-zero, we will fold it to `sub nuw` in InstCombine.
3589 case Intrinsic::ssub_sat:
3590 // For most types, if x != y then ssub.sat x, y != 0. But
3591 // ssub.sat.i1 0, -1 = 0, because 1 saturates to 0. This means
3592 // isNonZeroSub will do the wrong thing for ssub.sat.i1.
3593 if (BitWidth == 1)
3594 return false;
3595 return isNonZeroSub(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3596 Y: II->getArgOperand(i: 1), Depth);
3597 case Intrinsic::sadd_sat:
3598 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3599 Y: II->getArgOperand(i: 1),
3600 /*NSW=*/true, /* NUW=*/false, Depth);
3601 // Vec reverse preserves zero/non-zero status from input vec.
3602 case Intrinsic::vector_reverse:
3603 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
3604 Q, Depth);
3605 // umin/smin/smax/smin/or of all non-zero elements is always non-zero.
3606 case Intrinsic::vector_reduce_or:
3607 case Intrinsic::vector_reduce_umax:
3608 case Intrinsic::vector_reduce_umin:
3609 case Intrinsic::vector_reduce_smax:
3610 case Intrinsic::vector_reduce_smin:
3611 return isKnownNonZero(V: II->getArgOperand(i: 0), Q, Depth);
3612 case Intrinsic::umax:
3613 case Intrinsic::uadd_sat:
3614 // umax(X, (X != 0)) is non zero
3615 // X +usat (X != 0) is non zero
3616 if (matchOpWithOpEqZero(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1)))
3617 return true;
3618
3619 return isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth) ||
3620 isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3621 case Intrinsic::smax: {
3622 // If either arg is strictly positive the result is non-zero. Otherwise
3623 // the result is non-zero if both ops are non-zero.
3624 auto IsNonZero = [&](Value *Op, std::optional<bool> &OpNonZero,
3625 const KnownBits &OpKnown) {
3626 if (!OpNonZero.has_value())
3627 OpNonZero = OpKnown.isNonZero() ||
3628 isKnownNonZero(V: Op, DemandedElts, Q, Depth);
3629 return *OpNonZero;
3630 };
3631 // Avoid re-computing isKnownNonZero.
3632 std::optional<bool> Op0NonZero, Op1NonZero;
3633 KnownBits Op1Known =
3634 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3635 if (Op1Known.isNonNegative() &&
3636 IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known))
3637 return true;
3638 KnownBits Op0Known =
3639 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3640 if (Op0Known.isNonNegative() &&
3641 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known))
3642 return true;
3643 return IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known) &&
3644 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known);
3645 }
3646 case Intrinsic::smin: {
3647 // If either arg is negative the result is non-zero. Otherwise
3648 // the result is non-zero if both ops are non-zero.
3649 KnownBits Op1Known =
3650 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3651 if (Op1Known.isNegative())
3652 return true;
3653 KnownBits Op0Known =
3654 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3655 if (Op0Known.isNegative())
3656 return true;
3657
3658 if (Op1Known.isNonZero() && Op0Known.isNonZero())
3659 return true;
3660 }
3661 [[fallthrough]];
3662 case Intrinsic::umin:
3663 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth) &&
3664 isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3665 case Intrinsic::cttz:
3666 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3667 .Zero[0];
3668 case Intrinsic::ctlz:
3669 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3670 .isNonNegative();
3671 case Intrinsic::fshr:
3672 case Intrinsic::fshl:
3673 // If Op0 == Op1, this is a rotate. rotate(x, y) != 0 iff x != 0.
3674 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
3675 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3676 break;
3677 case Intrinsic::vscale:
3678 return true;
3679 case Intrinsic::experimental_get_vector_length:
3680 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3681 default:
3682 break;
3683 }
3684 break;
3685 }
3686
3687 return false;
3688 }
3689 }
3690
3691 KnownBits Known(BitWidth);
3692 computeKnownBits(V: I, DemandedElts, Known, Q, Depth);
3693 return Known.One != 0;
3694}
3695
3696/// Return true if the given value is known to be non-zero when defined. For
3697/// vectors, return true if every demanded element is known to be non-zero when
3698/// defined. For pointers, if the context instruction and dominator tree are
3699/// specified, perform context-sensitive analysis and return true if the
3700/// pointer couldn't possibly be null at the specified instruction.
3701/// Supports values with integer or pointer type and vectors of integers.
3702bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
3703 const SimplifyQuery &Q, unsigned Depth) {
3704 Type *Ty = V->getType();
3705
3706#ifndef NDEBUG
3707 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
3708
3709 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
3710 assert(
3711 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
3712 "DemandedElt width should equal the fixed vector number of elements");
3713 } else {
3714 assert(DemandedElts == APInt(1, 1) &&
3715 "DemandedElt width should be 1 for scalars");
3716 }
3717#endif
3718
3719 if (auto *C = dyn_cast<Constant>(Val: V)) {
3720 if (C->isNullValue())
3721 return false;
3722 if (isa<ConstantInt>(Val: C))
3723 // Must be non-zero due to null test above.
3724 return true;
3725
3726 // For constant vectors, check that all elements are poison or known
3727 // non-zero to determine that the whole vector is known non-zero.
3728 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Ty)) {
3729 for (unsigned i = 0, e = VecTy->getNumElements(); i != e; ++i) {
3730 if (!DemandedElts[i])
3731 continue;
3732 Constant *Elt = C->getAggregateElement(Elt: i);
3733 if (!Elt || Elt->isNullValue())
3734 return false;
3735 if (!isa<PoisonValue>(Val: Elt) && !isa<ConstantInt>(Val: Elt))
3736 return false;
3737 }
3738 return true;
3739 }
3740
3741 // Constant ptrauth can be null, iff the base pointer can be.
3742 if (auto *CPA = dyn_cast<ConstantPtrAuth>(Val: V))
3743 return isKnownNonZero(V: CPA->getPointer(), DemandedElts, Q, Depth);
3744
3745 // A global variable in address space 0 is non null unless extern weak
3746 // or an absolute symbol reference. Other address spaces may have null as a
3747 // valid address for a global, so we can't assume anything.
3748 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
3749 if (!GV->isAbsoluteSymbolRef() && !GV->hasExternalWeakLinkage() &&
3750 GV->getType()->getAddressSpace() == 0)
3751 return true;
3752 }
3753
3754 // For constant expressions, fall through to the Operator code below.
3755 if (!isa<ConstantExpr>(Val: V))
3756 return false;
3757 }
3758
3759 if (const auto *A = dyn_cast<Argument>(Val: V))
3760 if (std::optional<ConstantRange> Range = A->getRange()) {
3761 const APInt ZeroValue(Range->getBitWidth(), 0);
3762 if (!Range->contains(Val: ZeroValue))
3763 return true;
3764 }
3765
3766 if (!isa<Constant>(Val: V) && isKnownNonZeroFromAssume(V, Q))
3767 return true;
3768
3769 // Some of the tests below are recursive, so bail out if we hit the limit.
3770 if (Depth++ >= MaxAnalysisRecursionDepth)
3771 return false;
3772
3773 // Check for pointer simplifications.
3774
3775 if (PointerType *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
3776 // A byval, inalloca may not be null in a non-default addres space. A
3777 // nonnull argument is assumed never 0.
3778 if (const Argument *A = dyn_cast<Argument>(Val: V)) {
3779 if (((A->hasPassPointeeByValueCopyAttr() &&
3780 !NullPointerIsDefined(F: A->getParent(), AS: PtrTy->getAddressSpace())) ||
3781 A->hasNonNullAttr()))
3782 return true;
3783 }
3784 }
3785
3786 if (const auto *I = dyn_cast<Operator>(Val: V))
3787 if (isKnownNonZeroFromOperator(I, DemandedElts, Q, Depth))
3788 return true;
3789
3790 if (!isa<Constant>(Val: V) &&
3791 isKnownNonNullFromDominatingCondition(V, CtxI: Q.CxtI, DT: Q.DT))
3792 return true;
3793
3794 if (const Value *Stripped = stripNullTest(V))
3795 return isKnownNonZero(V: Stripped, DemandedElts, Q, Depth);
3796
3797 return false;
3798}
3799
3800bool llvm::isKnownNonZero(const Value *V, const SimplifyQuery &Q,
3801 unsigned Depth) {
3802 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
3803 APInt DemandedElts =
3804 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
3805 return ::isKnownNonZero(V, DemandedElts, Q, Depth);
3806}
3807
3808/// If the pair of operators are the same invertible function, return the
3809/// the operands of the function corresponding to each input. Otherwise,
3810/// return std::nullopt. An invertible function is one that is 1-to-1 and maps
3811/// every input value to exactly one output value. This is equivalent to
3812/// saying that Op1 and Op2 are equal exactly when the specified pair of
3813/// operands are equal, (except that Op1 and Op2 may be poison more often.)
3814static std::optional<std::pair<Value*, Value*>>
3815getInvertibleOperands(const Operator *Op1,
3816 const Operator *Op2) {
3817 if (Op1->getOpcode() != Op2->getOpcode())
3818 return std::nullopt;
3819
3820 auto getOperands = [&](unsigned OpNum) -> auto {
3821 return std::make_pair(x: Op1->getOperand(i: OpNum), y: Op2->getOperand(i: OpNum));
3822 };
3823
3824 switch (Op1->getOpcode()) {
3825 default:
3826 break;
3827 case Instruction::Or:
3828 if (!cast<PossiblyDisjointInst>(Val: Op1)->isDisjoint() ||
3829 !cast<PossiblyDisjointInst>(Val: Op2)->isDisjoint())
3830 break;
3831 [[fallthrough]];
3832 case Instruction::Xor:
3833 case Instruction::Add: {
3834 Value *Other;
3835 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 0)), R: m_Value(V&: Other))))
3836 return std::make_pair(x: Op1->getOperand(i: 1), y&: Other);
3837 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 1)), R: m_Value(V&: Other))))
3838 return std::make_pair(x: Op1->getOperand(i: 0), y&: Other);
3839 break;
3840 }
3841 case Instruction::Sub:
3842 if (Op1->getOperand(i: 0) == Op2->getOperand(i: 0))
3843 return getOperands(1);
3844 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3845 return getOperands(0);
3846 break;
3847 case Instruction::Mul: {
3848 // invertible if A * B == (A * B) mod 2^N where A, and B are integers
3849 // and N is the bitwdith. The nsw case is non-obvious, but proven by
3850 // alive2: https://alive2.llvm.org/ce/z/Z6D5qK
3851 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
3852 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
3853 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
3854 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
3855 break;
3856
3857 // Assume operand order has been canonicalized
3858 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1) &&
3859 isa<ConstantInt>(Val: Op1->getOperand(i: 1)) &&
3860 !cast<ConstantInt>(Val: Op1->getOperand(i: 1))->isZero())
3861 return getOperands(0);
3862 break;
3863 }
3864 case Instruction::Shl: {
3865 // Same as multiplies, with the difference that we don't need to check
3866 // for a non-zero multiply. Shifts always multiply by non-zero.
3867 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
3868 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
3869 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
3870 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
3871 break;
3872
3873 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3874 return getOperands(0);
3875 break;
3876 }
3877 case Instruction::AShr:
3878 case Instruction::LShr: {
3879 auto *PEO1 = cast<PossiblyExactOperator>(Val: Op1);
3880 auto *PEO2 = cast<PossiblyExactOperator>(Val: Op2);
3881 if (!PEO1->isExact() || !PEO2->isExact())
3882 break;
3883
3884 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3885 return getOperands(0);
3886 break;
3887 }
3888 case Instruction::SExt:
3889 case Instruction::ZExt:
3890 if (Op1->getOperand(i: 0)->getType() == Op2->getOperand(i: 0)->getType())
3891 return getOperands(0);
3892 break;
3893 case Instruction::PHI: {
3894 const PHINode *PN1 = cast<PHINode>(Val: Op1);
3895 const PHINode *PN2 = cast<PHINode>(Val: Op2);
3896
3897 // If PN1 and PN2 are both recurrences, can we prove the entire recurrences
3898 // are a single invertible function of the start values? Note that repeated
3899 // application of an invertible function is also invertible
3900 BinaryOperator *BO1 = nullptr;
3901 Value *Start1 = nullptr, *Step1 = nullptr;
3902 BinaryOperator *BO2 = nullptr;
3903 Value *Start2 = nullptr, *Step2 = nullptr;
3904 if (PN1->getParent() != PN2->getParent() ||
3905 !matchSimpleRecurrence(P: PN1, BO&: BO1, Start&: Start1, Step&: Step1) ||
3906 !matchSimpleRecurrence(P: PN2, BO&: BO2, Start&: Start2, Step&: Step2))
3907 break;
3908
3909 auto Values = getInvertibleOperands(Op1: cast<Operator>(Val: BO1),
3910 Op2: cast<Operator>(Val: BO2));
3911 if (!Values)
3912 break;
3913
3914 // We have to be careful of mutually defined recurrences here. Ex:
3915 // * X_i = X_(i-1) OP Y_(i-1), and Y_i = X_(i-1) OP V
3916 // * X_i = Y_i = X_(i-1) OP Y_(i-1)
3917 // The invertibility of these is complicated, and not worth reasoning
3918 // about (yet?).
3919 if (Values->first != PN1 || Values->second != PN2)
3920 break;
3921
3922 return std::make_pair(x&: Start1, y&: Start2);
3923 }
3924 }
3925 return std::nullopt;
3926}
3927
3928/// Return true if V1 == (binop V2, X), where X is known non-zero.
3929/// Only handle a small subset of binops where (binop V2, X) with non-zero X
3930/// implies V2 != V1.
3931static bool isModifyingBinopOfNonZero(const Value *V1, const Value *V2,
3932 const APInt &DemandedElts,
3933 const SimplifyQuery &Q, unsigned Depth) {
3934 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V1);
3935 if (!BO)
3936 return false;
3937 switch (BO->getOpcode()) {
3938 default:
3939 break;
3940 case Instruction::Or:
3941 if (!cast<PossiblyDisjointInst>(Val: V1)->isDisjoint())
3942 break;
3943 [[fallthrough]];
3944 case Instruction::Xor:
3945 case Instruction::Add:
3946 Value *Op = nullptr;
3947 if (V2 == BO->getOperand(i_nocapture: 0))
3948 Op = BO->getOperand(i_nocapture: 1);
3949 else if (V2 == BO->getOperand(i_nocapture: 1))
3950 Op = BO->getOperand(i_nocapture: 0);
3951 else
3952 return false;
3953 return isKnownNonZero(V: Op, DemandedElts, Q, Depth: Depth + 1);
3954 }
3955 return false;
3956}
3957
3958/// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
3959/// the multiplication is nuw or nsw.
3960static bool isNonEqualMul(const Value *V1, const Value *V2,
3961 const APInt &DemandedElts, const SimplifyQuery &Q,
3962 unsigned Depth) {
3963 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
3964 const APInt *C;
3965 return match(V: OBO, P: m_Mul(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
3966 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
3967 !C->isZero() && !C->isOne() &&
3968 isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
3969 }
3970 return false;
3971}
3972
3973/// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
3974/// the shift is nuw or nsw.
3975static bool isNonEqualShl(const Value *V1, const Value *V2,
3976 const APInt &DemandedElts, const SimplifyQuery &Q,
3977 unsigned Depth) {
3978 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
3979 const APInt *C;
3980 return match(V: OBO, P: m_Shl(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
3981 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
3982 !C->isZero() && isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
3983 }
3984 return false;
3985}
3986
3987static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
3988 const APInt &DemandedElts, const SimplifyQuery &Q,
3989 unsigned Depth) {
3990 // Check two PHIs are in same block.
3991 if (PN1->getParent() != PN2->getParent())
3992 return false;
3993
3994 SmallPtrSet<const BasicBlock *, 8> VisitedBBs;
3995 bool UsedFullRecursion = false;
3996 for (const BasicBlock *IncomBB : PN1->blocks()) {
3997 if (!VisitedBBs.insert(Ptr: IncomBB).second)
3998 continue; // Don't reprocess blocks that we have dealt with already.
3999 const Value *IV1 = PN1->getIncomingValueForBlock(BB: IncomBB);
4000 const Value *IV2 = PN2->getIncomingValueForBlock(BB: IncomBB);
4001 const APInt *C1, *C2;
4002 if (match(V: IV1, P: m_APInt(Res&: C1)) && match(V: IV2, P: m_APInt(Res&: C2)) && *C1 != *C2)
4003 continue;
4004
4005 // Only one pair of phi operands is allowed for full recursion.
4006 if (UsedFullRecursion)
4007 return false;
4008
4009 SimplifyQuery RecQ = Q.getWithoutCondContext();
4010 RecQ.CxtI = IncomBB->getTerminator();
4011 if (!isKnownNonEqual(V1: IV1, V2: IV2, DemandedElts, Q: RecQ, Depth: Depth + 1))
4012 return false;
4013 UsedFullRecursion = true;
4014 }
4015 return true;
4016}
4017
4018static bool isNonEqualSelect(const Value *V1, const Value *V2,
4019 const APInt &DemandedElts, const SimplifyQuery &Q,
4020 unsigned Depth) {
4021 const SelectInst *SI1 = dyn_cast<SelectInst>(Val: V1);
4022 if (!SI1)
4023 return false;
4024
4025 if (const SelectInst *SI2 = dyn_cast<SelectInst>(Val: V2)) {
4026 const Value *Cond1 = SI1->getCondition();
4027 const Value *Cond2 = SI2->getCondition();
4028 if (Cond1 == Cond2)
4029 return isKnownNonEqual(V1: SI1->getTrueValue(), V2: SI2->getTrueValue(),
4030 DemandedElts, Q, Depth: Depth + 1) &&
4031 isKnownNonEqual(V1: SI1->getFalseValue(), V2: SI2->getFalseValue(),
4032 DemandedElts, Q, Depth: Depth + 1);
4033 }
4034 return isKnownNonEqual(V1: SI1->getTrueValue(), V2, DemandedElts, Q, Depth: Depth + 1) &&
4035 isKnownNonEqual(V1: SI1->getFalseValue(), V2, DemandedElts, Q, Depth: Depth + 1);
4036}
4037
4038// Check to see if A is both a GEP and is the incoming value for a PHI in the
4039// loop, and B is either a ptr or another GEP. If the PHI has 2 incoming values,
4040// one of them being the recursive GEP A and the other a ptr at same base and at
4041// the same/higher offset than B we are only incrementing the pointer further in
4042// loop if offset of recursive GEP is greater than 0.
4043static bool isNonEqualPointersWithRecursiveGEP(const Value *A, const Value *B,
4044 const SimplifyQuery &Q) {
4045 if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
4046 return false;
4047
4048 auto *GEPA = dyn_cast<GEPOperator>(Val: A);
4049 if (!GEPA || GEPA->getNumIndices() != 1 || !isa<Constant>(Val: GEPA->idx_begin()))
4050 return false;
4051
4052 // Handle 2 incoming PHI values with one being a recursive GEP.
4053 auto *PN = dyn_cast<PHINode>(Val: GEPA->getPointerOperand());
4054 if (!PN || PN->getNumIncomingValues() != 2)
4055 return false;
4056
4057 // Search for the recursive GEP as an incoming operand, and record that as
4058 // Step.
4059 Value *Start = nullptr;
4060 Value *Step = const_cast<Value *>(A);
4061 if (PN->getIncomingValue(i: 0) == Step)
4062 Start = PN->getIncomingValue(i: 1);
4063 else if (PN->getIncomingValue(i: 1) == Step)
4064 Start = PN->getIncomingValue(i: 0);
4065 else
4066 return false;
4067
4068 // Other incoming node base should match the B base.
4069 // StartOffset >= OffsetB && StepOffset > 0?
4070 // StartOffset <= OffsetB && StepOffset < 0?
4071 // Is non-equal if above are true.
4072 // We use stripAndAccumulateInBoundsConstantOffsets to restrict the
4073 // optimisation to inbounds GEPs only.
4074 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: Start->getType());
4075 APInt StartOffset(IndexWidth, 0);
4076 Start = Start->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StartOffset);
4077 APInt StepOffset(IndexWidth, 0);
4078 Step = Step->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StepOffset);
4079
4080 // Check if Base Pointer of Step matches the PHI.
4081 if (Step != PN)
4082 return false;
4083 APInt OffsetB(IndexWidth, 0);
4084 B = B->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: OffsetB);
4085 return Start == B &&
4086 ((StartOffset.sge(RHS: OffsetB) && StepOffset.isStrictlyPositive()) ||
4087 (StartOffset.sle(RHS: OffsetB) && StepOffset.isNegative()));
4088}
4089
4090static bool isKnownNonEqualFromContext(const Value *V1, const Value *V2,
4091 const SimplifyQuery &Q, unsigned Depth) {
4092 if (!Q.CxtI)
4093 return false;
4094
4095 // Try to infer NonEqual based on information from dominating conditions.
4096 if (Q.DC && Q.DT) {
4097 auto IsKnownNonEqualFromDominatingCondition = [&](const Value *V) {
4098 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4099 Value *Cond = BI->getCondition();
4100 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4101 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()) &&
4102 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4103 /*LHSIsTrue=*/true, Depth)
4104 .value_or(u: false))
4105 return true;
4106
4107 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4108 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()) &&
4109 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4110 /*LHSIsTrue=*/false, Depth)
4111 .value_or(u: false))
4112 return true;
4113 }
4114
4115 return false;
4116 };
4117
4118 if (IsKnownNonEqualFromDominatingCondition(V1) ||
4119 IsKnownNonEqualFromDominatingCondition(V2))
4120 return true;
4121 }
4122
4123 if (!Q.AC)
4124 return false;
4125
4126 // Try to infer NonEqual based on information from assumptions.
4127 for (auto &AssumeVH : Q.AC->assumptionsFor(V: V1)) {
4128 if (!AssumeVH)
4129 continue;
4130 CallInst *I = cast<CallInst>(Val&: AssumeVH);
4131
4132 assert(I->getFunction() == Q.CxtI->getFunction() &&
4133 "Got assumption for the wrong function!");
4134 assert(I->getIntrinsicID() == Intrinsic::assume &&
4135 "must be an assume intrinsic");
4136
4137 if (isImpliedCondition(LHS: I->getArgOperand(i: 0), RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4138 /*LHSIsTrue=*/true, Depth)
4139 .value_or(u: false) &&
4140 isValidAssumeForContext(I, Q))
4141 return true;
4142 }
4143
4144 return false;
4145}
4146
4147/// Return true if it is known that V1 != V2.
4148static bool isKnownNonEqual(const Value *V1, const Value *V2,
4149 const APInt &DemandedElts, const SimplifyQuery &Q,
4150 unsigned Depth) {
4151 if (V1 == V2)
4152 return false;
4153 if (V1->getType() != V2->getType())
4154 // We can't look through casts yet.
4155 return false;
4156
4157 if (Depth >= MaxAnalysisRecursionDepth)
4158 return false;
4159
4160 // See if we can recurse through (exactly one of) our operands. This
4161 // requires our operation be 1-to-1 and map every input value to exactly
4162 // one output value. Such an operation is invertible.
4163 auto *O1 = dyn_cast<Operator>(Val: V1);
4164 auto *O2 = dyn_cast<Operator>(Val: V2);
4165 if (O1 && O2 && O1->getOpcode() == O2->getOpcode()) {
4166 if (auto Values = getInvertibleOperands(Op1: O1, Op2: O2))
4167 return isKnownNonEqual(V1: Values->first, V2: Values->second, DemandedElts, Q,
4168 Depth: Depth + 1);
4169
4170 if (const PHINode *PN1 = dyn_cast<PHINode>(Val: V1)) {
4171 const PHINode *PN2 = cast<PHINode>(Val: V2);
4172 // FIXME: This is missing a generalization to handle the case where one is
4173 // a PHI and another one isn't.
4174 if (isNonEqualPHIs(PN1, PN2, DemandedElts, Q, Depth))
4175 return true;
4176 };
4177 }
4178
4179 if (isModifyingBinopOfNonZero(V1, V2, DemandedElts, Q, Depth) ||
4180 isModifyingBinopOfNonZero(V1: V2, V2: V1, DemandedElts, Q, Depth))
4181 return true;
4182
4183 if (isNonEqualMul(V1, V2, DemandedElts, Q, Depth) ||
4184 isNonEqualMul(V1: V2, V2: V1, DemandedElts, Q, Depth))
4185 return true;
4186
4187 if (isNonEqualShl(V1, V2, DemandedElts, Q, Depth) ||
4188 isNonEqualShl(V1: V2, V2: V1, DemandedElts, Q, Depth))
4189 return true;
4190
4191 if (V1->getType()->isIntOrIntVectorTy()) {
4192 // Are any known bits in V1 contradictory to known bits in V2? If V1
4193 // has a known zero where V2 has a known one, they must not be equal.
4194 KnownBits Known1 = computeKnownBits(V: V1, DemandedElts, Q, Depth);
4195 if (!Known1.isUnknown()) {
4196 KnownBits Known2 = computeKnownBits(V: V2, DemandedElts, Q, Depth);
4197 if (Known1.Zero.intersects(RHS: Known2.One) ||
4198 Known2.Zero.intersects(RHS: Known1.One))
4199 return true;
4200 }
4201 }
4202
4203 if (isNonEqualSelect(V1, V2, DemandedElts, Q, Depth) ||
4204 isNonEqualSelect(V1: V2, V2: V1, DemandedElts, Q, Depth))
4205 return true;
4206
4207 if (isNonEqualPointersWithRecursiveGEP(A: V1, B: V2, Q) ||
4208 isNonEqualPointersWithRecursiveGEP(A: V2, B: V1, Q))
4209 return true;
4210
4211 Value *A, *B;
4212 // PtrToInts are NonEqual if their Ptrs are NonEqual.
4213 // Check PtrToInt type matches the pointer size.
4214 if (match(V: V1, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: A))) &&
4215 match(V: V2, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: B))))
4216 return isKnownNonEqual(V1: A, V2: B, DemandedElts, Q, Depth: Depth + 1);
4217
4218 if (isKnownNonEqualFromContext(V1, V2, Q, Depth))
4219 return true;
4220
4221 return false;
4222}
4223
4224/// For vector constants, loop over the elements and find the constant with the
4225/// minimum number of sign bits. Return 0 if the value is not a vector constant
4226/// or if any element was not analyzed; otherwise, return the count for the
4227/// element with the minimum number of sign bits.
4228static unsigned computeNumSignBitsVectorConstant(const Value *V,
4229 const APInt &DemandedElts,
4230 unsigned TyBits) {
4231 const auto *CV = dyn_cast<Constant>(Val: V);
4232 if (!CV || !isa<FixedVectorType>(Val: CV->getType()))
4233 return 0;
4234
4235 unsigned MinSignBits = TyBits;
4236 unsigned NumElts = cast<FixedVectorType>(Val: CV->getType())->getNumElements();
4237 for (unsigned i = 0; i != NumElts; ++i) {
4238 if (!DemandedElts[i])
4239 continue;
4240 // If we find a non-ConstantInt, bail out.
4241 auto *Elt = dyn_cast_or_null<ConstantInt>(Val: CV->getAggregateElement(Elt: i));
4242 if (!Elt)
4243 return 0;
4244
4245 MinSignBits = std::min(a: MinSignBits, b: Elt->getValue().getNumSignBits());
4246 }
4247
4248 return MinSignBits;
4249}
4250
4251static unsigned ComputeNumSignBitsImpl(const Value *V,
4252 const APInt &DemandedElts,
4253 const SimplifyQuery &Q, unsigned Depth);
4254
4255static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
4256 const SimplifyQuery &Q, unsigned Depth) {
4257 unsigned Result = ComputeNumSignBitsImpl(V, DemandedElts, Q, Depth);
4258 assert(Result > 0 && "At least one sign bit needs to be present!");
4259 return Result;
4260}
4261
4262/// Return the number of times the sign bit of the register is replicated into
4263/// the other bits. We know that at least 1 bit is always equal to the sign bit
4264/// (itself), but other cases can give us information. For example, immediately
4265/// after an "ashr X, 2", we know that the top 3 bits are all equal to each
4266/// other, so we return 3. For vectors, return the number of sign bits for the
4267/// vector element with the minimum number of known sign bits of the demanded
4268/// elements in the vector specified by DemandedElts.
4269static unsigned ComputeNumSignBitsImpl(const Value *V,
4270 const APInt &DemandedElts,
4271 const SimplifyQuery &Q, unsigned Depth) {
4272 Type *Ty = V->getType();
4273#ifndef NDEBUG
4274 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
4275
4276 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
4277 assert(
4278 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
4279 "DemandedElt width should equal the fixed vector number of elements");
4280 } else {
4281 assert(DemandedElts == APInt(1, 1) &&
4282 "DemandedElt width should be 1 for scalars");
4283 }
4284#endif
4285
4286 // We return the minimum number of sign bits that are guaranteed to be present
4287 // in V, so for undef we have to conservatively return 1. We don't have the
4288 // same behavior for poison though -- that's a FIXME today.
4289
4290 Type *ScalarTy = Ty->getScalarType();
4291 unsigned TyBits = ScalarTy->isPointerTy() ?
4292 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
4293 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
4294
4295 unsigned Tmp, Tmp2;
4296 unsigned FirstAnswer = 1;
4297
4298 // Note that ConstantInt is handled by the general computeKnownBits case
4299 // below.
4300
4301 if (Depth == MaxAnalysisRecursionDepth)
4302 return 1;
4303
4304 if (auto *U = dyn_cast<Operator>(Val: V)) {
4305 switch (Operator::getOpcode(V)) {
4306 default: break;
4307 case Instruction::BitCast: {
4308 Value *Src = U->getOperand(i: 0);
4309 Type *SrcTy = Src->getType();
4310
4311 // Skip if the source type is not an integer or integer vector type
4312 // This ensures we only process integer-like types
4313 if (!SrcTy->isIntOrIntVectorTy())
4314 break;
4315
4316 unsigned SrcBits = SrcTy->getScalarSizeInBits();
4317
4318 // Bitcast 'large element' scalar/vector to 'small element' vector.
4319 if ((SrcBits % TyBits) != 0)
4320 break;
4321
4322 // Only proceed if the destination type is a fixed-size vector
4323 if (isa<FixedVectorType>(Val: Ty)) {
4324 // Fast case - sign splat can be simply split across the small elements.
4325 // This works for both vector and scalar sources
4326 Tmp = ComputeNumSignBits(V: Src, Q, Depth: Depth + 1);
4327 if (Tmp == SrcBits)
4328 return TyBits;
4329 }
4330 break;
4331 }
4332 case Instruction::SExt:
4333 Tmp = TyBits - U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4334 return ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1) +
4335 Tmp;
4336
4337 case Instruction::SDiv: {
4338 const APInt *Denominator;
4339 // sdiv X, C -> adds log(C) sign bits.
4340 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4341
4342 // Ignore non-positive denominator.
4343 if (!Denominator->isStrictlyPositive())
4344 break;
4345
4346 // Calculate the incoming numerator bits.
4347 unsigned NumBits =
4348 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4349
4350 // Add floor(log(C)) bits to the numerator bits.
4351 return std::min(a: TyBits, b: NumBits + Denominator->logBase2());
4352 }
4353 break;
4354 }
4355
4356 case Instruction::SRem: {
4357 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4358
4359 const APInt *Denominator;
4360 // srem X, C -> we know that the result is within [-C+1,C) when C is a
4361 // positive constant. This let us put a lower bound on the number of sign
4362 // bits.
4363 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4364
4365 // Ignore non-positive denominator.
4366 if (Denominator->isStrictlyPositive()) {
4367 // Calculate the leading sign bit constraints by examining the
4368 // denominator. Given that the denominator is positive, there are two
4369 // cases:
4370 //
4371 // 1. The numerator is positive. The result range is [0,C) and
4372 // [0,C) u< (1 << ceilLogBase2(C)).
4373 //
4374 // 2. The numerator is negative. Then the result range is (-C,0] and
4375 // integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
4376 //
4377 // Thus a lower bound on the number of sign bits is `TyBits -
4378 // ceilLogBase2(C)`.
4379
4380 unsigned ResBits = TyBits - Denominator->ceilLogBase2();
4381 Tmp = std::max(a: Tmp, b: ResBits);
4382 }
4383 }
4384 return Tmp;
4385 }
4386
4387 case Instruction::AShr: {
4388 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4389 // ashr X, C -> adds C sign bits. Vectors too.
4390 const APInt *ShAmt;
4391 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4392 if (ShAmt->uge(RHS: TyBits))
4393 break; // Bad shift.
4394 unsigned ShAmtLimited = ShAmt->getZExtValue();
4395 Tmp += ShAmtLimited;
4396 if (Tmp > TyBits) Tmp = TyBits;
4397 }
4398 return Tmp;
4399 }
4400 case Instruction::Shl: {
4401 const APInt *ShAmt;
4402 Value *X = nullptr;
4403 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4404 // shl destroys sign bits.
4405 if (ShAmt->uge(RHS: TyBits))
4406 break; // Bad shift.
4407 // We can look through a zext (more or less treating it as a sext) if
4408 // all extended bits are shifted out.
4409 if (match(V: U->getOperand(i: 0), P: m_ZExt(Op: m_Value(V&: X))) &&
4410 ShAmt->uge(RHS: TyBits - X->getType()->getScalarSizeInBits())) {
4411 Tmp = ComputeNumSignBits(V: X, DemandedElts, Q, Depth: Depth + 1);
4412 Tmp += TyBits - X->getType()->getScalarSizeInBits();
4413 } else
4414 Tmp =
4415 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4416 if (ShAmt->uge(RHS: Tmp))
4417 break; // Shifted all sign bits out.
4418 Tmp2 = ShAmt->getZExtValue();
4419 return Tmp - Tmp2;
4420 }
4421 break;
4422 }
4423 case Instruction::And:
4424 case Instruction::Or:
4425 case Instruction::Xor: // NOT is handled here.
4426 // Logical binary ops preserve the number of sign bits at the worst.
4427 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4428 if (Tmp != 1) {
4429 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4430 FirstAnswer = std::min(a: Tmp, b: Tmp2);
4431 // We computed what we know about the sign bits as our first
4432 // answer. Now proceed to the generic code that uses
4433 // computeKnownBits, and pick whichever answer is better.
4434 }
4435 break;
4436
4437 case Instruction::Select: {
4438 // If we have a clamp pattern, we know that the number of sign bits will
4439 // be the minimum of the clamp min/max range.
4440 const Value *X;
4441 const APInt *CLow, *CHigh;
4442 if (isSignedMinMaxClamp(Select: U, In&: X, CLow, CHigh))
4443 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4444
4445 Tmp = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4446 if (Tmp == 1)
4447 break;
4448 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 2), DemandedElts, Q, Depth: Depth + 1);
4449 return std::min(a: Tmp, b: Tmp2);
4450 }
4451
4452 case Instruction::Add:
4453 // Add can have at most one carry bit. Thus we know that the output
4454 // is, at worst, one more bit than the inputs.
4455 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4456 if (Tmp == 1) break;
4457
4458 // Special case decrementing a value (ADD X, -1):
4459 if (const auto *CRHS = dyn_cast<Constant>(Val: U->getOperand(i: 1)))
4460 if (CRHS->isAllOnesValue()) {
4461 KnownBits Known(TyBits);
4462 computeKnownBits(V: U->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
4463
4464 // If the input is known to be 0 or 1, the output is 0/-1, which is
4465 // all sign bits set.
4466 if ((Known.Zero | 1).isAllOnes())
4467 return TyBits;
4468
4469 // If we are subtracting one from a positive number, there is no carry
4470 // out of the result.
4471 if (Known.isNonNegative())
4472 return Tmp;
4473 }
4474
4475 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4476 if (Tmp2 == 1)
4477 break;
4478 return std::min(a: Tmp, b: Tmp2) - 1;
4479
4480 case Instruction::Sub:
4481 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4482 if (Tmp2 == 1)
4483 break;
4484
4485 // Handle NEG.
4486 if (const auto *CLHS = dyn_cast<Constant>(Val: U->getOperand(i: 0)))
4487 if (CLHS->isNullValue()) {
4488 KnownBits Known(TyBits);
4489 computeKnownBits(V: U->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
4490 // If the input is known to be 0 or 1, the output is 0/-1, which is
4491 // all sign bits set.
4492 if ((Known.Zero | 1).isAllOnes())
4493 return TyBits;
4494
4495 // If the input is known to be positive (the sign bit is known clear),
4496 // the output of the NEG has the same number of sign bits as the
4497 // input.
4498 if (Known.isNonNegative())
4499 return Tmp2;
4500
4501 // Otherwise, we treat this like a SUB.
4502 }
4503
4504 // Sub can have at most one carry bit. Thus we know that the output
4505 // is, at worst, one more bit than the inputs.
4506 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4507 if (Tmp == 1)
4508 break;
4509 return std::min(a: Tmp, b: Tmp2) - 1;
4510
4511 case Instruction::Mul: {
4512 // The output of the Mul can be at most twice the valid bits in the
4513 // inputs.
4514 unsigned SignBitsOp0 =
4515 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4516 if (SignBitsOp0 == 1)
4517 break;
4518 unsigned SignBitsOp1 =
4519 ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4520 if (SignBitsOp1 == 1)
4521 break;
4522 unsigned OutValidBits =
4523 (TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
4524 return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
4525 }
4526
4527 case Instruction::PHI: {
4528 const PHINode *PN = cast<PHINode>(Val: U);
4529 unsigned NumIncomingValues = PN->getNumIncomingValues();
4530 // Don't analyze large in-degree PHIs.
4531 if (NumIncomingValues > 4) break;
4532 // Unreachable blocks may have zero-operand PHI nodes.
4533 if (NumIncomingValues == 0) break;
4534
4535 // Take the minimum of all incoming values. This can't infinitely loop
4536 // because of our depth threshold.
4537 SimplifyQuery RecQ = Q.getWithoutCondContext();
4538 Tmp = TyBits;
4539 for (unsigned i = 0, e = NumIncomingValues; i != e; ++i) {
4540 if (Tmp == 1) return Tmp;
4541 RecQ.CxtI = PN->getIncomingBlock(i)->getTerminator();
4542 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(V: PN->getIncomingValue(i),
4543 DemandedElts, Q: RecQ, Depth: Depth + 1));
4544 }
4545 return Tmp;
4546 }
4547
4548 case Instruction::Trunc: {
4549 // If the input contained enough sign bits that some remain after the
4550 // truncation, then we can make use of that. Otherwise we don't know
4551 // anything.
4552 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4553 unsigned OperandTyBits = U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4554 if (Tmp > (OperandTyBits - TyBits))
4555 return Tmp - (OperandTyBits - TyBits);
4556
4557 return 1;
4558 }
4559
4560 case Instruction::ExtractElement:
4561 // Look through extract element. At the moment we keep this simple and
4562 // skip tracking the specific element. But at least we might find
4563 // information valid for all elements of the vector (for example if vector
4564 // is sign extended, shifted, etc).
4565 return ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4566
4567 case Instruction::ShuffleVector: {
4568 // Collect the minimum number of sign bits that are shared by every vector
4569 // element referenced by the shuffle.
4570 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: U);
4571 if (!Shuf) {
4572 // FIXME: Add support for shufflevector constant expressions.
4573 return 1;
4574 }
4575 APInt DemandedLHS, DemandedRHS;
4576 // For undef elements, we don't know anything about the common state of
4577 // the shuffle result.
4578 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
4579 return 1;
4580 Tmp = std::numeric_limits<unsigned>::max();
4581 if (!!DemandedLHS) {
4582 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
4583 Tmp = ComputeNumSignBits(V: LHS, DemandedElts: DemandedLHS, Q, Depth: Depth + 1);
4584 }
4585 // If we don't know anything, early out and try computeKnownBits
4586 // fall-back.
4587 if (Tmp == 1)
4588 break;
4589 if (!!DemandedRHS) {
4590 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
4591 Tmp2 = ComputeNumSignBits(V: RHS, DemandedElts: DemandedRHS, Q, Depth: Depth + 1);
4592 Tmp = std::min(a: Tmp, b: Tmp2);
4593 }
4594 // If we don't know anything, early out and try computeKnownBits
4595 // fall-back.
4596 if (Tmp == 1)
4597 break;
4598 assert(Tmp <= TyBits && "Failed to determine minimum sign bits");
4599 return Tmp;
4600 }
4601 case Instruction::Call: {
4602 if (const auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
4603 switch (II->getIntrinsicID()) {
4604 default:
4605 break;
4606 case Intrinsic::abs:
4607 Tmp =
4608 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4609 if (Tmp == 1)
4610 break;
4611
4612 // Absolute value reduces number of sign bits by at most 1.
4613 return Tmp - 1;
4614 case Intrinsic::smin:
4615 case Intrinsic::smax: {
4616 const APInt *CLow, *CHigh;
4617 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
4618 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4619 }
4620 }
4621 }
4622 }
4623 }
4624 }
4625
4626 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4627 // use this information.
4628
4629 // If we can examine all elements of a vector constant successfully, we're
4630 // done (we can't do any better than that). If not, keep trying.
4631 if (unsigned VecSignBits =
4632 computeNumSignBitsVectorConstant(V, DemandedElts, TyBits))
4633 return VecSignBits;
4634
4635 KnownBits Known(TyBits);
4636 computeKnownBits(V, DemandedElts, Known, Q, Depth);
4637
4638 // If we know that the sign bit is either zero or one, determine the number of
4639 // identical bits in the top of the input value.
4640 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
4641}
4642
4643Intrinsic::ID llvm::getIntrinsicForCallSite(const CallBase &CB,
4644 const TargetLibraryInfo *TLI) {
4645 const Function *F = CB.getCalledFunction();
4646 if (!F)
4647 return Intrinsic::not_intrinsic;
4648
4649 if (F->isIntrinsic())
4650 return F->getIntrinsicID();
4651
4652 // We are going to infer semantics of a library function based on mapping it
4653 // to an LLVM intrinsic. Check that the library function is available from
4654 // this callbase and in this environment.
4655 LibFunc Func;
4656 if (F->hasLocalLinkage() || !TLI || !TLI->getLibFunc(CB, F&: Func) ||
4657 !CB.onlyReadsMemory())
4658 return Intrinsic::not_intrinsic;
4659
4660 switch (Func) {
4661 default:
4662 break;
4663 case LibFunc_sin:
4664 case LibFunc_sinf:
4665 case LibFunc_sinl:
4666 return Intrinsic::sin;
4667 case LibFunc_cos:
4668 case LibFunc_cosf:
4669 case LibFunc_cosl:
4670 return Intrinsic::cos;
4671 case LibFunc_tan:
4672 case LibFunc_tanf:
4673 case LibFunc_tanl:
4674 return Intrinsic::tan;
4675 case LibFunc_asin:
4676 case LibFunc_asinf:
4677 case LibFunc_asinl:
4678 return Intrinsic::asin;
4679 case LibFunc_acos:
4680 case LibFunc_acosf:
4681 case LibFunc_acosl:
4682 return Intrinsic::acos;
4683 case LibFunc_atan:
4684 case LibFunc_atanf:
4685 case LibFunc_atanl:
4686 return Intrinsic::atan;
4687 case LibFunc_atan2:
4688 case LibFunc_atan2f:
4689 case LibFunc_atan2l:
4690 return Intrinsic::atan2;
4691 case LibFunc_sinh:
4692 case LibFunc_sinhf:
4693 case LibFunc_sinhl:
4694 return Intrinsic::sinh;
4695 case LibFunc_cosh:
4696 case LibFunc_coshf:
4697 case LibFunc_coshl:
4698 return Intrinsic::cosh;
4699 case LibFunc_tanh:
4700 case LibFunc_tanhf:
4701 case LibFunc_tanhl:
4702 return Intrinsic::tanh;
4703 case LibFunc_exp:
4704 case LibFunc_expf:
4705 case LibFunc_expl:
4706 return Intrinsic::exp;
4707 case LibFunc_exp2:
4708 case LibFunc_exp2f:
4709 case LibFunc_exp2l:
4710 return Intrinsic::exp2;
4711 case LibFunc_exp10:
4712 case LibFunc_exp10f:
4713 case LibFunc_exp10l:
4714 return Intrinsic::exp10;
4715 case LibFunc_log:
4716 case LibFunc_logf:
4717 case LibFunc_logl:
4718 return Intrinsic::log;
4719 case LibFunc_log10:
4720 case LibFunc_log10f:
4721 case LibFunc_log10l:
4722 return Intrinsic::log10;
4723 case LibFunc_log2:
4724 case LibFunc_log2f:
4725 case LibFunc_log2l:
4726 return Intrinsic::log2;
4727 case LibFunc_fabs:
4728 case LibFunc_fabsf:
4729 case LibFunc_fabsl:
4730 return Intrinsic::fabs;
4731 case LibFunc_fmin:
4732 case LibFunc_fminf:
4733 case LibFunc_fminl:
4734 return Intrinsic::minnum;
4735 case LibFunc_fmax:
4736 case LibFunc_fmaxf:
4737 case LibFunc_fmaxl:
4738 return Intrinsic::maxnum;
4739 case LibFunc_copysign:
4740 case LibFunc_copysignf:
4741 case LibFunc_copysignl:
4742 return Intrinsic::copysign;
4743 case LibFunc_floor:
4744 case LibFunc_floorf:
4745 case LibFunc_floorl:
4746 return Intrinsic::floor;
4747 case LibFunc_ceil:
4748 case LibFunc_ceilf:
4749 case LibFunc_ceill:
4750 return Intrinsic::ceil;
4751 case LibFunc_trunc:
4752 case LibFunc_truncf:
4753 case LibFunc_truncl:
4754 return Intrinsic::trunc;
4755 case LibFunc_rint:
4756 case LibFunc_rintf:
4757 case LibFunc_rintl:
4758 return Intrinsic::rint;
4759 case LibFunc_nearbyint:
4760 case LibFunc_nearbyintf:
4761 case LibFunc_nearbyintl:
4762 return Intrinsic::nearbyint;
4763 case LibFunc_round:
4764 case LibFunc_roundf:
4765 case LibFunc_roundl:
4766 return Intrinsic::round;
4767 case LibFunc_roundeven:
4768 case LibFunc_roundevenf:
4769 case LibFunc_roundevenl:
4770 return Intrinsic::roundeven;
4771 case LibFunc_pow:
4772 case LibFunc_powf:
4773 case LibFunc_powl:
4774 return Intrinsic::pow;
4775 case LibFunc_sqrt:
4776 case LibFunc_sqrtf:
4777 case LibFunc_sqrtl:
4778 return Intrinsic::sqrt;
4779 }
4780
4781 return Intrinsic::not_intrinsic;
4782}
4783
4784/// Given an exploded icmp instruction, return true if the comparison only
4785/// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if
4786/// the result of the comparison is true when the input value is signed.
4787bool llvm::isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
4788 bool &TrueIfSigned) {
4789 switch (Pred) {
4790 case ICmpInst::ICMP_SLT: // True if LHS s< 0
4791 TrueIfSigned = true;
4792 return RHS.isZero();
4793 case ICmpInst::ICMP_SLE: // True if LHS s<= -1
4794 TrueIfSigned = true;
4795 return RHS.isAllOnes();
4796 case ICmpInst::ICMP_SGT: // True if LHS s> -1
4797 TrueIfSigned = false;
4798 return RHS.isAllOnes();
4799 case ICmpInst::ICMP_SGE: // True if LHS s>= 0
4800 TrueIfSigned = false;
4801 return RHS.isZero();
4802 case ICmpInst::ICMP_UGT:
4803 // True if LHS u> RHS and RHS == sign-bit-mask - 1
4804 TrueIfSigned = true;
4805 return RHS.isMaxSignedValue();
4806 case ICmpInst::ICMP_UGE:
4807 // True if LHS u>= RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
4808 TrueIfSigned = true;
4809 return RHS.isMinSignedValue();
4810 case ICmpInst::ICMP_ULT:
4811 // True if LHS u< RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
4812 TrueIfSigned = false;
4813 return RHS.isMinSignedValue();
4814 case ICmpInst::ICMP_ULE:
4815 // True if LHS u<= RHS and RHS == sign-bit-mask - 1
4816 TrueIfSigned = false;
4817 return RHS.isMaxSignedValue();
4818 default:
4819 return false;
4820 }
4821}
4822
4823static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
4824 bool CondIsTrue,
4825 const Instruction *CxtI,
4826 KnownFPClass &KnownFromContext,
4827 unsigned Depth = 0) {
4828 Value *A, *B;
4829 if (Depth < MaxAnalysisRecursionDepth &&
4830 (CondIsTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B)))
4831 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))))) {
4832 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue, CxtI, KnownFromContext,
4833 Depth: Depth + 1);
4834 computeKnownFPClassFromCond(V, Cond: B, CondIsTrue, CxtI, KnownFromContext,
4835 Depth: Depth + 1);
4836 return;
4837 }
4838 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A)))) {
4839 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue: !CondIsTrue, CxtI, KnownFromContext,
4840 Depth: Depth + 1);
4841 return;
4842 }
4843 CmpPredicate Pred;
4844 Value *LHS;
4845 uint64_t ClassVal = 0;
4846 const APFloat *CRHS;
4847 const APInt *RHS;
4848 if (match(V: Cond, P: m_FCmp(Pred, L: m_Value(V&: LHS), R: m_APFloat(Res&: CRHS)))) {
4849 auto [CmpVal, MaskIfTrue, MaskIfFalse] = fcmpImpliesClass(
4850 Pred, F: *cast<Instruction>(Val: Cond)->getParent()->getParent(), LHS, ConstRHS: *CRHS,
4851 LookThroughSrc: LHS != V);
4852 if (CmpVal == V)
4853 KnownFromContext.knownNot(RuleOut: ~(CondIsTrue ? MaskIfTrue : MaskIfFalse));
4854 } else if (match(V: Cond, P: m_Intrinsic<Intrinsic::is_fpclass>(
4855 Op0: m_Specific(V), Op1: m_ConstantInt(V&: ClassVal)))) {
4856 FPClassTest Mask = static_cast<FPClassTest>(ClassVal);
4857 KnownFromContext.knownNot(RuleOut: CondIsTrue ? ~Mask : Mask);
4858 } else if (match(V: Cond, P: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Specific(V)),
4859 R: m_APInt(Res&: RHS)))) {
4860 bool TrueIfSigned;
4861 if (!isSignBitCheck(Pred, RHS: *RHS, TrueIfSigned))
4862 return;
4863 if (TrueIfSigned == CondIsTrue)
4864 KnownFromContext.signBitMustBeOne();
4865 else
4866 KnownFromContext.signBitMustBeZero();
4867 }
4868}
4869
4870static KnownFPClass computeKnownFPClassFromContext(const Value *V,
4871 const SimplifyQuery &Q) {
4872 KnownFPClass KnownFromContext;
4873
4874 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
4875 computeKnownFPClassFromCond(V, Cond: Q.CC->Cond, CondIsTrue: !Q.CC->Invert, CxtI: Q.CxtI,
4876 KnownFromContext);
4877
4878 if (!Q.CxtI)
4879 return KnownFromContext;
4880
4881 if (Q.DC && Q.DT) {
4882 // Handle dominating conditions.
4883 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4884 Value *Cond = BI->getCondition();
4885
4886 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4887 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
4888 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, CxtI: Q.CxtI,
4889 KnownFromContext);
4890
4891 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4892 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
4893 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, CxtI: Q.CxtI,
4894 KnownFromContext);
4895 }
4896 }
4897
4898 if (!Q.AC)
4899 return KnownFromContext;
4900
4901 // Try to restrict the floating-point classes based on information from
4902 // assumptions.
4903 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
4904 if (!AssumeVH)
4905 continue;
4906 CallInst *I = cast<CallInst>(Val&: AssumeVH);
4907
4908 assert(I->getFunction() == Q.CxtI->getParent()->getParent() &&
4909 "Got assumption for the wrong function!");
4910 assert(I->getIntrinsicID() == Intrinsic::assume &&
4911 "must be an assume intrinsic");
4912
4913 if (!isValidAssumeForContext(I, Q))
4914 continue;
4915
4916 computeKnownFPClassFromCond(V, Cond: I->getArgOperand(i: 0),
4917 /*CondIsTrue=*/true, CxtI: Q.CxtI, KnownFromContext);
4918 }
4919
4920 return KnownFromContext;
4921}
4922
4923void llvm::adjustKnownFPClassForSelectArm(KnownFPClass &Known, Value *Cond,
4924 Value *Arm, bool Invert,
4925 const SimplifyQuery &SQ,
4926 unsigned Depth) {
4927
4928 KnownFPClass KnownSrc;
4929 computeKnownFPClassFromCond(V: Arm, Cond,
4930 /*CondIsTrue=*/!Invert, CxtI: SQ.CxtI, KnownFromContext&: KnownSrc,
4931 Depth: Depth + 1);
4932 KnownSrc = KnownSrc.unionWith(RHS: Known);
4933 if (KnownSrc.isUnknown())
4934 return;
4935
4936 if (isGuaranteedNotToBeUndef(V: Arm, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT, Depth: Depth + 1))
4937 Known = KnownSrc;
4938}
4939
4940void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
4941 FPClassTest InterestedClasses, KnownFPClass &Known,
4942 const SimplifyQuery &Q, unsigned Depth);
4943
4944static void computeKnownFPClass(const Value *V, KnownFPClass &Known,
4945 FPClassTest InterestedClasses,
4946 const SimplifyQuery &Q, unsigned Depth) {
4947 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
4948 APInt DemandedElts =
4949 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
4950 computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Q, Depth);
4951}
4952
4953static void computeKnownFPClassForFPTrunc(const Operator *Op,
4954 const APInt &DemandedElts,
4955 FPClassTest InterestedClasses,
4956 KnownFPClass &Known,
4957 const SimplifyQuery &Q,
4958 unsigned Depth) {
4959 if ((InterestedClasses &
4960 (KnownFPClass::OrderedLessThanZeroMask | fcNan)) == fcNone)
4961 return;
4962
4963 KnownFPClass KnownSrc;
4964 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
4965 Known&: KnownSrc, Q, Depth: Depth + 1);
4966 Known = KnownFPClass::fptrunc(KnownSrc);
4967}
4968
4969static constexpr KnownFPClass::MinMaxKind getMinMaxKind(Intrinsic::ID IID) {
4970 switch (IID) {
4971 case Intrinsic::minimum:
4972 return KnownFPClass::MinMaxKind::minimum;
4973 case Intrinsic::maximum:
4974 return KnownFPClass::MinMaxKind::maximum;
4975 case Intrinsic::minimumnum:
4976 return KnownFPClass::MinMaxKind::minimumnum;
4977 case Intrinsic::maximumnum:
4978 return KnownFPClass::MinMaxKind::maximumnum;
4979 case Intrinsic::minnum:
4980 return KnownFPClass::MinMaxKind::minnum;
4981 case Intrinsic::maxnum:
4982 return KnownFPClass::MinMaxKind::maxnum;
4983 default:
4984 llvm_unreachable("not a floating-point min-max intrinsic");
4985 }
4986}
4987
4988/// \return true if this is a floating point value that is known to have a
4989/// magnitude smaller than 1. i.e., fabs(X) <= 1.0 or is nan.
4990static bool isAbsoluteValueULEOne(const Value *V) {
4991 // TODO: Handle frexp
4992 // TODO: Other rounding intrinsics?
4993
4994 // fabs(x - floor(x)) <= 1
4995 const Value *SubFloorX;
4996 if (match(V, P: m_FSub(L: m_Value(V&: SubFloorX),
4997 R: m_Intrinsic<Intrinsic::floor>(Op0: m_Deferred(V: SubFloorX)))))
4998 return true;
4999
5000 return match(V, P: m_Intrinsic<Intrinsic::amdgcn_trig_preop>(Op0: m_Value())) ||
5001 match(V, P: m_Intrinsic<Intrinsic::amdgcn_fract>(Op0: m_Value()));
5002}
5003
5004void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5005 FPClassTest InterestedClasses, KnownFPClass &Known,
5006 const SimplifyQuery &Q, unsigned Depth) {
5007 assert(Known.isUnknown() && "should not be called with known information");
5008
5009 if (!DemandedElts) {
5010 // No demanded elts, better to assume we don't know anything.
5011 Known.resetAll();
5012 return;
5013 }
5014
5015 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
5016
5017 if (auto *CFP = dyn_cast<ConstantFP>(Val: V)) {
5018 Known = KnownFPClass(CFP->getValueAPF());
5019 return;
5020 }
5021
5022 if (isa<ConstantAggregateZero>(Val: V)) {
5023 Known.KnownFPClasses = fcPosZero;
5024 Known.SignBit = false;
5025 return;
5026 }
5027
5028 if (isa<PoisonValue>(Val: V)) {
5029 Known.KnownFPClasses = fcNone;
5030 Known.SignBit = false;
5031 return;
5032 }
5033
5034 // Try to handle fixed width vector constants
5035 auto *VFVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5036 const Constant *CV = dyn_cast<Constant>(Val: V);
5037 if (VFVTy && CV) {
5038 Known.KnownFPClasses = fcNone;
5039 bool SignBitAllZero = true;
5040 bool SignBitAllOne = true;
5041
5042 // For vectors, verify that each element is not NaN.
5043 unsigned NumElts = VFVTy->getNumElements();
5044 for (unsigned i = 0; i != NumElts; ++i) {
5045 if (!DemandedElts[i])
5046 continue;
5047
5048 Constant *Elt = CV->getAggregateElement(Elt: i);
5049 if (!Elt) {
5050 Known = KnownFPClass();
5051 return;
5052 }
5053 if (isa<PoisonValue>(Val: Elt))
5054 continue;
5055 auto *CElt = dyn_cast<ConstantFP>(Val: Elt);
5056 if (!CElt) {
5057 Known = KnownFPClass();
5058 return;
5059 }
5060
5061 const APFloat &C = CElt->getValueAPF();
5062 Known.KnownFPClasses |= C.classify();
5063 if (C.isNegative())
5064 SignBitAllZero = false;
5065 else
5066 SignBitAllOne = false;
5067 }
5068 if (SignBitAllOne != SignBitAllZero)
5069 Known.SignBit = SignBitAllOne;
5070 return;
5071 }
5072
5073 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) {
5074 Known.KnownFPClasses = fcNone;
5075 for (size_t I = 0, E = CDS->getNumElements(); I != E; ++I)
5076 Known |= CDS->getElementAsAPFloat(i: I).classify();
5077 return;
5078 }
5079
5080 if (const auto *CA = dyn_cast<ConstantAggregate>(Val: V)) {
5081 // TODO: Handle complex aggregates
5082 Known.KnownFPClasses = fcNone;
5083 for (const Use &Op : CA->operands()) {
5084 auto *CFP = dyn_cast<ConstantFP>(Val: Op.get());
5085 if (!CFP) {
5086 Known = KnownFPClass();
5087 return;
5088 }
5089
5090 Known |= CFP->getValueAPF().classify();
5091 }
5092
5093 return;
5094 }
5095
5096 FPClassTest KnownNotFromFlags = fcNone;
5097 if (const auto *CB = dyn_cast<CallBase>(Val: V))
5098 KnownNotFromFlags |= CB->getRetNoFPClass();
5099 else if (const auto *Arg = dyn_cast<Argument>(Val: V))
5100 KnownNotFromFlags |= Arg->getNoFPClass();
5101
5102 const Operator *Op = dyn_cast<Operator>(Val: V);
5103 if (const FPMathOperator *FPOp = dyn_cast_or_null<FPMathOperator>(Val: Op)) {
5104 if (FPOp->hasNoNaNs())
5105 KnownNotFromFlags |= fcNan;
5106 if (FPOp->hasNoInfs())
5107 KnownNotFromFlags |= fcInf;
5108 }
5109
5110 KnownFPClass AssumedClasses = computeKnownFPClassFromContext(V, Q);
5111 KnownNotFromFlags |= ~AssumedClasses.KnownFPClasses;
5112
5113 // We no longer need to find out about these bits from inputs if we can
5114 // assume this from flags/attributes.
5115 InterestedClasses &= ~KnownNotFromFlags;
5116
5117 llvm::scope_exit ClearClassesFromFlags([=, &Known] {
5118 Known.knownNot(RuleOut: KnownNotFromFlags);
5119 if (!Known.SignBit && AssumedClasses.SignBit) {
5120 if (*AssumedClasses.SignBit)
5121 Known.signBitMustBeOne();
5122 else
5123 Known.signBitMustBeZero();
5124 }
5125 });
5126
5127 if (!Op)
5128 return;
5129
5130 // All recursive calls that increase depth must come after this.
5131 if (Depth == MaxAnalysisRecursionDepth)
5132 return;
5133
5134 const unsigned Opc = Op->getOpcode();
5135 switch (Opc) {
5136 case Instruction::FNeg: {
5137 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5138 Known, Q, Depth: Depth + 1);
5139 Known.fneg();
5140 break;
5141 }
5142 case Instruction::Select: {
5143 auto ComputeForArm = [&](Value *Arm, bool Invert) {
5144 KnownFPClass Res;
5145 computeKnownFPClass(V: Arm, DemandedElts, InterestedClasses, Known&: Res, Q,
5146 Depth: Depth + 1);
5147 adjustKnownFPClassForSelectArm(Known&: Res, Cond: Op->getOperand(i: 0), Arm, Invert, SQ: Q,
5148 Depth);
5149 return Res;
5150 };
5151 // Only known if known in both the LHS and RHS.
5152 Known =
5153 ComputeForArm(Op->getOperand(i: 1), /*Invert=*/false)
5154 .intersectWith(RHS: ComputeForArm(Op->getOperand(i: 2), /*Invert=*/true));
5155 break;
5156 }
5157 case Instruction::Load: {
5158 const MDNode *NoFPClass =
5159 cast<LoadInst>(Val: Op)->getMetadata(KindID: LLVMContext::MD_nofpclass);
5160 if (!NoFPClass)
5161 break;
5162
5163 ConstantInt *MaskVal =
5164 mdconst::extract<ConstantInt>(MD: NoFPClass->getOperand(I: 0));
5165 Known.knownNot(RuleOut: static_cast<FPClassTest>(MaskVal->getZExtValue()));
5166 break;
5167 }
5168 case Instruction::Call: {
5169 const CallInst *II = cast<CallInst>(Val: Op);
5170 const Intrinsic::ID IID = II->getIntrinsicID();
5171 switch (IID) {
5172 case Intrinsic::fabs: {
5173 if ((InterestedClasses & (fcNan | fcPositive)) != fcNone) {
5174 // If we only care about the sign bit we don't need to inspect the
5175 // operand.
5176 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5177 InterestedClasses, Known, Q, Depth: Depth + 1);
5178 }
5179
5180 Known.fabs();
5181 break;
5182 }
5183 case Intrinsic::copysign: {
5184 KnownFPClass KnownSign;
5185
5186 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5187 Known, Q, Depth: Depth + 1);
5188 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5189 Known&: KnownSign, Q, Depth: Depth + 1);
5190 Known.copysign(Sign: KnownSign);
5191 break;
5192 }
5193 case Intrinsic::fma:
5194 case Intrinsic::fmuladd: {
5195 if ((InterestedClasses & fcNegative) == fcNone)
5196 break;
5197
5198 // FIXME: This should check isGuaranteedNotToBeUndef
5199 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) {
5200 KnownFPClass KnownSrc, KnownAddend;
5201 computeKnownFPClass(V: II->getArgOperand(i: 2), DemandedElts,
5202 InterestedClasses, Known&: KnownAddend, Q, Depth: Depth + 1);
5203 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5204 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5205
5206 const Function *F = II->getFunction();
5207 const fltSemantics &FltSem =
5208 II->getType()->getScalarType()->getFltSemantics();
5209 DenormalMode Mode =
5210 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5211
5212 if (KnownNotFromFlags & fcNan) {
5213 KnownSrc.knownNot(RuleOut: fcNan);
5214 KnownAddend.knownNot(RuleOut: fcNan);
5215 }
5216
5217 if (KnownNotFromFlags & fcInf) {
5218 KnownSrc.knownNot(RuleOut: fcInf);
5219 KnownAddend.knownNot(RuleOut: fcInf);
5220 }
5221
5222 Known = KnownFPClass::fma_square(Squared: KnownSrc, Addend: KnownAddend, Mode);
5223 break;
5224 }
5225
5226 KnownFPClass KnownSrc[3];
5227 for (int I = 0; I != 3; ++I) {
5228 computeKnownFPClass(V: II->getArgOperand(i: I), DemandedElts,
5229 InterestedClasses, Known&: KnownSrc[I], Q, Depth: Depth + 1);
5230 if (KnownSrc[I].isUnknown())
5231 return;
5232
5233 if (KnownNotFromFlags & fcNan)
5234 KnownSrc[I].knownNot(RuleOut: fcNan);
5235 if (KnownNotFromFlags & fcInf)
5236 KnownSrc[I].knownNot(RuleOut: fcInf);
5237 }
5238
5239 const Function *F = II->getFunction();
5240 const fltSemantics &FltSem =
5241 II->getType()->getScalarType()->getFltSemantics();
5242 DenormalMode Mode =
5243 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5244 Known = KnownFPClass::fma(LHS: KnownSrc[0], RHS: KnownSrc[1], Addend: KnownSrc[2], Mode);
5245 break;
5246 }
5247 case Intrinsic::sqrt:
5248 case Intrinsic::experimental_constrained_sqrt: {
5249 KnownFPClass KnownSrc;
5250 FPClassTest InterestedSrcs = InterestedClasses;
5251 if (InterestedClasses & fcNan)
5252 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5253
5254 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5255 Known&: KnownSrc, Q, Depth: Depth + 1);
5256
5257 DenormalMode Mode = DenormalMode::getDynamic();
5258
5259 bool HasNSZ = Q.IIQ.hasNoSignedZeros(Op: II);
5260 if (!HasNSZ) {
5261 const Function *F = II->getFunction();
5262 const fltSemantics &FltSem =
5263 II->getType()->getScalarType()->getFltSemantics();
5264 Mode = F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5265 }
5266
5267 Known = KnownFPClass::sqrt(Src: KnownSrc, Mode);
5268 if (HasNSZ)
5269 Known.knownNot(RuleOut: fcNegZero);
5270
5271 break;
5272 }
5273 case Intrinsic::sin: {
5274 KnownFPClass KnownSrc;
5275 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5276 Known&: KnownSrc, Q, Depth: Depth + 1);
5277 Known = KnownFPClass::sin(Src: KnownSrc);
5278 break;
5279 }
5280 case Intrinsic::cos: {
5281 KnownFPClass KnownSrc;
5282 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5283 Known&: KnownSrc, Q, Depth: Depth + 1);
5284 Known = KnownFPClass::cos(Src: KnownSrc);
5285 break;
5286 }
5287 case Intrinsic::tan: {
5288 KnownFPClass KnownSrc;
5289 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5290 Known&: KnownSrc, Q, Depth: Depth + 1);
5291 Known = KnownFPClass::tan(Src: KnownSrc);
5292 break;
5293 }
5294 case Intrinsic::sinh: {
5295 KnownFPClass KnownSrc;
5296 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5297 Known&: KnownSrc, Q, Depth: Depth + 1);
5298 Known = KnownFPClass::sinh(Src: KnownSrc);
5299 break;
5300 }
5301 case Intrinsic::cosh: {
5302 KnownFPClass KnownSrc;
5303 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5304 Known&: KnownSrc, Q, Depth: Depth + 1);
5305 Known = KnownFPClass::cosh(Src: KnownSrc);
5306 break;
5307 }
5308 case Intrinsic::tanh: {
5309 KnownFPClass KnownSrc;
5310 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5311 Known&: KnownSrc, Q, Depth: Depth + 1);
5312 Known = KnownFPClass::tanh(Src: KnownSrc);
5313 break;
5314 }
5315 case Intrinsic::asin: {
5316 KnownFPClass KnownSrc;
5317 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5318 Known&: KnownSrc, Q, Depth: Depth + 1);
5319 Known = KnownFPClass::asin(Src: KnownSrc);
5320 break;
5321 }
5322 case Intrinsic::acos: {
5323 KnownFPClass KnownSrc;
5324 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5325 Known&: KnownSrc, Q, Depth: Depth + 1);
5326 Known = KnownFPClass::acos(Src: KnownSrc);
5327 break;
5328 }
5329 case Intrinsic::atan: {
5330 KnownFPClass KnownSrc;
5331 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5332 Known&: KnownSrc, Q, Depth: Depth + 1);
5333 Known = KnownFPClass::atan(Src: KnownSrc);
5334 break;
5335 }
5336 case Intrinsic::atan2: {
5337 KnownFPClass KnownLHS, KnownRHS;
5338 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5339 Known&: KnownLHS, Q, Depth: Depth + 1);
5340 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5341 Known&: KnownRHS, Q, Depth: Depth + 1);
5342 Known = KnownFPClass::atan2(LHS: KnownLHS, RHS: KnownRHS);
5343 break;
5344 }
5345 case Intrinsic::maxnum:
5346 case Intrinsic::minnum:
5347 case Intrinsic::minimum:
5348 case Intrinsic::maximum:
5349 case Intrinsic::minimumnum:
5350 case Intrinsic::maximumnum: {
5351 KnownFPClass KnownLHS, KnownRHS;
5352 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5353 Known&: KnownLHS, Q, Depth: Depth + 1);
5354 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5355 Known&: KnownRHS, Q, Depth: Depth + 1);
5356
5357 const Function *F = II->getFunction();
5358
5359 DenormalMode Mode =
5360 F ? F->getDenormalMode(
5361 FPType: II->getType()->getScalarType()->getFltSemantics())
5362 : DenormalMode::getDynamic();
5363
5364 Known = KnownFPClass::minMaxLike(LHS: KnownLHS, RHS: KnownRHS, Kind: getMinMaxKind(IID),
5365 DenormMode: Mode);
5366 break;
5367 }
5368 case Intrinsic::canonicalize: {
5369 KnownFPClass KnownSrc;
5370 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5371 Known&: KnownSrc, Q, Depth: Depth + 1);
5372
5373 const Function *F = II->getFunction();
5374 DenormalMode DenormMode =
5375 F ? F->getDenormalMode(
5376 FPType: II->getType()->getScalarType()->getFltSemantics())
5377 : DenormalMode::getDynamic();
5378 Known = KnownFPClass::canonicalize(Src: KnownSrc, DenormMode);
5379 break;
5380 }
5381 case Intrinsic::vector_reduce_fmax:
5382 case Intrinsic::vector_reduce_fmin:
5383 case Intrinsic::vector_reduce_fmaximum:
5384 case Intrinsic::vector_reduce_fminimum: {
5385 // reduce min/max will choose an element from one of the vector elements,
5386 // so we can infer and class information that is common to all elements.
5387 Known = computeKnownFPClass(V: II->getArgOperand(i: 0), FMF: II->getFastMathFlags(),
5388 InterestedClasses, SQ: Q, Depth: Depth + 1);
5389 // Can only propagate sign if output is never NaN.
5390 if (!Known.isKnownNeverNaN())
5391 Known.SignBit.reset();
5392 break;
5393 }
5394 // reverse preserves all characteristics of the input vec's element.
5395 case Intrinsic::vector_reverse:
5396 Known = computeKnownFPClass(
5397 V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
5398 FMF: II->getFastMathFlags(), InterestedClasses, SQ: Q, Depth: Depth + 1);
5399 break;
5400 case Intrinsic::trunc:
5401 case Intrinsic::floor:
5402 case Intrinsic::ceil:
5403 case Intrinsic::rint:
5404 case Intrinsic::nearbyint:
5405 case Intrinsic::round:
5406 case Intrinsic::roundeven: {
5407 KnownFPClass KnownSrc;
5408 FPClassTest InterestedSrcs = InterestedClasses;
5409 if (InterestedSrcs & fcPosFinite)
5410 InterestedSrcs |= fcPosFinite;
5411 if (InterestedSrcs & fcNegFinite)
5412 InterestedSrcs |= fcNegFinite;
5413 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5414 Known&: KnownSrc, Q, Depth: Depth + 1);
5415
5416 Known = KnownFPClass::roundToIntegral(
5417 Src: KnownSrc, IsTrunc: IID == Intrinsic::trunc,
5418 IsMultiUnitFPType: V->getType()->getScalarType()->isMultiUnitFPType());
5419 break;
5420 }
5421 case Intrinsic::exp:
5422 case Intrinsic::exp2:
5423 case Intrinsic::exp10:
5424 case Intrinsic::amdgcn_exp2: {
5425 KnownFPClass KnownSrc;
5426 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5427 Known&: KnownSrc, Q, Depth: Depth + 1);
5428
5429 Known = KnownFPClass::exp(Src: KnownSrc);
5430
5431 Type *EltTy = II->getType()->getScalarType();
5432 if (IID == Intrinsic::amdgcn_exp2 && EltTy->isFloatTy())
5433 Known.knownNot(RuleOut: fcSubnormal);
5434
5435 break;
5436 }
5437 case Intrinsic::fptrunc_round: {
5438 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
5439 Q, Depth);
5440 break;
5441 }
5442 case Intrinsic::log:
5443 case Intrinsic::log10:
5444 case Intrinsic::log2:
5445 case Intrinsic::experimental_constrained_log:
5446 case Intrinsic::experimental_constrained_log10:
5447 case Intrinsic::experimental_constrained_log2:
5448 case Intrinsic::amdgcn_log: {
5449 Type *EltTy = II->getType()->getScalarType();
5450
5451 // log(+inf) -> +inf
5452 // log([+-]0.0) -> -inf
5453 // log(-inf) -> nan
5454 // log(-x) -> nan
5455 if ((InterestedClasses & (fcNan | fcInf)) != fcNone) {
5456 FPClassTest InterestedSrcs = InterestedClasses;
5457 if ((InterestedClasses & fcNegInf) != fcNone)
5458 InterestedSrcs |= fcZero | fcSubnormal;
5459 if ((InterestedClasses & fcNan) != fcNone)
5460 InterestedSrcs |= fcNan | fcNegative;
5461
5462 KnownFPClass KnownSrc;
5463 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5464 Known&: KnownSrc, Q, Depth: Depth + 1);
5465
5466 const Function *F = II->getFunction();
5467 DenormalMode Mode = F ? F->getDenormalMode(FPType: EltTy->getFltSemantics())
5468 : DenormalMode::getDynamic();
5469 Known = KnownFPClass::log(Src: KnownSrc, Mode);
5470 }
5471
5472 break;
5473 }
5474 case Intrinsic::powi: {
5475 if ((InterestedClasses & (fcNan | fcInf | fcNegative)) == fcNone)
5476 break;
5477
5478 const Value *Exp = II->getArgOperand(i: 1);
5479 Type *ExpTy = Exp->getType();
5480 unsigned BitWidth = ExpTy->getScalarType()->getIntegerBitWidth();
5481 KnownBits ExponentKnownBits(BitWidth);
5482 computeKnownBits(V: Exp, DemandedElts: isa<VectorType>(Val: ExpTy) ? DemandedElts : APInt(1, 1),
5483 Known&: ExponentKnownBits, Q, Depth: Depth + 1);
5484
5485 FPClassTest InterestedSrcs = fcNone;
5486 if (InterestedClasses & fcNan)
5487 InterestedSrcs |= fcNan;
5488 if (!ExponentKnownBits.isZero()) {
5489 if (InterestedClasses & fcInf)
5490 InterestedSrcs |= fcFinite | fcInf;
5491 if ((InterestedClasses & fcNegative) && !ExponentKnownBits.isEven())
5492 InterestedSrcs |= fcNegative;
5493 }
5494
5495 KnownFPClass KnownSrc;
5496 if (InterestedSrcs != fcNone)
5497 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5498 Known&: KnownSrc, Q, Depth: Depth + 1);
5499
5500 Known = KnownFPClass::powi(Src: KnownSrc, N: ExponentKnownBits);
5501 break;
5502 }
5503 case Intrinsic::ldexp: {
5504 KnownFPClass KnownSrc;
5505 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5506 Known&: KnownSrc, Q, Depth: Depth + 1);
5507 // Can refine inf/zero handling based on the exponent operand.
5508 const FPClassTest ExpInfoMask = fcZero | fcSubnormal | fcInf;
5509
5510 KnownBits ExpBits;
5511 if ((KnownSrc.KnownFPClasses & ExpInfoMask) != fcNone) {
5512 const Value *ExpArg = II->getArgOperand(i: 1);
5513 ExpBits = computeKnownBits(V: ExpArg, DemandedElts, Q, Depth: Depth + 1);
5514 }
5515
5516 const fltSemantics &Flt =
5517 II->getType()->getScalarType()->getFltSemantics();
5518
5519 const Function *F = II->getFunction();
5520 DenormalMode Mode =
5521 F ? F->getDenormalMode(FPType: Flt) : DenormalMode::getDynamic();
5522
5523 Known = KnownFPClass::ldexp(Src: KnownSrc, N: ExpBits, Flt, Mode);
5524 break;
5525 }
5526 case Intrinsic::arithmetic_fence: {
5527 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5528 Known, Q, Depth: Depth + 1);
5529 break;
5530 }
5531 case Intrinsic::experimental_constrained_sitofp:
5532 case Intrinsic::experimental_constrained_uitofp:
5533 // Cannot produce nan
5534 Known.knownNot(RuleOut: fcNan);
5535
5536 // sitofp and uitofp turn into +0.0 for zero.
5537 Known.knownNot(RuleOut: fcNegZero);
5538
5539 // Integers cannot be subnormal
5540 Known.knownNot(RuleOut: fcSubnormal);
5541
5542 if (IID == Intrinsic::experimental_constrained_uitofp)
5543 Known.signBitMustBeZero();
5544
5545 // TODO: Copy inf handling from instructions
5546 break;
5547
5548 case Intrinsic::amdgcn_fract: {
5549 Known.knownNot(RuleOut: fcInf);
5550
5551 if (InterestedClasses & fcNan) {
5552 KnownFPClass KnownSrc;
5553 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5554 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5555
5556 if (KnownSrc.isKnownNeverInfOrNaN())
5557 Known.knownNot(RuleOut: fcNan);
5558 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5559 Known.knownNot(RuleOut: fcSNan);
5560 }
5561
5562 break;
5563 }
5564 case Intrinsic::amdgcn_rcp: {
5565 KnownFPClass KnownSrc;
5566 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5567 Known&: KnownSrc, Q, Depth: Depth + 1);
5568
5569 Known.propagateNaN(Src: KnownSrc);
5570
5571 Type *EltTy = II->getType()->getScalarType();
5572
5573 // f32 denormal always flushed.
5574 if (EltTy->isFloatTy()) {
5575 Known.knownNot(RuleOut: fcSubnormal);
5576 KnownSrc.knownNot(RuleOut: fcSubnormal);
5577 }
5578
5579 if (KnownSrc.isKnownNever(Mask: fcNegative))
5580 Known.knownNot(RuleOut: fcNegative);
5581 if (KnownSrc.isKnownNever(Mask: fcPositive))
5582 Known.knownNot(RuleOut: fcPositive);
5583
5584 if (const Function *F = II->getFunction()) {
5585 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5586 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5587 Known.knownNot(RuleOut: fcPosInf);
5588 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5589 Known.knownNot(RuleOut: fcNegInf);
5590 }
5591
5592 break;
5593 }
5594 case Intrinsic::amdgcn_rsq: {
5595 KnownFPClass KnownSrc;
5596 // The only negative value that can be returned is -inf for -0 inputs.
5597 Known.knownNot(RuleOut: fcNegZero | fcNegSubnormal | fcNegNormal);
5598
5599 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5600 Known&: KnownSrc, Q, Depth: Depth + 1);
5601
5602 // Negative -> nan
5603 if (KnownSrc.isKnownNeverNaN() && KnownSrc.cannotBeOrderedLessThanZero())
5604 Known.knownNot(RuleOut: fcNan);
5605 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5606 Known.knownNot(RuleOut: fcSNan);
5607
5608 // +inf -> +0
5609 if (KnownSrc.isKnownNeverPosInfinity())
5610 Known.knownNot(RuleOut: fcPosZero);
5611
5612 Type *EltTy = II->getType()->getScalarType();
5613
5614 // f32 denormal always flushed.
5615 if (EltTy->isFloatTy())
5616 Known.knownNot(RuleOut: fcPosSubnormal);
5617
5618 if (const Function *F = II->getFunction()) {
5619 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5620
5621 // -0 -> -inf
5622 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5623 Known.knownNot(RuleOut: fcNegInf);
5624
5625 // +0 -> +inf
5626 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5627 Known.knownNot(RuleOut: fcPosInf);
5628 }
5629
5630 break;
5631 }
5632 case Intrinsic::amdgcn_trig_preop: {
5633 // Always returns a value [0, 1)
5634 Known.knownNot(RuleOut: fcNan | fcInf | fcNegative);
5635 break;
5636 }
5637 default:
5638 break;
5639 }
5640
5641 break;
5642 }
5643 case Instruction::FAdd:
5644 case Instruction::FSub: {
5645 KnownFPClass KnownLHS, KnownRHS;
5646 bool WantNegative =
5647 Op->getOpcode() == Instruction::FAdd &&
5648 (InterestedClasses & KnownFPClass::OrderedLessThanZeroMask) != fcNone;
5649 bool WantNaN = (InterestedClasses & fcNan) != fcNone;
5650 bool WantNegZero = (InterestedClasses & fcNegZero) != fcNone;
5651
5652 if (!WantNaN && !WantNegative && !WantNegZero)
5653 break;
5654
5655 FPClassTest InterestedSrcs = InterestedClasses;
5656 if (WantNegative)
5657 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5658 if (InterestedClasses & fcNan)
5659 InterestedSrcs |= fcInf;
5660 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: InterestedSrcs,
5661 Known&: KnownRHS, Q, Depth: Depth + 1);
5662
5663 // Special case fadd x, x, which is the canonical form of fmul x, 2.
5664 bool Self = Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
5665 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
5666 Depth: Depth + 1);
5667 if (Self)
5668 KnownLHS = KnownRHS;
5669
5670 if ((WantNaN && KnownRHS.isKnownNeverNaN()) ||
5671 (WantNegative && KnownRHS.cannotBeOrderedLessThanZero()) ||
5672 WantNegZero || Opc == Instruction::FSub) {
5673
5674 // FIXME: Context function should always be passed in separately
5675 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5676 const fltSemantics &FltSem =
5677 Op->getType()->getScalarType()->getFltSemantics();
5678 DenormalMode Mode =
5679 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5680
5681 if (Self && Opc == Instruction::FAdd) {
5682 Known = KnownFPClass::fadd_self(Src: KnownLHS, Mode);
5683 } else {
5684 // RHS is canonically cheaper to compute. Skip inspecting the LHS if
5685 // there's no point.
5686
5687 if (!Self) {
5688 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5689 Known&: KnownLHS, Q, Depth: Depth + 1);
5690 }
5691
5692 Known = Opc == Instruction::FAdd
5693 ? KnownFPClass::fadd(LHS: KnownLHS, RHS: KnownRHS, Mode)
5694 : KnownFPClass::fsub(LHS: KnownLHS, RHS: KnownRHS, Mode);
5695 }
5696 }
5697
5698 break;
5699 }
5700 case Instruction::FMul: {
5701 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5702 DenormalMode Mode =
5703 F ? F->getDenormalMode(
5704 FPType: Op->getType()->getScalarType()->getFltSemantics())
5705 : DenormalMode::getDynamic();
5706
5707 Value *LHS = Op->getOperand(i: 0);
5708 Value *RHS = Op->getOperand(i: 1);
5709 // X * X is always non-negative or a NaN.
5710 // FIXME: Should check isGuaranteedNotToBeUndef
5711 if (LHS == RHS) {
5712 KnownFPClass KnownSrc;
5713 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownSrc, Q,
5714 Depth: Depth + 1);
5715 Known = KnownFPClass::square(Src: KnownSrc, Mode);
5716 break;
5717 }
5718
5719 KnownFPClass KnownLHS, KnownRHS;
5720
5721 const APFloat *CRHS;
5722 if (match(V: RHS, P: m_APFloat(Res&: CRHS))) {
5723 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
5724 Depth: Depth + 1);
5725 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: *CRHS, Mode);
5726 } else {
5727 computeKnownFPClass(V: RHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS, Q,
5728 Depth: Depth + 1);
5729 // TODO: Improve accuracy in unfused FMA pattern. We can prove an
5730 // additional not-nan if the addend is known-not negative infinity if the
5731 // multiply is known-not infinity.
5732
5733 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
5734 Depth: Depth + 1);
5735 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: KnownRHS, Mode);
5736 }
5737
5738 /// Propgate no-infs if the other source is known smaller than one, such
5739 /// that this cannot introduce overflow.
5740 if (KnownLHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: RHS))
5741 Known.knownNot(RuleOut: fcInf);
5742 else if (KnownRHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: LHS))
5743 Known.knownNot(RuleOut: fcInf);
5744
5745 break;
5746 }
5747 case Instruction::FDiv:
5748 case Instruction::FRem: {
5749 const bool WantNan = (InterestedClasses & fcNan) != fcNone;
5750
5751 if (Op->getOpcode() == Instruction::FRem)
5752 Known.knownNot(RuleOut: fcInf);
5753
5754 if (Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
5755 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT)) {
5756 if (Op->getOpcode() == Instruction::FDiv) {
5757 // X / X is always exactly 1.0 or a NaN.
5758 Known.KnownFPClasses = fcNan | fcPosNormal;
5759 } else {
5760 // X % X is always exactly [+-]0.0 or a NaN.
5761 Known.KnownFPClasses = fcNan | fcZero;
5762 }
5763
5764 if (!WantNan)
5765 break;
5766
5767 KnownFPClass KnownSrc;
5768 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts,
5769 InterestedClasses: fcNan | fcInf | fcZero | fcSubnormal, Known&: KnownSrc, Q,
5770 Depth: Depth + 1);
5771 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5772 const fltSemantics &FltSem =
5773 Op->getType()->getScalarType()->getFltSemantics();
5774
5775 DenormalMode Mode =
5776 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5777
5778 Known = Op->getOpcode() == Instruction::FDiv
5779 ? KnownFPClass::fdiv_self(Src: KnownSrc, Mode)
5780 : KnownFPClass::frem_self(Src: KnownSrc, Mode);
5781 break;
5782 }
5783
5784 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
5785 const bool WantPositive =
5786 Opc == Instruction::FRem && (InterestedClasses & fcPositive) != fcNone;
5787 if (!WantNan && !WantNegative && !WantPositive)
5788 break;
5789
5790 KnownFPClass KnownLHS, KnownRHS;
5791
5792 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts,
5793 InterestedClasses: fcNan | fcInf | fcZero | fcNegative, Known&: KnownRHS, Q,
5794 Depth: Depth + 1);
5795
5796 bool KnowSomethingUseful = KnownRHS.isKnownNeverNaN() ||
5797 KnownRHS.isKnownNever(Mask: fcNegative) ||
5798 KnownRHS.isKnownNever(Mask: fcPositive);
5799
5800 if (KnowSomethingUseful || WantPositive) {
5801 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS,
5802 Q, Depth: Depth + 1);
5803 }
5804
5805 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5806 const fltSemantics &FltSem =
5807 Op->getType()->getScalarType()->getFltSemantics();
5808
5809 if (Op->getOpcode() == Instruction::FDiv) {
5810 DenormalMode Mode =
5811 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5812 Known = KnownFPClass::fdiv(LHS: KnownLHS, RHS: KnownRHS, Mode);
5813 } else {
5814 // Inf REM x and x REM 0 produce NaN.
5815 if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
5816 KnownLHS.isKnownNeverInfinity() && F &&
5817 KnownRHS.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem))) {
5818 Known.knownNot(RuleOut: fcNan);
5819 }
5820
5821 // The sign for frem is the same as the first operand.
5822 if (KnownLHS.cannotBeOrderedLessThanZero())
5823 Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
5824 if (KnownLHS.cannotBeOrderedGreaterThanZero())
5825 Known.knownNot(RuleOut: KnownFPClass::OrderedGreaterThanZeroMask);
5826
5827 // See if we can be more aggressive about the sign of 0.
5828 if (KnownLHS.isKnownNever(Mask: fcNegative))
5829 Known.knownNot(RuleOut: fcNegative);
5830 if (KnownLHS.isKnownNever(Mask: fcPositive))
5831 Known.knownNot(RuleOut: fcPositive);
5832 }
5833
5834 break;
5835 }
5836 case Instruction::FPExt: {
5837 KnownFPClass KnownSrc;
5838 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5839 Known&: KnownSrc, Q, Depth: Depth + 1);
5840
5841 const fltSemantics &DstTy =
5842 Op->getType()->getScalarType()->getFltSemantics();
5843 const fltSemantics &SrcTy =
5844 Op->getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
5845
5846 Known = KnownFPClass::fpext(KnownSrc, DstTy, SrcTy);
5847 break;
5848 }
5849 case Instruction::FPTrunc: {
5850 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known, Q,
5851 Depth);
5852 break;
5853 }
5854 case Instruction::SIToFP:
5855 case Instruction::UIToFP: {
5856 // Cannot produce nan
5857 Known.knownNot(RuleOut: fcNan);
5858
5859 // Integers cannot be subnormal
5860 Known.knownNot(RuleOut: fcSubnormal);
5861
5862 // sitofp and uitofp turn into +0.0 for zero.
5863 Known.knownNot(RuleOut: fcNegZero);
5864
5865 // UIToFP is always non-negative regardless of known bits.
5866 if (Op->getOpcode() == Instruction::UIToFP)
5867 Known.signBitMustBeZero();
5868
5869 // Only compute known bits if we can learn something useful from them.
5870 if (!(InterestedClasses & (fcPosZero | fcNormal | fcInf)))
5871 break;
5872
5873 KnownBits IntKnown =
5874 computeKnownBits(V: Op->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
5875
5876 // If the integer is non-zero, the result cannot be +0.0
5877 if (IntKnown.isNonZero())
5878 Known.knownNot(RuleOut: fcPosZero);
5879
5880 if (Op->getOpcode() == Instruction::SIToFP) {
5881 // If the signed integer is known non-negative, the result is
5882 // non-negative. If the signed integer is known negative, the result is
5883 // negative.
5884 if (IntKnown.isNonNegative()) {
5885 Known.signBitMustBeZero();
5886 } else if (IntKnown.isNegative()) {
5887 Known.signBitMustBeOne();
5888 }
5889 }
5890
5891 // Guard kept for ilogb()
5892 if (InterestedClasses & fcInf) {
5893 // Get width of largest magnitude integer known.
5894 // This still works for a signed minimum value because the largest FP
5895 // value is scaled by some fraction close to 2.0 (1.0 + 0.xxxx).
5896 int IntSize = IntKnown.getBitWidth();
5897 if (Op->getOpcode() == Instruction::UIToFP)
5898 IntSize -= IntKnown.countMinLeadingZeros();
5899 else if (Op->getOpcode() == Instruction::SIToFP)
5900 IntSize -= IntKnown.countMinSignBits();
5901
5902 // If the exponent of the largest finite FP value can hold the largest
5903 // integer, the result of the cast must be finite.
5904 Type *FPTy = Op->getType()->getScalarType();
5905 if (ilogb(Arg: APFloat::getLargest(Sem: FPTy->getFltSemantics())) >= IntSize)
5906 Known.knownNot(RuleOut: fcInf);
5907 }
5908
5909 break;
5910 }
5911 case Instruction::ExtractElement: {
5912 // Look through extract element. If the index is non-constant or
5913 // out-of-range demand all elements, otherwise just the extracted element.
5914 const Value *Vec = Op->getOperand(i: 0);
5915
5916 APInt DemandedVecElts;
5917 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
5918 unsigned NumElts = VecTy->getNumElements();
5919 DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
5920 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1));
5921 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
5922 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
5923 } else {
5924 DemandedVecElts = APInt(1, 1);
5925 }
5926
5927 return computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known,
5928 Q, Depth: Depth + 1);
5929 }
5930 case Instruction::InsertElement: {
5931 if (isa<ScalableVectorType>(Val: Op->getType()))
5932 return;
5933
5934 const Value *Vec = Op->getOperand(i: 0);
5935 const Value *Elt = Op->getOperand(i: 1);
5936 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 2));
5937 unsigned NumElts = DemandedElts.getBitWidth();
5938 APInt DemandedVecElts = DemandedElts;
5939 bool NeedsElt = true;
5940 // If we know the index we are inserting to, clear it from Vec check.
5941 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
5942 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
5943 NeedsElt = DemandedElts[CIdx->getZExtValue()];
5944 }
5945
5946 // Do we demand the inserted element?
5947 if (NeedsElt) {
5948 computeKnownFPClass(V: Elt, Known, InterestedClasses, Q, Depth: Depth + 1);
5949 // If we don't know any bits, early out.
5950 if (Known.isUnknown())
5951 break;
5952 } else {
5953 Known.KnownFPClasses = fcNone;
5954 }
5955
5956 // Do we need anymore elements from Vec?
5957 if (!DemandedVecElts.isZero()) {
5958 KnownFPClass Known2;
5959 computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known&: Known2, Q,
5960 Depth: Depth + 1);
5961 Known |= Known2;
5962 }
5963
5964 break;
5965 }
5966 case Instruction::ShuffleVector: {
5967 // Handle vector splat idiom
5968 if (Value *Splat = getSplatValue(V)) {
5969 computeKnownFPClass(V: Splat, Known, InterestedClasses, Q, Depth: Depth + 1);
5970 break;
5971 }
5972
5973 // For undef elements, we don't know anything about the common state of
5974 // the shuffle result.
5975 APInt DemandedLHS, DemandedRHS;
5976 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: Op);
5977 if (!Shuf || !getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
5978 return;
5979
5980 if (!!DemandedLHS) {
5981 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
5982 computeKnownFPClass(V: LHS, DemandedElts: DemandedLHS, InterestedClasses, Known, Q,
5983 Depth: Depth + 1);
5984
5985 // If we don't know any bits, early out.
5986 if (Known.isUnknown())
5987 break;
5988 } else {
5989 Known.KnownFPClasses = fcNone;
5990 }
5991
5992 if (!!DemandedRHS) {
5993 KnownFPClass Known2;
5994 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
5995 computeKnownFPClass(V: RHS, DemandedElts: DemandedRHS, InterestedClasses, Known&: Known2, Q,
5996 Depth: Depth + 1);
5997 Known |= Known2;
5998 }
5999
6000 break;
6001 }
6002 case Instruction::ExtractValue: {
6003 const ExtractValueInst *Extract = cast<ExtractValueInst>(Val: Op);
6004 ArrayRef<unsigned> Indices = Extract->getIndices();
6005 const Value *Src = Extract->getAggregateOperand();
6006 if (isa<StructType>(Val: Src->getType()) && Indices.size() == 1 &&
6007 Indices[0] == 0) {
6008 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Src)) {
6009 switch (II->getIntrinsicID()) {
6010 case Intrinsic::frexp: {
6011 Known.knownNot(RuleOut: fcSubnormal);
6012
6013 KnownFPClass KnownSrc;
6014 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
6015 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
6016
6017 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6018 const fltSemantics &FltSem =
6019 Op->getType()->getScalarType()->getFltSemantics();
6020
6021 DenormalMode Mode =
6022 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6023 Known = KnownFPClass::frexp_mant(Src: KnownSrc, Mode);
6024 return;
6025 }
6026 default:
6027 break;
6028 }
6029 }
6030 }
6031
6032 computeKnownFPClass(V: Src, DemandedElts, InterestedClasses, Known, Q,
6033 Depth: Depth + 1);
6034 break;
6035 }
6036 case Instruction::PHI: {
6037 const PHINode *P = cast<PHINode>(Val: Op);
6038 // Unreachable blocks may have zero-operand PHI nodes.
6039 if (P->getNumIncomingValues() == 0)
6040 break;
6041
6042 // Otherwise take the unions of the known bit sets of the operands,
6043 // taking conservative care to avoid excessive recursion.
6044 const unsigned PhiRecursionLimit = MaxAnalysisRecursionDepth - 2;
6045
6046 if (Depth < PhiRecursionLimit) {
6047 // Skip if every incoming value references to ourself.
6048 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
6049 break;
6050
6051 bool First = true;
6052
6053 for (const Use &U : P->operands()) {
6054 Value *IncValue;
6055 Instruction *CxtI;
6056 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CxtI);
6057 // Skip direct self references.
6058 if (IncValue == P)
6059 continue;
6060
6061 KnownFPClass KnownSrc;
6062 // Recurse, but cap the recursion to two levels, because we don't want
6063 // to waste time spinning around in loops. We need at least depth 2 to
6064 // detect known sign bits.
6065 computeKnownFPClass(V: IncValue, DemandedElts, InterestedClasses, Known&: KnownSrc,
6066 Q: Q.getWithoutCondContext().getWithInstruction(I: CxtI),
6067 Depth: PhiRecursionLimit);
6068
6069 if (First) {
6070 Known = KnownSrc;
6071 First = false;
6072 } else {
6073 Known |= KnownSrc;
6074 }
6075
6076 if (Known.KnownFPClasses == fcAllFlags)
6077 break;
6078 }
6079 }
6080
6081 // Look for the case of a for loop which has a positive
6082 // initial value and is incremented by a squared value.
6083 // This will propagate sign information out of such loops.
6084 if (P->getNumIncomingValues() != 2 || Known.cannotBeOrderedLessThanZero())
6085 break;
6086 for (unsigned I = 0; I < 2; I++) {
6087 Value *RecurValue = P->getIncomingValue(i: 1 - I);
6088 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: RecurValue);
6089 if (!II)
6090 continue;
6091 Value *R, *L, *Init;
6092 PHINode *PN;
6093 if (matchSimpleTernaryIntrinsicRecurrence(I: II, P&: PN, Init, OtherOp0&: L, OtherOp1&: R) &&
6094 PN == P) {
6095 switch (II->getIntrinsicID()) {
6096 case Intrinsic::fma:
6097 case Intrinsic::fmuladd: {
6098 KnownFPClass KnownStart;
6099 computeKnownFPClass(V: Init, DemandedElts, InterestedClasses, Known&: KnownStart,
6100 Q, Depth: Depth + 1);
6101 if (KnownStart.cannotBeOrderedLessThanZero() && L == R &&
6102 isGuaranteedNotToBeUndef(V: L, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1))
6103 Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
6104 break;
6105 }
6106 }
6107 }
6108 }
6109 break;
6110 }
6111 case Instruction::BitCast: {
6112 const Value *Src;
6113 if (!match(V: Op, P: m_ElementWiseBitCast(Op: m_Value(V&: Src))) ||
6114 !Src->getType()->isIntOrIntVectorTy())
6115 break;
6116
6117 const Type *Ty = Op->getType();
6118
6119 Value *CastLHS, *CastRHS;
6120
6121 // Match bitcast(umax(bitcast(a), bitcast(b)))
6122 if (match(V: Src, P: m_c_MaxOrMin(L: m_BitCast(Op: m_Value(V&: CastLHS)),
6123 R: m_BitCast(Op: m_Value(V&: CastRHS)))) &&
6124 CastLHS->getType() == Ty && CastRHS->getType() == Ty) {
6125 KnownFPClass KnownLHS, KnownRHS;
6126 computeKnownFPClass(V: CastRHS, DemandedElts, InterestedClasses, Known&: KnownRHS, Q,
6127 Depth: Depth + 1);
6128 if (!KnownRHS.isUnknown()) {
6129 computeKnownFPClass(V: CastLHS, DemandedElts, InterestedClasses, Known&: KnownLHS,
6130 Q, Depth: Depth + 1);
6131 Known = KnownLHS | KnownRHS;
6132 }
6133
6134 return;
6135 }
6136
6137 const Type *EltTy = Ty->getScalarType();
6138 KnownBits Bits(EltTy->getPrimitiveSizeInBits());
6139 computeKnownBits(V: Src, DemandedElts, Known&: Bits, Q, Depth: Depth + 1);
6140
6141 Known = KnownFPClass::bitcast(FltSemantics: EltTy->getFltSemantics(), Bits);
6142 break;
6143 }
6144 default:
6145 break;
6146 }
6147}
6148
6149KnownFPClass llvm::computeKnownFPClass(const Value *V,
6150 const APInt &DemandedElts,
6151 FPClassTest InterestedClasses,
6152 const SimplifyQuery &SQ,
6153 unsigned Depth) {
6154 KnownFPClass KnownClasses;
6155 ::computeKnownFPClass(V, DemandedElts, InterestedClasses, Known&: KnownClasses, Q: SQ,
6156 Depth);
6157 return KnownClasses;
6158}
6159
6160KnownFPClass llvm::computeKnownFPClass(const Value *V,
6161 FPClassTest InterestedClasses,
6162 const SimplifyQuery &SQ,
6163 unsigned Depth) {
6164 KnownFPClass Known;
6165 ::computeKnownFPClass(V, Known, InterestedClasses, Q: SQ, Depth);
6166 return Known;
6167}
6168
6169KnownFPClass llvm::computeKnownFPClass(
6170 const Value *V, const DataLayout &DL, FPClassTest InterestedClasses,
6171 const TargetLibraryInfo *TLI, AssumptionCache *AC, const Instruction *CxtI,
6172 const DominatorTree *DT, bool UseInstrInfo, unsigned Depth) {
6173 return computeKnownFPClass(V, InterestedClasses,
6174 SQ: SimplifyQuery(DL, TLI, DT, AC, CxtI, UseInstrInfo),
6175 Depth);
6176}
6177
6178KnownFPClass
6179llvm::computeKnownFPClass(const Value *V, const APInt &DemandedElts,
6180 FastMathFlags FMF, FPClassTest InterestedClasses,
6181 const SimplifyQuery &SQ, unsigned Depth) {
6182 if (FMF.noNaNs())
6183 InterestedClasses &= ~fcNan;
6184 if (FMF.noInfs())
6185 InterestedClasses &= ~fcInf;
6186
6187 KnownFPClass Result =
6188 computeKnownFPClass(V, DemandedElts, InterestedClasses, SQ, Depth);
6189
6190 if (FMF.noNaNs())
6191 Result.KnownFPClasses &= ~fcNan;
6192 if (FMF.noInfs())
6193 Result.KnownFPClasses &= ~fcInf;
6194 return Result;
6195}
6196
6197KnownFPClass llvm::computeKnownFPClass(const Value *V, FastMathFlags FMF,
6198 FPClassTest InterestedClasses,
6199 const SimplifyQuery &SQ,
6200 unsigned Depth) {
6201 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
6202 APInt DemandedElts =
6203 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
6204 return computeKnownFPClass(V, DemandedElts, FMF, InterestedClasses, SQ,
6205 Depth);
6206}
6207
6208bool llvm::cannotBeNegativeZero(const Value *V, const SimplifyQuery &SQ,
6209 unsigned Depth) {
6210 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNegZero, SQ, Depth);
6211 return Known.isKnownNeverNegZero();
6212}
6213
6214bool llvm::cannotBeOrderedLessThanZero(const Value *V, const SimplifyQuery &SQ,
6215 unsigned Depth) {
6216 KnownFPClass Known =
6217 computeKnownFPClass(V, InterestedClasses: KnownFPClass::OrderedLessThanZeroMask, SQ, Depth);
6218 return Known.cannotBeOrderedLessThanZero();
6219}
6220
6221bool llvm::isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ,
6222 unsigned Depth) {
6223 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf, SQ, Depth);
6224 return Known.isKnownNeverInfinity();
6225}
6226
6227/// Return true if the floating-point value can never contain a NaN or infinity.
6228bool llvm::isKnownNeverInfOrNaN(const Value *V, const SimplifyQuery &SQ,
6229 unsigned Depth) {
6230 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf | fcNan, SQ, Depth);
6231 return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity();
6232}
6233
6234/// Return true if the floating-point scalar value is not a NaN or if the
6235/// floating-point vector value has no NaN elements. Return false if a value
6236/// could ever be NaN.
6237bool llvm::isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ,
6238 unsigned Depth) {
6239 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNan, SQ, Depth);
6240 return Known.isKnownNeverNaN();
6241}
6242
6243/// Return false if we can prove that the specified FP value's sign bit is 0.
6244/// Return true if we can prove that the specified FP value's sign bit is 1.
6245/// Otherwise return std::nullopt.
6246std::optional<bool> llvm::computeKnownFPSignBit(const Value *V,
6247 const SimplifyQuery &SQ,
6248 unsigned Depth) {
6249 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcAllFlags, SQ, Depth);
6250 return Known.SignBit;
6251}
6252
6253bool llvm::canIgnoreSignBitOfZero(const Use &U) {
6254 auto *User = cast<Instruction>(Val: U.getUser());
6255 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6256 if (FPOp->hasNoSignedZeros())
6257 return true;
6258 }
6259
6260 switch (User->getOpcode()) {
6261 case Instruction::FPToSI:
6262 case Instruction::FPToUI:
6263 return true;
6264 case Instruction::FCmp:
6265 // fcmp treats both positive and negative zero as equal.
6266 return true;
6267 case Instruction::Call:
6268 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6269 switch (II->getIntrinsicID()) {
6270 case Intrinsic::fabs:
6271 return true;
6272 case Intrinsic::copysign:
6273 return U.getOperandNo() == 0;
6274 case Intrinsic::is_fpclass:
6275 case Intrinsic::vp_is_fpclass: {
6276 auto Test =
6277 static_cast<FPClassTest>(
6278 cast<ConstantInt>(Val: II->getArgOperand(i: 1))->getZExtValue()) &
6279 FPClassTest::fcZero;
6280 return Test == FPClassTest::fcZero || Test == FPClassTest::fcNone;
6281 }
6282 default:
6283 return false;
6284 }
6285 }
6286 return false;
6287 default:
6288 return false;
6289 }
6290}
6291
6292bool llvm::canIgnoreSignBitOfNaN(const Use &U) {
6293 auto *User = cast<Instruction>(Val: U.getUser());
6294 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6295 if (FPOp->hasNoNaNs())
6296 return true;
6297 }
6298
6299 switch (User->getOpcode()) {
6300 case Instruction::FPToSI:
6301 case Instruction::FPToUI:
6302 return true;
6303 // Proper FP math operations ignore the sign bit of NaN.
6304 case Instruction::FAdd:
6305 case Instruction::FSub:
6306 case Instruction::FMul:
6307 case Instruction::FDiv:
6308 case Instruction::FRem:
6309 case Instruction::FPTrunc:
6310 case Instruction::FPExt:
6311 case Instruction::FCmp:
6312 return true;
6313 // Bitwise FP operations should preserve the sign bit of NaN.
6314 case Instruction::FNeg:
6315 case Instruction::Select:
6316 case Instruction::PHI:
6317 return false;
6318 case Instruction::Ret:
6319 return User->getFunction()->getAttributes().getRetNoFPClass() &
6320 FPClassTest::fcNan;
6321 case Instruction::Call:
6322 case Instruction::Invoke: {
6323 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6324 switch (II->getIntrinsicID()) {
6325 case Intrinsic::fabs:
6326 return true;
6327 case Intrinsic::copysign:
6328 return U.getOperandNo() == 0;
6329 // Other proper FP math intrinsics ignore the sign bit of NaN.
6330 case Intrinsic::maxnum:
6331 case Intrinsic::minnum:
6332 case Intrinsic::maximum:
6333 case Intrinsic::minimum:
6334 case Intrinsic::maximumnum:
6335 case Intrinsic::minimumnum:
6336 case Intrinsic::canonicalize:
6337 case Intrinsic::fma:
6338 case Intrinsic::fmuladd:
6339 case Intrinsic::sqrt:
6340 case Intrinsic::pow:
6341 case Intrinsic::powi:
6342 case Intrinsic::fptoui_sat:
6343 case Intrinsic::fptosi_sat:
6344 case Intrinsic::is_fpclass:
6345 case Intrinsic::vp_is_fpclass:
6346 return true;
6347 default:
6348 return false;
6349 }
6350 }
6351
6352 FPClassTest NoFPClass =
6353 cast<CallBase>(Val: User)->getParamNoFPClass(i: U.getOperandNo());
6354 return NoFPClass & FPClassTest::fcNan;
6355 }
6356 default:
6357 return false;
6358 }
6359}
6360
6361bool llvm::isKnownIntegral(const Value *V, const SimplifyQuery &SQ,
6362 FastMathFlags FMF) {
6363 if (isa<PoisonValue>(Val: V))
6364 return true;
6365 if (isa<UndefValue>(Val: V))
6366 return false;
6367
6368 if (match(V, P: m_CheckedFp(CheckFn: [](const APFloat &Val) { return Val.isInteger(); })))
6369 return true;
6370
6371 const Instruction *I = dyn_cast<Instruction>(Val: V);
6372 if (!I)
6373 return false;
6374
6375 switch (I->getOpcode()) {
6376 case Instruction::SIToFP:
6377 case Instruction::UIToFP:
6378 // TODO: Could check nofpclass(inf) on incoming argument
6379 if (FMF.noInfs())
6380 return true;
6381
6382 // Need to check int size cannot produce infinity, which computeKnownFPClass
6383 // knows how to do already.
6384 return isKnownNeverInfinity(V: I, SQ);
6385 case Instruction::Call: {
6386 const CallInst *CI = cast<CallInst>(Val: I);
6387 switch (CI->getIntrinsicID()) {
6388 case Intrinsic::trunc:
6389 case Intrinsic::floor:
6390 case Intrinsic::ceil:
6391 case Intrinsic::rint:
6392 case Intrinsic::nearbyint:
6393 case Intrinsic::round:
6394 case Intrinsic::roundeven:
6395 return (FMF.noInfs() && FMF.noNaNs()) || isKnownNeverInfOrNaN(V: I, SQ);
6396 default:
6397 break;
6398 }
6399
6400 break;
6401 }
6402 default:
6403 break;
6404 }
6405
6406 return false;
6407}
6408
6409Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
6410
6411 // All byte-wide stores are splatable, even of arbitrary variables.
6412 if (V->getType()->isIntegerTy(BitWidth: 8))
6413 return V;
6414
6415 LLVMContext &Ctx = V->getContext();
6416
6417 // Undef don't care.
6418 auto *UndefInt8 = UndefValue::get(T: Type::getInt8Ty(C&: Ctx));
6419 if (isa<UndefValue>(Val: V))
6420 return UndefInt8;
6421
6422 // Return poison for zero-sized type.
6423 if (DL.getTypeStoreSize(Ty: V->getType()).isZero())
6424 return PoisonValue::get(T: Type::getInt8Ty(C&: Ctx));
6425
6426 Constant *C = dyn_cast<Constant>(Val: V);
6427 if (!C) {
6428 // Conceptually, we could handle things like:
6429 // %a = zext i8 %X to i16
6430 // %b = shl i16 %a, 8
6431 // %c = or i16 %a, %b
6432 // but until there is an example that actually needs this, it doesn't seem
6433 // worth worrying about.
6434 return nullptr;
6435 }
6436
6437 // Handle 'null' ConstantArrayZero etc.
6438 if (C->isNullValue())
6439 return Constant::getNullValue(Ty: Type::getInt8Ty(C&: Ctx));
6440
6441 // Constant floating-point values can be handled as integer values if the
6442 // corresponding integer value is "byteable". An important case is 0.0.
6443 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
6444 Type *ScalarTy = CFP->getType()->getScalarType();
6445 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
6446 return isBytewiseValue(
6447 V: ConstantInt::get(Context&: Ctx, V: CFP->getValue().bitcastToAPInt()), DL);
6448
6449 // Don't handle long double formats, which have strange constraints.
6450 return nullptr;
6451 }
6452
6453 // We can handle constant integers that are multiple of 8 bits.
6454 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
6455 if (CI->getBitWidth() % 8 == 0) {
6456 if (!CI->getValue().isSplat(SplatSizeInBits: 8))
6457 return nullptr;
6458 return ConstantInt::get(Context&: Ctx, V: CI->getValue().trunc(width: 8));
6459 }
6460 }
6461
6462 if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
6463 if (CE->getOpcode() == Instruction::IntToPtr) {
6464 if (auto *PtrTy = dyn_cast<PointerType>(Val: CE->getType())) {
6465 unsigned BitWidth = DL.getPointerSizeInBits(AS: PtrTy->getAddressSpace());
6466 if (Constant *Op = ConstantFoldIntegerCast(
6467 C: CE->getOperand(i_nocapture: 0), DestTy: Type::getIntNTy(C&: Ctx, N: BitWidth), IsSigned: false, DL))
6468 return isBytewiseValue(V: Op, DL);
6469 }
6470 }
6471 }
6472
6473 auto Merge = [&](Value *LHS, Value *RHS) -> Value * {
6474 if (LHS == RHS)
6475 return LHS;
6476 if (!LHS || !RHS)
6477 return nullptr;
6478 if (LHS == UndefInt8)
6479 return RHS;
6480 if (RHS == UndefInt8)
6481 return LHS;
6482 return nullptr;
6483 };
6484
6485 if (ConstantDataSequential *CA = dyn_cast<ConstantDataSequential>(Val: C)) {
6486 Value *Val = UndefInt8;
6487 for (uint64_t I = 0, E = CA->getNumElements(); I != E; ++I)
6488 if (!(Val = Merge(Val, isBytewiseValue(V: CA->getElementAsConstant(i: I), DL))))
6489 return nullptr;
6490 return Val;
6491 }
6492
6493 if (isa<ConstantAggregate>(Val: C)) {
6494 Value *Val = UndefInt8;
6495 for (Value *Op : C->operands())
6496 if (!(Val = Merge(Val, isBytewiseValue(V: Op, DL))))
6497 return nullptr;
6498 return Val;
6499 }
6500
6501 // Don't try to handle the handful of other constants.
6502 return nullptr;
6503}
6504
6505// This is the recursive version of BuildSubAggregate. It takes a few different
6506// arguments. Idxs is the index within the nested struct From that we are
6507// looking at now (which is of type IndexedType). IdxSkip is the number of
6508// indices from Idxs that should be left out when inserting into the resulting
6509// struct. To is the result struct built so far, new insertvalue instructions
6510// build on that.
6511static Value *BuildSubAggregate(Value *From, Value *To, Type *IndexedType,
6512 SmallVectorImpl<unsigned> &Idxs,
6513 unsigned IdxSkip,
6514 BasicBlock::iterator InsertBefore) {
6515 StructType *STy = dyn_cast<StructType>(Val: IndexedType);
6516 if (STy) {
6517 // Save the original To argument so we can modify it
6518 Value *OrigTo = To;
6519 // General case, the type indexed by Idxs is a struct
6520 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6521 // Process each struct element recursively
6522 Idxs.push_back(Elt: i);
6523 Value *PrevTo = To;
6524 To = BuildSubAggregate(From, To, IndexedType: STy->getElementType(N: i), Idxs, IdxSkip,
6525 InsertBefore);
6526 Idxs.pop_back();
6527 if (!To) {
6528 // Couldn't find any inserted value for this index? Cleanup
6529 while (PrevTo != OrigTo) {
6530 InsertValueInst* Del = cast<InsertValueInst>(Val: PrevTo);
6531 PrevTo = Del->getAggregateOperand();
6532 Del->eraseFromParent();
6533 }
6534 // Stop processing elements
6535 break;
6536 }
6537 }
6538 // If we successfully found a value for each of our subaggregates
6539 if (To)
6540 return To;
6541 }
6542 // Base case, the type indexed by SourceIdxs is not a struct, or not all of
6543 // the struct's elements had a value that was inserted directly. In the latter
6544 // case, perhaps we can't determine each of the subelements individually, but
6545 // we might be able to find the complete struct somewhere.
6546
6547 // Find the value that is at that particular spot
6548 Value *V = FindInsertedValue(V: From, idx_range: Idxs);
6549
6550 if (!V)
6551 return nullptr;
6552
6553 // Insert the value in the new (sub) aggregate
6554 return InsertValueInst::Create(Agg: To, Val: V, Idxs: ArrayRef(Idxs).slice(N: IdxSkip), NameStr: "tmp",
6555 InsertBefore);
6556}
6557
6558// This helper takes a nested struct and extracts a part of it (which is again a
6559// struct) into a new value. For example, given the struct:
6560// { a, { b, { c, d }, e } }
6561// and the indices "1, 1" this returns
6562// { c, d }.
6563//
6564// It does this by inserting an insertvalue for each element in the resulting
6565// struct, as opposed to just inserting a single struct. This will only work if
6566// each of the elements of the substruct are known (ie, inserted into From by an
6567// insertvalue instruction somewhere).
6568//
6569// All inserted insertvalue instructions are inserted before InsertBefore
6570static Value *BuildSubAggregate(Value *From, ArrayRef<unsigned> idx_range,
6571 BasicBlock::iterator InsertBefore) {
6572 Type *IndexedType = ExtractValueInst::getIndexedType(Agg: From->getType(),
6573 Idxs: idx_range);
6574 Value *To = PoisonValue::get(T: IndexedType);
6575 SmallVector<unsigned, 10> Idxs(idx_range);
6576 unsigned IdxSkip = Idxs.size();
6577
6578 return BuildSubAggregate(From, To, IndexedType, Idxs, IdxSkip, InsertBefore);
6579}
6580
6581/// Given an aggregate and a sequence of indices, see if the scalar value
6582/// indexed is already around as a register, for example if it was inserted
6583/// directly into the aggregate.
6584///
6585/// If InsertBefore is not null, this function will duplicate (modified)
6586/// insertvalues when a part of a nested struct is extracted.
6587Value *
6588llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
6589 std::optional<BasicBlock::iterator> InsertBefore) {
6590 // Nothing to index? Just return V then (this is useful at the end of our
6591 // recursion).
6592 if (idx_range.empty())
6593 return V;
6594 // We have indices, so V should have an indexable type.
6595 assert((V->getType()->isStructTy() || V->getType()->isArrayTy()) &&
6596 "Not looking at a struct or array?");
6597 assert(ExtractValueInst::getIndexedType(V->getType(), idx_range) &&
6598 "Invalid indices for type?");
6599
6600 if (Constant *C = dyn_cast<Constant>(Val: V)) {
6601 C = C->getAggregateElement(Elt: idx_range[0]);
6602 if (!C) return nullptr;
6603 return FindInsertedValue(V: C, idx_range: idx_range.slice(N: 1), InsertBefore);
6604 }
6605
6606 if (InsertValueInst *I = dyn_cast<InsertValueInst>(Val: V)) {
6607 // Loop the indices for the insertvalue instruction in parallel with the
6608 // requested indices
6609 const unsigned *req_idx = idx_range.begin();
6610 for (const unsigned *i = I->idx_begin(), *e = I->idx_end();
6611 i != e; ++i, ++req_idx) {
6612 if (req_idx == idx_range.end()) {
6613 // We can't handle this without inserting insertvalues
6614 if (!InsertBefore)
6615 return nullptr;
6616
6617 // The requested index identifies a part of a nested aggregate. Handle
6618 // this specially. For example,
6619 // %A = insertvalue { i32, {i32, i32 } } undef, i32 10, 1, 0
6620 // %B = insertvalue { i32, {i32, i32 } } %A, i32 11, 1, 1
6621 // %C = extractvalue {i32, { i32, i32 } } %B, 1
6622 // This can be changed into
6623 // %A = insertvalue {i32, i32 } undef, i32 10, 0
6624 // %C = insertvalue {i32, i32 } %A, i32 11, 1
6625 // which allows the unused 0,0 element from the nested struct to be
6626 // removed.
6627 return BuildSubAggregate(From: V, idx_range: ArrayRef(idx_range.begin(), req_idx),
6628 InsertBefore: *InsertBefore);
6629 }
6630
6631 // This insert value inserts something else than what we are looking for.
6632 // See if the (aggregate) value inserted into has the value we are
6633 // looking for, then.
6634 if (*req_idx != *i)
6635 return FindInsertedValue(V: I->getAggregateOperand(), idx_range,
6636 InsertBefore);
6637 }
6638 // If we end up here, the indices of the insertvalue match with those
6639 // requested (though possibly only partially). Now we recursively look at
6640 // the inserted value, passing any remaining indices.
6641 return FindInsertedValue(V: I->getInsertedValueOperand(),
6642 idx_range: ArrayRef(req_idx, idx_range.end()), InsertBefore);
6643 }
6644
6645 if (ExtractValueInst *I = dyn_cast<ExtractValueInst>(Val: V)) {
6646 // If we're extracting a value from an aggregate that was extracted from
6647 // something else, we can extract from that something else directly instead.
6648 // However, we will need to chain I's indices with the requested indices.
6649
6650 // Calculate the number of indices required
6651 unsigned size = I->getNumIndices() + idx_range.size();
6652 // Allocate some space to put the new indices in
6653 SmallVector<unsigned, 5> Idxs;
6654 Idxs.reserve(N: size);
6655 // Add indices from the extract value instruction
6656 Idxs.append(in_start: I->idx_begin(), in_end: I->idx_end());
6657
6658 // Add requested indices
6659 Idxs.append(in_start: idx_range.begin(), in_end: idx_range.end());
6660
6661 assert(Idxs.size() == size
6662 && "Number of indices added not correct?");
6663
6664 return FindInsertedValue(V: I->getAggregateOperand(), idx_range: Idxs, InsertBefore);
6665 }
6666 // Otherwise, we don't know (such as, extracting from a function return value
6667 // or load instruction)
6668 return nullptr;
6669}
6670
6671// If V refers to an initialized global constant, set Slice either to
6672// its initializer if the size of its elements equals ElementSize, or,
6673// for ElementSize == 8, to its representation as an array of unsiged
6674// char. Return true on success.
6675// Offset is in the unit "nr of ElementSize sized elements".
6676bool llvm::getConstantDataArrayInfo(const Value *V,
6677 ConstantDataArraySlice &Slice,
6678 unsigned ElementSize, uint64_t Offset) {
6679 assert(V && "V should not be null.");
6680 assert((ElementSize % 8) == 0 &&
6681 "ElementSize expected to be a multiple of the size of a byte.");
6682 unsigned ElementSizeInBytes = ElementSize / 8;
6683
6684 // Drill down into the pointer expression V, ignoring any intervening
6685 // casts, and determine the identity of the object it references along
6686 // with the cumulative byte offset into it.
6687 const GlobalVariable *GV =
6688 dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V));
6689 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
6690 // Fail if V is not based on constant global object.
6691 return false;
6692
6693 const DataLayout &DL = GV->getDataLayout();
6694 APInt Off(DL.getIndexTypeSizeInBits(Ty: V->getType()), 0);
6695
6696 if (GV != V->stripAndAccumulateConstantOffsets(DL, Offset&: Off,
6697 /*AllowNonInbounds*/ true))
6698 // Fail if a constant offset could not be determined.
6699 return false;
6700
6701 uint64_t StartIdx = Off.getLimitedValue();
6702 if (StartIdx == UINT64_MAX)
6703 // Fail if the constant offset is excessive.
6704 return false;
6705
6706 // Off/StartIdx is in the unit of bytes. So we need to convert to number of
6707 // elements. Simply bail out if that isn't possible.
6708 if ((StartIdx % ElementSizeInBytes) != 0)
6709 return false;
6710
6711 Offset += StartIdx / ElementSizeInBytes;
6712 ConstantDataArray *Array = nullptr;
6713 ArrayType *ArrayTy = nullptr;
6714
6715 if (GV->getInitializer()->isNullValue()) {
6716 Type *GVTy = GV->getValueType();
6717 uint64_t SizeInBytes = DL.getTypeStoreSize(Ty: GVTy).getFixedValue();
6718 uint64_t Length = SizeInBytes / ElementSizeInBytes;
6719
6720 Slice.Array = nullptr;
6721 Slice.Offset = 0;
6722 // Return an empty Slice for undersized constants to let callers
6723 // transform even undefined library calls into simpler, well-defined
6724 // expressions. This is preferable to making the calls although it
6725 // prevents sanitizers from detecting such calls.
6726 Slice.Length = Length < Offset ? 0 : Length - Offset;
6727 return true;
6728 }
6729
6730 auto *Init = const_cast<Constant *>(GV->getInitializer());
6731 if (auto *ArrayInit = dyn_cast<ConstantDataArray>(Val: Init)) {
6732 Type *InitElTy = ArrayInit->getElementType();
6733 if (InitElTy->isIntegerTy(BitWidth: ElementSize)) {
6734 // If Init is an initializer for an array of the expected type
6735 // and size, use it as is.
6736 Array = ArrayInit;
6737 ArrayTy = ArrayInit->getType();
6738 }
6739 }
6740
6741 if (!Array) {
6742 if (ElementSize != 8)
6743 // TODO: Handle conversions to larger integral types.
6744 return false;
6745
6746 // Otherwise extract the portion of the initializer starting
6747 // at Offset as an array of bytes, and reset Offset.
6748 Init = ReadByteArrayFromGlobal(GV, Offset);
6749 if (!Init)
6750 return false;
6751
6752 Offset = 0;
6753 Array = dyn_cast<ConstantDataArray>(Val: Init);
6754 ArrayTy = dyn_cast<ArrayType>(Val: Init->getType());
6755 }
6756
6757 uint64_t NumElts = ArrayTy->getArrayNumElements();
6758 if (Offset > NumElts)
6759 return false;
6760
6761 Slice.Array = Array;
6762 Slice.Offset = Offset;
6763 Slice.Length = NumElts - Offset;
6764 return true;
6765}
6766
6767/// Extract bytes from the initializer of the constant array V, which need
6768/// not be a nul-terminated string. On success, store the bytes in Str and
6769/// return true. When TrimAtNul is set, Str will contain only the bytes up
6770/// to but not including the first nul. Return false on failure.
6771bool llvm::getConstantStringInfo(const Value *V, StringRef &Str,
6772 bool TrimAtNul) {
6773 ConstantDataArraySlice Slice;
6774 if (!getConstantDataArrayInfo(V, Slice, ElementSize: 8))
6775 return false;
6776
6777 if (Slice.Array == nullptr) {
6778 if (TrimAtNul) {
6779 // Return a nul-terminated string even for an empty Slice. This is
6780 // safe because all existing SimplifyLibcalls callers require string
6781 // arguments and the behavior of the functions they fold is undefined
6782 // otherwise. Folding the calls this way is preferable to making
6783 // the undefined library calls, even though it prevents sanitizers
6784 // from reporting such calls.
6785 Str = StringRef();
6786 return true;
6787 }
6788 if (Slice.Length == 1) {
6789 Str = StringRef("", 1);
6790 return true;
6791 }
6792 // We cannot instantiate a StringRef as we do not have an appropriate string
6793 // of 0s at hand.
6794 return false;
6795 }
6796
6797 // Start out with the entire array in the StringRef.
6798 Str = Slice.Array->getAsString();
6799 // Skip over 'offset' bytes.
6800 Str = Str.substr(Start: Slice.Offset);
6801
6802 if (TrimAtNul) {
6803 // Trim off the \0 and anything after it. If the array is not nul
6804 // terminated, we just return the whole end of string. The client may know
6805 // some other way that the string is length-bound.
6806 Str = Str.substr(Start: 0, N: Str.find(C: '\0'));
6807 }
6808 return true;
6809}
6810
6811// These next two are very similar to the above, but also look through PHI
6812// nodes.
6813// TODO: See if we can integrate these two together.
6814
6815/// If we can compute the length of the string pointed to by
6816/// the specified pointer, return 'len+1'. If we can't, return 0.
6817static uint64_t GetStringLengthH(const Value *V,
6818 SmallPtrSetImpl<const PHINode*> &PHIs,
6819 unsigned CharSize) {
6820 // Look through noop bitcast instructions.
6821 V = V->stripPointerCasts();
6822
6823 // If this is a PHI node, there are two cases: either we have already seen it
6824 // or we haven't.
6825 if (const PHINode *PN = dyn_cast<PHINode>(Val: V)) {
6826 if (!PHIs.insert(Ptr: PN).second)
6827 return ~0ULL; // already in the set.
6828
6829 // If it was new, see if all the input strings are the same length.
6830 uint64_t LenSoFar = ~0ULL;
6831 for (Value *IncValue : PN->incoming_values()) {
6832 uint64_t Len = GetStringLengthH(V: IncValue, PHIs, CharSize);
6833 if (Len == 0) return 0; // Unknown length -> unknown.
6834
6835 if (Len == ~0ULL) continue;
6836
6837 if (Len != LenSoFar && LenSoFar != ~0ULL)
6838 return 0; // Disagree -> unknown.
6839 LenSoFar = Len;
6840 }
6841
6842 // Success, all agree.
6843 return LenSoFar;
6844 }
6845
6846 // strlen(select(c,x,y)) -> strlen(x) ^ strlen(y)
6847 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: V)) {
6848 uint64_t Len1 = GetStringLengthH(V: SI->getTrueValue(), PHIs, CharSize);
6849 if (Len1 == 0) return 0;
6850 uint64_t Len2 = GetStringLengthH(V: SI->getFalseValue(), PHIs, CharSize);
6851 if (Len2 == 0) return 0;
6852 if (Len1 == ~0ULL) return Len2;
6853 if (Len2 == ~0ULL) return Len1;
6854 if (Len1 != Len2) return 0;
6855 return Len1;
6856 }
6857
6858 // Otherwise, see if we can read the string.
6859 ConstantDataArraySlice Slice;
6860 if (!getConstantDataArrayInfo(V, Slice, ElementSize: CharSize))
6861 return 0;
6862
6863 if (Slice.Array == nullptr)
6864 // Zeroinitializer (including an empty one).
6865 return 1;
6866
6867 // Search for the first nul character. Return a conservative result even
6868 // when there is no nul. This is safe since otherwise the string function
6869 // being folded such as strlen is undefined, and can be preferable to
6870 // making the undefined library call.
6871 unsigned NullIndex = 0;
6872 for (unsigned E = Slice.Length; NullIndex < E; ++NullIndex) {
6873 if (Slice.Array->getElementAsInteger(i: Slice.Offset + NullIndex) == 0)
6874 break;
6875 }
6876
6877 return NullIndex + 1;
6878}
6879
6880/// If we can compute the length of the string pointed to by
6881/// the specified pointer, return 'len+1'. If we can't, return 0.
6882uint64_t llvm::GetStringLength(const Value *V, unsigned CharSize) {
6883 if (!V->getType()->isPointerTy())
6884 return 0;
6885
6886 SmallPtrSet<const PHINode*, 32> PHIs;
6887 uint64_t Len = GetStringLengthH(V, PHIs, CharSize);
6888 // If Len is ~0ULL, we had an infinite phi cycle: this is dead code, so return
6889 // an empty string as a length.
6890 return Len == ~0ULL ? 1 : Len;
6891}
6892
6893const Value *
6894llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call,
6895 bool MustPreserveOffset) {
6896 assert(Call &&
6897 "getArgumentAliasingToReturnedPointer only works on nonnull calls");
6898 if (const Value *RV = Call->getReturnedArgOperand())
6899 return RV;
6900 // This can be used only as a aliasing property.
6901 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
6902 Call, MustPreserveOffset))
6903 return Call->getArgOperand(i: 0);
6904 return nullptr;
6905}
6906
6907bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
6908 const CallBase *Call, bool MustPreserveOffset) {
6909 switch (Call->getIntrinsicID()) {
6910 case Intrinsic::launder_invariant_group:
6911 case Intrinsic::strip_invariant_group:
6912 case Intrinsic::aarch64_irg:
6913 case Intrinsic::aarch64_tagp:
6914 // The amdgcn_make_buffer_rsrc function does not alter the address of the
6915 // input pointer (and thus preserves the byte offset, which is the property
6916 // the MustPreserveOffset flag selects). However, it will not necessarily
6917 // map ptr addrspace(N) null to ptr addrspace(8) null, aka the "null
6918 // descriptor", which has "all loads return 0, all stores are dropped"
6919 // semantics. Given the context of this intrinsic list, no one should be
6920 // relying on such a strict bit-exact null mapping (and, at time of
6921 // writing, they are not), but we document this fact out of an abundance
6922 // of caution.
6923 case Intrinsic::amdgcn_make_buffer_rsrc:
6924 return true;
6925 case Intrinsic::ptrmask:
6926 return !MustPreserveOffset;
6927 case Intrinsic::threadlocal_address:
6928 // The underlying variable changes with thread ID. The Thread ID may change
6929 // at coroutine suspend points.
6930 return !Call->getParent()->getParent()->isPresplitCoroutine();
6931 default:
6932 return false;
6933 }
6934}
6935
6936/// \p PN defines a loop-variant pointer to an object. Check if the
6937/// previous iteration of the loop was referring to the same object as \p PN.
6938static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
6939 const LoopInfo *LI) {
6940 // Find the loop-defined value.
6941 Loop *L = LI->getLoopFor(BB: PN->getParent());
6942 if (PN->getNumIncomingValues() != 2)
6943 return true;
6944
6945 // Find the value from previous iteration.
6946 auto *PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 0));
6947 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
6948 PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 1));
6949 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
6950 return true;
6951
6952 // If a new pointer is loaded in the loop, the pointer references a different
6953 // object in every iteration. E.g.:
6954 // for (i)
6955 // int *p = a[i];
6956 // ...
6957 if (auto *Load = dyn_cast<LoadInst>(Val: PrevValue))
6958 if (!L->isLoopInvariant(V: Load->getPointerOperand()))
6959 return false;
6960 return true;
6961}
6962
6963const Value *llvm::getUnderlyingObject(const Value *V, unsigned MaxLookup) {
6964 for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
6965 if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
6966 const Value *PtrOp = GEP->getPointerOperand();
6967 if (!PtrOp->getType()->isPointerTy()) // Only handle scalar pointer base.
6968 return V;
6969 V = PtrOp;
6970 } else if (Operator::getOpcode(V) == Instruction::BitCast ||
6971 Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
6972 Value *NewV = cast<Operator>(Val: V)->getOperand(i: 0);
6973 if (!NewV->getType()->isPointerTy())
6974 return V;
6975 V = NewV;
6976 } else if (auto *GA = dyn_cast<GlobalAlias>(Val: V)) {
6977 if (GA->isInterposable())
6978 return V;
6979 V = GA->getAliasee();
6980 } else {
6981 if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
6982 // Look through single-arg phi nodes created by LCSSA.
6983 if (PHI->getNumIncomingValues() == 1) {
6984 V = PHI->getIncomingValue(i: 0);
6985 continue;
6986 }
6987 } else if (auto *Call = dyn_cast<CallBase>(Val: V)) {
6988 // CaptureTracking can know about special capturing properties of some
6989 // intrinsics like launder.invariant.group, that can't be expressed with
6990 // the attributes, but have properties like returning aliasing pointer.
6991 // Because some analysis may assume that nocaptured pointer is not
6992 // returned from some special intrinsic (because function would have to
6993 // be marked with returns attribute), it is crucial to use this function
6994 // because it should be in sync with CaptureTracking. Not using it may
6995 // cause weird miscompilations where 2 aliasing pointers are assumed to
6996 // noalias.
6997 if (auto *RP = getArgumentAliasingToReturnedPointer(
6998 Call, /*MustPreserveOffset=*/false)) {
6999 V = RP;
7000 continue;
7001 }
7002 }
7003
7004 return V;
7005 }
7006 assert(V->getType()->isPointerTy() && "Unexpected operand type!");
7007 }
7008 return V;
7009}
7010
7011void llvm::getUnderlyingObjects(const Value *V,
7012 SmallVectorImpl<const Value *> &Objects,
7013 const LoopInfo *LI, unsigned MaxLookup) {
7014 SmallPtrSet<const Value *, 4> Visited;
7015 SmallVector<const Value *, 4> Worklist;
7016 Worklist.push_back(Elt: V);
7017 do {
7018 const Value *P = Worklist.pop_back_val();
7019 P = getUnderlyingObject(V: P, MaxLookup);
7020
7021 if (!Visited.insert(Ptr: P).second)
7022 continue;
7023
7024 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7025 Worklist.push_back(Elt: SI->getTrueValue());
7026 Worklist.push_back(Elt: SI->getFalseValue());
7027 continue;
7028 }
7029
7030 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7031 // If this PHI changes the underlying object in every iteration of the
7032 // loop, don't look through it. Consider:
7033 // int **A;
7034 // for (i) {
7035 // Prev = Curr; // Prev = PHI (Prev_0, Curr)
7036 // Curr = A[i];
7037 // *Prev, *Curr;
7038 //
7039 // Prev is tracking Curr one iteration behind so they refer to different
7040 // underlying objects.
7041 if (!LI || !LI->isLoopHeader(BB: PN->getParent()) ||
7042 isSameUnderlyingObjectInLoop(PN, LI))
7043 append_range(C&: Worklist, R: PN->incoming_values());
7044 else
7045 Objects.push_back(Elt: P);
7046 continue;
7047 }
7048
7049 Objects.push_back(Elt: P);
7050 } while (!Worklist.empty());
7051}
7052
7053const Value *llvm::getUnderlyingObjectAggressive(const Value *V) {
7054 const unsigned MaxVisited = 8;
7055
7056 SmallPtrSet<const Value *, 8> Visited;
7057 SmallVector<const Value *, 8> Worklist;
7058 Worklist.push_back(Elt: V);
7059 const Value *Object = nullptr;
7060 // Used as fallback if we can't find a common underlying object through
7061 // recursion.
7062 bool First = true;
7063 const Value *FirstObject = getUnderlyingObject(V);
7064 do {
7065 const Value *P = Worklist.pop_back_val();
7066 P = First ? FirstObject : getUnderlyingObject(V: P);
7067 First = false;
7068
7069 if (!Visited.insert(Ptr: P).second)
7070 continue;
7071
7072 if (Visited.size() == MaxVisited)
7073 return FirstObject;
7074
7075 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7076 Worklist.push_back(Elt: SI->getTrueValue());
7077 Worklist.push_back(Elt: SI->getFalseValue());
7078 continue;
7079 }
7080
7081 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7082 append_range(C&: Worklist, R: PN->incoming_values());
7083 continue;
7084 }
7085
7086 if (!Object)
7087 Object = P;
7088 else if (Object != P)
7089 return FirstObject;
7090 } while (!Worklist.empty());
7091
7092 return Object ? Object : FirstObject;
7093}
7094
7095/// This is the function that does the work of looking through basic
7096/// ptrtoint+arithmetic+inttoptr sequences.
7097static const Value *getUnderlyingObjectFromInt(const Value *V) {
7098 do {
7099 if (const Operator *U = dyn_cast<Operator>(Val: V)) {
7100 // If we find a ptrtoint, we can transfer control back to the
7101 // regular getUnderlyingObjectFromInt.
7102 if (U->getOpcode() == Instruction::PtrToInt)
7103 return U->getOperand(i: 0);
7104 // If we find an add of a constant, a multiplied value, or a phi, it's
7105 // likely that the other operand will lead us to the base
7106 // object. We don't have to worry about the case where the
7107 // object address is somehow being computed by the multiply,
7108 // because our callers only care when the result is an
7109 // identifiable object.
7110 if (U->getOpcode() != Instruction::Add ||
7111 (!isa<ConstantInt>(Val: U->getOperand(i: 1)) &&
7112 Operator::getOpcode(V: U->getOperand(i: 1)) != Instruction::Mul &&
7113 !isa<PHINode>(Val: U->getOperand(i: 1))))
7114 return V;
7115 V = U->getOperand(i: 0);
7116 } else {
7117 return V;
7118 }
7119 assert(V->getType()->isIntegerTy() && "Unexpected operand type!");
7120 } while (true);
7121}
7122
7123/// This is a wrapper around getUnderlyingObjects and adds support for basic
7124/// ptrtoint+arithmetic+inttoptr sequences.
7125/// It returns false if unidentified object is found in getUnderlyingObjects.
7126bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
7127 SmallVectorImpl<Value *> &Objects) {
7128 SmallPtrSet<const Value *, 16> Visited;
7129 SmallVector<const Value *, 4> Working(1, V);
7130 do {
7131 V = Working.pop_back_val();
7132
7133 SmallVector<const Value *, 4> Objs;
7134 getUnderlyingObjects(V, Objects&: Objs);
7135
7136 for (const Value *V : Objs) {
7137 if (!Visited.insert(Ptr: V).second)
7138 continue;
7139 if (Operator::getOpcode(V) == Instruction::IntToPtr) {
7140 const Value *O =
7141 getUnderlyingObjectFromInt(V: cast<User>(Val: V)->getOperand(i: 0));
7142 if (O->getType()->isPointerTy()) {
7143 Working.push_back(Elt: O);
7144 continue;
7145 }
7146 }
7147 // If getUnderlyingObjects fails to find an identifiable object,
7148 // getUnderlyingObjectsForCodeGen also fails for safety.
7149 if (!isIdentifiedObject(V)) {
7150 Objects.clear();
7151 return false;
7152 }
7153 Objects.push_back(Elt: const_cast<Value *>(V));
7154 }
7155 } while (!Working.empty());
7156 return true;
7157}
7158
7159AllocaInst *llvm::findAllocaForValue(Value *V, bool OffsetZero) {
7160 AllocaInst *Result = nullptr;
7161 SmallPtrSet<Value *, 4> Visited;
7162 SmallVector<Value *, 4> Worklist;
7163
7164 auto AddWork = [&](Value *V) {
7165 if (Visited.insert(Ptr: V).second)
7166 Worklist.push_back(Elt: V);
7167 };
7168
7169 AddWork(V);
7170 do {
7171 V = Worklist.pop_back_val();
7172 assert(Visited.count(V));
7173
7174 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
7175 if (Result && Result != AI)
7176 return nullptr;
7177 Result = AI;
7178 } else if (CastInst *CI = dyn_cast<CastInst>(Val: V)) {
7179 AddWork(CI->getOperand(i_nocapture: 0));
7180 } else if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7181 for (Value *IncValue : PN->incoming_values())
7182 AddWork(IncValue);
7183 } else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
7184 AddWork(SI->getTrueValue());
7185 AddWork(SI->getFalseValue());
7186 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: V)) {
7187 if (OffsetZero && !GEP->hasAllZeroIndices())
7188 return nullptr;
7189 AddWork(GEP->getPointerOperand());
7190 } else if (CallBase *CB = dyn_cast<CallBase>(Val: V)) {
7191 Value *Returned = CB->getReturnedArgOperand();
7192 if (Returned)
7193 AddWork(Returned);
7194 else
7195 return nullptr;
7196 } else {
7197 return nullptr;
7198 }
7199 } while (!Worklist.empty());
7200
7201 return Result;
7202}
7203
7204static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7205 const Value *V, bool AllowLifetime, bool AllowDroppable) {
7206 for (const User *U : V->users()) {
7207 const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
7208 if (!II)
7209 return false;
7210
7211 if (AllowLifetime && II->isLifetimeStartOrEnd())
7212 continue;
7213
7214 if (AllowDroppable && II->isDroppable())
7215 continue;
7216
7217 return false;
7218 }
7219 return true;
7220}
7221
7222bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
7223 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7224 V, /* AllowLifetime */ true, /* AllowDroppable */ false);
7225}
7226bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) {
7227 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7228 V, /* AllowLifetime */ true, /* AllowDroppable */ true);
7229}
7230
7231bool llvm::isNotCrossLaneOperation(const Instruction *I) {
7232 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
7233 return isTriviallyVectorizable(ID: II->getIntrinsicID());
7234 auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: I);
7235 return (!Shuffle || Shuffle->isSelect()) &&
7236 !isa<CallBase, BitCastInst, ExtractElementInst>(Val: I);
7237}
7238
7239bool llvm::isSafeToSpeculativelyExecute(
7240 const Instruction *Inst, const Instruction *CtxI, AssumptionCache *AC,
7241 const DominatorTree *DT, const TargetLibraryInfo *TLI, bool UseVariableInfo,
7242 bool IgnoreUBImplyingAttrs) {
7243 return isSafeToSpeculativelyExecuteWithOpcode(Opcode: Inst->getOpcode(), Inst, CtxI,
7244 AC, DT, TLI, UseVariableInfo,
7245 IgnoreUBImplyingAttrs);
7246}
7247
7248bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
7249 unsigned Opcode, const Instruction *Inst, const Instruction *CtxI,
7250 AssumptionCache *AC, const DominatorTree *DT, const TargetLibraryInfo *TLI,
7251 bool UseVariableInfo, bool IgnoreUBImplyingAttrs) {
7252#ifndef NDEBUG
7253 if (Inst->getOpcode() != Opcode) {
7254 // Check that the operands are actually compatible with the Opcode override.
7255 auto hasEqualReturnAndLeadingOperandTypes =
7256 [](const Instruction *Inst, unsigned NumLeadingOperands) {
7257 if (Inst->getNumOperands() < NumLeadingOperands)
7258 return false;
7259 const Type *ExpectedType = Inst->getType();
7260 for (unsigned ItOp = 0; ItOp < NumLeadingOperands; ++ItOp)
7261 if (Inst->getOperand(ItOp)->getType() != ExpectedType)
7262 return false;
7263 return true;
7264 };
7265 assert(!Instruction::isBinaryOp(Opcode) ||
7266 hasEqualReturnAndLeadingOperandTypes(Inst, 2));
7267 assert(!Instruction::isUnaryOp(Opcode) ||
7268 hasEqualReturnAndLeadingOperandTypes(Inst, 1));
7269 }
7270#endif
7271
7272 switch (Opcode) {
7273 default:
7274 return true;
7275 case Instruction::UDiv:
7276 case Instruction::URem: {
7277 // x / y is undefined if y == 0.
7278 const APInt *V;
7279 if (match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: V)))
7280 return *V != 0;
7281 return false;
7282 }
7283 case Instruction::SDiv:
7284 case Instruction::SRem: {
7285 // x / y is undefined if y == 0 or x == INT_MIN and y == -1
7286 const APInt *Numerator, *Denominator;
7287 if (!match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: Denominator)))
7288 return false;
7289 // We cannot hoist this division if the denominator is 0.
7290 if (*Denominator == 0)
7291 return false;
7292 // It's safe to hoist if the denominator is not 0 or -1.
7293 if (!Denominator->isAllOnes())
7294 return true;
7295 // At this point we know that the denominator is -1. It is safe to hoist as
7296 // long we know that the numerator is not INT_MIN.
7297 if (match(V: Inst->getOperand(i: 0), P: m_APInt(Res&: Numerator)))
7298 return !Numerator->isMinSignedValue();
7299 // The numerator *might* be MinSignedValue.
7300 return false;
7301 }
7302 case Instruction::Load: {
7303 if (!UseVariableInfo)
7304 return false;
7305
7306 const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst);
7307 if (!LI)
7308 return false;
7309 if (mustSuppressSpeculation(LI: *LI))
7310 return false;
7311 const DataLayout &DL = LI->getDataLayout();
7312 return isDereferenceableAndAlignedPointer(
7313 V: LI->getPointerOperand(), Ty: LI->getType(), Alignment: LI->getAlign(),
7314 Q: SimplifyQuery(DL, TLI, DT, AC, CtxI));
7315 }
7316 case Instruction::Call: {
7317 auto *CI = dyn_cast<const CallInst>(Val: Inst);
7318 if (!CI)
7319 return false;
7320 const Function *Callee = CI->getCalledFunction();
7321
7322 // The called function could have undefined behavior or side-effects, even
7323 // if marked readnone nounwind.
7324 if (!Callee || !Callee->isSpeculatable())
7325 return false;
7326 // Since the operands may be changed after hoisting, undefined behavior may
7327 // be triggered by some UB-implying attributes.
7328 return IgnoreUBImplyingAttrs || !CI->hasUBImplyingAttrs();
7329 }
7330 case Instruction::VAArg:
7331 case Instruction::Alloca:
7332 case Instruction::Invoke:
7333 case Instruction::CallBr:
7334 case Instruction::PHI:
7335 case Instruction::Store:
7336 case Instruction::Ret:
7337 case Instruction::UncondBr:
7338 case Instruction::CondBr:
7339 case Instruction::IndirectBr:
7340 case Instruction::Switch:
7341 case Instruction::Unreachable:
7342 case Instruction::Fence:
7343 case Instruction::AtomicRMW:
7344 case Instruction::AtomicCmpXchg:
7345 case Instruction::LandingPad:
7346 case Instruction::Resume:
7347 case Instruction::CatchSwitch:
7348 case Instruction::CatchPad:
7349 case Instruction::CatchRet:
7350 case Instruction::CleanupPad:
7351 case Instruction::CleanupRet:
7352 return false; // Misc instructions which have effects
7353 }
7354}
7355
7356bool llvm::mayHaveNonDefUseDependency(const Instruction &I) {
7357 if (I.mayReadOrWriteMemory())
7358 // Memory dependency possible
7359 return true;
7360 if (!isSafeToSpeculativelyExecute(Inst: &I))
7361 // Can't move above a maythrow call or infinite loop. Or if an
7362 // inalloca alloca, above a stacksave call.
7363 return true;
7364 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7365 // 1) Can't reorder two inf-loop calls, even if readonly
7366 // 2) Also can't reorder an inf-loop call below a instruction which isn't
7367 // safe to speculative execute. (Inverse of above)
7368 return true;
7369 return false;
7370}
7371
7372/// Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
7373static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) {
7374 switch (OR) {
7375 case ConstantRange::OverflowResult::MayOverflow:
7376 return OverflowResult::MayOverflow;
7377 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
7378 return OverflowResult::AlwaysOverflowsLow;
7379 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
7380 return OverflowResult::AlwaysOverflowsHigh;
7381 case ConstantRange::OverflowResult::NeverOverflows:
7382 return OverflowResult::NeverOverflows;
7383 }
7384 llvm_unreachable("Unknown OverflowResult");
7385}
7386
7387/// Combine constant ranges from computeConstantRange() and computeKnownBits().
7388ConstantRange
7389llvm::computeConstantRangeIncludingKnownBits(const WithCache<const Value *> &V,
7390 bool ForSigned,
7391 const SimplifyQuery &SQ) {
7392 ConstantRange CR1 =
7393 ConstantRange::fromKnownBits(Known: V.getKnownBits(Q: SQ), IsSigned: ForSigned);
7394 ConstantRange CR2 = computeConstantRange(V, ForSigned, SQ);
7395 ConstantRange::PreferredRangeType RangeType =
7396 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
7397 return CR1.intersectWith(CR: CR2, Type: RangeType);
7398}
7399
7400OverflowResult llvm::computeOverflowForUnsignedMul(const Value *LHS,
7401 const Value *RHS,
7402 const SimplifyQuery &SQ,
7403 bool IsNSW) {
7404 ConstantRange LHSRange =
7405 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7406 ConstantRange RHSRange =
7407 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7408
7409 // mul nsw of two non-negative numbers is also nuw.
7410 if (IsNSW && LHSRange.isAllNonNegative() && RHSRange.isAllNonNegative())
7411 return OverflowResult::NeverOverflows;
7412
7413 return mapOverflowResult(OR: LHSRange.unsignedMulMayOverflow(Other: RHSRange));
7414}
7415
7416OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
7417 const Value *RHS,
7418 const SimplifyQuery &SQ) {
7419 // Multiplying n * m significant bits yields a result of n + m significant
7420 // bits. If the total number of significant bits does not exceed the
7421 // result bit width (minus 1), there is no overflow.
7422 // This means if we have enough leading sign bits in the operands
7423 // we can guarantee that the result does not overflow.
7424 // Ref: "Hacker's Delight" by Henry Warren
7425 unsigned BitWidth = LHS->getType()->getScalarSizeInBits();
7426
7427 // Note that underestimating the number of sign bits gives a more
7428 // conservative answer.
7429 unsigned SignBits =
7430 ::ComputeNumSignBits(V: LHS, Q: SQ) + ::ComputeNumSignBits(V: RHS, Q: SQ);
7431
7432 // First handle the easy case: if we have enough sign bits there's
7433 // definitely no overflow.
7434 if (SignBits > BitWidth + 1)
7435 return OverflowResult::NeverOverflows;
7436
7437 // There are two ambiguous cases where there can be no overflow:
7438 // SignBits == BitWidth + 1 and
7439 // SignBits == BitWidth
7440 // The second case is difficult to check, therefore we only handle the
7441 // first case.
7442 if (SignBits == BitWidth + 1) {
7443 // It overflows only when both arguments are negative and the true
7444 // product is exactly the minimum negative number.
7445 // E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
7446 // For simplicity we just check if at least one side is not negative.
7447 KnownBits LHSKnown = computeKnownBits(V: LHS, Q: SQ);
7448 KnownBits RHSKnown = computeKnownBits(V: RHS, Q: SQ);
7449 if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
7450 return OverflowResult::NeverOverflows;
7451 }
7452 return OverflowResult::MayOverflow;
7453}
7454
7455OverflowResult
7456llvm::computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
7457 const WithCache<const Value *> &RHS,
7458 const SimplifyQuery &SQ) {
7459 ConstantRange LHSRange =
7460 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7461 ConstantRange RHSRange =
7462 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7463 return mapOverflowResult(OR: LHSRange.unsignedAddMayOverflow(Other: RHSRange));
7464}
7465
7466static OverflowResult
7467computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
7468 const WithCache<const Value *> &RHS,
7469 const AddOperator *Add, const SimplifyQuery &SQ) {
7470 if (Add && Add->hasNoSignedWrap()) {
7471 return OverflowResult::NeverOverflows;
7472 }
7473
7474 // If LHS and RHS each have at least two sign bits, the addition will look
7475 // like
7476 //
7477 // XX..... +
7478 // YY.....
7479 //
7480 // If the carry into the most significant position is 0, X and Y can't both
7481 // be 1 and therefore the carry out of the addition is also 0.
7482 //
7483 // If the carry into the most significant position is 1, X and Y can't both
7484 // be 0 and therefore the carry out of the addition is also 1.
7485 //
7486 // Since the carry into the most significant position is always equal to
7487 // the carry out of the addition, there is no signed overflow.
7488 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7489 return OverflowResult::NeverOverflows;
7490
7491 ConstantRange LHSRange =
7492 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7493 ConstantRange RHSRange =
7494 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7495 OverflowResult OR =
7496 mapOverflowResult(OR: LHSRange.signedAddMayOverflow(Other: RHSRange));
7497 if (OR != OverflowResult::MayOverflow)
7498 return OR;
7499
7500 // The remaining code needs Add to be available. Early returns if not so.
7501 if (!Add)
7502 return OverflowResult::MayOverflow;
7503
7504 // If the sign of Add is the same as at least one of the operands, this add
7505 // CANNOT overflow. If this can be determined from the known bits of the
7506 // operands the above signedAddMayOverflow() check will have already done so.
7507 // The only other way to improve on the known bits is from an assumption, so
7508 // call computeKnownBitsFromContext() directly.
7509 bool LHSOrRHSKnownNonNegative =
7510 (LHSRange.isAllNonNegative() || RHSRange.isAllNonNegative());
7511 bool LHSOrRHSKnownNegative =
7512 (LHSRange.isAllNegative() || RHSRange.isAllNegative());
7513 if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) {
7514 KnownBits AddKnown(LHSRange.getBitWidth());
7515 computeKnownBitsFromContext(V: Add, Known&: AddKnown, Q: SQ);
7516 if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) ||
7517 (AddKnown.isNegative() && LHSOrRHSKnownNegative))
7518 return OverflowResult::NeverOverflows;
7519 }
7520
7521 return OverflowResult::MayOverflow;
7522}
7523
7524OverflowResult llvm::computeOverflowForUnsignedSub(const Value *LHS,
7525 const Value *RHS,
7526 const SimplifyQuery &SQ) {
7527 // X - (X % ?)
7528 // The remainder of a value can't have greater magnitude than itself,
7529 // so the subtraction can't overflow.
7530
7531 // X - (X -nuw ?)
7532 // In the minimal case, this would simplify to "?", so there's no subtract
7533 // at all. But if this analysis is used to peek through casts, for example,
7534 // then determining no-overflow may allow other transforms.
7535
7536 // TODO: There are other patterns like this.
7537 // See simplifyICmpWithBinOpOnLHS() for candidates.
7538 if (match(V: RHS, P: m_URem(L: m_Specific(V: LHS), R: m_Value())) ||
7539 match(V: RHS, P: m_NUWSub(L: m_Specific(V: LHS), R: m_Value())))
7540 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
7541 return OverflowResult::NeverOverflows;
7542
7543 if (auto C = isImpliedByDomCondition(Pred: CmpInst::ICMP_UGE, LHS, RHS, ContextI: SQ.CxtI,
7544 DL: SQ.DL)) {
7545 if (*C)
7546 return OverflowResult::NeverOverflows;
7547 return OverflowResult::AlwaysOverflowsLow;
7548 }
7549
7550 ConstantRange LHSRange =
7551 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7552 ConstantRange RHSRange =
7553 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7554 return mapOverflowResult(OR: LHSRange.unsignedSubMayOverflow(Other: RHSRange));
7555}
7556
7557OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS,
7558 const Value *RHS,
7559 const SimplifyQuery &SQ) {
7560 // X - (X % ?)
7561 // The remainder of a value can't have greater magnitude than itself,
7562 // so the subtraction can't overflow.
7563
7564 // X - (X -nsw ?)
7565 // In the minimal case, this would simplify to "?", so there's no subtract
7566 // at all. But if this analysis is used to peek through casts, for example,
7567 // then determining no-overflow may allow other transforms.
7568 if (match(V: RHS, P: m_SRem(L: m_Specific(V: LHS), R: m_Value())) ||
7569 match(V: RHS, P: m_NSWSub(L: m_Specific(V: LHS), R: m_Value())))
7570 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
7571 return OverflowResult::NeverOverflows;
7572
7573 // If LHS and RHS each have at least two sign bits, the subtraction
7574 // cannot overflow.
7575 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7576 return OverflowResult::NeverOverflows;
7577
7578 ConstantRange LHSRange =
7579 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7580 ConstantRange RHSRange =
7581 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7582 return mapOverflowResult(OR: LHSRange.signedSubMayOverflow(Other: RHSRange));
7583}
7584
7585bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO,
7586 const DominatorTree &DT) {
7587 SmallVector<const CondBrInst *, 2> GuardingBranches;
7588 SmallVector<const ExtractValueInst *, 2> Results;
7589
7590 for (const User *U : WO->users()) {
7591 if (const auto *EVI = dyn_cast<ExtractValueInst>(Val: U)) {
7592 assert(EVI->getNumIndices() == 1 && "Obvious from CI's type");
7593
7594 if (EVI->getIndices()[0] == 0)
7595 Results.push_back(Elt: EVI);
7596 else {
7597 assert(EVI->getIndices()[0] == 1 && "Obvious from CI's type");
7598
7599 for (const auto *U : EVI->users())
7600 if (const auto *B = dyn_cast<CondBrInst>(Val: U))
7601 GuardingBranches.push_back(Elt: B);
7602 }
7603 } else {
7604 // We are using the aggregate directly in a way we don't want to analyze
7605 // here (storing it to a global, say).
7606 return false;
7607 }
7608 }
7609
7610 auto AllUsesGuardedByBranch = [&](const CondBrInst *BI) {
7611 BasicBlockEdge NoWrapEdge(BI->getParent(), BI->getSuccessor(i: 1));
7612
7613 // Check if all users of the add are provably no-wrap.
7614 for (const auto *Result : Results) {
7615 // If the extractvalue itself is not executed on overflow, the we don't
7616 // need to check each use separately, since domination is transitive.
7617 if (DT.dominates(BBE: NoWrapEdge, BB: Result->getParent()))
7618 continue;
7619
7620 for (const auto &RU : Result->uses())
7621 if (!DT.dominates(BBE: NoWrapEdge, U: RU))
7622 return false;
7623 }
7624
7625 return true;
7626 };
7627
7628 return llvm::any_of(Range&: GuardingBranches, P: AllUsesGuardedByBranch);
7629}
7630
7631/// Shifts return poison if shiftwidth is larger than the bitwidth.
7632static bool shiftAmountKnownInRange(const Value *ShiftAmount) {
7633 auto *C = dyn_cast<Constant>(Val: ShiftAmount);
7634 if (!C)
7635 return false;
7636
7637 // Shifts return poison if shiftwidth is larger than the bitwidth.
7638 SmallVector<const Constant *, 4> ShiftAmounts;
7639 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: C->getType())) {
7640 unsigned NumElts = FVTy->getNumElements();
7641 for (unsigned i = 0; i < NumElts; ++i)
7642 ShiftAmounts.push_back(Elt: C->getAggregateElement(Elt: i));
7643 } else if (isa<ScalableVectorType>(Val: C->getType()))
7644 return false; // Can't tell, just return false to be safe
7645 else
7646 ShiftAmounts.push_back(Elt: C);
7647
7648 bool Safe = llvm::all_of(Range&: ShiftAmounts, P: [](const Constant *C) {
7649 auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
7650 return CI && CI->getValue().ult(RHS: C->getType()->getIntegerBitWidth());
7651 });
7652
7653 return Safe;
7654}
7655
7656static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
7657 bool ConsiderFlagsAndMetadata) {
7658
7659 if (ConsiderFlagsAndMetadata && includesPoison(Kind) &&
7660 Op->hasPoisonGeneratingAnnotations())
7661 return true;
7662
7663 unsigned Opcode = Op->getOpcode();
7664
7665 // Check whether opcode is a poison/undef-generating operation
7666 switch (Opcode) {
7667 case Instruction::Shl:
7668 case Instruction::AShr:
7669 case Instruction::LShr:
7670 return includesPoison(Kind) && !shiftAmountKnownInRange(ShiftAmount: Op->getOperand(i: 1));
7671 case Instruction::FPToSI:
7672 case Instruction::FPToUI:
7673 // fptosi/ui yields poison if the resulting value does not fit in the
7674 // destination type.
7675 return true;
7676 case Instruction::Call:
7677 if (auto *II = dyn_cast<IntrinsicInst>(Val: Op)) {
7678 switch (II->getIntrinsicID()) {
7679 // NOTE: Use IntrNoCreateUndefOrPoison when possible.
7680 case Intrinsic::ctlz:
7681 case Intrinsic::cttz:
7682 case Intrinsic::abs:
7683 // We're not considering flags so it is safe to just return false.
7684 return false;
7685 case Intrinsic::sshl_sat:
7686 case Intrinsic::ushl_sat:
7687 if (!includesPoison(Kind) ||
7688 shiftAmountKnownInRange(ShiftAmount: II->getArgOperand(i: 1)))
7689 return false;
7690 break;
7691 }
7692 }
7693 [[fallthrough]];
7694 case Instruction::CallBr:
7695 case Instruction::Invoke: {
7696 const auto *CB = cast<CallBase>(Val: Op);
7697 return !CB->hasRetAttr(Kind: Attribute::NoUndef) &&
7698 !CB->hasFnAttr(Kind: Attribute::NoCreateUndefOrPoison);
7699 }
7700 case Instruction::InsertElement:
7701 case Instruction::ExtractElement: {
7702 // If index exceeds the length of the vector, it returns poison
7703 auto *VTy = cast<VectorType>(Val: Op->getOperand(i: 0)->getType());
7704 unsigned IdxOp = Op->getOpcode() == Instruction::InsertElement ? 2 : 1;
7705 auto *Idx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: IdxOp));
7706 if (includesPoison(Kind))
7707 return !Idx ||
7708 Idx->getValue().uge(RHS: VTy->getElementCount().getKnownMinValue());
7709 return false;
7710 }
7711 case Instruction::ShuffleVector: {
7712 ArrayRef<int> Mask = isa<ConstantExpr>(Val: Op)
7713 ? cast<ConstantExpr>(Val: Op)->getShuffleMask()
7714 : cast<ShuffleVectorInst>(Val: Op)->getShuffleMask();
7715 return includesPoison(Kind) && is_contained(Range&: Mask, Element: PoisonMaskElem);
7716 }
7717 case Instruction::FNeg:
7718 case Instruction::PHI:
7719 case Instruction::Select:
7720 case Instruction::ExtractValue:
7721 case Instruction::InsertValue:
7722 case Instruction::Freeze:
7723 case Instruction::ICmp:
7724 case Instruction::FCmp:
7725 case Instruction::GetElementPtr:
7726 return false;
7727 case Instruction::AddrSpaceCast:
7728 return true;
7729 default: {
7730 const auto *CE = dyn_cast<ConstantExpr>(Val: Op);
7731 if (isa<CastInst>(Val: Op) || (CE && CE->isCast()))
7732 return false;
7733 else if (Instruction::isBinaryOp(Opcode))
7734 return false;
7735 // Be conservative and return true.
7736 return true;
7737 }
7738 }
7739}
7740
7741bool llvm::canCreateUndefOrPoison(const Operator *Op,
7742 bool ConsiderFlagsAndMetadata) {
7743 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::UndefOrPoison,
7744 ConsiderFlagsAndMetadata);
7745}
7746
7747bool llvm::canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata) {
7748 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::PoisonOnly,
7749 ConsiderFlagsAndMetadata);
7750}
7751
7752static bool directlyImpliesPoison(const Value *ValAssumedPoison, const Value *V,
7753 unsigned Depth) {
7754 if (ValAssumedPoison == V)
7755 return true;
7756
7757 const unsigned MaxDepth = 2;
7758 if (Depth >= MaxDepth)
7759 return false;
7760
7761 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
7762 if (any_of(Range: I->operands(), P: [=](const Use &Op) {
7763 return propagatesPoison(PoisonOp: Op) &&
7764 directlyImpliesPoison(ValAssumedPoison, V: Op, Depth: Depth + 1);
7765 }))
7766 return true;
7767
7768 // V = extractvalue V0, idx
7769 // V2 = extractvalue V0, idx2
7770 // V0's elements are all poison or not. (e.g., add_with_overflow)
7771 const WithOverflowInst *II;
7772 if (match(V: I, P: m_ExtractValue(V: m_WithOverflowInst(I&: II))) &&
7773 (match(V: ValAssumedPoison, P: m_ExtractValue(V: m_Specific(V: II))) ||
7774 llvm::is_contained(Range: II->args(), Element: ValAssumedPoison)))
7775 return true;
7776 }
7777 return false;
7778}
7779
7780static bool impliesPoison(const Value *ValAssumedPoison, const Value *V,
7781 unsigned Depth) {
7782 if (isGuaranteedNotToBePoison(V: ValAssumedPoison))
7783 return true;
7784
7785 if (directlyImpliesPoison(ValAssumedPoison, V, /* Depth */ 0))
7786 return true;
7787
7788 const unsigned MaxDepth = 2;
7789 if (Depth >= MaxDepth)
7790 return false;
7791
7792 const auto *I = dyn_cast<Instruction>(Val: ValAssumedPoison);
7793 if (I && !canCreatePoison(Op: cast<Operator>(Val: I))) {
7794 return all_of(Range: I->operands(), P: [=](const Value *Op) {
7795 return impliesPoison(ValAssumedPoison: Op, V, Depth: Depth + 1);
7796 });
7797 }
7798 return false;
7799}
7800
7801bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
7802 return ::impliesPoison(ValAssumedPoison, V, /* Depth */ 0);
7803}
7804
7805static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
7806
7807static bool isGuaranteedNotToBeUndefOrPoison(
7808 const Value *V, AssumptionCache *AC, const Instruction *CtxI,
7809 const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
7810 if (Depth >= MaxAnalysisRecursionDepth)
7811 return false;
7812
7813 if (isa<MetadataAsValue>(Val: V))
7814 return false;
7815
7816 if (const auto *A = dyn_cast<Argument>(Val: V)) {
7817 if (A->hasAttribute(Kind: Attribute::NoUndef) ||
7818 A->hasAttribute(Kind: Attribute::Dereferenceable) ||
7819 A->hasAttribute(Kind: Attribute::DereferenceableOrNull))
7820 return true;
7821 }
7822
7823 if (auto *C = dyn_cast<Constant>(Val: V)) {
7824 if (isa<PoisonValue>(Val: C))
7825 return !includesPoison(Kind);
7826
7827 if (isa<UndefValue>(Val: C))
7828 return !includesUndef(Kind);
7829
7830 if (isa<ConstantInt>(Val: C) || isa<GlobalVariable>(Val: C) || isa<ConstantFP>(Val: C) ||
7831 isa<ConstantPointerNull>(Val: C) || isa<Function>(Val: C))
7832 return true;
7833
7834 if (C->getType()->isVectorTy()) {
7835 if (isa<ConstantExpr>(Val: C)) {
7836 // Scalable vectors can use a ConstantExpr to build a splat.
7837 if (Constant *SplatC = C->getSplatValue())
7838 if (isa<ConstantInt>(Val: SplatC) || isa<ConstantFP>(Val: SplatC))
7839 return true;
7840 } else {
7841 if (includesUndef(Kind) && C->containsUndefElement())
7842 return false;
7843 if (includesPoison(Kind) && C->containsPoisonElement())
7844 return false;
7845 return !C->containsConstantExpression();
7846 }
7847 }
7848 }
7849
7850 // Strip cast operations from a pointer value.
7851 // Note that stripPointerCastsSameRepresentation can strip off getelementptr
7852 // inbounds with zero offset. To guarantee that the result isn't poison, the
7853 // stripped pointer is checked as it has to be pointing into an allocated
7854 // object or be null `null` to ensure `inbounds` getelement pointers with a
7855 // zero offset could not produce poison.
7856 // It can strip off addrspacecast that do not change bit representation as
7857 // well. We believe that such addrspacecast is equivalent to no-op.
7858 auto *StrippedV = V->stripPointerCastsSameRepresentation();
7859 if (isa<AllocaInst>(Val: StrippedV) || isa<GlobalVariable>(Val: StrippedV) ||
7860 isa<Function>(Val: StrippedV) || isa<ConstantPointerNull>(Val: StrippedV))
7861 return true;
7862
7863 auto OpCheck = [&](const Value *V) {
7864 return isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth: Depth + 1, Kind);
7865 };
7866
7867 if (auto *Opr = dyn_cast<Operator>(Val: V)) {
7868 // If the value is a freeze instruction, then it can never
7869 // be undef or poison.
7870 if (isa<FreezeInst>(Val: V))
7871 return true;
7872
7873 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
7874 if (CB->hasRetAttr(Kind: Attribute::NoUndef) ||
7875 CB->hasRetAttr(Kind: Attribute::Dereferenceable) ||
7876 CB->hasRetAttr(Kind: Attribute::DereferenceableOrNull))
7877 return true;
7878 }
7879
7880 if (!::canCreateUndefOrPoison(Op: Opr, Kind,
7881 /*ConsiderFlagsAndMetadata=*/true)) {
7882 if (const auto *PN = dyn_cast<PHINode>(Val: V)) {
7883 unsigned Num = PN->getNumIncomingValues();
7884 bool IsWellDefined = true;
7885 for (unsigned i = 0; i < Num; ++i) {
7886 if (PN == PN->getIncomingValue(i))
7887 continue;
7888 auto *TI = PN->getIncomingBlock(i)->getTerminator();
7889 if (!isGuaranteedNotToBeUndefOrPoison(V: PN->getIncomingValue(i), AC, CtxI: TI,
7890 DT, Depth: Depth + 1, Kind)) {
7891 IsWellDefined = false;
7892 break;
7893 }
7894 }
7895 if (IsWellDefined)
7896 return true;
7897 } else if (auto *Splat = isa<ShuffleVectorInst>(Val: Opr) ? getSplatValue(V: Opr)
7898 : nullptr) {
7899 // For splats we only need to check the value being splatted.
7900 if (OpCheck(Splat))
7901 return true;
7902 } else if (all_of(Range: Opr->operands(), P: OpCheck))
7903 return true;
7904 }
7905 }
7906
7907 if (auto *I = dyn_cast<LoadInst>(Val: V))
7908 if (I->hasMetadata(KindID: LLVMContext::MD_noundef) ||
7909 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable) ||
7910 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable_or_null))
7911 return true;
7912
7913 if (programUndefinedIfUndefOrPoison(V, PoisonOnly: !includesUndef(Kind)))
7914 return true;
7915
7916 // CxtI may be null or a cloned instruction.
7917 if (!CtxI || !CtxI->getParent() || !DT)
7918 return false;
7919
7920 auto *DNode = DT->getNode(BB: CtxI->getParent());
7921 if (!DNode)
7922 // Unreachable block
7923 return false;
7924
7925 // If V is used as a branch condition before reaching CtxI, V cannot be
7926 // undef or poison.
7927 // br V, BB1, BB2
7928 // BB1:
7929 // CtxI ; V cannot be undef or poison here
7930 auto *Dominator = DNode->getIDom();
7931 // This check is purely for compile time reasons: we can skip the IDom walk
7932 // if what we are checking for includes undef and the value is not an integer.
7933 if (!includesUndef(Kind) || V->getType()->isIntegerTy())
7934 while (Dominator) {
7935 auto *TI = Dominator->getBlock()->getTerminatorOrNull();
7936
7937 Value *Cond = nullptr;
7938 if (auto BI = dyn_cast_or_null<CondBrInst>(Val: TI)) {
7939 Cond = BI->getCondition();
7940 } else if (auto SI = dyn_cast_or_null<SwitchInst>(Val: TI)) {
7941 Cond = SI->getCondition();
7942 }
7943
7944 if (Cond) {
7945 if (Cond == V)
7946 return true;
7947 else if (!includesUndef(Kind) && isa<Operator>(Val: Cond)) {
7948 // For poison, we can analyze further
7949 auto *Opr = cast<Operator>(Val: Cond);
7950 if (any_of(Range: Opr->operands(), P: [V](const Use &U) {
7951 return V == U && propagatesPoison(PoisonOp: U);
7952 }))
7953 return true;
7954 }
7955 }
7956
7957 Dominator = Dominator->getIDom();
7958 }
7959
7960 if (AC && getKnowledgeValidInContext(V, AttrKinds: {Attribute::NoUndef}, AC&: *AC, CtxI, DT))
7961 return true;
7962
7963 return false;
7964}
7965
7966bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC,
7967 const Instruction *CtxI,
7968 const DominatorTree *DT,
7969 unsigned Depth) {
7970 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
7971 Kind: UndefPoisonKind::UndefOrPoison);
7972}
7973
7974bool llvm::isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC,
7975 const Instruction *CtxI,
7976 const DominatorTree *DT, unsigned Depth) {
7977 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
7978 Kind: UndefPoisonKind::PoisonOnly);
7979}
7980
7981bool llvm::isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC,
7982 const Instruction *CtxI,
7983 const DominatorTree *DT, unsigned Depth) {
7984 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
7985 Kind: UndefPoisonKind::UndefOnly);
7986}
7987
7988/// Return true if undefined behavior would provably be executed on the path to
7989/// OnPathTo if Root produced a posion result. Note that this doesn't say
7990/// anything about whether OnPathTo is actually executed or whether Root is
7991/// actually poison. This can be used to assess whether a new use of Root can
7992/// be added at a location which is control equivalent with OnPathTo (such as
7993/// immediately before it) without introducing UB which didn't previously
7994/// exist. Note that a false result conveys no information.
7995bool llvm::mustExecuteUBIfPoisonOnPathTo(Instruction *Root,
7996 Instruction *OnPathTo,
7997 DominatorTree *DT) {
7998 // Basic approach is to assume Root is poison, propagate poison forward
7999 // through all users we can easily track, and then check whether any of those
8000 // users are provable UB and must execute before out exiting block might
8001 // exit.
8002
8003 // The set of all recursive users we've visited (which are assumed to all be
8004 // poison because of said visit)
8005 SmallPtrSet<const Value *, 16> KnownPoison;
8006 SmallVector<const Instruction*, 16> Worklist;
8007 Worklist.push_back(Elt: Root);
8008 while (!Worklist.empty()) {
8009 const Instruction *I = Worklist.pop_back_val();
8010
8011 // If we know this must trigger UB on a path leading our target.
8012 if (mustTriggerUB(I, KnownPoison) && DT->dominates(Def: I, User: OnPathTo))
8013 return true;
8014
8015 // If we can't analyze propagation through this instruction, just skip it
8016 // and transitive users. Safe as false is a conservative result.
8017 if (I != Root && !any_of(Range: I->operands(), P: [&KnownPoison](const Use &U) {
8018 return KnownPoison.contains(Ptr: U) && propagatesPoison(PoisonOp: U);
8019 }))
8020 continue;
8021
8022 if (KnownPoison.insert(Ptr: I).second)
8023 for (const User *User : I->users())
8024 Worklist.push_back(Elt: cast<Instruction>(Val: User));
8025 }
8026
8027 // Might be non-UB, or might have a path we couldn't prove must execute on
8028 // way to exiting bb.
8029 return false;
8030}
8031
8032OverflowResult llvm::computeOverflowForSignedAdd(const AddOperator *Add,
8033 const SimplifyQuery &SQ) {
8034 return ::computeOverflowForSignedAdd(LHS: Add->getOperand(i_nocapture: 0), RHS: Add->getOperand(i_nocapture: 1),
8035 Add, SQ);
8036}
8037
8038OverflowResult
8039llvm::computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
8040 const WithCache<const Value *> &RHS,
8041 const SimplifyQuery &SQ) {
8042 return ::computeOverflowForSignedAdd(LHS, RHS, Add: nullptr, SQ);
8043}
8044
8045bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
8046 // Note: An atomic operation isn't guaranteed to return in a reasonable amount
8047 // of time because it's possible for another thread to interfere with it for an
8048 // arbitrary length of time, but programs aren't allowed to rely on that.
8049
8050 // If there is no successor, then execution can't transfer to it.
8051 if (isa<ReturnInst>(Val: I))
8052 return false;
8053 if (isa<UnreachableInst>(Val: I))
8054 return false;
8055
8056 // Note: Do not add new checks here; instead, change Instruction::mayThrow or
8057 // Instruction::willReturn.
8058 //
8059 // FIXME: Move this check into Instruction::willReturn.
8060 if (isa<CatchPadInst>(Val: I)) {
8061 switch (classifyEHPersonality(Pers: I->getFunction()->getPersonalityFn())) {
8062 default:
8063 // A catchpad may invoke exception object constructors and such, which
8064 // in some languages can be arbitrary code, so be conservative by default.
8065 return false;
8066 case EHPersonality::CoreCLR:
8067 // For CoreCLR, it just involves a type test.
8068 return true;
8069 }
8070 }
8071
8072 // An instruction that returns without throwing must transfer control flow
8073 // to a successor.
8074 return !I->mayThrow() && I->willReturn();
8075}
8076
8077bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) {
8078 // TODO: This is slightly conservative for invoke instruction since exiting
8079 // via an exception *is* normal control for them.
8080 for (const Instruction &I : *BB)
8081 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8082 return false;
8083 return true;
8084}
8085
8086bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8087 BasicBlock::const_iterator Begin, BasicBlock::const_iterator End,
8088 unsigned ScanLimit) {
8089 return isGuaranteedToTransferExecutionToSuccessor(Range: make_range(x: Begin, y: End),
8090 ScanLimit);
8091}
8092
8093bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8094 iterator_range<BasicBlock::const_iterator> Range, unsigned ScanLimit) {
8095 assert(ScanLimit && "scan limit must be non-zero");
8096 for (const Instruction &I : Range) {
8097 if (--ScanLimit == 0)
8098 return false;
8099 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8100 return false;
8101 }
8102 return true;
8103}
8104
8105bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
8106 const Loop *L) {
8107 // The loop header is guaranteed to be executed for every iteration.
8108 //
8109 // FIXME: Relax this constraint to cover all basic blocks that are
8110 // guaranteed to be executed at every iteration.
8111 if (I->getParent() != L->getHeader()) return false;
8112
8113 for (const Instruction &LI : *L->getHeader()) {
8114 if (&LI == I) return true;
8115 if (!isGuaranteedToTransferExecutionToSuccessor(I: &LI)) return false;
8116 }
8117 llvm_unreachable("Instruction not contained in its own parent basic block.");
8118}
8119
8120bool llvm::intrinsicPropagatesPoison(Intrinsic::ID IID) {
8121 switch (IID) {
8122 // TODO: Add more intrinsics.
8123 case Intrinsic::sadd_with_overflow:
8124 case Intrinsic::ssub_with_overflow:
8125 case Intrinsic::smul_with_overflow:
8126 case Intrinsic::uadd_with_overflow:
8127 case Intrinsic::usub_with_overflow:
8128 case Intrinsic::umul_with_overflow:
8129 // If an input is a vector containing a poison element, the
8130 // two output vectors (calculated results, overflow bits)'
8131 // corresponding lanes are poison.
8132 return true;
8133 case Intrinsic::ctpop:
8134 case Intrinsic::ctlz:
8135 case Intrinsic::cttz:
8136 case Intrinsic::abs:
8137 case Intrinsic::smax:
8138 case Intrinsic::smin:
8139 case Intrinsic::umax:
8140 case Intrinsic::umin:
8141 case Intrinsic::scmp:
8142 case Intrinsic::is_fpclass:
8143 case Intrinsic::ptrmask:
8144 case Intrinsic::ucmp:
8145 case Intrinsic::bitreverse:
8146 case Intrinsic::bswap:
8147 case Intrinsic::sadd_sat:
8148 case Intrinsic::ssub_sat:
8149 case Intrinsic::sshl_sat:
8150 case Intrinsic::uadd_sat:
8151 case Intrinsic::usub_sat:
8152 case Intrinsic::ushl_sat:
8153 case Intrinsic::smul_fix:
8154 case Intrinsic::smul_fix_sat:
8155 case Intrinsic::umul_fix:
8156 case Intrinsic::umul_fix_sat:
8157 case Intrinsic::pow:
8158 case Intrinsic::powi:
8159 case Intrinsic::sin:
8160 case Intrinsic::sinh:
8161 case Intrinsic::cos:
8162 case Intrinsic::cosh:
8163 case Intrinsic::sincos:
8164 case Intrinsic::sincospi:
8165 case Intrinsic::tan:
8166 case Intrinsic::tanh:
8167 case Intrinsic::asin:
8168 case Intrinsic::acos:
8169 case Intrinsic::atan:
8170 case Intrinsic::atan2:
8171 case Intrinsic::canonicalize:
8172 case Intrinsic::sqrt:
8173 case Intrinsic::exp:
8174 case Intrinsic::exp2:
8175 case Intrinsic::exp10:
8176 case Intrinsic::log:
8177 case Intrinsic::log2:
8178 case Intrinsic::log10:
8179 case Intrinsic::modf:
8180 case Intrinsic::floor:
8181 case Intrinsic::ceil:
8182 case Intrinsic::trunc:
8183 case Intrinsic::rint:
8184 case Intrinsic::nearbyint:
8185 case Intrinsic::round:
8186 case Intrinsic::roundeven:
8187 case Intrinsic::lrint:
8188 case Intrinsic::llrint:
8189 case Intrinsic::fshl:
8190 case Intrinsic::fshr:
8191 case Intrinsic::frexp:
8192 case Intrinsic::get_active_lane_mask:
8193 return true;
8194 default:
8195 return false;
8196 }
8197}
8198
8199bool llvm::propagatesPoison(const Use &PoisonOp) {
8200 const Operator *I = cast<Operator>(Val: PoisonOp.getUser());
8201 switch (I->getOpcode()) {
8202 case Instruction::Freeze:
8203 case Instruction::PHI:
8204 case Instruction::Invoke:
8205 return false;
8206 case Instruction::Select:
8207 return PoisonOp.getOperandNo() == 0;
8208 case Instruction::Call:
8209 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
8210 return intrinsicPropagatesPoison(IID: II->getIntrinsicID());
8211 return false;
8212 case Instruction::ICmp:
8213 case Instruction::FCmp:
8214 case Instruction::GetElementPtr:
8215 return true;
8216 default:
8217 if (isa<BinaryOperator>(Val: I) || isa<UnaryOperator>(Val: I) || isa<CastInst>(Val: I))
8218 return true;
8219
8220 // Be conservative and return false.
8221 return false;
8222 }
8223}
8224
8225/// Enumerates all operands of \p I that are guaranteed to not be undef or
8226/// poison. If the callback \p Handle returns true, stop processing and return
8227/// true. Otherwise, return false.
8228template <typename CallableT>
8229static bool handleGuaranteedWellDefinedOps(const Instruction *I,
8230 const CallableT &Handle) {
8231 switch (I->getOpcode()) {
8232 case Instruction::Store:
8233 if (Handle(cast<StoreInst>(Val: I)->getPointerOperand()))
8234 return true;
8235 break;
8236
8237 case Instruction::Load:
8238 if (Handle(cast<LoadInst>(Val: I)->getPointerOperand()))
8239 return true;
8240 break;
8241
8242 // Since dereferenceable attribute imply noundef, atomic operations
8243 // also implicitly have noundef pointers too
8244 case Instruction::AtomicCmpXchg:
8245 if (Handle(cast<AtomicCmpXchgInst>(Val: I)->getPointerOperand()))
8246 return true;
8247 break;
8248
8249 case Instruction::AtomicRMW:
8250 if (Handle(cast<AtomicRMWInst>(Val: I)->getPointerOperand()))
8251 return true;
8252 break;
8253
8254 case Instruction::Call:
8255 case Instruction::Invoke: {
8256 const CallBase *CB = cast<CallBase>(Val: I);
8257 if (CB->isIndirectCall() && Handle(CB->getCalledOperand()))
8258 return true;
8259 for (unsigned i = 0; i < CB->arg_size(); ++i)
8260 if ((CB->paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef) ||
8261 CB->paramHasAttr(ArgNo: i, Kind: Attribute::Dereferenceable) ||
8262 CB->paramHasAttr(ArgNo: i, Kind: Attribute::DereferenceableOrNull)) &&
8263 Handle(CB->getArgOperand(i)))
8264 return true;
8265 break;
8266 }
8267 case Instruction::Ret:
8268 if (I->getFunction()->hasRetAttribute(Kind: Attribute::NoUndef) &&
8269 Handle(I->getOperand(i: 0)))
8270 return true;
8271 break;
8272 case Instruction::Switch:
8273 if (Handle(cast<SwitchInst>(Val: I)->getCondition()))
8274 return true;
8275 break;
8276 case Instruction::CondBr:
8277 if (Handle(cast<CondBrInst>(Val: I)->getCondition()))
8278 return true;
8279 break;
8280 default:
8281 break;
8282 }
8283
8284 return false;
8285}
8286
8287/// Enumerates all operands of \p I that are guaranteed to not be poison.
8288template <typename CallableT>
8289static bool handleGuaranteedNonPoisonOps(const Instruction *I,
8290 const CallableT &Handle) {
8291 if (handleGuaranteedWellDefinedOps(I, Handle))
8292 return true;
8293 switch (I->getOpcode()) {
8294 // Divisors of these operations are allowed to be partially undef.
8295 case Instruction::UDiv:
8296 case Instruction::SDiv:
8297 case Instruction::URem:
8298 case Instruction::SRem:
8299 return Handle(I->getOperand(i: 1));
8300 default:
8301 return false;
8302 }
8303}
8304
8305bool llvm::mustTriggerUB(const Instruction *I,
8306 const SmallPtrSetImpl<const Value *> &KnownPoison) {
8307 return handleGuaranteedNonPoisonOps(
8308 I, Handle: [&](const Value *V) { return KnownPoison.count(Ptr: V); });
8309}
8310
8311static bool programUndefinedIfUndefOrPoison(const Value *V,
8312 bool PoisonOnly) {
8313 // We currently only look for uses of values within the same basic
8314 // block, as that makes it easier to guarantee that the uses will be
8315 // executed given that Inst is executed.
8316 //
8317 // FIXME: Expand this to consider uses beyond the same basic block. To do
8318 // this, look out for the distinction between post-dominance and strong
8319 // post-dominance.
8320 const BasicBlock *BB = nullptr;
8321 BasicBlock::const_iterator Begin;
8322 if (const auto *Inst = dyn_cast<Instruction>(Val: V)) {
8323 BB = Inst->getParent();
8324 Begin = Inst->getIterator();
8325 Begin++;
8326 } else if (const auto *Arg = dyn_cast<Argument>(Val: V)) {
8327 if (Arg->getParent()->isDeclaration())
8328 return false;
8329 BB = &Arg->getParent()->getEntryBlock();
8330 Begin = BB->begin();
8331 } else {
8332 return false;
8333 }
8334
8335 // Limit number of instructions we look at, to avoid scanning through large
8336 // blocks. The current limit is chosen arbitrarily.
8337 unsigned ScanLimit = 32;
8338 BasicBlock::const_iterator End = BB->end();
8339
8340 if (!PoisonOnly) {
8341 // Since undef does not propagate eagerly, be conservative & just check
8342 // whether a value is directly passed to an instruction that must take
8343 // well-defined operands.
8344
8345 for (const auto &I : make_range(x: Begin, y: End)) {
8346 if (--ScanLimit == 0)
8347 break;
8348
8349 if (handleGuaranteedWellDefinedOps(I: &I, Handle: [V](const Value *WellDefinedOp) {
8350 return WellDefinedOp == V;
8351 }))
8352 return true;
8353
8354 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8355 break;
8356 }
8357 return false;
8358 }
8359
8360 // Set of instructions that we have proved will yield poison if Inst
8361 // does.
8362 SmallPtrSet<const Value *, 16> YieldsPoison;
8363 SmallPtrSet<const BasicBlock *, 4> Visited;
8364
8365 YieldsPoison.insert(Ptr: V);
8366 Visited.insert(Ptr: BB);
8367
8368 while (true) {
8369 for (const auto &I : make_range(x: Begin, y: End)) {
8370 if (--ScanLimit == 0)
8371 return false;
8372 if (mustTriggerUB(I: &I, KnownPoison: YieldsPoison))
8373 return true;
8374 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8375 return false;
8376
8377 // If an operand is poison and propagates it, mark I as yielding poison.
8378 for (const Use &Op : I.operands()) {
8379 if (YieldsPoison.count(Ptr: Op) && propagatesPoison(PoisonOp: Op)) {
8380 YieldsPoison.insert(Ptr: &I);
8381 break;
8382 }
8383 }
8384
8385 // Special handling for select, which returns poison if its operand 0 is
8386 // poison (handled in the loop above) *or* if both its true/false operands
8387 // are poison (handled here).
8388 if (I.getOpcode() == Instruction::Select &&
8389 YieldsPoison.count(Ptr: I.getOperand(i: 1)) &&
8390 YieldsPoison.count(Ptr: I.getOperand(i: 2))) {
8391 YieldsPoison.insert(Ptr: &I);
8392 }
8393 }
8394
8395 BB = BB->getSingleSuccessor();
8396 if (!BB || !Visited.insert(Ptr: BB).second)
8397 break;
8398
8399 Begin = BB->getFirstNonPHIIt();
8400 End = BB->end();
8401 }
8402 return false;
8403}
8404
8405bool llvm::programUndefinedIfUndefOrPoison(const Instruction *Inst) {
8406 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: false);
8407}
8408
8409bool llvm::programUndefinedIfPoison(const Instruction *Inst) {
8410 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: true);
8411}
8412
8413static bool isKnownNonNaN(const Value *V, FastMathFlags FMF) {
8414 if (FMF.noNaNs())
8415 return true;
8416
8417 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8418 return !C->isNaN();
8419
8420 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8421 if (!C->getElementType()->isFloatingPointTy())
8422 return false;
8423 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8424 if (C->getElementAsAPFloat(i: I).isNaN())
8425 return false;
8426 }
8427 return true;
8428 }
8429
8430 if (isa<ConstantAggregateZero>(Val: V))
8431 return true;
8432
8433 return false;
8434}
8435
8436static bool isKnownNonZero(const Value *V) {
8437 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8438 return !C->isZero();
8439
8440 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8441 if (!C->getElementType()->isFloatingPointTy())
8442 return false;
8443 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8444 if (C->getElementAsAPFloat(i: I).isZero())
8445 return false;
8446 }
8447 return true;
8448 }
8449
8450 return false;
8451}
8452
8453/// Match clamp pattern for float types without care about NaNs or signed zeros.
8454/// Given non-min/max outer cmp/select from the clamp pattern this
8455/// function recognizes if it can be substitued by a "canonical" min/max
8456/// pattern.
8457static SelectPatternResult matchFastFloatClamp(CmpInst::Predicate Pred,
8458 Value *CmpLHS, Value *CmpRHS,
8459 Value *TrueVal, Value *FalseVal,
8460 Value *&LHS, Value *&RHS) {
8461 // Try to match
8462 // X < C1 ? C1 : Min(X, C2) --> Max(C1, Min(X, C2))
8463 // X > C1 ? C1 : Max(X, C2) --> Min(C1, Max(X, C2))
8464 // and return description of the outer Max/Min.
8465
8466 // First, check if select has inverse order:
8467 if (CmpRHS == FalseVal) {
8468 std::swap(a&: TrueVal, b&: FalseVal);
8469 Pred = CmpInst::getInversePredicate(pred: Pred);
8470 }
8471
8472 // Assume success now. If there's no match, callers should not use these anyway.
8473 LHS = TrueVal;
8474 RHS = FalseVal;
8475
8476 const APFloat *FC1;
8477 if (CmpRHS != TrueVal || !match(V: CmpRHS, P: m_APFloat(Res&: FC1)) || !FC1->isFinite())
8478 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8479
8480 const APFloat *FC2;
8481 switch (Pred) {
8482 case CmpInst::FCMP_OLT:
8483 case CmpInst::FCMP_OLE:
8484 case CmpInst::FCMP_ULT:
8485 case CmpInst::FCMP_ULE:
8486 if (match(V: FalseVal, P: m_OrdOrUnordFMin(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8487 *FC1 < *FC2)
8488 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8489 if (match(V: FalseVal, P: m_FMinNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
8490 *FC1 < *FC2)
8491 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8492 break;
8493 case CmpInst::FCMP_OGT:
8494 case CmpInst::FCMP_OGE:
8495 case CmpInst::FCMP_UGT:
8496 case CmpInst::FCMP_UGE:
8497 if (match(V: FalseVal, P: m_OrdOrUnordFMax(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8498 *FC1 > *FC2)
8499 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8500 if (match(V: FalseVal, P: m_FMaxNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
8501 *FC1 > *FC2)
8502 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8503 break;
8504 default:
8505 break;
8506 }
8507
8508 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8509}
8510
8511/// Recognize variations of:
8512/// CLAMP(v,l,h) ==> ((v) < (l) ? (l) : ((v) > (h) ? (h) : (v)))
8513static SelectPatternResult matchClamp(CmpInst::Predicate Pred,
8514 Value *CmpLHS, Value *CmpRHS,
8515 Value *TrueVal, Value *FalseVal) {
8516 // Swap the select operands and predicate to match the patterns below.
8517 if (CmpRHS != TrueVal) {
8518 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8519 std::swap(a&: TrueVal, b&: FalseVal);
8520 }
8521 const APInt *C1;
8522 if (CmpRHS == TrueVal && match(V: CmpRHS, P: m_APInt(Res&: C1))) {
8523 const APInt *C2;
8524 // (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
8525 if (match(V: FalseVal, P: m_SMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8526 C1->slt(RHS: *C2) && Pred == CmpInst::ICMP_SLT)
8527 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8528
8529 // (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
8530 if (match(V: FalseVal, P: m_SMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8531 C1->sgt(RHS: *C2) && Pred == CmpInst::ICMP_SGT)
8532 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8533
8534 // (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
8535 if (match(V: FalseVal, P: m_UMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8536 C1->ult(RHS: *C2) && Pred == CmpInst::ICMP_ULT)
8537 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8538
8539 // (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
8540 if (match(V: FalseVal, P: m_UMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8541 C1->ugt(RHS: *C2) && Pred == CmpInst::ICMP_UGT)
8542 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8543 }
8544 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8545}
8546
8547/// Recognize variations of:
8548/// a < c ? min(a,b) : min(b,c) ==> min(min(a,b),min(b,c))
8549static SelectPatternResult matchMinMaxOfMinMax(CmpInst::Predicate Pred,
8550 Value *CmpLHS, Value *CmpRHS,
8551 Value *TVal, Value *FVal,
8552 unsigned Depth) {
8553 // TODO: Allow FP min/max with nnan/nsz.
8554 assert(CmpInst::isIntPredicate(Pred) && "Expected integer comparison");
8555
8556 Value *A = nullptr, *B = nullptr;
8557 SelectPatternResult L = matchSelectPattern(V: TVal, LHS&: A, RHS&: B, CastOp: nullptr, Depth: Depth + 1);
8558 if (!SelectPatternResult::isMinOrMax(SPF: L.Flavor))
8559 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8560
8561 Value *C = nullptr, *D = nullptr;
8562 SelectPatternResult R = matchSelectPattern(V: FVal, LHS&: C, RHS&: D, CastOp: nullptr, Depth: Depth + 1);
8563 if (L.Flavor != R.Flavor)
8564 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8565
8566 // We have something like: x Pred y ? min(a, b) : min(c, d).
8567 // Try to match the compare to the min/max operations of the select operands.
8568 // First, make sure we have the right compare predicate.
8569 switch (L.Flavor) {
8570 case SPF_SMIN:
8571 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) {
8572 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8573 std::swap(a&: CmpLHS, b&: CmpRHS);
8574 }
8575 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
8576 break;
8577 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8578 case SPF_SMAX:
8579 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) {
8580 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8581 std::swap(a&: CmpLHS, b&: CmpRHS);
8582 }
8583 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)
8584 break;
8585 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8586 case SPF_UMIN:
8587 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
8588 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8589 std::swap(a&: CmpLHS, b&: CmpRHS);
8590 }
8591 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
8592 break;
8593 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8594 case SPF_UMAX:
8595 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
8596 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8597 std::swap(a&: CmpLHS, b&: CmpRHS);
8598 }
8599 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
8600 break;
8601 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8602 default:
8603 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8604 }
8605
8606 // If there is a common operand in the already matched min/max and the other
8607 // min/max operands match the compare operands (either directly or inverted),
8608 // then this is min/max of the same flavor.
8609
8610 // a pred c ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
8611 // ~c pred ~a ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
8612 if (D == B) {
8613 if ((CmpLHS == A && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8614 match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
8615 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8616 }
8617 // a pred d ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
8618 // ~d pred ~a ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
8619 if (C == B) {
8620 if ((CmpLHS == A && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8621 match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
8622 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8623 }
8624 // b pred c ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
8625 // ~c pred ~b ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
8626 if (D == A) {
8627 if ((CmpLHS == B && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8628 match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
8629 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8630 }
8631 // b pred d ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
8632 // ~d pred ~b ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
8633 if (C == A) {
8634 if ((CmpLHS == B && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8635 match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
8636 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8637 }
8638
8639 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8640}
8641
8642/// If the input value is the result of a 'not' op, constant integer, or vector
8643/// splat of a constant integer, return the bitwise-not source value.
8644/// TODO: This could be extended to handle non-splat vector integer constants.
8645static Value *getNotValue(Value *V) {
8646 Value *NotV;
8647 if (match(V, P: m_Not(V: m_Value(V&: NotV))))
8648 return NotV;
8649
8650 const APInt *C;
8651 if (match(V, P: m_APInt(Res&: C)))
8652 return ConstantInt::get(Ty: V->getType(), V: ~(*C));
8653
8654 return nullptr;
8655}
8656
8657/// Match non-obvious integer minimum and maximum sequences.
8658static SelectPatternResult matchMinMax(CmpInst::Predicate Pred,
8659 Value *CmpLHS, Value *CmpRHS,
8660 Value *TrueVal, Value *FalseVal,
8661 Value *&LHS, Value *&RHS,
8662 unsigned Depth) {
8663 // Assume success. If there's no match, callers should not use these anyway.
8664 LHS = TrueVal;
8665 RHS = FalseVal;
8666
8667 SelectPatternResult SPR = matchClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal);
8668 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
8669 return SPR;
8670
8671 SPR = matchMinMaxOfMinMax(Pred, CmpLHS, CmpRHS, TVal: TrueVal, FVal: FalseVal, Depth);
8672 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
8673 return SPR;
8674
8675 // Look through 'not' ops to find disguised min/max.
8676 // (X > Y) ? ~X : ~Y ==> (~X < ~Y) ? ~X : ~Y ==> MIN(~X, ~Y)
8677 // (X < Y) ? ~X : ~Y ==> (~X > ~Y) ? ~X : ~Y ==> MAX(~X, ~Y)
8678 if (CmpLHS == getNotValue(V: TrueVal) && CmpRHS == getNotValue(V: FalseVal)) {
8679 switch (Pred) {
8680 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8681 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8682 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8683 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8684 default: break;
8685 }
8686 }
8687
8688 // (X > Y) ? ~Y : ~X ==> (~X < ~Y) ? ~Y : ~X ==> MAX(~Y, ~X)
8689 // (X < Y) ? ~Y : ~X ==> (~X > ~Y) ? ~Y : ~X ==> MIN(~Y, ~X)
8690 if (CmpLHS == getNotValue(V: FalseVal) && CmpRHS == getNotValue(V: TrueVal)) {
8691 switch (Pred) {
8692 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8693 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8694 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8695 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8696 default: break;
8697 }
8698 }
8699
8700 if (Pred != CmpInst::ICMP_SGT && Pred != CmpInst::ICMP_SLT)
8701 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8702
8703 const APInt *C1;
8704 if (!match(V: CmpRHS, P: m_APInt(Res&: C1)))
8705 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8706
8707 // An unsigned min/max can be written with a signed compare.
8708 const APInt *C2;
8709 if ((CmpLHS == TrueVal && match(V: FalseVal, P: m_APInt(Res&: C2))) ||
8710 (CmpLHS == FalseVal && match(V: TrueVal, P: m_APInt(Res&: C2)))) {
8711 // Is the sign bit set?
8712 // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
8713 // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
8714 if (Pred == CmpInst::ICMP_SLT && C1->isZero() && C2->isMaxSignedValue())
8715 return {.Flavor: CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8716
8717 // Is the sign bit clear?
8718 // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
8719 // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
8720 if (Pred == CmpInst::ICMP_SGT && C1->isAllOnes() && C2->isMinSignedValue())
8721 return {.Flavor: CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8722 }
8723
8724 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8725}
8726
8727bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW,
8728 bool AllowPoison) {
8729 assert(X && Y && "Invalid operand");
8730
8731 auto IsNegationOf = [&](const Value *X, const Value *Y) {
8732 if (!match(V: X, P: m_Neg(V: m_Specific(V: Y))))
8733 return false;
8734
8735 auto *BO = cast<BinaryOperator>(Val: X);
8736 if (NeedNSW && !BO->hasNoSignedWrap())
8737 return false;
8738
8739 auto *Zero = cast<Constant>(Val: BO->getOperand(i_nocapture: 0));
8740 if (!AllowPoison && !Zero->isNullValue())
8741 return false;
8742
8743 return true;
8744 };
8745
8746 // X = -Y or Y = -X
8747 if (IsNegationOf(X, Y) || IsNegationOf(Y, X))
8748 return true;
8749
8750 // X = sub (A, B), Y = sub (B, A) || X = sub nsw (A, B), Y = sub nsw (B, A)
8751 Value *A, *B;
8752 return (!NeedNSW && (match(V: X, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
8753 match(V: Y, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))))) ||
8754 (NeedNSW && (match(V: X, P: m_NSWSub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
8755 match(V: Y, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A)))));
8756}
8757
8758bool llvm::isKnownInversion(const Value *X, const Value *Y) {
8759 // Handle X = icmp pred A, B, Y = icmp pred A, C.
8760 Value *A, *B, *C;
8761 CmpPredicate Pred1, Pred2;
8762 if (!match(V: X, P: m_ICmp(Pred&: Pred1, L: m_Value(V&: A), R: m_Value(V&: B))) ||
8763 !match(V: Y, P: m_c_ICmp(Pred&: Pred2, L: m_Specific(V: A), R: m_Value(V&: C))))
8764 return false;
8765
8766 // They must both have samesign flag or not.
8767 if (Pred1.hasSameSign() != Pred2.hasSameSign())
8768 return false;
8769
8770 if (B == C)
8771 return Pred1 == ICmpInst::getInversePredicate(pred: Pred2);
8772
8773 // Try to infer the relationship from constant ranges.
8774 const APInt *RHSC1, *RHSC2;
8775 if (!match(V: B, P: m_APInt(Res&: RHSC1)) || !match(V: C, P: m_APInt(Res&: RHSC2)))
8776 return false;
8777
8778 // Sign bits of two RHSCs should match.
8779 if (Pred1.hasSameSign() && RHSC1->isNonNegative() != RHSC2->isNonNegative())
8780 return false;
8781
8782 const auto CR1 = ConstantRange::makeExactICmpRegion(Pred: Pred1, Other: *RHSC1);
8783 const auto CR2 = ConstantRange::makeExactICmpRegion(Pred: Pred2, Other: *RHSC2);
8784
8785 return CR1.inverse() == CR2;
8786}
8787
8788SelectPatternResult llvm::getSelectPattern(CmpInst::Predicate Pred,
8789 SelectPatternNaNBehavior NaNBehavior,
8790 bool Ordered) {
8791 switch (Pred) {
8792 default:
8793 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false}; // Equality.
8794 case ICmpInst::ICMP_UGT:
8795 case ICmpInst::ICMP_UGE:
8796 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8797 case ICmpInst::ICMP_SGT:
8798 case ICmpInst::ICMP_SGE:
8799 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8800 case ICmpInst::ICMP_ULT:
8801 case ICmpInst::ICMP_ULE:
8802 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8803 case ICmpInst::ICMP_SLT:
8804 case ICmpInst::ICMP_SLE:
8805 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8806 case FCmpInst::FCMP_UGT:
8807 case FCmpInst::FCMP_UGE:
8808 case FCmpInst::FCMP_OGT:
8809 case FCmpInst::FCMP_OGE:
8810 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
8811 case FCmpInst::FCMP_ULT:
8812 case FCmpInst::FCMP_ULE:
8813 case FCmpInst::FCMP_OLT:
8814 case FCmpInst::FCMP_OLE:
8815 return {.Flavor: SPF_FMINNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
8816 }
8817}
8818
8819std::optional<std::pair<CmpPredicate, Constant *>>
8820llvm::getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred, Constant *C) {
8821 assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
8822 "Only for relational integer predicates.");
8823 if (isa<UndefValue>(Val: C))
8824 return std::nullopt;
8825
8826 Type *Type = C->getType();
8827 bool IsSigned = ICmpInst::isSigned(Pred);
8828
8829 CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred);
8830 bool WillIncrement =
8831 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
8832
8833 // Check if the constant operand can be safely incremented/decremented
8834 // without overflowing/underflowing.
8835 auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) {
8836 return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned);
8837 };
8838
8839 Constant *SafeReplacementConstant = nullptr;
8840 if (auto *CI = dyn_cast<ConstantInt>(Val: C)) {
8841 // Bail out if the constant can't be safely incremented/decremented.
8842 if (!ConstantIsOk(CI))
8843 return std::nullopt;
8844 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Type)) {
8845 unsigned NumElts = FVTy->getNumElements();
8846 for (unsigned i = 0; i != NumElts; ++i) {
8847 Constant *Elt = C->getAggregateElement(Elt: i);
8848 if (!Elt)
8849 return std::nullopt;
8850
8851 if (isa<UndefValue>(Val: Elt))
8852 continue;
8853
8854 // Bail out if we can't determine if this constant is min/max or if we
8855 // know that this constant is min/max.
8856 auto *CI = dyn_cast<ConstantInt>(Val: Elt);
8857 if (!CI || !ConstantIsOk(CI))
8858 return std::nullopt;
8859
8860 if (!SafeReplacementConstant)
8861 SafeReplacementConstant = CI;
8862 }
8863 } else if (isa<VectorType>(Val: C->getType())) {
8864 // Handle scalable splat
8865 Value *SplatC = C->getSplatValue();
8866 auto *CI = dyn_cast_or_null<ConstantInt>(Val: SplatC);
8867 // Bail out if the constant can't be safely incremented/decremented.
8868 if (!CI || !ConstantIsOk(CI))
8869 return std::nullopt;
8870 } else {
8871 // ConstantExpr?
8872 return std::nullopt;
8873 }
8874
8875 // It may not be safe to change a compare predicate in the presence of
8876 // undefined elements, so replace those elements with the first safe constant
8877 // that we found.
8878 // TODO: in case of poison, it is safe; let's replace undefs only.
8879 if (C->containsUndefOrPoisonElement()) {
8880 assert(SafeReplacementConstant && "Replacement constant not set");
8881 C = Constant::replaceUndefsWith(C, Replacement: SafeReplacementConstant);
8882 }
8883
8884 CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(pred: Pred);
8885
8886 // Increment or decrement the constant.
8887 Constant *OneOrNegOne = ConstantInt::get(Ty: Type, V: WillIncrement ? 1 : -1, IsSigned: true);
8888 Constant *NewC = ConstantExpr::getAdd(C1: C, C2: OneOrNegOne);
8889
8890 return std::make_pair(x&: NewPred, y&: NewC);
8891}
8892
8893static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
8894 FastMathFlags FMF,
8895 Value *CmpLHS, Value *CmpRHS,
8896 Value *TrueVal, Value *FalseVal,
8897 Value *&LHS, Value *&RHS,
8898 unsigned Depth) {
8899 bool HasMismatchedZeros = false;
8900 if (CmpInst::isFPPredicate(P: Pred)) {
8901 // IEEE-754 ignores the sign of 0.0 in comparisons. So if the select has one
8902 // 0.0 operand, set the compare's 0.0 operands to that same value for the
8903 // purpose of identifying min/max. Disregard vector constants with undefined
8904 // elements because those can not be back-propagated for analysis.
8905 Value *OutputZeroVal = nullptr;
8906 if (match(V: TrueVal, P: m_AnyZeroFP()) && !match(V: FalseVal, P: m_AnyZeroFP()) &&
8907 !cast<Constant>(Val: TrueVal)->containsUndefOrPoisonElement())
8908 OutputZeroVal = TrueVal;
8909 else if (match(V: FalseVal, P: m_AnyZeroFP()) && !match(V: TrueVal, P: m_AnyZeroFP()) &&
8910 !cast<Constant>(Val: FalseVal)->containsUndefOrPoisonElement())
8911 OutputZeroVal = FalseVal;
8912
8913 if (OutputZeroVal) {
8914 if (match(V: CmpLHS, P: m_AnyZeroFP()) && CmpLHS != OutputZeroVal) {
8915 HasMismatchedZeros = true;
8916 CmpLHS = OutputZeroVal;
8917 }
8918 if (match(V: CmpRHS, P: m_AnyZeroFP()) && CmpRHS != OutputZeroVal) {
8919 HasMismatchedZeros = true;
8920 CmpRHS = OutputZeroVal;
8921 }
8922 }
8923 }
8924
8925 LHS = CmpLHS;
8926 RHS = CmpRHS;
8927
8928 // Signed zero may return inconsistent results between implementations.
8929 // (0.0 <= -0.0) ? 0.0 : -0.0 // Returns 0.0
8930 // minNum(0.0, -0.0) // May return -0.0 or 0.0 (IEEE 754-2008 5.3.1)
8931 // Therefore, we behave conservatively and only proceed if at least one of the
8932 // operands is known to not be zero or if we don't care about signed zero.
8933 switch (Pred) {
8934 default: break;
8935 case CmpInst::FCMP_OGT: case CmpInst::FCMP_OLT:
8936 case CmpInst::FCMP_UGT: case CmpInst::FCMP_ULT:
8937 if (!HasMismatchedZeros)
8938 break;
8939 [[fallthrough]];
8940 case CmpInst::FCMP_OGE: case CmpInst::FCMP_OLE:
8941 case CmpInst::FCMP_UGE: case CmpInst::FCMP_ULE:
8942 if (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
8943 !isKnownNonZero(V: CmpRHS))
8944 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8945 }
8946
8947 SelectPatternNaNBehavior NaNBehavior = SPNB_NA;
8948 bool Ordered = false;
8949
8950 // When given one NaN and one non-NaN input:
8951 // - maxnum/minnum (C99 fmaxf()/fminf()) return the non-NaN input.
8952 // - A simple C99 (a < b ? a : b) construction will return 'b' (as the
8953 // ordered comparison fails), which could be NaN or non-NaN.
8954 // so here we discover exactly what NaN behavior is required/accepted.
8955 if (CmpInst::isFPPredicate(P: Pred)) {
8956 bool LHSSafe = isKnownNonNaN(V: CmpLHS, FMF);
8957 bool RHSSafe = isKnownNonNaN(V: CmpRHS, FMF);
8958
8959 if (LHSSafe && RHSSafe) {
8960 // Both operands are known non-NaN.
8961 NaNBehavior = SPNB_RETURNS_ANY;
8962 Ordered = CmpInst::isOrdered(predicate: Pred);
8963 } else if (CmpInst::isOrdered(predicate: Pred)) {
8964 // An ordered comparison will return false when given a NaN, so it
8965 // returns the RHS.
8966 Ordered = true;
8967 if (LHSSafe)
8968 // LHS is non-NaN, so if RHS is NaN then NaN will be returned.
8969 NaNBehavior = SPNB_RETURNS_NAN;
8970 else if (RHSSafe)
8971 NaNBehavior = SPNB_RETURNS_OTHER;
8972 else
8973 // Completely unsafe.
8974 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8975 } else {
8976 Ordered = false;
8977 // An unordered comparison will return true when given a NaN, so it
8978 // returns the LHS.
8979 if (LHSSafe)
8980 // LHS is non-NaN, so if RHS is NaN then non-NaN will be returned.
8981 NaNBehavior = SPNB_RETURNS_OTHER;
8982 else if (RHSSafe)
8983 NaNBehavior = SPNB_RETURNS_NAN;
8984 else
8985 // Completely unsafe.
8986 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8987 }
8988 }
8989
8990 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
8991 std::swap(a&: CmpLHS, b&: CmpRHS);
8992 Pred = CmpInst::getSwappedPredicate(pred: Pred);
8993 if (NaNBehavior == SPNB_RETURNS_NAN)
8994 NaNBehavior = SPNB_RETURNS_OTHER;
8995 else if (NaNBehavior == SPNB_RETURNS_OTHER)
8996 NaNBehavior = SPNB_RETURNS_NAN;
8997 Ordered = !Ordered;
8998 }
8999
9000 // ([if]cmp X, Y) ? X : Y
9001 if (TrueVal == CmpLHS && FalseVal == CmpRHS)
9002 return getSelectPattern(Pred, NaNBehavior, Ordered);
9003
9004 if (isKnownNegation(X: TrueVal, Y: FalseVal)) {
9005 // Sign-extending LHS does not change its sign, so TrueVal/FalseVal can
9006 // match against either LHS or sext(LHS).
9007 auto MaybeSExtCmpLHS =
9008 m_CombineOr(Ps: m_Specific(V: CmpLHS), Ps: m_SExt(Op: m_Specific(V: CmpLHS)));
9009 auto ZeroOrAllOnes = m_CombineOr(Ps: m_ZeroInt(), Ps: m_AllOnes());
9010 auto ZeroOrOne = m_CombineOr(Ps: m_ZeroInt(), Ps: m_One());
9011 if (match(V: TrueVal, P: MaybeSExtCmpLHS)) {
9012 // Set the return values. If the compare uses the negated value (-X >s 0),
9013 // swap the return values because the negated value is always 'RHS'.
9014 LHS = TrueVal;
9015 RHS = FalseVal;
9016 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: FalseVal))))
9017 std::swap(a&: LHS, b&: RHS);
9018
9019 // (X >s 0) ? X : -X or (X >s -1) ? X : -X --> ABS(X)
9020 // (-X >s 0) ? -X : X or (-X >s -1) ? -X : X --> ABS(X)
9021 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9022 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9023
9024 // (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X)
9025 if (Pred == ICmpInst::ICMP_SGE && match(V: CmpRHS, P: ZeroOrOne))
9026 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9027
9028 // (X <s 0) ? X : -X or (X <s 1) ? X : -X --> NABS(X)
9029 // (-X <s 0) ? -X : X or (-X <s 1) ? -X : X --> NABS(X)
9030 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9031 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9032 }
9033 else if (match(V: FalseVal, P: MaybeSExtCmpLHS)) {
9034 // Set the return values. If the compare uses the negated value (-X >s 0),
9035 // swap the return values because the negated value is always 'RHS'.
9036 LHS = FalseVal;
9037 RHS = TrueVal;
9038 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: TrueVal))))
9039 std::swap(a&: LHS, b&: RHS);
9040
9041 // (X >s 0) ? -X : X or (X >s -1) ? -X : X --> NABS(X)
9042 // (-X >s 0) ? X : -X or (-X >s -1) ? X : -X --> NABS(X)
9043 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9044 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9045
9046 // (X <s 0) ? -X : X or (X <s 1) ? -X : X --> ABS(X)
9047 // (-X <s 0) ? X : -X or (-X <s 1) ? X : -X --> ABS(X)
9048 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9049 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9050 }
9051 }
9052
9053 if (CmpInst::isIntPredicate(P: Pred))
9054 return matchMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS, Depth);
9055
9056 // According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar
9057 // may return either -0.0 or 0.0, so fcmp/select pair has stricter
9058 // semantics than minNum. Be conservative in such case.
9059 if (NaNBehavior != SPNB_RETURNS_ANY ||
9060 (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9061 !isKnownNonZero(V: CmpRHS)))
9062 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9063
9064 return matchFastFloatClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
9065}
9066
9067static Value *lookThroughCastConst(CmpInst *CmpI, Type *SrcTy, Constant *C,
9068 Instruction::CastOps *CastOp) {
9069 const DataLayout &DL = CmpI->getDataLayout();
9070
9071 Constant *CastedTo = nullptr;
9072 switch (*CastOp) {
9073 case Instruction::ZExt:
9074 if (CmpI->isUnsigned())
9075 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy);
9076 break;
9077 case Instruction::SExt:
9078 if (CmpI->isSigned())
9079 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy, OnlyIfReduced: true);
9080 break;
9081 case Instruction::Trunc:
9082 Constant *CmpConst;
9083 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_Constant(C&: CmpConst)) &&
9084 CmpConst->getType() == SrcTy) {
9085 // Here we have the following case:
9086 //
9087 // %cond = cmp iN %x, CmpConst
9088 // %tr = trunc iN %x to iK
9089 // %narrowsel = select i1 %cond, iK %t, iK C
9090 //
9091 // We can always move trunc after select operation:
9092 //
9093 // %cond = cmp iN %x, CmpConst
9094 // %widesel = select i1 %cond, iN %x, iN CmpConst
9095 // %tr = trunc iN %widesel to iK
9096 //
9097 // Note that C could be extended in any way because we don't care about
9098 // upper bits after truncation. It can't be abs pattern, because it would
9099 // look like:
9100 //
9101 // select i1 %cond, x, -x.
9102 //
9103 // So only min/max pattern could be matched. Such match requires widened C
9104 // == CmpConst. That is why set widened C = CmpConst, condition trunc
9105 // CmpConst == C is checked below.
9106 CastedTo = CmpConst;
9107 } else {
9108 unsigned ExtOp = CmpI->isSigned() ? Instruction::SExt : Instruction::ZExt;
9109 CastedTo = ConstantFoldCastOperand(Opcode: ExtOp, C, DestTy: SrcTy, DL);
9110 }
9111 break;
9112 case Instruction::FPTrunc:
9113 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPExt, C, DestTy: SrcTy, DL);
9114 break;
9115 case Instruction::FPExt:
9116 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPTrunc, C, DestTy: SrcTy, DL);
9117 break;
9118 case Instruction::FPToUI:
9119 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::UIToFP, C, DestTy: SrcTy, DL);
9120 break;
9121 case Instruction::FPToSI:
9122 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::SIToFP, C, DestTy: SrcTy, DL);
9123 break;
9124 case Instruction::UIToFP:
9125 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToUI, C, DestTy: SrcTy, DL);
9126 break;
9127 case Instruction::SIToFP:
9128 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToSI, C, DestTy: SrcTy, DL);
9129 break;
9130 default:
9131 break;
9132 }
9133
9134 if (!CastedTo)
9135 return nullptr;
9136
9137 // Make sure the cast doesn't lose any information.
9138 Constant *CastedBack =
9139 ConstantFoldCastOperand(Opcode: *CastOp, C: CastedTo, DestTy: C->getType(), DL);
9140 if (CastedBack && CastedBack != C)
9141 return nullptr;
9142
9143 return CastedTo;
9144}
9145
9146/// Helps to match a select pattern in case of a type mismatch.
9147///
9148/// The function processes the case when type of true and false values of a
9149/// select instruction differs from type of the cmp instruction operands because
9150/// of a cast instruction. The function checks if it is legal to move the cast
9151/// operation after "select". If yes, it returns the new second value of
9152/// "select" (with the assumption that cast is moved):
9153/// 1. As operand of cast instruction when both values of "select" are same cast
9154/// instructions.
9155/// 2. As restored constant (by applying reverse cast operation) when the first
9156/// value of the "select" is a cast operation and the second value is a
9157/// constant. It is implemented in lookThroughCastConst().
9158/// 3. As one operand is cast instruction and the other is not. The operands in
9159/// sel(cmp) are in different type integer.
9160/// NOTE: We return only the new second value because the first value could be
9161/// accessed as operand of cast instruction.
9162static Value *lookThroughCast(CmpInst *CmpI, Value *V1, Value *V2,
9163 Instruction::CastOps *CastOp) {
9164 auto *Cast1 = dyn_cast<CastInst>(Val: V1);
9165 if (!Cast1)
9166 return nullptr;
9167
9168 *CastOp = Cast1->getOpcode();
9169 Type *SrcTy = Cast1->getSrcTy();
9170 if (auto *Cast2 = dyn_cast<CastInst>(Val: V2)) {
9171 // If V1 and V2 are both the same cast from the same type, look through V1.
9172 if (*CastOp == Cast2->getOpcode() && SrcTy == Cast2->getSrcTy())
9173 return Cast2->getOperand(i_nocapture: 0);
9174 return nullptr;
9175 }
9176
9177 auto *C = dyn_cast<Constant>(Val: V2);
9178 if (C)
9179 return lookThroughCastConst(CmpI, SrcTy, C, CastOp);
9180
9181 Value *CastedTo = nullptr;
9182 if (*CastOp == Instruction::Trunc) {
9183 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_ZExtOrSExt(Op: m_Specific(V: V2)))) {
9184 // Here we have the following case:
9185 // %y_ext = sext iK %y to iN
9186 // %cond = cmp iN %x, %y_ext
9187 // %tr = trunc iN %x to iK
9188 // %narrowsel = select i1 %cond, iK %tr, iK %y
9189 //
9190 // We can always move trunc after select operation:
9191 // %y_ext = sext iK %y to iN
9192 // %cond = cmp iN %x, %y_ext
9193 // %widesel = select i1 %cond, iN %x, iN %y_ext
9194 // %tr = trunc iN %widesel to iK
9195 assert(V2->getType() == Cast1->getType() &&
9196 "V2 and Cast1 should be the same type.");
9197 CastedTo = CmpI->getOperand(i_nocapture: 1);
9198 }
9199 }
9200
9201 return CastedTo;
9202}
9203SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
9204 Instruction::CastOps *CastOp,
9205 unsigned Depth) {
9206 if (Depth >= MaxAnalysisRecursionDepth)
9207 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9208
9209 SelectInst *SI = dyn_cast<SelectInst>(Val: V);
9210 if (!SI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9211
9212 CmpInst *CmpI = dyn_cast<CmpInst>(Val: SI->getCondition());
9213 if (!CmpI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9214
9215 Value *TrueVal = SI->getTrueValue();
9216 Value *FalseVal = SI->getFalseValue();
9217
9218 return llvm::matchDecomposedSelectPattern(CmpI, TrueVal, FalseVal, LHS, RHS,
9219 FMF: SI->getFastMathFlagsOrNone(),
9220 CastOp, Depth);
9221}
9222
9223SelectPatternResult llvm::matchDecomposedSelectPattern(
9224 CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
9225 FastMathFlags FMF, Instruction::CastOps *CastOp, unsigned Depth) {
9226 CmpInst::Predicate Pred = CmpI->getPredicate();
9227 Value *CmpLHS = CmpI->getOperand(i_nocapture: 0);
9228 Value *CmpRHS = CmpI->getOperand(i_nocapture: 1);
9229 if (isa<FPMathOperator>(Val: CmpI) && CmpI->hasNoNaNs())
9230 FMF.setNoNaNs();
9231
9232 // Bail out early.
9233 if (CmpI->isEquality())
9234 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9235
9236 // Deal with type mismatches.
9237 if (CastOp && CmpLHS->getType() != TrueVal->getType()) {
9238 if (Value *C = lookThroughCast(CmpI, V1: TrueVal, V2: FalseVal, CastOp)) {
9239 // If this is a potential fmin/fmax with a cast to integer, then ignore
9240 // -0.0 because there is no corresponding integer value.
9241 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9242 FMF.setNoSignedZeros();
9243 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9244 TrueVal: cast<CastInst>(Val: TrueVal)->getOperand(i_nocapture: 0), FalseVal: C,
9245 LHS, RHS, Depth);
9246 }
9247 if (Value *C = lookThroughCast(CmpI, V1: FalseVal, V2: TrueVal, CastOp)) {
9248 // If this is a potential fmin/fmax with a cast to integer, then ignore
9249 // -0.0 because there is no corresponding integer value.
9250 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9251 FMF.setNoSignedZeros();
9252 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9253 TrueVal: C, FalseVal: cast<CastInst>(Val: FalseVal)->getOperand(i_nocapture: 0),
9254 LHS, RHS, Depth);
9255 }
9256 }
9257 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal, FalseVal,
9258 LHS, RHS, Depth);
9259}
9260
9261CmpInst::Predicate llvm::getMinMaxPred(SelectPatternFlavor SPF, bool Ordered) {
9262 if (SPF == SPF_SMIN) return ICmpInst::ICMP_SLT;
9263 if (SPF == SPF_UMIN) return ICmpInst::ICMP_ULT;
9264 if (SPF == SPF_SMAX) return ICmpInst::ICMP_SGT;
9265 if (SPF == SPF_UMAX) return ICmpInst::ICMP_UGT;
9266 if (SPF == SPF_FMINNUM)
9267 return Ordered ? FCmpInst::FCMP_OLT : FCmpInst::FCMP_ULT;
9268 if (SPF == SPF_FMAXNUM)
9269 return Ordered ? FCmpInst::FCMP_OGT : FCmpInst::FCMP_UGT;
9270 llvm_unreachable("unhandled!");
9271}
9272
9273Intrinsic::ID llvm::getMinMaxIntrinsic(SelectPatternFlavor SPF) {
9274 switch (SPF) {
9275 case SelectPatternFlavor::SPF_UMIN:
9276 return Intrinsic::umin;
9277 case SelectPatternFlavor::SPF_UMAX:
9278 return Intrinsic::umax;
9279 case SelectPatternFlavor::SPF_SMIN:
9280 return Intrinsic::smin;
9281 case SelectPatternFlavor::SPF_SMAX:
9282 return Intrinsic::smax;
9283 default:
9284 llvm_unreachable("Unexpected SPF");
9285 }
9286}
9287
9288SelectPatternFlavor llvm::getInverseMinMaxFlavor(SelectPatternFlavor SPF) {
9289 if (SPF == SPF_SMIN) return SPF_SMAX;
9290 if (SPF == SPF_UMIN) return SPF_UMAX;
9291 if (SPF == SPF_SMAX) return SPF_SMIN;
9292 if (SPF == SPF_UMAX) return SPF_UMIN;
9293 llvm_unreachable("unhandled!");
9294}
9295
9296Intrinsic::ID llvm::getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID) {
9297 switch (MinMaxID) {
9298 case Intrinsic::smax: return Intrinsic::smin;
9299 case Intrinsic::smin: return Intrinsic::smax;
9300 case Intrinsic::umax: return Intrinsic::umin;
9301 case Intrinsic::umin: return Intrinsic::umax;
9302 // Please note that next four intrinsics may produce the same result for
9303 // original and inverted case even if X != Y due to NaN is handled specially.
9304 case Intrinsic::maximum: return Intrinsic::minimum;
9305 case Intrinsic::minimum: return Intrinsic::maximum;
9306 case Intrinsic::maxnum: return Intrinsic::minnum;
9307 case Intrinsic::minnum: return Intrinsic::maxnum;
9308 case Intrinsic::maximumnum:
9309 return Intrinsic::minimumnum;
9310 case Intrinsic::minimumnum:
9311 return Intrinsic::maximumnum;
9312 default: llvm_unreachable("Unexpected intrinsic");
9313 }
9314}
9315
9316APInt llvm::getMinMaxLimit(SelectPatternFlavor SPF, unsigned BitWidth) {
9317 switch (SPF) {
9318 case SPF_SMAX: return APInt::getSignedMaxValue(numBits: BitWidth);
9319 case SPF_SMIN: return APInt::getSignedMinValue(numBits: BitWidth);
9320 case SPF_UMAX: return APInt::getMaxValue(numBits: BitWidth);
9321 case SPF_UMIN: return APInt::getMinValue(numBits: BitWidth);
9322 default: llvm_unreachable("Unexpected flavor");
9323 }
9324}
9325
9326std::pair<Intrinsic::ID, bool>
9327llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
9328 // Check if VL contains select instructions that can be folded into a min/max
9329 // vector intrinsic and return the intrinsic if it is possible.
9330 // TODO: Support floating point min/max.
9331 bool AllCmpSingleUse = true;
9332 SelectPatternResult SelectPattern;
9333 SelectPattern.Flavor = SPF_UNKNOWN;
9334 if (all_of(Range&: VL, P: [&SelectPattern, &AllCmpSingleUse](Value *I) {
9335 Value *LHS, *RHS;
9336 auto CurrentPattern = matchSelectPattern(V: I, LHS, RHS);
9337 if (!SelectPatternResult::isMinOrMax(SPF: CurrentPattern.Flavor))
9338 return false;
9339 if (SelectPattern.Flavor != SPF_UNKNOWN &&
9340 SelectPattern.Flavor != CurrentPattern.Flavor)
9341 return false;
9342 SelectPattern = CurrentPattern;
9343 AllCmpSingleUse &=
9344 match(V: I, P: m_Select(C: m_OneUse(SubPattern: m_Value()), L: m_Value(), R: m_Value()));
9345 return true;
9346 })) {
9347 switch (SelectPattern.Flavor) {
9348 case SPF_SMIN:
9349 return {Intrinsic::smin, AllCmpSingleUse};
9350 case SPF_UMIN:
9351 return {Intrinsic::umin, AllCmpSingleUse};
9352 case SPF_SMAX:
9353 return {Intrinsic::smax, AllCmpSingleUse};
9354 case SPF_UMAX:
9355 return {Intrinsic::umax, AllCmpSingleUse};
9356 case SPF_FMAXNUM:
9357 return {Intrinsic::maxnum, AllCmpSingleUse};
9358 case SPF_FMINNUM:
9359 return {Intrinsic::minnum, AllCmpSingleUse};
9360 default:
9361 llvm_unreachable("unexpected select pattern flavor");
9362 }
9363 }
9364 return {Intrinsic::not_intrinsic, false};
9365}
9366
9367template <typename InstTy>
9368static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
9369 Value *&Init, Value *&OtherOp) {
9370 // Handle the case of a simple two-predecessor recurrence PHI.
9371 // There's a lot more that could theoretically be done here, but
9372 // this is sufficient to catch some interesting cases.
9373 // TODO: Expand list -- gep, uadd.sat etc.
9374 if (PN->getNumIncomingValues() != 2)
9375 return false;
9376
9377 for (unsigned I = 0; I != 2; ++I) {
9378 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9379 Operation && Operation->getNumOperands() >= 2) {
9380 Value *LHS = Operation->getOperand(0);
9381 Value *RHS = Operation->getOperand(1);
9382 if (LHS != PN && RHS != PN)
9383 continue;
9384
9385 Inst = Operation;
9386 Init = PN->getIncomingValue(i: !I);
9387 OtherOp = (LHS == PN) ? RHS : LHS;
9388 return true;
9389 }
9390 }
9391 return false;
9392}
9393
9394template <typename InstTy>
9395static bool matchThreeInputRecurrence(const PHINode *PN, InstTy *&Inst,
9396 Value *&Init, Value *&OtherOp0,
9397 Value *&OtherOp1) {
9398 if (PN->getNumIncomingValues() != 2)
9399 return false;
9400
9401 for (unsigned I = 0; I != 2; ++I) {
9402 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9403 Operation && Operation->getNumOperands() >= 3) {
9404 Value *Op0 = Operation->getOperand(0);
9405 Value *Op1 = Operation->getOperand(1);
9406 Value *Op2 = Operation->getOperand(2);
9407
9408 if (Op0 != PN && Op1 != PN && Op2 != PN)
9409 continue;
9410
9411 Inst = Operation;
9412 Init = PN->getIncomingValue(i: !I);
9413 if (Op0 == PN) {
9414 OtherOp0 = Op1;
9415 OtherOp1 = Op2;
9416 } else if (Op1 == PN) {
9417 OtherOp0 = Op0;
9418 OtherOp1 = Op2;
9419 } else {
9420 OtherOp0 = Op0;
9421 OtherOp1 = Op1;
9422 }
9423 return true;
9424 }
9425 }
9426 return false;
9427}
9428bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
9429 Value *&Start, Value *&Step) {
9430 // We try to match a recurrence of the form:
9431 // %iv = [Start, %entry], [%iv.next, %backedge]
9432 // %iv.next = binop %iv, Step
9433 // Or:
9434 // %iv = [Start, %entry], [%iv.next, %backedge]
9435 // %iv.next = binop Step, %iv
9436 return matchTwoInputRecurrence(PN: P, Inst&: BO, Init&: Start, OtherOp&: Step);
9437}
9438
9439bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
9440 Value *&Start, Value *&Step) {
9441 BinaryOperator *BO = nullptr;
9442 P = dyn_cast<PHINode>(Val: I->getOperand(i_nocapture: 0));
9443 if (!P)
9444 P = dyn_cast<PHINode>(Val: I->getOperand(i_nocapture: 1));
9445 return P && matchSimpleRecurrence(P, BO, Start, Step) && BO == I;
9446}
9447
9448bool llvm::matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I,
9449 PHINode *&P, Value *&Init,
9450 Value *&OtherOp) {
9451 // Binary intrinsics only supported for now.
9452 if (I->arg_size() != 2 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9453 I->getType() != I->getArgOperand(i: 1)->getType())
9454 return false;
9455
9456 IntrinsicInst *II = nullptr;
9457 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9458 if (!P)
9459 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9460
9461 return P && matchTwoInputRecurrence(PN: P, Inst&: II, Init, OtherOp) && II == I;
9462}
9463
9464bool llvm::matchSimpleTernaryIntrinsicRecurrence(const IntrinsicInst *I,
9465 PHINode *&P, Value *&Init,
9466 Value *&OtherOp0,
9467 Value *&OtherOp1) {
9468 if (I->arg_size() != 3 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9469 I->getType() != I->getArgOperand(i: 1)->getType() ||
9470 I->getType() != I->getArgOperand(i: 2)->getType())
9471 return false;
9472 IntrinsicInst *II = nullptr;
9473 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9474 if (!P) {
9475 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9476 if (!P)
9477 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 2));
9478 }
9479 return P && matchThreeInputRecurrence(PN: P, Inst&: II, Init, OtherOp0, OtherOp1) &&
9480 II == I;
9481}
9482
9483/// Return true if "icmp Pred LHS RHS" is always true.
9484static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
9485 const Value *RHS) {
9486 if (ICmpInst::isTrueWhenEqual(predicate: Pred) && LHS == RHS)
9487 return true;
9488
9489 switch (Pred) {
9490 default:
9491 return false;
9492
9493 case CmpInst::ICMP_SLE: {
9494 const APInt *C;
9495
9496 // LHS s<= LHS +_{nsw} C if C >= 0
9497 // LHS s<= LHS | C if C >= 0
9498 if (match(V: RHS, P: m_NSWAdd(L: m_Specific(V: LHS), R: m_APInt(Res&: C))) ||
9499 match(V: RHS, P: m_Or(L: m_Specific(V: LHS), R: m_APInt(Res&: C))))
9500 return !C->isNegative();
9501
9502 // LHS s<= smax(LHS, V) for any V
9503 if (match(V: RHS, P: m_c_SMax(L: m_Specific(V: LHS), R: m_Value())))
9504 return true;
9505
9506 // smin(RHS, V) s<= RHS for any V
9507 if (match(V: LHS, P: m_c_SMin(L: m_Specific(V: RHS), R: m_Value())))
9508 return true;
9509
9510 // Match A to (X +_{nsw} CA) and B to (X +_{nsw} CB)
9511 const Value *X;
9512 const APInt *CLHS, *CRHS;
9513 if (match(V: LHS, P: m_NSWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9514 match(V: RHS, P: m_NSWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9515 return CLHS->sle(RHS: *CRHS);
9516
9517 return false;
9518 }
9519
9520 case CmpInst::ICMP_ULE: {
9521 // LHS u<= LHS +_{nuw} V for any V
9522 if (match(V: RHS, P: m_c_Add(L: m_Specific(V: LHS), R: m_Value())) &&
9523 cast<OverflowingBinaryOperator>(Val: RHS)->hasNoUnsignedWrap())
9524 return true;
9525
9526 // LHS u<= LHS | V for any V
9527 if (match(V: RHS, P: m_c_Or(L: m_Specific(V: LHS), R: m_Value())))
9528 return true;
9529
9530 // LHS u<= umax(LHS, V) for any V
9531 if (match(V: RHS, P: m_c_UMax(L: m_Specific(V: LHS), R: m_Value())))
9532 return true;
9533
9534 // RHS >> V u<= RHS for any V
9535 if (match(V: LHS, P: m_LShr(L: m_Specific(V: RHS), R: m_Value())))
9536 return true;
9537
9538 // RHS u/ C_ugt_1 u<= RHS
9539 const APInt *C;
9540 if (match(V: LHS, P: m_UDiv(L: m_Specific(V: RHS), R: m_APInt(Res&: C))) && C->ugt(RHS: 1))
9541 return true;
9542
9543 // RHS & V u<= RHS for any V
9544 if (match(V: LHS, P: m_c_And(L: m_Specific(V: RHS), R: m_Value())))
9545 return true;
9546
9547 // umin(RHS, V) u<= RHS for any V
9548 if (match(V: LHS, P: m_c_UMin(L: m_Specific(V: RHS), R: m_Value())))
9549 return true;
9550
9551 // Match A to (X +_{nuw} CA) and B to (X +_{nuw} CB)
9552 const Value *X;
9553 const APInt *CLHS, *CRHS;
9554 if (match(V: LHS, P: m_NUWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9555 match(V: RHS, P: m_NUWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9556 return CLHS->ule(RHS: *CRHS);
9557
9558 return false;
9559 }
9560 }
9561}
9562
9563/// Return true if "icmp Pred BLHS BRHS" is true whenever "icmp Pred
9564/// ALHS ARHS" is true. Otherwise, return std::nullopt.
9565static std::optional<bool>
9566isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS,
9567 const Value *ARHS, const Value *BLHS, const Value *BRHS) {
9568 switch (Pred) {
9569 default:
9570 return std::nullopt;
9571
9572 case CmpInst::ICMP_SLT:
9573 case CmpInst::ICMP_SLE:
9574 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BLHS, RHS: ALHS) &&
9575 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ARHS, RHS: BRHS))
9576 return true;
9577 return std::nullopt;
9578
9579 case CmpInst::ICMP_SGT:
9580 case CmpInst::ICMP_SGE:
9581 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ALHS, RHS: BLHS) &&
9582 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BRHS, RHS: ARHS))
9583 return true;
9584 return std::nullopt;
9585
9586 case CmpInst::ICMP_ULT:
9587 case CmpInst::ICMP_ULE:
9588 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BLHS, RHS: ALHS) &&
9589 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ARHS, RHS: BRHS))
9590 return true;
9591 return std::nullopt;
9592
9593 case CmpInst::ICMP_UGT:
9594 case CmpInst::ICMP_UGE:
9595 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ALHS, RHS: BLHS) &&
9596 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BRHS, RHS: ARHS))
9597 return true;
9598 return std::nullopt;
9599 }
9600}
9601
9602/// Return true if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is true.
9603/// Return false if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is false.
9604/// Otherwise, return std::nullopt if we can't infer anything.
9605static std::optional<bool>
9606isImpliedCondCommonOperandWithCR(CmpPredicate LPred, const ConstantRange &LCR,
9607 CmpPredicate RPred, const ConstantRange &RCR) {
9608 auto CRImpliesPred = [&](ConstantRange CR,
9609 CmpInst::Predicate Pred) -> std::optional<bool> {
9610 // If all true values for lhs and true for rhs, lhs implies rhs
9611 if (CR.icmp(Pred, Other: RCR))
9612 return true;
9613
9614 // If there is no overlap, lhs implies not rhs
9615 if (CR.icmp(Pred: CmpInst::getInversePredicate(pred: Pred), Other: RCR))
9616 return false;
9617
9618 return std::nullopt;
9619 };
9620 if (auto Res = CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
9621 RPred))
9622 return Res;
9623 if (LPred.hasSameSign() ^ RPred.hasSameSign()) {
9624 LPred = LPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: LPred)
9625 : LPred.dropSameSign();
9626 RPred = RPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: RPred)
9627 : RPred.dropSameSign();
9628 return CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
9629 RPred);
9630 }
9631 return std::nullopt;
9632}
9633
9634/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
9635/// is true. Return false if LHS implies RHS is false. Otherwise, return
9636/// std::nullopt if we can't infer anything.
9637static std::optional<bool>
9638isImpliedCondICmps(CmpPredicate LPred, const Value *L0, const Value *L1,
9639 CmpPredicate RPred, const Value *R0, const Value *R1,
9640 const DataLayout &DL, bool LHSIsTrue) {
9641 // The rest of the logic assumes the LHS condition is true. If that's not the
9642 // case, invert the predicate to make it so.
9643 if (!LHSIsTrue)
9644 LPred = ICmpInst::getInverseCmpPredicate(Pred: LPred);
9645
9646 // We can have non-canonical operands, so try to normalize any common operand
9647 // to L0/R0.
9648 if (L0 == R1) {
9649 std::swap(a&: R0, b&: R1);
9650 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
9651 }
9652 if (R0 == L1) {
9653 std::swap(a&: L0, b&: L1);
9654 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9655 }
9656 if (L1 == R1) {
9657 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
9658 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
9659 std::swap(a&: L0, b&: L1);
9660 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9661 std::swap(a&: R0, b&: R1);
9662 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
9663 }
9664 }
9665
9666 // See if we can infer anything if operand-0 matches and we have at least one
9667 // constant.
9668 const APInt *Unused;
9669 if (L0 == R0 && (match(V: L1, P: m_APInt(Res&: Unused)) || match(V: R1, P: m_APInt(Res&: Unused)))) {
9670 // Potential TODO: We could also further use the constant range of L0/R0 to
9671 // further constraint the constant ranges. At the moment this leads to
9672 // several regressions related to not transforming `multi_use(A + C0) eq/ne
9673 // C1` (see discussion: D58633).
9674 SimplifyQuery SQ(DL);
9675 ConstantRange LCR = computeConstantRange(V: L1, ForSigned: ICmpInst::isSigned(Pred: LPred), SQ,
9676 Depth: MaxAnalysisRecursionDepth - 1);
9677 ConstantRange RCR = computeConstantRange(V: R1, ForSigned: ICmpInst::isSigned(Pred: RPred), SQ,
9678 Depth: MaxAnalysisRecursionDepth - 1);
9679
9680 // Even if L1/R1 are not both constant, we can still sometimes deduce
9681 // relationship from a single constant. For example X u> Y implies X != 0.
9682 if (auto R = isImpliedCondCommonOperandWithCR(LPred, LCR, RPred, RCR))
9683 return R;
9684 // If both L1/R1 were exact constant ranges and we didn't get anything
9685 // here, we won't be able to deduce this.
9686 if (match(V: L1, P: m_APInt(Res&: Unused)) && match(V: R1, P: m_APInt(Res&: Unused)))
9687 return std::nullopt;
9688 }
9689
9690 // Can we infer anything when the two compares have matching operands?
9691 if (L0 == R0 && L1 == R1)
9692 return ICmpInst::isImpliedByMatchingCmp(Pred1: LPred, Pred2: RPred);
9693
9694 // It only really makes sense in the context of signed comparison for "X - Y
9695 // must be positive if X >= Y and no overflow".
9696 // Take SGT as an example: L0:x > L1:y and C >= 0
9697 // ==> R0:(x -nsw y) < R1:(-C) is false
9698 CmpInst::Predicate SignedLPred = LPred.getPreferredSignedPredicate();
9699 if ((SignedLPred == ICmpInst::ICMP_SGT ||
9700 SignedLPred == ICmpInst::ICMP_SGE) &&
9701 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
9702 if (match(V: R1, P: m_NonPositive()) &&
9703 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == false)
9704 return false;
9705 }
9706
9707 // Take SLT as an example: L0:x < L1:y and C <= 0
9708 // ==> R0:(x -nsw y) < R1:(-C) is true
9709 if ((SignedLPred == ICmpInst::ICMP_SLT ||
9710 SignedLPred == ICmpInst::ICMP_SLE) &&
9711 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
9712 if (match(V: R1, P: m_NonNegative()) &&
9713 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == true)
9714 return true;
9715 }
9716
9717 // a - b == NonZero -> a != b
9718 // ptrtoint(a) - ptrtoint(b) == NonZero -> a != b
9719 const APInt *L1C;
9720 Value *A, *B;
9721 if (LPred == ICmpInst::ICMP_EQ && ICmpInst::isEquality(P: RPred) &&
9722 match(V: L1, P: m_APInt(Res&: L1C)) && !L1C->isZero() &&
9723 match(V: L0, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9724 ((A == R0 && B == R1) || (A == R1 && B == R0) ||
9725 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0))) &&
9726 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1)))) ||
9727 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1))) &&
9728 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0)))))) {
9729 return RPred.dropSameSign() == ICmpInst::ICMP_NE;
9730 }
9731
9732 // L0 = R0 = L1 + R1, L0 >=u L1 implies R0 >=u R1, L0 <u L1 implies R0 <u R1
9733 if (L0 == R0 &&
9734 (LPred == ICmpInst::ICMP_ULT || LPred == ICmpInst::ICMP_UGE) &&
9735 (RPred == ICmpInst::ICMP_ULT || RPred == ICmpInst::ICMP_UGE) &&
9736 match(V: L0, P: m_c_Add(L: m_Specific(V: L1), R: m_Specific(V: R1))))
9737 return CmpPredicate::getMatching(A: LPred, B: RPred).has_value();
9738
9739 if (auto P = CmpPredicate::getMatching(A: LPred, B: RPred))
9740 return isImpliedCondOperands(Pred: *P, ALHS: L0, ARHS: L1, BLHS: R0, BRHS: R1);
9741
9742 return std::nullopt;
9743}
9744
9745/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
9746/// is true. Return false if LHS implies RHS is false. Otherwise, return
9747/// std::nullopt if we can't infer anything.
9748static std::optional<bool>
9749isImpliedCondFCmps(FCmpInst::Predicate LPred, const Value *L0, const Value *L1,
9750 FCmpInst::Predicate RPred, const Value *R0, const Value *R1,
9751 const DataLayout &DL, bool LHSIsTrue) {
9752 // The rest of the logic assumes the LHS condition is true. If that's not the
9753 // case, invert the predicate to make it so.
9754 if (!LHSIsTrue)
9755 LPred = FCmpInst::getInversePredicate(pred: LPred);
9756
9757 // We can have non-canonical operands, so try to normalize any common operand
9758 // to L0/R0.
9759 if (L0 == R1) {
9760 std::swap(a&: R0, b&: R1);
9761 RPred = FCmpInst::getSwappedPredicate(pred: RPred);
9762 }
9763 if (R0 == L1) {
9764 std::swap(a&: L0, b&: L1);
9765 LPred = FCmpInst::getSwappedPredicate(pred: LPred);
9766 }
9767 if (L1 == R1) {
9768 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
9769 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
9770 std::swap(a&: L0, b&: L1);
9771 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9772 std::swap(a&: R0, b&: R1);
9773 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
9774 }
9775 }
9776
9777 // Can we infer anything when the two compares have matching operands?
9778 if (L0 == R0 && L1 == R1) {
9779 if ((LPred & RPred) == LPred)
9780 return true;
9781 if ((LPred & ~RPred) == LPred)
9782 return false;
9783 }
9784
9785 // See if we can infer anything if operand-0 matches and we have at least one
9786 // constant.
9787 const APFloat *L1C, *R1C;
9788 if (L0 == R0 && match(V: L1, P: m_APFloat(Res&: L1C)) && match(V: R1, P: m_APFloat(Res&: R1C))) {
9789 if (std::optional<ConstantFPRange> DomCR =
9790 ConstantFPRange::makeExactFCmpRegion(Pred: LPred, Other: *L1C)) {
9791 if (std::optional<ConstantFPRange> ImpliedCR =
9792 ConstantFPRange::makeExactFCmpRegion(Pred: RPred, Other: *R1C)) {
9793 if (ImpliedCR->contains(CR: *DomCR))
9794 return true;
9795 }
9796 if (std::optional<ConstantFPRange> ImpliedCR =
9797 ConstantFPRange::makeExactFCmpRegion(
9798 Pred: FCmpInst::getInversePredicate(pred: RPred), Other: *R1C)) {
9799 if (ImpliedCR->contains(CR: *DomCR))
9800 return false;
9801 }
9802 }
9803 }
9804
9805 return std::nullopt;
9806}
9807
9808/// Return true if LHS implies RHS is true. Return false if LHS implies RHS is
9809/// false. Otherwise, return std::nullopt if we can't infer anything. We
9810/// expect the RHS to be an icmp and the LHS to be an 'and', 'or', or a 'select'
9811/// instruction.
9812static std::optional<bool>
9813isImpliedCondAndOr(const Instruction *LHS, CmpPredicate RHSPred,
9814 const Value *RHSOp0, const Value *RHSOp1,
9815 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
9816 // The LHS must be an 'or', 'and', or a 'select' instruction.
9817 assert((LHS->getOpcode() == Instruction::And ||
9818 LHS->getOpcode() == Instruction::Or ||
9819 LHS->getOpcode() == Instruction::Select) &&
9820 "Expected LHS to be 'and', 'or', or 'select'.");
9821
9822 assert(Depth <= MaxAnalysisRecursionDepth && "Hit recursion limit");
9823
9824 // If the result of an 'or' is false, then we know both legs of the 'or' are
9825 // false. Similarly, if the result of an 'and' is true, then we know both
9826 // legs of the 'and' are true.
9827 const Value *ALHS, *ARHS;
9828 if ((!LHSIsTrue && match(V: LHS, P: m_LogicalOr(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS)))) ||
9829 (LHSIsTrue && match(V: LHS, P: m_LogicalAnd(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS))))) {
9830 // FIXME: Make this non-recursion.
9831 if (std::optional<bool> Implication = isImpliedCondition(
9832 LHS: ALHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
9833 return Implication;
9834 if (std::optional<bool> Implication = isImpliedCondition(
9835 LHS: ARHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
9836 return Implication;
9837 return std::nullopt;
9838 }
9839 return std::nullopt;
9840}
9841
9842std::optional<bool>
9843llvm::isImpliedCondition(const Value *LHS, CmpPredicate RHSPred,
9844 const Value *RHSOp0, const Value *RHSOp1,
9845 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
9846 // Bail out when we hit the limit.
9847 if (Depth == MaxAnalysisRecursionDepth)
9848 return std::nullopt;
9849
9850 // A mismatch occurs when we compare a scalar cmp to a vector cmp, for
9851 // example.
9852 if (RHSOp0->getType()->isVectorTy() != LHS->getType()->isVectorTy())
9853 return std::nullopt;
9854
9855 assert(LHS->getType()->isIntOrIntVectorTy(1) &&
9856 "Expected integer type only!");
9857
9858 // Match not
9859 if (match(V: LHS, P: m_Not(V: m_Value(V&: LHS))))
9860 LHSIsTrue = !LHSIsTrue;
9861
9862 // Both LHS and RHS are icmps.
9863 if (RHSOp0->getType()->getScalarType()->isIntOrPtrTy()) {
9864 CmpPredicate LHSPred;
9865 Value *LHSOp0, *LHSOp1;
9866 if (match(V: LHS, P: m_ICmpLike(Pred&: LHSPred, L: m_Value(V&: LHSOp0), R: m_Value(V&: LHSOp1))))
9867 return isImpliedCondICmps(LPred: LHSPred, L0: LHSOp0, L1: LHSOp1, RPred: RHSPred, R0: RHSOp0,
9868 R1: RHSOp1, DL, LHSIsTrue);
9869 } else {
9870 assert(RHSOp0->getType()->isFPOrFPVectorTy() &&
9871 "Expected floating point type only!");
9872 if (const auto *LHSCmp = dyn_cast<FCmpInst>(Val: LHS))
9873 return isImpliedCondFCmps(LPred: LHSCmp->getPredicate(), L0: LHSCmp->getOperand(i_nocapture: 0),
9874 L1: LHSCmp->getOperand(i_nocapture: 1), RPred: RHSPred, R0: RHSOp0, R1: RHSOp1,
9875 DL, LHSIsTrue);
9876 }
9877
9878 /// The LHS should be an 'or', 'and', or a 'select' instruction. We expect
9879 /// the RHS to be an icmp.
9880 /// FIXME: Add support for and/or/select on the RHS.
9881 if (const Instruction *LHSI = dyn_cast<Instruction>(Val: LHS)) {
9882 if ((LHSI->getOpcode() == Instruction::And ||
9883 LHSI->getOpcode() == Instruction::Or ||
9884 LHSI->getOpcode() == Instruction::Select))
9885 return isImpliedCondAndOr(LHS: LHSI, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue,
9886 Depth);
9887 }
9888 return std::nullopt;
9889}
9890
9891std::optional<bool> llvm::isImpliedCondition(const Value *LHS, const Value *RHS,
9892 const DataLayout &DL,
9893 bool LHSIsTrue, unsigned Depth) {
9894 // LHS ==> RHS by definition
9895 if (LHS == RHS)
9896 return LHSIsTrue;
9897
9898 // Match not
9899 bool InvertRHS = false;
9900 if (match(V: RHS, P: m_Not(V: m_Value(V&: RHS)))) {
9901 if (LHS == RHS)
9902 return !LHSIsTrue;
9903 InvertRHS = true;
9904 }
9905
9906 CmpPredicate RHSPred;
9907 Value *RHSOp0, *RHSOp1;
9908 if (match(V: RHS, P: m_ICmpLike(Pred&: RHSPred, L: m_Value(V&: RHSOp0), R: m_Value(V&: RHSOp1)))) {
9909 if (auto Implied = isImpliedCondition(LHS, RHSPred, RHSOp0, RHSOp1, DL,
9910 LHSIsTrue, Depth))
9911 return InvertRHS ? !*Implied : *Implied;
9912 return std::nullopt;
9913 }
9914 if (const FCmpInst *RHSCmp = dyn_cast<FCmpInst>(Val: RHS)) {
9915 if (auto Implied = isImpliedCondition(
9916 LHS, RHSPred: RHSCmp->getPredicate(), RHSOp0: RHSCmp->getOperand(i_nocapture: 0),
9917 RHSOp1: RHSCmp->getOperand(i_nocapture: 1), DL, LHSIsTrue, Depth))
9918 return InvertRHS ? !*Implied : *Implied;
9919 return std::nullopt;
9920 }
9921
9922 if (Depth == MaxAnalysisRecursionDepth)
9923 return std::nullopt;
9924
9925 // LHS ==> (RHS1 || RHS2) if LHS ==> RHS1 or LHS ==> RHS2
9926 // LHS ==> !(RHS1 && RHS2) if LHS ==> !RHS1 or LHS ==> !RHS2
9927 const Value *RHS1, *RHS2;
9928 if (match(V: RHS, P: m_LogicalOr(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
9929 if (std::optional<bool> Imp =
9930 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
9931 if (*Imp == true)
9932 return !InvertRHS;
9933 if (std::optional<bool> Imp =
9934 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
9935 if (*Imp == true)
9936 return !InvertRHS;
9937 }
9938 if (match(V: RHS, P: m_LogicalAnd(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
9939 if (std::optional<bool> Imp =
9940 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
9941 if (*Imp == false)
9942 return InvertRHS;
9943 if (std::optional<bool> Imp =
9944 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
9945 if (*Imp == false)
9946 return InvertRHS;
9947 }
9948
9949 return std::nullopt;
9950}
9951
9952// Returns a pair (Condition, ConditionIsTrue), where Condition is a branch
9953// condition dominating ContextI or nullptr, if no condition is found.
9954static std::pair<Value *, bool>
9955getDomPredecessorCondition(const Instruction *ContextI) {
9956 if (!ContextI || !ContextI->getParent())
9957 return {nullptr, false};
9958
9959 // TODO: This is a poor/cheap way to determine dominance. Should we use a
9960 // dominator tree (eg, from a SimplifyQuery) instead?
9961 const BasicBlock *ContextBB = ContextI->getParent();
9962 const BasicBlock *PredBB = ContextBB->getSinglePredecessor();
9963 if (!PredBB)
9964 return {nullptr, false};
9965
9966 // We need a conditional branch in the predecessor.
9967 Value *PredCond;
9968 BasicBlock *TrueBB, *FalseBB;
9969 if (!match(V: PredBB->getTerminator(), P: m_Br(C: m_Value(V&: PredCond), T&: TrueBB, F&: FalseBB)))
9970 return {nullptr, false};
9971
9972 // The branch should get simplified. Don't bother simplifying this condition.
9973 if (TrueBB == FalseBB)
9974 return {nullptr, false};
9975
9976 assert((TrueBB == ContextBB || FalseBB == ContextBB) &&
9977 "Predecessor block does not point to successor?");
9978
9979 // Is this condition implied by the predecessor condition?
9980 return {PredCond, TrueBB == ContextBB};
9981}
9982
9983std::optional<bool> llvm::isImpliedByDomCondition(const Value *Cond,
9984 const Instruction *ContextI,
9985 const DataLayout &DL) {
9986 assert(Cond->getType()->isIntOrIntVectorTy(1) && "Condition must be bool");
9987 auto PredCond = getDomPredecessorCondition(ContextI);
9988 if (PredCond.first)
9989 return isImpliedCondition(LHS: PredCond.first, RHS: Cond, DL, LHSIsTrue: PredCond.second);
9990 return std::nullopt;
9991}
9992
9993std::optional<bool> llvm::isImpliedByDomCondition(CmpPredicate Pred,
9994 const Value *LHS,
9995 const Value *RHS,
9996 const Instruction *ContextI,
9997 const DataLayout &DL) {
9998 auto PredCond = getDomPredecessorCondition(ContextI);
9999 if (PredCond.first)
10000 return isImpliedCondition(LHS: PredCond.first, RHSPred: Pred, RHSOp0: LHS, RHSOp1: RHS, DL,
10001 LHSIsTrue: PredCond.second);
10002 return std::nullopt;
10003}
10004
10005static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
10006 APInt &Upper, const InstrInfoQuery &IIQ,
10007 bool PreferSignedRange) {
10008 unsigned Width = Lower.getBitWidth();
10009 const APInt *C;
10010 switch (BO.getOpcode()) {
10011 case Instruction::Sub:
10012 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10013 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10014 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10015
10016 // If the caller expects a signed compare, then try to use a signed range.
10017 // Otherwise if both no-wraps are set, use the unsigned range because it
10018 // is never larger than the signed range. Example:
10019 // "sub nuw nsw i8 -2, x" is unsigned [0, 254] vs. signed [-128, 126].
10020 // "sub nuw nsw i8 2, x" is unsigned [0, 2] vs. signed [-125, 127].
10021 if (PreferSignedRange && HasNSW && HasNUW)
10022 HasNUW = false;
10023
10024 if (HasNUW) {
10025 // 'sub nuw c, x' produces [0, C].
10026 Upper = *C + 1;
10027 } else if (HasNSW) {
10028 if (C->isNegative()) {
10029 // 'sub nsw -C, x' produces [SINT_MIN, -C - SINT_MIN].
10030 Lower = APInt::getSignedMinValue(numBits: Width);
10031 Upper = *C - APInt::getSignedMaxValue(numBits: Width);
10032 } else {
10033 // Note that sub 0, INT_MIN is not NSW. It techically is a signed wrap
10034 // 'sub nsw C, x' produces [C - SINT_MAX, SINT_MAX].
10035 Lower = *C - APInt::getSignedMaxValue(numBits: Width);
10036 Upper = APInt::getSignedMinValue(numBits: Width);
10037 }
10038 }
10039 }
10040 break;
10041 case Instruction::Add:
10042 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10043 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10044 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10045
10046 // If the caller expects a signed compare, then try to use a signed
10047 // range. Otherwise if both no-wraps are set, use the unsigned range
10048 // because it is never larger than the signed range. Example: "add nuw
10049 // nsw i8 X, -2" is unsigned [254,255] vs. signed [-128, 125].
10050 if (PreferSignedRange && HasNSW && HasNUW)
10051 HasNUW = false;
10052
10053 if (HasNUW) {
10054 // 'add nuw x, C' produces [C, UINT_MAX].
10055 Lower = *C;
10056 } else if (HasNSW) {
10057 if (C->isNegative()) {
10058 // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C].
10059 Lower = APInt::getSignedMinValue(numBits: Width);
10060 Upper = APInt::getSignedMaxValue(numBits: Width) + *C + 1;
10061 } else {
10062 // 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX].
10063 Lower = APInt::getSignedMinValue(numBits: Width) + *C;
10064 Upper = APInt::getSignedMaxValue(numBits: Width) + 1;
10065 }
10066 }
10067 }
10068 break;
10069
10070 case Instruction::And:
10071 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10072 // 'and x, C' produces [0, C].
10073 Upper = *C + 1;
10074 // X & -X is a power of two or zero. So we can cap the value at max power of
10075 // two.
10076 if (match(V: BO.getOperand(i_nocapture: 0), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 1)))) ||
10077 match(V: BO.getOperand(i_nocapture: 1), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 0)))))
10078 Upper = APInt::getSignedMinValue(numBits: Width) + 1;
10079 break;
10080
10081 case Instruction::Or:
10082 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10083 // 'or x, C' produces [C, UINT_MAX].
10084 Lower = *C;
10085 break;
10086
10087 case Instruction::AShr:
10088 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10089 // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C].
10090 Lower = APInt::getSignedMinValue(numBits: Width).ashr(ShiftAmt: *C);
10091 Upper = APInt::getSignedMaxValue(numBits: Width).ashr(ShiftAmt: *C) + 1;
10092 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10093 unsigned ShiftAmount = Width - 1;
10094 if (!C->isZero() && IIQ.isExact(Op: &BO))
10095 ShiftAmount = C->countr_zero();
10096 if (C->isNegative()) {
10097 // 'ashr C, x' produces [C, C >> (Width-1)]
10098 Lower = *C;
10099 Upper = C->ashr(ShiftAmt: ShiftAmount) + 1;
10100 } else {
10101 // 'ashr C, x' produces [C >> (Width-1), C]
10102 Lower = C->ashr(ShiftAmt: ShiftAmount);
10103 Upper = *C + 1;
10104 }
10105 }
10106 break;
10107
10108 case Instruction::LShr:
10109 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10110 // 'lshr x, C' produces [0, UINT_MAX >> C].
10111 Upper = APInt::getAllOnes(numBits: Width).lshr(ShiftAmt: *C) + 1;
10112 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10113 // 'lshr C, x' produces [C >> (Width-1), C].
10114 unsigned ShiftAmount = Width - 1;
10115 if (!C->isZero() && IIQ.isExact(Op: &BO))
10116 ShiftAmount = C->countr_zero();
10117 Lower = C->lshr(shiftAmt: ShiftAmount);
10118 Upper = *C + 1;
10119 }
10120 break;
10121
10122 case Instruction::Shl:
10123 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10124 if (IIQ.hasNoUnsignedWrap(Op: &BO)) {
10125 // 'shl nuw C, x' produces [C, C << CLZ(C)]
10126 Lower = *C;
10127 Upper = Lower.shl(shiftAmt: Lower.countl_zero()) + 1;
10128 } else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw?
10129 if (C->isNegative()) {
10130 // 'shl nsw C, x' produces [C << CLO(C)-1, C]
10131 unsigned ShiftAmount = C->countl_one() - 1;
10132 Lower = C->shl(shiftAmt: ShiftAmount);
10133 Upper = *C + 1;
10134 } else {
10135 // 'shl nsw C, x' produces [C, C << CLZ(C)-1]
10136 unsigned ShiftAmount = C->countl_zero() - 1;
10137 Lower = *C;
10138 Upper = C->shl(shiftAmt: ShiftAmount) + 1;
10139 }
10140 } else {
10141 // If lowbit is set, value can never be zero.
10142 if ((*C)[0])
10143 Lower = APInt::getOneBitSet(numBits: Width, BitNo: 0);
10144 // If we are shifting a constant the largest it can be is if the longest
10145 // sequence of consecutive ones is shifted to the highbits (breaking
10146 // ties for which sequence is higher). At the moment we take a liberal
10147 // upper bound on this by just popcounting the constant.
10148 // TODO: There may be a bitwise trick for it longest/highest
10149 // consecutative sequence of ones (naive method is O(Width) loop).
10150 Upper = APInt::getHighBitsSet(numBits: Width, hiBitsSet: C->popcount()) + 1;
10151 }
10152 } else if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10153 Upper = APInt::getBitsSetFrom(numBits: Width, loBit: C->getZExtValue()) + 1;
10154 }
10155 break;
10156
10157 case Instruction::SDiv:
10158 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10159 APInt IntMin = APInt::getSignedMinValue(numBits: Width);
10160 APInt IntMax = APInt::getSignedMaxValue(numBits: Width);
10161 if (C->isAllOnes()) {
10162 // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX]
10163 // where C != -1 and C != 0 and C != 1
10164 Lower = IntMin + 1;
10165 Upper = IntMax + 1;
10166 } else if (C->countl_zero() < Width - 1) {
10167 // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C]
10168 // where C != -1 and C != 0 and C != 1
10169 Lower = IntMin.sdiv(RHS: *C);
10170 Upper = IntMax.sdiv(RHS: *C);
10171 if (Lower.sgt(RHS: Upper))
10172 std::swap(a&: Lower, b&: Upper);
10173 Upper = Upper + 1;
10174 assert(Upper != Lower && "Upper part of range has wrapped!");
10175 }
10176 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10177 if (C->isMinSignedValue()) {
10178 // 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2].
10179 Lower = *C;
10180 Upper = Lower.lshr(shiftAmt: 1) + 1;
10181 } else {
10182 // 'sdiv C, x' produces [-|C|, |C|].
10183 Upper = C->abs() + 1;
10184 Lower = (-Upper) + 1;
10185 }
10186 }
10187 break;
10188
10189 case Instruction::UDiv:
10190 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10191 // 'udiv x, C' produces [0, UINT_MAX / C].
10192 Upper = APInt::getMaxValue(numBits: Width).udiv(RHS: *C) + 1;
10193 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10194 // 'udiv C, x' produces [0, C].
10195 Upper = *C + 1;
10196 }
10197 break;
10198
10199 case Instruction::SRem:
10200 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10201 // 'srem x, C' produces (-|C|, |C|).
10202 Upper = C->abs();
10203 Lower = (-Upper) + 1;
10204 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10205 if (C->isNegative()) {
10206 // 'srem -|C|, x' produces [-|C|, 0].
10207 Upper = 1;
10208 Lower = *C;
10209 } else {
10210 // 'srem |C|, x' produces [0, |C|].
10211 Upper = *C + 1;
10212 }
10213 }
10214 break;
10215
10216 case Instruction::URem:
10217 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10218 // 'urem x, C' produces [0, C).
10219 Upper = *C;
10220 else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10221 // 'urem C, x' produces [0, C].
10222 Upper = *C + 1;
10223 break;
10224
10225 default:
10226 break;
10227 }
10228}
10229
10230static ConstantRange getRangeForIntrinsic(const IntrinsicInst &II,
10231 bool UseInstrInfo) {
10232 unsigned Width = II.getType()->getScalarSizeInBits();
10233 const APInt *C;
10234 switch (II.getIntrinsicID()) {
10235 case Intrinsic::ctlz:
10236 case Intrinsic::cttz: {
10237 APInt Upper(Width, Width);
10238 if (!UseInstrInfo || !match(V: II.getArgOperand(i: 1), P: m_One()))
10239 Upper += 1;
10240 // Maximum of set/clear bits is the bit width.
10241 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper);
10242 }
10243 case Intrinsic::ctpop:
10244 // Maximum of set/clear bits is the bit width.
10245 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10246 Upper: APInt(Width, Width) + 1);
10247 case Intrinsic::uadd_sat:
10248 // uadd.sat(x, C) produces [C, UINT_MAX].
10249 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10250 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10251 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10252 break;
10253 case Intrinsic::sadd_sat:
10254 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10255 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10256 if (C->isNegative())
10257 // sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)].
10258 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10259 Upper: APInt::getSignedMaxValue(numBits: Width) + *C +
10260 1);
10261
10262 // sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX].
10263 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) + *C,
10264 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10265 }
10266 break;
10267 case Intrinsic::usub_sat:
10268 // usub.sat(C, x) produces [0, C].
10269 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10270 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10271
10272 // usub.sat(x, C) produces [0, UINT_MAX - C].
10273 if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10274 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10275 Upper: APInt::getMaxValue(numBits: Width) - *C + 1);
10276 break;
10277 case Intrinsic::ssub_sat:
10278 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10279 if (C->isNegative())
10280 // ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)].
10281 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10282 Upper: *C - APInt::getSignedMinValue(numBits: Width) +
10283 1);
10284
10285 // ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX].
10286 return ConstantRange::getNonEmpty(Lower: *C - APInt::getSignedMaxValue(numBits: Width),
10287 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10288 } else if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10289 if (C->isNegative())
10290 // ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]:
10291 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) - *C,
10292 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10293
10294 // ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C].
10295 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10296 Upper: APInt::getSignedMaxValue(numBits: Width) - *C +
10297 1);
10298 }
10299 break;
10300 case Intrinsic::umin:
10301 case Intrinsic::umax:
10302 case Intrinsic::smin:
10303 case Intrinsic::smax:
10304 if (!match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) &&
10305 !match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10306 break;
10307
10308 switch (II.getIntrinsicID()) {
10309 case Intrinsic::umin:
10310 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10311 case Intrinsic::umax:
10312 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10313 case Intrinsic::smin:
10314 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10315 Upper: *C + 1);
10316 case Intrinsic::smax:
10317 return ConstantRange::getNonEmpty(Lower: *C,
10318 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10319 default:
10320 llvm_unreachable("Must be min/max intrinsic");
10321 }
10322 break;
10323 case Intrinsic::abs:
10324 // If abs of SIGNED_MIN is poison, then the result is [0..SIGNED_MAX],
10325 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10326 if (match(V: II.getOperand(i_nocapture: 1), P: m_One()))
10327 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10328 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10329
10330 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10331 Upper: APInt::getSignedMinValue(numBits: Width) + 1);
10332 case Intrinsic::vscale:
10333 if (!II.getParent() || !II.getFunction())
10334 break;
10335 return getVScaleRange(F: II.getFunction(), BitWidth: Width);
10336 default:
10337 break;
10338 }
10339
10340 return ConstantRange::getFull(BitWidth: Width);
10341}
10342
10343static ConstantRange getRangeForSelectPattern(const SelectInst &SI,
10344 const InstrInfoQuery &IIQ) {
10345 unsigned BitWidth = SI.getType()->getScalarSizeInBits();
10346 const Value *LHS = nullptr, *RHS = nullptr;
10347 SelectPatternResult R = matchSelectPattern(V: &SI, LHS, RHS);
10348 if (R.Flavor == SPF_UNKNOWN)
10349 return ConstantRange::getFull(BitWidth);
10350
10351 if (R.Flavor == SelectPatternFlavor::SPF_ABS) {
10352 // If the negation part of the abs (in RHS) has the NSW flag,
10353 // then the result of abs(X) is [0..SIGNED_MAX],
10354 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10355 if (match(V: RHS, P: m_Neg(V: m_Specific(V: LHS))) &&
10356 IIQ.hasNoSignedWrap(Op: cast<Instruction>(Val: RHS)))
10357 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10358 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10359
10360 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10361 Upper: APInt::getSignedMinValue(numBits: BitWidth) + 1);
10362 }
10363
10364 if (R.Flavor == SelectPatternFlavor::SPF_NABS) {
10365 // The result of -abs(X) is <= 0.
10366 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10367 Upper: APInt(BitWidth, 1));
10368 }
10369
10370 const APInt *C;
10371 if (!match(V: LHS, P: m_APInt(Res&: C)) && !match(V: RHS, P: m_APInt(Res&: C)))
10372 return ConstantRange::getFull(BitWidth);
10373
10374 switch (R.Flavor) {
10375 case SPF_UMIN:
10376 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth), Upper: *C + 1);
10377 case SPF_UMAX:
10378 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: BitWidth));
10379 case SPF_SMIN:
10380 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10381 Upper: *C + 1);
10382 case SPF_SMAX:
10383 return ConstantRange::getNonEmpty(Lower: *C,
10384 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10385 default:
10386 return ConstantRange::getFull(BitWidth);
10387 }
10388}
10389
10390static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) {
10391 // The maximum representable value of a half is 65504. For floats the maximum
10392 // value is 3.4e38 which requires roughly 129 bits.
10393 unsigned BitWidth = I->getType()->getScalarSizeInBits();
10394 if (!I->getOperand(i: 0)->getType()->getScalarType()->isHalfTy())
10395 return;
10396 if (isa<FPToSIInst>(Val: I) && BitWidth >= 17) {
10397 Lower = APInt(BitWidth, -65504, true);
10398 Upper = APInt(BitWidth, 65505);
10399 }
10400
10401 if (isa<FPToUIInst>(Val: I) && BitWidth >= 16) {
10402 // For a fptoui the lower limit is left as 0.
10403 Upper = APInt(BitWidth, 65505);
10404 }
10405}
10406
10407ConstantRange llvm::computeConstantRange(const Value *V, bool ForSigned,
10408 const SimplifyQuery &SQ,
10409 unsigned Depth) {
10410 assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction");
10411
10412 if (Depth == MaxAnalysisRecursionDepth)
10413 return ConstantRange::getFull(BitWidth: V->getType()->getScalarSizeInBits());
10414
10415 if (auto *C = dyn_cast<Constant>(Val: V))
10416 return C->toConstantRange();
10417
10418 unsigned BitWidth = V->getType()->getScalarSizeInBits();
10419 ConstantRange CR = ConstantRange::getFull(BitWidth);
10420 if (auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
10421 APInt Lower = APInt(BitWidth, 0);
10422 APInt Upper = APInt(BitWidth, 0);
10423 // TODO: Return ConstantRange.
10424 setLimitsForBinOp(BO: *BO, Lower, Upper, IIQ: SQ.IIQ, PreferSignedRange: ForSigned);
10425 CR = ConstantRange::getNonEmpty(Lower, Upper);
10426 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
10427 CR = getRangeForIntrinsic(II: *II, UseInstrInfo: SQ.IIQ.UseInstrInfo);
10428 else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
10429 ConstantRange CRTrue =
10430 computeConstantRange(V: SI->getTrueValue(), ForSigned, SQ, Depth: Depth + 1);
10431 ConstantRange CRFalse =
10432 computeConstantRange(V: SI->getFalseValue(), ForSigned, SQ, Depth: Depth + 1);
10433 CR = CRTrue.unionWith(CR: CRFalse);
10434 CR = CR.intersectWith(CR: getRangeForSelectPattern(SI: *SI, IIQ: SQ.IIQ));
10435 } else if (auto *TI = dyn_cast<TruncInst>(Val: V)) {
10436 ConstantRange SrcCR =
10437 computeConstantRange(V: TI->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
10438 CR = SrcCR.truncate(BitWidth);
10439 } else if (isa<FPToUIInst>(Val: V) || isa<FPToSIInst>(Val: V)) {
10440 APInt Lower = APInt(BitWidth, 0);
10441 APInt Upper = APInt(BitWidth, 0);
10442 // TODO: Return ConstantRange.
10443 setLimitForFPToI(I: cast<Instruction>(Val: V), Lower, Upper);
10444 CR = ConstantRange::getNonEmpty(Lower, Upper);
10445 } else if (const auto *A = dyn_cast<Argument>(Val: V))
10446 if (std::optional<ConstantRange> Range = A->getRange())
10447 CR = *Range;
10448
10449 if (auto *I = dyn_cast<Instruction>(Val: V)) {
10450 if (auto *Range = SQ.IIQ.getMetadata(I, KindID: LLVMContext::MD_range))
10451 CR = CR.intersectWith(CR: getConstantRangeFromMetadata(RangeMD: *Range));
10452
10453 Value *FrexpSrc;
10454 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
10455 if (std::optional<ConstantRange> Range = CB->getRange())
10456 CR = CR.intersectWith(CR: *Range);
10457 } else if (match(V: I, P: m_ExtractValue<1>(V: m_Intrinsic<Intrinsic::frexp>(
10458 Op0: m_Value(V&: FrexpSrc))))) {
10459 const fltSemantics &FltSem =
10460 FrexpSrc->getType()->getScalarType()->getFltSemantics();
10461 // It should be possible to implement this for any type, but this logic
10462 // only computes the range assuming standard subnormal handling.
10463 if (APFloat::isIEEELikeFP(FltSem)) {
10464 KnownFPClass KnownSrc =
10465 computeKnownFPClass(V: FrexpSrc, InterestedClasses: fcSubnormal, SQ, Depth: Depth + 1);
10466
10467 // Exponent result is (src == 0) ? 0 : ilogb(src) + 1, and unspecified
10468 // for inf/nan.
10469 int MinExp = APFloat::semanticsMinExponent(FltSem) + 1;
10470
10471 // Offset to find the true minimum exponent value for a denormal.
10472 if (!KnownSrc.isKnownNeverSubnormal())
10473 MinExp -= (APFloat::semanticsPrecision(FltSem) - 1);
10474
10475 int MaxExp = APFloat::semanticsMaxExponent(FltSem) + 1;
10476 CR = ConstantRange::getNonEmpty(
10477 Lower: APInt(BitWidth, MinExp, /*isSigned=*/true),
10478 Upper: APInt(BitWidth, MaxExp + 1, /*isSigned=*/true));
10479 }
10480 }
10481 }
10482
10483 if (SQ.CxtI && SQ.AC) {
10484 // Try to restrict the range based on information from assumptions.
10485 for (auto &AssumeVH : SQ.AC->assumptionsFor(V)) {
10486 if (!AssumeVH)
10487 continue;
10488 CallInst *I = cast<CallInst>(Val&: AssumeVH);
10489 assert(I->getParent()->getParent() == SQ.CxtI->getParent()->getParent() &&
10490 "Got assumption for the wrong function!");
10491 assert(I->getIntrinsicID() == Intrinsic::assume &&
10492 "must be an assume intrinsic");
10493
10494 if (!isValidAssumeForContext(I, Q: SQ))
10495 continue;
10496 Value *Arg = I->getArgOperand(i: 0);
10497 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
10498 // Currently we just use information from comparisons.
10499 if (!Cmp || Cmp->getOperand(i_nocapture: 0) != V)
10500 continue;
10501 // TODO: Set "ForSigned" parameter via Cmp->isSigned()?
10502 ConstantRange RHS =
10503 computeConstantRange(V: Cmp->getOperand(i_nocapture: 1), /*ForSigned=*/false,
10504 SQ: SQ.getWithInstruction(I), Depth: Depth + 1);
10505 CR = CR.intersectWith(
10506 CR: ConstantRange::makeAllowedICmpRegion(Pred: Cmp->getCmpPredicate(), Other: RHS));
10507 }
10508 }
10509
10510 return CR;
10511}
10512
10513static void
10514addValueAffectedByCondition(Value *V,
10515 function_ref<void(Value *)> InsertAffected) {
10516 assert(V != nullptr);
10517 if (isa<Argument>(Val: V) || isa<GlobalValue>(Val: V)) {
10518 InsertAffected(V);
10519 } else if (auto *I = dyn_cast<Instruction>(Val: V)) {
10520 InsertAffected(V);
10521
10522 // Peek through unary operators to find the source of the condition.
10523 Value *Op;
10524 if (match(V: I, P: m_CombineOr(Ps: m_PtrToIntOrAddr(Op: m_Value(V&: Op)),
10525 Ps: m_Trunc(Op: m_Value(V&: Op))))) {
10526 if (isa<Instruction>(Val: Op) || isa<Argument>(Val: Op))
10527 InsertAffected(Op);
10528 }
10529 }
10530}
10531
10532void llvm::findValuesAffectedByCondition(
10533 Value *Cond, bool IsAssume, function_ref<void(Value *)> InsertAffected) {
10534 auto AddAffected = [&InsertAffected](Value *V) {
10535 addValueAffectedByCondition(V, InsertAffected);
10536 };
10537
10538 auto AddCmpOperands = [&AddAffected, IsAssume](Value *LHS, Value *RHS) {
10539 if (IsAssume) {
10540 AddAffected(LHS);
10541 AddAffected(RHS);
10542 } else if (match(V: RHS, P: m_Constant()))
10543 AddAffected(LHS);
10544 };
10545
10546 SmallVector<Value *, 8> Worklist;
10547 SmallPtrSet<Value *, 8> Visited;
10548 Worklist.push_back(Elt: Cond);
10549 while (!Worklist.empty()) {
10550 Value *V = Worklist.pop_back_val();
10551 if (!Visited.insert(Ptr: V).second)
10552 continue;
10553
10554 CmpPredicate Pred;
10555 Value *A, *B, *X;
10556
10557 if (IsAssume) {
10558 AddAffected(V);
10559 if (match(V, P: m_Not(V: m_Value(V&: X))))
10560 AddAffected(X);
10561 }
10562
10563 if (match(V, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
10564 // assume(A && B) is split to -> assume(A); assume(B);
10565 // assume(!(A || B)) is split to -> assume(!A); assume(!B);
10566 // Finally, assume(A || B) / assume(!(A && B)) generally don't provide
10567 // enough information to be worth handling (intersection of information as
10568 // opposed to union).
10569 if (!IsAssume) {
10570 Worklist.push_back(Elt: A);
10571 Worklist.push_back(Elt: B);
10572 }
10573 } else if (match(V, P: m_ICmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
10574 bool HasRHSC = match(V: B, P: m_ConstantInt());
10575 if (ICmpInst::isEquality(P: Pred)) {
10576 AddAffected(A);
10577 if (IsAssume)
10578 AddAffected(B);
10579 if (HasRHSC) {
10580 Value *Y;
10581 // (X << C) or (X >>_s C) or (X >>_u C).
10582 if (match(V: A, P: m_Shift(L: m_Value(V&: X), R: m_ConstantInt())))
10583 AddAffected(X);
10584 // (X & C) or (X | C).
10585 else if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10586 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10587 AddAffected(X);
10588 AddAffected(Y);
10589 }
10590 // X - Y
10591 else if (match(V: A, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10592 AddAffected(X);
10593 AddAffected(Y);
10594 }
10595 }
10596 } else {
10597 AddCmpOperands(A, B);
10598 if (HasRHSC) {
10599 // Handle (A + C1) u< C2, which is the canonical form of
10600 // A > C3 && A < C4.
10601 if (match(V: A, P: m_AddLike(L: m_Value(V&: X), R: m_ConstantInt())))
10602 AddAffected(X);
10603
10604 if (ICmpInst::isUnsigned(Pred)) {
10605 Value *Y;
10606 // X & Y u> C -> X >u C && Y >u C
10607 // X | Y u< C -> X u< C && Y u< C
10608 // X nuw+ Y u< C -> X u< C && Y u< C
10609 if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10610 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10611 match(V: A, P: m_NUWAdd(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10612 AddAffected(X);
10613 AddAffected(Y);
10614 }
10615 // X nuw- Y u> C -> X u> C
10616 if (match(V: A, P: m_NUWSub(L: m_Value(V&: X), R: m_Value())))
10617 AddAffected(X);
10618 }
10619 }
10620
10621 // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported
10622 // by computeKnownFPClass().
10623 if (match(V: A, P: m_ElementWiseBitCast(Op: m_Value(V&: X)))) {
10624 if (Pred == ICmpInst::ICMP_SLT && match(V: B, P: m_Zero()))
10625 InsertAffected(X);
10626 else if (Pred == ICmpInst::ICMP_SGT && match(V: B, P: m_AllOnes()))
10627 InsertAffected(X);
10628 }
10629 }
10630
10631 if (HasRHSC && match(V: A, P: m_Ctpop(Op0: m_Value(V&: X))))
10632 AddAffected(X);
10633 } else if (match(V, P: m_FCmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
10634 AddCmpOperands(A, B);
10635
10636 // fcmp fneg(x), y
10637 // fcmp fabs(x), y
10638 // fcmp fneg(fabs(x)), y
10639 if (match(V: A, P: m_FNeg(X: m_Value(V&: A))))
10640 AddAffected(A);
10641 if (match(V: A, P: m_FAbs(Op0: m_Value(V&: A))))
10642 AddAffected(A);
10643
10644 } else if (match(V, P: m_Intrinsic<Intrinsic::is_fpclass>(Op0: m_Value(V&: A),
10645 Op1: m_Value()))) {
10646 // Handle patterns that computeKnownFPClass() support.
10647 AddAffected(A);
10648 } else if (!IsAssume && match(V, P: m_Trunc(Op: m_Value(V&: X)))) {
10649 // Assume is checked here as X is already added above for assumes in
10650 // addValueAffectedByCondition
10651 AddAffected(X);
10652 } else if (!IsAssume && match(V, P: m_Not(V: m_Value(V&: X)))) {
10653 // Assume is checked here to avoid issues with ephemeral values
10654 Worklist.push_back(Elt: X);
10655 }
10656 }
10657}
10658
10659const Value *llvm::stripNullTest(const Value *V) {
10660 // (X >> C) or/add (X & mask(C) != 0)
10661 if (const auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
10662 if (BO->getOpcode() == Instruction::Add ||
10663 BO->getOpcode() == Instruction::Or) {
10664 const Value *X;
10665 const APInt *C1, *C2;
10666 if (match(V: BO, P: m_c_BinOp(L: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: C1)),
10667 R: m_ZExt(Op: m_SpecificICmp(
10668 MatchPred: ICmpInst::ICMP_NE,
10669 L: m_And(L: m_Deferred(V: X), R: m_LowBitMask(V&: C2)),
10670 R: m_Zero())))) &&
10671 C2->popcount() == C1->getZExtValue())
10672 return X;
10673 }
10674 }
10675 return nullptr;
10676}
10677
10678Value *llvm::stripNullTest(Value *V) {
10679 return const_cast<Value *>(stripNullTest(V: const_cast<const Value *>(V)));
10680}
10681
10682bool llvm::collectPossibleValues(const Value *V,
10683 SmallPtrSetImpl<const Constant *> &Constants,
10684 unsigned MaxCount, bool AllowUndefOrPoison) {
10685 SmallPtrSet<const Instruction *, 8> Visited;
10686 SmallVector<const Instruction *, 8> Worklist;
10687 auto Push = [&](const Value *V) -> bool {
10688 Constant *C;
10689 if (match(V: const_cast<Value *>(V), P: m_ImmConstant(C))) {
10690 if (!AllowUndefOrPoison && !isGuaranteedNotToBeUndefOrPoison(V: C))
10691 return false;
10692 // Check existence first to avoid unnecessary allocations.
10693 if (Constants.contains(Ptr: C))
10694 return true;
10695 if (Constants.size() == MaxCount)
10696 return false;
10697 Constants.insert(Ptr: C);
10698 return true;
10699 }
10700
10701 if (auto *Inst = dyn_cast<Instruction>(Val: V)) {
10702 if (Visited.insert(Ptr: Inst).second)
10703 Worklist.push_back(Elt: Inst);
10704 return true;
10705 }
10706 return false;
10707 };
10708 if (!Push(V))
10709 return false;
10710 while (!Worklist.empty()) {
10711 const Instruction *CurInst = Worklist.pop_back_val();
10712 switch (CurInst->getOpcode()) {
10713 case Instruction::Select:
10714 if (!Push(CurInst->getOperand(i: 1)))
10715 return false;
10716 if (!Push(CurInst->getOperand(i: 2)))
10717 return false;
10718 break;
10719 case Instruction::PHI:
10720 for (Value *IncomingValue : cast<PHINode>(Val: CurInst)->incoming_values()) {
10721 // Fast path for recurrence PHI.
10722 if (IncomingValue == CurInst)
10723 continue;
10724 if (!Push(IncomingValue))
10725 return false;
10726 }
10727 break;
10728 default:
10729 return false;
10730 }
10731 }
10732 return true;
10733}
10734