1//===- ValueTracking.cpp - Walk computations to compute properties --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains routines that help analyze properties that chains of
10// computations have.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ValueTracking.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/FloatingPointMode.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/ScopeExit.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/ADT/iterator_range.h"
25#include "llvm/Analysis/AliasAnalysis.h"
26#include "llvm/Analysis/AssumeBundleQueries.h"
27#include "llvm/Analysis/AssumptionCache.h"
28#include "llvm/Analysis/ConstantFolding.h"
29#include "llvm/Analysis/DomConditionCache.h"
30#include "llvm/Analysis/FloatingPointPredicateUtils.h"
31#include "llvm/Analysis/GuardUtils.h"
32#include "llvm/Analysis/InstructionSimplify.h"
33#include "llvm/Analysis/Loads.h"
34#include "llvm/Analysis/LoopInfo.h"
35#include "llvm/Analysis/TargetLibraryInfo.h"
36#include "llvm/Analysis/VectorUtils.h"
37#include "llvm/Analysis/WithCache.h"
38#include "llvm/IR/Argument.h"
39#include "llvm/IR/Attributes.h"
40#include "llvm/IR/BasicBlock.h"
41#include "llvm/IR/BundleAttributes.h"
42#include "llvm/IR/Constant.h"
43#include "llvm/IR/ConstantFPRange.h"
44#include "llvm/IR/ConstantRange.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/DiagnosticInfo.h"
48#include "llvm/IR/Dominators.h"
49#include "llvm/IR/EHPersonalities.h"
50#include "llvm/IR/Function.h"
51#include "llvm/IR/GetElementPtrTypeIterator.h"
52#include "llvm/IR/GlobalAlias.h"
53#include "llvm/IR/GlobalValue.h"
54#include "llvm/IR/GlobalVariable.h"
55#include "llvm/IR/InstrTypes.h"
56#include "llvm/IR/Instruction.h"
57#include "llvm/IR/Instructions.h"
58#include "llvm/IR/IntrinsicInst.h"
59#include "llvm/IR/Intrinsics.h"
60#include "llvm/IR/IntrinsicsAArch64.h"
61#include "llvm/IR/IntrinsicsAMDGPU.h"
62#include "llvm/IR/IntrinsicsRISCV.h"
63#include "llvm/IR/IntrinsicsX86.h"
64#include "llvm/IR/LLVMContext.h"
65#include "llvm/IR/Metadata.h"
66#include "llvm/IR/Module.h"
67#include "llvm/IR/Operator.h"
68#include "llvm/IR/PatternMatch.h"
69#include "llvm/IR/Type.h"
70#include "llvm/IR/User.h"
71#include "llvm/IR/Value.h"
72#include "llvm/Support/Casting.h"
73#include "llvm/Support/CommandLine.h"
74#include "llvm/Support/Compiler.h"
75#include "llvm/Support/ErrorHandling.h"
76#include "llvm/Support/KnownBits.h"
77#include "llvm/Support/KnownFPClass.h"
78#include "llvm/Support/MathExtras.h"
79#include "llvm/Support/UndefPoison.h"
80#include "llvm/TargetParser/RISCVTargetParser.h"
81#include <algorithm>
82#include <cassert>
83#include <cstdint>
84#include <optional>
85#include <utility>
86
87using namespace llvm;
88using namespace llvm::PatternMatch;
89
90// Controls the number of uses of the value searched for possible
91// dominating comparisons.
92static cl::opt<unsigned> DomConditionsMaxUses("dom-conditions-max-uses",
93 cl::Hidden, cl::init(Val: 20));
94
95/// Maximum number of instructions to check between assume and context
96/// instruction.
97static constexpr unsigned MaxInstrsToCheckForFree = 32;
98
99/// Returns the bitwidth of the given scalar or pointer type. For vector types,
100/// returns the element type's bitwidth.
101static unsigned getBitWidth(Type *Ty, const DataLayout &DL) {
102 if (unsigned BitWidth = Ty->getScalarSizeInBits())
103 return BitWidth;
104
105 return DL.getPointerTypeSizeInBits(Ty);
106}
107
108// Given the provided Value and, potentially, a context instruction, return
109// the preferred context instruction (if any).
110static const Instruction *safeCxtI(const Value *V, const Instruction *CxtI) {
111 // If we've been provided with a context instruction, then use that (provided
112 // it has been inserted).
113 if (CxtI && CxtI->getParent())
114 return CxtI;
115
116 // If the value is really an already-inserted instruction, then use that.
117 CxtI = dyn_cast<Instruction>(Val: V);
118 if (CxtI && CxtI->getParent())
119 return CxtI;
120
121 return nullptr;
122}
123
124static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
125 const APInt &DemandedElts,
126 APInt &DemandedLHS, APInt &DemandedRHS) {
127 if (isa<ScalableVectorType>(Val: Shuf->getType())) {
128 assert(DemandedElts == APInt(1,1));
129 DemandedLHS = DemandedRHS = DemandedElts;
130 return true;
131 }
132
133 int NumElts =
134 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
135 return llvm::getShuffleDemandedElts(SrcWidth: NumElts, Mask: Shuf->getShuffleMask(),
136 DemandedElts, DemandedLHS, DemandedRHS);
137}
138
139static void computeKnownBits(const Value *V, const APInt &DemandedElts,
140 KnownBits &Known, const SimplifyQuery &Q,
141 unsigned Depth);
142
143void llvm::computeKnownBits(const Value *V, KnownBits &Known,
144 const SimplifyQuery &Q, unsigned Depth) {
145 // Since the number of lanes in a scalable vector is unknown at compile time,
146 // we track one bit which is implicitly broadcast to all lanes. This means
147 // that all lanes in a scalable vector are considered demanded.
148 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
149 APInt DemandedElts =
150 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
151 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
152}
153
154void llvm::computeKnownBits(const Value *V, KnownBits &Known,
155 const DataLayout &DL, AssumptionCache *AC,
156 const Instruction *CxtI, const DominatorTree *DT,
157 bool UseInstrInfo, unsigned Depth) {
158 computeKnownBits(V, Known,
159 Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo),
160 Depth);
161}
162
163KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
164 AssumptionCache *AC, const Instruction *CxtI,
165 const DominatorTree *DT, bool UseInstrInfo,
166 unsigned Depth) {
167 return computeKnownBits(
168 V, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
169}
170
171KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
172 const DataLayout &DL, AssumptionCache *AC,
173 const Instruction *CxtI,
174 const DominatorTree *DT, bool UseInstrInfo,
175 unsigned Depth) {
176 return computeKnownBits(
177 V, DemandedElts,
178 Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
179}
180
181static NoCommonBitsSetResult
182haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
183 const SimplifyQuery &SQ) {
184 // Look for an inverted mask: (X & ~M) op (Y & M).
185 {
186 Value *M;
187 if (match(V: LHS, P: m_c_And(L: m_Not(V: m_Value(V&: M)), R: m_Value())) &&
188 match(V: RHS, P: m_c_And(L: m_Specific(V: M), R: m_Value())))
189 return isGuaranteedNotToBeUndef(V: M, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT)
190 ? NoCommonBitsSetResult::Known
191 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
192 }
193
194 // X op (Y & ~X)
195 if (match(V: RHS, P: m_c_And(L: m_Not(V: m_Specific(V: LHS)), R: m_Value())))
196 return isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT)
197 ? NoCommonBitsSetResult::Known
198 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
199
200 // X op ((X & Y) ^ Y) -- this is the canonical form of the previous pattern
201 // for constant Y.
202 Value *Y;
203 if (match(V: RHS,
204 P: m_c_Xor(L: m_c_And(L: m_Specific(V: LHS), R: m_Value(V&: Y)), R: m_Deferred(V: Y)))) {
205 bool IsNoUndef = isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT) &&
206 isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT);
207 return IsNoUndef ? NoCommonBitsSetResult::Known
208 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
209 }
210
211 // Peek through extends to find a 'not' of the other side:
212 // (ext Y) op ext(~Y)
213 if (match(V: LHS, P: m_ZExtOrSExt(Op: m_Value(V&: Y))) &&
214 match(V: RHS, P: m_ZExtOrSExt(Op: m_Not(V: m_Specific(V: Y)))))
215 return isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT)
216 ? NoCommonBitsSetResult::Known
217 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
218
219 // Look for: (A & B) op ~(A | B)
220 {
221 Value *A, *B;
222 if (match(V: LHS, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
223 match(V: RHS, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))) {
224 bool IsNoUndef = isGuaranteedNotToBeUndef(V: A, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT) &&
225 isGuaranteedNotToBeUndef(V: B, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT);
226 return IsNoUndef ? NoCommonBitsSetResult::Known
227 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
228 }
229 }
230
231 // Look for: (X << V) op (Y >> (BitWidth - V))
232 // or (X >> V) op (Y << (BitWidth - V))
233 {
234 const Value *V;
235 const APInt *R;
236 if (((match(V: RHS, P: m_Shl(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
237 match(V: LHS, P: m_LShr(L: m_Value(), R: m_Specific(V)))) ||
238 (match(V: RHS, P: m_LShr(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
239 match(V: LHS, P: m_Shl(L: m_Value(), R: m_Specific(V))))) &&
240 R->uge(RHS: LHS->getType()->getScalarSizeInBits()))
241 return NoCommonBitsSetResult::Known;
242 }
243
244 return NoCommonBitsSetResult::Unknown;
245}
246
247NoCommonBitsSetResult
248llvm::getNoCommonBitsSetResult(const WithCache<const Value *> &LHSCache,
249 const WithCache<const Value *> &RHSCache,
250 const SimplifyQuery &SQ) {
251 const Value *LHS = LHSCache.getValue();
252 const Value *RHS = RHSCache.getValue();
253
254 assert(LHS->getType() == RHS->getType() &&
255 "LHS and RHS should have the same type");
256 assert(LHS->getType()->isIntOrIntVectorTy() &&
257 "LHS and RHS should be integers");
258
259 NoCommonBitsSetResult Result = haveNoCommonBitsSetSpecialCases(LHS, RHS, SQ);
260 if (Result == NoCommonBitsSetResult::Known)
261 return NoCommonBitsSetResult::Known;
262
263 NoCommonBitsSetResult CommuteResult =
264 haveNoCommonBitsSetSpecialCases(LHS: RHS, RHS: LHS, SQ);
265 if (CommuteResult == NoCommonBitsSetResult::Known)
266 return NoCommonBitsSetResult::Known;
267
268 if (KnownBits::haveNoCommonBitsSet(LHS: LHSCache.getKnownBits(Q: SQ),
269 RHS: RHSCache.getKnownBits(Q: SQ)))
270 return NoCommonBitsSetResult::Known;
271
272 if (Result == NoCommonBitsSetResult::OnlyIfUndefIgnored ||
273 CommuteResult == NoCommonBitsSetResult::OnlyIfUndefIgnored)
274 return NoCommonBitsSetResult::OnlyIfUndefIgnored;
275
276 return NoCommonBitsSetResult::Unknown;
277}
278
279bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
280 const WithCache<const Value *> &RHSCache,
281 const SimplifyQuery &SQ) {
282 NoCommonBitsSetResult Result =
283 getNoCommonBitsSetResult(LHSCache, RHSCache, SQ);
284 return Result == NoCommonBitsSetResult::Known;
285}
286
287bool llvm::isOnlyUsedInZeroComparison(const Instruction *I) {
288 return !I->user_empty() &&
289 all_of(Range: I->users(), P: match_fn(P: m_ICmp(L: m_Value(), R: m_Zero())));
290}
291
292bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) {
293 return !I->user_empty() && all_of(Range: I->users(), P: [](const User *U) {
294 CmpPredicate P;
295 return match(V: U, P: m_ICmp(Pred&: P, L: m_Value(), R: m_Zero())) && ICmpInst::isEquality(P);
296 });
297}
298
299bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
300 bool OrZero, AssumptionCache *AC,
301 const Instruction *CxtI,
302 const DominatorTree *DT, bool UseInstrInfo,
303 unsigned Depth) {
304 return ::isKnownToBeAPowerOfTwo(
305 V, OrZero, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo),
306 Depth);
307}
308
309static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
310 const SimplifyQuery &Q, unsigned Depth);
311
312bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
313 unsigned Depth) {
314 return computeKnownBits(V, Q: SQ, Depth).isNonNegative();
315}
316
317bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
318 unsigned Depth) {
319 if (auto *CI = dyn_cast<ConstantInt>(Val: V))
320 return CI->getValue().isStrictlyPositive();
321
322 // If `isKnownNonNegative` ever becomes more sophisticated, make sure to keep
323 // this updated.
324 KnownBits Known = computeKnownBits(V, Q: SQ, Depth);
325 return Known.isNonNegative() &&
326 (Known.isNonZero() || isKnownNonZero(V, Q: SQ, Depth));
327}
328
329bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ,
330 unsigned Depth) {
331 return computeKnownBits(V, Q: SQ, Depth).isNegative();
332}
333
334static bool isKnownNonEqual(const Value *V1, const Value *V2,
335 const APInt &DemandedElts, const SimplifyQuery &Q,
336 unsigned Depth);
337
338bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
339 const SimplifyQuery &Q, unsigned Depth) {
340 // We don't support looking through casts.
341 if (V1 == V2 || V1->getType() != V2->getType())
342 return false;
343 auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType());
344 APInt DemandedElts =
345 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
346 return ::isKnownNonEqual(V1, V2, DemandedElts, Q, Depth);
347}
348
349bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
350 const SimplifyQuery &SQ, unsigned Depth) {
351 KnownBits Known(Mask.getBitWidth());
352 computeKnownBits(V, Known, Q: SQ, Depth);
353 return Mask.isSubsetOf(RHS: Known.Zero);
354}
355
356static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
357 const SimplifyQuery &Q, unsigned Depth);
358
359static unsigned ComputeNumSignBits(const Value *V, const SimplifyQuery &Q,
360 unsigned Depth = 0) {
361 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
362 APInt DemandedElts =
363 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
364 return ComputeNumSignBits(V, DemandedElts, Q, Depth);
365}
366
367unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL,
368 AssumptionCache *AC, const Instruction *CxtI,
369 const DominatorTree *DT, bool UseInstrInfo,
370 unsigned Depth) {
371 return ::ComputeNumSignBits(
372 V, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
373}
374
375unsigned llvm::ComputeMaxSignificantBits(const Value *V, const DataLayout &DL,
376 AssumptionCache *AC,
377 const Instruction *CxtI,
378 const DominatorTree *DT,
379 unsigned Depth) {
380 unsigned SignBits = ComputeNumSignBits(V, DL, AC, CxtI, DT, UseInstrInfo: Depth);
381 return V->getType()->getScalarSizeInBits() - SignBits + 1;
382}
383
384/// Try to detect the lerp pattern: a * (b - c) + c * d
385/// where a >= 0, b >= 0, c >= 0, d >= 0, and b >= c.
386///
387/// In that particular case, we can use the following chain of reasoning:
388///
389/// a * (b - c) + c * d <= a' * (b - c) + a' * c = a' * b where a' = max(a, d)
390///
391/// Since that is true for arbitrary a, b, c and d within our constraints, we
392/// can conclude that:
393///
394/// max(a * (b - c) + c * d) <= max(max(a), max(d)) * max(b) = U
395///
396/// Considering that any result of the lerp would be less or equal to U, it
397/// would have at least the number of leading 0s as in U.
398///
399/// While being quite a specific situation, it is fairly common in computer
400/// graphics in the shape of alpha blending.
401///
402/// Modifies given KnownOut in-place with the inferred information.
403static void computeKnownBitsFromLerpPattern(const Value *Op0, const Value *Op1,
404 const APInt &DemandedElts,
405 KnownBits &KnownOut,
406 const SimplifyQuery &Q,
407 unsigned Depth) {
408
409 Type *Ty = Op0->getType();
410 const unsigned BitWidth = Ty->getScalarSizeInBits();
411
412 // Only handle scalar types for now
413 if (Ty->isVectorTy())
414 return;
415
416 // Try to match: a * (b - c) + c * d.
417 // When a == 1 => A == nullptr, the same applies to d/D as well.
418 const Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
419 const Instruction *SubBC = nullptr;
420
421 const auto MatchSubBC = [&]() {
422 // (b - c) can have two forms that interest us:
423 //
424 // 1. sub nuw %b, %c
425 // 2. xor %c, %b
426 //
427 // For the first case, nuw flag guarantees our requirement b >= c.
428 //
429 // The second case might happen when the analysis can infer that b is a mask
430 // for c and we can transform sub operation into xor (that is usually true
431 // for constant b's). Even though xor is symmetrical, canonicalization
432 // ensures that the constant will be the RHS. We have additional checks
433 // later on to ensure that this xor operation is equivalent to subtraction.
434 return m_Instruction(I&: SubBC, P: m_CombineOr(Ps: m_NUWSub(L: m_Value(V&: B), R: m_Value(V&: C)),
435 Ps: m_Xor(L: m_Value(V&: C), R: m_Value(V&: B))));
436 };
437
438 const auto MatchASubBC = [&]() {
439 // Cases:
440 // - a * (b - c)
441 // - (b - c) * a
442 // - (b - c) <- a implicitly equals 1
443 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: A), R: MatchSubBC()), Ps: MatchSubBC());
444 };
445
446 const auto MatchCD = [&]() {
447 // Cases:
448 // - d * c
449 // - c * d
450 // - c <- d implicitly equals 1
451 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: D), R: m_Specific(V: C)), Ps: m_Specific(V: C));
452 };
453
454 const auto Match = [&](const Value *LHS, const Value *RHS) {
455 // We do use m_Specific(C) in MatchCD, so we have to make sure that
456 // it's bound to anything and match(LHS, MatchASubBC()) absolutely
457 // has to evaluate first and return true.
458 //
459 // If Match returns true, it is guaranteed that B != nullptr, C != nullptr.
460 return match(V: LHS, P: MatchASubBC()) && match(V: RHS, P: MatchCD());
461 };
462
463 if (!Match(Op0, Op1) && !Match(Op1, Op0))
464 return;
465
466 const auto ComputeKnownBitsOrOne = [&](const Value *V) {
467 // For some of the values we use the convention of leaving
468 // it nullptr to signify an implicit constant 1.
469 return V ? computeKnownBits(V, DemandedElts, Q, Depth: Depth + 1)
470 : KnownBits::makeConstant(C: APInt(BitWidth, 1));
471 };
472
473 // Check that all operands are non-negative
474 const KnownBits KnownA = ComputeKnownBitsOrOne(A);
475 if (!KnownA.isNonNegative())
476 return;
477
478 const KnownBits KnownD = ComputeKnownBitsOrOne(D);
479 if (!KnownD.isNonNegative())
480 return;
481
482 const KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
483 if (!KnownB.isNonNegative())
484 return;
485
486 const KnownBits KnownC = computeKnownBits(V: C, DemandedElts, Q, Depth: Depth + 1);
487 if (!KnownC.isNonNegative())
488 return;
489
490 // If we matched subtraction as xor, we need to actually check that xor
491 // is semantically equivalent to subtraction.
492 //
493 // For that to be true, b has to be a mask for c or that b's known
494 // ones cover all known and possible ones of c.
495 if (SubBC->getOpcode() == Instruction::Xor &&
496 !KnownC.getMaxValue().isSubsetOf(RHS: KnownB.getMinValue()))
497 return;
498
499 const APInt MaxA = KnownA.getMaxValue();
500 const APInt MaxD = KnownD.getMaxValue();
501 const APInt MaxAD = APIntOps::umax(A: MaxA, B: MaxD);
502 const APInt MaxB = KnownB.getMaxValue();
503
504 // We can't infer leading zeros info if the upper-bound estimate wraps.
505 bool Overflow;
506 const APInt UpperBound = MaxAD.umul_ov(RHS: MaxB, Overflow);
507
508 if (Overflow)
509 return;
510
511 // If we know that x <= y and both are positive than x has at least the same
512 // number of leading zeros as y.
513 const unsigned MinimumNumberOfLeadingZeros = UpperBound.countl_zero();
514 KnownOut.Zero.setHighBits(MinimumNumberOfLeadingZeros);
515}
516
517static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
518 bool NSW, bool NUW,
519 const APInt &DemandedElts,
520 KnownBits &KnownOut, KnownBits &Known2,
521 const SimplifyQuery &Q, unsigned Depth) {
522 computeKnownBits(V: Op1, DemandedElts, Known&: KnownOut, Q, Depth: Depth + 1);
523
524 // If one operand is unknown and we have no nowrap information,
525 // the result will be unknown independently of the second operand.
526 if (KnownOut.isUnknown() && !NSW && !NUW)
527 return;
528
529 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
530 KnownOut = KnownBits::computeForAddSub(Add, NSW, NUW, LHS: Known2, RHS: KnownOut);
531
532 if (!Add && NSW && !KnownOut.isNonNegative() &&
533 (isImpliedByDomCondition(Pred: ICmpInst::ICMP_SLE, LHS: Op1, RHS: Op0, ContextI: Q.CxtI, DL: Q.DL)
534 .value_or(u: false) ||
535 match(V: Op1, P: m_c_SMin(L: m_Specific(V: Op0), R: m_Value()))))
536 KnownOut.makeNonNegative();
537
538 if (Add)
539 // Try to match lerp pattern and combine results
540 computeKnownBitsFromLerpPattern(Op0, Op1, DemandedElts, KnownOut, Q, Depth);
541}
542
543static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
544 bool NUW, const APInt &DemandedElts,
545 KnownBits &Known, KnownBits &Known2,
546 const SimplifyQuery &Q, unsigned Depth) {
547 computeKnownBits(V: Op1, DemandedElts, Known, Q, Depth: Depth + 1);
548 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
549
550 bool isKnownNegative = false;
551 bool isKnownNonNegative = false;
552 // If the multiplication is known not to overflow, compute the sign bit.
553 if (NSW) {
554 if (Op0 == Op1) {
555 // The product of a number with itself is non-negative.
556 isKnownNonNegative = true;
557 } else {
558 bool isKnownNonNegativeOp1 = Known.isNonNegative();
559 bool isKnownNonNegativeOp0 = Known2.isNonNegative();
560 bool isKnownNegativeOp1 = Known.isNegative();
561 bool isKnownNegativeOp0 = Known2.isNegative();
562 // The product of two numbers with the same sign is non-negative.
563 isKnownNonNegative = (isKnownNegativeOp1 && isKnownNegativeOp0) ||
564 (isKnownNonNegativeOp1 && isKnownNonNegativeOp0);
565 if (!isKnownNonNegative && NUW) {
566 // mul nuw nsw with a factor > 1 is non-negative.
567 KnownBits One = KnownBits::makeConstant(C: APInt(Known.getBitWidth(), 1));
568 isKnownNonNegative = KnownBits::sgt(LHS: Known, RHS: One).value_or(u: false) ||
569 KnownBits::sgt(LHS: Known2, RHS: One).value_or(u: false);
570 }
571
572 // The product of a negative number and a non-negative number is either
573 // negative or zero.
574 if (!isKnownNonNegative)
575 isKnownNegative =
576 (isKnownNegativeOp1 && isKnownNonNegativeOp0 &&
577 Known2.isNonZero()) ||
578 (isKnownNegativeOp0 && isKnownNonNegativeOp1 && Known.isNonZero());
579 }
580 }
581
582 bool SelfMultiply = Op0 == Op1;
583 if (SelfMultiply)
584 SelfMultiply &=
585 isGuaranteedNotToBeUndef(V: Op0, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1);
586 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
587
588 if (SelfMultiply) {
589 unsigned SignBits = ComputeNumSignBits(V: Op0, DemandedElts, Q, Depth: Depth + 1);
590 unsigned TyBits = Op0->getType()->getScalarSizeInBits();
591 unsigned OutValidBits = 2 * (TyBits - SignBits + 1);
592
593 if (OutValidBits < TyBits) {
594 APInt KnownZeroMask =
595 APInt::getHighBitsSet(numBits: TyBits, hiBitsSet: TyBits - OutValidBits + 1);
596 Known.Zero |= KnownZeroMask;
597 }
598 }
599
600 // Only make use of no-wrap flags if we failed to compute the sign bit
601 // directly. This matters if the multiplication always overflows, in
602 // which case we prefer to follow the result of the direct computation,
603 // though as the program is invoking undefined behaviour we can choose
604 // whatever we like here.
605 if (isKnownNonNegative && !Known.isNegative())
606 Known.makeNonNegative();
607 else if (isKnownNegative && !Known.isNonNegative())
608 Known.makeNegative();
609}
610
611void llvm::computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
612 KnownBits &Known) {
613 unsigned BitWidth = Known.getBitWidth();
614 unsigned NumRanges = Ranges.getNumOperands() / 2;
615 assert(NumRanges >= 1);
616
617 Known.setAllConflict();
618
619 for (unsigned i = 0; i < NumRanges; ++i) {
620 ConstantInt *Lower =
621 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 0));
622 ConstantInt *Upper =
623 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 1));
624 ConstantRange Range(Lower->getValue(), Upper->getValue());
625 // BitWidth must equal the Ranges BitWidth for the correct number of high
626 // bits to be set.
627 assert(BitWidth == Range.getBitWidth() &&
628 "Known bit width must match range bit width!");
629
630 // The first CommonPrefixBits of all values in Range are equal.
631 unsigned CommonPrefixBits =
632 (Range.getUnsignedMax() ^ Range.getUnsignedMin()).countl_zero();
633 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: CommonPrefixBits);
634 APInt UnsignedMax = Range.getUnsignedMax().zextOrTrunc(width: BitWidth);
635 Known.One &= UnsignedMax & Mask;
636 Known.Zero &= ~UnsignedMax & Mask;
637 }
638}
639
640static bool isEphemeralValueOf(const Instruction *I, const Value *E) {
641 // The instruction defining an assumption's condition itself is always
642 // considered ephemeral to that assumption (even if it has other
643 // non-ephemeral users). See r246696's test case for an example.
644 if (is_contained(Range: I->operands(), Element: E))
645 return true;
646
647 const auto *EI = dyn_cast<Instruction>(Val: E);
648 if (!EI)
649 return false;
650
651 if (EI == I)
652 return true;
653
654 SmallPtrSet<const Instruction *, 16> Visited;
655 SmallVector<const Instruction *, 16> WorkList;
656 Visited.insert(Ptr: EI);
657 WorkList.push_back(Elt: EI);
658 bool ReachesI = false;
659 while (!WorkList.empty()) {
660 const Instruction *V = WorkList.pop_back_val();
661 for (const User *U : V->users()) {
662 const auto *UI = cast<Instruction>(Val: U);
663 if (UI == I) {
664 ReachesI = true;
665 continue;
666 }
667 if (UI->mayHaveSideEffects() || UI->isTerminator())
668 return false;
669 if (Visited.insert(Ptr: UI).second)
670 WorkList.push_back(Elt: UI);
671 }
672 }
673 return ReachesI;
674}
675
676// Is this an intrinsic that cannot be speculated but also cannot trap?
677bool llvm::isAssumeLikeIntrinsic(const Instruction *I) {
678 if (const IntrinsicInst *CI = dyn_cast<IntrinsicInst>(Val: I))
679 return CI->isAssumeLikeIntrinsic();
680
681 return false;
682}
683
684bool llvm::isValidAssumeForContext(const Instruction *Inv,
685 const Instruction *CxtI,
686 const DominatorTree *DT,
687 bool AllowEphemerals) {
688 // There are two restrictions on the use of an assume:
689 // 1. The assume must dominate the context (or the control flow must
690 // reach the assume whenever it reaches the context).
691 // 2. The context must not be in the assume's set of ephemeral values
692 // (otherwise we will use the assume to prove that the condition
693 // feeding the assume is trivially true, thus causing the removal of
694 // the assume).
695
696 if (Inv->getParent() == CxtI->getParent()) {
697 // If Inv and CtxI are in the same block, check if the assume (Inv) is first
698 // in the BB.
699 if (Inv->comesBefore(Other: CxtI))
700 return true;
701
702 // Don't let an assume affect itself - this would cause the problems
703 // `isEphemeralValueOf` is trying to prevent, and it would also make
704 // the loop below go out of bounds.
705 if (!AllowEphemerals && Inv == CxtI)
706 return false;
707
708 // The context comes first, but they're both in the same block.
709 // Make sure there is nothing in between that might interrupt
710 // the control flow, not even CxtI itself.
711 // We limit the scan distance between the assume and its context instruction
712 // to avoid a compile-time explosion. This limit is chosen arbitrarily, so
713 // it can be adjusted if needed (could be turned into a cl::opt).
714 auto Range = make_range(x: CxtI->getIterator(), y: Inv->getIterator());
715 if (!isGuaranteedToTransferExecutionToSuccessor(Range, ScanLimit: 15))
716 return false;
717
718 return AllowEphemerals || !isEphemeralValueOf(I: Inv, E: CxtI);
719 }
720
721 // Inv and CxtI are in different blocks.
722 if (DT) {
723 if (DT->dominates(Def: Inv, User: CxtI))
724 return true;
725 } else if (Inv->getParent() == CxtI->getParent()->getSinglePredecessor() ||
726 Inv->getParent()->isEntryBlock()) {
727 // We don't have a DT, but this trivially dominates.
728 return true;
729 }
730
731 return false;
732}
733
734bool llvm::willNotFreeBetween(const Instruction *Assume,
735 const Instruction *CtxI) {
736 // Helper to check if there are any calls in the range that may free memory.
737 unsigned NumChecked = 0;
738 auto hasNoFreeInRange = [&NumChecked](auto Range) {
739 for (const Instruction &I : Range) {
740 if (NumChecked++ > MaxInstrsToCheckForFree)
741 return false;
742
743 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
744 if (!CB->hasFnAttr(Kind: Attribute::NoFree))
745 return false;
746 } else if (I.maySynchronize())
747 return false;
748 }
749 return true;
750 };
751
752 const BasicBlock *CtxBB = CtxI->getParent();
753 const BasicBlock *AssumeBB = Assume->getParent();
754 BasicBlock::const_iterator CtxIter = CtxI->getIterator();
755 if (CtxBB == AssumeBB) {
756 // Same block case: check that Assume comes before CtxI.
757 if (Assume != CtxI && !Assume->comesBefore(Other: CtxI))
758 return false;
759 return hasNoFreeInRange(make_range(x: Assume->getIterator(), y: CtxIter));
760 }
761
762 // Handle chain of single-predecessor blocks.
763 const BasicBlock *CurBB = CtxBB;
764 while (true) {
765 if (CurBB == AssumeBB)
766 return hasNoFreeInRange(
767 make_range(x: Assume->getIterator(), y: AssumeBB->end()));
768
769 const BasicBlock *PredBB = CurBB->getSinglePredecessor();
770 if (!PredBB)
771 return false;
772
773 if (!hasNoFreeInRange(make_range(x: CurBB->begin(),
774 y: CurBB == CtxBB ? CtxIter : CurBB->end())))
775 return false;
776 CurBB = PredBB;
777 }
778}
779
780// TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but
781// we still have enough information about `RHS` to conclude non-zero. For
782// example Pred=EQ, RHS=isKnownNonZero. cmpExcludesZero is called in loops
783// so the extra compile time may not be worth it, but possibly a second API
784// should be created for use outside of loops.
785static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) {
786 // v u> y implies v != 0.
787 if (Pred == ICmpInst::ICMP_UGT)
788 return true;
789
790 // Special-case v != 0 to also handle v != null.
791 if (Pred == ICmpInst::ICMP_NE)
792 return match(V: RHS, P: m_Zero());
793
794 // All other predicates - rely on generic ConstantRange handling.
795 const APInt *C;
796 auto Zero = APInt::getZero(numBits: RHS->getType()->getScalarSizeInBits());
797 if (match(V: RHS, P: m_APInt(Res&: C))) {
798 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
799 return !TrueValues.contains(Val: Zero);
800 }
801
802 auto *VC = dyn_cast<ConstantDataVector>(Val: RHS);
803 if (VC == nullptr)
804 return false;
805
806 for (unsigned ElemIdx = 0, NElem = VC->getNumElements(); ElemIdx < NElem;
807 ++ElemIdx) {
808 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(
809 Pred, Other: VC->getElementAsAPInt(i: ElemIdx));
810 if (TrueValues.contains(Val: Zero))
811 return false;
812 }
813 return true;
814}
815
816static void breakSelfRecursivePHI(const Use *U, const PHINode *PHI,
817 Value *&ValOut, Instruction *&CtxIOut,
818 const PHINode **PhiOut = nullptr) {
819 ValOut = U->get();
820 if (ValOut == PHI)
821 return;
822 CtxIOut = PHI->getIncomingBlock(U: *U)->getTerminator();
823 if (PhiOut)
824 *PhiOut = PHI;
825 Value *V;
826 // If the Use is a select of this phi, compute analysis on other arm to break
827 // recursion.
828 // TODO: Min/Max
829 if (match(V: ValOut, P: m_Select(C: m_Value(), L: m_Specific(V: PHI), R: m_Value(V))) ||
830 match(V: ValOut, P: m_Select(C: m_Value(), L: m_Value(V), R: m_Specific(V: PHI))))
831 ValOut = V;
832
833 // Same for select, if this phi is 2-operand phi, compute analysis on other
834 // incoming value to break recursion.
835 // TODO: We could handle any number of incoming edges as long as we only have
836 // two unique values.
837 if (auto *IncPhi = dyn_cast<PHINode>(Val: ValOut);
838 IncPhi && IncPhi->getNumIncomingValues() == 2) {
839 for (int Idx = 0; Idx < 2; ++Idx) {
840 if (IncPhi->getIncomingValue(i: Idx) == PHI) {
841 ValOut = IncPhi->getIncomingValue(i: 1 - Idx);
842 if (PhiOut)
843 *PhiOut = IncPhi;
844 CtxIOut = IncPhi->getIncomingBlock(i: 1 - Idx)->getTerminator();
845 break;
846 }
847 }
848 }
849}
850
851static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) {
852 // Use of assumptions is context-sensitive. If we don't have a context, we
853 // cannot use them!
854 if (!Q.AC || !Q.CxtI)
855 return false;
856
857 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
858 if (!Elem.Assume)
859 continue;
860
861 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
862 assert(I->getFunction() == Q.CxtI->getFunction() &&
863 "Got assumption for the wrong function!");
864
865 if (Elem.Index != AssumptionCache::ExprResultIdx) {
866 if (assumeBundleImpliesNonNull(Val: V, Context: Q.CxtI->getFunction(),
867 OBU: I->getOperandBundleAt(Index: Elem.Index)) &&
868 isValidAssumeForContext(I, Q))
869 return true;
870 continue;
871 }
872
873 // Warning: This loop can end up being somewhat performance sensitive.
874 // We're running this loop for once for each value queried resulting in a
875 // runtime of ~O(#assumes * #values).
876
877 Value *RHS;
878 CmpPredicate Pred;
879 auto m_V = m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToInt(Op: m_Specific(V)));
880 if (!match(V: I->getArgOperand(i: 0), P: m_c_ICmp(Pred, L: m_V, R: m_Value(V&: RHS))))
881 continue;
882
883 if (cmpExcludesZero(Pred, RHS) && isValidAssumeForContext(I, Q))
884 return true;
885 }
886
887 return false;
888}
889
890static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
891 Value *LHS, Value *RHS, KnownBits &Known,
892 const SimplifyQuery &Q) {
893 if (RHS->getType()->isPointerTy()) {
894 // Handle comparison of pointer to null explicitly, as it will not be
895 // covered by the m_APInt() logic below.
896 if (LHS == V && match(V: RHS, P: m_Zero())) {
897 switch (Pred) {
898 case ICmpInst::ICMP_EQ:
899 Known.setAllZero();
900 break;
901 case ICmpInst::ICMP_SGE:
902 case ICmpInst::ICMP_SGT:
903 Known.makeNonNegative();
904 break;
905 case ICmpInst::ICMP_SLT:
906 Known.makeNegative();
907 break;
908 default:
909 break;
910 }
911 }
912 return;
913 }
914
915 unsigned BitWidth = Known.getBitWidth();
916 auto m_V =
917 m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToIntSameSize(DL: Q.DL, Op: m_Specific(V)));
918
919 Value *Y;
920 const APInt *Mask, *C;
921 if (!match(V: RHS, P: m_APInt(Res&: C)))
922 return;
923
924 uint64_t ShAmt;
925 switch (Pred) {
926 case ICmpInst::ICMP_EQ:
927 // assume(V = C)
928 if (match(V: LHS, P: m_V)) {
929 Known = Known.unionWith(RHS: KnownBits::makeConstant(C: *C));
930 // assume(V & Mask = C)
931 } else if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value(V&: Y)))) {
932 // For one bits in Mask, we can propagate bits from C to V.
933 Known.One |= *C;
934 if (match(V: Y, P: m_APInt(Res&: Mask)))
935 Known.Zero |= ~*C & *Mask;
936 // assume(V | Mask = C)
937 } else if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value(V&: Y)))) {
938 // For zero bits in Mask, we can propagate bits from C to V.
939 Known.Zero |= ~*C;
940 if (match(V: Y, P: m_APInt(Res&: Mask)))
941 Known.One |= *C & ~*Mask;
942 // assume(V << ShAmt = C)
943 } else if (match(V: LHS, P: m_Shl(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
944 ShAmt < BitWidth) {
945 // For those bits in C that are known, we can propagate them to known
946 // bits in V shifted to the right by ShAmt.
947 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
948 RHSKnown >>= ShAmt;
949 Known = Known.unionWith(RHS: RHSKnown);
950 // assume(V >> ShAmt = C)
951 } else if (match(V: LHS, P: m_Shr(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
952 ShAmt < BitWidth) {
953 // For those bits in RHS that are known, we can propagate them to known
954 // bits in V shifted to the right by C.
955 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
956 RHSKnown <<= ShAmt;
957 Known = Known.unionWith(RHS: RHSKnown);
958 }
959 break;
960 case ICmpInst::ICMP_NE: {
961 // assume (V & B != 0) where B is a power of 2
962 const APInt *BPow2;
963 if (C->isZero() && match(V: LHS, P: m_And(L: m_V, R: m_Power2(V&: BPow2))))
964 Known.One |= *BPow2;
965 break;
966 }
967 default: {
968 const APInt *Offset = nullptr;
969 if (match(V: LHS, P: m_CombineOr(Ps: m_V, Ps: m_AddLike(L: m_V, R: m_APInt(Res&: Offset))))) {
970 ConstantRange LHSRange = ConstantRange::makeAllowedICmpRegion(Pred, Other: *C);
971 if (Offset)
972 LHSRange = LHSRange.sub(Other: *Offset);
973 Known = Known.unionWith(RHS: LHSRange.toKnownBits());
974 }
975 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
976 // X & Y u> C -> X u> C && Y u> C
977 // X nuw- Y u> C -> X u> C
978 if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value())) ||
979 match(V: LHS, P: m_NUWSub(L: m_V, R: m_Value())))
980 Known.One.setHighBits(
981 (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
982 }
983 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
984 // X | Y u< C -> X u< C && Y u< C
985 // X nuw+ Y u< C -> X u< C && Y u< C
986 if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value())) ||
987 match(V: LHS, P: m_c_NUWAdd(L: m_V, R: m_Value()))) {
988 Known.Zero.setHighBits(
989 (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
990 }
991 }
992 } break;
993 }
994}
995
996static void computeKnownBitsFromICmpCond(const Value *V, ICmpInst *Cmp,
997 KnownBits &Known,
998 const SimplifyQuery &SQ, bool Invert) {
999 ICmpInst::Predicate Pred =
1000 Invert ? Cmp->getInversePredicate() : Cmp->getPredicate();
1001 Value *LHS = Cmp->getOperand(i_nocapture: 0);
1002 Value *RHS = Cmp->getOperand(i_nocapture: 1);
1003
1004 // Handle icmp pred (trunc V), C
1005 if (match(V: LHS, P: m_Trunc(Op: m_Specific(V)))) {
1006 KnownBits DstKnown(LHS->getType()->getScalarSizeInBits());
1007 computeKnownBitsFromCmp(V: LHS, Pred, LHS, RHS, Known&: DstKnown, Q: SQ);
1008 if (cast<TruncInst>(Val: LHS)->hasNoUnsignedWrap())
1009 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1010 else
1011 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1012 return;
1013 }
1014
1015 computeKnownBitsFromCmp(V, Pred, LHS, RHS, Known, Q: SQ);
1016}
1017
1018static void computeKnownBitsFromCond(const Value *V, Value *Cond,
1019 KnownBits &Known, const SimplifyQuery &SQ,
1020 bool Invert, unsigned Depth) {
1021 Value *A, *B;
1022 if (Depth < MaxAnalysisRecursionDepth &&
1023 match(V: Cond, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
1024 KnownBits Known2(Known.getBitWidth());
1025 KnownBits Known3(Known.getBitWidth());
1026 computeKnownBitsFromCond(V, Cond: A, Known&: Known2, SQ, Invert, Depth: Depth + 1);
1027 computeKnownBitsFromCond(V, Cond: B, Known&: Known3, SQ, Invert, Depth: Depth + 1);
1028 if (Invert ? match(V: Cond, P: m_LogicalOr(L: m_Value(), R: m_Value()))
1029 : match(V: Cond, P: m_LogicalAnd(L: m_Value(), R: m_Value())))
1030 Known2 = Known2.unionWith(RHS: Known3);
1031 else
1032 Known2 = Known2.intersectWith(RHS: Known3);
1033 Known = Known.unionWith(RHS: Known2);
1034 return;
1035 }
1036
1037 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
1038 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ, Invert);
1039 return;
1040 }
1041
1042 if (match(V: Cond, P: m_Trunc(Op: m_Specific(V)))) {
1043 KnownBits DstKnown(1);
1044 if (Invert) {
1045 DstKnown.setAllZero();
1046 } else {
1047 DstKnown.setAllOnes();
1048 }
1049 if (cast<TruncInst>(Val: Cond)->hasNoUnsignedWrap()) {
1050 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1051 return;
1052 }
1053 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1054 return;
1055 }
1056
1057 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A))))
1058 computeKnownBitsFromCond(V, Cond: A, Known, SQ, Invert: !Invert, Depth: Depth + 1);
1059}
1060
1061void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
1062 const SimplifyQuery &Q, unsigned Depth) {
1063 // Handle injected condition.
1064 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
1065 computeKnownBitsFromCond(V, Cond: Q.CC->Cond, Known, SQ: Q, Invert: Q.CC->Invert, Depth);
1066
1067 if (!Q.CxtI)
1068 return;
1069
1070 if (Q.DC && Q.DT) {
1071 // Handle dominating conditions.
1072 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
1073 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
1074 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
1075 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1076 /*Invert*/ false, Depth);
1077
1078 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
1079 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
1080 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1081 /*Invert*/ true, Depth);
1082 }
1083
1084 if (Known.hasConflict())
1085 Known.resetAll();
1086 }
1087
1088 if (!Q.AC)
1089 return;
1090
1091 unsigned BitWidth = Known.getBitWidth();
1092
1093 // Note that the patterns below need to be kept in sync with the code
1094 // in AssumptionCache::updateAffectedValues.
1095
1096 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
1097 if (!Elem.Assume)
1098 continue;
1099
1100 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
1101 assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() &&
1102 "Got assumption for the wrong function!");
1103
1104 if (Elem.Index != AssumptionCache::ExprResultIdx) {
1105 if (auto OBU = I->getOperandBundleAt(Index: Elem.Index);
1106 getBundleAttrFromOBU(OBU) == BundleAttr::Align) {
1107 auto [Ptr, _, _2, Alignment, Offset] = getAssumeAlignInfo(OBU);
1108 if (Ptr == V && Alignment && Offset && isPowerOf2_64(Value: *Alignment) &&
1109 isValidAssumeForContext(I, Q)) {
1110 Known.Zero |= (*Alignment - 1) & ~*Offset;
1111 Known.One |= (*Alignment - 1) & *Offset;
1112 }
1113 }
1114 continue;
1115 }
1116
1117 // Warning: This loop can end up being somewhat performance sensitive.
1118 // We're running this loop for once for each value queried resulting in a
1119 // runtime of ~O(#assumes * #values).
1120
1121 Value *Arg = I->getArgOperand(i: 0);
1122
1123 if (Arg == V && isValidAssumeForContext(I, Q)) {
1124 assert(BitWidth == 1 && "assume operand is not i1?");
1125 (void)BitWidth;
1126 Known.setAllOnes();
1127 return;
1128 }
1129 if (match(V: Arg, P: m_Not(V: m_Specific(V))) &&
1130 isValidAssumeForContext(I, Q)) {
1131 assert(BitWidth == 1 && "assume operand is not i1?");
1132 (void)BitWidth;
1133 Known.setAllZero();
1134 return;
1135 }
1136 auto *Trunc = dyn_cast<TruncInst>(Val: Arg);
1137 if (Trunc && Trunc->getOperand(i_nocapture: 0) == V &&
1138 isValidAssumeForContext(I, Q)) {
1139 if (Trunc->hasNoUnsignedWrap()) {
1140 Known = KnownBits::makeConstant(C: APInt(BitWidth, 1));
1141 return;
1142 }
1143 Known.One.setBit(0);
1144 return;
1145 }
1146
1147 // The remaining tests are all recursive, so bail out if we hit the limit.
1148 if (Depth == MaxAnalysisRecursionDepth)
1149 continue;
1150
1151 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
1152 if (!Cmp)
1153 continue;
1154
1155 if (!isValidAssumeForContext(I, Q))
1156 continue;
1157
1158 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ: Q, /*Invert=*/false);
1159 }
1160
1161 // Conflicting assumption: Undefined behavior will occur on this execution
1162 // path.
1163 if (Known.hasConflict())
1164 Known.resetAll();
1165}
1166
1167/// Compute known bits from a shift operator, including those with a
1168/// non-constant shift amount. Known is the output of this function. Known2 is a
1169/// pre-allocated temporary with the same bit width as Known and on return
1170/// contains the known bit of the shift value source. KF is an
1171/// operator-specific function that, given the known-bits and a shift amount,
1172/// compute the implied known-bits of the shift operator's result respectively
1173/// for that shift amount. The results from calling KF are conservatively
1174/// combined for all permitted shift amounts.
1175static void computeKnownBitsFromShiftOperator(
1176 const Operator *I, const APInt &DemandedElts, KnownBits &Known,
1177 KnownBits &Known2, const SimplifyQuery &Q, unsigned Depth,
1178 function_ref<KnownBits(const KnownBits &, const KnownBits &, bool)> KF) {
1179 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1180 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1181 // To limit compile-time impact, only query isKnownNonZero() if we know at
1182 // least something about the shift amount.
1183 bool ShAmtNonZero =
1184 Known.isNonZero() ||
1185 (Known.getMaxValue().ult(RHS: Known.getBitWidth()) &&
1186 isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1));
1187 Known = KF(Known2, Known, ShAmtNonZero);
1188}
1189
1190static KnownBits
1191getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
1192 const KnownBits &KnownLHS, const KnownBits &KnownRHS,
1193 const SimplifyQuery &Q, unsigned Depth) {
1194 unsigned BitWidth = KnownLHS.getBitWidth();
1195 KnownBits KnownOut(BitWidth);
1196 bool IsAnd = false;
1197 bool HasKnownOne = !KnownLHS.One.isZero() || !KnownRHS.One.isZero();
1198 Value *X = nullptr, *Y = nullptr;
1199
1200 switch (I->getOpcode()) {
1201 case Instruction::And:
1202 KnownOut = KnownLHS & KnownRHS;
1203 IsAnd = true;
1204 // and(x, -x) is common idioms that will clear all but lowest set
1205 // bit. If we have a single known bit in x, we can clear all bits
1206 // above it.
1207 // TODO: instcombine often reassociates independent `and` which can hide
1208 // this pattern. Try to match and(x, and(-x, y)) / and(and(x, y), -x).
1209 if (HasKnownOne && match(V: I, P: m_c_And(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X))))) {
1210 // -(-x) == x so using whichever (LHS/RHS) gets us a better result.
1211 if (KnownLHS.countMaxTrailingZeros() <= KnownRHS.countMaxTrailingZeros())
1212 KnownOut = KnownLHS.blsi();
1213 else
1214 KnownOut = KnownRHS.blsi();
1215 }
1216 break;
1217 case Instruction::Or:
1218 KnownOut = KnownLHS | KnownRHS;
1219 break;
1220 case Instruction::Xor:
1221 KnownOut = KnownLHS ^ KnownRHS;
1222 // xor(x, x-1) is common idioms that will clear all but lowest set
1223 // bit. If we have a single known bit in x, we can clear all bits
1224 // above it.
1225 // TODO: xor(x, x-1) is often rewritting as xor(x, x-C) where C !=
1226 // -1 but for the purpose of demanded bits (xor(x, x-C) &
1227 // Demanded) == (xor(x, x-1) & Demanded). Extend the xor pattern
1228 // to use arbitrary C if xor(x, x-C) as the same as xor(x, x-1).
1229 if (HasKnownOne &&
1230 match(V: I, P: m_c_Xor(L: m_Value(V&: X), R: m_Add(L: m_Deferred(V: X), R: m_AllOnes())))) {
1231 const KnownBits &XBits = I->getOperand(i: 0) == X ? KnownLHS : KnownRHS;
1232 KnownOut = XBits.blsmsk();
1233 }
1234 break;
1235 default:
1236 llvm_unreachable("Invalid Op used in 'analyzeKnownBitsFromAndXorOr'");
1237 }
1238
1239 // and(x, add (x, -1)) is a common idiom that always clears the low bit;
1240 // xor/or(x, add (x, -1)) is an idiom that will always set the low bit.
1241 // here we handle the more general case of adding any odd number by
1242 // matching the form and/xor/or(x, add(x, y)) where y is odd.
1243 // TODO: This could be generalized to clearing any bit set in y where the
1244 // following bit is known to be unset in y.
1245 if (!KnownOut.Zero[0] && !KnownOut.One[0] &&
1246 (match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_c_Add(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1247 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1248 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Value(V&: Y), R: m_Deferred(V: X)))))) {
1249 KnownBits KnownY(BitWidth);
1250 computeKnownBits(V: Y, DemandedElts, Known&: KnownY, Q, Depth: Depth + 1);
1251 if (KnownY.countMinTrailingOnes() > 0) {
1252 if (IsAnd)
1253 KnownOut.Zero.setBit(0);
1254 else
1255 KnownOut.One.setBit(0);
1256 }
1257 }
1258 return KnownOut;
1259}
1260
1261static KnownBits computeKnownBitsForHorizontalOperation(
1262 const Operator *I, const APInt &DemandedElts, const SimplifyQuery &Q,
1263 unsigned Depth,
1264 const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
1265 KnownBitsFunc) {
1266 APInt DemandedEltsLHS, DemandedEltsRHS;
1267 getHorizDemandedEltsForFirstOperand(VectorBitWidth: Q.DL.getTypeSizeInBits(Ty: I->getType()),
1268 DemandedElts, DemandedLHS&: DemandedEltsLHS,
1269 DemandedRHS&: DemandedEltsRHS);
1270
1271 const auto ComputeForSingleOpFunc =
1272 [Depth, &Q, KnownBitsFunc](const Value *Op, APInt &DemandedEltsOp) {
1273 return KnownBitsFunc(
1274 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp, Q, Depth: Depth + 1),
1275 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp << 1, Q, Depth: Depth + 1));
1276 };
1277
1278 if (DemandedEltsRHS.isZero())
1279 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS);
1280 if (DemandedEltsLHS.isZero())
1281 return ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS);
1282
1283 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS)
1284 .intersectWith(RHS: ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS));
1285}
1286
1287// Public so this can be used in `SimplifyDemandedUseBits`.
1288KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
1289 const KnownBits &KnownLHS,
1290 const KnownBits &KnownRHS,
1291 const SimplifyQuery &SQ,
1292 unsigned Depth) {
1293 auto *FVTy = dyn_cast<FixedVectorType>(Val: I->getType());
1294 APInt DemandedElts =
1295 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
1296
1297 return getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS, KnownRHS, Q: SQ,
1298 Depth);
1299}
1300
1301ConstantRange llvm::getVScaleRange(const Function *F, unsigned BitWidth) {
1302 Attribute Attr = F->getFnAttribute(Kind: Attribute::VScaleRange);
1303 // Without vscale_range, we only know that vscale is non-zero.
1304 if (!Attr.isValid())
1305 return ConstantRange(APInt(BitWidth, 1), APInt::getZero(numBits: BitWidth));
1306
1307 unsigned AttrMin = Attr.getVScaleRangeMin();
1308 // Minimum is larger than vscale width, result is always poison.
1309 if ((unsigned)llvm::bit_width(Value: AttrMin) > BitWidth)
1310 return ConstantRange::getEmpty(BitWidth);
1311
1312 APInt Min(BitWidth, AttrMin);
1313 std::optional<unsigned> AttrMax = Attr.getVScaleRangeMax();
1314 if (!AttrMax || (unsigned)llvm::bit_width(Value: *AttrMax) > BitWidth)
1315 return ConstantRange(Min, APInt::getZero(numBits: BitWidth));
1316
1317 return ConstantRange(Min, APInt(BitWidth, *AttrMax) + 1);
1318}
1319
1320void llvm::adjustKnownBitsForSelectArm(KnownBits &Known, Value *Cond,
1321 Value *Arm, bool Invert,
1322 const SimplifyQuery &Q, unsigned Depth) {
1323 // If we have a constant arm, we are done.
1324 if (Known.isConstant())
1325 return;
1326
1327 // See what condition implies about the bits of the select arm.
1328 KnownBits CondRes(Known.getBitWidth());
1329 computeKnownBitsFromCond(V: Arm, Cond, Known&: CondRes, SQ: Q, Invert, Depth: Depth + 1);
1330 // If we don't get any information from the condition, no reason to
1331 // proceed.
1332 if (CondRes.isUnknown())
1333 return;
1334
1335 // We can have conflict if the condition is dead. I.e if we have
1336 // (x | 64) < 32 ? (x | 64) : y
1337 // we will have conflict at bit 6 from the condition/the `or`.
1338 // In that case just return. Its not particularly important
1339 // what we do, as this select is going to be simplified soon.
1340 CondRes = CondRes.unionWith(RHS: Known);
1341 if (CondRes.hasConflict())
1342 return;
1343
1344 // Finally make sure the information we found is valid. This is relatively
1345 // expensive so it's left for the very end.
1346 if (!isGuaranteedNotToBeUndef(V: Arm, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1))
1347 return;
1348
1349 // Finally, we know we get information from the condition and its valid,
1350 // so return it.
1351 Known = std::move(CondRes);
1352}
1353
1354// Match a signed min+max clamp pattern like smax(smin(In, CHigh), CLow).
1355// Returns the input and lower/upper bounds.
1356static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
1357 const APInt *&CLow, const APInt *&CHigh) {
1358 assert(isa<Operator>(Select) &&
1359 cast<Operator>(Select)->getOpcode() == Instruction::Select &&
1360 "Input should be a Select!");
1361
1362 const Value *LHS = nullptr, *RHS = nullptr;
1363 SelectPatternFlavor SPF = matchSelectPattern(V: Select, LHS, RHS).Flavor;
1364 if (SPF != SPF_SMAX && SPF != SPF_SMIN)
1365 return false;
1366
1367 if (!match(V: RHS, P: m_APInt(Res&: CLow)))
1368 return false;
1369
1370 const Value *LHS2 = nullptr, *RHS2 = nullptr;
1371 SelectPatternFlavor SPF2 = matchSelectPattern(V: LHS, LHS&: LHS2, RHS&: RHS2).Flavor;
1372 if (getInverseMinMaxFlavor(SPF) != SPF2)
1373 return false;
1374
1375 if (!match(V: RHS2, P: m_APInt(Res&: CHigh)))
1376 return false;
1377
1378 if (SPF == SPF_SMIN)
1379 std::swap(a&: CLow, b&: CHigh);
1380
1381 In = LHS2;
1382 return CLow->sle(RHS: *CHigh);
1383}
1384
1385static bool isSignedMinMaxIntrinsicClamp(const IntrinsicInst *II,
1386 const APInt *&CLow,
1387 const APInt *&CHigh) {
1388 assert((II->getIntrinsicID() == Intrinsic::smin ||
1389 II->getIntrinsicID() == Intrinsic::smax) &&
1390 "Must be smin/smax");
1391
1392 Intrinsic::ID InverseID = getInverseMinMaxIntrinsic(MinMaxID: II->getIntrinsicID());
1393 auto *InnerII = dyn_cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
1394 if (!InnerII || InnerII->getIntrinsicID() != InverseID ||
1395 !match(V: II->getArgOperand(i: 1), P: m_APInt(Res&: CLow)) ||
1396 !match(V: InnerII->getArgOperand(i: 1), P: m_APInt(Res&: CHigh)))
1397 return false;
1398
1399 if (II->getIntrinsicID() == Intrinsic::smin)
1400 std::swap(a&: CLow, b&: CHigh);
1401 return CLow->sle(RHS: *CHigh);
1402}
1403
1404static void unionWithMinMaxIntrinsicClamp(const IntrinsicInst *II,
1405 KnownBits &Known) {
1406 const APInt *CLow, *CHigh;
1407 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
1408 Known = Known.unionWith(
1409 RHS: ConstantRange::getNonEmpty(Lower: *CLow, Upper: *CHigh + 1).toKnownBits());
1410}
1411
1412static void computeKnownBitsFromOperator(const Operator *I,
1413 const APInt &DemandedElts,
1414 KnownBits &Known,
1415 const SimplifyQuery &Q,
1416 unsigned Depth) {
1417 unsigned BitWidth = Known.getBitWidth();
1418
1419 KnownBits Known2(BitWidth);
1420 switch (I->getOpcode()) {
1421 default: break;
1422 case Instruction::Load:
1423 if (MDNode *MD =
1424 Q.IIQ.getMetadata(I: cast<LoadInst>(Val: I), KindID: LLVMContext::MD_range))
1425 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
1426 break;
1427 case Instruction::And:
1428 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1429 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1430
1431 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1432 break;
1433 case Instruction::Or:
1434 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1435 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1436
1437 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1438 break;
1439 case Instruction::Xor:
1440 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1441 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1442
1443 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1444 break;
1445 case Instruction::Mul: {
1446 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1447 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1448 computeKnownBitsMul(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1449 DemandedElts, Known, Known2, Q, Depth);
1450 break;
1451 }
1452 case Instruction::UDiv: {
1453 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1454 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1455 Known =
1456 KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1457 break;
1458 }
1459 case Instruction::SDiv: {
1460 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1461 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1462 Known =
1463 KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1464 break;
1465 }
1466 case Instruction::Select: {
1467 auto ComputeForArm = [&](Value *Arm, bool Invert) {
1468 KnownBits Res(Known.getBitWidth());
1469 computeKnownBits(V: Arm, DemandedElts, Known&: Res, Q, Depth: Depth + 1);
1470 adjustKnownBitsForSelectArm(Known&: Res, Cond: I->getOperand(i: 0), Arm, Invert, Q, Depth);
1471 return Res;
1472 };
1473 // Only known if known in both the LHS and RHS.
1474 Known =
1475 ComputeForArm(I->getOperand(i: 1), /*Invert=*/false)
1476 .intersectWith(RHS: ComputeForArm(I->getOperand(i: 2), /*Invert=*/true));
1477 break;
1478 }
1479 case Instruction::FPTrunc:
1480 case Instruction::FPExt:
1481 case Instruction::FPToUI:
1482 case Instruction::FPToSI:
1483 case Instruction::SIToFP:
1484 case Instruction::UIToFP:
1485 break; // Can't work with floating point.
1486 case Instruction::PtrToInt:
1487 case Instruction::PtrToAddr:
1488 case Instruction::IntToPtr:
1489 // Fall through and handle them the same as zext/trunc.
1490 [[fallthrough]];
1491 case Instruction::ZExt:
1492 case Instruction::Trunc: {
1493 Type *SrcTy = I->getOperand(i: 0)->getType();
1494
1495 unsigned SrcBitWidth;
1496 // Note that we handle pointer operands here because of inttoptr/ptrtoint
1497 // which fall through here.
1498 Type *ScalarTy = SrcTy->getScalarType();
1499 SrcBitWidth = ScalarTy->isPointerTy() ?
1500 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
1501 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
1502
1503 assert(SrcBitWidth && "SrcBitWidth can't be zero");
1504 Known = Known.anyextOrTrunc(BitWidth: SrcBitWidth);
1505 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1506 if (auto *Inst = dyn_cast<PossiblyNonNegInst>(Val: I);
1507 Inst && Inst->hasNonNeg() && !Known.isNegative())
1508 Known.makeNonNegative();
1509 Known = Known.zextOrTrunc(BitWidth);
1510 break;
1511 }
1512 case Instruction::BitCast: {
1513 Type *SrcTy = I->getOperand(i: 0)->getType();
1514 if (SrcTy->isIntOrPtrTy() &&
1515 // TODO: For now, not handling conversions like:
1516 // (bitcast i64 %x to <2 x i32>)
1517 !I->getType()->isVectorTy()) {
1518 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1519 break;
1520 }
1521
1522 const Value *V;
1523 // Handle bitcast from floating point to integer.
1524 if (match(V: I, P: m_ElementWiseBitCast(Op: m_Value(V))) &&
1525 V->getType()->isFPOrFPVectorTy()) {
1526 Type *FPType = V->getType()->getScalarType();
1527 KnownFPClass Result =
1528 computeKnownFPClass(V, DemandedElts, InterestedClasses: fcAllFlags, SQ: Q, Depth: Depth + 1);
1529 FPClassTest FPClasses = Result.KnownFPClasses;
1530
1531 // TODO: Treat it as zero/poison if the use of I is unreachable.
1532 if (FPClasses == fcNone)
1533 break;
1534
1535 if (Result.isKnownNever(Mask: fcNormal | fcSubnormal | fcNan)) {
1536 Known.setAllConflict();
1537
1538 if (FPClasses & fcInf)
1539 Known = Known.intersectWith(RHS: KnownBits::makeConstant(
1540 C: APFloat::getInf(Sem: FPType->getFltSemantics()).bitcastToAPInt()));
1541
1542 if (FPClasses & fcZero)
1543 Known = Known.intersectWith(RHS: KnownBits::makeConstant(
1544 C: APInt::getZero(numBits: FPType->getScalarSizeInBits())));
1545
1546 Known.Zero.clearSignBit();
1547 Known.One.clearSignBit();
1548 }
1549
1550 if (Result.SignBit) {
1551 if (*Result.SignBit)
1552 Known.makeNegative();
1553 else
1554 Known.makeNonNegative();
1555 }
1556
1557 break;
1558 }
1559
1560 // Handle cast from vector integer type to scalar or vector integer.
1561 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy);
1562 if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
1563 !I->getType()->isIntOrIntVectorTy() ||
1564 isa<ScalableVectorType>(Val: I->getType()))
1565 break;
1566
1567 unsigned NumElts = DemandedElts.getBitWidth();
1568 bool IsLE = Q.DL.isLittleEndian();
1569 // Look through a cast from narrow vector elements to wider type.
1570 // Examples: v4i32 -> v2i64, v3i8 -> v24
1571 unsigned SubBitWidth = SrcVecTy->getScalarSizeInBits();
1572 if (BitWidth % SubBitWidth == 0) {
1573 // Known bits are automatically intersected across demanded elements of a
1574 // vector. So for example, if a bit is computed as known zero, it must be
1575 // zero across all demanded elements of the vector.
1576 //
1577 // For this bitcast, each demanded element of the output is sub-divided
1578 // across a set of smaller vector elements in the source vector. To get
1579 // the known bits for an entire element of the output, compute the known
1580 // bits for each sub-element sequentially. This is done by shifting the
1581 // one-set-bit demanded elements parameter across the sub-elements for
1582 // consecutive calls to computeKnownBits. We are using the demanded
1583 // elements parameter as a mask operator.
1584 //
1585 // The known bits of each sub-element are then inserted into place
1586 // (dependent on endian) to form the full result of known bits.
1587 unsigned SubScale = BitWidth / SubBitWidth;
1588 APInt SubDemandedElts = APInt::getZero(numBits: NumElts * SubScale);
1589 for (unsigned i = 0; i != NumElts; ++i) {
1590 if (DemandedElts[i])
1591 SubDemandedElts.setBit(i * SubScale);
1592 }
1593
1594 KnownBits KnownSrc(SubBitWidth);
1595 for (unsigned i = 0; i != SubScale; ++i) {
1596 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts.shl(shiftAmt: i), Known&: KnownSrc, Q,
1597 Depth: Depth + 1);
1598 unsigned ShiftElt = IsLE ? i : SubScale - 1 - i;
1599 Known.insertBits(SubBits: KnownSrc, BitPosition: ShiftElt * SubBitWidth);
1600 }
1601 }
1602 // Look through a cast from wider vector elements to narrow type.
1603 // Examples: v2i64 -> v4i32
1604 if (SubBitWidth % BitWidth == 0) {
1605 unsigned SubScale = SubBitWidth / BitWidth;
1606 KnownBits KnownSrc(SubBitWidth);
1607 APInt SubDemandedElts =
1608 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
1609 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts, Known&: KnownSrc, Q,
1610 Depth: Depth + 1);
1611
1612 Known.setAllConflict();
1613 for (unsigned i = 0; i != NumElts; ++i) {
1614 if (DemandedElts[i]) {
1615 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
1616 unsigned Offset = (Shifts % SubScale) * BitWidth;
1617 Known = Known.intersectWith(RHS: KnownSrc.extractBits(NumBits: BitWidth, BitPosition: Offset));
1618 if (Known.isUnknown())
1619 break;
1620 }
1621 }
1622 }
1623 break;
1624 }
1625 case Instruction::SExt: {
1626 // Compute the bits in the result that are not present in the input.
1627 unsigned SrcBitWidth = I->getOperand(i: 0)->getType()->getScalarSizeInBits();
1628
1629 Known = Known.trunc(BitWidth: SrcBitWidth);
1630 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1631 // If the sign bit of the input is known set or clear, then we know the
1632 // top bits of the result.
1633 Known = Known.sext(BitWidth);
1634 break;
1635 }
1636 case Instruction::Shl: {
1637 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1638 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1639 auto KF = [NUW, NSW](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1640 bool ShAmtNonZero) {
1641 return KnownBits::shl(LHS: KnownVal, RHS: KnownAmt, NUW, NSW, ShAmtNonZero);
1642 };
1643 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1644 KF);
1645 // Trailing zeros of a right-shifted constant never decrease.
1646 const APInt *C;
1647 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1648 Known.Zero.setLowBits(C->countr_zero());
1649
1650 // shl X, sub(Y, xor(ctlz(X, true), BitWidth-1)) shifts X so that its MSB
1651 // lands at bit Y, when BitWidth is a power of 2.
1652 const APInt *YC;
1653 Value *X = I->getOperand(i: 0);
1654 if (isPowerOf2_32(Value: BitWidth) &&
1655 match(V: I->getOperand(i: 1),
1656 P: m_Sub(L: m_APInt(Res&: YC), R: m_Xor(L: m_Ctlz(Op0: m_Specific(V: X), Op1: m_One()),
1657 R: m_SpecificInt(V: BitWidth - 1)))) &&
1658 YC->ult(RHS: BitWidth - 1)) {
1659 unsigned Y = YC->getZExtValue();
1660 Known.One.setBit(Y);
1661 Known.Zero.setBitsFrom(Y + 1);
1662 }
1663 break;
1664 }
1665 case Instruction::LShr: {
1666 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1667 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1668 bool ShAmtNonZero) {
1669 return KnownBits::lshr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1670 };
1671 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1672 KF);
1673 // Leading zeros of a left-shifted constant never decrease.
1674 const APInt *C;
1675 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1676 Known.Zero.setHighBits(C->countl_zero());
1677 break;
1678 }
1679 case Instruction::AShr: {
1680 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1681 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1682 bool ShAmtNonZero) {
1683 return KnownBits::ashr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1684 };
1685 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1686 KF);
1687 break;
1688 }
1689 case Instruction::Sub: {
1690 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1691 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1692 computeKnownBitsAddSub(Add: false, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1693 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1694 break;
1695 }
1696 case Instruction::Add: {
1697 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1698 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1699 computeKnownBitsAddSub(Add: true, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1700 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1701 break;
1702 }
1703 case Instruction::SRem:
1704 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1705 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1706 Known = KnownBits::srem(LHS: Known, RHS: Known2);
1707 break;
1708
1709 case Instruction::URem:
1710 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1711 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1712 Known = KnownBits::urem(LHS: Known, RHS: Known2);
1713 break;
1714 case Instruction::Alloca:
1715 Known.Zero.setLowBits(Log2(A: cast<AllocaInst>(Val: I)->getAlign()));
1716 break;
1717 case Instruction::GetElementPtr: {
1718 // Analyze all of the subscripts of this getelementptr instruction
1719 // to determine if we can prove known low zero bits.
1720 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1721 // Accumulate the constant indices in a separate variable
1722 // to minimize the number of calls to computeForAddSub.
1723 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: I->getType());
1724 APInt AccConstIndices(IndexWidth, 0);
1725
1726 auto AddIndexToKnown = [&](KnownBits IndexBits) {
1727 if (IndexWidth == BitWidth) {
1728 // Note that inbounds does *not* guarantee nsw for the addition, as only
1729 // the offset is signed, while the base address is unsigned.
1730 Known = KnownBits::add(LHS: Known, RHS: IndexBits);
1731 } else {
1732 // If the index width is smaller than the pointer width, only add the
1733 // value to the low bits.
1734 assert(IndexWidth < BitWidth &&
1735 "Index width can't be larger than pointer width");
1736 Known.insertBits(SubBits: KnownBits::add(LHS: Known.trunc(BitWidth: IndexWidth), RHS: IndexBits), BitPosition: 0);
1737 }
1738 };
1739
1740 gep_type_iterator GTI = gep_type_begin(GEP: I);
1741 for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
1742 // TrailZ can only become smaller, short-circuit if we hit zero.
1743 if (Known.isUnknown())
1744 break;
1745
1746 Value *Index = I->getOperand(i);
1747
1748 // Handle case when index is zero.
1749 Constant *CIndex = dyn_cast<Constant>(Val: Index);
1750 if (CIndex && CIndex->isNullValue())
1751 continue;
1752
1753 if (StructType *STy = GTI.getStructTypeOrNull()) {
1754 // Handle struct member offset arithmetic.
1755
1756 assert(CIndex &&
1757 "Access to structure field must be known at compile time");
1758
1759 if (CIndex->getType()->isVectorTy())
1760 Index = CIndex->getSplatValue();
1761
1762 unsigned Idx = cast<ConstantInt>(Val: Index)->getZExtValue();
1763 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
1764 uint64_t Offset = SL->getElementOffset(Idx);
1765 AccConstIndices += Offset;
1766 continue;
1767 }
1768
1769 // Handle array index arithmetic.
1770 Type *IndexedTy = GTI.getIndexedType();
1771 if (!IndexedTy->isSized()) {
1772 Known.resetAll();
1773 break;
1774 }
1775
1776 TypeSize Stride = GTI.getSequentialElementStride(DL: Q.DL);
1777 uint64_t StrideInBytes = Stride.getKnownMinValue();
1778 if (!Stride.isScalable()) {
1779 // Fast path for constant offset.
1780 if (auto *CI = dyn_cast<ConstantInt>(Val: Index)) {
1781 AccConstIndices +=
1782 CI->getValue().sextOrTrunc(width: IndexWidth) * StrideInBytes;
1783 continue;
1784 }
1785 }
1786
1787 KnownBits IndexBits =
1788 computeKnownBits(V: Index, Q, Depth: Depth + 1).sextOrTrunc(BitWidth: IndexWidth);
1789 KnownBits ScalingFactor(IndexWidth);
1790 // Multiply by current sizeof type.
1791 // &A[i] == A + i * sizeof(*A[i]).
1792 if (Stride.isScalable()) {
1793 // For scalable types the only thing we know about sizeof is
1794 // that this is a multiple of the minimum size.
1795 ScalingFactor.Zero.setLowBits(llvm::countr_zero(Val: StrideInBytes));
1796 } else {
1797 ScalingFactor =
1798 KnownBits::makeConstant(C: APInt(IndexWidth, StrideInBytes));
1799 }
1800 AddIndexToKnown(KnownBits::mul(LHS: IndexBits, RHS: ScalingFactor));
1801 }
1802 if (!Known.isUnknown() && !AccConstIndices.isZero())
1803 AddIndexToKnown(KnownBits::makeConstant(C: AccConstIndices));
1804 break;
1805 }
1806 case Instruction::PHI: {
1807 const PHINode *P = cast<PHINode>(Val: I);
1808 BinaryOperator *BO = nullptr;
1809 Value *R = nullptr, *L = nullptr;
1810 if (matchSimpleRecurrence(P, BO, Start&: R, Step&: L)) {
1811 // Handle the case of a simple two-predecessor recurrence PHI.
1812 // There's a lot more that could theoretically be done here, but
1813 // this is sufficient to catch some interesting cases.
1814 unsigned Opcode = BO->getOpcode();
1815
1816 switch (Opcode) {
1817 // If this is a shift recurrence, we know the bits being shifted in. We
1818 // can combine that with information about the start value of the
1819 // recurrence to conclude facts about the result. If this is a udiv
1820 // recurrence, we know that the result can never exceed either the
1821 // numerator or the start value, whichever is greater.
1822 case Instruction::LShr:
1823 case Instruction::AShr:
1824 case Instruction::Shl:
1825 case Instruction::UDiv:
1826 if (BO->getOperand(i_nocapture: 0) != I)
1827 break;
1828 [[fallthrough]];
1829
1830 // For a urem recurrence, the result can never exceed the start value. The
1831 // phi could either be the numerator or the denominator.
1832 case Instruction::URem: {
1833 // We have matched a recurrence of the form:
1834 // %iv = [R, %entry], [%iv.next, %backedge]
1835 // %iv.next = shift_op %iv, L
1836
1837 // Recurse with the phi context to avoid concern about whether facts
1838 // inferred hold at original context instruction. TODO: It may be
1839 // correct to use the original context. IF warranted, explore and
1840 // add sufficient tests to cover.
1841 SimplifyQuery RecQ = Q.getWithoutCondContext();
1842 RecQ.CxtI = P;
1843 computeKnownBits(V: R, DemandedElts, Known&: Known2, Q: RecQ, Depth: Depth + 1);
1844 switch (Opcode) {
1845 case Instruction::Shl:
1846 // A shl recurrence will only increase the tailing zeros
1847 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
1848 break;
1849 case Instruction::LShr:
1850 case Instruction::UDiv:
1851 case Instruction::URem:
1852 // lshr, udiv, and urem recurrences will preserve the leading zeros of
1853 // the start value.
1854 Known.Zero.setHighBits(Known2.countMinLeadingZeros());
1855 break;
1856 case Instruction::AShr:
1857 // An ashr recurrence will extend the initial sign bit
1858 Known.Zero.setHighBits(Known2.countMinLeadingZeros());
1859 Known.One.setHighBits(Known2.countMinLeadingOnes());
1860 break;
1861 }
1862 break;
1863 }
1864
1865 // Check for operations that have the property that if
1866 // both their operands have low zero bits, the result
1867 // will have low zero bits.
1868 case Instruction::Add:
1869 case Instruction::Sub:
1870 case Instruction::And:
1871 case Instruction::Or:
1872 case Instruction::Mul: {
1873 // Change the context instruction to the "edge" that flows into the
1874 // phi. This is important because that is where the value is actually
1875 // "evaluated" even though it is used later somewhere else. (see also
1876 // D69571).
1877 SimplifyQuery RecQ = Q.getWithoutCondContext();
1878
1879 unsigned OpNum = P->getOperand(i_nocapture: 0) == R ? 0 : 1;
1880 Instruction *RInst = P->getIncomingBlock(i: OpNum)->getTerminator();
1881 Instruction *LInst = P->getIncomingBlock(i: 1 - OpNum)->getTerminator();
1882
1883 // Ok, we have a PHI of the form L op= R. Check for low
1884 // zero bits.
1885 RecQ.CxtI = RInst;
1886 computeKnownBits(V: R, DemandedElts, Known&: Known2, Q: RecQ, Depth: Depth + 1);
1887
1888 // We need to take the minimum number of known bits
1889 KnownBits Known3(BitWidth);
1890 RecQ.CxtI = LInst;
1891 computeKnownBits(V: L, DemandedElts, Known&: Known3, Q: RecQ, Depth: Depth + 1);
1892
1893 Known.Zero.setLowBits(std::min(a: Known2.countMinTrailingZeros(),
1894 b: Known3.countMinTrailingZeros()));
1895
1896 auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(Val: BO);
1897 if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(Op: OverflowOp))
1898 break;
1899
1900 switch (Opcode) {
1901 // If initial value of recurrence is nonnegative, and we are adding
1902 // a nonnegative number with nsw, the result can only be nonnegative
1903 // or poison value regardless of the number of times we execute the
1904 // add in phi recurrence. If initial value is negative and we are
1905 // adding a negative number with nsw, the result can only be
1906 // negative or poison value. Similar arguments apply to sub and mul.
1907 //
1908 // (add non-negative, non-negative) --> non-negative
1909 // (add negative, negative) --> negative
1910 case Instruction::Add: {
1911 if (Known2.isNonNegative() && Known3.isNonNegative())
1912 Known.makeNonNegative();
1913 else if (Known2.isNegative() && Known3.isNegative())
1914 Known.makeNegative();
1915 break;
1916 }
1917
1918 // (sub nsw non-negative, negative) --> non-negative
1919 // (sub nsw negative, non-negative) --> negative
1920 case Instruction::Sub: {
1921 if (BO->getOperand(i_nocapture: 0) != I)
1922 break;
1923 if (Known2.isNonNegative() && Known3.isNegative())
1924 Known.makeNonNegative();
1925 else if (Known2.isNegative() && Known3.isNonNegative())
1926 Known.makeNegative();
1927 break;
1928 }
1929
1930 // (mul nsw non-negative, non-negative) --> non-negative
1931 case Instruction::Mul:
1932 if (Known2.isNonNegative() && Known3.isNonNegative())
1933 Known.makeNonNegative();
1934 break;
1935
1936 default:
1937 break;
1938 }
1939 break;
1940 }
1941
1942 default:
1943 break;
1944 }
1945 }
1946
1947 // Unreachable blocks may have zero-operand PHI nodes.
1948 if (P->getNumIncomingValues() == 0)
1949 break;
1950
1951 // Otherwise take the unions of the known bit sets of the operands,
1952 // taking conservative care to avoid excessive recursion.
1953 if (Depth < MaxAnalysisRecursionDepth - 1 && Known.isUnknown()) {
1954 // Skip if every incoming value references to ourself.
1955 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
1956 break;
1957
1958 Known.setAllConflict();
1959 for (const Use &U : P->operands()) {
1960 Value *IncValue;
1961 const PHINode *CxtPhi;
1962 Instruction *CxtI;
1963 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CxtI, PhiOut: &CxtPhi);
1964 // Skip direct self references.
1965 if (IncValue == P)
1966 continue;
1967
1968 // Change the context instruction to the "edge" that flows into the
1969 // phi. This is important because that is where the value is actually
1970 // "evaluated" even though it is used later somewhere else. (see also
1971 // D69571).
1972 SimplifyQuery RecQ = Q.getWithoutCondContext().getWithInstruction(I: CxtI);
1973
1974 Known2 = KnownBits(BitWidth);
1975
1976 // Recurse, but cap the recursion to one level, because we don't
1977 // want to waste time spinning around in loops.
1978 // TODO: See if we can base recursion limiter on number of incoming phi
1979 // edges so we don't overly clamp analysis.
1980 computeKnownBits(V: IncValue, DemandedElts, Known&: Known2, Q: RecQ,
1981 Depth: MaxAnalysisRecursionDepth - 1);
1982
1983 // See if we can further use a conditional branch into the phi
1984 // to help us determine the range of the value.
1985 if (!Known2.isConstant()) {
1986 CmpPredicate Pred;
1987 const APInt *RHSC;
1988 BasicBlock *TrueSucc, *FalseSucc;
1989 // TODO: Use RHS Value and compute range from its known bits.
1990 if (match(V: RecQ.CxtI,
1991 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: IncValue), R: m_APInt(Res&: RHSC)),
1992 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
1993 // Check for cases of duplicate successors.
1994 if ((TrueSucc == CxtPhi->getParent()) !=
1995 (FalseSucc == CxtPhi->getParent())) {
1996 // If we're using the false successor, invert the predicate.
1997 if (FalseSucc == CxtPhi->getParent())
1998 Pred = CmpInst::getInversePredicate(pred: Pred);
1999 // Get the knownbits implied by the incoming phi condition.
2000 auto CR = ConstantRange::makeExactICmpRegion(Pred, Other: *RHSC);
2001 KnownBits KnownUnion = Known2.unionWith(RHS: CR.toKnownBits());
2002 // We can have conflicts here if we are analyzing deadcode (its
2003 // impossible for us reach this BB based the icmp).
2004 if (KnownUnion.hasConflict()) {
2005 // No reason to continue analyzing in a known dead region, so
2006 // just resetAll and break. This will cause us to also exit the
2007 // outer loop.
2008 Known.resetAll();
2009 break;
2010 }
2011 Known2 = KnownUnion;
2012 }
2013 }
2014 }
2015
2016 Known = Known.intersectWith(RHS: Known2);
2017 // If all bits have been ruled out, there's no need to check
2018 // more operands.
2019 if (Known.isUnknown())
2020 break;
2021 }
2022 }
2023 break;
2024 }
2025 case Instruction::Call:
2026 case Instruction::Invoke: {
2027 // If range metadata is attached to this call, set known bits from that,
2028 // and then intersect with known bits based on other properties of the
2029 // function.
2030 if (MDNode *MD =
2031 Q.IIQ.getMetadata(I: cast<Instruction>(Val: I), KindID: LLVMContext::MD_range))
2032 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
2033
2034 const auto *CB = cast<CallBase>(Val: I);
2035
2036 if (std::optional<ConstantRange> Range = CB->getRange())
2037 Known = Known.unionWith(RHS: Range->toKnownBits());
2038
2039 if (const Value *RV = CB->getReturnedArgOperand()) {
2040 if (RV->getType() == I->getType()) {
2041 computeKnownBits(V: RV, Known&: Known2, Q, Depth: Depth + 1);
2042 Known = Known.unionWith(RHS: Known2);
2043 // If the function doesn't return properly for all input values
2044 // (e.g. unreachable exits) then there might be conflicts between the
2045 // argument value and the range metadata. Simply discard the known bits
2046 // in case of conflicts.
2047 if (Known.hasConflict())
2048 Known.resetAll();
2049 }
2050 }
2051 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
2052 switch (II->getIntrinsicID()) {
2053 default:
2054 break;
2055 case Intrinsic::abs: {
2056 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2057 bool IntMinIsPoison = match(V: II->getArgOperand(i: 1), P: m_One());
2058 Known = Known.unionWith(RHS: Known2.abs(IntMinIsPoison));
2059 break;
2060 }
2061 case Intrinsic::bitreverse:
2062 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2063 Known = Known.unionWith(RHS: Known2.reverseBits());
2064 break;
2065 case Intrinsic::bswap:
2066 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2067 Known = Known.unionWith(RHS: Known2.byteSwap());
2068 break;
2069 case Intrinsic::ctlz: {
2070 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2071 // If we have a known 1, its position is our upper bound.
2072 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2073 // If this call is poison for 0 input, the result will be less than 2^n.
2074 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2075 PossibleLZ = std::min(a: PossibleLZ, b: BitWidth - 1);
2076 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
2077 Known.Zero.setBitsFrom(LowBits);
2078 break;
2079 }
2080 case Intrinsic::cttz: {
2081 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2082 // If we have a known 1, its position is our upper bound.
2083 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2084 // If this call is poison for 0 input, the result will be less than 2^n.
2085 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2086 PossibleTZ = std::min(a: PossibleTZ, b: BitWidth - 1);
2087 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
2088 Known.Zero.setBitsFrom(LowBits);
2089 break;
2090 }
2091 case Intrinsic::ctpop: {
2092 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2093 // We can bound the space the count needs. Also, bits known to be zero
2094 // can't contribute to the population.
2095 unsigned BitsPossiblySet = Known2.countMaxPopulation();
2096 unsigned LowBits = llvm::bit_width(Value: BitsPossiblySet);
2097 Known.Zero.setBitsFrom(LowBits);
2098 // TODO: we could bound KnownOne using the lower bound on the number
2099 // of bits which might be set provided by popcnt KnownOne2.
2100 break;
2101 }
2102 case Intrinsic::fshr:
2103 case Intrinsic::fshl: {
2104 const APInt *SA;
2105 if (!match(V: I->getOperand(i: 2), P: m_APInt(Res&: SA)))
2106 break;
2107
2108 KnownBits Known3(BitWidth);
2109 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2110 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known3, Q, Depth: Depth + 1);
2111 Known = II->getIntrinsicID() == Intrinsic::fshl
2112 ? KnownBits::fshl(LHS: Known2, RHS: Known3, Amt: *SA)
2113 : KnownBits::fshr(LHS: Known2, RHS: Known3, Amt: *SA);
2114 break;
2115 }
2116 case Intrinsic::clmul:
2117 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2118 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2119 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
2120 break;
2121 case Intrinsic::pext:
2122 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2123 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2124 Known = KnownBits::pext(Val: Known, Mask: Known2);
2125 break;
2126 case Intrinsic::pdep:
2127 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2128 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2129 Known = KnownBits::pdep(Val: Known, Mask: Known2);
2130 break;
2131 case Intrinsic::uadd_sat:
2132 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2133 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2134 Known = KnownBits::uadd_sat(LHS: Known, RHS: Known2);
2135 break;
2136 case Intrinsic::usub_sat:
2137 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2138 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2139 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
2140 break;
2141 case Intrinsic::sadd_sat:
2142 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2143 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2144 Known = KnownBits::sadd_sat(LHS: Known, RHS: Known2);
2145 break;
2146 case Intrinsic::ssub_sat:
2147 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2148 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2149 Known = KnownBits::ssub_sat(LHS: Known, RHS: Known2);
2150 break;
2151 // Vec reverse preserves bits from input vec.
2152 case Intrinsic::vector_reverse:
2153 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: DemandedElts.reverseBits(), Known, Q,
2154 Depth: Depth + 1);
2155 break;
2156 // for min/max/and/or reduce, any bit common to each element in the
2157 // input vec is set in the output.
2158 case Intrinsic::vector_reduce_and:
2159 case Intrinsic::vector_reduce_or:
2160 case Intrinsic::vector_reduce_umax:
2161 case Intrinsic::vector_reduce_umin:
2162 case Intrinsic::vector_reduce_smax:
2163 case Intrinsic::vector_reduce_smin:
2164 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2165 break;
2166 case Intrinsic::vector_reduce_xor: {
2167 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2168 // The zeros common to all vecs are zero in the output.
2169 // If the number of elements is odd, then the common ones remain. If the
2170 // number of elements is even, then the common ones becomes zeros.
2171 auto *VecTy = cast<VectorType>(Val: I->getOperand(i: 0)->getType());
2172 // Even, so the ones become zeros.
2173 bool EvenCnt = VecTy->getElementCount().isKnownEven();
2174 if (EvenCnt)
2175 Known.Zero |= Known.One;
2176 // Maybe even element count so need to clear ones.
2177 if (VecTy->isScalableTy() || EvenCnt)
2178 Known.One.clearAllBits();
2179 break;
2180 }
2181 case Intrinsic::vector_reduce_add: {
2182 auto *VecTy = dyn_cast<FixedVectorType>(Val: I->getOperand(i: 0)->getType());
2183 if (!VecTy)
2184 break;
2185 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2186 Known = Known.reduceAdd(NumElts: VecTy->getNumElements());
2187 break;
2188 }
2189 case Intrinsic::umin:
2190 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2191 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2192 Known = KnownBits::umin(LHS: Known, RHS: Known2);
2193 break;
2194 case Intrinsic::umax:
2195 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2196 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2197 Known = KnownBits::umax(LHS: Known, RHS: Known2);
2198 break;
2199 case Intrinsic::smin:
2200 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2201 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2202 Known = KnownBits::smin(LHS: Known, RHS: Known2);
2203 unionWithMinMaxIntrinsicClamp(II, Known);
2204 break;
2205 case Intrinsic::smax:
2206 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2207 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2208 Known = KnownBits::smax(LHS: Known, RHS: Known2);
2209 unionWithMinMaxIntrinsicClamp(II, Known);
2210 break;
2211 case Intrinsic::ptrmask: {
2212 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2213
2214 const Value *Mask = I->getOperand(i: 1);
2215 Known2 = KnownBits(Mask->getType()->getScalarSizeInBits());
2216 computeKnownBits(V: Mask, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2217 // TODO: 1-extend would be more precise.
2218 Known &= Known2.anyextOrTrunc(BitWidth);
2219 break;
2220 }
2221 case Intrinsic::x86_sse2_pmulh_w:
2222 case Intrinsic::x86_avx2_pmulh_w:
2223 case Intrinsic::x86_avx512_pmulh_w_512:
2224 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2225 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2226 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
2227 break;
2228 case Intrinsic::x86_sse2_pmulhu_w:
2229 case Intrinsic::x86_avx2_pmulhu_w:
2230 case Intrinsic::x86_avx512_pmulhu_w_512:
2231 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2232 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2233 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
2234 break;
2235 case Intrinsic::x86_sse42_crc32_64_64:
2236 Known.Zero.setBitsFrom(32);
2237 break;
2238 case Intrinsic::x86_ssse3_phadd_d_128:
2239 case Intrinsic::x86_ssse3_phadd_w_128:
2240 case Intrinsic::x86_avx2_phadd_d:
2241 case Intrinsic::x86_avx2_phadd_w: {
2242 Known = computeKnownBitsForHorizontalOperation(
2243 I, DemandedElts, Q, Depth,
2244 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2245 return KnownBits::add(LHS: KnownLHS, RHS: KnownRHS);
2246 });
2247 break;
2248 }
2249 case Intrinsic::x86_ssse3_phadd_sw_128:
2250 case Intrinsic::x86_avx2_phadd_sw: {
2251 Known = computeKnownBitsForHorizontalOperation(
2252 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::sadd_sat);
2253 break;
2254 }
2255 case Intrinsic::x86_ssse3_phsub_d_128:
2256 case Intrinsic::x86_ssse3_phsub_w_128:
2257 case Intrinsic::x86_avx2_phsub_d:
2258 case Intrinsic::x86_avx2_phsub_w: {
2259 Known = computeKnownBitsForHorizontalOperation(
2260 I, DemandedElts, Q, Depth,
2261 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2262 return KnownBits::sub(LHS: KnownLHS, RHS: KnownRHS);
2263 });
2264 break;
2265 }
2266 case Intrinsic::x86_ssse3_phsub_sw_128:
2267 case Intrinsic::x86_avx2_phsub_sw: {
2268 Known = computeKnownBitsForHorizontalOperation(
2269 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::ssub_sat);
2270 break;
2271 }
2272 case Intrinsic::riscv_vsetvli:
2273 case Intrinsic::riscv_vsetvlimax: {
2274 bool HasAVL = II->getIntrinsicID() == Intrinsic::riscv_vsetvli;
2275 const ConstantRange Range = getVScaleRange(F: II->getFunction(), BitWidth);
2276 uint64_t SEW = RISCVVType::decodeVSEW(
2277 VSEW: cast<ConstantInt>(Val: II->getArgOperand(i: HasAVL))->getZExtValue());
2278 RISCVVType::VLMUL VLMUL = static_cast<RISCVVType::VLMUL>(
2279 cast<ConstantInt>(Val: II->getArgOperand(i: 1 + HasAVL))->getZExtValue());
2280 uint64_t MaxVLEN =
2281 Range.getUnsignedMax().getZExtValue() * RISCV::RVVBitsPerBlock;
2282 uint64_t MaxVL = MaxVLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
2283
2284 // Result of vsetvli must be not larger than AVL.
2285 if (HasAVL)
2286 if (auto *CI = dyn_cast<ConstantInt>(Val: II->getArgOperand(i: 0)))
2287 MaxVL = std::min(a: MaxVL, b: CI->getZExtValue());
2288
2289 unsigned KnownZeroFirstBit = Log2_32(Value: MaxVL) + 1;
2290 if (BitWidth > KnownZeroFirstBit)
2291 Known.Zero.setBitsFrom(KnownZeroFirstBit);
2292 break;
2293 }
2294 case Intrinsic::amdgcn_mbcnt_hi:
2295 case Intrinsic::amdgcn_mbcnt_lo: {
2296 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
2297 // most 31 + src1.
2298 Known.Zero.setBitsFrom(
2299 II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo ? 6 : 5);
2300 computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Q, Depth: Depth + 1);
2301 Known = KnownBits::add(LHS: Known, RHS: Known2);
2302 break;
2303 }
2304 case Intrinsic::vscale: {
2305 if (!II->getParent() || !II->getFunction())
2306 break;
2307
2308 Known = getVScaleRange(F: II->getFunction(), BitWidth).toKnownBits();
2309 break;
2310 }
2311 }
2312 }
2313 break;
2314 }
2315 case Instruction::ShuffleVector: {
2316 if (auto *Splat = getSplatValue(V: I)) {
2317 computeKnownBits(V: Splat, Known, Q, Depth: Depth + 1);
2318 break;
2319 }
2320
2321 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
2322 // FIXME: Do we need to handle ConstantExpr involving shufflevectors?
2323 if (!Shuf) {
2324 Known.resetAll();
2325 return;
2326 }
2327 // For undef elements, we don't know anything about the common state of
2328 // the shuffle result.
2329 APInt DemandedLHS, DemandedRHS;
2330 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS)) {
2331 Known.resetAll();
2332 return;
2333 }
2334 Known.setAllConflict();
2335 if (!!DemandedLHS) {
2336 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
2337 computeKnownBits(V: LHS, DemandedElts: DemandedLHS, Known, Q, Depth: Depth + 1);
2338 // If we don't know any bits, early out.
2339 if (Known.isUnknown())
2340 break;
2341 }
2342 if (!!DemandedRHS) {
2343 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
2344 computeKnownBits(V: RHS, DemandedElts: DemandedRHS, Known&: Known2, Q, Depth: Depth + 1);
2345 Known = Known.intersectWith(RHS: Known2);
2346 }
2347 break;
2348 }
2349 case Instruction::InsertElement: {
2350 if (isa<ScalableVectorType>(Val: I->getType())) {
2351 Known.resetAll();
2352 return;
2353 }
2354 const Value *Vec = I->getOperand(i: 0);
2355 const Value *Elt = I->getOperand(i: 1);
2356 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
2357 unsigned NumElts = DemandedElts.getBitWidth();
2358 APInt DemandedVecElts = DemandedElts;
2359 bool NeedsElt = true;
2360 // If we know the index we are inserting too, clear it from Vec check.
2361 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
2362 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
2363 NeedsElt = DemandedElts[CIdx->getZExtValue()];
2364 }
2365
2366 Known.setAllConflict();
2367 if (NeedsElt) {
2368 computeKnownBits(V: Elt, Known, Q, Depth: Depth + 1);
2369 // If we don't know any bits, early out.
2370 if (Known.isUnknown())
2371 break;
2372 }
2373
2374 if (!DemandedVecElts.isZero()) {
2375 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known&: Known2, Q, Depth: Depth + 1);
2376 Known = Known.intersectWith(RHS: Known2);
2377 }
2378 break;
2379 }
2380 case Instruction::ExtractElement: {
2381 // Look through extract element. If the index is non-constant or
2382 // out-of-range demand all elements, otherwise just the extracted element.
2383 const Value *Vec = I->getOperand(i: 0);
2384 const Value *Idx = I->getOperand(i: 1);
2385 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
2386 if (isa<ScalableVectorType>(Val: Vec->getType())) {
2387 // FIXME: there's probably *something* we can do with scalable vectors
2388 Known.resetAll();
2389 break;
2390 }
2391 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2392 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
2393 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
2394 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
2395 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known, Q, Depth: Depth + 1);
2396 break;
2397 }
2398 case Instruction::ExtractValue:
2399 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I->getOperand(i: 0))) {
2400 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: I);
2401 if (EVI->getNumIndices() != 1) break;
2402 if (EVI->getIndices()[0] == 0) {
2403 switch (II->getIntrinsicID()) {
2404 default: break;
2405 case Intrinsic::uadd_with_overflow:
2406 case Intrinsic::sadd_with_overflow:
2407 computeKnownBitsAddSub(
2408 Add: true, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2409 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2410 break;
2411 case Intrinsic::usub_with_overflow:
2412 case Intrinsic::ssub_with_overflow:
2413 computeKnownBitsAddSub(
2414 Add: false, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2415 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2416 break;
2417 case Intrinsic::umul_with_overflow:
2418 case Intrinsic::smul_with_overflow:
2419 computeKnownBitsMul(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), NSW: false,
2420 NUW: false, DemandedElts, Known, Known2, Q, Depth);
2421 break;
2422 }
2423 }
2424 }
2425 break;
2426 case Instruction::Freeze:
2427 if (isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
2428 Depth: Depth + 1))
2429 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2430 break;
2431 }
2432}
2433
2434/// Determine which bits of V are known to be either zero or one and return
2435/// them.
2436KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
2437 const SimplifyQuery &Q, unsigned Depth) {
2438 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2439 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
2440 return Known;
2441}
2442
2443/// Determine which bits of V are known to be either zero or one and return
2444/// them.
2445KnownBits llvm::computeKnownBits(const Value *V, const SimplifyQuery &Q,
2446 unsigned Depth) {
2447 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2448 computeKnownBits(V, Known, Q, Depth);
2449 return Known;
2450}
2451
2452/// Determine which bits of V are known to be either zero or one and return
2453/// them in the Known bit set.
2454///
2455/// NOTE: we cannot consider 'undef' to be "IsZero" here. The problem is that
2456/// we cannot optimize based on the assumption that it is zero without changing
2457/// it to be an explicit zero. If we don't change it to zero, other code could
2458/// optimized based on the contradictory assumption that it is non-zero.
2459/// Because instcombine aggressively folds operations with undef args anyway,
2460/// this won't lose us code quality.
2461///
2462/// This function is defined on values with integer type, values with pointer
2463/// type, and vectors of integers. In the case
2464/// where V is a vector, known zero, and known one values are the
2465/// same width as the vector element, and the bit is set only if it is true
2466/// for all of the demanded elements in the vector specified by DemandedElts.
2467void computeKnownBits(const Value *V, const APInt &DemandedElts,
2468 KnownBits &Known, const SimplifyQuery &Q,
2469 unsigned Depth) {
2470 if (!DemandedElts) {
2471 // No demanded elts, better to assume we don't know anything.
2472 Known.resetAll();
2473 return;
2474 }
2475
2476 assert(V && "No Value?");
2477 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2478
2479#ifndef NDEBUG
2480 Type *Ty = V->getType();
2481 unsigned BitWidth = Known.getBitWidth();
2482
2483 assert((Ty->isIntOrIntVectorTy(BitWidth) || Ty->isPtrOrPtrVectorTy()) &&
2484 "Not integer or pointer type!");
2485
2486 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
2487 assert(
2488 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
2489 "DemandedElt width should equal the fixed vector number of elements");
2490 } else {
2491 assert(DemandedElts == APInt(1, 1) &&
2492 "DemandedElt width should be 1 for scalars or scalable vectors");
2493 }
2494
2495 Type *ScalarTy = Ty->getScalarType();
2496 if (ScalarTy->isPointerTy()) {
2497 assert(BitWidth == Q.DL.getPointerTypeSizeInBits(ScalarTy) &&
2498 "V and Known should have same BitWidth");
2499 } else {
2500 assert(BitWidth == Q.DL.getTypeSizeInBits(ScalarTy) &&
2501 "V and Known should have same BitWidth");
2502 }
2503#endif
2504
2505 const APInt *C;
2506 if (match(V, P: m_APInt(Res&: C))) {
2507 // We know all of the bits for a scalar constant or a splat vector constant!
2508 Known = KnownBits::makeConstant(C: *C);
2509 return;
2510 }
2511 // Null and aggregate-zero are all-zeros.
2512 if (isa<ConstantPointerNull>(Val: V) || isa<ConstantAggregateZero>(Val: V)) {
2513 Known.setAllZero();
2514 return;
2515 }
2516 // Handle a constant vector by taking the intersection of the known bits of
2517 // each element.
2518 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: V)) {
2519 assert(!isa<ScalableVectorType>(V->getType()));
2520 // We know that CDV must be a vector of integers. Take the intersection of
2521 // each element.
2522 Known.setAllConflict();
2523 for (unsigned i = 0, e = CDV->getNumElements(); i != e; ++i) {
2524 if (!DemandedElts[i])
2525 continue;
2526 APInt Elt = CDV->getElementAsAPInt(i);
2527 Known.Zero &= ~Elt;
2528 Known.One &= Elt;
2529 }
2530 if (Known.hasConflict())
2531 Known.resetAll();
2532 return;
2533 }
2534
2535 if (const auto *CV = dyn_cast<ConstantVector>(Val: V)) {
2536 assert(!isa<ScalableVectorType>(V->getType()));
2537 // We know that CV must be a vector of integers. Take the intersection of
2538 // each element.
2539 Known.setAllConflict();
2540 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
2541 if (!DemandedElts[i])
2542 continue;
2543 Constant *Element = CV->getAggregateElement(Elt: i);
2544 if (isa<PoisonValue>(Val: Element))
2545 continue;
2546 auto *ElementCI = dyn_cast_or_null<ConstantInt>(Val: Element);
2547 if (!ElementCI) {
2548 Known.resetAll();
2549 return;
2550 }
2551 const APInt &Elt = ElementCI->getValue();
2552 Known.Zero &= ~Elt;
2553 Known.One &= Elt;
2554 }
2555 if (Known.hasConflict())
2556 Known.resetAll();
2557 return;
2558 }
2559
2560 // Start out not knowing anything.
2561 Known.resetAll();
2562
2563 // We can't imply anything about undefs.
2564 if (isa<UndefValue>(Val: V))
2565 return;
2566
2567 // There's no point in looking through other users of ConstantData for
2568 // assumptions. Confirm that we've handled them all.
2569 assert(!isa<ConstantData>(V) && "Unhandled constant data!");
2570
2571 if (const auto *A = dyn_cast<Argument>(Val: V))
2572 if (std::optional<ConstantRange> Range = A->getRange())
2573 Known = Range->toKnownBits();
2574
2575 // All recursive calls that increase depth must come after this.
2576 if (Depth == MaxAnalysisRecursionDepth)
2577 return;
2578
2579 // A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
2580 // the bits of its aliasee.
2581 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: V)) {
2582 if (!GA->isInterposable())
2583 computeKnownBits(V: GA->getAliasee(), Known, Q, Depth: Depth + 1);
2584 return;
2585 }
2586
2587 if (const Operator *I = dyn_cast<Operator>(Val: V))
2588 computeKnownBitsFromOperator(I, DemandedElts, Known, Q, Depth);
2589 else if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
2590 if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
2591 Known = CR->toKnownBits();
2592 }
2593
2594 // Aligned pointers have trailing zeros - refine Known.Zero set
2595 if (isa<PointerType>(Val: V->getType())) {
2596 Align Alignment = V->getPointerAlignment(DL: Q.DL);
2597 Known.Zero.setLowBits(Log2(A: Alignment));
2598 }
2599
2600 // computeKnownBitsFromContext strictly refines Known.
2601 // Therefore, we run them after computeKnownBitsFromOperator.
2602
2603 // Check whether we can determine known bits from context such as assumes.
2604 computeKnownBitsFromContext(V, Known, Q, Depth);
2605}
2606
2607/// Try to detect a recurrence that the value of the induction variable is
2608/// always a power of two (or zero).
2609static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
2610 SimplifyQuery &Q, unsigned Depth) {
2611 BinaryOperator *BO = nullptr;
2612 Value *Start = nullptr, *Step = nullptr;
2613 if (!matchSimpleRecurrence(P: PN, BO, Start, Step))
2614 return false;
2615
2616 // Initial value must be a power of two.
2617 for (const Use &U : PN->operands()) {
2618 if (U.get() == Start) {
2619 // Initial value comes from a different BB, need to adjust context
2620 // instruction for analysis.
2621 Q.CxtI = PN->getIncomingBlock(U)->getTerminator();
2622 if (!isKnownToBeAPowerOfTwo(V: Start, OrZero, Q, Depth))
2623 return false;
2624 }
2625 }
2626
2627 // Except for Mul, the induction variable must be on the left side of the
2628 // increment expression, otherwise its value can be arbitrary.
2629 if (BO->getOpcode() != Instruction::Mul && BO->getOperand(i_nocapture: 1) != Step)
2630 return false;
2631
2632 Q.CxtI = BO->getParent()->getTerminator();
2633 switch (BO->getOpcode()) {
2634 case Instruction::Mul:
2635 // Power of two is closed under multiplication.
2636 return (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) ||
2637 Q.IIQ.hasNoSignedWrap(Op: BO)) &&
2638 isKnownToBeAPowerOfTwo(V: Step, OrZero, Q, Depth);
2639 case Instruction::SDiv:
2640 // Start value must not be signmask for signed division, so simply being a
2641 // power of two is not sufficient, and it has to be a constant.
2642 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2643 return false;
2644 [[fallthrough]];
2645 case Instruction::UDiv:
2646 // Divisor must be a power of two.
2647 // If OrZero is false, cannot guarantee induction variable is non-zero after
2648 // division, same for Shr, unless it is exact division.
2649 return (OrZero || Q.IIQ.isExact(Op: BO)) &&
2650 isKnownToBeAPowerOfTwo(V: Step, OrZero: false, Q, Depth);
2651 case Instruction::Shl:
2652 return OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO);
2653 case Instruction::AShr:
2654 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2655 return false;
2656 [[fallthrough]];
2657 case Instruction::LShr:
2658 return OrZero || Q.IIQ.isExact(Op: BO);
2659 default:
2660 return false;
2661 }
2662}
2663
2664/// Return true if we can infer that \p V is known to be a power of 2 from
2665/// dominating condition \p Cond (e.g., ctpop(V) == 1).
2666static bool isImpliedToBeAPowerOfTwoFromCond(const Value *V, bool OrZero,
2667 const Value *Cond,
2668 bool CondIsTrue) {
2669 CmpPredicate Pred;
2670 const APInt *RHSC;
2671 if (!match(V: Cond, P: m_ICmp(Pred, L: m_Ctpop(Op0: m_Specific(V)), R: m_APInt(Res&: RHSC))))
2672 return false;
2673 if (!CondIsTrue)
2674 Pred = ICmpInst::getInversePredicate(pred: Pred);
2675 // ctpop(V) u< 2
2676 if (OrZero && Pred == ICmpInst::ICMP_ULT && *RHSC == 2)
2677 return true;
2678 // ctpop(V) == 1
2679 return Pred == ICmpInst::ICMP_EQ && *RHSC == 1;
2680}
2681
2682/// Return true if the given value is known to have exactly one
2683/// bit set when defined. For vectors return true if every element is known to
2684/// be a power of two when defined. Supports values with integer or pointer
2685/// types and vectors of integers.
2686bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero,
2687 const SimplifyQuery &Q, unsigned Depth) {
2688 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2689
2690 if (isa<Constant>(Val: V))
2691 return OrZero ? match(V, P: m_Power2OrZero()) : match(V, P: m_Power2());
2692
2693 // i1 is by definition a power of 2 or zero.
2694 if (OrZero && V->getType()->getScalarSizeInBits() == 1)
2695 return true;
2696
2697 // Try to infer from assumptions.
2698 if (Q.AC && Q.CxtI) {
2699 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
2700 if (!AssumeVH)
2701 continue;
2702 CallInst *I = cast<CallInst>(Val&: AssumeVH);
2703 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond: I->getArgOperand(i: 0),
2704 /*CondIsTrue=*/true) &&
2705 isValidAssumeForContext(I, Q))
2706 return true;
2707 }
2708 }
2709
2710 // Handle dominating conditions.
2711 if (Q.DC && Q.CxtI && Q.DT) {
2712 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
2713 Value *Cond = BI->getCondition();
2714
2715 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
2716 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2717 /*CondIsTrue=*/true) &&
2718 Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
2719 return true;
2720
2721 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
2722 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2723 /*CondIsTrue=*/false) &&
2724 Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
2725 return true;
2726 }
2727 }
2728
2729 auto *I = dyn_cast<Instruction>(Val: V);
2730 if (!I)
2731 return false;
2732
2733 if (Q.CxtI && match(V, P: m_VScale())) {
2734 const Function *F = Q.CxtI->getFunction();
2735 // The vscale_range indicates vscale is a power-of-two.
2736 return F->hasFnAttribute(Kind: Attribute::VScaleRange);
2737 }
2738
2739 // 1 << X is clearly a power of two if the one is not shifted off the end. If
2740 // it is shifted off the end then the result is undefined.
2741 if (match(V: I, P: m_Shl(L: m_One(), R: m_Value())))
2742 return true;
2743
2744 // (signmask) >>l X is clearly a power of two if the one is not shifted off
2745 // the bottom. If it is shifted off the bottom then the result is undefined.
2746 if (match(V: I, P: m_LShr(L: m_SignMask(), R: m_Value())))
2747 return true;
2748
2749 // The remaining tests are all recursive, so bail out if we hit the limit.
2750 if (Depth++ == MaxAnalysisRecursionDepth)
2751 return false;
2752
2753 switch (I->getOpcode()) {
2754 case Instruction::ZExt:
2755 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2756 case Instruction::Trunc:
2757 return OrZero && isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2758 case Instruction::Shl:
2759 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: I) || Q.IIQ.hasNoSignedWrap(Op: I))
2760 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2761 return false;
2762 case Instruction::LShr:
2763 if (OrZero || Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2764 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2765 return false;
2766 case Instruction::UDiv:
2767 if (Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2768 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2769 return false;
2770 case Instruction::Mul:
2771 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2772 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth) &&
2773 (OrZero || isKnownNonZero(V: I, Q, Depth));
2774 case Instruction::And:
2775 // A power of two and'd with anything is a power of two or zero.
2776 if (OrZero &&
2777 (isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), /*OrZero*/ true, Q, Depth) ||
2778 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), /*OrZero*/ true, Q, Depth)))
2779 return true;
2780 // X & (-X) is always a power of two or zero.
2781 if (match(V: I->getOperand(i: 0), P: m_Neg(V: m_Specific(V: I->getOperand(i: 1)))) ||
2782 match(V: I->getOperand(i: 1), P: m_Neg(V: m_Specific(V: I->getOperand(i: 0)))))
2783 return OrZero || isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
2784 return false;
2785 case Instruction::Add: {
2786 // Adding a power-of-two or zero to the same power-of-two or zero yields
2787 // either the original power-of-two, a larger power-of-two or zero.
2788 const OverflowingBinaryOperator *VOBO = cast<OverflowingBinaryOperator>(Val: V);
2789 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO) ||
2790 Q.IIQ.hasNoSignedWrap(Op: VOBO)) {
2791 if (match(V: I->getOperand(i: 0),
2792 P: m_c_And(L: m_Specific(V: I->getOperand(i: 1)), R: m_Value())) &&
2793 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth))
2794 return true;
2795 if (match(V: I->getOperand(i: 1),
2796 P: m_c_And(L: m_Specific(V: I->getOperand(i: 0)), R: m_Value())) &&
2797 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth))
2798 return true;
2799
2800 unsigned BitWidth = V->getType()->getScalarSizeInBits();
2801 KnownBits LHSBits(BitWidth);
2802 computeKnownBits(V: I->getOperand(i: 0), Known&: LHSBits, Q, Depth);
2803
2804 KnownBits RHSBits(BitWidth);
2805 computeKnownBits(V: I->getOperand(i: 1), Known&: RHSBits, Q, Depth);
2806 // If i8 V is a power of two or zero:
2807 // ZeroBits: 1 1 1 0 1 1 1 1
2808 // ~ZeroBits: 0 0 0 1 0 0 0 0
2809 if ((~(LHSBits.Zero & RHSBits.Zero)).isPowerOf2())
2810 // If OrZero isn't set, we cannot give back a zero result.
2811 // Make sure either the LHS or RHS has a bit set.
2812 if (OrZero || RHSBits.One.getBoolValue() || LHSBits.One.getBoolValue())
2813 return true;
2814 }
2815
2816 // LShr(UINT_MAX, Y) + 1 is a power of two (if add is nuw) or zero.
2817 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO))
2818 if (match(V: I, P: m_Add(L: m_LShr(L: m_AllOnes(), R: m_Value()), R: m_One())))
2819 return true;
2820 return false;
2821 }
2822 case Instruction::Select:
2823 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2824 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 2), OrZero, Q, Depth);
2825 case Instruction::PHI: {
2826 // A PHI node is power of two if all incoming values are power of two, or if
2827 // it is an induction variable where in each step its value is a power of
2828 // two.
2829 auto *PN = cast<PHINode>(Val: I);
2830 SimplifyQuery RecQ = Q.getWithoutCondContext();
2831
2832 // Check if it is an induction variable and always power of two.
2833 if (isPowerOfTwoRecurrence(PN, OrZero, Q&: RecQ, Depth))
2834 return true;
2835
2836 // Recursively check all incoming values. Limit recursion to 2 levels, so
2837 // that search complexity is limited to number of operands^2.
2838 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
2839 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
2840 // Value is power of 2 if it is coming from PHI node itself by induction.
2841 if (U.get() == PN)
2842 return true;
2843
2844 // Change the context instruction to the incoming block where it is
2845 // evaluated.
2846 RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
2847 return isKnownToBeAPowerOfTwo(V: U.get(), OrZero, Q: RecQ, Depth: NewDepth);
2848 });
2849 }
2850 case Instruction::Invoke:
2851 case Instruction::Call: {
2852 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2853 switch (II->getIntrinsicID()) {
2854 case Intrinsic::umax:
2855 case Intrinsic::smax:
2856 case Intrinsic::umin:
2857 case Intrinsic::smin:
2858 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 1), OrZero, Q, Depth) &&
2859 isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2860 // bswap/bitreverse just move around bits, but don't change any 1s/0s
2861 // thus dont change pow2/non-pow2 status.
2862 case Intrinsic::bitreverse:
2863 case Intrinsic::bswap:
2864 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2865 case Intrinsic::fshr:
2866 case Intrinsic::fshl:
2867 // If Op0 == Op1, this is a rotate. is_pow2(rotate(x, y)) == is_pow2(x)
2868 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
2869 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2870 break;
2871 default:
2872 break;
2873 }
2874 }
2875 return false;
2876 }
2877 default:
2878 return false;
2879 }
2880}
2881
2882/// Test whether a GEP's result is known to be non-null.
2883///
2884/// Uses properties inherent in a GEP to try to determine whether it is known
2885/// to be non-null.
2886///
2887/// Currently this routine does not support vector GEPs.
2888static bool isGEPKnownNonNull(const GEPOperator *GEP, const SimplifyQuery &Q,
2889 unsigned Depth) {
2890 const Function *F = nullptr;
2891 if (const Instruction *I = dyn_cast<Instruction>(Val: GEP))
2892 F = I->getFunction();
2893
2894 // If the gep is nuw or inbounds with invalid null pointer, then the GEP
2895 // may be null iff the base pointer is null and the offset is zero.
2896 if (!GEP->hasNoUnsignedWrap() &&
2897 !(GEP->isInBounds() &&
2898 !NullPointerIsDefined(F, AS: GEP->getPointerAddressSpace())))
2899 return false;
2900
2901 // FIXME: Support vector-GEPs.
2902 assert(GEP->getType()->isPointerTy() && "We only support plain pointer GEP");
2903
2904 // If the base pointer is non-null, we cannot walk to a null address with an
2905 // inbounds GEP in address space zero.
2906 if (isKnownNonZero(V: GEP->getPointerOperand(), Q, Depth))
2907 return true;
2908
2909 // Walk the GEP operands and see if any operand introduces a non-zero offset.
2910 // If so, then the GEP cannot produce a null pointer, as doing so would
2911 // inherently violate the inbounds contract within address space zero.
2912 for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
2913 GTI != GTE; ++GTI) {
2914 // Struct types are easy -- they must always be indexed by a constant.
2915 if (StructType *STy = GTI.getStructTypeOrNull()) {
2916 ConstantInt *OpC = cast<ConstantInt>(Val: GTI.getOperand());
2917 unsigned ElementIdx = OpC->getZExtValue();
2918 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
2919 uint64_t ElementOffset = SL->getElementOffset(Idx: ElementIdx);
2920 if (ElementOffset > 0)
2921 return true;
2922 continue;
2923 }
2924
2925 // If we have a zero-sized type, the index doesn't matter. Keep looping.
2926 if (GTI.getSequentialElementStride(DL: Q.DL).isZero())
2927 continue;
2928
2929 // Fast path the constant operand case both for efficiency and so we don't
2930 // increment Depth when just zipping down an all-constant GEP.
2931 if (ConstantInt *OpC = dyn_cast<ConstantInt>(Val: GTI.getOperand())) {
2932 if (!OpC->isZero())
2933 return true;
2934 continue;
2935 }
2936
2937 // We post-increment Depth here because while isKnownNonZero increments it
2938 // as well, when we pop back up that increment won't persist. We don't want
2939 // to recurse 10k times just because we have 10k GEP operands. We don't
2940 // bail completely out because we want to handle constant GEPs regardless
2941 // of depth.
2942 if (Depth++ >= MaxAnalysisRecursionDepth)
2943 continue;
2944
2945 if (isKnownNonZero(V: GTI.getOperand(), Q, Depth))
2946 return true;
2947 }
2948
2949 return false;
2950}
2951
2952static bool isKnownNonNullFromDominatingCondition(const Value *V,
2953 const Instruction *CtxI,
2954 const DominatorTree *DT) {
2955 assert(!isa<Constant>(V) && "Called for constant?");
2956
2957 if (!CtxI || !DT)
2958 return false;
2959
2960 unsigned NumUsesExplored = 0;
2961 for (auto &U : V->uses()) {
2962 // Avoid massive lists
2963 if (NumUsesExplored >= DomConditionsMaxUses)
2964 break;
2965 NumUsesExplored++;
2966
2967 const Instruction *UI = cast<Instruction>(Val: U.getUser());
2968 // If the value is used as an argument to a call or invoke, then argument
2969 // attributes may provide an answer about null-ness.
2970 if (V->getType()->isPointerTy()) {
2971 if (const auto *CB = dyn_cast<CallBase>(Val: UI)) {
2972 if (CB->isArgOperand(U: &U) &&
2973 CB->paramHasNonNullAttr(ArgNo: CB->getArgOperandNo(U: &U),
2974 /*AllowUndefOrPoison=*/false) &&
2975 DT->dominates(Def: CB, User: CtxI))
2976 return true;
2977 }
2978 }
2979
2980 // If the value is used as a load/store, then the pointer must be non null.
2981 if (V == getLoadStorePointerOperand(V: UI)) {
2982 if (!NullPointerIsDefined(F: UI->getFunction(),
2983 AS: V->getType()->getPointerAddressSpace()) &&
2984 DT->dominates(Def: UI, User: CtxI))
2985 return true;
2986 }
2987
2988 if ((match(V: UI, P: m_IDiv(L: m_Value(), R: m_Specific(V))) ||
2989 match(V: UI, P: m_IRem(L: m_Value(), R: m_Specific(V)))) &&
2990 isValidAssumeForContext(Inv: UI, CxtI: CtxI, DT))
2991 return true;
2992
2993 // Consider only compare instructions uniquely controlling a branch
2994 Value *RHS;
2995 CmpPredicate Pred;
2996 if (!match(V: UI, P: m_c_ICmp(Pred, L: m_Specific(V), R: m_Value(V&: RHS))))
2997 continue;
2998
2999 bool NonNullIfTrue;
3000 if (cmpExcludesZero(Pred, RHS))
3001 NonNullIfTrue = true;
3002 else if (cmpExcludesZero(Pred: CmpInst::getInversePredicate(pred: Pred), RHS))
3003 NonNullIfTrue = false;
3004 else
3005 continue;
3006
3007 SmallVector<const User *, 4> WorkList;
3008 SmallPtrSet<const User *, 4> Visited;
3009 for (const auto *CmpU : UI->users()) {
3010 assert(WorkList.empty() && "Should be!");
3011 if (Visited.insert(Ptr: CmpU).second)
3012 WorkList.push_back(Elt: CmpU);
3013
3014 while (!WorkList.empty()) {
3015 auto *Curr = WorkList.pop_back_val();
3016
3017 // If a user is an AND, add all its users to the work list. We only
3018 // propagate "pred != null" condition through AND because it is only
3019 // correct to assume that all conditions of AND are met in true branch.
3020 // TODO: Support similar logic of OR and EQ predicate?
3021 if (NonNullIfTrue)
3022 if (match(V: Curr, P: m_LogicalAnd(L: m_Value(), R: m_Value()))) {
3023 for (const auto *CurrU : Curr->users())
3024 if (Visited.insert(Ptr: CurrU).second)
3025 WorkList.push_back(Elt: CurrU);
3026 continue;
3027 }
3028
3029 if (const CondBrInst *BI = dyn_cast<CondBrInst>(Val: Curr)) {
3030 BasicBlock *NonNullSuccessor =
3031 BI->getSuccessor(i: NonNullIfTrue ? 0 : 1);
3032 BasicBlockEdge Edge(BI->getParent(), NonNullSuccessor);
3033 if (DT->dominates(BBE: Edge, BB: CtxI->getParent()))
3034 return true;
3035 } else if (NonNullIfTrue && isGuard(U: Curr) &&
3036 DT->dominates(Def: cast<Instruction>(Val: Curr), User: CtxI)) {
3037 return true;
3038 }
3039 }
3040 }
3041 }
3042
3043 return false;
3044}
3045
3046/// Does the 'Range' metadata (which must be a valid MD_range operand list)
3047/// ensure that the value it's attached to is never Value? 'RangeType' is
3048/// is the type of the value described by the range.
3049static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value) {
3050 const unsigned NumRanges = Ranges->getNumOperands() / 2;
3051 assert(NumRanges >= 1);
3052 for (unsigned i = 0; i < NumRanges; ++i) {
3053 ConstantInt *Lower =
3054 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 0));
3055 ConstantInt *Upper =
3056 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 1));
3057 ConstantRange Range(Lower->getValue(), Upper->getValue());
3058 if (Range.contains(Val: Value))
3059 return false;
3060 }
3061 return true;
3062}
3063
3064/// Try to detect a recurrence that monotonically increases/decreases from a
3065/// non-zero starting value. These are common as induction variables.
3066static bool isNonZeroRecurrence(const PHINode *PN) {
3067 BinaryOperator *BO = nullptr;
3068 Value *Start = nullptr, *Step = nullptr;
3069 const APInt *StartC, *StepC;
3070 if (!matchSimpleRecurrence(P: PN, BO, Start, Step) ||
3071 !match(V: Start, P: m_APInt(Res&: StartC)) || StartC->isZero())
3072 return false;
3073
3074 switch (BO->getOpcode()) {
3075 case Instruction::Add:
3076 // Starting from non-zero and stepping away from zero can never wrap back
3077 // to zero.
3078 return BO->hasNoUnsignedWrap() ||
3079 (BO->hasNoSignedWrap() && match(V: Step, P: m_APInt(Res&: StepC)) &&
3080 StartC->isNegative() == StepC->isNegative());
3081 case Instruction::Mul:
3082 return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) &&
3083 match(V: Step, P: m_APInt(Res&: StepC)) && !StepC->isZero();
3084 case Instruction::Shl:
3085 return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap();
3086 case Instruction::AShr:
3087 case Instruction::LShr:
3088 return BO->isExact();
3089 default:
3090 return false;
3091 }
3092}
3093
3094static bool matchOpWithOpEqZero(Value *Op0, Value *Op1) {
3095 return match(V: Op0, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3096 L: m_Specific(V: Op1), R: m_Zero()))) ||
3097 match(V: Op1, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3098 L: m_Specific(V: Op0), R: m_Zero())));
3099}
3100
3101static bool isNonZeroAdd(const APInt &DemandedElts, const SimplifyQuery &Q,
3102 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3103 bool NUW, unsigned Depth) {
3104 // (X + (X != 0)) is non zero
3105 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3106 return true;
3107
3108 if (NUW)
3109 return isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3110 isKnownNonZero(V: X, DemandedElts, Q, Depth);
3111
3112 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3113 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3114
3115 // If X and Y are both non-negative (as signed values) then their sum is not
3116 // zero unless both X and Y are zero.
3117 if (XKnown.isNonNegative() && YKnown.isNonNegative())
3118 if (isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3119 isKnownNonZero(V: X, DemandedElts, Q, Depth))
3120 return true;
3121
3122 // If X and Y are both negative (as signed values) then their sum is not
3123 // zero unless both X and Y equal INT_MIN.
3124 if (XKnown.isNegative() && YKnown.isNegative()) {
3125 APInt Mask = APInt::getSignedMaxValue(numBits: BitWidth);
3126 // The sign bit of X is set. If some other bit is set then X is not equal
3127 // to INT_MIN.
3128 if (XKnown.One.intersects(RHS: Mask))
3129 return true;
3130 // The sign bit of Y is set. If some other bit is set then Y is not equal
3131 // to INT_MIN.
3132 if (YKnown.One.intersects(RHS: Mask))
3133 return true;
3134 }
3135
3136 // The sum of a non-negative number and a power of two is not zero.
3137 if (XKnown.isNonNegative() &&
3138 isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ false, Q, Depth))
3139 return true;
3140 if (YKnown.isNonNegative() &&
3141 isKnownToBeAPowerOfTwo(V: X, /*OrZero*/ false, Q, Depth))
3142 return true;
3143
3144 return KnownBits::add(LHS: XKnown, RHS: YKnown, NSW, NUW).isNonZero();
3145}
3146
3147static bool isNonZeroSub(const APInt &DemandedElts, const SimplifyQuery &Q,
3148 unsigned BitWidth, Value *X, Value *Y,
3149 unsigned Depth) {
3150 // (X - (X != 0)) is non zero
3151 // ((X != 0) - X) is non zero
3152 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3153 return true;
3154
3155 // TODO: Move this case into isKnownNonEqual().
3156 if (auto *C = dyn_cast<Constant>(Val: X))
3157 if (C->isNullValue() && isKnownNonZero(V: Y, DemandedElts, Q, Depth))
3158 return true;
3159
3160 return ::isKnownNonEqual(V1: X, V2: Y, DemandedElts, Q, Depth);
3161}
3162
3163static bool isNonZeroMul(const APInt &DemandedElts, const SimplifyQuery &Q,
3164 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3165 bool NUW, unsigned Depth) {
3166 // If X and Y are non-zero then so is X * Y as long as the multiplication
3167 // does not overflow.
3168 if (NSW || NUW)
3169 return isKnownNonZero(V: X, DemandedElts, Q, Depth) &&
3170 isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3171
3172 // If either X or Y is odd, then if the other is non-zero the result can't
3173 // be zero.
3174 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3175 if (XKnown.One[0])
3176 return isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3177
3178 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3179 if (YKnown.One[0])
3180 return XKnown.isNonZero() || isKnownNonZero(V: X, DemandedElts, Q, Depth);
3181
3182 // If there exists any subset of X (sX) and subset of Y (sY) s.t sX * sY is
3183 // non-zero, then X * Y is non-zero. We can find sX and sY by just taking
3184 // the lowest known One of X and Y. If they are non-zero, the result
3185 // must be non-zero. We can check if LSB(X) * LSB(Y) != 0 by doing
3186 // X.CountLeadingZeros + Y.CountLeadingZeros < BitWidth.
3187 return (XKnown.countMaxTrailingZeros() + YKnown.countMaxTrailingZeros()) <
3188 BitWidth;
3189}
3190
3191static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
3192 const SimplifyQuery &Q, const KnownBits &KnownVal,
3193 unsigned Depth) {
3194 auto ShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3195 switch (I->getOpcode()) {
3196 case Instruction::Shl:
3197 return Lhs.shl(ShiftAmt: Rhs);
3198 case Instruction::LShr:
3199 return Lhs.lshr(ShiftAmt: Rhs);
3200 case Instruction::AShr:
3201 return Lhs.ashr(ShiftAmt: Rhs);
3202 default:
3203 llvm_unreachable("Unknown Shift Opcode");
3204 }
3205 };
3206
3207 auto InvShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3208 switch (I->getOpcode()) {
3209 case Instruction::Shl:
3210 return Lhs.lshr(ShiftAmt: Rhs);
3211 case Instruction::LShr:
3212 case Instruction::AShr:
3213 return Lhs.shl(ShiftAmt: Rhs);
3214 default:
3215 llvm_unreachable("Unknown Shift Opcode");
3216 }
3217 };
3218
3219 if (KnownVal.isUnknown())
3220 return false;
3221
3222 KnownBits KnownCnt =
3223 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3224 APInt MaxShift = KnownCnt.getMaxValue();
3225 unsigned NumBits = KnownVal.getBitWidth();
3226 if (MaxShift.uge(RHS: NumBits))
3227 return false;
3228
3229 if (!ShiftOp(KnownVal.One, MaxShift).isZero())
3230 return true;
3231
3232 // If all of the bits shifted out are known to be zero, and Val is known
3233 // non-zero then at least one non-zero bit must remain.
3234 if (InvShiftOp(KnownVal.Zero, NumBits - MaxShift)
3235 .eq(RHS: InvShiftOp(APInt::getAllOnes(numBits: NumBits), NumBits - MaxShift)) &&
3236 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth))
3237 return true;
3238
3239 return false;
3240}
3241
3242static bool isKnownNonZeroFromOperator(const Operator *I,
3243 const APInt &DemandedElts,
3244 const SimplifyQuery &Q, unsigned Depth) {
3245 unsigned BitWidth = getBitWidth(Ty: I->getType()->getScalarType(), DL: Q.DL);
3246 switch (I->getOpcode()) {
3247 case Instruction::Alloca:
3248 // Alloca never returns null, malloc might.
3249 return I->getType()->getPointerAddressSpace() == 0;
3250 case Instruction::GetElementPtr:
3251 if (I->getType()->isPointerTy())
3252 return isGEPKnownNonNull(GEP: cast<GEPOperator>(Val: I), Q, Depth);
3253 break;
3254 case Instruction::BitCast: {
3255 // We need to be a bit careful here. We can only peek through the bitcast
3256 // if the scalar size of elements in the operand are smaller than and a
3257 // multiple of the size they are casting too. Take three cases:
3258 //
3259 // 1) Unsafe:
3260 // bitcast <2 x i16> %NonZero to <4 x i8>
3261 //
3262 // %NonZero can have 2 non-zero i16 elements, but isKnownNonZero on a
3263 // <4 x i8> requires that all 4 i8 elements be non-zero which isn't
3264 // guranteed (imagine just sign bit set in the 2 i16 elements).
3265 //
3266 // 2) Unsafe:
3267 // bitcast <4 x i3> %NonZero to <3 x i4>
3268 //
3269 // Even though the scalar size of the src (`i3`) is smaller than the
3270 // scalar size of the dst `i4`, because `i3` is not a multiple of `i4`
3271 // its possible for the `3 x i4` elements to be zero because there are
3272 // some elements in the destination that don't contain any full src
3273 // element.
3274 //
3275 // 3) Safe:
3276 // bitcast <4 x i8> %NonZero to <2 x i16>
3277 //
3278 // This is always safe as non-zero in the 4 i8 elements implies
3279 // non-zero in the combination of any two adjacent ones. Since i8 is a
3280 // multiple of i16, each i16 is guranteed to have 2 full i8 elements.
3281 // This all implies the 2 i16 elements are non-zero.
3282 Type *FromTy = I->getOperand(i: 0)->getType();
3283 if ((FromTy->isIntOrIntVectorTy() || FromTy->isPtrOrPtrVectorTy()) &&
3284 (BitWidth % getBitWidth(Ty: FromTy->getScalarType(), DL: Q.DL)) == 0)
3285 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3286 } break;
3287 case Instruction::IntToPtr:
3288 // Note that we have to take special care to avoid looking through
3289 // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
3290 // as casts that can alter the value, e.g., AddrSpaceCasts.
3291 if (!isa<ScalableVectorType>(Val: I->getType()) &&
3292 Q.DL.getTypeSizeInBits(Ty: I->getOperand(i: 0)->getType()).getFixedValue() <=
3293 Q.DL.getTypeSizeInBits(Ty: I->getType()).getFixedValue())
3294 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3295 break;
3296 case Instruction::PtrToAddr:
3297 // isKnownNonZero() for pointers refers to the address bits being non-zero,
3298 // so we can directly forward.
3299 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3300 case Instruction::PtrToInt:
3301 // For inttoptr, make sure the result size is >= the address size. If the
3302 // address is non-zero, any larger value is also non-zero.
3303 if (Q.DL.getAddressSizeInBits(Ty: I->getOperand(i: 0)->getType()) <=
3304 I->getType()->getScalarSizeInBits())
3305 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3306 break;
3307 case Instruction::Trunc:
3308 // nuw/nsw trunc preserves zero/non-zero status of input.
3309 if (auto *TI = dyn_cast<TruncInst>(Val: I))
3310 if (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap())
3311 return isKnownNonZero(V: TI->getOperand(i_nocapture: 0), DemandedElts, Q, Depth);
3312 break;
3313
3314 // Iff x - y != 0, then x ^ y != 0
3315 // Therefore we can do the same exact checks
3316 case Instruction::Xor:
3317 case Instruction::Sub:
3318 return isNonZeroSub(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3319 Y: I->getOperand(i: 1), Depth);
3320 case Instruction::Or:
3321 // (X | (X != 0)) is non zero
3322 if (matchOpWithOpEqZero(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1)))
3323 return true;
3324 // X | Y != 0 if X != Y.
3325 if (isKnownNonEqual(V1: I->getOperand(i: 0), V2: I->getOperand(i: 1), DemandedElts, Q,
3326 Depth))
3327 return true;
3328 // X | Y != 0 if X != 0 or Y != 0.
3329 return isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth) ||
3330 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3331 case Instruction::SExt:
3332 case Instruction::ZExt:
3333 // ext X != 0 if X != 0.
3334 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3335
3336 case Instruction::Shl: {
3337 // shl nsw/nuw can't remove any non-zero bits.
3338 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3339 if (Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO))
3340 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3341
3342 // shl X, Y != 0 if X is odd. Note that the value of the shift is undefined
3343 // if the lowest bit is shifted off the end.
3344 KnownBits Known(BitWidth);
3345 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth);
3346 if (Known.One[0])
3347 return true;
3348
3349 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3350 }
3351 case Instruction::LShr:
3352 case Instruction::AShr: {
3353 // shr exact can only shift out zero bits.
3354 const PossiblyExactOperator *BO = cast<PossiblyExactOperator>(Val: I);
3355 if (BO->isExact())
3356 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3357
3358 // shr X, Y != 0 if X is negative. Note that the value of the shift is not
3359 // defined if the sign bit is shifted off the end.
3360 KnownBits Known =
3361 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3362 if (Known.isNegative())
3363 return true;
3364
3365 // shr (add nuw A, B), C is non-zero if A or B has a known-one bit at
3366 // position >= C, because the sum >= max(A, B).
3367 Value *A, *B;
3368 const APInt *C;
3369 if (Depth + 1 < MaxAnalysisRecursionDepth &&
3370 match(V: I->getOperand(i: 0), P: m_NUWAdd(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3371 match(V: I->getOperand(i: 1), P: m_APInt(Res&: C)) && C->ult(RHS: BitWidth)) {
3372 KnownBits KnownA = computeKnownBits(V: A, DemandedElts, Q, Depth: Depth + 1);
3373 if (!KnownA.One.lshr(ShiftAmt: *C).isZero())
3374 return true;
3375 KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
3376 if (!KnownB.One.lshr(ShiftAmt: *C).isZero())
3377 return true;
3378 }
3379
3380 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3381 }
3382 case Instruction::UDiv:
3383 case Instruction::SDiv: {
3384 // X / Y
3385 // div exact can only produce a zero if the dividend is zero.
3386 if (cast<PossiblyExactOperator>(Val: I)->isExact())
3387 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3388
3389 KnownBits XKnown =
3390 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3391 // If X is fully unknown we won't be able to figure anything out so don't
3392 // both computing knownbits for Y.
3393 if (XKnown.isUnknown())
3394 return false;
3395
3396 KnownBits YKnown =
3397 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3398 if (I->getOpcode() == Instruction::SDiv) {
3399 // For signed division need to compare abs value of the operands.
3400 XKnown = XKnown.abs(/*IntMinIsPoison*/ false);
3401 YKnown = YKnown.abs(/*IntMinIsPoison*/ false);
3402 }
3403 // If X u>= Y then div is non zero (0/0 is UB).
3404 std::optional<bool> XUgeY = KnownBits::uge(LHS: XKnown, RHS: YKnown);
3405 // If X is total unknown or X u< Y we won't be able to prove non-zero
3406 // with compute known bits so just return early.
3407 return XUgeY && *XUgeY;
3408 }
3409 case Instruction::Add: {
3410 // X + Y.
3411
3412 // If Add has nuw wrap flag, then if either X or Y is non-zero the result is
3413 // non-zero.
3414 auto *BO = cast<OverflowingBinaryOperator>(Val: I);
3415 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3416 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3417 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3418 }
3419 case Instruction::Mul: {
3420 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3421 return isNonZeroMul(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3422 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3423 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3424 }
3425 case Instruction::Select: {
3426 // (C ? X : Y) != 0 if X != 0 and Y != 0.
3427
3428 // First check if the arm is non-zero using `isKnownNonZero`. If that fails,
3429 // then see if the select condition implies the arm is non-zero. For example
3430 // (X != 0 ? X : Y), we know the true arm is non-zero as the `X` "return" is
3431 // dominated by `X != 0`.
3432 auto SelectArmIsNonZero = [&](bool IsTrueArm) {
3433 Value *Op;
3434 Op = IsTrueArm ? I->getOperand(i: 1) : I->getOperand(i: 2);
3435 // Op is trivially non-zero.
3436 if (isKnownNonZero(V: Op, DemandedElts, Q, Depth))
3437 return true;
3438
3439 // The condition of the select dominates the true/false arm. Check if the
3440 // condition implies that a given arm is non-zero.
3441 Value *X;
3442 CmpPredicate Pred;
3443 if (!match(V: I->getOperand(i: 0), P: m_c_ICmp(Pred, L: m_Specific(V: Op), R: m_Value(V&: X))))
3444 return false;
3445
3446 if (!IsTrueArm)
3447 Pred = ICmpInst::getInversePredicate(pred: Pred);
3448
3449 return cmpExcludesZero(Pred, RHS: X);
3450 };
3451
3452 if (SelectArmIsNonZero(/* IsTrueArm */ true) &&
3453 SelectArmIsNonZero(/* IsTrueArm */ false))
3454 return true;
3455 break;
3456 }
3457 case Instruction::PHI: {
3458 auto *PN = cast<PHINode>(Val: I);
3459 if (Q.IIQ.UseInstrInfo && isNonZeroRecurrence(PN))
3460 return true;
3461
3462 // Check if all incoming values are non-zero using recursion.
3463 SimplifyQuery RecQ = Q.getWithoutCondContext();
3464 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
3465 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
3466 if (U.get() == PN)
3467 return true;
3468 RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
3469 // Check if the branch on the phi excludes zero.
3470 CmpPredicate Pred;
3471 Value *X;
3472 BasicBlock *TrueSucc, *FalseSucc;
3473 if (match(V: RecQ.CxtI,
3474 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: U.get()), R: m_Value(V&: X)),
3475 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
3476 // Check for cases of duplicate successors.
3477 if ((TrueSucc == PN->getParent()) != (FalseSucc == PN->getParent())) {
3478 // If we're using the false successor, invert the predicate.
3479 if (FalseSucc == PN->getParent())
3480 Pred = CmpInst::getInversePredicate(pred: Pred);
3481 if (cmpExcludesZero(Pred, RHS: X))
3482 return true;
3483 }
3484 }
3485 // Finally recurse on the edge and check it directly.
3486 return isKnownNonZero(V: U.get(), DemandedElts, Q: RecQ, Depth: NewDepth);
3487 });
3488 }
3489 case Instruction::InsertElement: {
3490 if (isa<ScalableVectorType>(Val: I->getType()))
3491 break;
3492
3493 const Value *Vec = I->getOperand(i: 0);
3494 const Value *Elt = I->getOperand(i: 1);
3495 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
3496
3497 unsigned NumElts = DemandedElts.getBitWidth();
3498 APInt DemandedVecElts = DemandedElts;
3499 bool SkipElt = false;
3500 // If we know the index we are inserting too, clear it from Vec check.
3501 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
3502 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
3503 SkipElt = !DemandedElts[CIdx->getZExtValue()];
3504 }
3505
3506 // Result is zero if Elt is non-zero and rest of the demanded elts in Vec
3507 // are non-zero.
3508 return (SkipElt || isKnownNonZero(V: Elt, Q, Depth)) &&
3509 (DemandedVecElts.isZero() ||
3510 isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth));
3511 }
3512 case Instruction::ExtractElement:
3513 if (const auto *EEI = dyn_cast<ExtractElementInst>(Val: I)) {
3514 const Value *Vec = EEI->getVectorOperand();
3515 const Value *Idx = EEI->getIndexOperand();
3516 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
3517 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
3518 unsigned NumElts = VecTy->getNumElements();
3519 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
3520 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
3521 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
3522 return isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth);
3523 }
3524 }
3525 break;
3526 case Instruction::ShuffleVector: {
3527 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
3528 if (!Shuf)
3529 break;
3530 APInt DemandedLHS, DemandedRHS;
3531 // For undef elements, we don't know anything about the common state of
3532 // the shuffle result.
3533 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
3534 break;
3535 // If demanded elements for both vecs are non-zero, the shuffle is non-zero.
3536 return (DemandedRHS.isZero() ||
3537 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 1), DemandedElts: DemandedRHS, Q, Depth)) &&
3538 (DemandedLHS.isZero() ||
3539 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 0), DemandedElts: DemandedLHS, Q, Depth));
3540 }
3541 case Instruction::Freeze:
3542 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth) &&
3543 isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
3544 Depth);
3545 case Instruction::Load: {
3546 auto *LI = cast<LoadInst>(Val: I);
3547 // A Load tagged with nonnull or dereferenceable with null pointer undefined
3548 // is never null.
3549 if (auto *PtrT = dyn_cast<PointerType>(Val: I->getType())) {
3550 if (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_nonnull) ||
3551 (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_dereferenceable) &&
3552 !NullPointerIsDefined(F: LI->getFunction(), AS: PtrT->getAddressSpace())))
3553 return true;
3554 } else if (MDNode *Ranges = Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_range)) {
3555 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3556 }
3557
3558 // No need to fall through to computeKnownBits as range metadata is already
3559 // handled in isKnownNonZero.
3560 return false;
3561 }
3562 case Instruction::ExtractValue: {
3563 const WithOverflowInst *WO;
3564 if (match(V: I, P: m_ExtractValue<0>(V: m_WithOverflowInst(I&: WO)))) {
3565 switch (WO->getBinaryOp()) {
3566 default:
3567 break;
3568 case Instruction::Add:
3569 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3570 Y: WO->getArgOperand(i: 1),
3571 /*NSW=*/false,
3572 /*NUW=*/false, Depth);
3573 case Instruction::Sub:
3574 return isNonZeroSub(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3575 Y: WO->getArgOperand(i: 1), Depth);
3576 case Instruction::Mul:
3577 return isNonZeroMul(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3578 Y: WO->getArgOperand(i: 1),
3579 /*NSW=*/false, /*NUW=*/false, Depth);
3580 break;
3581 }
3582 }
3583 break;
3584 }
3585 case Instruction::Call:
3586 case Instruction::Invoke: {
3587 const auto *Call = cast<CallBase>(Val: I);
3588 if (I->getType()->isPointerTy()) {
3589 if (Call->isReturnNonNull())
3590 return true;
3591 if (const auto *RP = getArgumentAliasingToReturnedPointer(
3592 Call, /*MustPreserveOffset=*/true))
3593 return isKnownNonZero(V: RP, Q, Depth);
3594 } else {
3595 if (MDNode *Ranges = Q.IIQ.getMetadata(I: Call, KindID: LLVMContext::MD_range))
3596 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3597 if (std::optional<ConstantRange> Range = Call->getRange()) {
3598 const APInt ZeroValue(Range->getBitWidth(), 0);
3599 if (!Range->contains(Val: ZeroValue))
3600 return true;
3601 }
3602 if (const Value *RV = Call->getReturnedArgOperand())
3603 if (RV->getType() == I->getType() && isKnownNonZero(V: RV, Q, Depth))
3604 return true;
3605 }
3606
3607 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3608 switch (II->getIntrinsicID()) {
3609 case Intrinsic::sshl_sat:
3610 case Intrinsic::ushl_sat:
3611 case Intrinsic::abs:
3612 case Intrinsic::bitreverse:
3613 case Intrinsic::bswap:
3614 case Intrinsic::ctpop:
3615 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3616 // NB: We don't do usub_sat here as in any case we can prove its
3617 // non-zero, we will fold it to `sub nuw` in InstCombine.
3618 case Intrinsic::ssub_sat:
3619 // For most types, if x != y then ssub.sat x, y != 0. But
3620 // ssub.sat.i1 0, -1 = 0, because 1 saturates to 0. This means
3621 // isNonZeroSub will do the wrong thing for ssub.sat.i1.
3622 if (BitWidth == 1)
3623 return false;
3624 return isNonZeroSub(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3625 Y: II->getArgOperand(i: 1), Depth);
3626 case Intrinsic::sadd_sat:
3627 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3628 Y: II->getArgOperand(i: 1),
3629 /*NSW=*/true, /* NUW=*/false, Depth);
3630 // Vec reverse preserves zero/non-zero status from input vec.
3631 case Intrinsic::vector_reverse:
3632 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
3633 Q, Depth);
3634 // umin/smin/smax/smin/or of all non-zero elements is always non-zero.
3635 case Intrinsic::vector_reduce_or:
3636 case Intrinsic::vector_reduce_umax:
3637 case Intrinsic::vector_reduce_umin:
3638 case Intrinsic::vector_reduce_smax:
3639 case Intrinsic::vector_reduce_smin:
3640 return isKnownNonZero(V: II->getArgOperand(i: 0), Q, Depth);
3641 case Intrinsic::umax:
3642 case Intrinsic::uadd_sat:
3643 // umax(X, (X != 0)) is non zero
3644 // X +usat (X != 0) is non zero
3645 if (matchOpWithOpEqZero(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1)))
3646 return true;
3647
3648 return isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth) ||
3649 isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3650 case Intrinsic::smax: {
3651 // If either arg is strictly positive the result is non-zero. Otherwise
3652 // the result is non-zero if both ops are non-zero.
3653 auto IsNonZero = [&](Value *Op, std::optional<bool> &OpNonZero,
3654 const KnownBits &OpKnown) {
3655 if (!OpNonZero.has_value())
3656 OpNonZero = OpKnown.isNonZero() ||
3657 isKnownNonZero(V: Op, DemandedElts, Q, Depth);
3658 return *OpNonZero;
3659 };
3660 // Avoid re-computing isKnownNonZero.
3661 std::optional<bool> Op0NonZero, Op1NonZero;
3662 KnownBits Op1Known =
3663 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3664 if (Op1Known.isNonNegative() &&
3665 IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known))
3666 return true;
3667 KnownBits Op0Known =
3668 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3669 if (Op0Known.isNonNegative() &&
3670 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known))
3671 return true;
3672 return IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known) &&
3673 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known);
3674 }
3675 case Intrinsic::smin: {
3676 // If either arg is negative the result is non-zero. Otherwise
3677 // the result is non-zero if both ops are non-zero.
3678 KnownBits Op1Known =
3679 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3680 if (Op1Known.isNegative())
3681 return true;
3682 KnownBits Op0Known =
3683 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3684 if (Op0Known.isNegative())
3685 return true;
3686
3687 if (Op1Known.isNonZero() && Op0Known.isNonZero())
3688 return true;
3689 }
3690 [[fallthrough]];
3691 case Intrinsic::umin:
3692 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth) &&
3693 isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3694 case Intrinsic::cttz:
3695 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3696 .Zero[0];
3697 case Intrinsic::ctlz:
3698 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3699 .isNonNegative();
3700 case Intrinsic::fshr:
3701 case Intrinsic::fshl:
3702 // If Op0 == Op1, this is a rotate. rotate(x, y) != 0 iff x != 0.
3703 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
3704 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3705 break;
3706 case Intrinsic::vscale:
3707 return true;
3708 case Intrinsic::experimental_get_vector_length:
3709 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3710 default:
3711 break;
3712 }
3713 break;
3714 }
3715
3716 return false;
3717 }
3718 }
3719
3720 KnownBits Known(BitWidth);
3721 computeKnownBits(V: I, DemandedElts, Known, Q, Depth);
3722 return Known.One != 0;
3723}
3724
3725/// Return true if the given value is known to be non-zero when defined. For
3726/// vectors, return true if every demanded element is known to be non-zero when
3727/// defined. For pointers, if the context instruction and dominator tree are
3728/// specified, perform context-sensitive analysis and return true if the
3729/// pointer couldn't possibly be null at the specified instruction.
3730/// Supports values with integer or pointer type and vectors of integers.
3731bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
3732 const SimplifyQuery &Q, unsigned Depth) {
3733 Type *Ty = V->getType();
3734
3735#ifndef NDEBUG
3736 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
3737
3738 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
3739 assert(
3740 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
3741 "DemandedElt width should equal the fixed vector number of elements");
3742 } else {
3743 assert(DemandedElts == APInt(1, 1) &&
3744 "DemandedElt width should be 1 for scalars");
3745 }
3746#endif
3747
3748 if (auto *C = dyn_cast<Constant>(Val: V)) {
3749 if (C->isNullValue())
3750 return false;
3751 if (isa<ConstantInt>(Val: C))
3752 // Must be non-zero due to null test above.
3753 return true;
3754
3755 // For constant vectors, check that all elements are poison or known
3756 // non-zero to determine that the whole vector is known non-zero.
3757 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Ty)) {
3758 for (unsigned i = 0, e = VecTy->getNumElements(); i != e; ++i) {
3759 if (!DemandedElts[i])
3760 continue;
3761 Constant *Elt = C->getAggregateElement(Elt: i);
3762 if (!Elt || Elt->isNullValue())
3763 return false;
3764 if (!isa<PoisonValue>(Val: Elt) && !isa<ConstantInt>(Val: Elt))
3765 return false;
3766 }
3767 return true;
3768 }
3769
3770 // Constant ptrauth can be null, iff the base pointer can be.
3771 if (auto *CPA = dyn_cast<ConstantPtrAuth>(Val: V))
3772 return isKnownNonZero(V: CPA->getPointer(), DemandedElts, Q, Depth);
3773
3774 // A global variable in address space 0 is non null unless extern weak
3775 // or an absolute symbol reference. Other address spaces may have null as a
3776 // valid address for a global, so we can't assume anything.
3777 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
3778 if (!GV->isAbsoluteSymbolRef() && !GV->hasExternalWeakLinkage() &&
3779 GV->getType()->getAddressSpace() == 0)
3780 return true;
3781 }
3782
3783 // For constant expressions, fall through to the Operator code below.
3784 if (!isa<ConstantExpr>(Val: V))
3785 return false;
3786 }
3787
3788 if (const auto *A = dyn_cast<Argument>(Val: V))
3789 if (std::optional<ConstantRange> Range = A->getRange()) {
3790 const APInt ZeroValue(Range->getBitWidth(), 0);
3791 if (!Range->contains(Val: ZeroValue))
3792 return true;
3793 }
3794
3795 if (!isa<Constant>(Val: V) && isKnownNonZeroFromAssume(V, Q))
3796 return true;
3797
3798 // Some of the tests below are recursive, so bail out if we hit the limit.
3799 if (Depth++ >= MaxAnalysisRecursionDepth)
3800 return false;
3801
3802 // Check for pointer simplifications.
3803
3804 if (PointerType *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
3805 // A byval, inalloca may not be null in a non-default addres space. A
3806 // nonnull argument is assumed never 0.
3807 if (const Argument *A = dyn_cast<Argument>(Val: V)) {
3808 if (((A->hasPassPointeeByValueCopyAttr() &&
3809 !NullPointerIsDefined(F: A->getParent(), AS: PtrTy->getAddressSpace())) ||
3810 A->hasNonNullAttr()))
3811 return true;
3812 }
3813 }
3814
3815 if (const auto *I = dyn_cast<Operator>(Val: V))
3816 if (isKnownNonZeroFromOperator(I, DemandedElts, Q, Depth))
3817 return true;
3818
3819 if (!isa<Constant>(Val: V) &&
3820 isKnownNonNullFromDominatingCondition(V, CtxI: Q.CxtI, DT: Q.DT))
3821 return true;
3822
3823 if (const Value *Stripped = stripNullTest(V))
3824 return isKnownNonZero(V: Stripped, DemandedElts, Q, Depth);
3825
3826 return false;
3827}
3828
3829bool llvm::isKnownNonZero(const Value *V, const SimplifyQuery &Q,
3830 unsigned Depth) {
3831 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
3832 APInt DemandedElts =
3833 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
3834 return ::isKnownNonZero(V, DemandedElts, Q, Depth);
3835}
3836
3837/// If the pair of operators are the same invertible function, return the
3838/// the operands of the function corresponding to each input. Otherwise,
3839/// return std::nullopt. An invertible function is one that is 1-to-1 and maps
3840/// every input value to exactly one output value. This is equivalent to
3841/// saying that Op1 and Op2 are equal exactly when the specified pair of
3842/// operands are equal, (except that Op1 and Op2 may be poison more often.)
3843static std::optional<std::pair<Value*, Value*>>
3844getInvertibleOperands(const Operator *Op1,
3845 const Operator *Op2) {
3846 if (Op1->getOpcode() != Op2->getOpcode())
3847 return std::nullopt;
3848
3849 auto getOperands = [&](unsigned OpNum) -> auto {
3850 return std::make_pair(x: Op1->getOperand(i: OpNum), y: Op2->getOperand(i: OpNum));
3851 };
3852
3853 switch (Op1->getOpcode()) {
3854 default:
3855 break;
3856 case Instruction::Or:
3857 if (!cast<PossiblyDisjointInst>(Val: Op1)->isDisjoint() ||
3858 !cast<PossiblyDisjointInst>(Val: Op2)->isDisjoint())
3859 break;
3860 [[fallthrough]];
3861 case Instruction::Xor:
3862 case Instruction::Add: {
3863 Value *Other;
3864 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 0)), R: m_Value(V&: Other))))
3865 return std::make_pair(x: Op1->getOperand(i: 1), y&: Other);
3866 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 1)), R: m_Value(V&: Other))))
3867 return std::make_pair(x: Op1->getOperand(i: 0), y&: Other);
3868 break;
3869 }
3870 case Instruction::Sub:
3871 if (Op1->getOperand(i: 0) == Op2->getOperand(i: 0))
3872 return getOperands(1);
3873 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3874 return getOperands(0);
3875 break;
3876 case Instruction::Mul: {
3877 // invertible if A * B == (A * B) mod 2^N where A, and B are integers
3878 // and N is the bitwdith. The nsw case is non-obvious, but proven by
3879 // alive2: https://alive2.llvm.org/ce/z/Z6D5qK
3880 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
3881 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
3882 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
3883 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
3884 break;
3885
3886 // Assume operand order has been canonicalized
3887 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1) &&
3888 isa<ConstantInt>(Val: Op1->getOperand(i: 1)) &&
3889 !cast<ConstantInt>(Val: Op1->getOperand(i: 1))->isZero())
3890 return getOperands(0);
3891 break;
3892 }
3893 case Instruction::Shl: {
3894 // Same as multiplies, with the difference that we don't need to check
3895 // for a non-zero multiply. Shifts always multiply by non-zero.
3896 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
3897 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
3898 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
3899 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
3900 break;
3901
3902 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3903 return getOperands(0);
3904 break;
3905 }
3906 case Instruction::AShr:
3907 case Instruction::LShr: {
3908 auto *PEO1 = cast<PossiblyExactOperator>(Val: Op1);
3909 auto *PEO2 = cast<PossiblyExactOperator>(Val: Op2);
3910 if (!PEO1->isExact() || !PEO2->isExact())
3911 break;
3912
3913 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3914 return getOperands(0);
3915 break;
3916 }
3917 case Instruction::SExt:
3918 case Instruction::ZExt:
3919 if (Op1->getOperand(i: 0)->getType() == Op2->getOperand(i: 0)->getType())
3920 return getOperands(0);
3921 break;
3922 case Instruction::PHI: {
3923 const PHINode *PN1 = cast<PHINode>(Val: Op1);
3924 const PHINode *PN2 = cast<PHINode>(Val: Op2);
3925
3926 // If PN1 and PN2 are both recurrences, can we prove the entire recurrences
3927 // are a single invertible function of the start values? Note that repeated
3928 // application of an invertible function is also invertible
3929 BinaryOperator *BO1 = nullptr;
3930 Value *Start1 = nullptr, *Step1 = nullptr;
3931 BinaryOperator *BO2 = nullptr;
3932 Value *Start2 = nullptr, *Step2 = nullptr;
3933 if (PN1->getParent() != PN2->getParent() ||
3934 !matchSimpleRecurrence(P: PN1, BO&: BO1, Start&: Start1, Step&: Step1) ||
3935 !matchSimpleRecurrence(P: PN2, BO&: BO2, Start&: Start2, Step&: Step2))
3936 break;
3937
3938 auto Values = getInvertibleOperands(Op1: cast<Operator>(Val: BO1),
3939 Op2: cast<Operator>(Val: BO2));
3940 if (!Values)
3941 break;
3942
3943 // We have to be careful of mutually defined recurrences here. Ex:
3944 // * X_i = X_(i-1) OP Y_(i-1), and Y_i = X_(i-1) OP V
3945 // * X_i = Y_i = X_(i-1) OP Y_(i-1)
3946 // The invertibility of these is complicated, and not worth reasoning
3947 // about (yet?).
3948 if (Values->first != PN1 || Values->second != PN2)
3949 break;
3950
3951 return std::make_pair(x&: Start1, y&: Start2);
3952 }
3953 }
3954 return std::nullopt;
3955}
3956
3957/// Return true if V1 == (binop V2, X), where X is known non-zero.
3958/// Only handle a small subset of binops where (binop V2, X) with non-zero X
3959/// implies V2 != V1.
3960static bool isModifyingBinopOfNonZero(const Value *V1, const Value *V2,
3961 const APInt &DemandedElts,
3962 const SimplifyQuery &Q, unsigned Depth) {
3963 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V1);
3964 if (!BO)
3965 return false;
3966 switch (BO->getOpcode()) {
3967 default:
3968 break;
3969 case Instruction::Or:
3970 if (!cast<PossiblyDisjointInst>(Val: V1)->isDisjoint())
3971 break;
3972 [[fallthrough]];
3973 case Instruction::Xor:
3974 case Instruction::Add:
3975 Value *Op = nullptr;
3976 if (V2 == BO->getOperand(i_nocapture: 0))
3977 Op = BO->getOperand(i_nocapture: 1);
3978 else if (V2 == BO->getOperand(i_nocapture: 1))
3979 Op = BO->getOperand(i_nocapture: 0);
3980 else
3981 return false;
3982 return isKnownNonZero(V: Op, DemandedElts, Q, Depth: Depth + 1);
3983 }
3984 return false;
3985}
3986
3987/// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
3988/// the multiplication is nuw or nsw.
3989static bool isNonEqualMul(const Value *V1, const Value *V2,
3990 const APInt &DemandedElts, const SimplifyQuery &Q,
3991 unsigned Depth) {
3992 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
3993 const APInt *C;
3994 return match(V: OBO, P: m_Mul(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
3995 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
3996 !C->isZero() && !C->isOne() &&
3997 isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
3998 }
3999 return false;
4000}
4001
4002/// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
4003/// the shift is nuw or nsw.
4004static bool isNonEqualShl(const Value *V1, const Value *V2,
4005 const APInt &DemandedElts, const SimplifyQuery &Q,
4006 unsigned Depth) {
4007 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
4008 const APInt *C;
4009 return match(V: OBO, P: m_Shl(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
4010 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
4011 !C->isZero() && isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
4012 }
4013 return false;
4014}
4015
4016static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
4017 const APInt &DemandedElts, const SimplifyQuery &Q,
4018 unsigned Depth) {
4019 // Check two PHIs are in same block.
4020 if (PN1->getParent() != PN2->getParent())
4021 return false;
4022
4023 SmallPtrSet<const BasicBlock *, 8> VisitedBBs;
4024 bool UsedFullRecursion = false;
4025 for (const BasicBlock *IncomBB : PN1->blocks()) {
4026 if (!VisitedBBs.insert(Ptr: IncomBB).second)
4027 continue; // Don't reprocess blocks that we have dealt with already.
4028 const Value *IV1 = PN1->getIncomingValueForBlock(BB: IncomBB);
4029 const Value *IV2 = PN2->getIncomingValueForBlock(BB: IncomBB);
4030 const APInt *C1, *C2;
4031 if (match(V: IV1, P: m_APInt(Res&: C1)) && match(V: IV2, P: m_APInt(Res&: C2)) && *C1 != *C2)
4032 continue;
4033
4034 // Only one pair of phi operands is allowed for full recursion.
4035 if (UsedFullRecursion)
4036 return false;
4037
4038 SimplifyQuery RecQ = Q.getWithoutCondContext();
4039 RecQ.CxtI = IncomBB->getTerminator();
4040 if (!isKnownNonEqual(V1: IV1, V2: IV2, DemandedElts, Q: RecQ, Depth: Depth + 1))
4041 return false;
4042 UsedFullRecursion = true;
4043 }
4044 return true;
4045}
4046
4047static bool isNonEqualSelect(const Value *V1, const Value *V2,
4048 const APInt &DemandedElts, const SimplifyQuery &Q,
4049 unsigned Depth) {
4050 const SelectInst *SI1 = dyn_cast<SelectInst>(Val: V1);
4051 if (!SI1)
4052 return false;
4053
4054 if (const SelectInst *SI2 = dyn_cast<SelectInst>(Val: V2)) {
4055 const Value *Cond1 = SI1->getCondition();
4056 const Value *Cond2 = SI2->getCondition();
4057 if (Cond1 == Cond2)
4058 return isKnownNonEqual(V1: SI1->getTrueValue(), V2: SI2->getTrueValue(),
4059 DemandedElts, Q, Depth: Depth + 1) &&
4060 isKnownNonEqual(V1: SI1->getFalseValue(), V2: SI2->getFalseValue(),
4061 DemandedElts, Q, Depth: Depth + 1);
4062 }
4063 return isKnownNonEqual(V1: SI1->getTrueValue(), V2, DemandedElts, Q, Depth: Depth + 1) &&
4064 isKnownNonEqual(V1: SI1->getFalseValue(), V2, DemandedElts, Q, Depth: Depth + 1);
4065}
4066
4067// Check to see if A is both a GEP and is the incoming value for a PHI in the
4068// loop, and B is either a ptr or another GEP. If the PHI has 2 incoming values,
4069// one of them being the recursive GEP A and the other a ptr at same base and at
4070// the same/higher offset than B we are only incrementing the pointer further in
4071// loop if offset of recursive GEP is greater than 0.
4072static bool isNonEqualPointersWithRecursiveGEP(const Value *A, const Value *B,
4073 const SimplifyQuery &Q) {
4074 if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
4075 return false;
4076
4077 auto *GEPA = dyn_cast<GEPOperator>(Val: A);
4078 if (!GEPA || GEPA->getNumIndices() != 1 || !isa<Constant>(Val: GEPA->idx_begin()))
4079 return false;
4080
4081 // Handle 2 incoming PHI values with one being a recursive GEP.
4082 auto *PN = dyn_cast<PHINode>(Val: GEPA->getPointerOperand());
4083 if (!PN || PN->getNumIncomingValues() != 2)
4084 return false;
4085
4086 // Search for the recursive GEP as an incoming operand, and record that as
4087 // Step.
4088 Value *Start = nullptr;
4089 Value *Step = const_cast<Value *>(A);
4090 if (PN->getIncomingValue(i: 0) == Step)
4091 Start = PN->getIncomingValue(i: 1);
4092 else if (PN->getIncomingValue(i: 1) == Step)
4093 Start = PN->getIncomingValue(i: 0);
4094 else
4095 return false;
4096
4097 // Other incoming node base should match the B base.
4098 // StartOffset >= OffsetB && StepOffset > 0?
4099 // StartOffset <= OffsetB && StepOffset < 0?
4100 // Is non-equal if above are true.
4101 // We use stripAndAccumulateInBoundsConstantOffsets to restrict the
4102 // optimisation to inbounds GEPs only.
4103 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: Start->getType());
4104 APInt StartOffset(IndexWidth, 0);
4105 Start = Start->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StartOffset);
4106 APInt StepOffset(IndexWidth, 0);
4107 Step = Step->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StepOffset);
4108
4109 // Check if Base Pointer of Step matches the PHI.
4110 if (Step != PN)
4111 return false;
4112 APInt OffsetB(IndexWidth, 0);
4113 B = B->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: OffsetB);
4114 return Start == B &&
4115 ((StartOffset.sge(RHS: OffsetB) && StepOffset.isStrictlyPositive()) ||
4116 (StartOffset.sle(RHS: OffsetB) && StepOffset.isNegative()));
4117}
4118
4119static bool isKnownNonEqualFromContext(const Value *V1, const Value *V2,
4120 const SimplifyQuery &Q, unsigned Depth) {
4121 if (!Q.CxtI)
4122 return false;
4123
4124 // Try to infer NonEqual based on information from dominating conditions.
4125 if (Q.DC && Q.DT) {
4126 auto IsKnownNonEqualFromDominatingCondition = [&](const Value *V) {
4127 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4128 Value *Cond = BI->getCondition();
4129 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4130 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()) &&
4131 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4132 /*LHSIsTrue=*/true, Depth)
4133 .value_or(u: false))
4134 return true;
4135
4136 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4137 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()) &&
4138 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4139 /*LHSIsTrue=*/false, Depth)
4140 .value_or(u: false))
4141 return true;
4142 }
4143
4144 return false;
4145 };
4146
4147 if (IsKnownNonEqualFromDominatingCondition(V1) ||
4148 IsKnownNonEqualFromDominatingCondition(V2))
4149 return true;
4150 }
4151
4152 if (!Q.AC)
4153 return false;
4154
4155 // Try to infer NonEqual based on information from assumptions.
4156 for (auto &AssumeVH : Q.AC->assumptionsFor(V: V1)) {
4157 if (!AssumeVH)
4158 continue;
4159 CallInst *I = cast<CallInst>(Val&: AssumeVH);
4160
4161 assert(I->getFunction() == Q.CxtI->getFunction() &&
4162 "Got assumption for the wrong function!");
4163 assert(I->getIntrinsicID() == Intrinsic::assume &&
4164 "must be an assume intrinsic");
4165
4166 if (isImpliedCondition(LHS: I->getArgOperand(i: 0), RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4167 /*LHSIsTrue=*/true, Depth)
4168 .value_or(u: false) &&
4169 isValidAssumeForContext(I, Q))
4170 return true;
4171 }
4172
4173 return false;
4174}
4175
4176/// Return true if it is known that V1 != V2.
4177static bool isKnownNonEqual(const Value *V1, const Value *V2,
4178 const APInt &DemandedElts, const SimplifyQuery &Q,
4179 unsigned Depth) {
4180 if (V1 == V2)
4181 return false;
4182 if (V1->getType() != V2->getType())
4183 // We can't look through casts yet.
4184 return false;
4185
4186 if (Depth >= MaxAnalysisRecursionDepth)
4187 return false;
4188
4189 // See if we can recurse through (exactly one of) our operands. This
4190 // requires our operation be 1-to-1 and map every input value to exactly
4191 // one output value. Such an operation is invertible.
4192 auto *O1 = dyn_cast<Operator>(Val: V1);
4193 auto *O2 = dyn_cast<Operator>(Val: V2);
4194 if (O1 && O2 && O1->getOpcode() == O2->getOpcode()) {
4195 if (auto Values = getInvertibleOperands(Op1: O1, Op2: O2))
4196 return isKnownNonEqual(V1: Values->first, V2: Values->second, DemandedElts, Q,
4197 Depth: Depth + 1);
4198
4199 if (const PHINode *PN1 = dyn_cast<PHINode>(Val: V1)) {
4200 const PHINode *PN2 = cast<PHINode>(Val: V2);
4201 // FIXME: This is missing a generalization to handle the case where one is
4202 // a PHI and another one isn't.
4203 if (isNonEqualPHIs(PN1, PN2, DemandedElts, Q, Depth))
4204 return true;
4205 };
4206 }
4207
4208 if (isModifyingBinopOfNonZero(V1, V2, DemandedElts, Q, Depth) ||
4209 isModifyingBinopOfNonZero(V1: V2, V2: V1, DemandedElts, Q, Depth))
4210 return true;
4211
4212 if (isNonEqualMul(V1, V2, DemandedElts, Q, Depth) ||
4213 isNonEqualMul(V1: V2, V2: V1, DemandedElts, Q, Depth))
4214 return true;
4215
4216 if (isNonEqualShl(V1, V2, DemandedElts, Q, Depth) ||
4217 isNonEqualShl(V1: V2, V2: V1, DemandedElts, Q, Depth))
4218 return true;
4219
4220 if (V1->getType()->isIntOrIntVectorTy()) {
4221 // Are any known bits in V1 contradictory to known bits in V2? If V1
4222 // has a known zero where V2 has a known one, they must not be equal.
4223 KnownBits Known1 = computeKnownBits(V: V1, DemandedElts, Q, Depth);
4224 if (!Known1.isUnknown()) {
4225 KnownBits Known2 = computeKnownBits(V: V2, DemandedElts, Q, Depth);
4226 if (Known1.Zero.intersects(RHS: Known2.One) ||
4227 Known2.Zero.intersects(RHS: Known1.One))
4228 return true;
4229 }
4230 }
4231
4232 if (isNonEqualSelect(V1, V2, DemandedElts, Q, Depth) ||
4233 isNonEqualSelect(V1: V2, V2: V1, DemandedElts, Q, Depth))
4234 return true;
4235
4236 if (isNonEqualPointersWithRecursiveGEP(A: V1, B: V2, Q) ||
4237 isNonEqualPointersWithRecursiveGEP(A: V2, B: V1, Q))
4238 return true;
4239
4240 Value *A, *B;
4241 // PtrToInts are NonEqual if their Ptrs are NonEqual.
4242 // Check PtrToInt type matches the pointer size.
4243 if (match(V: V1, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: A))) &&
4244 match(V: V2, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: B))))
4245 return isKnownNonEqual(V1: A, V2: B, DemandedElts, Q, Depth: Depth + 1);
4246
4247 if (isKnownNonEqualFromContext(V1, V2, Q, Depth))
4248 return true;
4249
4250 return false;
4251}
4252
4253/// For vector constants, loop over the elements and find the constant with the
4254/// minimum number of sign bits. Return 0 if the value is not a vector constant
4255/// or if any element was not analyzed; otherwise, return the count for the
4256/// element with the minimum number of sign bits.
4257static unsigned computeNumSignBitsVectorConstant(const Value *V,
4258 const APInt &DemandedElts,
4259 unsigned TyBits) {
4260 const auto *CV = dyn_cast<Constant>(Val: V);
4261 if (!CV || !isa<FixedVectorType>(Val: CV->getType()))
4262 return 0;
4263
4264 unsigned MinSignBits = TyBits;
4265 unsigned NumElts = cast<FixedVectorType>(Val: CV->getType())->getNumElements();
4266 for (unsigned i = 0; i != NumElts; ++i) {
4267 if (!DemandedElts[i])
4268 continue;
4269 // If we find a non-ConstantInt, bail out.
4270 auto *Elt = dyn_cast_or_null<ConstantInt>(Val: CV->getAggregateElement(Elt: i));
4271 if (!Elt)
4272 return 0;
4273
4274 MinSignBits = std::min(a: MinSignBits, b: Elt->getValue().getNumSignBits());
4275 }
4276
4277 return MinSignBits;
4278}
4279
4280static unsigned ComputeNumSignBitsImpl(const Value *V,
4281 const APInt &DemandedElts,
4282 const SimplifyQuery &Q, unsigned Depth);
4283
4284static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
4285 const SimplifyQuery &Q, unsigned Depth) {
4286 unsigned Result = ComputeNumSignBitsImpl(V, DemandedElts, Q, Depth);
4287 assert(Result > 0 && "At least one sign bit needs to be present!");
4288 return Result;
4289}
4290
4291/// Return the number of times the sign bit of the register is replicated into
4292/// the other bits. We know that at least 1 bit is always equal to the sign bit
4293/// (itself), but other cases can give us information. For example, immediately
4294/// after an "ashr X, 2", we know that the top 3 bits are all equal to each
4295/// other, so we return 3. For vectors, return the number of sign bits for the
4296/// vector element with the minimum number of known sign bits of the demanded
4297/// elements in the vector specified by DemandedElts.
4298static unsigned ComputeNumSignBitsImpl(const Value *V,
4299 const APInt &DemandedElts,
4300 const SimplifyQuery &Q, unsigned Depth) {
4301 Type *Ty = V->getType();
4302#ifndef NDEBUG
4303 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
4304
4305 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
4306 assert(
4307 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
4308 "DemandedElt width should equal the fixed vector number of elements");
4309 } else {
4310 assert(DemandedElts == APInt(1, 1) &&
4311 "DemandedElt width should be 1 for scalars");
4312 }
4313#endif
4314
4315 // We return the minimum number of sign bits that are guaranteed to be present
4316 // in V, so for undef we have to conservatively return 1. We don't have the
4317 // same behavior for poison though -- that's a FIXME today.
4318
4319 Type *ScalarTy = Ty->getScalarType();
4320 unsigned TyBits = ScalarTy->isPointerTy() ?
4321 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
4322 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
4323
4324 unsigned Tmp, Tmp2;
4325 unsigned FirstAnswer = 1;
4326
4327 // Note that ConstantInt is handled by the general computeKnownBits case
4328 // below.
4329
4330 if (Depth == MaxAnalysisRecursionDepth)
4331 return 1;
4332
4333 if (auto *U = dyn_cast<Operator>(Val: V)) {
4334 switch (Operator::getOpcode(V)) {
4335 default: break;
4336 case Instruction::BitCast: {
4337 Value *Src = U->getOperand(i: 0);
4338 Type *SrcTy = Src->getType();
4339
4340 // Skip if the source type is not an integer or integer vector type
4341 // This ensures we only process integer-like types
4342 if (!SrcTy->isIntOrIntVectorTy())
4343 break;
4344
4345 unsigned SrcBits = SrcTy->getScalarSizeInBits();
4346
4347 // Bitcast 'large element' scalar/vector to 'small element' vector.
4348 if ((SrcBits % TyBits) != 0)
4349 break;
4350
4351 // Only proceed if the destination type is a fixed-size vector
4352 if (isa<FixedVectorType>(Val: Ty)) {
4353 // Fast case - sign splat can be simply split across the small elements.
4354 // This works for both vector and scalar sources
4355 Tmp = ComputeNumSignBits(V: Src, Q, Depth: Depth + 1);
4356 if (Tmp == SrcBits)
4357 return TyBits;
4358 }
4359 break;
4360 }
4361 case Instruction::SExt:
4362 Tmp = TyBits - U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4363 return ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1) +
4364 Tmp;
4365
4366 case Instruction::SDiv: {
4367 const APInt *Denominator;
4368 // sdiv X, C -> adds log(C) sign bits.
4369 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4370
4371 // Ignore non-positive denominator.
4372 if (!Denominator->isStrictlyPositive())
4373 break;
4374
4375 // Calculate the incoming numerator bits.
4376 unsigned NumBits =
4377 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4378
4379 // Add floor(log(C)) bits to the numerator bits.
4380 return std::min(a: TyBits, b: NumBits + Denominator->logBase2());
4381 }
4382 break;
4383 }
4384
4385 case Instruction::SRem: {
4386 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4387
4388 const APInt *Denominator;
4389 // srem X, C -> we know that the result is within [-C+1,C) when C is a
4390 // positive constant. This let us put a lower bound on the number of sign
4391 // bits.
4392 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4393
4394 // Ignore non-positive denominator.
4395 if (Denominator->isStrictlyPositive()) {
4396 // Calculate the leading sign bit constraints by examining the
4397 // denominator. Given that the denominator is positive, there are two
4398 // cases:
4399 //
4400 // 1. The numerator is positive. The result range is [0,C) and
4401 // [0,C) u< (1 << ceilLogBase2(C)).
4402 //
4403 // 2. The numerator is negative. Then the result range is (-C,0] and
4404 // integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
4405 //
4406 // Thus a lower bound on the number of sign bits is `TyBits -
4407 // ceilLogBase2(C)`.
4408
4409 unsigned ResBits = TyBits - Denominator->ceilLogBase2();
4410 Tmp = std::max(a: Tmp, b: ResBits);
4411 }
4412 }
4413 return Tmp;
4414 }
4415
4416 case Instruction::AShr: {
4417 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4418 // ashr X, C -> adds C sign bits. Vectors too.
4419 const APInt *ShAmt;
4420 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4421 if (ShAmt->uge(RHS: TyBits))
4422 break; // Bad shift.
4423 unsigned ShAmtLimited = ShAmt->getZExtValue();
4424 Tmp += ShAmtLimited;
4425 if (Tmp > TyBits) Tmp = TyBits;
4426 }
4427 return Tmp;
4428 }
4429 case Instruction::Shl: {
4430 const APInt *ShAmt;
4431 Value *X = nullptr;
4432 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4433 // shl destroys sign bits.
4434 if (ShAmt->uge(RHS: TyBits))
4435 break; // Bad shift.
4436 // We can look through a zext (more or less treating it as a sext) if
4437 // all extended bits are shifted out.
4438 if (match(V: U->getOperand(i: 0), P: m_ZExt(Op: m_Value(V&: X))) &&
4439 ShAmt->uge(RHS: TyBits - X->getType()->getScalarSizeInBits())) {
4440 Tmp = ComputeNumSignBits(V: X, DemandedElts, Q, Depth: Depth + 1);
4441 Tmp += TyBits - X->getType()->getScalarSizeInBits();
4442 } else
4443 Tmp =
4444 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4445 if (ShAmt->uge(RHS: Tmp))
4446 break; // Shifted all sign bits out.
4447 Tmp2 = ShAmt->getZExtValue();
4448 return Tmp - Tmp2;
4449 }
4450 break;
4451 }
4452 case Instruction::And:
4453 case Instruction::Or:
4454 case Instruction::Xor: // NOT is handled here.
4455 // Logical binary ops preserve the number of sign bits at the worst.
4456 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4457 if (Tmp != 1) {
4458 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4459 FirstAnswer = std::min(a: Tmp, b: Tmp2);
4460 // We computed what we know about the sign bits as our first
4461 // answer. Now proceed to the generic code that uses
4462 // computeKnownBits, and pick whichever answer is better.
4463 }
4464 break;
4465
4466 case Instruction::Select: {
4467 // If we have a clamp pattern, we know that the number of sign bits will
4468 // be the minimum of the clamp min/max range.
4469 const Value *X;
4470 const APInt *CLow, *CHigh;
4471 if (isSignedMinMaxClamp(Select: U, In&: X, CLow, CHigh))
4472 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4473
4474 Tmp = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4475 if (Tmp == 1)
4476 break;
4477 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 2), DemandedElts, Q, Depth: Depth + 1);
4478 return std::min(a: Tmp, b: Tmp2);
4479 }
4480
4481 case Instruction::Add:
4482 // Add can have at most one carry bit. Thus we know that the output
4483 // is, at worst, one more bit than the inputs.
4484 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4485 if (Tmp == 1) break;
4486
4487 // Special case decrementing a value (ADD X, -1):
4488 if (const auto *CRHS = dyn_cast<Constant>(Val: U->getOperand(i: 1)))
4489 if (CRHS->isAllOnesValue()) {
4490 KnownBits Known(TyBits);
4491 computeKnownBits(V: U->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
4492
4493 // If the input is known to be 0 or 1, the output is 0/-1, which is
4494 // all sign bits set.
4495 if ((Known.Zero | 1).isAllOnes())
4496 return TyBits;
4497
4498 // If we are subtracting one from a positive number, there is no carry
4499 // out of the result.
4500 if (Known.isNonNegative())
4501 return Tmp;
4502 }
4503
4504 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4505 if (Tmp2 == 1)
4506 break;
4507 return std::min(a: Tmp, b: Tmp2) - 1;
4508
4509 case Instruction::Sub:
4510 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4511 if (Tmp2 == 1)
4512 break;
4513
4514 // Handle NEG.
4515 if (const auto *CLHS = dyn_cast<Constant>(Val: U->getOperand(i: 0)))
4516 if (CLHS->isNullValue()) {
4517 KnownBits Known(TyBits);
4518 computeKnownBits(V: U->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
4519 // If the input is known to be 0 or 1, the output is 0/-1, which is
4520 // all sign bits set.
4521 if ((Known.Zero | 1).isAllOnes())
4522 return TyBits;
4523
4524 // If the input is known to be positive (the sign bit is known clear),
4525 // the output of the NEG has the same number of sign bits as the
4526 // input.
4527 if (Known.isNonNegative())
4528 return Tmp2;
4529
4530 // Otherwise, we treat this like a SUB.
4531 }
4532
4533 // Sub can have at most one carry bit. Thus we know that the output
4534 // is, at worst, one more bit than the inputs.
4535 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4536 if (Tmp == 1)
4537 break;
4538 return std::min(a: Tmp, b: Tmp2) - 1;
4539
4540 case Instruction::Mul: {
4541 // The output of the Mul can be at most twice the valid bits in the
4542 // inputs.
4543 unsigned SignBitsOp0 =
4544 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4545 if (SignBitsOp0 == 1)
4546 break;
4547 unsigned SignBitsOp1 =
4548 ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4549 if (SignBitsOp1 == 1)
4550 break;
4551 unsigned OutValidBits =
4552 (TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
4553 return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
4554 }
4555
4556 case Instruction::PHI: {
4557 const PHINode *PN = cast<PHINode>(Val: U);
4558 unsigned NumIncomingValues = PN->getNumIncomingValues();
4559 // Don't analyze large in-degree PHIs.
4560 if (NumIncomingValues > 4) break;
4561 // Unreachable blocks may have zero-operand PHI nodes.
4562 if (NumIncomingValues == 0) break;
4563
4564 // Take the minimum of all incoming values. This can't infinitely loop
4565 // because of our depth threshold.
4566 SimplifyQuery RecQ = Q.getWithoutCondContext();
4567 Tmp = TyBits;
4568 for (unsigned i = 0, e = NumIncomingValues; i != e; ++i) {
4569 if (Tmp == 1) return Tmp;
4570 RecQ.CxtI = PN->getIncomingBlock(i)->getTerminator();
4571 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(V: PN->getIncomingValue(i),
4572 DemandedElts, Q: RecQ, Depth: Depth + 1));
4573 }
4574 return Tmp;
4575 }
4576
4577 case Instruction::Trunc: {
4578 // If the input contained enough sign bits that some remain after the
4579 // truncation, then we can make use of that. Otherwise we don't know
4580 // anything.
4581 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4582 unsigned OperandTyBits = U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4583 if (Tmp > (OperandTyBits - TyBits))
4584 return Tmp - (OperandTyBits - TyBits);
4585
4586 return 1;
4587 }
4588
4589 case Instruction::ExtractElement:
4590 // Look through extract element. At the moment we keep this simple and
4591 // skip tracking the specific element. But at least we might find
4592 // information valid for all elements of the vector (for example if vector
4593 // is sign extended, shifted, etc).
4594 return ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4595
4596 case Instruction::ShuffleVector: {
4597 // Collect the minimum number of sign bits that are shared by every vector
4598 // element referenced by the shuffle.
4599 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: U);
4600 if (!Shuf) {
4601 // FIXME: Add support for shufflevector constant expressions.
4602 return 1;
4603 }
4604 APInt DemandedLHS, DemandedRHS;
4605 // For undef elements, we don't know anything about the common state of
4606 // the shuffle result.
4607 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
4608 return 1;
4609 Tmp = std::numeric_limits<unsigned>::max();
4610 if (!!DemandedLHS) {
4611 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
4612 Tmp = ComputeNumSignBits(V: LHS, DemandedElts: DemandedLHS, Q, Depth: Depth + 1);
4613 }
4614 // If we don't know anything, early out and try computeKnownBits
4615 // fall-back.
4616 if (Tmp == 1)
4617 break;
4618 if (!!DemandedRHS) {
4619 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
4620 Tmp2 = ComputeNumSignBits(V: RHS, DemandedElts: DemandedRHS, Q, Depth: Depth + 1);
4621 Tmp = std::min(a: Tmp, b: Tmp2);
4622 }
4623 // If we don't know anything, early out and try computeKnownBits
4624 // fall-back.
4625 if (Tmp == 1)
4626 break;
4627 assert(Tmp <= TyBits && "Failed to determine minimum sign bits");
4628 return Tmp;
4629 }
4630 case Instruction::Call: {
4631 if (const auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
4632 switch (II->getIntrinsicID()) {
4633 default:
4634 break;
4635 case Intrinsic::abs:
4636 Tmp =
4637 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4638 if (Tmp == 1)
4639 break;
4640
4641 // Absolute value reduces number of sign bits by at most 1.
4642 return Tmp - 1;
4643 case Intrinsic::smin:
4644 case Intrinsic::smax: {
4645 const APInt *CLow, *CHigh;
4646 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
4647 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4648 }
4649 }
4650 }
4651 }
4652 }
4653 }
4654
4655 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4656 // use this information.
4657
4658 // If we can examine all elements of a vector constant successfully, we're
4659 // done (we can't do any better than that). If not, keep trying.
4660 if (unsigned VecSignBits =
4661 computeNumSignBitsVectorConstant(V, DemandedElts, TyBits))
4662 return VecSignBits;
4663
4664 KnownBits Known(TyBits);
4665 computeKnownBits(V, DemandedElts, Known, Q, Depth);
4666
4667 // If we know that the sign bit is either zero or one, determine the number of
4668 // identical bits in the top of the input value.
4669 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
4670}
4671
4672Intrinsic::ID llvm::getIntrinsicForCallSite(const CallBase &CB,
4673 const TargetLibraryInfo *TLI) {
4674 const Function *F = CB.getCalledFunction();
4675 if (!F)
4676 return Intrinsic::not_intrinsic;
4677
4678 if (F->isIntrinsic())
4679 return F->getIntrinsicID();
4680
4681 // We are going to infer semantics of a library function based on mapping it
4682 // to an LLVM intrinsic. Check that the library function is available from
4683 // this callbase and in this environment.
4684 LibFunc Func;
4685 if (F->hasLocalLinkage() || !TLI || !TLI->getLibFunc(CB, F&: Func) ||
4686 !CB.onlyReadsMemory())
4687 return Intrinsic::not_intrinsic;
4688
4689 switch (Func) {
4690 default:
4691 break;
4692 case LibFunc_sin:
4693 case LibFunc_sinf:
4694 case LibFunc_sinl:
4695 return Intrinsic::sin;
4696 case LibFunc_cos:
4697 case LibFunc_cosf:
4698 case LibFunc_cosl:
4699 return Intrinsic::cos;
4700 case LibFunc_tan:
4701 case LibFunc_tanf:
4702 case LibFunc_tanl:
4703 return Intrinsic::tan;
4704 case LibFunc_asin:
4705 case LibFunc_asinf:
4706 case LibFunc_asinl:
4707 return Intrinsic::asin;
4708 case LibFunc_acos:
4709 case LibFunc_acosf:
4710 case LibFunc_acosl:
4711 return Intrinsic::acos;
4712 case LibFunc_atan:
4713 case LibFunc_atanf:
4714 case LibFunc_atanl:
4715 return Intrinsic::atan;
4716 case LibFunc_atan2:
4717 case LibFunc_atan2f:
4718 case LibFunc_atan2l:
4719 return Intrinsic::atan2;
4720 case LibFunc_sinh:
4721 case LibFunc_sinhf:
4722 case LibFunc_sinhl:
4723 return Intrinsic::sinh;
4724 case LibFunc_cosh:
4725 case LibFunc_coshf:
4726 case LibFunc_coshl:
4727 return Intrinsic::cosh;
4728 case LibFunc_tanh:
4729 case LibFunc_tanhf:
4730 case LibFunc_tanhl:
4731 return Intrinsic::tanh;
4732 case LibFunc_exp:
4733 case LibFunc_expf:
4734 case LibFunc_expl:
4735 return Intrinsic::exp;
4736 case LibFunc_exp2:
4737 case LibFunc_exp2f:
4738 case LibFunc_exp2l:
4739 return Intrinsic::exp2;
4740 case LibFunc_exp10:
4741 case LibFunc_exp10f:
4742 case LibFunc_exp10l:
4743 return Intrinsic::exp10;
4744 case LibFunc_log:
4745 case LibFunc_logf:
4746 case LibFunc_logl:
4747 return Intrinsic::log;
4748 case LibFunc_log10:
4749 case LibFunc_log10f:
4750 case LibFunc_log10l:
4751 return Intrinsic::log10;
4752 case LibFunc_log2:
4753 case LibFunc_log2f:
4754 case LibFunc_log2l:
4755 return Intrinsic::log2;
4756 case LibFunc_fabs:
4757 case LibFunc_fabsf:
4758 case LibFunc_fabsl:
4759 return Intrinsic::fabs;
4760 case LibFunc_fmin:
4761 case LibFunc_fminf:
4762 case LibFunc_fminl:
4763 return Intrinsic::minnum;
4764 case LibFunc_fmax:
4765 case LibFunc_fmaxf:
4766 case LibFunc_fmaxl:
4767 return Intrinsic::maxnum;
4768 case LibFunc_copysign:
4769 case LibFunc_copysignf:
4770 case LibFunc_copysignl:
4771 return Intrinsic::copysign;
4772 case LibFunc_floor:
4773 case LibFunc_floorf:
4774 case LibFunc_floorl:
4775 return Intrinsic::floor;
4776 case LibFunc_ceil:
4777 case LibFunc_ceilf:
4778 case LibFunc_ceill:
4779 return Intrinsic::ceil;
4780 case LibFunc_trunc:
4781 case LibFunc_truncf:
4782 case LibFunc_truncl:
4783 return Intrinsic::trunc;
4784 case LibFunc_rint:
4785 case LibFunc_rintf:
4786 case LibFunc_rintl:
4787 return Intrinsic::rint;
4788 case LibFunc_nearbyint:
4789 case LibFunc_nearbyintf:
4790 case LibFunc_nearbyintl:
4791 return Intrinsic::nearbyint;
4792 case LibFunc_round:
4793 case LibFunc_roundf:
4794 case LibFunc_roundl:
4795 return Intrinsic::round;
4796 case LibFunc_roundeven:
4797 case LibFunc_roundevenf:
4798 case LibFunc_roundevenl:
4799 return Intrinsic::roundeven;
4800 case LibFunc_pow:
4801 case LibFunc_powf:
4802 case LibFunc_powl:
4803 return Intrinsic::pow;
4804 case LibFunc_sqrt:
4805 case LibFunc_sqrtf:
4806 case LibFunc_sqrtl:
4807 return Intrinsic::sqrt;
4808 }
4809
4810 return Intrinsic::not_intrinsic;
4811}
4812
4813/// Given an exploded icmp instruction, return true if the comparison only
4814/// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if
4815/// the result of the comparison is true when the input value is signed.
4816bool llvm::isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
4817 bool &TrueIfSigned) {
4818 switch (Pred) {
4819 case ICmpInst::ICMP_SLT: // True if LHS s< 0
4820 TrueIfSigned = true;
4821 return RHS.isZero();
4822 case ICmpInst::ICMP_SLE: // True if LHS s<= -1
4823 TrueIfSigned = true;
4824 return RHS.isAllOnes();
4825 case ICmpInst::ICMP_SGT: // True if LHS s> -1
4826 TrueIfSigned = false;
4827 return RHS.isAllOnes();
4828 case ICmpInst::ICMP_SGE: // True if LHS s>= 0
4829 TrueIfSigned = false;
4830 return RHS.isZero();
4831 case ICmpInst::ICMP_UGT:
4832 // True if LHS u> RHS and RHS == sign-bit-mask - 1
4833 TrueIfSigned = true;
4834 return RHS.isMaxSignedValue();
4835 case ICmpInst::ICMP_UGE:
4836 // True if LHS u>= RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
4837 TrueIfSigned = true;
4838 return RHS.isMinSignedValue();
4839 case ICmpInst::ICMP_ULT:
4840 // True if LHS u< RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
4841 TrueIfSigned = false;
4842 return RHS.isMinSignedValue();
4843 case ICmpInst::ICMP_ULE:
4844 // True if LHS u<= RHS and RHS == sign-bit-mask - 1
4845 TrueIfSigned = false;
4846 return RHS.isMaxSignedValue();
4847 default:
4848 return false;
4849 }
4850}
4851
4852static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
4853 bool CondIsTrue,
4854 const Instruction *CxtI,
4855 KnownFPClass &KnownFromContext,
4856 unsigned Depth = 0) {
4857 Value *A, *B;
4858 if (Depth < MaxAnalysisRecursionDepth &&
4859 (CondIsTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B)))
4860 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))))) {
4861 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue, CxtI, KnownFromContext,
4862 Depth: Depth + 1);
4863 computeKnownFPClassFromCond(V, Cond: B, CondIsTrue, CxtI, KnownFromContext,
4864 Depth: Depth + 1);
4865 return;
4866 }
4867 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A)))) {
4868 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue: !CondIsTrue, CxtI, KnownFromContext,
4869 Depth: Depth + 1);
4870 return;
4871 }
4872 CmpPredicate Pred;
4873 Value *LHS;
4874 uint64_t ClassVal = 0;
4875 const APFloat *CRHS;
4876 const APInt *RHS;
4877 if (match(V: Cond, P: m_FCmp(Pred, L: m_Value(V&: LHS), R: m_APFloat(Res&: CRHS)))) {
4878 auto [CmpVal, MaskIfTrue, MaskIfFalse] = fcmpImpliesClass(
4879 Pred, F: *cast<Instruction>(Val: Cond)->getParent()->getParent(), LHS, ConstRHS: *CRHS,
4880 LookThroughSrc: LHS != V);
4881 if (CmpVal == V)
4882 KnownFromContext.knownNot(RuleOut: ~(CondIsTrue ? MaskIfTrue : MaskIfFalse));
4883 } else if (match(V: Cond, P: m_Intrinsic<Intrinsic::is_fpclass>(
4884 Ops: m_Specific(V), Ops: m_ConstantInt(V&: ClassVal)))) {
4885 FPClassTest Mask = static_cast<FPClassTest>(ClassVal);
4886 KnownFromContext.knownNot(RuleOut: CondIsTrue ? ~Mask : Mask);
4887 } else if (match(V: Cond, P: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Specific(V)),
4888 R: m_APInt(Res&: RHS)))) {
4889 bool TrueIfSigned;
4890 if (!isSignBitCheck(Pred, RHS: *RHS, TrueIfSigned))
4891 return;
4892 if (TrueIfSigned == CondIsTrue)
4893 KnownFromContext.signBitMustBeOne();
4894 else
4895 KnownFromContext.signBitMustBeZero();
4896 }
4897}
4898
4899/// Compute the minimum and maximum values (inclusive) for the exponent of \p V,
4900/// assuming it is not nan. Returns {min, max, max-assuming-nonzero}. A value
4901/// frexp(0) = 0, so the tighter max-assuming-nonzero bound is only usable when
4902/// \p V is known not to be a logical zero (e.g., for fabs(x) < 0.25, the non-0
4903/// exponent range is [-149, -2], but the 0 edge case is above this range).
4904static std::tuple<int, int, int>
4905computeKnownExponentRangeFromContext(const Value *V, const SimplifyQuery &Q) {
4906 if (!Q.CxtI || !Q.DC || !Q.DT)
4907 return {APFloat::IEK_NaN, APFloat::IEK_Inf, APFloat::IEK_Inf};
4908
4909 // Intersect the bounds implied by every dominating condition, keeping the
4910 // tightest maximum. A value may participate in multiple compares
4911 // (e.g. fabs(x) < 2.0 and fabs(x) < 1.0), and the tighter one wins.
4912 int MaxExp = APFloat::IEK_Inf;
4913 int MaxExpNonZero = APFloat::IEK_Inf;
4914
4915 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4916 CmpPredicate Pred;
4917 const APFloat *LimitC;
4918 if (!match(V: BI->getCondition(),
4919 P: m_FCmp(Pred, L: m_FAbs(Op0: m_Specific(V)), R: m_Finite(V&: LimitC))))
4920 continue;
4921
4922 if (Pred == FCmpInst::FCMP_ORD || Pred == FCmpInst::FCMP_UNO ||
4923 Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE)
4924 continue;
4925
4926 // If fabs(x) <= K, implies the exponent min exp range.
4927 // if fabs(x) >= K, swap the successor
4928 bool IsLessEqual =
4929 Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE ||
4930 Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE ||
4931 Pred == FCmpInst::FCMP_OEQ || Pred == FCmpInst::FCMP_UEQ;
4932
4933 bool KnownStrictlyLess =
4934 Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_ULT ||
4935 Pred == FCmpInst::FCMP_OGE || Pred == FCmpInst::FCMP_UGE;
4936
4937 BasicBlockEdge Edge1(BI->getParent(),
4938 BI->getSuccessor(i: IsLessEqual ? 0 : 1));
4939 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent())) {
4940 // frexp returns an exponent one greater than ilogb.
4941 int Exp = ilogb(Arg: *LimitC) + 1;
4942
4943 // A strict bound fabs(V) < 2^n forces ilogb(V) <= n - 1, so the max frexp
4944 // exponent drops by one when K is exact power of two.
4945 if (KnownStrictlyLess && LimitC->getExactLog2Abs() != INT_MIN)
4946 --Exp;
4947
4948 // frexp(0) = 0, which the bound above (assuming a normal nonzero value)
4949 // may exclude.
4950
4951 // TODO: Figure out lower bound to detect no-underflow.
4952 MaxExpNonZero = std::min(a: MaxExpNonZero, b: Exp);
4953 MaxExp = std::min(a: MaxExp, b: std::max(a: Exp, b: 0));
4954 }
4955 }
4956
4957 return {APFloat::IEK_NaN, MaxExp, MaxExpNonZero};
4958}
4959
4960static KnownFPClass computeKnownFPClassFromContext(const Value *V,
4961 const SimplifyQuery &Q) {
4962 KnownFPClass KnownFromContext;
4963
4964 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
4965 computeKnownFPClassFromCond(V, Cond: Q.CC->Cond, CondIsTrue: !Q.CC->Invert, CxtI: Q.CxtI,
4966 KnownFromContext);
4967
4968 if (!Q.CxtI)
4969 return KnownFromContext;
4970
4971 if (Q.DC && Q.DT) {
4972 // Handle dominating conditions.
4973 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4974 Value *Cond = BI->getCondition();
4975
4976 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4977 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
4978 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, CxtI: Q.CxtI,
4979 KnownFromContext);
4980
4981 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4982 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
4983 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, CxtI: Q.CxtI,
4984 KnownFromContext);
4985 }
4986 }
4987
4988 if (!Q.AC)
4989 return KnownFromContext;
4990
4991 // Try to restrict the floating-point classes based on information from
4992 // assumptions.
4993 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
4994 if (!AssumeVH)
4995 continue;
4996 CallInst *I = cast<CallInst>(Val&: AssumeVH);
4997
4998 assert(I->getFunction() == Q.CxtI->getParent()->getParent() &&
4999 "Got assumption for the wrong function!");
5000 assert(I->getIntrinsicID() == Intrinsic::assume &&
5001 "must be an assume intrinsic");
5002
5003 if (!isValidAssumeForContext(I, Q))
5004 continue;
5005
5006 computeKnownFPClassFromCond(V, Cond: I->getArgOperand(i: 0),
5007 /*CondIsTrue=*/true, CxtI: Q.CxtI, KnownFromContext);
5008 }
5009
5010 return KnownFromContext;
5011}
5012
5013void llvm::adjustKnownFPClassForSelectArm(KnownFPClass &Known, Value *Cond,
5014 Value *Arm, bool Invert,
5015 const SimplifyQuery &SQ,
5016 unsigned Depth) {
5017
5018 KnownFPClass KnownSrc;
5019 computeKnownFPClassFromCond(V: Arm, Cond,
5020 /*CondIsTrue=*/!Invert, CxtI: SQ.CxtI, KnownFromContext&: KnownSrc,
5021 Depth: Depth + 1);
5022 KnownSrc = KnownSrc.unionWith(RHS: Known);
5023 if (KnownSrc.isUnknown())
5024 return;
5025
5026 if (isGuaranteedNotToBeUndef(V: Arm, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT, Depth: Depth + 1))
5027 Known = KnownSrc;
5028}
5029
5030void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5031 FPClassTest InterestedClasses, KnownFPClass &Known,
5032 const SimplifyQuery &Q, unsigned Depth);
5033
5034static void computeKnownFPClass(const Value *V, KnownFPClass &Known,
5035 FPClassTest InterestedClasses,
5036 const SimplifyQuery &Q, unsigned Depth) {
5037 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5038 APInt DemandedElts =
5039 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
5040 computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Q, Depth);
5041}
5042
5043static void computeKnownFPClassForFPTrunc(const Operator *Op,
5044 const APInt &DemandedElts,
5045 FPClassTest InterestedClasses,
5046 KnownFPClass &Known,
5047 const SimplifyQuery &Q,
5048 unsigned Depth) {
5049 if ((InterestedClasses &
5050 (KnownFPClass::OrderedLessThanZeroMask | fcNan)) == fcNone)
5051 return;
5052
5053 KnownFPClass KnownSrc;
5054 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5055 Known&: KnownSrc, Q, Depth: Depth + 1);
5056 Known = KnownFPClass::fptrunc(KnownSrc);
5057}
5058
5059static constexpr KnownFPClass::MinMaxKind getMinMaxKind(Intrinsic::ID IID) {
5060 switch (IID) {
5061 case Intrinsic::minimum:
5062 return KnownFPClass::MinMaxKind::minimum;
5063 case Intrinsic::maximum:
5064 return KnownFPClass::MinMaxKind::maximum;
5065 case Intrinsic::minimumnum:
5066 return KnownFPClass::MinMaxKind::minimumnum;
5067 case Intrinsic::maximumnum:
5068 return KnownFPClass::MinMaxKind::maximumnum;
5069 case Intrinsic::minnum:
5070 return KnownFPClass::MinMaxKind::minnum;
5071 case Intrinsic::maxnum:
5072 return KnownFPClass::MinMaxKind::maxnum;
5073 default:
5074 llvm_unreachable("not a floating-point min-max intrinsic");
5075 }
5076}
5077
5078/// \return true if this is a floating point value that is known to have a
5079/// magnitude smaller than 1. i.e., fabs(X) <= 1.0 or is nan.
5080static bool isAbsoluteValueULEOne(const Value *V) {
5081 // TODO: Handle frexp
5082 // TODO: Other rounding intrinsics?
5083 // TODO: Try computeKnownExponentRangeFromContext
5084
5085 // fabs(x - floor(x)) <= 1
5086 const Value *SubFloorX;
5087 if (match(V, P: m_FSub(L: m_Value(V&: SubFloorX),
5088 R: m_Intrinsic<Intrinsic::floor>(Ops: m_Deferred(V: SubFloorX)))))
5089 return true;
5090
5091 return match(V, P: m_Intrinsic<Intrinsic::amdgcn_trig_preop>(Ops: m_Value())) ||
5092 match(V, P: m_Intrinsic<Intrinsic::amdgcn_fract>(Ops: m_Value()));
5093}
5094
5095void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5096 FPClassTest InterestedClasses, KnownFPClass &Known,
5097 const SimplifyQuery &Q, unsigned Depth) {
5098 assert(Known.isUnknown() && "should not be called with known information");
5099
5100 if (!DemandedElts) {
5101 // No demanded elts, better to assume we don't know anything.
5102 Known.resetAll();
5103 return;
5104 }
5105
5106 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
5107
5108 if (auto *CFP = dyn_cast<ConstantFP>(Val: V)) {
5109 Known = KnownFPClass(CFP->getValueAPF());
5110 return;
5111 }
5112
5113 if (isa<ConstantAggregateZero>(Val: V)) {
5114 Known.KnownFPClasses = fcPosZero;
5115 Known.SignBit = false;
5116 return;
5117 }
5118
5119 if (isa<PoisonValue>(Val: V)) {
5120 Known.KnownFPClasses = fcNone;
5121 Known.SignBit = false;
5122 return;
5123 }
5124
5125 // Try to handle fixed width vector constants
5126 auto *VFVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5127 const Constant *CV = dyn_cast<Constant>(Val: V);
5128 if (VFVTy && CV) {
5129 Known.KnownFPClasses = fcNone;
5130 bool SignBitAllZero = true;
5131 bool SignBitAllOne = true;
5132
5133 // For vectors, verify that each element is not NaN.
5134 unsigned NumElts = VFVTy->getNumElements();
5135 for (unsigned i = 0; i != NumElts; ++i) {
5136 if (!DemandedElts[i])
5137 continue;
5138
5139 Constant *Elt = CV->getAggregateElement(Elt: i);
5140 if (!Elt) {
5141 Known = KnownFPClass();
5142 return;
5143 }
5144 if (isa<PoisonValue>(Val: Elt))
5145 continue;
5146 auto *CElt = dyn_cast<ConstantFP>(Val: Elt);
5147 if (!CElt) {
5148 Known = KnownFPClass();
5149 return;
5150 }
5151
5152 const APFloat &C = CElt->getValueAPF();
5153 Known.KnownFPClasses |= C.classify();
5154 if (C.isNegative())
5155 SignBitAllZero = false;
5156 else
5157 SignBitAllOne = false;
5158 }
5159 if (SignBitAllOne != SignBitAllZero)
5160 Known.SignBit = SignBitAllOne;
5161 return;
5162 }
5163
5164 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) {
5165 Known.KnownFPClasses = fcNone;
5166 for (size_t I = 0, E = CDS->getNumElements(); I != E; ++I)
5167 Known |= CDS->getElementAsAPFloat(i: I).classify();
5168 return;
5169 }
5170
5171 if (const auto *CA = dyn_cast<ConstantAggregate>(Val: V)) {
5172 // TODO: Handle complex aggregates
5173 Known.KnownFPClasses = fcNone;
5174 for (const Use &Op : CA->operands()) {
5175 auto *CFP = dyn_cast<ConstantFP>(Val: Op.get());
5176 if (!CFP) {
5177 Known = KnownFPClass();
5178 return;
5179 }
5180
5181 Known |= CFP->getValueAPF().classify();
5182 }
5183
5184 return;
5185 }
5186
5187 FPClassTest KnownNotFromFlags = fcNone;
5188 if (const auto *CB = dyn_cast<CallBase>(Val: V))
5189 KnownNotFromFlags |= CB->getRetNoFPClass();
5190 else if (const auto *Arg = dyn_cast<Argument>(Val: V))
5191 KnownNotFromFlags |= Arg->getNoFPClass();
5192
5193 const Operator *Op = dyn_cast<Operator>(Val: V);
5194 if (const FPMathOperator *FPOp = dyn_cast_or_null<FPMathOperator>(Val: Op)) {
5195 if (FPOp->hasNoNaNs())
5196 KnownNotFromFlags |= fcNan;
5197 if (FPOp->hasNoInfs())
5198 KnownNotFromFlags |= fcInf;
5199 }
5200
5201 KnownFPClass AssumedClasses = computeKnownFPClassFromContext(V, Q);
5202 KnownNotFromFlags |= ~AssumedClasses.KnownFPClasses;
5203
5204 // We no longer need to find out about these bits from inputs if we can
5205 // assume this from flags/attributes.
5206 InterestedClasses &= ~KnownNotFromFlags;
5207
5208 llvm::scope_exit ClearClassesFromFlags([=, &Known] {
5209 Known.knownNot(RuleOut: KnownNotFromFlags);
5210 if (!Known.SignBit && AssumedClasses.SignBit) {
5211 if (*AssumedClasses.SignBit)
5212 Known.signBitMustBeOne();
5213 else
5214 Known.signBitMustBeZero();
5215 }
5216 });
5217
5218 if (!Op)
5219 return;
5220
5221 // All recursive calls that increase depth must come after this.
5222 if (Depth == MaxAnalysisRecursionDepth)
5223 return;
5224
5225 const unsigned Opc = Op->getOpcode();
5226 switch (Opc) {
5227 case Instruction::FNeg: {
5228 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5229 Known, Q, Depth: Depth + 1);
5230 Known.fneg();
5231 break;
5232 }
5233 case Instruction::Select: {
5234 auto ComputeForArm = [&](Value *Arm, bool Invert) {
5235 KnownFPClass Res;
5236 computeKnownFPClass(V: Arm, DemandedElts, InterestedClasses, Known&: Res, Q,
5237 Depth: Depth + 1);
5238 adjustKnownFPClassForSelectArm(Known&: Res, Cond: Op->getOperand(i: 0), Arm, Invert, SQ: Q,
5239 Depth);
5240 return Res;
5241 };
5242 // Only known if known in both the LHS and RHS.
5243 Known =
5244 ComputeForArm(Op->getOperand(i: 1), /*Invert=*/false)
5245 .intersectWith(RHS: ComputeForArm(Op->getOperand(i: 2), /*Invert=*/true));
5246 break;
5247 }
5248 case Instruction::Load: {
5249 const MDNode *NoFPClass =
5250 cast<LoadInst>(Val: Op)->getMetadata(KindID: LLVMContext::MD_nofpclass);
5251 if (!NoFPClass)
5252 break;
5253
5254 ConstantInt *MaskVal =
5255 mdconst::extract<ConstantInt>(MD: NoFPClass->getOperand(I: 0));
5256 Known.knownNot(RuleOut: static_cast<FPClassTest>(MaskVal->getZExtValue()));
5257 break;
5258 }
5259 case Instruction::Call: {
5260 const CallInst *II = cast<CallInst>(Val: Op);
5261 const Intrinsic::ID IID = II->getIntrinsicID();
5262 switch (IID) {
5263 case Intrinsic::fabs: {
5264 if ((InterestedClasses & (fcNan | fcPositive)) != fcNone) {
5265 // If we only care about the sign bit we don't need to inspect the
5266 // operand.
5267 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5268 InterestedClasses, Known, Q, Depth: Depth + 1);
5269 }
5270
5271 Known.fabs();
5272 break;
5273 }
5274 case Intrinsic::copysign: {
5275 KnownFPClass KnownSign;
5276
5277 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5278 Known, Q, Depth: Depth + 1);
5279 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5280 Known&: KnownSign, Q, Depth: Depth + 1);
5281 Known.copysign(Sign: KnownSign);
5282 break;
5283 }
5284 case Intrinsic::fma:
5285 case Intrinsic::fmuladd: {
5286 if ((InterestedClasses & fcNegative) == fcNone)
5287 break;
5288
5289 // FIXME: This should check isGuaranteedNotToBeUndef
5290 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) {
5291 KnownFPClass KnownSrc, KnownAddend;
5292 computeKnownFPClass(V: II->getArgOperand(i: 2), DemandedElts,
5293 InterestedClasses, Known&: KnownAddend, Q, Depth: Depth + 1);
5294 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5295 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5296
5297 const Function *F = II->getFunction();
5298 const fltSemantics &FltSem =
5299 II->getType()->getScalarType()->getFltSemantics();
5300 DenormalMode Mode =
5301 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5302
5303 if (KnownNotFromFlags & fcNan) {
5304 KnownSrc.knownNot(RuleOut: fcNan);
5305 KnownAddend.knownNot(RuleOut: fcNan);
5306 }
5307
5308 if (KnownNotFromFlags & fcInf) {
5309 KnownSrc.knownNot(RuleOut: fcInf);
5310 KnownAddend.knownNot(RuleOut: fcInf);
5311 }
5312
5313 Known = KnownFPClass::fma_square(Squared: KnownSrc, Addend: KnownAddend, Mode);
5314 break;
5315 }
5316
5317 KnownFPClass KnownSrc[3];
5318 for (int I = 0; I != 3; ++I) {
5319 computeKnownFPClass(V: II->getArgOperand(i: I), DemandedElts,
5320 InterestedClasses, Known&: KnownSrc[I], Q, Depth: Depth + 1);
5321 if (KnownSrc[I].isUnknown())
5322 return;
5323
5324 if (KnownNotFromFlags & fcNan)
5325 KnownSrc[I].knownNot(RuleOut: fcNan);
5326 if (KnownNotFromFlags & fcInf)
5327 KnownSrc[I].knownNot(RuleOut: fcInf);
5328 }
5329
5330 const Function *F = II->getFunction();
5331 const fltSemantics &FltSem =
5332 II->getType()->getScalarType()->getFltSemantics();
5333 DenormalMode Mode =
5334 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5335 Known = KnownFPClass::fma(LHS: KnownSrc[0], RHS: KnownSrc[1], Addend: KnownSrc[2], Mode);
5336 break;
5337 }
5338 case Intrinsic::sqrt:
5339 case Intrinsic::experimental_constrained_sqrt: {
5340 KnownFPClass KnownSrc;
5341 FPClassTest InterestedSrcs = InterestedClasses;
5342 if (InterestedClasses & fcNan)
5343 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5344
5345 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5346 Known&: KnownSrc, Q, Depth: Depth + 1);
5347
5348 DenormalMode Mode = DenormalMode::getDynamic();
5349
5350 bool HasNSZ = Q.IIQ.hasNoSignedZeros(Op: II);
5351 if (!HasNSZ) {
5352 const Function *F = II->getFunction();
5353 const fltSemantics &FltSem =
5354 II->getType()->getScalarType()->getFltSemantics();
5355 Mode = F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5356 }
5357
5358 Known = KnownFPClass::sqrt(Src: KnownSrc, Mode);
5359 if (HasNSZ)
5360 Known.knownNot(RuleOut: fcNegZero);
5361
5362 break;
5363 }
5364 case Intrinsic::sin: {
5365 KnownFPClass KnownSrc;
5366 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5367 Known&: KnownSrc, Q, Depth: Depth + 1);
5368 Known = KnownFPClass::sin(Src: KnownSrc);
5369 break;
5370 }
5371 case Intrinsic::cos: {
5372 KnownFPClass KnownSrc;
5373 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5374 Known&: KnownSrc, Q, Depth: Depth + 1);
5375 Known = KnownFPClass::cos(Src: KnownSrc);
5376 break;
5377 }
5378 case Intrinsic::tan: {
5379 KnownFPClass KnownSrc;
5380 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5381 Known&: KnownSrc, Q, Depth: Depth + 1);
5382 Known = KnownFPClass::tan(Src: KnownSrc);
5383 break;
5384 }
5385 case Intrinsic::sinh: {
5386 KnownFPClass KnownSrc;
5387 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5388 Known&: KnownSrc, Q, Depth: Depth + 1);
5389 Known = KnownFPClass::sinh(Src: KnownSrc);
5390 break;
5391 }
5392 case Intrinsic::cosh: {
5393 KnownFPClass KnownSrc;
5394 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5395 Known&: KnownSrc, Q, Depth: Depth + 1);
5396 Known = KnownFPClass::cosh(Src: KnownSrc);
5397 break;
5398 }
5399 case Intrinsic::tanh: {
5400 KnownFPClass KnownSrc;
5401 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5402 Known&: KnownSrc, Q, Depth: Depth + 1);
5403 Known = KnownFPClass::tanh(Src: KnownSrc);
5404 break;
5405 }
5406 case Intrinsic::asin: {
5407 KnownFPClass KnownSrc;
5408 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5409 Known&: KnownSrc, Q, Depth: Depth + 1);
5410 Known = KnownFPClass::asin(Src: KnownSrc);
5411 break;
5412 }
5413 case Intrinsic::acos: {
5414 KnownFPClass KnownSrc;
5415 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5416 Known&: KnownSrc, Q, Depth: Depth + 1);
5417 Known = KnownFPClass::acos(Src: KnownSrc);
5418 break;
5419 }
5420 case Intrinsic::atan: {
5421 KnownFPClass KnownSrc;
5422 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5423 Known&: KnownSrc, Q, Depth: Depth + 1);
5424 Known = KnownFPClass::atan(Src: KnownSrc);
5425 break;
5426 }
5427 case Intrinsic::atan2: {
5428 KnownFPClass KnownLHS, KnownRHS;
5429 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5430 Known&: KnownLHS, Q, Depth: Depth + 1);
5431 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5432 Known&: KnownRHS, Q, Depth: Depth + 1);
5433 Known = KnownFPClass::atan2(LHS: KnownLHS, RHS: KnownRHS);
5434 break;
5435 }
5436 case Intrinsic::maxnum:
5437 case Intrinsic::minnum:
5438 case Intrinsic::minimum:
5439 case Intrinsic::maximum:
5440 case Intrinsic::minimumnum:
5441 case Intrinsic::maximumnum: {
5442 KnownFPClass KnownLHS, KnownRHS;
5443 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5444 Known&: KnownLHS, Q, Depth: Depth + 1);
5445 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5446 Known&: KnownRHS, Q, Depth: Depth + 1);
5447
5448 const Function *F = II->getFunction();
5449
5450 DenormalMode Mode =
5451 F ? F->getDenormalMode(
5452 FPType: II->getType()->getScalarType()->getFltSemantics())
5453 : DenormalMode::getDynamic();
5454
5455 Known = KnownFPClass::minMaxLike(LHS: KnownLHS, RHS: KnownRHS, Kind: getMinMaxKind(IID),
5456 DenormMode: Mode);
5457 break;
5458 }
5459 case Intrinsic::canonicalize: {
5460 KnownFPClass KnownSrc;
5461 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5462 Known&: KnownSrc, Q, Depth: Depth + 1);
5463
5464 const Function *F = II->getFunction();
5465 DenormalMode DenormMode =
5466 F ? F->getDenormalMode(
5467 FPType: II->getType()->getScalarType()->getFltSemantics())
5468 : DenormalMode::getDynamic();
5469 Known = KnownFPClass::canonicalize(Src: KnownSrc, DenormMode);
5470 break;
5471 }
5472 case Intrinsic::vector_reduce_fmax:
5473 case Intrinsic::vector_reduce_fmin:
5474 case Intrinsic::vector_reduce_fmaximum:
5475 case Intrinsic::vector_reduce_fminimum: {
5476 // reduce min/max will choose an element from one of the vector elements,
5477 // so we can infer and class information that is common to all elements.
5478 Known = computeKnownFPClass(V: II->getArgOperand(i: 0), FMF: II->getFastMathFlags(),
5479 InterestedClasses, SQ: Q, Depth: Depth + 1);
5480 // Can only propagate sign if output is never NaN.
5481 if (!Known.isKnownNeverNaN())
5482 Known.SignBit.reset();
5483 break;
5484 }
5485 // reverse preserves all characteristics of the input vec's element.
5486 case Intrinsic::vector_reverse:
5487 Known = computeKnownFPClass(
5488 V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
5489 FMF: II->getFastMathFlags(), InterestedClasses, SQ: Q, Depth: Depth + 1);
5490 break;
5491 case Intrinsic::trunc:
5492 case Intrinsic::floor:
5493 case Intrinsic::ceil:
5494 case Intrinsic::rint:
5495 case Intrinsic::nearbyint:
5496 case Intrinsic::round:
5497 case Intrinsic::roundeven: {
5498 KnownFPClass KnownSrc;
5499 FPClassTest InterestedSrcs = InterestedClasses;
5500 if (InterestedSrcs & fcPosFinite)
5501 InterestedSrcs |= fcPosFinite;
5502 if (InterestedSrcs & fcNegFinite)
5503 InterestedSrcs |= fcNegFinite;
5504 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5505 Known&: KnownSrc, Q, Depth: Depth + 1);
5506
5507 Known = KnownFPClass::roundToIntegral(
5508 Src: KnownSrc, IsTrunc: IID == Intrinsic::trunc,
5509 IsMultiUnitFPType: V->getType()->getScalarType()->isMultiUnitFPType());
5510 break;
5511 }
5512 case Intrinsic::exp:
5513 case Intrinsic::exp2:
5514 case Intrinsic::exp10:
5515 case Intrinsic::amdgcn_exp2: {
5516 KnownFPClass KnownSrc;
5517 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5518 Known&: KnownSrc, Q, Depth: Depth + 1);
5519
5520 Known = KnownFPClass::exp(Src: KnownSrc);
5521
5522 Type *EltTy = II->getType()->getScalarType();
5523 if (IID == Intrinsic::amdgcn_exp2 && EltTy->isFloatTy())
5524 Known.knownNot(RuleOut: fcSubnormal);
5525
5526 break;
5527 }
5528 case Intrinsic::fptrunc_round: {
5529 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
5530 Q, Depth);
5531 break;
5532 }
5533 case Intrinsic::log:
5534 case Intrinsic::log10:
5535 case Intrinsic::log2:
5536 case Intrinsic::experimental_constrained_log:
5537 case Intrinsic::experimental_constrained_log10:
5538 case Intrinsic::experimental_constrained_log2:
5539 case Intrinsic::amdgcn_log: {
5540 Type *EltTy = II->getType()->getScalarType();
5541
5542 // log(+inf) -> +inf
5543 // log([+-]0.0) -> -inf
5544 // log(-inf) -> nan
5545 // log(-x) -> nan
5546 if ((InterestedClasses & (fcNan | fcInf)) != fcNone) {
5547 FPClassTest InterestedSrcs = InterestedClasses;
5548 if ((InterestedClasses & fcNegInf) != fcNone)
5549 InterestedSrcs |= fcZero | fcSubnormal;
5550 if ((InterestedClasses & fcNan) != fcNone)
5551 InterestedSrcs |= fcNan | fcNegative;
5552
5553 KnownFPClass KnownSrc;
5554 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5555 Known&: KnownSrc, Q, Depth: Depth + 1);
5556
5557 const Function *F = II->getFunction();
5558 DenormalMode Mode = F ? F->getDenormalMode(FPType: EltTy->getFltSemantics())
5559 : DenormalMode::getDynamic();
5560 Known = KnownFPClass::log(Src: KnownSrc, Mode);
5561 }
5562
5563 break;
5564 }
5565 case Intrinsic::powi: {
5566 if ((InterestedClasses & (fcNan | fcInf | fcNegative)) == fcNone)
5567 break;
5568
5569 const Value *Exp = II->getArgOperand(i: 1);
5570 Type *ExpTy = Exp->getType();
5571 unsigned BitWidth = ExpTy->getScalarType()->getIntegerBitWidth();
5572 KnownBits ExponentKnownBits(BitWidth);
5573 computeKnownBits(V: Exp, DemandedElts: isa<VectorType>(Val: ExpTy) ? DemandedElts : APInt(1, 1),
5574 Known&: ExponentKnownBits, Q, Depth: Depth + 1);
5575
5576 FPClassTest InterestedSrcs = fcNone;
5577 if (InterestedClasses & fcNan)
5578 InterestedSrcs |= fcNan;
5579 if (!ExponentKnownBits.isZero()) {
5580 if (InterestedClasses & fcInf)
5581 InterestedSrcs |= fcFinite | fcInf;
5582 if ((InterestedClasses & fcNegative) && !ExponentKnownBits.isEven())
5583 InterestedSrcs |= fcNegative;
5584 }
5585
5586 KnownFPClass KnownSrc;
5587 if (InterestedSrcs != fcNone)
5588 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5589 Known&: KnownSrc, Q, Depth: Depth + 1);
5590
5591 Known = KnownFPClass::powi(Src: KnownSrc, N: ExponentKnownBits);
5592 break;
5593 }
5594 case Intrinsic::ldexp: {
5595 KnownFPClass KnownSrc;
5596 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5597 Known&: KnownSrc, Q, Depth: Depth + 1);
5598 // Can refine inf/zero handling based on the exponent operand.
5599 const FPClassTest ExpInfoMask = fcZero | fcSubnormal | fcInf;
5600
5601 const Value *ExpArg = II->getArgOperand(i: 1);
5602 ConstantRange ExpKnownRange =
5603 ((KnownSrc.KnownFPClasses & ExpInfoMask) != fcNone)
5604 ? computeConstantRange(V: ExpArg, /*ForSigned=*/true, SQ: Q, Depth: Depth + 1)
5605 : ConstantRange::getFull(
5606 BitWidth: ExpArg->getType()->getScalarSizeInBits());
5607
5608 const fltSemantics &Flt =
5609 II->getType()->getScalarType()->getFltSemantics();
5610
5611 const Function *F = II->getFunction();
5612 DenormalMode Mode =
5613 F ? F->getDenormalMode(FPType: Flt) : DenormalMode::getDynamic();
5614
5615 Known = KnownFPClass::ldexp(Src: KnownSrc, ConstantRangeMin: ExpKnownRange.getSignedMin(),
5616 ConstantRangeMax: ExpKnownRange.getSignedMax(), Flt, Mode);
5617 break;
5618 }
5619 case Intrinsic::arithmetic_fence: {
5620 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5621 Known, Q, Depth: Depth + 1);
5622 break;
5623 }
5624 case Intrinsic::experimental_constrained_sitofp:
5625 case Intrinsic::experimental_constrained_uitofp:
5626 // Cannot produce nan
5627 Known.knownNot(RuleOut: fcNan);
5628
5629 // sitofp and uitofp turn into +0.0 for zero.
5630 Known.knownNot(RuleOut: fcNegZero);
5631
5632 // Integers cannot be subnormal
5633 Known.knownNot(RuleOut: fcSubnormal);
5634
5635 if (IID == Intrinsic::experimental_constrained_uitofp)
5636 Known.signBitMustBeZero();
5637
5638 // TODO: Copy inf handling from instructions
5639 break;
5640
5641 case Intrinsic::amdgcn_fract: {
5642 Known.knownNot(RuleOut: fcInf);
5643
5644 if (InterestedClasses & fcNan) {
5645 KnownFPClass KnownSrc;
5646 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5647 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5648
5649 if (KnownSrc.isKnownNeverInfOrNaN())
5650 Known.knownNot(RuleOut: fcNan);
5651 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5652 Known.knownNot(RuleOut: fcSNan);
5653 }
5654
5655 break;
5656 }
5657 case Intrinsic::amdgcn_rcp: {
5658 KnownFPClass KnownSrc;
5659 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5660 Known&: KnownSrc, Q, Depth: Depth + 1);
5661
5662 Known.propagateNaN(Src: KnownSrc);
5663
5664 Type *EltTy = II->getType()->getScalarType();
5665
5666 // f32 denormal always flushed.
5667 if (EltTy->isFloatTy()) {
5668 Known.knownNot(RuleOut: fcSubnormal);
5669 KnownSrc.knownNot(RuleOut: fcSubnormal);
5670 }
5671
5672 if (KnownSrc.isKnownNever(Mask: fcNegative))
5673 Known.knownNot(RuleOut: fcNegative);
5674 if (KnownSrc.isKnownNever(Mask: fcPositive))
5675 Known.knownNot(RuleOut: fcPositive);
5676
5677 if (const Function *F = II->getFunction()) {
5678 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5679 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5680 Known.knownNot(RuleOut: fcPosInf);
5681 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5682 Known.knownNot(RuleOut: fcNegInf);
5683 }
5684
5685 break;
5686 }
5687 case Intrinsic::amdgcn_rsq: {
5688 KnownFPClass KnownSrc;
5689 // The only negative value that can be returned is -inf for -0 inputs.
5690 Known.knownNot(RuleOut: fcNegZero | fcNegSubnormal | fcNegNormal);
5691
5692 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5693 Known&: KnownSrc, Q, Depth: Depth + 1);
5694
5695 // Negative -> nan
5696 if (KnownSrc.isKnownNeverNaN() && KnownSrc.cannotBeOrderedLessThanZero())
5697 Known.knownNot(RuleOut: fcNan);
5698 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5699 Known.knownNot(RuleOut: fcSNan);
5700
5701 // +inf -> +0
5702 if (KnownSrc.isKnownNeverPosInfinity())
5703 Known.knownNot(RuleOut: fcPosZero);
5704
5705 Type *EltTy = II->getType()->getScalarType();
5706
5707 // f32 denormal always flushed.
5708 if (EltTy->isFloatTy())
5709 Known.knownNot(RuleOut: fcPosSubnormal);
5710
5711 if (const Function *F = II->getFunction()) {
5712 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5713
5714 // -0 -> -inf
5715 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5716 Known.knownNot(RuleOut: fcNegInf);
5717
5718 // +0 -> +inf
5719 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5720 Known.knownNot(RuleOut: fcPosInf);
5721 }
5722
5723 break;
5724 }
5725 case Intrinsic::amdgcn_trig_preop: {
5726 // Always returns a value [0, 1)
5727 Known.knownNot(RuleOut: fcNan | fcInf | fcNegative);
5728 break;
5729 }
5730 default:
5731 break;
5732 }
5733
5734 break;
5735 }
5736 case Instruction::FAdd:
5737 case Instruction::FSub: {
5738 KnownFPClass KnownLHS, KnownRHS;
5739 bool WantNegative =
5740 Op->getOpcode() == Instruction::FAdd &&
5741 (InterestedClasses & KnownFPClass::OrderedLessThanZeroMask) != fcNone;
5742 bool WantNaN = (InterestedClasses & fcNan) != fcNone;
5743 bool WantNegZero = (InterestedClasses & fcNegZero) != fcNone;
5744
5745 if (!WantNaN && !WantNegative && !WantNegZero)
5746 break;
5747
5748 FPClassTest InterestedSrcs = InterestedClasses;
5749 if (WantNegative)
5750 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5751 if (InterestedClasses & fcNan)
5752 InterestedSrcs |= fcInf;
5753 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: InterestedSrcs,
5754 Known&: KnownRHS, Q, Depth: Depth + 1);
5755
5756 // Special case fadd x, x, which is the canonical form of fmul x, 2.
5757 bool Self = Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
5758 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
5759 Depth: Depth + 1);
5760 if (Self)
5761 KnownLHS = KnownRHS;
5762
5763 if ((WantNaN && KnownRHS.isKnownNeverNaN()) ||
5764 (WantNegative && KnownRHS.cannotBeOrderedLessThanZero()) ||
5765 WantNegZero || Opc == Instruction::FSub) {
5766
5767 // FIXME: Context function should always be passed in separately
5768 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5769 const fltSemantics &FltSem =
5770 Op->getType()->getScalarType()->getFltSemantics();
5771 DenormalMode Mode =
5772 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5773
5774 if (Self && Opc == Instruction::FAdd) {
5775 Known = KnownFPClass::fadd_self(Src: KnownLHS, Mode);
5776 } else {
5777 // RHS is canonically cheaper to compute. Skip inspecting the LHS if
5778 // there's no point.
5779
5780 if (!Self) {
5781 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5782 Known&: KnownLHS, Q, Depth: Depth + 1);
5783 }
5784
5785 Known = Opc == Instruction::FAdd
5786 ? KnownFPClass::fadd(LHS: KnownLHS, RHS: KnownRHS, Mode)
5787 : KnownFPClass::fsub(LHS: KnownLHS, RHS: KnownRHS, Mode);
5788 }
5789 }
5790
5791 break;
5792 }
5793 case Instruction::FMul: {
5794 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5795 DenormalMode Mode =
5796 F ? F->getDenormalMode(
5797 FPType: Op->getType()->getScalarType()->getFltSemantics())
5798 : DenormalMode::getDynamic();
5799
5800 Value *LHS = Op->getOperand(i: 0);
5801 Value *RHS = Op->getOperand(i: 1);
5802 // X * X is always non-negative or a NaN.
5803 // FIXME: Should check isGuaranteedNotToBeUndef
5804 if (LHS == RHS) {
5805 KnownFPClass KnownSrc;
5806 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownSrc, Q,
5807 Depth: Depth + 1);
5808 Known = KnownFPClass::square(Src: KnownSrc, Mode);
5809 break;
5810 }
5811
5812 KnownFPClass KnownLHS, KnownRHS;
5813
5814 const APFloat *CRHS;
5815 if (match(V: RHS, P: m_APFloat(Res&: CRHS))) {
5816 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
5817 Depth: Depth + 1);
5818 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: *CRHS, Mode);
5819 } else {
5820 computeKnownFPClass(V: RHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS, Q,
5821 Depth: Depth + 1);
5822 // TODO: Improve accuracy in unfused FMA pattern. We can prove an
5823 // additional not-nan if the addend is known-not negative infinity if the
5824 // multiply is known-not infinity.
5825
5826 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
5827 Depth: Depth + 1);
5828 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: KnownRHS, Mode);
5829 }
5830
5831 /// Propgate no-infs if the other source is known smaller than one, such
5832 /// that this cannot introduce overflow.
5833 if (KnownLHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: RHS))
5834 Known.knownNot(RuleOut: fcInf);
5835 else if (KnownRHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: LHS))
5836 Known.knownNot(RuleOut: fcInf);
5837
5838 break;
5839 }
5840 case Instruction::FDiv:
5841 case Instruction::FRem: {
5842 const bool WantNan = (InterestedClasses & fcNan) != fcNone;
5843
5844 if (Op->getOpcode() == Instruction::FRem)
5845 Known.knownNot(RuleOut: fcInf);
5846
5847 if (Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
5848 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT)) {
5849 if (Op->getOpcode() == Instruction::FDiv) {
5850 // X / X is always exactly 1.0 or a NaN.
5851 Known.KnownFPClasses = fcNan | fcPosNormal;
5852 } else {
5853 // X % X is always exactly [+-]0.0 or a NaN.
5854 Known.KnownFPClasses = fcNan | fcZero;
5855 }
5856
5857 if (!WantNan)
5858 break;
5859
5860 KnownFPClass KnownSrc;
5861 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts,
5862 InterestedClasses: fcNan | fcInf | fcZero | fcSubnormal, Known&: KnownSrc, Q,
5863 Depth: Depth + 1);
5864 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5865 const fltSemantics &FltSem =
5866 Op->getType()->getScalarType()->getFltSemantics();
5867
5868 DenormalMode Mode =
5869 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5870
5871 Known = Op->getOpcode() == Instruction::FDiv
5872 ? KnownFPClass::fdiv_self(Src: KnownSrc, Mode)
5873 : KnownFPClass::frem_self(Src: KnownSrc, Mode);
5874 break;
5875 }
5876
5877 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
5878 const bool WantPositive =
5879 Opc == Instruction::FRem && (InterestedClasses & fcPositive) != fcNone;
5880 if (!WantNan && !WantNegative && !WantPositive)
5881 break;
5882
5883 KnownFPClass KnownLHS, KnownRHS;
5884
5885 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts,
5886 InterestedClasses: fcNan | fcInf | fcZero | fcNegative, Known&: KnownRHS, Q,
5887 Depth: Depth + 1);
5888
5889 bool KnowSomethingUseful = KnownRHS.isKnownNeverNaN() ||
5890 KnownRHS.isKnownNever(Mask: fcNegative) ||
5891 KnownRHS.isKnownNever(Mask: fcPositive);
5892
5893 if (KnowSomethingUseful || WantPositive) {
5894 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS,
5895 Q, Depth: Depth + 1);
5896 }
5897
5898 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5899 const fltSemantics &FltSem =
5900 Op->getType()->getScalarType()->getFltSemantics();
5901
5902 if (Op->getOpcode() == Instruction::FDiv) {
5903 DenormalMode Mode =
5904 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5905 Known = KnownFPClass::fdiv(LHS: KnownLHS, RHS: KnownRHS, Mode);
5906 } else {
5907 // Inf REM x and x REM 0 produce NaN.
5908 if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
5909 KnownLHS.isKnownNeverInfinity() && F &&
5910 KnownRHS.isKnownNeverLogicalZero(Mode: F->getDenormalMode(FPType: FltSem))) {
5911 Known.knownNot(RuleOut: fcNan);
5912 }
5913
5914 // The sign for frem is the same as the first operand.
5915 if (KnownLHS.cannotBeOrderedLessThanZero())
5916 Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
5917 if (KnownLHS.cannotBeOrderedGreaterThanZero())
5918 Known.knownNot(RuleOut: KnownFPClass::OrderedGreaterThanZeroMask);
5919
5920 // See if we can be more aggressive about the sign of 0.
5921 if (KnownLHS.isKnownNever(Mask: fcNegative))
5922 Known.knownNot(RuleOut: fcNegative);
5923 if (KnownLHS.isKnownNever(Mask: fcPositive))
5924 Known.knownNot(RuleOut: fcPositive);
5925 }
5926
5927 break;
5928 }
5929 case Instruction::FPExt: {
5930 KnownFPClass KnownSrc;
5931 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5932 Known&: KnownSrc, Q, Depth: Depth + 1);
5933
5934 const fltSemantics &DstTy =
5935 Op->getType()->getScalarType()->getFltSemantics();
5936 const fltSemantics &SrcTy =
5937 Op->getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
5938
5939 Known = KnownFPClass::fpext(KnownSrc, DstTy, SrcTy);
5940 break;
5941 }
5942 case Instruction::FPTrunc: {
5943 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known, Q,
5944 Depth);
5945 break;
5946 }
5947 case Instruction::SIToFP:
5948 case Instruction::UIToFP: {
5949 // Cannot produce nan
5950 Known.knownNot(RuleOut: fcNan);
5951
5952 // Integers cannot be subnormal
5953 Known.knownNot(RuleOut: fcSubnormal);
5954
5955 // sitofp and uitofp turn into +0.0 for zero.
5956 Known.knownNot(RuleOut: fcNegZero);
5957
5958 // UIToFP is always non-negative regardless of known bits.
5959 if (Op->getOpcode() == Instruction::UIToFP)
5960 Known.signBitMustBeZero();
5961
5962 // Only compute known bits if we can learn something useful from them.
5963 if (!(InterestedClasses & (fcPosZero | fcNormal | fcInf)))
5964 break;
5965
5966 KnownBits IntKnown =
5967 computeKnownBits(V: Op->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
5968
5969 // If the integer is non-zero, the result cannot be +0.0
5970 if (IntKnown.isNonZero())
5971 Known.knownNot(RuleOut: fcPosZero);
5972
5973 if (Op->getOpcode() == Instruction::SIToFP) {
5974 // If the signed integer is known non-negative, the result is
5975 // non-negative. If the signed integer is known negative, the result is
5976 // negative.
5977 if (IntKnown.isNonNegative()) {
5978 Known.signBitMustBeZero();
5979 } else if (IntKnown.isNegative()) {
5980 Known.signBitMustBeOne();
5981 }
5982 }
5983
5984 // Guard kept for ilogb()
5985 if (InterestedClasses & fcInf) {
5986 // Get width of largest magnitude integer known.
5987 // This still works for a signed minimum value because the largest FP
5988 // value is scaled by some fraction close to 2.0 (1.0 + 0.xxxx).
5989 int IntSize = IntKnown.getBitWidth();
5990 if (Op->getOpcode() == Instruction::UIToFP)
5991 IntSize -= IntKnown.countMinLeadingZeros();
5992 else if (Op->getOpcode() == Instruction::SIToFP)
5993 IntSize -= IntKnown.countMinSignBits();
5994
5995 // If the exponent of the largest finite FP value can hold the largest
5996 // integer, the result of the cast must be finite.
5997 Type *FPTy = Op->getType()->getScalarType();
5998 if (ilogb(Arg: APFloat::getLargest(Sem: FPTy->getFltSemantics())) >= IntSize)
5999 Known.knownNot(RuleOut: fcInf);
6000 }
6001
6002 break;
6003 }
6004 case Instruction::ExtractElement: {
6005 // Look through extract element. If the index is non-constant or
6006 // out-of-range demand all elements, otherwise just the extracted element.
6007 const Value *Vec = Op->getOperand(i: 0);
6008
6009 APInt DemandedVecElts;
6010 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
6011 unsigned NumElts = VecTy->getNumElements();
6012 DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
6013 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1));
6014 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
6015 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
6016 } else {
6017 DemandedVecElts = APInt(1, 1);
6018 }
6019
6020 return computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known,
6021 Q, Depth: Depth + 1);
6022 }
6023 case Instruction::InsertElement: {
6024 if (isa<ScalableVectorType>(Val: Op->getType()))
6025 return;
6026
6027 const Value *Vec = Op->getOperand(i: 0);
6028 const Value *Elt = Op->getOperand(i: 1);
6029 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 2));
6030 unsigned NumElts = DemandedElts.getBitWidth();
6031 APInt DemandedVecElts = DemandedElts;
6032 bool NeedsElt = true;
6033 // If we know the index we are inserting to, clear it from Vec check.
6034 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
6035 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
6036 NeedsElt = DemandedElts[CIdx->getZExtValue()];
6037 }
6038
6039 // Do we demand the inserted element?
6040 if (NeedsElt) {
6041 computeKnownFPClass(V: Elt, Known, InterestedClasses, Q, Depth: Depth + 1);
6042 // If we don't know any bits, early out.
6043 if (Known.isUnknown())
6044 break;
6045 } else {
6046 Known.KnownFPClasses = fcNone;
6047 }
6048
6049 // Do we need anymore elements from Vec?
6050 if (!DemandedVecElts.isZero()) {
6051 KnownFPClass Known2;
6052 computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known&: Known2, Q,
6053 Depth: Depth + 1);
6054 Known |= Known2;
6055 }
6056
6057 break;
6058 }
6059 case Instruction::ShuffleVector: {
6060 // Handle vector splat idiom
6061 if (Value *Splat = getSplatValue(V)) {
6062 computeKnownFPClass(V: Splat, Known, InterestedClasses, Q, Depth: Depth + 1);
6063 break;
6064 }
6065
6066 // For undef elements, we don't know anything about the common state of
6067 // the shuffle result.
6068 APInt DemandedLHS, DemandedRHS;
6069 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: Op);
6070 if (!Shuf || !getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
6071 return;
6072
6073 if (!!DemandedLHS) {
6074 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
6075 computeKnownFPClass(V: LHS, DemandedElts: DemandedLHS, InterestedClasses, Known, Q,
6076 Depth: Depth + 1);
6077
6078 // If we don't know any bits, early out.
6079 if (Known.isUnknown())
6080 break;
6081 } else {
6082 Known.KnownFPClasses = fcNone;
6083 }
6084
6085 if (!!DemandedRHS) {
6086 KnownFPClass Known2;
6087 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
6088 computeKnownFPClass(V: RHS, DemandedElts: DemandedRHS, InterestedClasses, Known&: Known2, Q,
6089 Depth: Depth + 1);
6090 Known |= Known2;
6091 }
6092
6093 break;
6094 }
6095 case Instruction::ExtractValue: {
6096 const ExtractValueInst *Extract = cast<ExtractValueInst>(Val: Op);
6097 ArrayRef<unsigned> Indices = Extract->getIndices();
6098 const Value *Src = Extract->getAggregateOperand();
6099 if (isa<StructType>(Val: Src->getType()) && Indices.size() == 1 &&
6100 Indices[0] == 0) {
6101 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Src)) {
6102 switch (II->getIntrinsicID()) {
6103 case Intrinsic::frexp: {
6104 Known.knownNot(RuleOut: fcSubnormal);
6105
6106 KnownFPClass KnownSrc;
6107 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
6108 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
6109
6110 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6111 const fltSemantics &FltSem =
6112 Op->getType()->getScalarType()->getFltSemantics();
6113
6114 DenormalMode Mode =
6115 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6116 Known = KnownFPClass::frexp_mant(Src: KnownSrc, Mode);
6117 return;
6118 }
6119 default:
6120 break;
6121 }
6122 }
6123 }
6124
6125 computeKnownFPClass(V: Src, DemandedElts, InterestedClasses, Known, Q,
6126 Depth: Depth + 1);
6127 break;
6128 }
6129 case Instruction::PHI: {
6130 const PHINode *P = cast<PHINode>(Val: Op);
6131 // Unreachable blocks may have zero-operand PHI nodes.
6132 if (P->getNumIncomingValues() == 0)
6133 break;
6134
6135 // Otherwise take the unions of the known bit sets of the operands,
6136 // taking conservative care to avoid excessive recursion.
6137 const unsigned PhiRecursionLimit = MaxAnalysisRecursionDepth - 2;
6138
6139 if (Depth < PhiRecursionLimit) {
6140 // Skip if every incoming value references to ourself.
6141 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
6142 break;
6143
6144 bool First = true;
6145
6146 for (const Use &U : P->operands()) {
6147 Value *IncValue;
6148 Instruction *CxtI;
6149 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CxtI);
6150 // Skip direct self references.
6151 if (IncValue == P)
6152 continue;
6153
6154 KnownFPClass KnownSrc;
6155 // Recurse, but cap the recursion to two levels, because we don't want
6156 // to waste time spinning around in loops. We need at least depth 2 to
6157 // detect known sign bits.
6158 computeKnownFPClass(V: IncValue, DemandedElts, InterestedClasses, Known&: KnownSrc,
6159 Q: Q.getWithoutCondContext().getWithInstruction(I: CxtI),
6160 Depth: PhiRecursionLimit);
6161
6162 if (First) {
6163 Known = KnownSrc;
6164 First = false;
6165 } else {
6166 Known |= KnownSrc;
6167 }
6168
6169 if (Known.KnownFPClasses == fcAllFlags)
6170 break;
6171 }
6172 }
6173
6174 // Look for the case of a for loop which has a positive
6175 // initial value and is incremented by a squared value.
6176 // This will propagate sign information out of such loops.
6177 if (P->getNumIncomingValues() != 2 || Known.cannotBeOrderedLessThanZero())
6178 break;
6179 for (unsigned I = 0; I < 2; I++) {
6180 Value *RecurValue = P->getIncomingValue(i: 1 - I);
6181 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: RecurValue);
6182 if (!II)
6183 continue;
6184 Value *R, *L, *Init;
6185 PHINode *PN;
6186 if (matchSimpleTernaryIntrinsicRecurrence(I: II, P&: PN, Init, OtherOp0&: L, OtherOp1&: R) &&
6187 PN == P) {
6188 switch (II->getIntrinsicID()) {
6189 case Intrinsic::fma:
6190 case Intrinsic::fmuladd: {
6191 KnownFPClass KnownStart;
6192 computeKnownFPClass(V: Init, DemandedElts, InterestedClasses, Known&: KnownStart,
6193 Q, Depth: Depth + 1);
6194 if (KnownStart.cannotBeOrderedLessThanZero() && L == R &&
6195 isGuaranteedNotToBeUndef(V: L, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1))
6196 Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
6197 break;
6198 }
6199 }
6200 }
6201 }
6202 break;
6203 }
6204 case Instruction::BitCast: {
6205 const Value *Src;
6206 if (!match(V: Op, P: m_ElementWiseBitCast(Op: m_Value(V&: Src))) ||
6207 !Src->getType()->isIntOrIntVectorTy())
6208 break;
6209
6210 const Type *Ty = Op->getType();
6211
6212 Value *CastLHS, *CastRHS;
6213
6214 // Match bitcast(umax(bitcast(a), bitcast(b)))
6215 if (match(V: Src, P: m_c_MaxOrMin(L: m_BitCast(Op: m_Value(V&: CastLHS)),
6216 R: m_BitCast(Op: m_Value(V&: CastRHS)))) &&
6217 CastLHS->getType() == Ty && CastRHS->getType() == Ty) {
6218 KnownFPClass KnownLHS, KnownRHS;
6219 computeKnownFPClass(V: CastRHS, DemandedElts, InterestedClasses, Known&: KnownRHS, Q,
6220 Depth: Depth + 1);
6221 if (!KnownRHS.isUnknown()) {
6222 computeKnownFPClass(V: CastLHS, DemandedElts, InterestedClasses, Known&: KnownLHS,
6223 Q, Depth: Depth + 1);
6224 Known = KnownLHS | KnownRHS;
6225 }
6226
6227 return;
6228 }
6229
6230 const Type *EltTy = Ty->getScalarType();
6231 KnownBits Bits(EltTy->getPrimitiveSizeInBits());
6232 computeKnownBits(V: Src, DemandedElts, Known&: Bits, Q, Depth: Depth + 1);
6233
6234 Known = KnownFPClass::bitcast(FltSemantics: EltTy->getFltSemantics(), Bits);
6235 break;
6236 }
6237 default:
6238 break;
6239 }
6240}
6241
6242KnownFPClass llvm::computeKnownFPClass(const Value *V,
6243 const APInt &DemandedElts,
6244 FPClassTest InterestedClasses,
6245 const SimplifyQuery &SQ,
6246 unsigned Depth) {
6247 KnownFPClass KnownClasses;
6248 ::computeKnownFPClass(V, DemandedElts, InterestedClasses, Known&: KnownClasses, Q: SQ,
6249 Depth);
6250 return KnownClasses;
6251}
6252
6253KnownFPClass llvm::computeKnownFPClass(const Value *V,
6254 FPClassTest InterestedClasses,
6255 const SimplifyQuery &SQ,
6256 unsigned Depth) {
6257 KnownFPClass Known;
6258 ::computeKnownFPClass(V, Known, InterestedClasses, Q: SQ, Depth);
6259 return Known;
6260}
6261
6262KnownFPClass llvm::computeKnownFPClass(
6263 const Value *V, const DataLayout &DL, FPClassTest InterestedClasses,
6264 const TargetLibraryInfo *TLI, AssumptionCache *AC, const Instruction *CxtI,
6265 const DominatorTree *DT, bool UseInstrInfo, unsigned Depth) {
6266 return computeKnownFPClass(V, InterestedClasses,
6267 SQ: SimplifyQuery(DL, TLI, DT, AC, CxtI, UseInstrInfo),
6268 Depth);
6269}
6270
6271KnownFPClass
6272llvm::computeKnownFPClass(const Value *V, const APInt &DemandedElts,
6273 FastMathFlags FMF, FPClassTest InterestedClasses,
6274 const SimplifyQuery &SQ, unsigned Depth) {
6275 if (FMF.noNaNs())
6276 InterestedClasses &= ~fcNan;
6277 if (FMF.noInfs())
6278 InterestedClasses &= ~fcInf;
6279
6280 KnownFPClass Result =
6281 computeKnownFPClass(V, DemandedElts, InterestedClasses, SQ, Depth);
6282
6283 if (FMF.noNaNs())
6284 Result.KnownFPClasses &= ~fcNan;
6285 if (FMF.noInfs())
6286 Result.KnownFPClasses &= ~fcInf;
6287 return Result;
6288}
6289
6290KnownFPClass llvm::computeKnownFPClass(const Value *V, FastMathFlags FMF,
6291 FPClassTest InterestedClasses,
6292 const SimplifyQuery &SQ,
6293 unsigned Depth) {
6294 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
6295 APInt DemandedElts =
6296 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
6297 return computeKnownFPClass(V, DemandedElts, FMF, InterestedClasses, SQ,
6298 Depth);
6299}
6300
6301bool llvm::cannotBeNegativeZero(const Value *V, const SimplifyQuery &SQ,
6302 unsigned Depth) {
6303 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNegZero, SQ, Depth);
6304 return Known.isKnownNeverNegZero();
6305}
6306
6307bool llvm::cannotBeOrderedLessThanZero(const Value *V, const SimplifyQuery &SQ,
6308 unsigned Depth) {
6309 KnownFPClass Known =
6310 computeKnownFPClass(V, InterestedClasses: KnownFPClass::OrderedLessThanZeroMask, SQ, Depth);
6311 return Known.cannotBeOrderedLessThanZero();
6312}
6313
6314bool llvm::isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ,
6315 unsigned Depth) {
6316 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf, SQ, Depth);
6317 return Known.isKnownNeverInfinity();
6318}
6319
6320/// Return true if the floating-point value can never contain a NaN or infinity.
6321bool llvm::isKnownNeverInfOrNaN(const Value *V, const SimplifyQuery &SQ,
6322 unsigned Depth) {
6323 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf | fcNan, SQ, Depth);
6324 return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity();
6325}
6326
6327/// Return true if the floating-point scalar value is not a NaN or if the
6328/// floating-point vector value has no NaN elements. Return false if a value
6329/// could ever be NaN.
6330bool llvm::isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ,
6331 unsigned Depth) {
6332 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNan, SQ, Depth);
6333 return Known.isKnownNeverNaN();
6334}
6335
6336/// Return false if we can prove that the specified FP value's sign bit is 0.
6337/// Return true if we can prove that the specified FP value's sign bit is 1.
6338/// Otherwise return std::nullopt.
6339std::optional<bool> llvm::computeKnownFPSignBit(const Value *V,
6340 const SimplifyQuery &SQ,
6341 unsigned Depth) {
6342 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcAllFlags, SQ, Depth);
6343 return Known.SignBit;
6344}
6345
6346bool llvm::canIgnoreSignBitOfZero(const Use &U) {
6347 auto *User = cast<Instruction>(Val: U.getUser());
6348 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6349 if (FPOp->hasNoSignedZeros())
6350 return true;
6351 }
6352
6353 switch (User->getOpcode()) {
6354 case Instruction::FPToSI:
6355 case Instruction::FPToUI:
6356 return true;
6357 case Instruction::FCmp:
6358 // fcmp treats both positive and negative zero as equal.
6359 return true;
6360 case Instruction::Call:
6361 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6362 switch (II->getIntrinsicID()) {
6363 case Intrinsic::fabs:
6364 return true;
6365 case Intrinsic::copysign:
6366 return U.getOperandNo() == 0;
6367 case Intrinsic::is_fpclass:
6368 case Intrinsic::vp_is_fpclass: {
6369 auto Test =
6370 static_cast<FPClassTest>(
6371 cast<ConstantInt>(Val: II->getArgOperand(i: 1))->getZExtValue()) &
6372 FPClassTest::fcZero;
6373 return Test == FPClassTest::fcZero || Test == FPClassTest::fcNone;
6374 }
6375 default:
6376 return false;
6377 }
6378 }
6379 return false;
6380 default:
6381 return false;
6382 }
6383}
6384
6385bool llvm::canIgnoreSignBitOfNaN(const Use &U) {
6386 auto *User = cast<Instruction>(Val: U.getUser());
6387 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6388 if (FPOp->hasNoNaNs())
6389 return true;
6390 }
6391
6392 switch (User->getOpcode()) {
6393 case Instruction::FPToSI:
6394 case Instruction::FPToUI:
6395 return true;
6396 // Proper FP math operations ignore the sign bit of NaN.
6397 case Instruction::FAdd:
6398 case Instruction::FSub:
6399 case Instruction::FMul:
6400 case Instruction::FDiv:
6401 case Instruction::FRem:
6402 case Instruction::FPTrunc:
6403 case Instruction::FPExt:
6404 case Instruction::FCmp:
6405 return true;
6406 // Bitwise FP operations should preserve the sign bit of NaN.
6407 case Instruction::FNeg:
6408 case Instruction::Select:
6409 case Instruction::PHI:
6410 return false;
6411 case Instruction::Ret:
6412 return User->getFunction()->getAttributes().getRetNoFPClass() &
6413 FPClassTest::fcNan;
6414 case Instruction::Call:
6415 case Instruction::Invoke: {
6416 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6417 switch (II->getIntrinsicID()) {
6418 case Intrinsic::fabs:
6419 return true;
6420 case Intrinsic::copysign:
6421 return U.getOperandNo() == 0;
6422 // Other proper FP math intrinsics ignore the sign bit of NaN.
6423 case Intrinsic::maxnum:
6424 case Intrinsic::minnum:
6425 case Intrinsic::maximum:
6426 case Intrinsic::minimum:
6427 case Intrinsic::maximumnum:
6428 case Intrinsic::minimumnum:
6429 case Intrinsic::canonicalize:
6430 case Intrinsic::fma:
6431 case Intrinsic::fmuladd:
6432 case Intrinsic::sqrt:
6433 case Intrinsic::pow:
6434 case Intrinsic::powi:
6435 case Intrinsic::fptoui_sat:
6436 case Intrinsic::fptosi_sat:
6437 case Intrinsic::is_fpclass:
6438 case Intrinsic::vp_is_fpclass:
6439 return true;
6440 default:
6441 return false;
6442 }
6443 }
6444
6445 FPClassTest NoFPClass =
6446 cast<CallBase>(Val: User)->getParamNoFPClass(i: U.getOperandNo());
6447 return NoFPClass & FPClassTest::fcNan;
6448 }
6449 default:
6450 return false;
6451 }
6452}
6453
6454bool llvm::isKnownIntegral(const Value *V, const SimplifyQuery &SQ,
6455 FastMathFlags FMF) {
6456 if (isa<PoisonValue>(Val: V))
6457 return true;
6458 if (isa<UndefValue>(Val: V))
6459 return false;
6460
6461 if (match(V, P: m_CheckedFp(CheckFn: [](const APFloat &Val) { return Val.isInteger(); })))
6462 return true;
6463
6464 const Instruction *I = dyn_cast<Instruction>(Val: V);
6465 if (!I)
6466 return false;
6467
6468 switch (I->getOpcode()) {
6469 case Instruction::SIToFP:
6470 case Instruction::UIToFP:
6471 // TODO: Could check nofpclass(inf) on incoming argument
6472 if (FMF.noInfs())
6473 return true;
6474
6475 // Need to check int size cannot produce infinity, which computeKnownFPClass
6476 // knows how to do already.
6477 return isKnownNeverInfinity(V: I, SQ);
6478 case Instruction::Call: {
6479 const CallInst *CI = cast<CallInst>(Val: I);
6480 switch (CI->getIntrinsicID()) {
6481 case Intrinsic::trunc:
6482 case Intrinsic::floor:
6483 case Intrinsic::ceil:
6484 case Intrinsic::rint:
6485 case Intrinsic::nearbyint:
6486 case Intrinsic::round:
6487 case Intrinsic::roundeven:
6488 return (FMF.noInfs() && FMF.noNaNs()) || isKnownNeverInfOrNaN(V: I, SQ);
6489 default:
6490 break;
6491 }
6492
6493 break;
6494 }
6495 default:
6496 break;
6497 }
6498
6499 return false;
6500}
6501
6502Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
6503
6504 // All byte-wide stores are splatable, even of arbitrary variables.
6505 if (V->getType()->isIntegerTy(BitWidth: 8))
6506 return V;
6507
6508 LLVMContext &Ctx = V->getContext();
6509
6510 // Undef don't care.
6511 auto *UndefInt8 = UndefValue::get(T: Type::getInt8Ty(C&: Ctx));
6512 if (isa<UndefValue>(Val: V))
6513 return UndefInt8;
6514
6515 // Return poison for zero-sized type.
6516 if (DL.getTypeStoreSize(Ty: V->getType()).isZero())
6517 return PoisonValue::get(T: Type::getInt8Ty(C&: Ctx));
6518
6519 Constant *C = dyn_cast<Constant>(Val: V);
6520 if (!C) {
6521 // Conceptually, we could handle things like:
6522 // %a = zext i8 %X to i16
6523 // %b = shl i16 %a, 8
6524 // %c = or i16 %a, %b
6525 // but until there is an example that actually needs this, it doesn't seem
6526 // worth worrying about.
6527 return nullptr;
6528 }
6529
6530 // Handle 'null' ConstantArrayZero etc.
6531 if (C->isNullValue())
6532 return Constant::getNullValue(Ty: Type::getInt8Ty(C&: Ctx));
6533
6534 // Constant floating-point values can be handled as integer values if the
6535 // corresponding integer value is "byteable". An important case is 0.0.
6536 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
6537 Type *ScalarTy = CFP->getType()->getScalarType();
6538 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
6539 return isBytewiseValue(
6540 V: ConstantInt::get(Context&: Ctx, V: CFP->getValue().bitcastToAPInt()), DL);
6541
6542 // Don't handle long double formats, which have strange constraints.
6543 return nullptr;
6544 }
6545
6546 // We can handle constant integers that are multiple of 8 bits.
6547 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
6548 if (CI->getBitWidth() % 8 == 0) {
6549 if (!CI->getValue().isSplat(SplatSizeInBits: 8))
6550 return nullptr;
6551 return ConstantInt::get(Context&: Ctx, V: CI->getValue().trunc(width: 8));
6552 }
6553 }
6554
6555 if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
6556 if (CE->getOpcode() == Instruction::IntToPtr) {
6557 if (auto *PtrTy = dyn_cast<PointerType>(Val: CE->getType())) {
6558 unsigned BitWidth = DL.getPointerSizeInBits(AS: PtrTy->getAddressSpace());
6559 if (Constant *Op = ConstantFoldIntegerCast(
6560 C: CE->getOperand(i_nocapture: 0), DestTy: Type::getIntNTy(C&: Ctx, N: BitWidth), IsSigned: false, DL))
6561 return isBytewiseValue(V: Op, DL);
6562 }
6563 }
6564 }
6565
6566 auto Merge = [&](Value *LHS, Value *RHS) -> Value * {
6567 if (LHS == RHS)
6568 return LHS;
6569 if (!LHS || !RHS)
6570 return nullptr;
6571 if (LHS == UndefInt8)
6572 return RHS;
6573 if (RHS == UndefInt8)
6574 return LHS;
6575 return nullptr;
6576 };
6577
6578 if (ConstantDataSequential *CA = dyn_cast<ConstantDataSequential>(Val: C)) {
6579 Value *Val = UndefInt8;
6580 for (uint64_t I = 0, E = CA->getNumElements(); I != E; ++I)
6581 if (!(Val = Merge(Val, isBytewiseValue(V: CA->getElementAsConstant(i: I), DL))))
6582 return nullptr;
6583 return Val;
6584 }
6585
6586 if (isa<ConstantAggregate>(Val: C)) {
6587 Value *Val = UndefInt8;
6588 for (Value *Op : C->operands())
6589 if (!(Val = Merge(Val, isBytewiseValue(V: Op, DL))))
6590 return nullptr;
6591 return Val;
6592 }
6593
6594 // Don't try to handle the handful of other constants.
6595 return nullptr;
6596}
6597
6598// This is the recursive version of BuildSubAggregate. It takes a few different
6599// arguments. Idxs is the index within the nested struct From that we are
6600// looking at now (which is of type IndexedType). IdxSkip is the number of
6601// indices from Idxs that should be left out when inserting into the resulting
6602// struct. To is the result struct built so far, new insertvalue instructions
6603// build on that.
6604static Value *BuildSubAggregate(Value *From, Value *To, Type *IndexedType,
6605 SmallVectorImpl<unsigned> &Idxs,
6606 unsigned IdxSkip,
6607 BasicBlock::iterator InsertBefore) {
6608 StructType *STy = dyn_cast<StructType>(Val: IndexedType);
6609 if (STy) {
6610 // Save the original To argument so we can modify it
6611 Value *OrigTo = To;
6612 // General case, the type indexed by Idxs is a struct
6613 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6614 // Process each struct element recursively
6615 Idxs.push_back(Elt: i);
6616 Value *PrevTo = To;
6617 To = BuildSubAggregate(From, To, IndexedType: STy->getElementType(N: i), Idxs, IdxSkip,
6618 InsertBefore);
6619 Idxs.pop_back();
6620 if (!To) {
6621 // Couldn't find any inserted value for this index? Cleanup
6622 while (PrevTo != OrigTo) {
6623 InsertValueInst* Del = cast<InsertValueInst>(Val: PrevTo);
6624 PrevTo = Del->getAggregateOperand();
6625 Del->eraseFromParent();
6626 }
6627 // Stop processing elements
6628 break;
6629 }
6630 }
6631 // If we successfully found a value for each of our subaggregates
6632 if (To)
6633 return To;
6634 }
6635 // Base case, the type indexed by SourceIdxs is not a struct, or not all of
6636 // the struct's elements had a value that was inserted directly. In the latter
6637 // case, perhaps we can't determine each of the subelements individually, but
6638 // we might be able to find the complete struct somewhere.
6639
6640 // Find the value that is at that particular spot
6641 Value *V = FindInsertedValue(V: From, idx_range: Idxs);
6642
6643 if (!V)
6644 return nullptr;
6645
6646 // Insert the value in the new (sub) aggregate
6647 return InsertValueInst::Create(Agg: To, Val: V, Idxs: ArrayRef(Idxs).slice(N: IdxSkip), NameStr: "tmp",
6648 InsertBefore);
6649}
6650
6651// This helper takes a nested struct and extracts a part of it (which is again a
6652// struct) into a new value. For example, given the struct:
6653// { a, { b, { c, d }, e } }
6654// and the indices "1, 1" this returns
6655// { c, d }.
6656//
6657// It does this by inserting an insertvalue for each element in the resulting
6658// struct, as opposed to just inserting a single struct. This will only work if
6659// each of the elements of the substruct are known (ie, inserted into From by an
6660// insertvalue instruction somewhere).
6661//
6662// All inserted insertvalue instructions are inserted before InsertBefore
6663static Value *BuildSubAggregate(Value *From, ArrayRef<unsigned> idx_range,
6664 BasicBlock::iterator InsertBefore) {
6665 Type *IndexedType = ExtractValueInst::getIndexedType(Agg: From->getType(),
6666 Idxs: idx_range);
6667 Value *To = PoisonValue::get(T: IndexedType);
6668 SmallVector<unsigned, 10> Idxs(idx_range);
6669 unsigned IdxSkip = Idxs.size();
6670
6671 return BuildSubAggregate(From, To, IndexedType, Idxs, IdxSkip, InsertBefore);
6672}
6673
6674/// Given an aggregate and a sequence of indices, see if the scalar value
6675/// indexed is already around as a register, for example if it was inserted
6676/// directly into the aggregate.
6677///
6678/// If InsertBefore is not null, this function will duplicate (modified)
6679/// insertvalues when a part of a nested struct is extracted.
6680Value *
6681llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
6682 std::optional<BasicBlock::iterator> InsertBefore) {
6683 // Nothing to index? Just return V then (this is useful at the end of our
6684 // recursion).
6685 if (idx_range.empty())
6686 return V;
6687 // We have indices, so V should have an indexable type.
6688 assert((V->getType()->isStructTy() || V->getType()->isArrayTy()) &&
6689 "Not looking at a struct or array?");
6690 assert(ExtractValueInst::getIndexedType(V->getType(), idx_range) &&
6691 "Invalid indices for type?");
6692
6693 if (Constant *C = dyn_cast<Constant>(Val: V)) {
6694 C = C->getAggregateElement(Elt: idx_range[0]);
6695 if (!C) return nullptr;
6696 return FindInsertedValue(V: C, idx_range: idx_range.slice(N: 1), InsertBefore);
6697 }
6698
6699 if (InsertValueInst *I = dyn_cast<InsertValueInst>(Val: V)) {
6700 // Loop the indices for the insertvalue instruction in parallel with the
6701 // requested indices
6702 const unsigned *req_idx = idx_range.begin();
6703 for (const unsigned *i = I->idx_begin(), *e = I->idx_end();
6704 i != e; ++i, ++req_idx) {
6705 if (req_idx == idx_range.end()) {
6706 // We can't handle this without inserting insertvalues
6707 if (!InsertBefore)
6708 return nullptr;
6709
6710 // The requested index identifies a part of a nested aggregate. Handle
6711 // this specially. For example,
6712 // %A = insertvalue { i32, {i32, i32 } } undef, i32 10, 1, 0
6713 // %B = insertvalue { i32, {i32, i32 } } %A, i32 11, 1, 1
6714 // %C = extractvalue {i32, { i32, i32 } } %B, 1
6715 // This can be changed into
6716 // %A = insertvalue {i32, i32 } undef, i32 10, 0
6717 // %C = insertvalue {i32, i32 } %A, i32 11, 1
6718 // which allows the unused 0,0 element from the nested struct to be
6719 // removed.
6720 return BuildSubAggregate(From: V, idx_range: ArrayRef(idx_range.begin(), req_idx),
6721 InsertBefore: *InsertBefore);
6722 }
6723
6724 // This insert value inserts something else than what we are looking for.
6725 // See if the (aggregate) value inserted into has the value we are
6726 // looking for, then.
6727 if (*req_idx != *i)
6728 return FindInsertedValue(V: I->getAggregateOperand(), idx_range,
6729 InsertBefore);
6730 }
6731 // If we end up here, the indices of the insertvalue match with those
6732 // requested (though possibly only partially). Now we recursively look at
6733 // the inserted value, passing any remaining indices.
6734 return FindInsertedValue(V: I->getInsertedValueOperand(),
6735 idx_range: ArrayRef(req_idx, idx_range.end()), InsertBefore);
6736 }
6737
6738 if (ExtractValueInst *I = dyn_cast<ExtractValueInst>(Val: V)) {
6739 // If we're extracting a value from an aggregate that was extracted from
6740 // something else, we can extract from that something else directly instead.
6741 // However, we will need to chain I's indices with the requested indices.
6742
6743 // Calculate the number of indices required
6744 unsigned size = I->getNumIndices() + idx_range.size();
6745 // Allocate some space to put the new indices in
6746 SmallVector<unsigned, 5> Idxs;
6747 Idxs.reserve(N: size);
6748 // Add indices from the extract value instruction
6749 Idxs.append(in_start: I->idx_begin(), in_end: I->idx_end());
6750
6751 // Add requested indices
6752 Idxs.append(in_start: idx_range.begin(), in_end: idx_range.end());
6753
6754 assert(Idxs.size() == size
6755 && "Number of indices added not correct?");
6756
6757 return FindInsertedValue(V: I->getAggregateOperand(), idx_range: Idxs, InsertBefore);
6758 }
6759 // Otherwise, we don't know (such as, extracting from a function return value
6760 // or load instruction)
6761 return nullptr;
6762}
6763
6764// If V refers to an initialized global constant, set Slice either to
6765// its initializer if the size of its elements equals ElementSize, or,
6766// for ElementSize == 8, to its representation as an array of unsiged
6767// char. Return true on success.
6768// Offset is in the unit "nr of ElementSize sized elements".
6769bool llvm::getConstantDataArrayInfo(const Value *V,
6770 ConstantDataArraySlice &Slice,
6771 unsigned ElementSize, uint64_t Offset) {
6772 assert(V && "V should not be null.");
6773 assert((ElementSize % 8) == 0 &&
6774 "ElementSize expected to be a multiple of the size of a byte.");
6775 unsigned ElementSizeInBytes = ElementSize / 8;
6776
6777 // Drill down into the pointer expression V, ignoring any intervening
6778 // casts, and determine the identity of the object it references along
6779 // with the cumulative byte offset into it.
6780 const GlobalVariable *GV =
6781 dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V));
6782 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
6783 // Fail if V is not based on constant global object.
6784 return false;
6785
6786 const DataLayout &DL = GV->getDataLayout();
6787 APInt Off(DL.getIndexTypeSizeInBits(Ty: V->getType()), 0);
6788
6789 if (GV != V->stripAndAccumulateConstantOffsets(DL, Offset&: Off,
6790 /*AllowNonInbounds*/ true))
6791 // Fail if a constant offset could not be determined.
6792 return false;
6793
6794 uint64_t StartIdx = Off.getLimitedValue();
6795 if (StartIdx == UINT64_MAX)
6796 // Fail if the constant offset is excessive.
6797 return false;
6798
6799 // Off/StartIdx is in the unit of bytes. So we need to convert to number of
6800 // elements. Simply bail out if that isn't possible.
6801 if ((StartIdx % ElementSizeInBytes) != 0)
6802 return false;
6803
6804 Offset += StartIdx / ElementSizeInBytes;
6805 ConstantDataArray *Array = nullptr;
6806 ArrayType *ArrayTy = nullptr;
6807
6808 if (GV->getInitializer()->isNullValue()) {
6809 Type *GVTy = GV->getValueType();
6810 uint64_t SizeInBytes = DL.getTypeStoreSize(Ty: GVTy).getFixedValue();
6811 uint64_t Length = SizeInBytes / ElementSizeInBytes;
6812
6813 Slice.Array = nullptr;
6814 Slice.Offset = 0;
6815 // Return an empty Slice for undersized constants to let callers
6816 // transform even undefined library calls into simpler, well-defined
6817 // expressions. This is preferable to making the calls although it
6818 // prevents sanitizers from detecting such calls.
6819 Slice.Length = Length < Offset ? 0 : Length - Offset;
6820 return true;
6821 }
6822
6823 auto *Init = const_cast<Constant *>(GV->getInitializer());
6824 if (auto *ArrayInit = dyn_cast<ConstantDataArray>(Val: Init)) {
6825 Type *InitElTy = ArrayInit->getElementType();
6826 if (InitElTy->isIntegerTy(BitWidth: ElementSize)) {
6827 // If Init is an initializer for an array of the expected type
6828 // and size, use it as is.
6829 Array = ArrayInit;
6830 ArrayTy = ArrayInit->getType();
6831 }
6832 }
6833
6834 if (!Array) {
6835 if (ElementSize != 8)
6836 // TODO: Handle conversions to larger integral types.
6837 return false;
6838
6839 // Otherwise extract the portion of the initializer starting
6840 // at Offset as an array of bytes, and reset Offset.
6841 Init = ReadByteArrayFromGlobal(GV, Offset);
6842 if (!Init)
6843 return false;
6844
6845 Offset = 0;
6846 Array = dyn_cast<ConstantDataArray>(Val: Init);
6847 ArrayTy = dyn_cast<ArrayType>(Val: Init->getType());
6848 }
6849
6850 uint64_t NumElts = ArrayTy->getArrayNumElements();
6851 if (Offset > NumElts)
6852 return false;
6853
6854 Slice.Array = Array;
6855 Slice.Offset = Offset;
6856 Slice.Length = NumElts - Offset;
6857 return true;
6858}
6859
6860/// Extract bytes from the initializer of the constant array V, which need
6861/// not be a nul-terminated string. On success, store the bytes in Str and
6862/// return true. When TrimAtNul is set, Str will contain only the bytes up
6863/// to but not including the first nul. Return false on failure.
6864bool llvm::getConstantStringInfo(const Value *V, StringRef &Str,
6865 bool TrimAtNul) {
6866 ConstantDataArraySlice Slice;
6867 if (!getConstantDataArrayInfo(V, Slice, ElementSize: 8))
6868 return false;
6869
6870 if (Slice.Array == nullptr) {
6871 if (TrimAtNul) {
6872 // Return a nul-terminated string even for an empty Slice. This is
6873 // safe because all existing SimplifyLibcalls callers require string
6874 // arguments and the behavior of the functions they fold is undefined
6875 // otherwise. Folding the calls this way is preferable to making
6876 // the undefined library calls, even though it prevents sanitizers
6877 // from reporting such calls.
6878 Str = StringRef();
6879 return true;
6880 }
6881 if (Slice.Length == 1) {
6882 Str = StringRef("", 1);
6883 return true;
6884 }
6885 // We cannot instantiate a StringRef as we do not have an appropriate string
6886 // of 0s at hand.
6887 return false;
6888 }
6889
6890 // Start out with the entire array in the StringRef.
6891 Str = Slice.Array->getAsString();
6892 // Skip over 'offset' bytes.
6893 Str = Str.substr(Start: Slice.Offset);
6894
6895 if (TrimAtNul) {
6896 // Trim off the \0 and anything after it. If the array is not nul
6897 // terminated, we just return the whole end of string. The client may know
6898 // some other way that the string is length-bound.
6899 Str = Str.substr(Start: 0, N: Str.find(C: '\0'));
6900 }
6901 return true;
6902}
6903
6904// These next two are very similar to the above, but also look through PHI
6905// nodes.
6906// TODO: See if we can integrate these two together.
6907
6908/// If we can compute the length of the string pointed to by
6909/// the specified pointer, return 'len+1'. If we can't, return 0.
6910static uint64_t GetStringLengthH(const Value *V,
6911 SmallPtrSetImpl<const PHINode*> &PHIs,
6912 unsigned CharSize) {
6913 // Look through noop bitcast instructions.
6914 V = V->stripPointerCasts();
6915
6916 // If this is a PHI node, there are two cases: either we have already seen it
6917 // or we haven't.
6918 if (const PHINode *PN = dyn_cast<PHINode>(Val: V)) {
6919 if (!PHIs.insert(Ptr: PN).second)
6920 return ~0ULL; // already in the set.
6921
6922 // If it was new, see if all the input strings are the same length.
6923 uint64_t LenSoFar = ~0ULL;
6924 for (Value *IncValue : PN->incoming_values()) {
6925 uint64_t Len = GetStringLengthH(V: IncValue, PHIs, CharSize);
6926 if (Len == 0) return 0; // Unknown length -> unknown.
6927
6928 if (Len == ~0ULL) continue;
6929
6930 if (Len != LenSoFar && LenSoFar != ~0ULL)
6931 return 0; // Disagree -> unknown.
6932 LenSoFar = Len;
6933 }
6934
6935 // Success, all agree.
6936 return LenSoFar;
6937 }
6938
6939 // strlen(select(c,x,y)) -> strlen(x) ^ strlen(y)
6940 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: V)) {
6941 uint64_t Len1 = GetStringLengthH(V: SI->getTrueValue(), PHIs, CharSize);
6942 if (Len1 == 0) return 0;
6943 uint64_t Len2 = GetStringLengthH(V: SI->getFalseValue(), PHIs, CharSize);
6944 if (Len2 == 0) return 0;
6945 if (Len1 == ~0ULL) return Len2;
6946 if (Len2 == ~0ULL) return Len1;
6947 if (Len1 != Len2) return 0;
6948 return Len1;
6949 }
6950
6951 // Otherwise, see if we can read the string.
6952 ConstantDataArraySlice Slice;
6953 if (!getConstantDataArrayInfo(V, Slice, ElementSize: CharSize))
6954 return 0;
6955
6956 if (Slice.Array == nullptr)
6957 // Zeroinitializer (including an empty one).
6958 return 1;
6959
6960 // Search for the first nul character. Return a conservative result even
6961 // when there is no nul. This is safe since otherwise the string function
6962 // being folded such as strlen is undefined, and can be preferable to
6963 // making the undefined library call.
6964 unsigned NullIndex = 0;
6965 for (unsigned E = Slice.Length; NullIndex < E; ++NullIndex) {
6966 if (Slice.Array->getElementAsInteger(i: Slice.Offset + NullIndex) == 0)
6967 break;
6968 }
6969
6970 return NullIndex + 1;
6971}
6972
6973/// If we can compute the length of the string pointed to by
6974/// the specified pointer, return 'len+1'. If we can't, return 0.
6975uint64_t llvm::GetStringLength(const Value *V, unsigned CharSize) {
6976 if (!V->getType()->isPointerTy())
6977 return 0;
6978
6979 SmallPtrSet<const PHINode*, 32> PHIs;
6980 uint64_t Len = GetStringLengthH(V, PHIs, CharSize);
6981 // If Len is ~0ULL, we had an infinite phi cycle: this is dead code, so return
6982 // an empty string as a length.
6983 return Len == ~0ULL ? 1 : Len;
6984}
6985
6986const Value *
6987llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call,
6988 bool MustPreserveOffset) {
6989 assert(Call &&
6990 "getArgumentAliasingToReturnedPointer only works on nonnull calls");
6991 if (const Value *RV = Call->getReturnedArgOperand())
6992 return RV;
6993 // This can be used only as a aliasing property.
6994 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
6995 Call, MustPreserveOffset))
6996 return Call->getArgOperand(i: 0);
6997 return nullptr;
6998}
6999
7000bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
7001 const CallBase *Call, bool MustPreserveOffset) {
7002 switch (Call->getIntrinsicID()) {
7003 case Intrinsic::launder_invariant_group:
7004 case Intrinsic::strip_invariant_group:
7005 case Intrinsic::aarch64_irg:
7006 case Intrinsic::aarch64_tagp:
7007 // The amdgcn_make_buffer_rsrc function does not alter the address of the
7008 // input pointer (and thus preserves the byte offset, which is the property
7009 // the MustPreserveOffset flag selects). However, it will not necessarily
7010 // map ptr addrspace(N) null to ptr addrspace(8) null, aka the "null
7011 // descriptor", which has "all loads return 0, all stores are dropped"
7012 // semantics. Given the context of this intrinsic list, no one should be
7013 // relying on such a strict bit-exact null mapping (and, at time of
7014 // writing, they are not), but we document this fact out of an abundance
7015 // of caution.
7016 case Intrinsic::amdgcn_make_buffer_rsrc:
7017 return true;
7018 case Intrinsic::ptrmask:
7019 return !MustPreserveOffset;
7020 case Intrinsic::threadlocal_address:
7021 // The underlying variable changes with thread ID. The Thread ID may change
7022 // at coroutine suspend points.
7023 return !Call->getParent()->getParent()->isPresplitCoroutine();
7024 default:
7025 return false;
7026 }
7027}
7028
7029/// \p PN defines a loop-variant pointer to an object. Check if the
7030/// previous iteration of the loop was referring to the same object as \p PN.
7031static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
7032 const LoopInfo *LI) {
7033 // Find the loop-defined value.
7034 Loop *L = LI->getLoopFor(BB: PN->getParent());
7035 if (PN->getNumIncomingValues() != 2)
7036 return true;
7037
7038 // Find the value from previous iteration.
7039 auto *PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 0));
7040 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
7041 PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 1));
7042 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
7043 return true;
7044
7045 // If a new pointer is loaded in the loop, the pointer references a different
7046 // object in every iteration. E.g.:
7047 // for (i)
7048 // int *p = a[i];
7049 // ...
7050 if (auto *Load = dyn_cast<LoadInst>(Val: PrevValue))
7051 if (!L->isLoopInvariant(V: Load->getPointerOperand()))
7052 return false;
7053 return true;
7054}
7055
7056const Value *llvm::getUnderlyingObject(const Value *V, unsigned MaxLookup) {
7057 for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
7058 if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
7059 const Value *PtrOp = GEP->getPointerOperand();
7060 if (!PtrOp->getType()->isPointerTy()) // Only handle scalar pointer base.
7061 return V;
7062 V = PtrOp;
7063 } else if (Operator::getOpcode(V) == Instruction::BitCast ||
7064 Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
7065 Value *NewV = cast<Operator>(Val: V)->getOperand(i: 0);
7066 if (!NewV->getType()->isPointerTy())
7067 return V;
7068 V = NewV;
7069 } else if (auto *GA = dyn_cast<GlobalAlias>(Val: V)) {
7070 if (GA->isInterposable())
7071 return V;
7072 V = GA->getAliasee();
7073 } else {
7074 if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
7075 // Look through single-arg phi nodes created by LCSSA.
7076 if (PHI->getNumIncomingValues() == 1) {
7077 V = PHI->getIncomingValue(i: 0);
7078 continue;
7079 }
7080 } else if (auto *Call = dyn_cast<CallBase>(Val: V)) {
7081 // CaptureTracking can know about special capturing properties of some
7082 // intrinsics like launder.invariant.group, that can't be expressed with
7083 // the attributes, but have properties like returning aliasing pointer.
7084 // Because some analysis may assume that nocaptured pointer is not
7085 // returned from some special intrinsic (because function would have to
7086 // be marked with returns attribute), it is crucial to use this function
7087 // because it should be in sync with CaptureTracking. Not using it may
7088 // cause weird miscompilations where 2 aliasing pointers are assumed to
7089 // noalias.
7090 if (auto *RP = getArgumentAliasingToReturnedPointer(
7091 Call, /*MustPreserveOffset=*/false)) {
7092 V = RP;
7093 continue;
7094 }
7095 }
7096
7097 return V;
7098 }
7099 assert(V->getType()->isPointerTy() && "Unexpected operand type!");
7100 }
7101 return V;
7102}
7103
7104void llvm::getUnderlyingObjects(const Value *V,
7105 SmallVectorImpl<const Value *> &Objects,
7106 const LoopInfo *LI, unsigned MaxLookup) {
7107 SmallPtrSet<const Value *, 4> Visited;
7108 SmallVector<const Value *, 4> Worklist;
7109 Worklist.push_back(Elt: V);
7110 do {
7111 const Value *P = Worklist.pop_back_val();
7112 P = getUnderlyingObject(V: P, MaxLookup);
7113
7114 if (!Visited.insert(Ptr: P).second)
7115 continue;
7116
7117 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7118 Worklist.push_back(Elt: SI->getTrueValue());
7119 Worklist.push_back(Elt: SI->getFalseValue());
7120 continue;
7121 }
7122
7123 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7124 // If this PHI changes the underlying object in every iteration of the
7125 // loop, don't look through it. Consider:
7126 // int **A;
7127 // for (i) {
7128 // Prev = Curr; // Prev = PHI (Prev_0, Curr)
7129 // Curr = A[i];
7130 // *Prev, *Curr;
7131 //
7132 // Prev is tracking Curr one iteration behind so they refer to different
7133 // underlying objects.
7134 if (!LI || !LI->isLoopHeader(BB: PN->getParent()) ||
7135 isSameUnderlyingObjectInLoop(PN, LI))
7136 append_range(C&: Worklist, R: PN->incoming_values());
7137 else
7138 Objects.push_back(Elt: P);
7139 continue;
7140 }
7141
7142 Objects.push_back(Elt: P);
7143 } while (!Worklist.empty());
7144}
7145
7146const Value *llvm::getUnderlyingObjectAggressive(const Value *V) {
7147 const unsigned MaxVisited = 8;
7148
7149 SmallPtrSet<const Value *, 8> Visited;
7150 SmallVector<const Value *, 8> Worklist;
7151 Worklist.push_back(Elt: V);
7152 const Value *Object = nullptr;
7153 // Used as fallback if we can't find a common underlying object through
7154 // recursion.
7155 bool First = true;
7156 const Value *FirstObject = getUnderlyingObject(V);
7157 do {
7158 const Value *P = Worklist.pop_back_val();
7159 P = First ? FirstObject : getUnderlyingObject(V: P);
7160 First = false;
7161
7162 if (!Visited.insert(Ptr: P).second)
7163 continue;
7164
7165 if (Visited.size() == MaxVisited)
7166 return FirstObject;
7167
7168 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7169 Worklist.push_back(Elt: SI->getTrueValue());
7170 Worklist.push_back(Elt: SI->getFalseValue());
7171 continue;
7172 }
7173
7174 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7175 append_range(C&: Worklist, R: PN->incoming_values());
7176 continue;
7177 }
7178
7179 if (!Object)
7180 Object = P;
7181 else if (Object != P)
7182 return FirstObject;
7183 } while (!Worklist.empty());
7184
7185 return Object ? Object : FirstObject;
7186}
7187
7188/// This is the function that does the work of looking through basic
7189/// ptrtoint+arithmetic+inttoptr sequences.
7190static const Value *getUnderlyingObjectFromInt(const Value *V) {
7191 do {
7192 if (const Operator *U = dyn_cast<Operator>(Val: V)) {
7193 // If we find a ptrtoint, we can transfer control back to the
7194 // regular getUnderlyingObjectFromInt.
7195 if (U->getOpcode() == Instruction::PtrToInt)
7196 return U->getOperand(i: 0);
7197 // If we find an add of a constant, a multiplied value, or a phi, it's
7198 // likely that the other operand will lead us to the base
7199 // object. We don't have to worry about the case where the
7200 // object address is somehow being computed by the multiply,
7201 // because our callers only care when the result is an
7202 // identifiable object.
7203 if (U->getOpcode() != Instruction::Add ||
7204 (!isa<ConstantInt>(Val: U->getOperand(i: 1)) &&
7205 Operator::getOpcode(V: U->getOperand(i: 1)) != Instruction::Mul &&
7206 !isa<PHINode>(Val: U->getOperand(i: 1))))
7207 return V;
7208 V = U->getOperand(i: 0);
7209 } else {
7210 return V;
7211 }
7212 assert(V->getType()->isIntegerTy() && "Unexpected operand type!");
7213 } while (true);
7214}
7215
7216/// This is a wrapper around getUnderlyingObjects and adds support for basic
7217/// ptrtoint+arithmetic+inttoptr sequences.
7218/// It returns false if unidentified object is found in getUnderlyingObjects.
7219bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
7220 SmallVectorImpl<Value *> &Objects) {
7221 SmallPtrSet<const Value *, 16> Visited;
7222 SmallVector<const Value *, 4> Working(1, V);
7223 do {
7224 V = Working.pop_back_val();
7225
7226 SmallVector<const Value *, 4> Objs;
7227 getUnderlyingObjects(V, Objects&: Objs);
7228
7229 for (const Value *V : Objs) {
7230 if (!Visited.insert(Ptr: V).second)
7231 continue;
7232 if (Operator::getOpcode(V) == Instruction::IntToPtr) {
7233 const Value *O =
7234 getUnderlyingObjectFromInt(V: cast<User>(Val: V)->getOperand(i: 0));
7235 if (O->getType()->isPointerTy()) {
7236 Working.push_back(Elt: O);
7237 continue;
7238 }
7239 }
7240 // If getUnderlyingObjects fails to find an identifiable object,
7241 // getUnderlyingObjectsForCodeGen also fails for safety.
7242 if (!isIdentifiedObject(V)) {
7243 Objects.clear();
7244 return false;
7245 }
7246 Objects.push_back(Elt: const_cast<Value *>(V));
7247 }
7248 } while (!Working.empty());
7249 return true;
7250}
7251
7252AllocaInst *llvm::findAllocaForValue(Value *V, bool OffsetZero) {
7253 AllocaInst *Result = nullptr;
7254 SmallPtrSet<Value *, 4> Visited;
7255 SmallVector<Value *, 4> Worklist;
7256
7257 auto AddWork = [&](Value *V) {
7258 if (Visited.insert(Ptr: V).second)
7259 Worklist.push_back(Elt: V);
7260 };
7261
7262 AddWork(V);
7263 do {
7264 V = Worklist.pop_back_val();
7265 assert(Visited.count(V));
7266
7267 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
7268 if (Result && Result != AI)
7269 return nullptr;
7270 Result = AI;
7271 } else if (CastInst *CI = dyn_cast<CastInst>(Val: V)) {
7272 AddWork(CI->getOperand(i_nocapture: 0));
7273 } else if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7274 for (Value *IncValue : PN->incoming_values())
7275 AddWork(IncValue);
7276 } else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
7277 AddWork(SI->getTrueValue());
7278 AddWork(SI->getFalseValue());
7279 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: V)) {
7280 if (OffsetZero && !GEP->hasAllZeroIndices())
7281 return nullptr;
7282 AddWork(GEP->getPointerOperand());
7283 } else if (CallBase *CB = dyn_cast<CallBase>(Val: V)) {
7284 Value *Returned = CB->getReturnedArgOperand();
7285 if (Returned)
7286 AddWork(Returned);
7287 else
7288 return nullptr;
7289 } else {
7290 return nullptr;
7291 }
7292 } while (!Worklist.empty());
7293
7294 return Result;
7295}
7296
7297static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7298 const Value *V, bool AllowLifetime, bool AllowDroppable) {
7299 for (const User *U : V->users()) {
7300 const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
7301 if (!II)
7302 return false;
7303
7304 if (AllowLifetime && II->isLifetimeStartOrEnd())
7305 continue;
7306
7307 if (AllowDroppable && II->isDroppable())
7308 continue;
7309
7310 return false;
7311 }
7312 return true;
7313}
7314
7315bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
7316 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7317 V, /* AllowLifetime */ true, /* AllowDroppable */ false);
7318}
7319bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) {
7320 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7321 V, /* AllowLifetime */ true, /* AllowDroppable */ true);
7322}
7323
7324bool llvm::isNotCrossLaneOperation(const Instruction *I) {
7325 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
7326 return isTriviallyVectorizable(ID: II->getIntrinsicID());
7327 auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: I);
7328 return (!Shuffle || Shuffle->isSelect()) &&
7329 !isa<CallBase, BitCastInst, ExtractElementInst>(Val: I);
7330}
7331
7332bool llvm::isSafeToSpeculativelyExecute(
7333 const Instruction *Inst, const Instruction *CtxI, AssumptionCache *AC,
7334 const DominatorTree *DT, const TargetLibraryInfo *TLI, bool UseVariableInfo,
7335 bool IgnoreUBImplyingAttrs) {
7336 return isSafeToSpeculativelyExecuteWithOpcode(Opcode: Inst->getOpcode(), Inst, CtxI,
7337 AC, DT, TLI, UseVariableInfo,
7338 IgnoreUBImplyingAttrs);
7339}
7340
7341bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
7342 unsigned Opcode, const Instruction *Inst, const Instruction *CtxI,
7343 AssumptionCache *AC, const DominatorTree *DT, const TargetLibraryInfo *TLI,
7344 bool UseVariableInfo, bool IgnoreUBImplyingAttrs) {
7345#ifndef NDEBUG
7346 if (Inst->getOpcode() != Opcode) {
7347 // Check that the operands are actually compatible with the Opcode override.
7348 auto hasEqualReturnAndLeadingOperandTypes =
7349 [](const Instruction *Inst, unsigned NumLeadingOperands) {
7350 if (Inst->getNumOperands() < NumLeadingOperands)
7351 return false;
7352 const Type *ExpectedType = Inst->getType();
7353 for (unsigned ItOp = 0; ItOp < NumLeadingOperands; ++ItOp)
7354 if (Inst->getOperand(ItOp)->getType() != ExpectedType)
7355 return false;
7356 return true;
7357 };
7358 assert(!Instruction::isBinaryOp(Opcode) ||
7359 hasEqualReturnAndLeadingOperandTypes(Inst, 2));
7360 assert(!Instruction::isUnaryOp(Opcode) ||
7361 hasEqualReturnAndLeadingOperandTypes(Inst, 1));
7362 }
7363#endif
7364
7365 switch (Opcode) {
7366 default:
7367 return true;
7368 case Instruction::UDiv:
7369 case Instruction::URem: {
7370 // x / y is undefined if y == 0.
7371 const APInt *V;
7372 if (match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: V)))
7373 return *V != 0;
7374 return false;
7375 }
7376 case Instruction::SDiv:
7377 case Instruction::SRem: {
7378 // x / y is undefined if y == 0 or x == INT_MIN and y == -1
7379 const APInt *Numerator, *Denominator;
7380 if (!match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: Denominator)))
7381 return false;
7382 // We cannot hoist this division if the denominator is 0.
7383 if (*Denominator == 0)
7384 return false;
7385 // It's safe to hoist if the denominator is not 0 or -1.
7386 if (!Denominator->isAllOnes())
7387 return true;
7388 // At this point we know that the denominator is -1. It is safe to hoist as
7389 // long we know that the numerator is not INT_MIN.
7390 if (match(V: Inst->getOperand(i: 0), P: m_APInt(Res&: Numerator)))
7391 return !Numerator->isMinSignedValue();
7392 // The numerator *might* be MinSignedValue.
7393 return false;
7394 }
7395 case Instruction::Load: {
7396 if (!UseVariableInfo)
7397 return false;
7398
7399 const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst);
7400 if (!LI)
7401 return false;
7402 if (mustSuppressSpeculation(LI: *LI))
7403 return false;
7404 const DataLayout &DL = LI->getDataLayout();
7405 return isDereferenceableAndAlignedPointer(
7406 V: LI->getPointerOperand(), Ty: LI->getType(), Alignment: LI->getAlign(),
7407 Q: SimplifyQuery(DL, TLI, DT, AC, CtxI));
7408 }
7409 case Instruction::Call: {
7410 auto *CI = dyn_cast<const CallInst>(Val: Inst);
7411 if (!CI)
7412 return false;
7413 const Function *Callee = CI->getCalledFunction();
7414
7415 // The called function could have undefined behavior or side-effects, even
7416 // if marked readnone nounwind.
7417 if (!Callee || !Callee->isSpeculatable())
7418 return false;
7419 // Since the operands may be changed after hoisting, undefined behavior may
7420 // be triggered by some UB-implying attributes.
7421 return IgnoreUBImplyingAttrs || !CI->hasUBImplyingAttrs();
7422 }
7423 case Instruction::VAArg:
7424 case Instruction::Alloca:
7425 case Instruction::Invoke:
7426 case Instruction::CallBr:
7427 case Instruction::PHI:
7428 case Instruction::Store:
7429 case Instruction::Ret:
7430 case Instruction::UncondBr:
7431 case Instruction::CondBr:
7432 case Instruction::IndirectBr:
7433 case Instruction::Switch:
7434 case Instruction::Unreachable:
7435 case Instruction::Fence:
7436 case Instruction::AtomicRMW:
7437 case Instruction::AtomicCmpXchg:
7438 case Instruction::LandingPad:
7439 case Instruction::Resume:
7440 case Instruction::CatchSwitch:
7441 case Instruction::CatchPad:
7442 case Instruction::CatchRet:
7443 case Instruction::CleanupPad:
7444 case Instruction::CleanupRet:
7445 return false; // Misc instructions which have effects
7446 }
7447}
7448
7449bool llvm::mayHaveNonDefUseDependency(const Instruction &I) {
7450 if (I.mayReadOrWriteMemory())
7451 // Memory dependency possible
7452 return true;
7453 if (!isSafeToSpeculativelyExecute(Inst: &I))
7454 // Can't move above a maythrow call or infinite loop. Or if an
7455 // inalloca alloca, above a stacksave call.
7456 return true;
7457 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7458 // 1) Can't reorder two inf-loop calls, even if readonly
7459 // 2) Also can't reorder an inf-loop call below a instruction which isn't
7460 // safe to speculative execute. (Inverse of above)
7461 return true;
7462 return false;
7463}
7464
7465/// Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
7466static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) {
7467 switch (OR) {
7468 case ConstantRange::OverflowResult::MayOverflow:
7469 return OverflowResult::MayOverflow;
7470 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
7471 return OverflowResult::AlwaysOverflowsLow;
7472 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
7473 return OverflowResult::AlwaysOverflowsHigh;
7474 case ConstantRange::OverflowResult::NeverOverflows:
7475 return OverflowResult::NeverOverflows;
7476 }
7477 llvm_unreachable("Unknown OverflowResult");
7478}
7479
7480/// Combine constant ranges from computeConstantRange() and computeKnownBits().
7481ConstantRange
7482llvm::computeConstantRangeIncludingKnownBits(const WithCache<const Value *> &V,
7483 bool ForSigned,
7484 const SimplifyQuery &SQ) {
7485 ConstantRange CR1 =
7486 ConstantRange::fromKnownBits(Known: V.getKnownBits(Q: SQ), IsSigned: ForSigned);
7487 ConstantRange CR2 = computeConstantRange(V, ForSigned, SQ);
7488 ConstantRange::PreferredRangeType RangeType =
7489 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
7490 return CR1.intersectWith(CR: CR2, Type: RangeType);
7491}
7492
7493OverflowResult llvm::computeOverflowForUnsignedMul(const Value *LHS,
7494 const Value *RHS,
7495 const SimplifyQuery &SQ,
7496 bool IsNSW) {
7497 ConstantRange LHSRange =
7498 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7499 ConstantRange RHSRange =
7500 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7501
7502 // mul nsw of two non-negative numbers is also nuw.
7503 if (IsNSW && LHSRange.isAllNonNegative() && RHSRange.isAllNonNegative())
7504 return OverflowResult::NeverOverflows;
7505
7506 return mapOverflowResult(OR: LHSRange.unsignedMulMayOverflow(Other: RHSRange));
7507}
7508
7509OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
7510 const Value *RHS,
7511 const SimplifyQuery &SQ) {
7512 // Multiplying n * m significant bits yields a result of n + m significant
7513 // bits. If the total number of significant bits does not exceed the
7514 // result bit width (minus 1), there is no overflow.
7515 // This means if we have enough leading sign bits in the operands
7516 // we can guarantee that the result does not overflow.
7517 // Ref: "Hacker's Delight" by Henry Warren
7518 unsigned BitWidth = LHS->getType()->getScalarSizeInBits();
7519
7520 // Note that underestimating the number of sign bits gives a more
7521 // conservative answer.
7522 unsigned SignBits =
7523 ::ComputeNumSignBits(V: LHS, Q: SQ) + ::ComputeNumSignBits(V: RHS, Q: SQ);
7524
7525 // First handle the easy case: if we have enough sign bits there's
7526 // definitely no overflow.
7527 if (SignBits > BitWidth + 1)
7528 return OverflowResult::NeverOverflows;
7529
7530 // There are two ambiguous cases where there can be no overflow:
7531 // SignBits == BitWidth + 1 and
7532 // SignBits == BitWidth
7533 // The second case is difficult to check, therefore we only handle the
7534 // first case.
7535 if (SignBits == BitWidth + 1) {
7536 // It overflows only when both arguments are negative and the true
7537 // product is exactly the minimum negative number.
7538 // E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
7539 // For simplicity we just check if at least one side is not negative.
7540 KnownBits LHSKnown = computeKnownBits(V: LHS, Q: SQ);
7541 KnownBits RHSKnown = computeKnownBits(V: RHS, Q: SQ);
7542 if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
7543 return OverflowResult::NeverOverflows;
7544 }
7545 return OverflowResult::MayOverflow;
7546}
7547
7548OverflowResult
7549llvm::computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
7550 const WithCache<const Value *> &RHS,
7551 const SimplifyQuery &SQ) {
7552 ConstantRange LHSRange =
7553 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7554 ConstantRange RHSRange =
7555 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7556 return mapOverflowResult(OR: LHSRange.unsignedAddMayOverflow(Other: RHSRange));
7557}
7558
7559static OverflowResult
7560computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
7561 const WithCache<const Value *> &RHS,
7562 const AddOperator *Add, const SimplifyQuery &SQ) {
7563 if (Add && Add->hasNoSignedWrap()) {
7564 return OverflowResult::NeverOverflows;
7565 }
7566
7567 // If LHS and RHS each have at least two sign bits, the addition will look
7568 // like
7569 //
7570 // XX..... +
7571 // YY.....
7572 //
7573 // If the carry into the most significant position is 0, X and Y can't both
7574 // be 1 and therefore the carry out of the addition is also 0.
7575 //
7576 // If the carry into the most significant position is 1, X and Y can't both
7577 // be 0 and therefore the carry out of the addition is also 1.
7578 //
7579 // Since the carry into the most significant position is always equal to
7580 // the carry out of the addition, there is no signed overflow.
7581 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7582 return OverflowResult::NeverOverflows;
7583
7584 ConstantRange LHSRange =
7585 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7586 ConstantRange RHSRange =
7587 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7588 OverflowResult OR =
7589 mapOverflowResult(OR: LHSRange.signedAddMayOverflow(Other: RHSRange));
7590 if (OR != OverflowResult::MayOverflow)
7591 return OR;
7592
7593 // The remaining code needs Add to be available. Early returns if not so.
7594 if (!Add)
7595 return OverflowResult::MayOverflow;
7596
7597 // If the sign of Add is the same as at least one of the operands, this add
7598 // CANNOT overflow. If this can be determined from the known bits of the
7599 // operands the above signedAddMayOverflow() check will have already done so.
7600 // The only other way to improve on the known bits is from an assumption, so
7601 // call computeKnownBitsFromContext() directly.
7602 bool LHSOrRHSKnownNonNegative =
7603 (LHSRange.isAllNonNegative() || RHSRange.isAllNonNegative());
7604 bool LHSOrRHSKnownNegative =
7605 (LHSRange.isAllNegative() || RHSRange.isAllNegative());
7606 if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) {
7607 KnownBits AddKnown(LHSRange.getBitWidth());
7608 computeKnownBitsFromContext(V: Add, Known&: AddKnown, Q: SQ);
7609 if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) ||
7610 (AddKnown.isNegative() && LHSOrRHSKnownNegative))
7611 return OverflowResult::NeverOverflows;
7612 }
7613
7614 return OverflowResult::MayOverflow;
7615}
7616
7617OverflowResult llvm::computeOverflowForUnsignedSub(const Value *LHS,
7618 const Value *RHS,
7619 const SimplifyQuery &SQ) {
7620 // X - (X % ?)
7621 // The remainder of a value can't have greater magnitude than itself,
7622 // so the subtraction can't overflow.
7623
7624 // X - (X -nuw ?)
7625 // In the minimal case, this would simplify to "?", so there's no subtract
7626 // at all. But if this analysis is used to peek through casts, for example,
7627 // then determining no-overflow may allow other transforms.
7628
7629 // TODO: There are other patterns like this.
7630 // See simplifyICmpWithBinOpOnLHS() for candidates.
7631 if (match(V: RHS, P: m_URem(L: m_Specific(V: LHS), R: m_Value())) ||
7632 match(V: RHS, P: m_NUWSub(L: m_Specific(V: LHS), R: m_Value())))
7633 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
7634 return OverflowResult::NeverOverflows;
7635
7636 if (auto C = isImpliedByDomCondition(Pred: CmpInst::ICMP_UGE, LHS, RHS, ContextI: SQ.CxtI,
7637 DL: SQ.DL)) {
7638 if (*C)
7639 return OverflowResult::NeverOverflows;
7640 return OverflowResult::AlwaysOverflowsLow;
7641 }
7642
7643 ConstantRange LHSRange =
7644 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7645 ConstantRange RHSRange =
7646 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7647 return mapOverflowResult(OR: LHSRange.unsignedSubMayOverflow(Other: RHSRange));
7648}
7649
7650OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS,
7651 const Value *RHS,
7652 const SimplifyQuery &SQ) {
7653 // X - (X % ?)
7654 // The remainder of a value can't have greater magnitude than itself,
7655 // so the subtraction can't overflow.
7656
7657 // X - (X -nsw ?)
7658 // In the minimal case, this would simplify to "?", so there's no subtract
7659 // at all. But if this analysis is used to peek through casts, for example,
7660 // then determining no-overflow may allow other transforms.
7661 if (match(V: RHS, P: m_SRem(L: m_Specific(V: LHS), R: m_Value())) ||
7662 match(V: RHS, P: m_NSWSub(L: m_Specific(V: LHS), R: m_Value())))
7663 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
7664 return OverflowResult::NeverOverflows;
7665
7666 // If LHS and RHS each have at least two sign bits, the subtraction
7667 // cannot overflow.
7668 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7669 return OverflowResult::NeverOverflows;
7670
7671 ConstantRange LHSRange =
7672 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7673 ConstantRange RHSRange =
7674 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7675 return mapOverflowResult(OR: LHSRange.signedSubMayOverflow(Other: RHSRange));
7676}
7677
7678bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO,
7679 const DominatorTree &DT) {
7680 SmallVector<const CondBrInst *, 2> GuardingBranches;
7681 SmallVector<const ExtractValueInst *, 2> Results;
7682
7683 for (const User *U : WO->users()) {
7684 if (const auto *EVI = dyn_cast<ExtractValueInst>(Val: U)) {
7685 assert(EVI->getNumIndices() == 1 && "Obvious from CI's type");
7686
7687 if (EVI->getIndices()[0] == 0)
7688 Results.push_back(Elt: EVI);
7689 else {
7690 assert(EVI->getIndices()[0] == 1 && "Obvious from CI's type");
7691
7692 for (const auto *U : EVI->users())
7693 if (const auto *B = dyn_cast<CondBrInst>(Val: U))
7694 GuardingBranches.push_back(Elt: B);
7695 }
7696 } else {
7697 // We are using the aggregate directly in a way we don't want to analyze
7698 // here (storing it to a global, say).
7699 return false;
7700 }
7701 }
7702
7703 auto AllUsesGuardedByBranch = [&](const CondBrInst *BI) {
7704 BasicBlockEdge NoWrapEdge(BI->getParent(), BI->getSuccessor(i: 1));
7705
7706 // Check if all users of the add are provably no-wrap.
7707 for (const auto *Result : Results) {
7708 // If the extractvalue itself is not executed on overflow, the we don't
7709 // need to check each use separately, since domination is transitive.
7710 if (DT.dominates(BBE: NoWrapEdge, BB: Result->getParent()))
7711 continue;
7712
7713 for (const auto &RU : Result->uses())
7714 if (!DT.dominates(BBE: NoWrapEdge, U: RU))
7715 return false;
7716 }
7717
7718 return true;
7719 };
7720
7721 return llvm::any_of(Range&: GuardingBranches, P: AllUsesGuardedByBranch);
7722}
7723
7724/// Shifts return poison if shiftwidth is larger than the bitwidth.
7725static bool shiftAmountKnownInRange(const Value *ShiftAmount) {
7726 auto *C = dyn_cast<Constant>(Val: ShiftAmount);
7727 if (!C)
7728 return false;
7729
7730 // Shifts return poison if shiftwidth is larger than the bitwidth.
7731 SmallVector<const Constant *, 4> ShiftAmounts;
7732 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: C->getType())) {
7733 unsigned NumElts = FVTy->getNumElements();
7734 for (unsigned i = 0; i < NumElts; ++i)
7735 ShiftAmounts.push_back(Elt: C->getAggregateElement(Elt: i));
7736 } else if (isa<ScalableVectorType>(Val: C->getType()))
7737 return false; // Can't tell, just return false to be safe
7738 else
7739 ShiftAmounts.push_back(Elt: C);
7740
7741 bool Safe = llvm::all_of(Range&: ShiftAmounts, P: [](const Constant *C) {
7742 auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
7743 return CI && CI->getValue().ult(RHS: C->getType()->getIntegerBitWidth());
7744 });
7745
7746 return Safe;
7747}
7748
7749static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
7750 bool ConsiderFlagsAndMetadata) {
7751
7752 if (ConsiderFlagsAndMetadata && includesPoison(Kind) &&
7753 Op->hasPoisonGeneratingAnnotations())
7754 return true;
7755
7756 unsigned Opcode = Op->getOpcode();
7757
7758 // Check whether opcode is a poison/undef-generating operation
7759 switch (Opcode) {
7760 case Instruction::Shl:
7761 case Instruction::AShr:
7762 case Instruction::LShr:
7763 return includesPoison(Kind) && !shiftAmountKnownInRange(ShiftAmount: Op->getOperand(i: 1));
7764 case Instruction::FPToSI:
7765 case Instruction::FPToUI:
7766 // fptosi/ui yields poison if the resulting value does not fit in the
7767 // destination type.
7768 return true;
7769 case Instruction::Call:
7770 if (auto *II = dyn_cast<IntrinsicInst>(Val: Op)) {
7771 switch (II->getIntrinsicID()) {
7772 // NOTE: Use IntrNoCreateUndefOrPoison when possible.
7773 case Intrinsic::ctlz:
7774 case Intrinsic::cttz:
7775 case Intrinsic::abs:
7776 // We're not considering flags so it is safe to just return false.
7777 return false;
7778 case Intrinsic::sshl_sat:
7779 case Intrinsic::ushl_sat:
7780 if (!includesPoison(Kind) ||
7781 shiftAmountKnownInRange(ShiftAmount: II->getArgOperand(i: 1)))
7782 return false;
7783 break;
7784 }
7785 }
7786 [[fallthrough]];
7787 case Instruction::CallBr:
7788 case Instruction::Invoke: {
7789 const auto *CB = cast<CallBase>(Val: Op);
7790 return !CB->hasRetAttr(Kind: Attribute::NoUndef) &&
7791 !CB->hasFnAttr(Kind: Attribute::NoCreateUndefOrPoison);
7792 }
7793 case Instruction::InsertElement:
7794 case Instruction::ExtractElement: {
7795 // If index exceeds the length of the vector, it returns poison
7796 auto *VTy = cast<VectorType>(Val: Op->getOperand(i: 0)->getType());
7797 unsigned IdxOp = Op->getOpcode() == Instruction::InsertElement ? 2 : 1;
7798 auto *Idx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: IdxOp));
7799 if (includesPoison(Kind))
7800 return !Idx ||
7801 Idx->getValue().uge(RHS: VTy->getElementCount().getKnownMinValue());
7802 return false;
7803 }
7804 case Instruction::ShuffleVector: {
7805 ArrayRef<int> Mask = isa<ConstantExpr>(Val: Op)
7806 ? cast<ConstantExpr>(Val: Op)->getShuffleMask()
7807 : cast<ShuffleVectorInst>(Val: Op)->getShuffleMask();
7808 return includesPoison(Kind) && is_contained(Range&: Mask, Element: PoisonMaskElem);
7809 }
7810 case Instruction::FNeg:
7811 case Instruction::PHI:
7812 case Instruction::Select:
7813 case Instruction::ExtractValue:
7814 case Instruction::InsertValue:
7815 case Instruction::Freeze:
7816 case Instruction::ICmp:
7817 case Instruction::FCmp:
7818 case Instruction::GetElementPtr:
7819 return false;
7820 case Instruction::AddrSpaceCast:
7821 return true;
7822 default: {
7823 const auto *CE = dyn_cast<ConstantExpr>(Val: Op);
7824 if (isa<CastInst>(Val: Op) || (CE && CE->isCast()))
7825 return false;
7826 else if (Instruction::isBinaryOp(Opcode))
7827 return false;
7828 // Be conservative and return true.
7829 return true;
7830 }
7831 }
7832}
7833
7834bool llvm::canCreateUndefOrPoison(const Operator *Op,
7835 bool ConsiderFlagsAndMetadata) {
7836 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::UndefOrPoison,
7837 ConsiderFlagsAndMetadata);
7838}
7839
7840bool llvm::canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata) {
7841 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::PoisonOnly,
7842 ConsiderFlagsAndMetadata);
7843}
7844
7845static bool directlyImpliesPoison(const Value *ValAssumedPoison, const Value *V,
7846 unsigned Depth) {
7847 if (ValAssumedPoison == V)
7848 return true;
7849
7850 const unsigned MaxDepth = 2;
7851 if (Depth >= MaxDepth)
7852 return false;
7853
7854 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
7855 if (any_of(Range: I->operands(), P: [=](const Use &Op) {
7856 return propagatesPoison(PoisonOp: Op) &&
7857 directlyImpliesPoison(ValAssumedPoison, V: Op, Depth: Depth + 1);
7858 }))
7859 return true;
7860
7861 // V = extractvalue V0, idx
7862 // V2 = extractvalue V0, idx2
7863 // V0's elements are all poison or not. (e.g., add_with_overflow)
7864 const WithOverflowInst *II;
7865 if (match(V: I, P: m_ExtractValue(V: m_WithOverflowInst(I&: II))) &&
7866 (match(V: ValAssumedPoison, P: m_ExtractValue(V: m_Specific(V: II))) ||
7867 llvm::is_contained(Range: II->args(), Element: ValAssumedPoison)))
7868 return true;
7869 }
7870 return false;
7871}
7872
7873static bool impliesPoison(const Value *ValAssumedPoison, const Value *V,
7874 unsigned Depth) {
7875 if (isGuaranteedNotToBePoison(V: ValAssumedPoison))
7876 return true;
7877
7878 if (directlyImpliesPoison(ValAssumedPoison, V, /* Depth */ 0))
7879 return true;
7880
7881 const unsigned MaxDepth = 2;
7882 if (Depth >= MaxDepth)
7883 return false;
7884
7885 const auto *I = dyn_cast<Instruction>(Val: ValAssumedPoison);
7886 if (I && !canCreatePoison(Op: cast<Operator>(Val: I))) {
7887 return all_of(Range: I->operands(), P: [=](const Value *Op) {
7888 return impliesPoison(ValAssumedPoison: Op, V, Depth: Depth + 1);
7889 });
7890 }
7891 return false;
7892}
7893
7894bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
7895 return ::impliesPoison(ValAssumedPoison, V, /* Depth */ 0);
7896}
7897
7898static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
7899
7900static bool isGuaranteedNotToBeUndefOrPoison(
7901 const Value *V, AssumptionCache *AC, const Instruction *CtxI,
7902 const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
7903 if (Depth >= MaxAnalysisRecursionDepth)
7904 return false;
7905
7906 if (isa<MetadataAsValue>(Val: V))
7907 return false;
7908
7909 if (const auto *A = dyn_cast<Argument>(Val: V)) {
7910 if (A->hasAttribute(Kind: Attribute::NoUndef) ||
7911 A->hasAttribute(Kind: Attribute::Dereferenceable) ||
7912 A->hasAttribute(Kind: Attribute::DereferenceableOrNull))
7913 return true;
7914 }
7915
7916 if (auto *C = dyn_cast<Constant>(Val: V)) {
7917 if (isa<PoisonValue>(Val: C))
7918 return !includesPoison(Kind);
7919
7920 if (isa<UndefValue>(Val: C))
7921 return !includesUndef(Kind);
7922
7923 if (isa<ConstantInt>(Val: C) || isa<GlobalVariable>(Val: C) || isa<ConstantFP>(Val: C) ||
7924 isa<ConstantPointerNull>(Val: C) || isa<Function>(Val: C))
7925 return true;
7926
7927 if (C->getType()->isVectorTy()) {
7928 if (isa<ConstantExpr>(Val: C)) {
7929 // Scalable vectors can use a ConstantExpr to build a splat.
7930 if (Constant *SplatC = C->getSplatValue())
7931 if (isa<ConstantInt>(Val: SplatC) || isa<ConstantFP>(Val: SplatC))
7932 return true;
7933 } else {
7934 if (includesUndef(Kind) && C->containsUndefElement())
7935 return false;
7936 if (includesPoison(Kind) && C->containsPoisonElement())
7937 return false;
7938 return !C->containsConstantExpression();
7939 }
7940 }
7941 }
7942
7943 // Strip cast operations from a pointer value.
7944 // Note that stripPointerCastsSameRepresentation can strip off getelementptr
7945 // inbounds with zero offset. To guarantee that the result isn't poison, the
7946 // stripped pointer is checked as it has to be pointing into an allocated
7947 // object or be null `null` to ensure `inbounds` getelement pointers with a
7948 // zero offset could not produce poison.
7949 // It can strip off addrspacecast that do not change bit representation as
7950 // well. We believe that such addrspacecast is equivalent to no-op.
7951 auto *StrippedV = V->stripPointerCastsSameRepresentation();
7952 if (isa<AllocaInst>(Val: StrippedV) || isa<GlobalVariable>(Val: StrippedV) ||
7953 isa<Function>(Val: StrippedV) || isa<ConstantPointerNull>(Val: StrippedV))
7954 return true;
7955
7956 auto OpCheck = [&](const Value *V) {
7957 return isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth: Depth + 1, Kind);
7958 };
7959
7960 if (auto *Opr = dyn_cast<Operator>(Val: V)) {
7961 // If the value is a freeze instruction, then it can never
7962 // be undef or poison.
7963 if (isa<FreezeInst>(Val: V))
7964 return true;
7965
7966 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
7967 if (CB->hasRetAttr(Kind: Attribute::NoUndef) ||
7968 CB->hasRetAttr(Kind: Attribute::Dereferenceable) ||
7969 CB->hasRetAttr(Kind: Attribute::DereferenceableOrNull))
7970 return true;
7971 }
7972
7973 if (!::canCreateUndefOrPoison(Op: Opr, Kind,
7974 /*ConsiderFlagsAndMetadata=*/true)) {
7975 if (const auto *PN = dyn_cast<PHINode>(Val: V)) {
7976 unsigned Num = PN->getNumIncomingValues();
7977 bool IsWellDefined = true;
7978 for (unsigned i = 0; i < Num; ++i) {
7979 if (PN == PN->getIncomingValue(i))
7980 continue;
7981 auto *TI = PN->getIncomingBlock(i)->getTerminator();
7982 if (!isGuaranteedNotToBeUndefOrPoison(V: PN->getIncomingValue(i), AC, CtxI: TI,
7983 DT, Depth: Depth + 1, Kind)) {
7984 IsWellDefined = false;
7985 break;
7986 }
7987 }
7988 if (IsWellDefined)
7989 return true;
7990 } else if (auto *Splat = isa<ShuffleVectorInst>(Val: Opr) ? getSplatValue(V: Opr)
7991 : nullptr) {
7992 // For splats we only need to check the value being splatted.
7993 if (OpCheck(Splat))
7994 return true;
7995 } else if (all_of(Range: Opr->operands(), P: OpCheck))
7996 return true;
7997 }
7998 }
7999
8000 if (auto *I = dyn_cast<LoadInst>(Val: V))
8001 if (I->hasMetadata(KindID: LLVMContext::MD_noundef) ||
8002 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable) ||
8003 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable_or_null))
8004 return true;
8005
8006 if (programUndefinedIfUndefOrPoison(V, PoisonOnly: !includesUndef(Kind)))
8007 return true;
8008
8009 // CxtI may be null or a cloned instruction.
8010 if (!CtxI || !CtxI->getParent() || !DT)
8011 return false;
8012
8013 auto *DNode = DT->getNode(BB: CtxI->getParent());
8014 if (!DNode)
8015 // Unreachable block
8016 return false;
8017
8018 // If V is used as a branch condition before reaching CtxI, V cannot be
8019 // undef or poison.
8020 // br V, BB1, BB2
8021 // BB1:
8022 // CtxI ; V cannot be undef or poison here
8023 auto *Dominator = DNode->getIDom();
8024 // This check is purely for compile time reasons: we can skip the IDom walk
8025 // if what we are checking for includes undef and the value is not an integer.
8026 if (!includesUndef(Kind) || V->getType()->isIntegerTy())
8027 while (Dominator) {
8028 auto *TI = Dominator->getBlock()->getTerminatorOrNull();
8029
8030 Value *Cond = nullptr;
8031 if (auto BI = dyn_cast_or_null<CondBrInst>(Val: TI)) {
8032 Cond = BI->getCondition();
8033 } else if (auto SI = dyn_cast_or_null<SwitchInst>(Val: TI)) {
8034 Cond = SI->getCondition();
8035 }
8036
8037 if (Cond) {
8038 if (Cond == V)
8039 return true;
8040 else if (!includesUndef(Kind) && isa<Operator>(Val: Cond)) {
8041 // For poison, we can analyze further
8042 auto *Opr = cast<Operator>(Val: Cond);
8043 if (any_of(Range: Opr->operands(), P: [V](const Use &U) {
8044 return V == U && propagatesPoison(PoisonOp: U);
8045 }))
8046 return true;
8047 }
8048 }
8049
8050 Dominator = Dominator->getIDom();
8051 }
8052
8053 if (AC && getKnowledgeValidInContext(V, AttrKinds: {Attribute::NoUndef}, AC&: *AC, CtxI, DT))
8054 return true;
8055
8056 return false;
8057}
8058
8059bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC,
8060 const Instruction *CtxI,
8061 const DominatorTree *DT,
8062 unsigned Depth) {
8063 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8064 Kind: UndefPoisonKind::UndefOrPoison);
8065}
8066
8067bool llvm::isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC,
8068 const Instruction *CtxI,
8069 const DominatorTree *DT, unsigned Depth) {
8070 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8071 Kind: UndefPoisonKind::PoisonOnly);
8072}
8073
8074bool llvm::isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC,
8075 const Instruction *CtxI,
8076 const DominatorTree *DT, unsigned Depth) {
8077 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8078 Kind: UndefPoisonKind::UndefOnly);
8079}
8080
8081/// Return true if undefined behavior would provably be executed on the path to
8082/// OnPathTo if Root produced a posion result. Note that this doesn't say
8083/// anything about whether OnPathTo is actually executed or whether Root is
8084/// actually poison. This can be used to assess whether a new use of Root can
8085/// be added at a location which is control equivalent with OnPathTo (such as
8086/// immediately before it) without introducing UB which didn't previously
8087/// exist. Note that a false result conveys no information.
8088bool llvm::mustExecuteUBIfPoisonOnPathTo(Instruction *Root,
8089 Instruction *OnPathTo,
8090 DominatorTree *DT) {
8091 // Basic approach is to assume Root is poison, propagate poison forward
8092 // through all users we can easily track, and then check whether any of those
8093 // users are provable UB and must execute before out exiting block might
8094 // exit.
8095
8096 // The set of all recursive users we've visited (which are assumed to all be
8097 // poison because of said visit)
8098 SmallPtrSet<const Value *, 16> KnownPoison;
8099 SmallVector<const Instruction*, 16> Worklist;
8100 Worklist.push_back(Elt: Root);
8101 while (!Worklist.empty()) {
8102 const Instruction *I = Worklist.pop_back_val();
8103
8104 // If we know this must trigger UB on a path leading our target.
8105 if (mustTriggerUB(I, KnownPoison) && DT->dominates(Def: I, User: OnPathTo))
8106 return true;
8107
8108 // If we can't analyze propagation through this instruction, just skip it
8109 // and transitive users. Safe as false is a conservative result.
8110 if (I != Root && !any_of(Range: I->operands(), P: [&KnownPoison](const Use &U) {
8111 return KnownPoison.contains(Ptr: U) && propagatesPoison(PoisonOp: U);
8112 }))
8113 continue;
8114
8115 if (KnownPoison.insert(Ptr: I).second)
8116 for (const User *User : I->users())
8117 Worklist.push_back(Elt: cast<Instruction>(Val: User));
8118 }
8119
8120 // Might be non-UB, or might have a path we couldn't prove must execute on
8121 // way to exiting bb.
8122 return false;
8123}
8124
8125OverflowResult llvm::computeOverflowForSignedAdd(const AddOperator *Add,
8126 const SimplifyQuery &SQ) {
8127 return ::computeOverflowForSignedAdd(LHS: Add->getOperand(i_nocapture: 0), RHS: Add->getOperand(i_nocapture: 1),
8128 Add, SQ);
8129}
8130
8131OverflowResult
8132llvm::computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
8133 const WithCache<const Value *> &RHS,
8134 const SimplifyQuery &SQ) {
8135 return ::computeOverflowForSignedAdd(LHS, RHS, Add: nullptr, SQ);
8136}
8137
8138bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
8139 // Note: An atomic operation isn't guaranteed to return in a reasonable amount
8140 // of time because it's possible for another thread to interfere with it for an
8141 // arbitrary length of time, but programs aren't allowed to rely on that.
8142
8143 // If there is no successor, then execution can't transfer to it.
8144 if (isa<ReturnInst>(Val: I))
8145 return false;
8146 if (isa<UnreachableInst>(Val: I))
8147 return false;
8148
8149 // Note: Do not add new checks here; instead, change Instruction::mayThrow or
8150 // Instruction::willReturn.
8151 //
8152 // FIXME: Move this check into Instruction::willReturn.
8153 if (isa<CatchPadInst>(Val: I)) {
8154 switch (classifyEHPersonality(Pers: I->getFunction()->getPersonalityFn())) {
8155 default:
8156 // A catchpad may invoke exception object constructors and such, which
8157 // in some languages can be arbitrary code, so be conservative by default.
8158 return false;
8159 case EHPersonality::CoreCLR:
8160 // For CoreCLR, it just involves a type test.
8161 return true;
8162 }
8163 }
8164
8165 // An instruction that returns without throwing must transfer control flow
8166 // to a successor.
8167 return !I->mayThrow() && I->willReturn();
8168}
8169
8170bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) {
8171 // TODO: This is slightly conservative for invoke instruction since exiting
8172 // via an exception *is* normal control for them.
8173 for (const Instruction &I : *BB)
8174 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8175 return false;
8176 return true;
8177}
8178
8179bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8180 BasicBlock::const_iterator Begin, BasicBlock::const_iterator End,
8181 unsigned ScanLimit) {
8182 return isGuaranteedToTransferExecutionToSuccessor(Range: make_range(x: Begin, y: End),
8183 ScanLimit);
8184}
8185
8186bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8187 iterator_range<BasicBlock::const_iterator> Range, unsigned ScanLimit) {
8188 assert(ScanLimit && "scan limit must be non-zero");
8189 for (const Instruction &I : Range) {
8190 if (--ScanLimit == 0)
8191 return false;
8192 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8193 return false;
8194 }
8195 return true;
8196}
8197
8198bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
8199 const Loop *L) {
8200 // The loop header is guaranteed to be executed for every iteration.
8201 //
8202 // FIXME: Relax this constraint to cover all basic blocks that are
8203 // guaranteed to be executed at every iteration.
8204 if (I->getParent() != L->getHeader()) return false;
8205
8206 for (const Instruction &LI : *L->getHeader()) {
8207 if (&LI == I) return true;
8208 if (!isGuaranteedToTransferExecutionToSuccessor(I: &LI)) return false;
8209 }
8210 llvm_unreachable("Instruction not contained in its own parent basic block.");
8211}
8212
8213bool llvm::intrinsicPropagatesPoison(Intrinsic::ID IID) {
8214 switch (IID) {
8215 // TODO: Add more intrinsics.
8216 case Intrinsic::sadd_with_overflow:
8217 case Intrinsic::ssub_with_overflow:
8218 case Intrinsic::smul_with_overflow:
8219 case Intrinsic::uadd_with_overflow:
8220 case Intrinsic::usub_with_overflow:
8221 case Intrinsic::umul_with_overflow:
8222 // If an input is a vector containing a poison element, the
8223 // two output vectors (calculated results, overflow bits)'
8224 // corresponding lanes are poison.
8225 return true;
8226 case Intrinsic::ctpop:
8227 case Intrinsic::ctlz:
8228 case Intrinsic::cttz:
8229 case Intrinsic::abs:
8230 case Intrinsic::smax:
8231 case Intrinsic::smin:
8232 case Intrinsic::umax:
8233 case Intrinsic::umin:
8234 case Intrinsic::scmp:
8235 case Intrinsic::is_fpclass:
8236 case Intrinsic::ptrmask:
8237 case Intrinsic::ucmp:
8238 case Intrinsic::bitreverse:
8239 case Intrinsic::bswap:
8240 case Intrinsic::sadd_sat:
8241 case Intrinsic::ssub_sat:
8242 case Intrinsic::sshl_sat:
8243 case Intrinsic::uadd_sat:
8244 case Intrinsic::usub_sat:
8245 case Intrinsic::ushl_sat:
8246 case Intrinsic::smul_fix:
8247 case Intrinsic::smul_fix_sat:
8248 case Intrinsic::umul_fix:
8249 case Intrinsic::umul_fix_sat:
8250 case Intrinsic::pow:
8251 case Intrinsic::powi:
8252 case Intrinsic::sin:
8253 case Intrinsic::sinh:
8254 case Intrinsic::cos:
8255 case Intrinsic::cosh:
8256 case Intrinsic::sincos:
8257 case Intrinsic::sincospi:
8258 case Intrinsic::tan:
8259 case Intrinsic::tanh:
8260 case Intrinsic::asin:
8261 case Intrinsic::acos:
8262 case Intrinsic::atan:
8263 case Intrinsic::atan2:
8264 case Intrinsic::canonicalize:
8265 case Intrinsic::sqrt:
8266 case Intrinsic::exp:
8267 case Intrinsic::exp2:
8268 case Intrinsic::exp10:
8269 case Intrinsic::log:
8270 case Intrinsic::log2:
8271 case Intrinsic::log10:
8272 case Intrinsic::modf:
8273 case Intrinsic::floor:
8274 case Intrinsic::ceil:
8275 case Intrinsic::trunc:
8276 case Intrinsic::rint:
8277 case Intrinsic::nearbyint:
8278 case Intrinsic::round:
8279 case Intrinsic::roundeven:
8280 case Intrinsic::lrint:
8281 case Intrinsic::llrint:
8282 case Intrinsic::fshl:
8283 case Intrinsic::fshr:
8284 case Intrinsic::frexp:
8285 case Intrinsic::get_active_lane_mask:
8286 return true;
8287 default:
8288 return false;
8289 }
8290}
8291
8292bool llvm::propagatesPoison(const Use &PoisonOp) {
8293 const Operator *I = cast<Operator>(Val: PoisonOp.getUser());
8294 switch (I->getOpcode()) {
8295 case Instruction::Freeze:
8296 case Instruction::PHI:
8297 case Instruction::Invoke:
8298 return false;
8299 case Instruction::Select:
8300 return PoisonOp.getOperandNo() == 0;
8301 case Instruction::Call:
8302 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
8303 return intrinsicPropagatesPoison(IID: II->getIntrinsicID());
8304 return false;
8305 case Instruction::ICmp:
8306 case Instruction::FCmp:
8307 case Instruction::GetElementPtr:
8308 return true;
8309 default:
8310 if (isa<BinaryOperator>(Val: I) || isa<UnaryOperator>(Val: I) || isa<CastInst>(Val: I))
8311 return true;
8312
8313 // Be conservative and return false.
8314 return false;
8315 }
8316}
8317
8318/// Enumerates all operands of \p I that are guaranteed to not be undef or
8319/// poison. If the callback \p Handle returns true, stop processing and return
8320/// true. Otherwise, return false.
8321template <typename CallableT>
8322static bool handleGuaranteedWellDefinedOps(const Instruction *I,
8323 const CallableT &Handle) {
8324 switch (I->getOpcode()) {
8325 case Instruction::Store:
8326 if (Handle(cast<StoreInst>(Val: I)->getPointerOperand()))
8327 return true;
8328 break;
8329
8330 case Instruction::Load:
8331 if (Handle(cast<LoadInst>(Val: I)->getPointerOperand()))
8332 return true;
8333 break;
8334
8335 // Since dereferenceable attribute imply noundef, atomic operations
8336 // also implicitly have noundef pointers too
8337 case Instruction::AtomicCmpXchg:
8338 if (Handle(cast<AtomicCmpXchgInst>(Val: I)->getPointerOperand()))
8339 return true;
8340 break;
8341
8342 case Instruction::AtomicRMW:
8343 if (Handle(cast<AtomicRMWInst>(Val: I)->getPointerOperand()))
8344 return true;
8345 break;
8346
8347 case Instruction::Call:
8348 case Instruction::Invoke: {
8349 const CallBase *CB = cast<CallBase>(Val: I);
8350 if (CB->isIndirectCall() && Handle(CB->getCalledOperand()))
8351 return true;
8352 for (unsigned i = 0; i < CB->arg_size(); ++i)
8353 if ((CB->paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef) ||
8354 CB->paramHasAttr(ArgNo: i, Kind: Attribute::Dereferenceable) ||
8355 CB->paramHasAttr(ArgNo: i, Kind: Attribute::DereferenceableOrNull)) &&
8356 Handle(CB->getArgOperand(i)))
8357 return true;
8358 break;
8359 }
8360 case Instruction::Ret:
8361 if (I->getFunction()->hasRetAttribute(Kind: Attribute::NoUndef) &&
8362 Handle(I->getOperand(i: 0)))
8363 return true;
8364 break;
8365 case Instruction::Switch:
8366 if (Handle(cast<SwitchInst>(Val: I)->getCondition()))
8367 return true;
8368 break;
8369 case Instruction::CondBr:
8370 if (Handle(cast<CondBrInst>(Val: I)->getCondition()))
8371 return true;
8372 break;
8373 default:
8374 break;
8375 }
8376
8377 return false;
8378}
8379
8380/// Enumerates all operands of \p I that are guaranteed to not be poison.
8381template <typename CallableT>
8382static bool handleGuaranteedNonPoisonOps(const Instruction *I,
8383 const CallableT &Handle) {
8384 if (handleGuaranteedWellDefinedOps(I, Handle))
8385 return true;
8386 switch (I->getOpcode()) {
8387 // Divisors of these operations are allowed to be partially undef.
8388 case Instruction::UDiv:
8389 case Instruction::SDiv:
8390 case Instruction::URem:
8391 case Instruction::SRem:
8392 return Handle(I->getOperand(i: 1));
8393 default:
8394 return false;
8395 }
8396}
8397
8398bool llvm::mustTriggerUB(const Instruction *I,
8399 const SmallPtrSetImpl<const Value *> &KnownPoison) {
8400 return handleGuaranteedNonPoisonOps(
8401 I, Handle: [&](const Value *V) { return KnownPoison.count(Ptr: V); });
8402}
8403
8404static bool programUndefinedIfUndefOrPoison(const Value *V,
8405 bool PoisonOnly) {
8406 // We currently only look for uses of values within the same basic
8407 // block, as that makes it easier to guarantee that the uses will be
8408 // executed given that Inst is executed.
8409 //
8410 // FIXME: Expand this to consider uses beyond the same basic block. To do
8411 // this, look out for the distinction between post-dominance and strong
8412 // post-dominance.
8413 const BasicBlock *BB = nullptr;
8414 BasicBlock::const_iterator Begin;
8415 if (const auto *Inst = dyn_cast<Instruction>(Val: V)) {
8416 BB = Inst->getParent();
8417 Begin = Inst->getIterator();
8418 Begin++;
8419 } else if (const auto *Arg = dyn_cast<Argument>(Val: V)) {
8420 if (Arg->getParent()->isDeclaration())
8421 return false;
8422 BB = &Arg->getParent()->getEntryBlock();
8423 Begin = BB->begin();
8424 } else {
8425 return false;
8426 }
8427
8428 // Limit number of instructions we look at, to avoid scanning through large
8429 // blocks. The current limit is chosen arbitrarily.
8430 unsigned ScanLimit = 32;
8431 BasicBlock::const_iterator End = BB->end();
8432
8433 if (!PoisonOnly) {
8434 // Since undef does not propagate eagerly, be conservative & just check
8435 // whether a value is directly passed to an instruction that must take
8436 // well-defined operands.
8437
8438 for (const auto &I : make_range(x: Begin, y: End)) {
8439 if (--ScanLimit == 0)
8440 break;
8441
8442 if (handleGuaranteedWellDefinedOps(I: &I, Handle: [V](const Value *WellDefinedOp) {
8443 return WellDefinedOp == V;
8444 }))
8445 return true;
8446
8447 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8448 break;
8449 }
8450 return false;
8451 }
8452
8453 // Set of instructions that we have proved will yield poison if Inst
8454 // does.
8455 SmallPtrSet<const Value *, 16> YieldsPoison;
8456 SmallPtrSet<const BasicBlock *, 4> Visited;
8457
8458 YieldsPoison.insert(Ptr: V);
8459 Visited.insert(Ptr: BB);
8460
8461 while (true) {
8462 for (const auto &I : make_range(x: Begin, y: End)) {
8463 if (--ScanLimit == 0)
8464 return false;
8465 if (mustTriggerUB(I: &I, KnownPoison: YieldsPoison))
8466 return true;
8467 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8468 return false;
8469
8470 // If an operand is poison and propagates it, mark I as yielding poison.
8471 for (const Use &Op : I.operands()) {
8472 if (YieldsPoison.count(Ptr: Op) && propagatesPoison(PoisonOp: Op)) {
8473 YieldsPoison.insert(Ptr: &I);
8474 break;
8475 }
8476 }
8477
8478 // Special handling for select, which returns poison if its operand 0 is
8479 // poison (handled in the loop above) *or* if both its true/false operands
8480 // are poison (handled here).
8481 if (I.getOpcode() == Instruction::Select &&
8482 YieldsPoison.count(Ptr: I.getOperand(i: 1)) &&
8483 YieldsPoison.count(Ptr: I.getOperand(i: 2))) {
8484 YieldsPoison.insert(Ptr: &I);
8485 }
8486 }
8487
8488 BB = BB->getSingleSuccessor();
8489 if (!BB || !Visited.insert(Ptr: BB).second)
8490 break;
8491
8492 Begin = BB->getFirstNonPHIIt();
8493 End = BB->end();
8494 }
8495 return false;
8496}
8497
8498bool llvm::programUndefinedIfUndefOrPoison(const Instruction *Inst) {
8499 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: false);
8500}
8501
8502bool llvm::programUndefinedIfPoison(const Instruction *Inst) {
8503 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: true);
8504}
8505
8506static bool isKnownNonNaN(const Value *V, FastMathFlags FMF) {
8507 if (FMF.noNaNs())
8508 return true;
8509
8510 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8511 return !C->isNaN();
8512
8513 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8514 if (!C->getElementType()->isFloatingPointTy())
8515 return false;
8516 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8517 if (C->getElementAsAPFloat(i: I).isNaN())
8518 return false;
8519 }
8520 return true;
8521 }
8522
8523 if (isa<ConstantAggregateZero>(Val: V))
8524 return true;
8525
8526 return false;
8527}
8528
8529static bool isKnownNonZero(const Value *V) {
8530 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8531 return !C->isZero();
8532
8533 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8534 if (!C->getElementType()->isFloatingPointTy())
8535 return false;
8536 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8537 if (C->getElementAsAPFloat(i: I).isZero())
8538 return false;
8539 }
8540 return true;
8541 }
8542
8543 return false;
8544}
8545
8546/// Match clamp pattern for float types without care about NaNs or signed zeros.
8547/// Given non-min/max outer cmp/select from the clamp pattern this
8548/// function recognizes if it can be substitued by a "canonical" min/max
8549/// pattern.
8550static SelectPatternResult matchFastFloatClamp(CmpInst::Predicate Pred,
8551 Value *CmpLHS, Value *CmpRHS,
8552 Value *TrueVal, Value *FalseVal,
8553 Value *&LHS, Value *&RHS) {
8554 // Try to match
8555 // X < C1 ? C1 : Min(X, C2) --> Max(C1, Min(X, C2))
8556 // X > C1 ? C1 : Max(X, C2) --> Min(C1, Max(X, C2))
8557 // and return description of the outer Max/Min.
8558
8559 // First, check if select has inverse order:
8560 if (CmpRHS == FalseVal) {
8561 std::swap(a&: TrueVal, b&: FalseVal);
8562 Pred = CmpInst::getInversePredicate(pred: Pred);
8563 }
8564
8565 // Assume success now. If there's no match, callers should not use these anyway.
8566 LHS = TrueVal;
8567 RHS = FalseVal;
8568
8569 const APFloat *FC1;
8570 if (CmpRHS != TrueVal || !match(V: CmpRHS, P: m_APFloat(Res&: FC1)) || !FC1->isFinite())
8571 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8572
8573 const APFloat *FC2;
8574 switch (Pred) {
8575 case CmpInst::FCMP_OLT:
8576 case CmpInst::FCMP_OLE:
8577 case CmpInst::FCMP_ULT:
8578 case CmpInst::FCMP_ULE:
8579 if (match(V: FalseVal, P: m_OrdOrUnordFMin(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8580 *FC1 < *FC2)
8581 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8582 if (match(V: FalseVal, P: m_FMinNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
8583 *FC1 < *FC2)
8584 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8585 break;
8586 case CmpInst::FCMP_OGT:
8587 case CmpInst::FCMP_OGE:
8588 case CmpInst::FCMP_UGT:
8589 case CmpInst::FCMP_UGE:
8590 if (match(V: FalseVal, P: m_OrdOrUnordFMax(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8591 *FC1 > *FC2)
8592 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8593 if (match(V: FalseVal, P: m_FMaxNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
8594 *FC1 > *FC2)
8595 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8596 break;
8597 default:
8598 break;
8599 }
8600
8601 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8602}
8603
8604/// Recognize variations of:
8605/// CLAMP(v,l,h) ==> ((v) < (l) ? (l) : ((v) > (h) ? (h) : (v)))
8606static SelectPatternResult matchClamp(CmpInst::Predicate Pred,
8607 Value *CmpLHS, Value *CmpRHS,
8608 Value *TrueVal, Value *FalseVal) {
8609 // Swap the select operands and predicate to match the patterns below.
8610 if (CmpRHS != TrueVal) {
8611 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8612 std::swap(a&: TrueVal, b&: FalseVal);
8613 }
8614 const APInt *C1;
8615 if (CmpRHS == TrueVal && match(V: CmpRHS, P: m_APInt(Res&: C1))) {
8616 const APInt *C2;
8617 // (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
8618 if (match(V: FalseVal, P: m_SMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8619 C1->slt(RHS: *C2) && Pred == CmpInst::ICMP_SLT)
8620 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8621
8622 // (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
8623 if (match(V: FalseVal, P: m_SMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8624 C1->sgt(RHS: *C2) && Pred == CmpInst::ICMP_SGT)
8625 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8626
8627 // (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
8628 if (match(V: FalseVal, P: m_UMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8629 C1->ult(RHS: *C2) && Pred == CmpInst::ICMP_ULT)
8630 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8631
8632 // (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
8633 if (match(V: FalseVal, P: m_UMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8634 C1->ugt(RHS: *C2) && Pred == CmpInst::ICMP_UGT)
8635 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8636 }
8637 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8638}
8639
8640/// Recognize variations of:
8641/// a < c ? min(a,b) : min(b,c) ==> min(min(a,b),min(b,c))
8642static SelectPatternResult matchMinMaxOfMinMax(CmpInst::Predicate Pred,
8643 Value *CmpLHS, Value *CmpRHS,
8644 Value *TVal, Value *FVal,
8645 unsigned Depth) {
8646 // TODO: Allow FP min/max with nnan/nsz.
8647 assert(CmpInst::isIntPredicate(Pred) && "Expected integer comparison");
8648
8649 Value *A = nullptr, *B = nullptr;
8650 SelectPatternResult L = matchSelectPattern(V: TVal, LHS&: A, RHS&: B, CastOp: nullptr, Depth: Depth + 1);
8651 if (!SelectPatternResult::isMinOrMax(SPF: L.Flavor))
8652 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8653
8654 Value *C = nullptr, *D = nullptr;
8655 SelectPatternResult R = matchSelectPattern(V: FVal, LHS&: C, RHS&: D, CastOp: nullptr, Depth: Depth + 1);
8656 if (L.Flavor != R.Flavor)
8657 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8658
8659 // We have something like: x Pred y ? min(a, b) : min(c, d).
8660 // Try to match the compare to the min/max operations of the select operands.
8661 // First, make sure we have the right compare predicate.
8662 switch (L.Flavor) {
8663 case SPF_SMIN:
8664 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) {
8665 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8666 std::swap(a&: CmpLHS, b&: CmpRHS);
8667 }
8668 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
8669 break;
8670 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8671 case SPF_SMAX:
8672 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) {
8673 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8674 std::swap(a&: CmpLHS, b&: CmpRHS);
8675 }
8676 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)
8677 break;
8678 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8679 case SPF_UMIN:
8680 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
8681 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8682 std::swap(a&: CmpLHS, b&: CmpRHS);
8683 }
8684 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
8685 break;
8686 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8687 case SPF_UMAX:
8688 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
8689 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8690 std::swap(a&: CmpLHS, b&: CmpRHS);
8691 }
8692 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
8693 break;
8694 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8695 default:
8696 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8697 }
8698
8699 // If there is a common operand in the already matched min/max and the other
8700 // min/max operands match the compare operands (either directly or inverted),
8701 // then this is min/max of the same flavor.
8702
8703 // a pred c ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
8704 // ~c pred ~a ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
8705 if (D == B) {
8706 if ((CmpLHS == A && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8707 match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
8708 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8709 }
8710 // a pred d ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
8711 // ~d pred ~a ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
8712 if (C == B) {
8713 if ((CmpLHS == A && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8714 match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
8715 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8716 }
8717 // b pred c ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
8718 // ~c pred ~b ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
8719 if (D == A) {
8720 if ((CmpLHS == B && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8721 match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
8722 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8723 }
8724 // b pred d ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
8725 // ~d pred ~b ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
8726 if (C == A) {
8727 if ((CmpLHS == B && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8728 match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
8729 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8730 }
8731
8732 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8733}
8734
8735/// If the input value is the result of a 'not' op, constant integer, or vector
8736/// splat of a constant integer, return the bitwise-not source value.
8737/// TODO: This could be extended to handle non-splat vector integer constants.
8738static Value *getNotValue(Value *V) {
8739 Value *NotV;
8740 if (match(V, P: m_Not(V: m_Value(V&: NotV))))
8741 return NotV;
8742
8743 const APInt *C;
8744 if (match(V, P: m_APInt(Res&: C)))
8745 return ConstantInt::get(Ty: V->getType(), V: ~(*C));
8746
8747 return nullptr;
8748}
8749
8750/// Match non-obvious integer minimum and maximum sequences.
8751static SelectPatternResult matchMinMax(CmpInst::Predicate Pred,
8752 Value *CmpLHS, Value *CmpRHS,
8753 Value *TrueVal, Value *FalseVal,
8754 Value *&LHS, Value *&RHS,
8755 unsigned Depth) {
8756 // Assume success. If there's no match, callers should not use these anyway.
8757 LHS = TrueVal;
8758 RHS = FalseVal;
8759
8760 SelectPatternResult SPR = matchClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal);
8761 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
8762 return SPR;
8763
8764 SPR = matchMinMaxOfMinMax(Pred, CmpLHS, CmpRHS, TVal: TrueVal, FVal: FalseVal, Depth);
8765 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
8766 return SPR;
8767
8768 // Look through 'not' ops to find disguised min/max.
8769 // (X > Y) ? ~X : ~Y ==> (~X < ~Y) ? ~X : ~Y ==> MIN(~X, ~Y)
8770 // (X < Y) ? ~X : ~Y ==> (~X > ~Y) ? ~X : ~Y ==> MAX(~X, ~Y)
8771 if (CmpLHS == getNotValue(V: TrueVal) && CmpRHS == getNotValue(V: FalseVal)) {
8772 switch (Pred) {
8773 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8774 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8775 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8776 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8777 default: break;
8778 }
8779 }
8780
8781 // (X > Y) ? ~Y : ~X ==> (~X < ~Y) ? ~Y : ~X ==> MAX(~Y, ~X)
8782 // (X < Y) ? ~Y : ~X ==> (~X > ~Y) ? ~Y : ~X ==> MIN(~Y, ~X)
8783 if (CmpLHS == getNotValue(V: FalseVal) && CmpRHS == getNotValue(V: TrueVal)) {
8784 switch (Pred) {
8785 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8786 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8787 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8788 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8789 default: break;
8790 }
8791 }
8792
8793 if (Pred != CmpInst::ICMP_SGT && Pred != CmpInst::ICMP_SLT)
8794 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8795
8796 const APInt *C1;
8797 if (!match(V: CmpRHS, P: m_APInt(Res&: C1)))
8798 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8799
8800 // An unsigned min/max can be written with a signed compare.
8801 const APInt *C2;
8802 if ((CmpLHS == TrueVal && match(V: FalseVal, P: m_APInt(Res&: C2))) ||
8803 (CmpLHS == FalseVal && match(V: TrueVal, P: m_APInt(Res&: C2)))) {
8804 // Is the sign bit set?
8805 // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
8806 // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
8807 if (Pred == CmpInst::ICMP_SLT && C1->isZero() && C2->isMaxSignedValue())
8808 return {.Flavor: CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8809
8810 // Is the sign bit clear?
8811 // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
8812 // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
8813 if (Pred == CmpInst::ICMP_SGT && C1->isAllOnes() && C2->isMinSignedValue())
8814 return {.Flavor: CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8815 }
8816
8817 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8818}
8819
8820bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW,
8821 bool AllowPoison) {
8822 assert(X && Y && "Invalid operand");
8823
8824 auto IsNegationOf = [&](const Value *X, const Value *Y) {
8825 if (!match(V: X, P: m_Neg(V: m_Specific(V: Y))))
8826 return false;
8827
8828 auto *BO = cast<BinaryOperator>(Val: X);
8829 if (NeedNSW && !BO->hasNoSignedWrap())
8830 return false;
8831
8832 auto *Zero = cast<Constant>(Val: BO->getOperand(i_nocapture: 0));
8833 if (!AllowPoison && !Zero->isNullValue())
8834 return false;
8835
8836 return true;
8837 };
8838
8839 // X = -Y or Y = -X
8840 if (IsNegationOf(X, Y) || IsNegationOf(Y, X))
8841 return true;
8842
8843 // X = sub (A, B), Y = sub (B, A) || X = sub nsw (A, B), Y = sub nsw (B, A)
8844 Value *A, *B;
8845 return (!NeedNSW && (match(V: X, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
8846 match(V: Y, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))))) ||
8847 (NeedNSW && (match(V: X, P: m_NSWSub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
8848 match(V: Y, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A)))));
8849}
8850
8851bool llvm::isKnownInversion(const Value *X, const Value *Y) {
8852 // Handle X = icmp pred A, B, Y = icmp pred A, C.
8853 Value *A, *B, *C;
8854 CmpPredicate Pred1, Pred2;
8855 if (!match(V: X, P: m_ICmp(Pred&: Pred1, L: m_Value(V&: A), R: m_Value(V&: B))) ||
8856 !match(V: Y, P: m_c_ICmp(Pred&: Pred2, L: m_Specific(V: A), R: m_Value(V&: C))))
8857 return false;
8858
8859 // They must both have samesign flag or not.
8860 if (Pred1.hasSameSign() != Pred2.hasSameSign())
8861 return false;
8862
8863 if (B == C)
8864 return Pred1 == ICmpInst::getInversePredicate(pred: Pred2);
8865
8866 // Try to infer the relationship from constant ranges.
8867 const APInt *RHSC1, *RHSC2;
8868 if (!match(V: B, P: m_APInt(Res&: RHSC1)) || !match(V: C, P: m_APInt(Res&: RHSC2)))
8869 return false;
8870
8871 // Sign bits of two RHSCs should match.
8872 if (Pred1.hasSameSign() && RHSC1->isNonNegative() != RHSC2->isNonNegative())
8873 return false;
8874
8875 const auto CR1 = ConstantRange::makeExactICmpRegion(Pred: Pred1, Other: *RHSC1);
8876 const auto CR2 = ConstantRange::makeExactICmpRegion(Pred: Pred2, Other: *RHSC2);
8877
8878 return CR1.inverse() == CR2;
8879}
8880
8881SelectPatternResult llvm::getSelectPattern(CmpInst::Predicate Pred,
8882 SelectPatternNaNBehavior NaNBehavior,
8883 bool Ordered) {
8884 switch (Pred) {
8885 default:
8886 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false}; // Equality.
8887 case ICmpInst::ICMP_UGT:
8888 case ICmpInst::ICMP_UGE:
8889 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8890 case ICmpInst::ICMP_SGT:
8891 case ICmpInst::ICMP_SGE:
8892 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8893 case ICmpInst::ICMP_ULT:
8894 case ICmpInst::ICMP_ULE:
8895 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8896 case ICmpInst::ICMP_SLT:
8897 case ICmpInst::ICMP_SLE:
8898 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8899 case FCmpInst::FCMP_UGT:
8900 case FCmpInst::FCMP_UGE:
8901 case FCmpInst::FCMP_OGT:
8902 case FCmpInst::FCMP_OGE:
8903 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
8904 case FCmpInst::FCMP_ULT:
8905 case FCmpInst::FCMP_ULE:
8906 case FCmpInst::FCMP_OLT:
8907 case FCmpInst::FCMP_OLE:
8908 return {.Flavor: SPF_FMINNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
8909 }
8910}
8911
8912std::optional<std::pair<CmpPredicate, Constant *>>
8913llvm::getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred, Constant *C) {
8914 assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
8915 "Only for relational integer predicates.");
8916 if (isa<UndefValue>(Val: C))
8917 return std::nullopt;
8918
8919 Type *Type = C->getType();
8920 bool IsSigned = ICmpInst::isSigned(Pred);
8921
8922 CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred);
8923 bool WillIncrement =
8924 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
8925
8926 // Check if the constant operand can be safely incremented/decremented
8927 // without overflowing/underflowing.
8928 auto ConstantIsOk = [WillIncrement, IsSigned](ConstantInt *C) {
8929 return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned);
8930 };
8931
8932 Constant *SafeReplacementConstant = nullptr;
8933 if (auto *CI = dyn_cast<ConstantInt>(Val: C)) {
8934 // Bail out if the constant can't be safely incremented/decremented.
8935 if (!ConstantIsOk(CI))
8936 return std::nullopt;
8937 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Type)) {
8938 unsigned NumElts = FVTy->getNumElements();
8939 for (unsigned i = 0; i != NumElts; ++i) {
8940 Constant *Elt = C->getAggregateElement(Elt: i);
8941 if (!Elt)
8942 return std::nullopt;
8943
8944 if (isa<UndefValue>(Val: Elt))
8945 continue;
8946
8947 // Bail out if we can't determine if this constant is min/max or if we
8948 // know that this constant is min/max.
8949 auto *CI = dyn_cast<ConstantInt>(Val: Elt);
8950 if (!CI || !ConstantIsOk(CI))
8951 return std::nullopt;
8952
8953 if (!SafeReplacementConstant)
8954 SafeReplacementConstant = CI;
8955 }
8956 } else if (isa<VectorType>(Val: C->getType())) {
8957 // Handle scalable splat
8958 Value *SplatC = C->getSplatValue();
8959 auto *CI = dyn_cast_or_null<ConstantInt>(Val: SplatC);
8960 // Bail out if the constant can't be safely incremented/decremented.
8961 if (!CI || !ConstantIsOk(CI))
8962 return std::nullopt;
8963 } else {
8964 // ConstantExpr?
8965 return std::nullopt;
8966 }
8967
8968 // It may not be safe to change a compare predicate in the presence of
8969 // undefined elements, so replace those elements with the first safe constant
8970 // that we found.
8971 // TODO: in case of poison, it is safe; let's replace undefs only.
8972 if (C->containsUndefOrPoisonElement()) {
8973 assert(SafeReplacementConstant && "Replacement constant not set");
8974 C = Constant::replaceUndefsWith(C, Replacement: SafeReplacementConstant);
8975 }
8976
8977 CmpInst::Predicate NewPred = CmpInst::getFlippedStrictnessPredicate(pred: Pred);
8978
8979 // Increment or decrement the constant.
8980 Constant *OneOrNegOne = ConstantInt::get(Ty: Type, V: WillIncrement ? 1 : -1, IsSigned: true);
8981 Constant *NewC = ConstantExpr::getAdd(C1: C, C2: OneOrNegOne);
8982
8983 return std::make_pair(x&: NewPred, y&: NewC);
8984}
8985
8986static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
8987 FastMathFlags FMF,
8988 Value *CmpLHS, Value *CmpRHS,
8989 Value *TrueVal, Value *FalseVal,
8990 Value *&LHS, Value *&RHS,
8991 unsigned Depth) {
8992 bool HasMismatchedZeros = false;
8993 if (CmpInst::isFPPredicate(P: Pred)) {
8994 // IEEE-754 ignores the sign of 0.0 in comparisons. So if the select has one
8995 // 0.0 operand, set the compare's 0.0 operands to that same value for the
8996 // purpose of identifying min/max. Disregard vector constants with undefined
8997 // elements because those can not be back-propagated for analysis.
8998 Value *OutputZeroVal = nullptr;
8999 if (match(V: TrueVal, P: m_AnyZeroFP()) && !match(V: FalseVal, P: m_AnyZeroFP()) &&
9000 !cast<Constant>(Val: TrueVal)->containsUndefOrPoisonElement())
9001 OutputZeroVal = TrueVal;
9002 else if (match(V: FalseVal, P: m_AnyZeroFP()) && !match(V: TrueVal, P: m_AnyZeroFP()) &&
9003 !cast<Constant>(Val: FalseVal)->containsUndefOrPoisonElement())
9004 OutputZeroVal = FalseVal;
9005
9006 if (OutputZeroVal) {
9007 if (match(V: CmpLHS, P: m_AnyZeroFP()) && CmpLHS != OutputZeroVal) {
9008 HasMismatchedZeros = true;
9009 CmpLHS = OutputZeroVal;
9010 }
9011 if (match(V: CmpRHS, P: m_AnyZeroFP()) && CmpRHS != OutputZeroVal) {
9012 HasMismatchedZeros = true;
9013 CmpRHS = OutputZeroVal;
9014 }
9015 }
9016 }
9017
9018 LHS = CmpLHS;
9019 RHS = CmpRHS;
9020
9021 // Signed zero may return inconsistent results between implementations.
9022 // (0.0 <= -0.0) ? 0.0 : -0.0 // Returns 0.0
9023 // minNum(0.0, -0.0) // May return -0.0 or 0.0 (IEEE 754-2008 5.3.1)
9024 // Therefore, we behave conservatively and only proceed if at least one of the
9025 // operands is known to not be zero or if we don't care about signed zero.
9026 switch (Pred) {
9027 default: break;
9028 case CmpInst::FCMP_OGT: case CmpInst::FCMP_OLT:
9029 case CmpInst::FCMP_UGT: case CmpInst::FCMP_ULT:
9030 if (!HasMismatchedZeros)
9031 break;
9032 [[fallthrough]];
9033 case CmpInst::FCMP_OGE: case CmpInst::FCMP_OLE:
9034 case CmpInst::FCMP_UGE: case CmpInst::FCMP_ULE:
9035 if (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9036 !isKnownNonZero(V: CmpRHS))
9037 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9038 }
9039
9040 SelectPatternNaNBehavior NaNBehavior = SPNB_NA;
9041 bool Ordered = false;
9042
9043 // When given one NaN and one non-NaN input:
9044 // - maxnum/minnum (C99 fmaxf()/fminf()) return the non-NaN input.
9045 // - A simple C99 (a < b ? a : b) construction will return 'b' (as the
9046 // ordered comparison fails), which could be NaN or non-NaN.
9047 // so here we discover exactly what NaN behavior is required/accepted.
9048 if (CmpInst::isFPPredicate(P: Pred)) {
9049 bool LHSSafe = isKnownNonNaN(V: CmpLHS, FMF);
9050 bool RHSSafe = isKnownNonNaN(V: CmpRHS, FMF);
9051
9052 if (LHSSafe && RHSSafe) {
9053 // Both operands are known non-NaN.
9054 NaNBehavior = SPNB_RETURNS_ANY;
9055 Ordered = CmpInst::isOrdered(predicate: Pred);
9056 } else if (CmpInst::isOrdered(predicate: Pred)) {
9057 // An ordered comparison will return false when given a NaN, so it
9058 // returns the RHS.
9059 Ordered = true;
9060 if (LHSSafe)
9061 // LHS is non-NaN, so if RHS is NaN then NaN will be returned.
9062 NaNBehavior = SPNB_RETURNS_NAN;
9063 else if (RHSSafe)
9064 NaNBehavior = SPNB_RETURNS_OTHER;
9065 else
9066 // Completely unsafe.
9067 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9068 } else {
9069 Ordered = false;
9070 // An unordered comparison will return true when given a NaN, so it
9071 // returns the LHS.
9072 if (LHSSafe)
9073 // LHS is non-NaN, so if RHS is NaN then non-NaN will be returned.
9074 NaNBehavior = SPNB_RETURNS_OTHER;
9075 else if (RHSSafe)
9076 NaNBehavior = SPNB_RETURNS_NAN;
9077 else
9078 // Completely unsafe.
9079 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9080 }
9081 }
9082
9083 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
9084 std::swap(a&: CmpLHS, b&: CmpRHS);
9085 Pred = CmpInst::getSwappedPredicate(pred: Pred);
9086 if (NaNBehavior == SPNB_RETURNS_NAN)
9087 NaNBehavior = SPNB_RETURNS_OTHER;
9088 else if (NaNBehavior == SPNB_RETURNS_OTHER)
9089 NaNBehavior = SPNB_RETURNS_NAN;
9090 Ordered = !Ordered;
9091 }
9092
9093 // ([if]cmp X, Y) ? X : Y
9094 if (TrueVal == CmpLHS && FalseVal == CmpRHS)
9095 return getSelectPattern(Pred, NaNBehavior, Ordered);
9096
9097 if (isKnownNegation(X: TrueVal, Y: FalseVal)) {
9098 // Sign-extending LHS does not change its sign, so TrueVal/FalseVal can
9099 // match against either LHS or sign-preserving operations on LHS, like
9100 // sext(LHS), or binary ops that do not wrap in signed sense.
9101 auto CmpLHSOrSExt =
9102 m_CombineOr(Ps: m_Specific(V: CmpLHS), Ps: m_SExt(Op: m_Specific(V: CmpLHS)));
9103 auto MaybeSExtOrMulCmpLHS =
9104 m_CombineOr(Ps: CmpLHSOrSExt, Ps: m_NSWMul(L: CmpLHSOrSExt, R: m_StrictlyPositive()),
9105 Ps: m_NSWShl(L: CmpLHSOrSExt, R: m_Value()));
9106 auto ZeroOrAllOnes = m_CombineOr(Ps: m_ZeroInt(), Ps: m_AllOnes());
9107 auto ZeroOrOne = m_CombineOr(Ps: m_ZeroInt(), Ps: m_One());
9108 if (match(V: TrueVal, P: MaybeSExtOrMulCmpLHS)) {
9109 // Set the return values. If the compare uses the negated value (-X >s 0),
9110 // swap the return values because the negated value is always 'RHS'.
9111 LHS = TrueVal;
9112 RHS = FalseVal;
9113 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: FalseVal))))
9114 std::swap(a&: LHS, b&: RHS);
9115
9116 // (X >s 0) ? X : -X or (X >s -1) ? X : -X --> ABS(X)
9117 // (-X >s 0) ? -X : X or (-X >s -1) ? -X : X --> ABS(X)
9118 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9119 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9120
9121 // (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X)
9122 if (Pred == ICmpInst::ICMP_SGE && match(V: CmpRHS, P: ZeroOrOne))
9123 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9124
9125 // (X <s 0) ? X : -X or (X <s 1) ? X : -X --> NABS(X)
9126 // (-X <s 0) ? -X : X or (-X <s 1) ? -X : X --> NABS(X)
9127 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9128 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9129 } else if (match(V: FalseVal, P: MaybeSExtOrMulCmpLHS)) {
9130 // Set the return values. If the compare uses the negated value (-X >s 0),
9131 // swap the return values because the negated value is always 'RHS'.
9132 LHS = FalseVal;
9133 RHS = TrueVal;
9134 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: TrueVal))))
9135 std::swap(a&: LHS, b&: RHS);
9136
9137 // (X >s 0) ? -X : X or (X >s -1) ? -X : X --> NABS(X)
9138 // (-X >s 0) ? X : -X or (-X >s -1) ? X : -X --> NABS(X)
9139 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9140 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9141
9142 // (X <s 0) ? -X : X or (X <s 1) ? -X : X --> ABS(X)
9143 // (-X <s 0) ? X : -X or (-X <s 1) ? X : -X --> ABS(X)
9144 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9145 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9146 }
9147 }
9148
9149 if (CmpInst::isIntPredicate(P: Pred))
9150 return matchMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS, Depth);
9151
9152 // According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar
9153 // may return either -0.0 or 0.0, so fcmp/select pair has stricter
9154 // semantics than minNum. Be conservative in such case.
9155 if (NaNBehavior != SPNB_RETURNS_ANY ||
9156 (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9157 !isKnownNonZero(V: CmpRHS)))
9158 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9159
9160 return matchFastFloatClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
9161}
9162
9163static Value *lookThroughCastConst(CmpInst *CmpI, Type *SrcTy, Constant *C,
9164 Instruction::CastOps *CastOp) {
9165 const DataLayout &DL = CmpI->getDataLayout();
9166
9167 Constant *CastedTo = nullptr;
9168 switch (*CastOp) {
9169 case Instruction::ZExt:
9170 if (CmpI->isUnsigned())
9171 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy);
9172 break;
9173 case Instruction::SExt:
9174 if (CmpI->isSigned())
9175 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy, OnlyIfReduced: true);
9176 break;
9177 case Instruction::Trunc:
9178 Constant *CmpConst;
9179 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_Constant(C&: CmpConst)) &&
9180 CmpConst->getType() == SrcTy) {
9181 // Here we have the following case:
9182 //
9183 // %cond = cmp iN %x, CmpConst
9184 // %tr = trunc iN %x to iK
9185 // %narrowsel = select i1 %cond, iK %t, iK C
9186 //
9187 // We can always move trunc after select operation:
9188 //
9189 // %cond = cmp iN %x, CmpConst
9190 // %widesel = select i1 %cond, iN %x, iN CmpConst
9191 // %tr = trunc iN %widesel to iK
9192 //
9193 // Note that C could be extended in any way because we don't care about
9194 // upper bits after truncation. It can't be abs pattern, because it would
9195 // look like:
9196 //
9197 // select i1 %cond, x, -x.
9198 //
9199 // So only min/max pattern could be matched. Such match requires widened C
9200 // == CmpConst. That is why set widened C = CmpConst, condition trunc
9201 // CmpConst == C is checked below.
9202 CastedTo = CmpConst;
9203 } else {
9204 unsigned ExtOp = CmpI->isSigned() ? Instruction::SExt : Instruction::ZExt;
9205 CastedTo = ConstantFoldCastOperand(Opcode: ExtOp, C, DestTy: SrcTy, DL);
9206 }
9207 break;
9208 case Instruction::FPTrunc:
9209 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPExt, C, DestTy: SrcTy, DL);
9210 break;
9211 case Instruction::FPExt:
9212 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPTrunc, C, DestTy: SrcTy, DL);
9213 break;
9214 case Instruction::FPToUI:
9215 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::UIToFP, C, DestTy: SrcTy, DL);
9216 break;
9217 case Instruction::FPToSI:
9218 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::SIToFP, C, DestTy: SrcTy, DL);
9219 break;
9220 case Instruction::UIToFP:
9221 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToUI, C, DestTy: SrcTy, DL);
9222 break;
9223 case Instruction::SIToFP:
9224 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToSI, C, DestTy: SrcTy, DL);
9225 break;
9226 default:
9227 break;
9228 }
9229
9230 if (!CastedTo)
9231 return nullptr;
9232
9233 // Make sure the cast doesn't lose any information.
9234 Constant *CastedBack =
9235 ConstantFoldCastOperand(Opcode: *CastOp, C: CastedTo, DestTy: C->getType(), DL);
9236 if (CastedBack && CastedBack != C)
9237 return nullptr;
9238
9239 return CastedTo;
9240}
9241
9242/// Helps to match a select pattern in case of a type mismatch.
9243///
9244/// The function processes the case when type of true and false values of a
9245/// select instruction differs from type of the cmp instruction operands because
9246/// of a cast instruction. The function checks if it is legal to move the cast
9247/// operation after "select". If yes, it returns the new second value of
9248/// "select" (with the assumption that cast is moved):
9249/// 1. As operand of cast instruction when both values of "select" are same cast
9250/// instructions.
9251/// 2. As restored constant (by applying reverse cast operation) when the first
9252/// value of the "select" is a cast operation and the second value is a
9253/// constant. It is implemented in lookThroughCastConst().
9254/// 3. As one operand is cast instruction and the other is not. The operands in
9255/// sel(cmp) are in different type integer.
9256/// NOTE: We return only the new second value because the first value could be
9257/// accessed as operand of cast instruction.
9258static Value *lookThroughCast(CmpInst *CmpI, Value *V1, Value *V2,
9259 Instruction::CastOps *CastOp) {
9260 auto *Cast1 = dyn_cast<CastInst>(Val: V1);
9261 if (!Cast1)
9262 return nullptr;
9263
9264 *CastOp = Cast1->getOpcode();
9265 Type *SrcTy = Cast1->getSrcTy();
9266 if (auto *Cast2 = dyn_cast<CastInst>(Val: V2)) {
9267 // If V1 and V2 are both the same cast from the same type, look through V1.
9268 if (*CastOp == Cast2->getOpcode() && SrcTy == Cast2->getSrcTy())
9269 return Cast2->getOperand(i_nocapture: 0);
9270 return nullptr;
9271 }
9272
9273 auto *C = dyn_cast<Constant>(Val: V2);
9274 if (C)
9275 return lookThroughCastConst(CmpI, SrcTy, C, CastOp);
9276
9277 Value *CastedTo = nullptr;
9278 if (*CastOp == Instruction::Trunc) {
9279 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_ZExtOrSExt(Op: m_Specific(V: V2)))) {
9280 // Here we have the following case:
9281 // %y_ext = sext iK %y to iN
9282 // %cond = cmp iN %x, %y_ext
9283 // %tr = trunc iN %x to iK
9284 // %narrowsel = select i1 %cond, iK %tr, iK %y
9285 //
9286 // We can always move trunc after select operation:
9287 // %y_ext = sext iK %y to iN
9288 // %cond = cmp iN %x, %y_ext
9289 // %widesel = select i1 %cond, iN %x, iN %y_ext
9290 // %tr = trunc iN %widesel to iK
9291 assert(V2->getType() == Cast1->getType() &&
9292 "V2 and Cast1 should be the same type.");
9293 CastedTo = CmpI->getOperand(i_nocapture: 1);
9294 }
9295 }
9296
9297 return CastedTo;
9298}
9299SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
9300 Instruction::CastOps *CastOp,
9301 unsigned Depth) {
9302 if (Depth >= MaxAnalysisRecursionDepth)
9303 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9304
9305 SelectInst *SI = dyn_cast<SelectInst>(Val: V);
9306 if (!SI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9307
9308 CmpInst *CmpI = dyn_cast<CmpInst>(Val: SI->getCondition());
9309 if (!CmpI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9310
9311 Value *TrueVal = SI->getTrueValue();
9312 Value *FalseVal = SI->getFalseValue();
9313
9314 return llvm::matchDecomposedSelectPattern(CmpI, TrueVal, FalseVal, LHS, RHS,
9315 FMF: SI->getFastMathFlagsOrNone(),
9316 CastOp, Depth);
9317}
9318
9319SelectPatternResult llvm::matchDecomposedSelectPattern(
9320 CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
9321 FastMathFlags FMF, Instruction::CastOps *CastOp, unsigned Depth) {
9322 CmpInst::Predicate Pred = CmpI->getPredicate();
9323 Value *CmpLHS = CmpI->getOperand(i_nocapture: 0);
9324 Value *CmpRHS = CmpI->getOperand(i_nocapture: 1);
9325 if (isa<FPMathOperator>(Val: CmpI) && CmpI->hasNoNaNs())
9326 FMF.setNoNaNs();
9327
9328 // Bail out early.
9329 if (CmpI->isEquality())
9330 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9331
9332 // Deal with type mismatches.
9333 if (CastOp && CmpLHS->getType() != TrueVal->getType()) {
9334 if (Value *C = lookThroughCast(CmpI, V1: TrueVal, V2: FalseVal, CastOp)) {
9335 // If this is a potential fmin/fmax with a cast to integer, then ignore
9336 // -0.0 because there is no corresponding integer value.
9337 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9338 FMF.setNoSignedZeros();
9339 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9340 TrueVal: cast<CastInst>(Val: TrueVal)->getOperand(i_nocapture: 0), FalseVal: C,
9341 LHS, RHS, Depth);
9342 }
9343 if (Value *C = lookThroughCast(CmpI, V1: FalseVal, V2: TrueVal, CastOp)) {
9344 // If this is a potential fmin/fmax with a cast to integer, then ignore
9345 // -0.0 because there is no corresponding integer value.
9346 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9347 FMF.setNoSignedZeros();
9348 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9349 TrueVal: C, FalseVal: cast<CastInst>(Val: FalseVal)->getOperand(i_nocapture: 0),
9350 LHS, RHS, Depth);
9351 }
9352 }
9353 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal, FalseVal,
9354 LHS, RHS, Depth);
9355}
9356
9357CmpInst::Predicate llvm::getMinMaxPred(SelectPatternFlavor SPF, bool Ordered) {
9358 if (SPF == SPF_SMIN) return ICmpInst::ICMP_SLT;
9359 if (SPF == SPF_UMIN) return ICmpInst::ICMP_ULT;
9360 if (SPF == SPF_SMAX) return ICmpInst::ICMP_SGT;
9361 if (SPF == SPF_UMAX) return ICmpInst::ICMP_UGT;
9362 if (SPF == SPF_FMINNUM)
9363 return Ordered ? FCmpInst::FCMP_OLT : FCmpInst::FCMP_ULT;
9364 if (SPF == SPF_FMAXNUM)
9365 return Ordered ? FCmpInst::FCMP_OGT : FCmpInst::FCMP_UGT;
9366 llvm_unreachable("unhandled!");
9367}
9368
9369Intrinsic::ID llvm::getMinMaxIntrinsic(SelectPatternFlavor SPF) {
9370 switch (SPF) {
9371 case SelectPatternFlavor::SPF_UMIN:
9372 return Intrinsic::umin;
9373 case SelectPatternFlavor::SPF_UMAX:
9374 return Intrinsic::umax;
9375 case SelectPatternFlavor::SPF_SMIN:
9376 return Intrinsic::smin;
9377 case SelectPatternFlavor::SPF_SMAX:
9378 return Intrinsic::smax;
9379 default:
9380 llvm_unreachable("Unexpected SPF");
9381 }
9382}
9383
9384SelectPatternFlavor llvm::getInverseMinMaxFlavor(SelectPatternFlavor SPF) {
9385 if (SPF == SPF_SMIN) return SPF_SMAX;
9386 if (SPF == SPF_UMIN) return SPF_UMAX;
9387 if (SPF == SPF_SMAX) return SPF_SMIN;
9388 if (SPF == SPF_UMAX) return SPF_UMIN;
9389 llvm_unreachable("unhandled!");
9390}
9391
9392Intrinsic::ID llvm::getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID) {
9393 switch (MinMaxID) {
9394 case Intrinsic::smax: return Intrinsic::smin;
9395 case Intrinsic::smin: return Intrinsic::smax;
9396 case Intrinsic::umax: return Intrinsic::umin;
9397 case Intrinsic::umin: return Intrinsic::umax;
9398 // Please note that next four intrinsics may produce the same result for
9399 // original and inverted case even if X != Y due to NaN is handled specially.
9400 case Intrinsic::maximum: return Intrinsic::minimum;
9401 case Intrinsic::minimum: return Intrinsic::maximum;
9402 case Intrinsic::maxnum: return Intrinsic::minnum;
9403 case Intrinsic::minnum: return Intrinsic::maxnum;
9404 case Intrinsic::maximumnum:
9405 return Intrinsic::minimumnum;
9406 case Intrinsic::minimumnum:
9407 return Intrinsic::maximumnum;
9408 default: llvm_unreachable("Unexpected intrinsic");
9409 }
9410}
9411
9412APInt llvm::getMinMaxLimit(SelectPatternFlavor SPF, unsigned BitWidth) {
9413 switch (SPF) {
9414 case SPF_SMAX: return APInt::getSignedMaxValue(numBits: BitWidth);
9415 case SPF_SMIN: return APInt::getSignedMinValue(numBits: BitWidth);
9416 case SPF_UMAX: return APInt::getMaxValue(numBits: BitWidth);
9417 case SPF_UMIN: return APInt::getMinValue(numBits: BitWidth);
9418 default: llvm_unreachable("Unexpected flavor");
9419 }
9420}
9421
9422std::pair<Intrinsic::ID, bool>
9423llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
9424 // Check if VL contains select instructions that can be folded into a min/max
9425 // vector intrinsic and return the intrinsic if it is possible.
9426 // TODO: Support floating point min/max.
9427 bool AllCmpSingleUse = true;
9428 SelectPatternResult SelectPattern;
9429 SelectPattern.Flavor = SPF_UNKNOWN;
9430 if (all_of(Range&: VL, P: [&SelectPattern, &AllCmpSingleUse](Value *I) {
9431 Value *LHS, *RHS;
9432 auto CurrentPattern = matchSelectPattern(V: I, LHS, RHS);
9433 if (!SelectPatternResult::isMinOrMax(SPF: CurrentPattern.Flavor))
9434 return false;
9435 if (SelectPattern.Flavor != SPF_UNKNOWN &&
9436 SelectPattern.Flavor != CurrentPattern.Flavor)
9437 return false;
9438 SelectPattern = CurrentPattern;
9439 AllCmpSingleUse &=
9440 match(V: I, P: m_Select(C: m_OneUse(SubPattern: m_Value()), L: m_Value(), R: m_Value()));
9441 return true;
9442 })) {
9443 switch (SelectPattern.Flavor) {
9444 case SPF_SMIN:
9445 return {Intrinsic::smin, AllCmpSingleUse};
9446 case SPF_UMIN:
9447 return {Intrinsic::umin, AllCmpSingleUse};
9448 case SPF_SMAX:
9449 return {Intrinsic::smax, AllCmpSingleUse};
9450 case SPF_UMAX:
9451 return {Intrinsic::umax, AllCmpSingleUse};
9452 case SPF_FMAXNUM:
9453 return {Intrinsic::maxnum, AllCmpSingleUse};
9454 case SPF_FMINNUM:
9455 return {Intrinsic::minnum, AllCmpSingleUse};
9456 default:
9457 llvm_unreachable("unexpected select pattern flavor");
9458 }
9459 }
9460 return {Intrinsic::not_intrinsic, false};
9461}
9462
9463template <typename InstTy>
9464static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
9465 Value *&Init, Value *&OtherOp) {
9466 // Handle the case of a simple two-predecessor recurrence PHI.
9467 // There's a lot more that could theoretically be done here, but
9468 // this is sufficient to catch some interesting cases.
9469 // TODO: Expand list -- gep, uadd.sat etc.
9470 if (PN->getNumIncomingValues() != 2)
9471 return false;
9472
9473 for (unsigned I = 0; I != 2; ++I) {
9474 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9475 Operation && Operation->getNumOperands() >= 2) {
9476 Value *LHS = Operation->getOperand(0);
9477 Value *RHS = Operation->getOperand(1);
9478 if (LHS != PN && RHS != PN)
9479 continue;
9480
9481 Inst = Operation;
9482 Init = PN->getIncomingValue(i: !I);
9483 OtherOp = (LHS == PN) ? RHS : LHS;
9484 return true;
9485 }
9486 }
9487 return false;
9488}
9489
9490template <typename InstTy>
9491static bool matchThreeInputRecurrence(const PHINode *PN, InstTy *&Inst,
9492 Value *&Init, Value *&OtherOp0,
9493 Value *&OtherOp1) {
9494 if (PN->getNumIncomingValues() != 2)
9495 return false;
9496
9497 for (unsigned I = 0; I != 2; ++I) {
9498 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9499 Operation && Operation->getNumOperands() >= 3) {
9500 Value *Op0 = Operation->getOperand(0);
9501 Value *Op1 = Operation->getOperand(1);
9502 Value *Op2 = Operation->getOperand(2);
9503
9504 if (Op0 != PN && Op1 != PN && Op2 != PN)
9505 continue;
9506
9507 Inst = Operation;
9508 Init = PN->getIncomingValue(i: !I);
9509 if (Op0 == PN) {
9510 OtherOp0 = Op1;
9511 OtherOp1 = Op2;
9512 } else if (Op1 == PN) {
9513 OtherOp0 = Op0;
9514 OtherOp1 = Op2;
9515 } else {
9516 OtherOp0 = Op0;
9517 OtherOp1 = Op1;
9518 }
9519 return true;
9520 }
9521 }
9522 return false;
9523}
9524bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
9525 Value *&Start, Value *&Step) {
9526 // We try to match a recurrence of the form:
9527 // %iv = [Start, %entry], [%iv.next, %backedge]
9528 // %iv.next = binop %iv, Step
9529 // Or:
9530 // %iv = [Start, %entry], [%iv.next, %backedge]
9531 // %iv.next = binop Step, %iv
9532 return matchTwoInputRecurrence(PN: P, Inst&: BO, Init&: Start, OtherOp&: Step);
9533}
9534
9535bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
9536 Value *&Start, Value *&Step) {
9537 BinaryOperator *BO = nullptr;
9538 return match(V: I, P: m_c_BinOp(L: m_Phi(PN&: P), R: m_Value())) &&
9539 matchSimpleRecurrence(P, BO, Start, Step) && BO == I;
9540}
9541
9542bool llvm::matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I,
9543 PHINode *&P, Value *&Init,
9544 Value *&OtherOp) {
9545 // Binary intrinsics only supported for now.
9546 if (I->arg_size() != 2 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9547 I->getType() != I->getArgOperand(i: 1)->getType())
9548 return false;
9549
9550 IntrinsicInst *II = nullptr;
9551 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9552 if (!P)
9553 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9554
9555 return P && matchTwoInputRecurrence(PN: P, Inst&: II, Init, OtherOp) && II == I;
9556}
9557
9558bool llvm::matchSimpleTernaryIntrinsicRecurrence(const IntrinsicInst *I,
9559 PHINode *&P, Value *&Init,
9560 Value *&OtherOp0,
9561 Value *&OtherOp1) {
9562 if (I->arg_size() != 3 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9563 I->getType() != I->getArgOperand(i: 1)->getType() ||
9564 I->getType() != I->getArgOperand(i: 2)->getType())
9565 return false;
9566 IntrinsicInst *II = nullptr;
9567 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9568 if (!P) {
9569 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9570 if (!P)
9571 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 2));
9572 }
9573 return P && matchThreeInputRecurrence(PN: P, Inst&: II, Init, OtherOp0, OtherOp1) &&
9574 II == I;
9575}
9576
9577/// Return true if "icmp Pred LHS RHS" is always true.
9578static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
9579 const Value *RHS) {
9580 if (ICmpInst::isTrueWhenEqual(predicate: Pred) && LHS == RHS)
9581 return true;
9582
9583 switch (Pred) {
9584 default:
9585 return false;
9586
9587 case CmpInst::ICMP_SLE: {
9588 const APInt *C;
9589
9590 // LHS s<= LHS +_{nsw} C if C >= 0
9591 // LHS s<= LHS | C if C >= 0
9592 if (match(V: RHS, P: m_NSWAdd(L: m_Specific(V: LHS), R: m_APInt(Res&: C))) ||
9593 match(V: RHS, P: m_Or(L: m_Specific(V: LHS), R: m_APInt(Res&: C))))
9594 return !C->isNegative();
9595
9596 // LHS s<= smax(LHS, V) for any V
9597 if (match(V: RHS, P: m_c_SMax(L: m_Specific(V: LHS), R: m_Value())))
9598 return true;
9599
9600 // smin(RHS, V) s<= RHS for any V
9601 if (match(V: LHS, P: m_c_SMin(L: m_Specific(V: RHS), R: m_Value())))
9602 return true;
9603
9604 // Match A to (X +_{nsw} CA) and B to (X +_{nsw} CB)
9605 const Value *X;
9606 const APInt *CLHS, *CRHS;
9607 if (match(V: LHS, P: m_NSWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9608 match(V: RHS, P: m_NSWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9609 return CLHS->sle(RHS: *CRHS);
9610
9611 return false;
9612 }
9613
9614 case CmpInst::ICMP_ULE: {
9615 // LHS u<= LHS +_{nuw} V for any V
9616 if (match(V: RHS, P: m_c_Add(L: m_Specific(V: LHS), R: m_Value())) &&
9617 cast<OverflowingBinaryOperator>(Val: RHS)->hasNoUnsignedWrap())
9618 return true;
9619
9620 // LHS u<= LHS | V for any V
9621 if (match(V: RHS, P: m_c_Or(L: m_Specific(V: LHS), R: m_Value())))
9622 return true;
9623
9624 // LHS u<= umax(LHS, V) for any V
9625 if (match(V: RHS, P: m_c_UMax(L: m_Specific(V: LHS), R: m_Value())))
9626 return true;
9627
9628 // RHS >> V u<= RHS for any V
9629 if (match(V: LHS, P: m_LShr(L: m_Specific(V: RHS), R: m_Value())))
9630 return true;
9631
9632 // RHS u/ C_ugt_1 u<= RHS
9633 const APInt *C;
9634 if (match(V: LHS, P: m_UDiv(L: m_Specific(V: RHS), R: m_APInt(Res&: C))) && C->ugt(RHS: 1))
9635 return true;
9636
9637 // RHS & V u<= RHS for any V
9638 if (match(V: LHS, P: m_c_And(L: m_Specific(V: RHS), R: m_Value())))
9639 return true;
9640
9641 // umin(RHS, V) u<= RHS for any V
9642 if (match(V: LHS, P: m_c_UMin(L: m_Specific(V: RHS), R: m_Value())))
9643 return true;
9644
9645 // Match A to (X +_{nuw} CA) and B to (X +_{nuw} CB)
9646 const Value *X;
9647 const APInt *CLHS, *CRHS;
9648 if (match(V: LHS, P: m_NUWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9649 match(V: RHS, P: m_NUWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9650 return CLHS->ule(RHS: *CRHS);
9651
9652 return false;
9653 }
9654 }
9655}
9656
9657/// Return true if "icmp Pred BLHS BRHS" is true whenever "icmp Pred
9658/// ALHS ARHS" is true. Otherwise, return std::nullopt.
9659static std::optional<bool>
9660isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS,
9661 const Value *ARHS, const Value *BLHS, const Value *BRHS) {
9662 switch (Pred) {
9663 default:
9664 return std::nullopt;
9665
9666 case CmpInst::ICMP_SLT:
9667 case CmpInst::ICMP_SLE:
9668 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BLHS, RHS: ALHS) &&
9669 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ARHS, RHS: BRHS))
9670 return true;
9671 return std::nullopt;
9672
9673 case CmpInst::ICMP_SGT:
9674 case CmpInst::ICMP_SGE:
9675 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ALHS, RHS: BLHS) &&
9676 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BRHS, RHS: ARHS))
9677 return true;
9678 return std::nullopt;
9679
9680 case CmpInst::ICMP_ULT:
9681 case CmpInst::ICMP_ULE:
9682 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BLHS, RHS: ALHS) &&
9683 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ARHS, RHS: BRHS))
9684 return true;
9685 return std::nullopt;
9686
9687 case CmpInst::ICMP_UGT:
9688 case CmpInst::ICMP_UGE:
9689 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ALHS, RHS: BLHS) &&
9690 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BRHS, RHS: ARHS))
9691 return true;
9692 return std::nullopt;
9693 }
9694}
9695
9696/// Return true if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is true.
9697/// Return false if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is false.
9698/// Otherwise, return std::nullopt if we can't infer anything.
9699static std::optional<bool>
9700isImpliedCondCommonOperandWithCR(CmpPredicate LPred, const ConstantRange &LCR,
9701 CmpPredicate RPred, const ConstantRange &RCR) {
9702 auto CRImpliesPred = [&](ConstantRange CR,
9703 CmpInst::Predicate Pred) -> std::optional<bool> {
9704 // If all true values for lhs and true for rhs, lhs implies rhs
9705 if (CR.icmp(Pred, Other: RCR))
9706 return true;
9707
9708 // If there is no overlap, lhs implies not rhs
9709 if (CR.icmp(Pred: CmpInst::getInversePredicate(pred: Pred), Other: RCR))
9710 return false;
9711
9712 return std::nullopt;
9713 };
9714 if (auto Res = CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
9715 RPred))
9716 return Res;
9717 if (LPred.hasSameSign() ^ RPred.hasSameSign()) {
9718 LPred = LPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: LPred)
9719 : LPred.dropSameSign();
9720 RPred = RPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: RPred)
9721 : RPred.dropSameSign();
9722 return CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
9723 RPred);
9724 }
9725 return std::nullopt;
9726}
9727
9728/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
9729/// is true. Return false if LHS implies RHS is false. Otherwise, return
9730/// std::nullopt if we can't infer anything.
9731static std::optional<bool>
9732isImpliedCondICmps(CmpPredicate LPred, const Value *L0, const Value *L1,
9733 CmpPredicate RPred, const Value *R0, const Value *R1,
9734 const DataLayout &DL, bool LHSIsTrue) {
9735 // The rest of the logic assumes the LHS condition is true. If that's not the
9736 // case, invert the predicate to make it so.
9737 if (!LHSIsTrue)
9738 LPred = ICmpInst::getInverseCmpPredicate(Pred: LPred);
9739
9740 // We can have non-canonical operands, so try to normalize any common operand
9741 // to L0/R0.
9742 if (L0 == R1) {
9743 std::swap(a&: R0, b&: R1);
9744 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
9745 }
9746 if (R0 == L1) {
9747 std::swap(a&: L0, b&: L1);
9748 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9749 }
9750 if (L1 == R1) {
9751 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
9752 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
9753 std::swap(a&: L0, b&: L1);
9754 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9755 std::swap(a&: R0, b&: R1);
9756 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
9757 }
9758 }
9759
9760 // See if we can infer anything if operand-0 matches and we have at least one
9761 // constant.
9762 const APInt *Unused;
9763 if (L0 == R0 && (match(V: L1, P: m_APInt(Res&: Unused)) || match(V: R1, P: m_APInt(Res&: Unused)))) {
9764 // Potential TODO: We could also further use the constant range of L0/R0 to
9765 // further constraint the constant ranges. At the moment this leads to
9766 // several regressions related to not transforming `multi_use(A + C0) eq/ne
9767 // C1` (see discussion: D58633).
9768 SimplifyQuery SQ(DL);
9769 ConstantRange LCR = computeConstantRange(V: L1, ForSigned: ICmpInst::isSigned(Pred: LPred), SQ,
9770 Depth: MaxAnalysisRecursionDepth - 1);
9771 ConstantRange RCR = computeConstantRange(V: R1, ForSigned: ICmpInst::isSigned(Pred: RPred), SQ,
9772 Depth: MaxAnalysisRecursionDepth - 1);
9773
9774 // Even if L1/R1 are not both constant, we can still sometimes deduce
9775 // relationship from a single constant. For example X u> Y implies X != 0.
9776 if (auto R = isImpliedCondCommonOperandWithCR(LPred, LCR, RPred, RCR))
9777 return R;
9778 // If both L1/R1 were exact constant ranges and we didn't get anything
9779 // here, we won't be able to deduce this.
9780 if (match(V: L1, P: m_APInt(Res&: Unused)) && match(V: R1, P: m_APInt(Res&: Unused)))
9781 return std::nullopt;
9782 }
9783
9784 // Can we infer anything when the two compares have matching operands?
9785 if (L0 == R0 && L1 == R1)
9786 return ICmpInst::isImpliedByMatchingCmp(Pred1: LPred, Pred2: RPred);
9787
9788 // It only really makes sense in the context of signed comparison for "X - Y
9789 // must be positive if X >= Y and no overflow".
9790 // Take SGT as an example: L0:x > L1:y and C >= 0
9791 // ==> R0:(x -nsw y) < R1:(-C) is false
9792 CmpInst::Predicate SignedLPred = LPred.getPreferredSignedPredicate();
9793 if ((SignedLPred == ICmpInst::ICMP_SGT ||
9794 SignedLPred == ICmpInst::ICMP_SGE) &&
9795 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
9796 if (match(V: R1, P: m_NonPositive()) &&
9797 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == false)
9798 return false;
9799 }
9800
9801 // Take SLT as an example: L0:x < L1:y and C <= 0
9802 // ==> R0:(x -nsw y) < R1:(-C) is true
9803 if ((SignedLPred == ICmpInst::ICMP_SLT ||
9804 SignedLPred == ICmpInst::ICMP_SLE) &&
9805 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
9806 if (match(V: R1, P: m_NonNegative()) &&
9807 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == true)
9808 return true;
9809 }
9810
9811 // a - b == NonZero -> a != b
9812 // ptrtoint(a) - ptrtoint(b) == NonZero -> a != b
9813 const APInt *L1C;
9814 Value *A, *B;
9815 if (LPred == ICmpInst::ICMP_EQ && ICmpInst::isEquality(P: RPred) &&
9816 match(V: L1, P: m_APInt(Res&: L1C)) && !L1C->isZero() &&
9817 match(V: L0, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9818 ((A == R0 && B == R1) || (A == R1 && B == R0) ||
9819 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0))) &&
9820 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1)))) ||
9821 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1))) &&
9822 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0)))))) {
9823 return RPred.dropSameSign() == ICmpInst::ICMP_NE;
9824 }
9825
9826 // L0 = R0 = L1 + R1, L0 >=u L1 implies R0 >=u R1, L0 <u L1 implies R0 <u R1
9827 if (L0 == R0 &&
9828 (LPred == ICmpInst::ICMP_ULT || LPred == ICmpInst::ICMP_UGE) &&
9829 (RPred == ICmpInst::ICMP_ULT || RPred == ICmpInst::ICMP_UGE) &&
9830 match(V: L0, P: m_c_Add(L: m_Specific(V: L1), R: m_Specific(V: R1))))
9831 return CmpPredicate::getMatching(A: LPred, B: RPred).has_value();
9832
9833 if (auto P = CmpPredicate::getMatching(A: LPred, B: RPred))
9834 return isImpliedCondOperands(Pred: *P, ALHS: L0, ARHS: L1, BLHS: R0, BRHS: R1);
9835
9836 return std::nullopt;
9837}
9838
9839/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
9840/// is true. Return false if LHS implies RHS is false. Otherwise, return
9841/// std::nullopt if we can't infer anything.
9842static std::optional<bool>
9843isImpliedCondFCmps(FCmpInst::Predicate LPred, const Value *L0, const Value *L1,
9844 FCmpInst::Predicate RPred, const Value *R0, const Value *R1,
9845 const DataLayout &DL, bool LHSIsTrue) {
9846 // The rest of the logic assumes the LHS condition is true. If that's not the
9847 // case, invert the predicate to make it so.
9848 if (!LHSIsTrue)
9849 LPred = FCmpInst::getInversePredicate(pred: LPred);
9850
9851 // We can have non-canonical operands, so try to normalize any common operand
9852 // to L0/R0.
9853 if (L0 == R1) {
9854 std::swap(a&: R0, b&: R1);
9855 RPred = FCmpInst::getSwappedPredicate(pred: RPred);
9856 }
9857 if (R0 == L1) {
9858 std::swap(a&: L0, b&: L1);
9859 LPred = FCmpInst::getSwappedPredicate(pred: LPred);
9860 }
9861 if (L1 == R1) {
9862 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
9863 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
9864 std::swap(a&: L0, b&: L1);
9865 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9866 std::swap(a&: R0, b&: R1);
9867 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
9868 }
9869 }
9870
9871 // Can we infer anything when the two compares have matching operands?
9872 if (L0 == R0 && L1 == R1) {
9873 if ((LPred & RPred) == LPred)
9874 return true;
9875 if ((LPred & ~RPred) == LPred)
9876 return false;
9877 }
9878
9879 // See if we can infer anything if operand-0 matches and we have at least one
9880 // constant.
9881 const APFloat *L1C, *R1C;
9882 if (L0 == R0 && match(V: L1, P: m_APFloat(Res&: L1C)) && match(V: R1, P: m_APFloat(Res&: R1C))) {
9883 if (std::optional<ConstantFPRange> DomCR =
9884 ConstantFPRange::makeExactFCmpRegion(Pred: LPred, Other: *L1C)) {
9885 if (std::optional<ConstantFPRange> ImpliedCR =
9886 ConstantFPRange::makeExactFCmpRegion(Pred: RPred, Other: *R1C)) {
9887 if (ImpliedCR->contains(CR: *DomCR))
9888 return true;
9889 }
9890 if (std::optional<ConstantFPRange> ImpliedCR =
9891 ConstantFPRange::makeExactFCmpRegion(
9892 Pred: FCmpInst::getInversePredicate(pred: RPred), Other: *R1C)) {
9893 if (ImpliedCR->contains(CR: *DomCR))
9894 return false;
9895 }
9896 }
9897 }
9898
9899 return std::nullopt;
9900}
9901
9902/// Return true if LHS implies RHS is true. Return false if LHS implies RHS is
9903/// false. Otherwise, return std::nullopt if we can't infer anything. We
9904/// expect the RHS to be an icmp and the LHS to be an 'and', 'or', or a 'select'
9905/// instruction.
9906static std::optional<bool>
9907isImpliedCondAndOr(const Instruction *LHS, CmpPredicate RHSPred,
9908 const Value *RHSOp0, const Value *RHSOp1,
9909 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
9910 // The LHS must be an 'or', 'and', or a 'select' instruction.
9911 assert((LHS->getOpcode() == Instruction::And ||
9912 LHS->getOpcode() == Instruction::Or ||
9913 LHS->getOpcode() == Instruction::Select) &&
9914 "Expected LHS to be 'and', 'or', or 'select'.");
9915
9916 assert(Depth <= MaxAnalysisRecursionDepth && "Hit recursion limit");
9917
9918 // If the result of an 'or' is false, then we know both legs of the 'or' are
9919 // false. Similarly, if the result of an 'and' is true, then we know both
9920 // legs of the 'and' are true.
9921 const Value *ALHS, *ARHS;
9922 if ((!LHSIsTrue && match(V: LHS, P: m_LogicalOr(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS)))) ||
9923 (LHSIsTrue && match(V: LHS, P: m_LogicalAnd(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS))))) {
9924 // FIXME: Make this non-recursion.
9925 if (std::optional<bool> Implication = isImpliedCondition(
9926 LHS: ALHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
9927 return Implication;
9928 if (std::optional<bool> Implication = isImpliedCondition(
9929 LHS: ARHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
9930 return Implication;
9931 return std::nullopt;
9932 }
9933 return std::nullopt;
9934}
9935
9936std::optional<bool>
9937llvm::isImpliedCondition(const Value *LHS, CmpPredicate RHSPred,
9938 const Value *RHSOp0, const Value *RHSOp1,
9939 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
9940 // Bail out when we hit the limit.
9941 if (Depth == MaxAnalysisRecursionDepth)
9942 return std::nullopt;
9943
9944 // A mismatch occurs when we compare a scalar cmp to a vector cmp, for
9945 // example.
9946 if (RHSOp0->getType()->isVectorTy() != LHS->getType()->isVectorTy())
9947 return std::nullopt;
9948
9949 assert(LHS->getType()->isIntOrIntVectorTy(1) &&
9950 "Expected integer type only!");
9951
9952 // Match not
9953 if (match(V: LHS, P: m_Not(V: m_Value(V&: LHS))))
9954 LHSIsTrue = !LHSIsTrue;
9955
9956 // Both LHS and RHS are icmps.
9957 if (RHSOp0->getType()->getScalarType()->isIntOrPtrTy()) {
9958 CmpPredicate LHSPred;
9959 Value *LHSOp0, *LHSOp1;
9960 if (match(V: LHS, P: m_ICmpLike(Pred&: LHSPred, L: m_Value(V&: LHSOp0), R: m_Value(V&: LHSOp1))))
9961 return isImpliedCondICmps(LPred: LHSPred, L0: LHSOp0, L1: LHSOp1, RPred: RHSPred, R0: RHSOp0,
9962 R1: RHSOp1, DL, LHSIsTrue);
9963 } else {
9964 assert(RHSOp0->getType()->isFPOrFPVectorTy() &&
9965 "Expected floating point type only!");
9966 if (const auto *LHSCmp = dyn_cast<FCmpInst>(Val: LHS))
9967 return isImpliedCondFCmps(LPred: LHSCmp->getPredicate(), L0: LHSCmp->getOperand(i_nocapture: 0),
9968 L1: LHSCmp->getOperand(i_nocapture: 1), RPred: RHSPred, R0: RHSOp0, R1: RHSOp1,
9969 DL, LHSIsTrue);
9970 }
9971
9972 /// The LHS should be an 'or', 'and', or a 'select' instruction. We expect
9973 /// the RHS to be an icmp.
9974 /// FIXME: Add support for and/or/select on the RHS.
9975 if (const Instruction *LHSI = dyn_cast<Instruction>(Val: LHS)) {
9976 if ((LHSI->getOpcode() == Instruction::And ||
9977 LHSI->getOpcode() == Instruction::Or ||
9978 LHSI->getOpcode() == Instruction::Select))
9979 return isImpliedCondAndOr(LHS: LHSI, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue,
9980 Depth);
9981 }
9982 return std::nullopt;
9983}
9984
9985std::optional<bool> llvm::isImpliedCondition(const Value *LHS, const Value *RHS,
9986 const DataLayout &DL,
9987 bool LHSIsTrue, unsigned Depth) {
9988 // LHS ==> RHS by definition
9989 if (LHS == RHS)
9990 return LHSIsTrue;
9991
9992 // Match not
9993 bool InvertRHS = false;
9994 if (match(V: RHS, P: m_Not(V: m_Value(V&: RHS)))) {
9995 if (LHS == RHS)
9996 return !LHSIsTrue;
9997 InvertRHS = true;
9998 }
9999
10000 CmpPredicate RHSPred;
10001 Value *RHSOp0, *RHSOp1;
10002 if (match(V: RHS, P: m_ICmpLike(Pred&: RHSPred, L: m_Value(V&: RHSOp0), R: m_Value(V&: RHSOp1)))) {
10003 if (auto Implied = isImpliedCondition(LHS, RHSPred, RHSOp0, RHSOp1, DL,
10004 LHSIsTrue, Depth))
10005 return InvertRHS ? !*Implied : *Implied;
10006 return std::nullopt;
10007 }
10008 if (const FCmpInst *RHSCmp = dyn_cast<FCmpInst>(Val: RHS)) {
10009 if (auto Implied = isImpliedCondition(
10010 LHS, RHSPred: RHSCmp->getPredicate(), RHSOp0: RHSCmp->getOperand(i_nocapture: 0),
10011 RHSOp1: RHSCmp->getOperand(i_nocapture: 1), DL, LHSIsTrue, Depth))
10012 return InvertRHS ? !*Implied : *Implied;
10013 return std::nullopt;
10014 }
10015
10016 if (Depth == MaxAnalysisRecursionDepth)
10017 return std::nullopt;
10018
10019 // LHS ==> (RHS1 || RHS2) if LHS ==> RHS1 or LHS ==> RHS2
10020 // LHS ==> !(RHS1 && RHS2) if LHS ==> !RHS1 or LHS ==> !RHS2
10021 const Value *RHS1, *RHS2;
10022 if (match(V: RHS, P: m_LogicalOr(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
10023 if (std::optional<bool> Imp =
10024 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
10025 if (*Imp == true)
10026 return !InvertRHS;
10027 if (std::optional<bool> Imp =
10028 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
10029 if (*Imp == true)
10030 return !InvertRHS;
10031 }
10032 if (match(V: RHS, P: m_LogicalAnd(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
10033 if (std::optional<bool> Imp =
10034 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
10035 if (*Imp == false)
10036 return InvertRHS;
10037 if (std::optional<bool> Imp =
10038 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
10039 if (*Imp == false)
10040 return InvertRHS;
10041 }
10042
10043 return std::nullopt;
10044}
10045
10046// Returns a pair (Condition, ConditionIsTrue), where Condition is a branch
10047// condition dominating ContextI or nullptr, if no condition is found.
10048static std::pair<Value *, bool>
10049getDomPredecessorCondition(const Instruction *ContextI) {
10050 if (!ContextI || !ContextI->getParent())
10051 return {nullptr, false};
10052
10053 // TODO: This is a poor/cheap way to determine dominance. Should we use a
10054 // dominator tree (eg, from a SimplifyQuery) instead?
10055 const BasicBlock *ContextBB = ContextI->getParent();
10056 const BasicBlock *PredBB = ContextBB->getSinglePredecessor();
10057 if (!PredBB)
10058 return {nullptr, false};
10059
10060 // We need a conditional branch in the predecessor.
10061 Value *PredCond;
10062 BasicBlock *TrueBB, *FalseBB;
10063 if (!match(V: PredBB->getTerminator(), P: m_Br(C: m_Value(V&: PredCond), T&: TrueBB, F&: FalseBB)))
10064 return {nullptr, false};
10065
10066 // The branch should get simplified. Don't bother simplifying this condition.
10067 if (TrueBB == FalseBB)
10068 return {nullptr, false};
10069
10070 assert((TrueBB == ContextBB || FalseBB == ContextBB) &&
10071 "Predecessor block does not point to successor?");
10072
10073 // Is this condition implied by the predecessor condition?
10074 return {PredCond, TrueBB == ContextBB};
10075}
10076
10077std::optional<bool> llvm::isImpliedByDomCondition(const Value *Cond,
10078 const Instruction *ContextI,
10079 const DataLayout &DL) {
10080 assert(Cond->getType()->isIntOrIntVectorTy(1) && "Condition must be bool");
10081 auto PredCond = getDomPredecessorCondition(ContextI);
10082 if (PredCond.first)
10083 return isImpliedCondition(LHS: PredCond.first, RHS: Cond, DL, LHSIsTrue: PredCond.second);
10084 return std::nullopt;
10085}
10086
10087std::optional<bool> llvm::isImpliedByDomCondition(CmpPredicate Pred,
10088 const Value *LHS,
10089 const Value *RHS,
10090 const Instruction *ContextI,
10091 const DataLayout &DL) {
10092 auto PredCond = getDomPredecessorCondition(ContextI);
10093 if (PredCond.first)
10094 return isImpliedCondition(LHS: PredCond.first, RHSPred: Pred, RHSOp0: LHS, RHSOp1: RHS, DL,
10095 LHSIsTrue: PredCond.second);
10096 return std::nullopt;
10097}
10098
10099static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
10100 APInt &Upper, const InstrInfoQuery &IIQ,
10101 bool PreferSignedRange) {
10102 unsigned Width = Lower.getBitWidth();
10103 const APInt *C;
10104 switch (BO.getOpcode()) {
10105 case Instruction::Sub:
10106 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10107 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10108 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10109
10110 // If the caller expects a signed compare, then try to use a signed range.
10111 // Otherwise if both no-wraps are set, use the unsigned range because it
10112 // is never larger than the signed range. Example:
10113 // "sub nuw nsw i8 -2, x" is unsigned [0, 254] vs. signed [-128, 126].
10114 // "sub nuw nsw i8 2, x" is unsigned [0, 2] vs. signed [-125, 127].
10115 if (PreferSignedRange && HasNSW && HasNUW)
10116 HasNUW = false;
10117
10118 if (HasNUW) {
10119 // 'sub nuw c, x' produces [0, C].
10120 Upper = *C + 1;
10121 } else if (HasNSW) {
10122 if (C->isNegative()) {
10123 // 'sub nsw -C, x' produces [SINT_MIN, -C - SINT_MIN].
10124 Lower = APInt::getSignedMinValue(numBits: Width);
10125 Upper = *C - APInt::getSignedMaxValue(numBits: Width);
10126 } else {
10127 // Note that sub 0, INT_MIN is not NSW. It techically is a signed wrap
10128 // 'sub nsw C, x' produces [C - SINT_MAX, SINT_MAX].
10129 Lower = *C - APInt::getSignedMaxValue(numBits: Width);
10130 Upper = APInt::getSignedMinValue(numBits: Width);
10131 }
10132 }
10133 }
10134 break;
10135 case Instruction::Add:
10136 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10137 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10138 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10139
10140 // If the caller expects a signed compare, then try to use a signed
10141 // range. Otherwise if both no-wraps are set, use the unsigned range
10142 // because it is never larger than the signed range. Example: "add nuw
10143 // nsw i8 X, -2" is unsigned [254,255] vs. signed [-128, 125].
10144 if (PreferSignedRange && HasNSW && HasNUW)
10145 HasNUW = false;
10146
10147 if (HasNUW) {
10148 // 'add nuw x, C' produces [C, UINT_MAX].
10149 Lower = *C;
10150 } else if (HasNSW) {
10151 if (C->isNegative()) {
10152 // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C].
10153 Lower = APInt::getSignedMinValue(numBits: Width);
10154 Upper = APInt::getSignedMaxValue(numBits: Width) + *C + 1;
10155 } else {
10156 // 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX].
10157 Lower = APInt::getSignedMinValue(numBits: Width) + *C;
10158 Upper = APInt::getSignedMaxValue(numBits: Width) + 1;
10159 }
10160 }
10161 }
10162 break;
10163
10164 case Instruction::And:
10165 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10166 // 'and x, C' produces [0, C].
10167 Upper = *C + 1;
10168 // X & -X is a power of two or zero. So we can cap the value at max power of
10169 // two.
10170 if (match(V: BO.getOperand(i_nocapture: 0), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 1)))) ||
10171 match(V: BO.getOperand(i_nocapture: 1), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 0)))))
10172 Upper = APInt::getSignedMinValue(numBits: Width) + 1;
10173 break;
10174
10175 case Instruction::Or:
10176 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10177 // 'or x, C' produces [C, UINT_MAX].
10178 Lower = *C;
10179 break;
10180
10181 case Instruction::AShr:
10182 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10183 // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C].
10184 Lower = APInt::getSignedMinValue(numBits: Width).ashr(ShiftAmt: *C);
10185 Upper = APInt::getSignedMaxValue(numBits: Width).ashr(ShiftAmt: *C) + 1;
10186 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10187 unsigned ShiftAmount = Width - 1;
10188 if (!C->isZero() && IIQ.isExact(Op: &BO))
10189 ShiftAmount = C->countr_zero();
10190 if (C->isNegative()) {
10191 // 'ashr C, x' produces [C, C >> (Width-1)]
10192 Lower = *C;
10193 Upper = C->ashr(ShiftAmt: ShiftAmount) + 1;
10194 } else {
10195 // 'ashr C, x' produces [C >> (Width-1), C]
10196 Lower = C->ashr(ShiftAmt: ShiftAmount);
10197 Upper = *C + 1;
10198 }
10199 }
10200 break;
10201
10202 case Instruction::LShr:
10203 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10204 // 'lshr x, C' produces [0, UINT_MAX >> C].
10205 Upper = APInt::getAllOnes(numBits: Width).lshr(ShiftAmt: *C) + 1;
10206 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10207 // 'lshr C, x' produces [C >> (Width-1), C].
10208 unsigned ShiftAmount = Width - 1;
10209 if (!C->isZero() && IIQ.isExact(Op: &BO))
10210 ShiftAmount = C->countr_zero();
10211 Lower = C->lshr(shiftAmt: ShiftAmount);
10212 Upper = *C + 1;
10213 }
10214 break;
10215
10216 case Instruction::Shl:
10217 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10218 if (IIQ.hasNoUnsignedWrap(Op: &BO)) {
10219 // 'shl nuw C, x' produces [C, C << CLZ(C)]
10220 Lower = *C;
10221 Upper = Lower.shl(shiftAmt: Lower.countl_zero()) + 1;
10222 } else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw?
10223 if (C->isNegative()) {
10224 // 'shl nsw C, x' produces [C << CLO(C)-1, C]
10225 unsigned ShiftAmount = C->countl_one() - 1;
10226 Lower = C->shl(shiftAmt: ShiftAmount);
10227 Upper = *C + 1;
10228 } else {
10229 // 'shl nsw C, x' produces [C, C << CLZ(C)-1]
10230 unsigned ShiftAmount = C->countl_zero() - 1;
10231 Lower = *C;
10232 Upper = C->shl(shiftAmt: ShiftAmount) + 1;
10233 }
10234 } else {
10235 // If lowbit is set, value can never be zero.
10236 if ((*C)[0])
10237 Lower = APInt::getOneBitSet(numBits: Width, BitNo: 0);
10238 // If we are shifting a constant the largest it can be is if the longest
10239 // sequence of consecutive ones is shifted to the highbits (breaking
10240 // ties for which sequence is higher). At the moment we take a liberal
10241 // upper bound on this by just popcounting the constant.
10242 // TODO: There may be a bitwise trick for it longest/highest
10243 // consecutative sequence of ones (naive method is O(Width) loop).
10244 Upper = APInt::getHighBitsSet(numBits: Width, hiBitsSet: C->popcount()) + 1;
10245 }
10246 } else if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10247 Upper = APInt::getBitsSetFrom(numBits: Width, loBit: C->getZExtValue()) + 1;
10248 }
10249 break;
10250
10251 case Instruction::SDiv:
10252 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10253 APInt IntMin = APInt::getSignedMinValue(numBits: Width);
10254 APInt IntMax = APInt::getSignedMaxValue(numBits: Width);
10255 if (C->isAllOnes()) {
10256 // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX]
10257 // where C != -1 and C != 0 and C != 1
10258 Lower = IntMin + 1;
10259 Upper = IntMax + 1;
10260 } else if (C->countl_zero() < Width - 1) {
10261 // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C]
10262 // where C != -1 and C != 0 and C != 1
10263 Lower = IntMin.sdiv(RHS: *C);
10264 Upper = IntMax.sdiv(RHS: *C);
10265 if (Lower.sgt(RHS: Upper))
10266 std::swap(a&: Lower, b&: Upper);
10267 Upper = Upper + 1;
10268 assert(Upper != Lower && "Upper part of range has wrapped!");
10269 }
10270 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10271 if (C->isMinSignedValue()) {
10272 // 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2].
10273 Lower = *C;
10274 Upper = Lower.lshr(shiftAmt: 1) + 1;
10275 } else {
10276 // 'sdiv C, x' produces [-|C|, |C|].
10277 Upper = C->abs() + 1;
10278 Lower = (-Upper) + 1;
10279 }
10280 }
10281 break;
10282
10283 case Instruction::UDiv:
10284 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10285 // 'udiv x, C' produces [0, UINT_MAX / C].
10286 Upper = APInt::getMaxValue(numBits: Width).udiv(RHS: *C) + 1;
10287 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10288 // 'udiv C, x' produces [0, C].
10289 Upper = *C + 1;
10290 }
10291 break;
10292
10293 case Instruction::SRem:
10294 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10295 // 'srem x, C' produces (-|C|, |C|).
10296 Upper = C->abs();
10297 Lower = (-Upper) + 1;
10298 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10299 if (C->isNegative()) {
10300 // 'srem -|C|, x' produces [-|C|, 0].
10301 Upper = 1;
10302 Lower = *C;
10303 } else {
10304 // 'srem |C|, x' produces [0, |C|].
10305 Upper = *C + 1;
10306 }
10307 }
10308 break;
10309
10310 case Instruction::URem:
10311 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10312 // 'urem x, C' produces [0, C).
10313 Upper = *C;
10314 else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10315 // 'urem C, x' produces [0, C].
10316 Upper = *C + 1;
10317 break;
10318
10319 default:
10320 break;
10321 }
10322}
10323
10324static ConstantRange getRangeForIntrinsic(const IntrinsicInst &II,
10325 bool UseInstrInfo) {
10326 unsigned Width = II.getType()->getScalarSizeInBits();
10327 const APInt *C;
10328 switch (II.getIntrinsicID()) {
10329 case Intrinsic::ctlz:
10330 case Intrinsic::cttz: {
10331 APInt Upper(Width, Width);
10332 if (!UseInstrInfo || !match(V: II.getArgOperand(i: 1), P: m_One()))
10333 Upper += 1;
10334 // Maximum of set/clear bits is the bit width.
10335 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper);
10336 }
10337 case Intrinsic::ctpop:
10338 // Maximum of set/clear bits is the bit width.
10339 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10340 Upper: APInt(Width, Width) + 1);
10341 case Intrinsic::uadd_sat:
10342 // uadd.sat(x, C) produces [C, UINT_MAX].
10343 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10344 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10345 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10346 break;
10347 case Intrinsic::sadd_sat:
10348 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10349 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10350 if (C->isNegative())
10351 // sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)].
10352 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10353 Upper: APInt::getSignedMaxValue(numBits: Width) + *C +
10354 1);
10355
10356 // sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX].
10357 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) + *C,
10358 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10359 }
10360 break;
10361 case Intrinsic::usub_sat:
10362 // usub.sat(C, x) produces [0, C].
10363 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10364 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10365
10366 // usub.sat(x, C) produces [0, UINT_MAX - C].
10367 if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10368 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10369 Upper: APInt::getMaxValue(numBits: Width) - *C + 1);
10370 break;
10371 case Intrinsic::ssub_sat:
10372 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10373 if (C->isNegative())
10374 // ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)].
10375 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10376 Upper: *C - APInt::getSignedMinValue(numBits: Width) +
10377 1);
10378
10379 // ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX].
10380 return ConstantRange::getNonEmpty(Lower: *C - APInt::getSignedMaxValue(numBits: Width),
10381 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10382 } else if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10383 if (C->isNegative())
10384 // ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]:
10385 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) - *C,
10386 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10387
10388 // ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C].
10389 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10390 Upper: APInt::getSignedMaxValue(numBits: Width) - *C +
10391 1);
10392 }
10393 break;
10394 case Intrinsic::umin:
10395 case Intrinsic::umax:
10396 case Intrinsic::smin:
10397 case Intrinsic::smax:
10398 if (!match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) &&
10399 !match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10400 break;
10401
10402 switch (II.getIntrinsicID()) {
10403 case Intrinsic::umin:
10404 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10405 case Intrinsic::umax:
10406 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10407 case Intrinsic::smin:
10408 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10409 Upper: *C + 1);
10410 case Intrinsic::smax:
10411 return ConstantRange::getNonEmpty(Lower: *C,
10412 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10413 default:
10414 llvm_unreachable("Must be min/max intrinsic");
10415 }
10416 break;
10417 case Intrinsic::abs:
10418 // If abs of SIGNED_MIN is poison, then the result is [0..SIGNED_MAX],
10419 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10420 if (match(V: II.getOperand(i_nocapture: 1), P: m_One()))
10421 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10422 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10423
10424 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10425 Upper: APInt::getSignedMinValue(numBits: Width) + 1);
10426 case Intrinsic::vscale:
10427 if (!II.getParent() || !II.getFunction())
10428 break;
10429 return getVScaleRange(F: II.getFunction(), BitWidth: Width);
10430 default:
10431 break;
10432 }
10433
10434 return ConstantRange::getFull(BitWidth: Width);
10435}
10436
10437static ConstantRange getRangeForSelectPattern(const SelectInst &SI,
10438 const InstrInfoQuery &IIQ) {
10439 unsigned BitWidth = SI.getType()->getScalarSizeInBits();
10440 const Value *LHS = nullptr, *RHS = nullptr;
10441 SelectPatternResult R = matchSelectPattern(V: &SI, LHS, RHS);
10442 if (R.Flavor == SPF_UNKNOWN)
10443 return ConstantRange::getFull(BitWidth);
10444
10445 if (R.Flavor == SelectPatternFlavor::SPF_ABS) {
10446 // If the negation part of the abs (in RHS) has the NSW flag,
10447 // then the result of abs(X) is [0..SIGNED_MAX],
10448 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10449 if (match(V: RHS, P: m_Neg(V: m_Specific(V: LHS))) &&
10450 IIQ.hasNoSignedWrap(Op: cast<Instruction>(Val: RHS)))
10451 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10452 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10453
10454 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10455 Upper: APInt::getSignedMinValue(numBits: BitWidth) + 1);
10456 }
10457
10458 if (R.Flavor == SelectPatternFlavor::SPF_NABS) {
10459 // The result of -abs(X) is <= 0.
10460 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10461 Upper: APInt(BitWidth, 1));
10462 }
10463
10464 const APInt *C;
10465 if (!match(V: LHS, P: m_APInt(Res&: C)) && !match(V: RHS, P: m_APInt(Res&: C)))
10466 return ConstantRange::getFull(BitWidth);
10467
10468 switch (R.Flavor) {
10469 case SPF_UMIN:
10470 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth), Upper: *C + 1);
10471 case SPF_UMAX:
10472 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: BitWidth));
10473 case SPF_SMIN:
10474 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10475 Upper: *C + 1);
10476 case SPF_SMAX:
10477 return ConstantRange::getNonEmpty(Lower: *C,
10478 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10479 default:
10480 return ConstantRange::getFull(BitWidth);
10481 }
10482}
10483
10484static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) {
10485 // The maximum representable value of a half is 65504. For floats the maximum
10486 // value is 3.4e38 which requires roughly 129 bits.
10487 unsigned BitWidth = I->getType()->getScalarSizeInBits();
10488 if (!I->getOperand(i: 0)->getType()->getScalarType()->isHalfTy())
10489 return;
10490 if (isa<FPToSIInst>(Val: I) && BitWidth >= 17) {
10491 Lower = APInt(BitWidth, -65504, true);
10492 Upper = APInt(BitWidth, 65505);
10493 }
10494
10495 if (isa<FPToUIInst>(Val: I) && BitWidth >= 16) {
10496 // For a fptoui the lower limit is left as 0.
10497 Upper = APInt(BitWidth, 65505);
10498 }
10499}
10500
10501ConstantRange llvm::computeConstantRange(const Value *V, bool ForSigned,
10502 const SimplifyQuery &SQ,
10503 unsigned Depth) {
10504 assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction");
10505
10506 if (Depth == MaxAnalysisRecursionDepth)
10507 return ConstantRange::getFull(BitWidth: V->getType()->getScalarSizeInBits());
10508
10509 if (auto *C = dyn_cast<Constant>(Val: V))
10510 return C->toConstantRange();
10511
10512 unsigned BitWidth = V->getType()->getScalarSizeInBits();
10513 ConstantRange CR = ConstantRange::getFull(BitWidth);
10514 if (auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
10515 APInt Lower = APInt(BitWidth, 0);
10516 APInt Upper = APInt(BitWidth, 0);
10517 // TODO: Return ConstantRange.
10518 setLimitsForBinOp(BO: *BO, Lower, Upper, IIQ: SQ.IIQ, PreferSignedRange: ForSigned);
10519 CR = ConstantRange::getNonEmpty(Lower, Upper);
10520 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
10521 CR = getRangeForIntrinsic(II: *II, UseInstrInfo: SQ.IIQ.UseInstrInfo);
10522 else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
10523 ConstantRange CRTrue =
10524 computeConstantRange(V: SI->getTrueValue(), ForSigned, SQ, Depth: Depth + 1);
10525 ConstantRange CRFalse =
10526 computeConstantRange(V: SI->getFalseValue(), ForSigned, SQ, Depth: Depth + 1);
10527 CR = CRTrue.unionWith(CR: CRFalse);
10528 CR = CR.intersectWith(CR: getRangeForSelectPattern(SI: *SI, IIQ: SQ.IIQ));
10529 } else if (auto *TI = dyn_cast<TruncInst>(Val: V)) {
10530 ConstantRange SrcCR =
10531 computeConstantRange(V: TI->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
10532 CR = SrcCR.truncate(BitWidth);
10533 } else if (isa<FPToUIInst>(Val: V) || isa<FPToSIInst>(Val: V)) {
10534 APInt Lower = APInt(BitWidth, 0);
10535 APInt Upper = APInt(BitWidth, 0);
10536 // TODO: Return ConstantRange.
10537 setLimitForFPToI(I: cast<Instruction>(Val: V), Lower, Upper);
10538 CR = ConstantRange::getNonEmpty(Lower, Upper);
10539 } else if (const auto *A = dyn_cast<Argument>(Val: V))
10540 if (std::optional<ConstantRange> Range = A->getRange())
10541 CR = *Range;
10542
10543 if (auto *I = dyn_cast<Instruction>(Val: V)) {
10544 if (auto *Range = SQ.IIQ.getMetadata(I, KindID: LLVMContext::MD_range))
10545 CR = CR.intersectWith(CR: getConstantRangeFromMetadata(RangeMD: *Range));
10546
10547 Value *FrexpSrc;
10548 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
10549 if (std::optional<ConstantRange> Range = CB->getRange())
10550 CR = CR.intersectWith(CR: *Range);
10551 } else if (match(V: I, P: m_ExtractValue<1>(V: m_Intrinsic<Intrinsic::frexp>(
10552 Ops: m_Value(V&: FrexpSrc))))) {
10553 const fltSemantics &FltSem =
10554 FrexpSrc->getType()->getScalarType()->getFltSemantics();
10555 // It should be possible to implement this for any type, but this logic
10556 // only computes the range assuming standard subnormal handling.
10557 if (APFloat::isIEEELikeFP(FltSem)) {
10558 KnownFPClass KnownSrc = computeKnownFPClass(
10559 V: FrexpSrc, InterestedClasses: fcSubnormal | fcZero | fcNan | fcInf, SQ, Depth: Depth + 1);
10560
10561 // The exponent of frexp(NaN) and frexp(Inf) is unspecified. Only
10562 // constrain its range when the source can be neither.
10563 if (KnownSrc.isKnownNeverInfOrNaN()) {
10564 int MinExp = APFloat::semanticsMinExponent(FltSem) + 1;
10565
10566 // Offset to find the true minimum exponent value for a denormal.
10567 if (!KnownSrc.isKnownNeverSubnormal())
10568 MinExp -= (APFloat::semanticsPrecision(FltSem) - 1);
10569
10570 int MaxExp = APFloat::semanticsMaxExponent(FltSem) + 1;
10571
10572 auto [AdjustedMin, AdjustedMax, AdjustedMaxNonZero] =
10573 computeKnownExponentRangeFromContext(V: FrexpSrc, Q: SQ);
10574
10575 DenormalMode Mode = I->getFunction()->getDenormalMode(FPType: FltSem);
10576 bool NeverLogicalZero = KnownSrc.isKnownNeverLogicalZero(Mode);
10577
10578 MinExp = std::max(a: AdjustedMin, b: MinExp);
10579 MaxExp = std::min(a: NeverLogicalZero ? AdjustedMaxNonZero : AdjustedMax,
10580 b: MaxExp);
10581
10582 CR = ConstantRange::getNonEmpty(
10583 Lower: APInt(BitWidth, static_cast<int64_t>(MinExp), /*isSigned=*/true),
10584 Upper: APInt(BitWidth, static_cast<int64_t>(MaxExp) + 1,
10585 /*isSigned=*/true));
10586 }
10587 }
10588 }
10589 }
10590
10591 if (SQ.CxtI && SQ.AC) {
10592 // Try to restrict the range based on information from assumptions.
10593 for (auto &AssumeVH : SQ.AC->assumptionsFor(V)) {
10594 if (!AssumeVH)
10595 continue;
10596 CallInst *I = cast<CallInst>(Val&: AssumeVH);
10597 assert(I->getParent()->getParent() == SQ.CxtI->getParent()->getParent() &&
10598 "Got assumption for the wrong function!");
10599 assert(I->getIntrinsicID() == Intrinsic::assume &&
10600 "must be an assume intrinsic");
10601
10602 if (!isValidAssumeForContext(I, Q: SQ))
10603 continue;
10604 Value *Arg = I->getArgOperand(i: 0);
10605 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
10606 // Currently we just use information from comparisons.
10607 if (!Cmp || Cmp->getOperand(i_nocapture: 0) != V)
10608 continue;
10609 // TODO: Set "ForSigned" parameter via Cmp->isSigned()?
10610 ConstantRange RHS =
10611 computeConstantRange(V: Cmp->getOperand(i_nocapture: 1), /*ForSigned=*/false,
10612 SQ: SQ.getWithInstruction(I), Depth: Depth + 1);
10613 CR = CR.intersectWith(
10614 CR: ConstantRange::makeAllowedICmpRegion(Pred: Cmp->getCmpPredicate(), Other: RHS));
10615 }
10616 }
10617
10618 return CR;
10619}
10620
10621static void
10622addValueAffectedByCondition(Value *V,
10623 function_ref<void(Value *)> InsertAffected) {
10624 assert(V != nullptr);
10625 if (isa<Argument>(Val: V) || isa<GlobalValue>(Val: V)) {
10626 InsertAffected(V);
10627 } else if (auto *I = dyn_cast<Instruction>(Val: V)) {
10628 InsertAffected(V);
10629
10630 // Peek through unary operators to find the source of the condition.
10631 Value *Op;
10632 if (match(V: I, P: m_CombineOr(Ps: m_PtrToIntOrAddr(Op: m_Value(V&: Op)),
10633 Ps: m_Trunc(Op: m_Value(V&: Op))))) {
10634 if (isa<Instruction>(Val: Op) || isa<Argument>(Val: Op))
10635 InsertAffected(Op);
10636 }
10637 }
10638}
10639
10640void llvm::findValuesAffectedByCondition(
10641 Value *Cond, bool IsAssume, function_ref<void(Value *)> InsertAffected) {
10642 auto AddAffected = [&InsertAffected](Value *V) {
10643 addValueAffectedByCondition(V, InsertAffected);
10644 };
10645
10646 auto AddCmpOperands = [&AddAffected, IsAssume](Value *LHS, Value *RHS) {
10647 if (IsAssume) {
10648 AddAffected(LHS);
10649 AddAffected(RHS);
10650 } else if (match(V: RHS, P: m_Constant()))
10651 AddAffected(LHS);
10652 };
10653
10654 SmallVector<Value *, 8> Worklist;
10655 SmallPtrSet<Value *, 8> Visited;
10656 Worklist.push_back(Elt: Cond);
10657 while (!Worklist.empty()) {
10658 Value *V = Worklist.pop_back_val();
10659 if (!Visited.insert(Ptr: V).second)
10660 continue;
10661
10662 CmpPredicate Pred;
10663 Value *A, *B, *X;
10664
10665 if (IsAssume) {
10666 AddAffected(V);
10667 if (match(V, P: m_Not(V: m_Value(V&: X))))
10668 AddAffected(X);
10669 }
10670
10671 if (match(V, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
10672 // assume(A && B) is split to -> assume(A); assume(B);
10673 // assume(!(A || B)) is split to -> assume(!A); assume(!B);
10674 // Finally, assume(A || B) / assume(!(A && B)) generally don't provide
10675 // enough information to be worth handling (intersection of information as
10676 // opposed to union).
10677 if (!IsAssume) {
10678 Worklist.push_back(Elt: A);
10679 Worklist.push_back(Elt: B);
10680 }
10681 } else if (match(V, P: m_ICmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
10682 bool HasRHSC = match(V: B, P: m_ConstantInt());
10683 if (ICmpInst::isEquality(P: Pred)) {
10684 AddAffected(A);
10685 if (IsAssume)
10686 AddAffected(B);
10687 if (HasRHSC) {
10688 Value *Y;
10689 // (X << C) or (X >>_s C) or (X >>_u C).
10690 if (match(V: A, P: m_Shift(L: m_Value(V&: X), R: m_ConstantInt())))
10691 AddAffected(X);
10692 // (X & C) or (X | C).
10693 else if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10694 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10695 AddAffected(X);
10696 AddAffected(Y);
10697 }
10698 // X - Y
10699 else if (match(V: A, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10700 AddAffected(X);
10701 AddAffected(Y);
10702 }
10703 }
10704 } else {
10705 AddCmpOperands(A, B);
10706 if (HasRHSC) {
10707 // Handle (A + C1) u< C2, which is the canonical form of
10708 // A > C3 && A < C4.
10709 if (match(V: A, P: m_AddLike(L: m_Value(V&: X), R: m_ConstantInt())))
10710 AddAffected(X);
10711
10712 if (ICmpInst::isUnsigned(Pred)) {
10713 Value *Y;
10714 // X & Y u> C -> X >u C && Y >u C
10715 // X | Y u< C -> X u< C && Y u< C
10716 // X nuw+ Y u< C -> X u< C && Y u< C
10717 if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10718 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10719 match(V: A, P: m_NUWAdd(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10720 AddAffected(X);
10721 AddAffected(Y);
10722 }
10723 // X nuw- Y u> C -> X u> C
10724 if (match(V: A, P: m_NUWSub(L: m_Value(V&: X), R: m_Value())))
10725 AddAffected(X);
10726 }
10727 }
10728
10729 // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported
10730 // by computeKnownFPClass().
10731 if (match(V: A, P: m_ElementWiseBitCast(Op: m_Value(V&: X)))) {
10732 if (Pred == ICmpInst::ICMP_SLT && match(V: B, P: m_Zero()))
10733 InsertAffected(X);
10734 else if (Pred == ICmpInst::ICMP_SGT && match(V: B, P: m_AllOnes()))
10735 InsertAffected(X);
10736 }
10737 }
10738
10739 if (HasRHSC && match(V: A, P: m_Ctpop(Op0: m_Value(V&: X))))
10740 AddAffected(X);
10741 } else if (match(V, P: m_FCmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
10742 AddCmpOperands(A, B);
10743
10744 // fcmp fneg(x), y
10745 // fcmp fabs(x), y
10746 // fcmp fneg(fabs(x)), y
10747 if (match(V: A, P: m_FNeg(X: m_Value(V&: A))))
10748 AddAffected(A);
10749 if (match(V: A, P: m_FAbs(Op0: m_Value(V&: A))))
10750 AddAffected(A);
10751
10752 } else if (match(V, P: m_Intrinsic<Intrinsic::is_fpclass>(Ops: m_Value(V&: A),
10753 Ops: m_Value()))) {
10754 // Handle patterns that computeKnownFPClass() support.
10755 AddAffected(A);
10756 } else if (!IsAssume && match(V, P: m_Trunc(Op: m_Value(V&: X)))) {
10757 // Assume is checked here as X is already added above for assumes in
10758 // addValueAffectedByCondition
10759 AddAffected(X);
10760 } else if (!IsAssume && match(V, P: m_Not(V: m_Value(V&: X)))) {
10761 // Assume is checked here to avoid issues with ephemeral values
10762 Worklist.push_back(Elt: X);
10763 }
10764 }
10765}
10766
10767const Value *llvm::stripNullTest(const Value *V) {
10768 // (X >> C) or/add (X & mask(C) != 0)
10769 if (const auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
10770 if (BO->getOpcode() == Instruction::Add ||
10771 BO->getOpcode() == Instruction::Or) {
10772 const Value *X;
10773 const APInt *C1, *C2;
10774 if (match(V: BO, P: m_c_BinOp(L: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: C1)),
10775 R: m_ZExt(Op: m_SpecificICmp(
10776 MatchPred: ICmpInst::ICMP_NE,
10777 L: m_And(L: m_Deferred(V: X), R: m_LowBitMask(V&: C2)),
10778 R: m_Zero())))) &&
10779 C2->popcount() == C1->getZExtValue())
10780 return X;
10781 }
10782 }
10783 return nullptr;
10784}
10785
10786Value *llvm::stripNullTest(Value *V) {
10787 return const_cast<Value *>(stripNullTest(V: const_cast<const Value *>(V)));
10788}
10789
10790bool llvm::collectPossibleValues(const Value *V,
10791 SmallPtrSetImpl<const Constant *> &Constants,
10792 unsigned MaxCount, bool AllowUndefOrPoison) {
10793 SmallPtrSet<const Instruction *, 8> Visited;
10794 SmallVector<const Instruction *, 8> Worklist;
10795 auto Push = [&](const Value *V) -> bool {
10796 Constant *C;
10797 if (match(V: const_cast<Value *>(V), P: m_ImmConstant(C))) {
10798 if (!AllowUndefOrPoison && !isGuaranteedNotToBeUndefOrPoison(V: C))
10799 return false;
10800 // Check existence first to avoid unnecessary allocations.
10801 if (Constants.contains(Ptr: C))
10802 return true;
10803 if (Constants.size() == MaxCount)
10804 return false;
10805 Constants.insert(Ptr: C);
10806 return true;
10807 }
10808
10809 if (auto *Inst = dyn_cast<Instruction>(Val: V)) {
10810 if (Visited.insert(Ptr: Inst).second)
10811 Worklist.push_back(Elt: Inst);
10812 return true;
10813 }
10814 return false;
10815 };
10816 if (!Push(V))
10817 return false;
10818 while (!Worklist.empty()) {
10819 const Instruction *CurInst = Worklist.pop_back_val();
10820 switch (CurInst->getOpcode()) {
10821 case Instruction::Select:
10822 if (!Push(CurInst->getOperand(i: 1)))
10823 return false;
10824 if (!Push(CurInst->getOperand(i: 2)))
10825 return false;
10826 break;
10827 case Instruction::PHI:
10828 for (Value *IncomingValue : cast<PHINode>(Val: CurInst)->incoming_values()) {
10829 // Fast path for recurrence PHI.
10830 if (IncomingValue == CurInst)
10831 continue;
10832 if (!Push(IncomingValue))
10833 return false;
10834 }
10835 break;
10836 default:
10837 return false;
10838 }
10839 }
10840 return true;
10841}
10842