1//===- ValueTracking.cpp - Walk computations to compute properties --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains routines that help analyze properties that chains of
10// computations have.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ValueTracking.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/FloatingPointMode.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/ADT/SmallPtrSet.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/ADT/iterator_range.h"
24#include "llvm/Analysis/AliasAnalysis.h"
25#include "llvm/Analysis/AssumeBundleQueries.h"
26#include "llvm/Analysis/AssumptionCache.h"
27#include "llvm/Analysis/ConstantFolding.h"
28#include "llvm/Analysis/DomConditionCache.h"
29#include "llvm/Analysis/FloatingPointPredicateUtils.h"
30#include "llvm/Analysis/GuardUtils.h"
31#include "llvm/Analysis/InstructionSimplify.h"
32#include "llvm/Analysis/Loads.h"
33#include "llvm/Analysis/LoopInfo.h"
34#include "llvm/Analysis/TargetLibraryInfo.h"
35#include "llvm/Analysis/VectorUtils.h"
36#include "llvm/Analysis/WithCache.h"
37#include "llvm/IR/Argument.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/BasicBlock.h"
40#include "llvm/IR/BundleAttributes.h"
41#include "llvm/IR/Constant.h"
42#include "llvm/IR/ConstantFPRange.h"
43#include "llvm/IR/ConstantRange.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DerivedTypes.h"
46#include "llvm/IR/DiagnosticInfo.h"
47#include "llvm/IR/Dominators.h"
48#include "llvm/IR/EHPersonalities.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GetElementPtrTypeIterator.h"
51#include "llvm/IR/GlobalAlias.h"
52#include "llvm/IR/GlobalValue.h"
53#include "llvm/IR/GlobalVariable.h"
54#include "llvm/IR/InstrTypes.h"
55#include "llvm/IR/Instruction.h"
56#include "llvm/IR/Instructions.h"
57#include "llvm/IR/IntrinsicInst.h"
58#include "llvm/IR/Intrinsics.h"
59#include "llvm/IR/IntrinsicsAArch64.h"
60#include "llvm/IR/IntrinsicsAMDGPU.h"
61#include "llvm/IR/IntrinsicsRISCV.h"
62#include "llvm/IR/IntrinsicsX86.h"
63#include "llvm/IR/LLVMContext.h"
64#include "llvm/IR/Metadata.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/Operator.h"
67#include "llvm/IR/PatternMatch.h"
68#include "llvm/IR/Type.h"
69#include "llvm/IR/User.h"
70#include "llvm/IR/Value.h"
71#include "llvm/Support/Casting.h"
72#include "llvm/Support/CommandLine.h"
73#include "llvm/Support/Compiler.h"
74#include "llvm/Support/ErrorHandling.h"
75#include "llvm/Support/KnownBits.h"
76#include "llvm/Support/KnownFPClass.h"
77#include "llvm/Support/MathExtras.h"
78#include "llvm/Support/UndefPoison.h"
79#include "llvm/TargetParser/RISCVTargetParser.h"
80#include <algorithm>
81#include <cassert>
82#include <cstdint>
83#include <optional>
84#include <utility>
85
86using namespace llvm;
87using namespace llvm::PatternMatch;
88
89// Controls the number of uses of the value searched for possible
90// dominating comparisons.
91static cl::opt<unsigned> DomConditionsMaxUses("dom-conditions-max-uses",
92 cl::Hidden, cl::init(Val: 20));
93
94/// Maximum number of instructions to check between assume and context
95/// instruction.
96static constexpr unsigned MaxInstrsToCheckForFree = 32;
97
98template <typename InstTy>
99static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
100 Value *&Init, Value *&OtherOp);
101
102/// Returns the bitwidth of the given scalar or pointer type. For vector types,
103/// returns the element type's bitwidth.
104static unsigned getBitWidth(Type *Ty, const DataLayout &DL) {
105 if (unsigned BitWidth = Ty->getScalarSizeInBits())
106 return BitWidth;
107
108 return DL.getPointerTypeSizeInBits(Ty);
109}
110
111// Given the provided Value and, potentially, a context instruction, return
112// the preferred context instruction (if any).
113static const Instruction *safeCtxI(const Value *V, const Instruction *CtxI) {
114 // If we've been provided with a context instruction, then use that (provided
115 // it has been inserted).
116 if (CtxI && CtxI->getParent())
117 return CtxI;
118
119 // If the value is really an already-inserted instruction, then use that.
120 CtxI = dyn_cast<Instruction>(Val: V);
121 if (CtxI && CtxI->getParent())
122 return CtxI;
123
124 return nullptr;
125}
126
127static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
128 const APInt &DemandedElts,
129 APInt &DemandedLHS, APInt &DemandedRHS) {
130 if (isa<ScalableVectorType>(Val: Shuf->getType())) {
131 assert(DemandedElts == APInt(1,1));
132 DemandedLHS = DemandedRHS = DemandedElts;
133 return true;
134 }
135
136 int NumElts =
137 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
138 return llvm::getShuffleDemandedElts(SrcWidth: NumElts, Mask: Shuf->getShuffleMask(),
139 DemandedElts, DemandedLHS, DemandedRHS);
140}
141
142static void computeKnownBits(const Value *V, const APInt &DemandedElts,
143 KnownBits &Known, const SimplifyQuery &Q,
144 unsigned Depth);
145
146void llvm::computeKnownBits(const Value *V, KnownBits &Known,
147 const SimplifyQuery &Q, unsigned Depth) {
148 // Since the number of lanes in a scalable vector is unknown at compile time,
149 // we track one bit which is implicitly broadcast to all lanes. This means
150 // that all lanes in a scalable vector are considered demanded.
151 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
152 APInt DemandedElts =
153 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
154 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
155}
156
157void llvm::computeKnownBits(const Value *V, KnownBits &Known,
158 const DataLayout &DL, AssumptionCache *AC,
159 const Instruction *CtxI, const DominatorTree *DT,
160 bool UseInstrInfo, unsigned Depth) {
161 computeKnownBits(V, Known,
162 Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo),
163 Depth);
164}
165
166KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
167 AssumptionCache *AC, const Instruction *CtxI,
168 const DominatorTree *DT, bool UseInstrInfo,
169 unsigned Depth) {
170 return computeKnownBits(
171 V, Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo), Depth);
172}
173
174static NoCommonBitsSetResult
175haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
176 const SimplifyQuery &SQ) {
177 // Look for an inverted mask: (X & ~M) op (Y & M).
178 {
179 Value *M;
180 if (match(V: LHS, P: m_c_And(L: m_Not(V: m_Value(V&: M)), R: m_Value())) &&
181 match(V: RHS, P: m_c_And(L: m_Specific(V: M), R: m_Value())))
182 return isGuaranteedNotToBeUndef(V: M, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)
183 ? NoCommonBitsSetResult::Known
184 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
185 }
186
187 // X op (Y & ~X)
188 if (match(V: RHS, P: m_c_And(L: m_Not(V: m_Specific(V: LHS)), R: m_Value())))
189 return isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)
190 ? NoCommonBitsSetResult::Known
191 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
192
193 // X op ((X & Y) ^ Y) -- this is the canonical form of the previous pattern
194 // for constant Y.
195 Value *Y;
196 if (match(V: RHS,
197 P: m_c_Xor(L: m_c_And(L: m_Specific(V: LHS), R: m_Value(V&: Y)), R: m_Deferred(V: Y)))) {
198 bool IsNoUndef = isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT) &&
199 isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT);
200 return IsNoUndef ? NoCommonBitsSetResult::Known
201 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
202 }
203
204 // Peek through extends to find a 'not' of the other side:
205 // (ext Y) op ext(~Y)
206 if (match(V: LHS, P: m_ZExtOrSExt(Op: m_Value(V&: Y))) &&
207 match(V: RHS, P: m_ZExtOrSExt(Op: m_Not(V: m_Specific(V: Y)))))
208 return isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)
209 ? NoCommonBitsSetResult::Known
210 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
211
212 // Look for: (A & B) op ~(A | B)
213 {
214 Value *A, *B;
215 if (match(V: LHS, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
216 match(V: RHS, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))) {
217 bool IsNoUndef = isGuaranteedNotToBeUndef(V: A, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT) &&
218 isGuaranteedNotToBeUndef(V: B, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT);
219 return IsNoUndef ? NoCommonBitsSetResult::Known
220 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
221 }
222 }
223
224 // Look for: (X << V) op (Y >> (BitWidth - V))
225 // or (X >> V) op (Y << (BitWidth - V))
226 {
227 const Value *V;
228 const APInt *R;
229 if (((match(V: RHS, P: m_Shl(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
230 match(V: LHS, P: m_LShr(L: m_Value(), R: m_Specific(V)))) ||
231 (match(V: RHS, P: m_LShr(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
232 match(V: LHS, P: m_Shl(L: m_Value(), R: m_Specific(V))))) &&
233 R->uge(RHS: LHS->getType()->getScalarSizeInBits()))
234 return NoCommonBitsSetResult::Known;
235 }
236
237 return NoCommonBitsSetResult::Unknown;
238}
239
240NoCommonBitsSetResult
241llvm::getNoCommonBitsSetResult(const WithCache<const Value *> &LHSCache,
242 const WithCache<const Value *> &RHSCache,
243 const SimplifyQuery &SQ) {
244 const Value *LHS = LHSCache.getValue();
245 const Value *RHS = RHSCache.getValue();
246
247 assert(LHS->getType() == RHS->getType() &&
248 "LHS and RHS should have the same type");
249 assert(LHS->getType()->isIntOrIntVectorTy() &&
250 "LHS and RHS should be integers");
251
252 NoCommonBitsSetResult Result = haveNoCommonBitsSetSpecialCases(LHS, RHS, SQ);
253 if (Result == NoCommonBitsSetResult::Known)
254 return NoCommonBitsSetResult::Known;
255
256 NoCommonBitsSetResult CommuteResult =
257 haveNoCommonBitsSetSpecialCases(LHS: RHS, RHS: LHS, SQ);
258 if (CommuteResult == NoCommonBitsSetResult::Known)
259 return NoCommonBitsSetResult::Known;
260
261 if (KnownBits::haveNoCommonBitsSet(LHS: LHSCache.getKnownBits(Q: SQ),
262 RHS: RHSCache.getKnownBits(Q: SQ)))
263 return NoCommonBitsSetResult::Known;
264
265 if (Result == NoCommonBitsSetResult::OnlyIfUndefIgnored ||
266 CommuteResult == NoCommonBitsSetResult::OnlyIfUndefIgnored)
267 return NoCommonBitsSetResult::OnlyIfUndefIgnored;
268
269 return NoCommonBitsSetResult::Unknown;
270}
271
272bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
273 const WithCache<const Value *> &RHSCache,
274 const SimplifyQuery &SQ) {
275 NoCommonBitsSetResult Result =
276 getNoCommonBitsSetResult(LHSCache, RHSCache, SQ);
277 return Result == NoCommonBitsSetResult::Known;
278}
279
280bool llvm::isOnlyUsedInZeroComparison(const Instruction *I) {
281 return !I->user_empty() &&
282 all_of(Range: I->users(), P: match_fn(P: m_ICmp(L: m_Value(), R: m_Zero())));
283}
284
285bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) {
286 return !I->user_empty() && all_of(Range: I->users(), P: [](const User *U) {
287 CmpPredicate P;
288 return match(V: U, P: m_ICmp(Pred&: P, L: m_Value(), R: m_Zero())) && ICmpInst::isEquality(P);
289 });
290}
291
292bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
293 bool OrZero, AssumptionCache *AC,
294 const Instruction *CtxI,
295 const DominatorTree *DT, bool UseInstrInfo,
296 unsigned Depth) {
297 return ::isKnownToBeAPowerOfTwo(
298 V, OrZero, Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo),
299 Depth);
300}
301
302static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
303 const SimplifyQuery &Q, unsigned Depth);
304
305bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
306 unsigned Depth) {
307 return computeKnownBits(V, Q: SQ, Depth).isNonNegative();
308}
309
310bool llvm::isKnownPositive(const WithCache<const Value *> &V,
311 const SimplifyQuery &SQ, unsigned Depth) {
312 if (auto *CI = dyn_cast<ConstantInt>(Val: V.getValue()))
313 return CI->getValue().isStrictlyPositive();
314
315 // If `isKnownNonNegative` ever becomes more sophisticated, make sure to keep
316 // this updated.
317 KnownBits Known = Depth == 0 ? V.getKnownBits(Q: SQ)
318 : computeKnownBits(V: V.getValue(), Q: SQ, Depth);
319 return Known.isNonNegative() &&
320 (Known.isNonZero() || isKnownNonZero(V: V.getValue(), Q: SQ, Depth));
321}
322
323bool llvm::isKnownNegative(const WithCache<const Value *> &V,
324 const SimplifyQuery &SQ, unsigned Depth) {
325 if (Depth == 0)
326 return V.getKnownBits(Q: SQ).isNegative();
327 return computeKnownBits(V: V.getValue(), Q: SQ, Depth).isNegative();
328}
329
330static bool isKnownNonEqual(const Value *V1, const Value *V2,
331 const APInt &DemandedElts, const SimplifyQuery &Q,
332 unsigned Depth);
333
334static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
335 const Value *RHS);
336
337bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
338 const SimplifyQuery &Q, unsigned Depth) {
339 // We don't support looking through casts.
340 if (V1 == V2 || V1->getType() != V2->getType())
341 return false;
342 auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType());
343 APInt DemandedElts =
344 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
345 return ::isKnownNonEqual(V1, V2, DemandedElts, Q, Depth);
346}
347
348bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
349 const SimplifyQuery &SQ, unsigned Depth) {
350 KnownBits Known(Mask.getBitWidth());
351 computeKnownBits(V, Known, Q: SQ, Depth);
352 return Mask.isSubsetOf(RHS: Known.Zero);
353}
354
355static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
356 const SimplifyQuery &Q, unsigned Depth);
357
358static unsigned ComputeNumSignBits(const Value *V, const SimplifyQuery &Q,
359 unsigned Depth = 0) {
360 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
361 APInt DemandedElts =
362 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
363 return ComputeNumSignBits(V, DemandedElts, Q, Depth);
364}
365
366unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL,
367 AssumptionCache *AC, const Instruction *CtxI,
368 const DominatorTree *DT, bool UseInstrInfo,
369 unsigned Depth) {
370 return ::ComputeNumSignBits(
371 V, Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo), Depth);
372}
373
374unsigned llvm::ComputeMaxSignificantBits(const Value *V, const DataLayout &DL,
375 AssumptionCache *AC,
376 const Instruction *CtxI,
377 const DominatorTree *DT,
378 unsigned Depth) {
379 unsigned SignBits = ComputeNumSignBits(V, DL, AC, CtxI, DT, UseInstrInfo: Depth);
380 return V->getType()->getScalarSizeInBits() - SignBits + 1;
381}
382
383/// Try to detect the lerp pattern: a * (b - c) + c * d
384/// where a >= 0, b >= 0, c >= 0, d >= 0, and b >= c.
385///
386/// In that particular case, we can use the following chain of reasoning:
387///
388/// a * (b - c) + c * d <= a' * (b - c) + a' * c = a' * b where a' = max(a, d)
389///
390/// Since that is true for arbitrary a, b, c and d within our constraints, we
391/// can conclude that:
392///
393/// max(a * (b - c) + c * d) <= max(max(a), max(d)) * max(b) = U
394///
395/// Considering that any result of the lerp would be less or equal to U, it
396/// would have at least the number of leading 0s as in U.
397///
398/// While being quite a specific situation, it is fairly common in computer
399/// graphics in the shape of alpha blending.
400///
401/// Modifies given KnownOut in-place with the inferred information.
402static void computeKnownBitsFromLerpPattern(const Value *Op0, const Value *Op1,
403 const APInt &DemandedElts,
404 KnownBits &KnownOut,
405 const SimplifyQuery &Q,
406 unsigned Depth) {
407
408 Type *Ty = Op0->getType();
409 const unsigned BitWidth = Ty->getScalarSizeInBits();
410
411 // Only handle scalar types for now
412 if (Ty->isVectorTy())
413 return;
414
415 // Try to match: a * (b - c) + c * d.
416 // When a == 1 => A == nullptr, the same applies to d/D as well.
417 const Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
418 const Instruction *SubBC = nullptr;
419
420 const auto MatchSubBC = [&]() {
421 // (b - c) can have two forms that interest us:
422 //
423 // 1. sub nuw %b, %c
424 // 2. xor %c, %b
425 //
426 // For the first case, nuw flag guarantees our requirement b >= c.
427 //
428 // The second case might happen when the analysis can infer that b is a mask
429 // for c and we can transform sub operation into xor (that is usually true
430 // for constant b's). Even though xor is symmetrical, canonicalization
431 // ensures that the constant will be the RHS. We have additional checks
432 // later on to ensure that this xor operation is equivalent to subtraction.
433 return m_Instruction(I&: SubBC, P: m_CombineOr(Ps: m_NUWSub(L: m_Value(V&: B), R: m_Value(V&: C)),
434 Ps: m_Xor(L: m_Value(V&: C), R: m_Value(V&: B))));
435 };
436
437 const auto MatchASubBC = [&]() {
438 // Cases:
439 // - a * (b - c)
440 // - (b - c) * a
441 // - (b - c) <- a implicitly equals 1
442 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: A), R: MatchSubBC()), Ps: MatchSubBC());
443 };
444
445 const auto MatchCD = [&]() {
446 // Cases:
447 // - d * c
448 // - c * d
449 // - c <- d implicitly equals 1
450 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: D), R: m_Specific(V: C)), Ps: m_Specific(V: C));
451 };
452
453 const auto Match = [&](const Value *LHS, const Value *RHS) {
454 // We do use m_Specific(C) in MatchCD, so we have to make sure that
455 // it's bound to anything and match(LHS, MatchASubBC()) absolutely
456 // has to evaluate first and return true.
457 //
458 // If Match returns true, it is guaranteed that B != nullptr, C != nullptr.
459 return match(V: LHS, P: MatchASubBC()) && match(V: RHS, P: MatchCD());
460 };
461
462 if (!Match(Op0, Op1) && !Match(Op1, Op0))
463 return;
464
465 const auto ComputeKnownBitsOrOne = [&](const Value *V) {
466 // For some of the values we use the convention of leaving
467 // it nullptr to signify an implicit constant 1.
468 return V ? computeKnownBits(V, DemandedElts, Q, Depth: Depth + 1)
469 : KnownBits::makeConstant(C: APInt(BitWidth, 1));
470 };
471
472 // Check that all operands are non-negative
473 const KnownBits KnownA = ComputeKnownBitsOrOne(A);
474 if (!KnownA.isNonNegative())
475 return;
476
477 const KnownBits KnownD = ComputeKnownBitsOrOne(D);
478 if (!KnownD.isNonNegative())
479 return;
480
481 const KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
482 if (!KnownB.isNonNegative())
483 return;
484
485 const KnownBits KnownC = computeKnownBits(V: C, DemandedElts, Q, Depth: Depth + 1);
486 if (!KnownC.isNonNegative())
487 return;
488
489 // If we matched subtraction as xor, we need to actually check that xor
490 // is semantically equivalent to subtraction.
491 //
492 // For that to be true, b has to be a mask for c or that b's known
493 // ones cover all known and possible ones of c.
494 if (SubBC->getOpcode() == Instruction::Xor &&
495 !KnownC.getMaxValue().isSubsetOf(RHS: KnownB.getMinValue()))
496 return;
497
498 const APInt MaxA = KnownA.getMaxValue();
499 const APInt MaxD = KnownD.getMaxValue();
500 const APInt MaxAD = APIntOps::umax(A: MaxA, B: MaxD);
501 const APInt MaxB = KnownB.getMaxValue();
502
503 // We can't infer leading zeros info if the upper-bound estimate wraps.
504 bool Overflow;
505 const APInt UpperBound = MaxAD.umul_ov(RHS: MaxB, Overflow);
506
507 if (Overflow)
508 return;
509
510 // If we know that x <= y and both are positive than x has at least the same
511 // number of leading zeros as y.
512 const unsigned MinimumNumberOfLeadingZeros = UpperBound.countl_zero();
513 KnownOut.Zero.setHighBits(MinimumNumberOfLeadingZeros);
514}
515
516/// Infer non-negativity for R + (R < 0 ? A : 0), where R = srem X, Y.
517/// A must be a non-negative Y or abs(Y).
518static void computeKnownBitsFromEuclideanRem(const Value *Op0, const Value *Op1,
519 KnownBits &KnownOut,
520 const SimplifyQuery &Q,
521 unsigned Depth) {
522 Type *Ty = Op0->getType();
523
524 // Only handle scalars for now.
525 if (Ty->isVectorTy())
526 return;
527
528 const Value *Divisor = nullptr, *Cond = nullptr, *Remainder = nullptr,
529 *AdjustmentValue = nullptr;
530 CmpPredicate Pred;
531 const APInt *Threshold = nullptr;
532 bool TrueIfNegative = false;
533
534 const auto Match = [&](const Value *Rem, const Value *Adjustment) {
535 Remainder = Rem;
536
537 // Match an adjustment added only when this remainder is negative.
538 return match(V: Rem, P: m_SRem(L: m_Value(), R: m_Value(V&: Divisor))) &&
539 match(V: Adjustment,
540 P: m_Select(C: m_Value(V&: Cond), L: m_Value(V&: AdjustmentValue), R: m_Zero())) &&
541 match(V: Cond, P: m_ICmp(Pred, L: m_Specific(V: Rem), R: m_APInt(Res&: Threshold))) &&
542 isSignBitCheck(Pred, RHS: *Threshold, TrueIfSigned&: TrueIfNegative) && TrueIfNegative;
543 };
544
545 if (!Match(Op0, Op1) && !Match(Op1, Op0))
546 return;
547
548 // Accept Y itself if non-negative, or abs(Y).
549 bool IsPositiveDivisorCase =
550 Divisor == AdjustmentValue && isKnownNonNegative(V: Divisor, SQ: Q, Depth: Depth + 1);
551 bool IsAbsoluteValueCase =
552 match(V: AdjustmentValue,
553 P: m_Intrinsic<Intrinsic::abs>(Ops: m_Specific(V: Divisor), Ops: m_Value()));
554
555 if (!IsPositiveDivisorCase && !IsAbsoluteValueCase)
556 return;
557
558 // Repeated uses must agree. An undef value could break this relationship.
559 if (!isGuaranteedNotToBeUndef(V: Remainder, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1) ||
560 !isGuaranteedNotToBeUndef(V: Divisor, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1))
561 return;
562
563 // For Y other than signed minimum, adding abs(Y) to a negative remainder
564 // gives a non-negative result without overflow. For signed minimum, the
565 // wrapping addition clears the remainder's sign bit. Poison-producing cases
566 // do not invalidate this deduction.
567 KnownOut.makeNonNegative();
568}
569
570static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
571 bool NSW, bool NUW,
572 const APInt &DemandedElts,
573 KnownBits &KnownOut, KnownBits &Known2,
574 const SimplifyQuery &Q, unsigned Depth) {
575 computeKnownBits(V: Op1, DemandedElts, Known&: KnownOut, Q, Depth: Depth + 1);
576
577 if (KnownOut.isUnknown() && !NSW && !NUW) {
578 if (Add) {
579 // Independent operand bits give no result, but the Euclidean remainder
580 // pattern can establish non-negativity through the operands'
581 // relationship.
582 computeKnownBitsFromEuclideanRem(Op0, Op1, KnownOut, Q, Depth);
583 }
584 return;
585 }
586
587 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
588 KnownOut = KnownBits::computeForAddSub(Add, NSW, NUW, LHS: Known2, RHS: KnownOut);
589
590 if (!Add && NSW && !KnownOut.isNonNegative() &&
591 (isImpliedByDomCondition(Pred: ICmpInst::ICMP_SLE, LHS: Op1, RHS: Op0, ContextI: Q.CtxI, DL: Q.DL)
592 .value_or(u: false) ||
593 match(V: Op1, P: m_c_SMin(L: m_Specific(V: Op0), R: m_Value()))))
594 KnownOut.makeNonNegative();
595
596 if (Add) {
597 // Try to match lerp pattern and combine results
598 computeKnownBitsFromLerpPattern(Op0, Op1, DemandedElts, KnownOut, Q, Depth);
599
600 // Use the relationship between the remainder and its adjustment.
601 computeKnownBitsFromEuclideanRem(Op0, Op1, KnownOut, Q, Depth);
602 }
603}
604
605static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
606 bool NUW, const APInt &DemandedElts,
607 KnownBits &Known, KnownBits &Known2,
608 const SimplifyQuery &Q, unsigned Depth) {
609 computeKnownBits(V: Op1, DemandedElts, Known, Q, Depth: Depth + 1);
610 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
611
612 bool isKnownNegative = false;
613 bool isKnownNonNegative = false;
614 // If the multiplication is known not to overflow, compute the sign bit.
615 if (NSW) {
616 if (Op0 == Op1) {
617 // The product of a number with itself is non-negative.
618 isKnownNonNegative = true;
619 } else {
620 bool isKnownNonNegativeOp1 = Known.isNonNegative();
621 bool isKnownNonNegativeOp0 = Known2.isNonNegative();
622 bool isKnownNegativeOp1 = Known.isNegative();
623 bool isKnownNegativeOp0 = Known2.isNegative();
624 // The product of two numbers with the same sign is non-negative.
625 isKnownNonNegative = (isKnownNegativeOp1 && isKnownNegativeOp0) ||
626 (isKnownNonNegativeOp1 && isKnownNonNegativeOp0);
627 if (!isKnownNonNegative && NUW) {
628 // mul nuw nsw with a factor > 1 is non-negative.
629 KnownBits One = KnownBits::makeConstant(C: APInt(Known.getBitWidth(), 1));
630 isKnownNonNegative = KnownBits::sgt(LHS: Known, RHS: One).value_or(u: false) ||
631 KnownBits::sgt(LHS: Known2, RHS: One).value_or(u: false);
632 }
633
634 // The product of a negative number and a non-negative number is either
635 // negative or zero.
636 if (!isKnownNonNegative)
637 isKnownNegative =
638 (isKnownNegativeOp1 && isKnownNonNegativeOp0 &&
639 Known2.isNonZero()) ||
640 (isKnownNegativeOp0 && isKnownNonNegativeOp1 && Known.isNonZero());
641 }
642 }
643
644 bool SelfMultiply = Op0 == Op1;
645 if (SelfMultiply)
646 SelfMultiply &=
647 isGuaranteedNotToBeUndef(V: Op0, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1);
648 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
649
650 if (SelfMultiply) {
651 unsigned SignBits = ComputeNumSignBits(V: Op0, DemandedElts, Q, Depth: Depth + 1);
652 unsigned TyBits = Op0->getType()->getScalarSizeInBits();
653 unsigned OutValidBits = 2 * (TyBits - SignBits + 1);
654
655 if (OutValidBits < TyBits) {
656 APInt KnownZeroMask =
657 APInt::getHighBitsSet(numBits: TyBits, hiBitsSet: TyBits - OutValidBits + 1);
658 Known.Zero |= KnownZeroMask;
659 }
660 }
661
662 // Only make use of no-wrap flags if we failed to compute the sign bit
663 // directly. This matters if the multiplication always overflows, in
664 // which case we prefer to follow the result of the direct computation,
665 // though as the program is invoking undefined behaviour we can choose
666 // whatever we like here.
667 if (isKnownNonNegative && !Known.isNegative())
668 Known.makeNonNegative();
669 else if (isKnownNegative && !Known.isNonNegative())
670 Known.makeNegative();
671}
672
673void llvm::computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
674 KnownBits &Known) {
675 unsigned BitWidth = Known.getBitWidth();
676 unsigned NumRanges = Ranges.getNumOperands() / 2;
677 assert(NumRanges >= 1);
678
679 Known.setAllConflict();
680
681 for (unsigned i = 0; i < NumRanges; ++i) {
682 ConstantInt *Lower =
683 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 0));
684 ConstantInt *Upper =
685 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 1));
686 ConstantRange Range(Lower->getValue(), Upper->getValue());
687 // BitWidth must equal the Ranges BitWidth for the correct number of high
688 // bits to be set.
689 assert(BitWidth == Range.getBitWidth() &&
690 "Known bit width must match range bit width!");
691
692 // The first CommonPrefixBits of all values in Range are equal.
693 unsigned CommonPrefixBits =
694 (Range.getUnsignedMax() ^ Range.getUnsignedMin()).countl_zero();
695 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: CommonPrefixBits);
696 APInt UnsignedMax = Range.getUnsignedMax().zextOrTrunc(width: BitWidth);
697 Known.One &= UnsignedMax & Mask;
698 Known.Zero &= ~UnsignedMax & Mask;
699 }
700}
701
702static bool isEphemeralValueOf(const Instruction *I, const Value *E) {
703 // The instruction defining an assumption's condition itself is always
704 // considered ephemeral to that assumption (even if it has other
705 // non-ephemeral users). See r246696's test case for an example.
706 if (is_contained(Range: I->operands(), Element: E))
707 return true;
708
709 const auto *EI = dyn_cast<Instruction>(Val: E);
710 if (!EI)
711 return false;
712
713 if (EI == I)
714 return true;
715
716 SmallPtrSet<const Instruction *, 16> Visited;
717 SmallVector<const Instruction *, 16> WorkList;
718 Visited.insert(Ptr: EI);
719 WorkList.push_back(Elt: EI);
720 bool ReachesI = false;
721 while (!WorkList.empty()) {
722 const Instruction *V = WorkList.pop_back_val();
723 for (const User *U : V->users()) {
724 const auto *UI = cast<Instruction>(Val: U);
725 if (UI == I) {
726 ReachesI = true;
727 continue;
728 }
729 if (UI->mayHaveSideEffects() || UI->isTerminator())
730 return false;
731 if (Visited.insert(Ptr: UI).second)
732 WorkList.push_back(Elt: UI);
733 }
734 }
735 return ReachesI;
736}
737
738// Is this an intrinsic that cannot be speculated but also cannot trap?
739bool llvm::isAssumeLikeIntrinsic(const Instruction *I) {
740 if (const IntrinsicInst *CI = dyn_cast<IntrinsicInst>(Val: I))
741 return CI->isAssumeLikeIntrinsic();
742
743 return false;
744}
745
746bool llvm::isValidAssumeForContext(const Instruction *Inv,
747 const Instruction *CtxI,
748 const DominatorTree *DT,
749 bool AllowEphemerals) {
750 // There are two restrictions on the use of an assume:
751 // 1. The assume must dominate the context (or the control flow must
752 // reach the assume whenever it reaches the context).
753 // 2. The context must not be in the assume's set of ephemeral values
754 // (otherwise we will use the assume to prove that the condition
755 // feeding the assume is trivially true, thus causing the removal of
756 // the assume).
757
758 // If the checked instruction and the assume are the same, it's valid exactly
759 // if ephemeral values are allowed.
760 if (Inv == CtxI)
761 return AllowEphemerals;
762
763 if (Inv->getParent() == CtxI->getParent()) {
764 // If Inv and CtxI are in the same block, check if the assume (Inv) is first
765 // in the BB.
766 if (Inv->comesBefore(Other: CtxI))
767 return true;
768
769 // The context comes first, but they're both in the same block.
770 // Make sure there is nothing in between that might interrupt
771 // the control flow, not even CtxI itself.
772 // We limit the scan distance between the assume and its context instruction
773 // to avoid a compile-time explosion. This limit is chosen arbitrarily, so
774 // it can be adjusted if needed (could be turned into a cl::opt).
775 auto Range = make_range(x: CtxI->getIterator(), y: Inv->getIterator());
776 if (!isGuaranteedToTransferExecutionToSuccessor(Range, ScanLimit: 15))
777 return false;
778
779 return AllowEphemerals || !isEphemeralValueOf(I: Inv, E: CtxI);
780 }
781
782 // Inv and CtxI are in different blocks.
783 if (DT) {
784 if (DT->dominates(Def: Inv, User: CtxI))
785 return true;
786 } else if (Inv->getParent() == CtxI->getParent()->getSinglePredecessor() ||
787 Inv->getParent()->isEntryBlock()) {
788 // We don't have a DT, but this trivially dominates.
789 return true;
790 }
791
792 return false;
793}
794
795static bool hasNoFreeInRange(BasicBlock::const_iterator Begin,
796 BasicBlock::const_iterator End,
797 unsigned &NumChecked) {
798 for (const Instruction &I : make_range(x: Begin, y: End)) {
799 if (NumChecked++ > MaxInstrsToCheckForFree)
800 return false;
801 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
802 if (!CB->hasFnAttr(Kind: Attribute::NoFree))
803 return false;
804 } else if (I.maySynchronize()) {
805 return false;
806 }
807 }
808 return true;
809}
810
811bool llvm::willNotFreeBetween(const Instruction *Assume,
812 const Instruction *CtxI,
813 const DominatorTree *DT) {
814 const BasicBlock *CtxBB = CtxI->getParent();
815 const BasicBlock *AssumeBB = Assume->getParent();
816 unsigned NumChecked = 0;
817 BasicBlock::const_iterator CtxIter = CtxI->getIterator();
818 if (CtxBB == AssumeBB) {
819 if (Assume != CtxI && !Assume->comesBefore(Other: CtxI))
820 return false;
821 return hasNoFreeInRange(Begin: Assume->getIterator(), End: CtxIter, NumChecked);
822 }
823 if (DT && !DT->dominates(Def: Assume, User: CtxI))
824 return false;
825 if (!hasNoFreeInRange(Begin: CtxBB->begin(), End: CtxIter, NumChecked))
826 return false;
827 if (pred_empty(BB: CtxBB))
828 return false;
829
830 // Note: CtxBB is NOT pre-inserted into Visited to ensure that loop
831 // backedges returning to CtxBB are enqueued and checked correctly.
832 SmallVector<const BasicBlock *, 16> Worklist(predecessors(BB: CtxBB));
833 SmallPtrSet<const BasicBlock *, 16> Visited;
834 while (!Worklist.empty()) {
835 const BasicBlock *CurBB = Worklist.pop_back_val();
836 if (!Visited.insert(Ptr: CurBB).second)
837 continue;
838
839 if (CurBB == AssumeBB) {
840 if (!hasNoFreeInRange(Begin: Assume->getIterator(), End: AssumeBB->end(), NumChecked))
841 return false;
842 continue;
843 }
844 assert((!DT || DT->dominates(AssumeBB, CurBB)) &&
845 "Blocks between Assume and CtxI must be dominated by AssumeBB");
846
847 if (pred_empty(BB: CurBB)) {
848 // Reaching the entry block means there is a path to CtxI that does not
849 // pass through Assume.
850 if (CurBB->isEntryBlock())
851 return false;
852
853 // CurBB is unreachable (for example, emptied by SimplifyCFG but not yet
854 // removed), so nothing in it can execute a free.
855 continue;
856 }
857
858 // If CurBB == CtxBB (due to a loop backedge targeting CtxBB), check
859 // instructions from CtxIter to the end of CtxBB (instructions before
860 // CtxIter were checked above). Otherwise, check the entire block.
861 auto StartIt = (CurBB == CtxBB) ? CtxIter : CurBB->begin();
862 if (!hasNoFreeInRange(Begin: StartIt, End: CurBB->end(), NumChecked))
863 return false;
864 append_range(C&: Worklist, R: predecessors(BB: CurBB));
865 }
866 return true;
867}
868
869// TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but
870// we still have enough information about `RHS` to conclude non-zero. For
871// example Pred=EQ, RHS=isKnownNonZero. cmpExcludesZero is called in loops
872// so the extra compile time may not be worth it, but possibly a second API
873// should be created for use outside of loops.
874static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) {
875 // v u> y implies v != 0.
876 if (Pred == ICmpInst::ICMP_UGT)
877 return true;
878
879 // Special-case v != 0 to also handle v != null.
880 if (Pred == ICmpInst::ICMP_NE)
881 return match(V: RHS, P: m_Zero());
882
883 // All other predicates - rely on generic ConstantRange handling.
884 const APInt *C;
885 auto Zero = APInt::getZero(numBits: RHS->getType()->getScalarSizeInBits());
886 if (match(V: RHS, P: m_APInt(Res&: C))) {
887 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
888 return !TrueValues.contains(Val: Zero);
889 }
890
891 auto *VC = dyn_cast<ConstantDataVector>(Val: RHS);
892 if (VC == nullptr)
893 return false;
894
895 for (unsigned ElemIdx = 0, NElem = VC->getNumElements(); ElemIdx < NElem;
896 ++ElemIdx) {
897 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(
898 Pred, Other: VC->getElementAsAPInt(i: ElemIdx));
899 if (TrueValues.contains(Val: Zero))
900 return false;
901 }
902 return true;
903}
904
905static void breakSelfRecursivePHI(const Use *U, const PHINode *PHI,
906 Value *&ValOut, Instruction *&CtxIOut,
907 const PHINode **PhiOut = nullptr) {
908 ValOut = U->get();
909 if (ValOut == PHI)
910 return;
911 CtxIOut = PHI->getIncomingBlock(U: *U)->getTerminator();
912 if (PhiOut)
913 *PhiOut = PHI;
914 Value *V;
915 // If the Use is a select of this phi, compute analysis on other arm to break
916 // recursion.
917 // TODO: Min/Max
918 if (match(V: ValOut, P: m_Select(C: m_Value(), L: m_Specific(V: PHI), R: m_Value(V))) ||
919 match(V: ValOut, P: m_Select(C: m_Value(), L: m_Value(V), R: m_Specific(V: PHI))))
920 ValOut = V;
921
922 // Same for select, if this phi is 2-operand phi, compute analysis on other
923 // incoming value to break recursion.
924 // TODO: We could handle any number of incoming edges as long as we only have
925 // two unique values.
926 if (auto *IncPhi = dyn_cast<PHINode>(Val: ValOut);
927 IncPhi && IncPhi->getNumIncomingValues() == 2) {
928 for (int Idx = 0; Idx < 2; ++Idx) {
929 if (IncPhi->getIncomingValue(i: Idx) == PHI) {
930 ValOut = IncPhi->getIncomingValue(i: 1 - Idx);
931 if (PhiOut)
932 *PhiOut = IncPhi;
933 CtxIOut = IncPhi->getIncomingBlock(i: 1 - Idx)->getTerminator();
934 break;
935 }
936 }
937 }
938}
939
940static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) {
941 // Use of assumptions is context-sensitive. If we don't have a context, we
942 // cannot use them!
943 if (!Q.AC || !Q.CtxI)
944 return false;
945
946 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
947 if (!Elem.Assume)
948 continue;
949
950 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
951 assert(I->getFunction() == Q.CtxI->getFunction() &&
952 "Got assumption for the wrong function!");
953
954 if (Elem.Index != AssumptionCache::ExprResultIdx) {
955 if (assumeBundleImpliesNonNull(Val: V, Context: Q.CtxI->getFunction(),
956 OBU: I->getOperandBundleAt(Index: Elem.Index)) &&
957 isValidAssumeForContext(I, Q))
958 return true;
959 continue;
960 }
961
962 // Warning: This loop can end up being somewhat performance sensitive.
963 // We're running this loop for once for each value queried resulting in a
964 // runtime of ~O(#assumes * #values).
965
966 if (match(V: I->getArgOperand(i: 0), P: m_Trunc(Op: m_Specific(V))) &&
967 isValidAssumeForContext(I, Q))
968 return true;
969
970 Value *RHS;
971 CmpPredicate Pred;
972 auto m_V = m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToInt(Op: m_Specific(V)));
973 if (!match(V: I->getArgOperand(i: 0), P: m_c_ICmp(Pred, L: m_V, R: m_Value(V&: RHS))))
974 continue;
975
976 if (cmpExcludesZero(Pred, RHS) && isValidAssumeForContext(I, Q))
977 return true;
978 }
979
980 return false;
981}
982
983static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
984 Value *LHS, Value *RHS, KnownBits &Known,
985 const SimplifyQuery &Q) {
986 if (RHS->getType()->isPointerTy()) {
987 // Handle comparison of pointer to null explicitly, as it will not be
988 // covered by the m_APInt() logic below.
989 if (LHS == V && match(V: RHS, P: m_Zero())) {
990 switch (Pred) {
991 case ICmpInst::ICMP_EQ:
992 Known.setAllZero();
993 break;
994 case ICmpInst::ICMP_SGE:
995 case ICmpInst::ICMP_SGT:
996 Known.makeNonNegative();
997 break;
998 case ICmpInst::ICMP_SLT:
999 Known.makeNegative();
1000 break;
1001 default:
1002 break;
1003 }
1004 }
1005 return;
1006 }
1007
1008 unsigned BitWidth = Known.getBitWidth();
1009 auto m_V =
1010 m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToIntSameSize(DL: Q.DL, Op: m_Specific(V)));
1011
1012 Value *Y;
1013 const APInt *Mask, *C;
1014 if (!match(V: RHS, P: m_APInt(Res&: C)))
1015 return;
1016
1017 uint64_t ShAmt;
1018 switch (Pred) {
1019 case ICmpInst::ICMP_EQ:
1020 // assume(V = C)
1021 if (match(V: LHS, P: m_V)) {
1022 Known = Known.unionWith(RHS: KnownBits::makeConstant(C: *C));
1023 // assume(V & Mask = C)
1024 } else if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value(V&: Y)))) {
1025 // For one bits in Mask, we can propagate bits from C to V.
1026 Known.One |= *C;
1027 if (match(V: Y, P: m_APInt(Res&: Mask)))
1028 Known.Zero |= ~*C & *Mask;
1029 // assume(V | Mask = C)
1030 } else if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value(V&: Y)))) {
1031 // For zero bits in Mask, we can propagate bits from C to V.
1032 Known.Zero |= ~*C;
1033 if (match(V: Y, P: m_APInt(Res&: Mask)))
1034 Known.One |= *C & ~*Mask;
1035 // assume(V << ShAmt = C)
1036 } else if (match(V: LHS, P: m_Shl(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
1037 ShAmt < BitWidth) {
1038 // For those bits in C that are known, we can propagate them to known
1039 // bits in V shifted to the right by ShAmt.
1040 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
1041 RHSKnown >>= ShAmt;
1042 Known = Known.unionWith(RHS: RHSKnown);
1043 // assume(V >> ShAmt = C)
1044 } else if (match(V: LHS, P: m_Shr(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
1045 ShAmt < BitWidth) {
1046 // For those bits in RHS that are known, we can propagate them to known
1047 // bits in V shifted to the right by C.
1048 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
1049 RHSKnown <<= ShAmt;
1050 Known = Known.unionWith(RHS: RHSKnown);
1051 }
1052 break;
1053 case ICmpInst::ICMP_NE: {
1054 // assume (V & B != 0) where B is a power of 2
1055 const APInt *BPow2;
1056 if (C->isZero() && match(V: LHS, P: m_And(L: m_V, R: m_Power2(V&: BPow2))))
1057 Known.One |= *BPow2;
1058 break;
1059 }
1060 default: {
1061 const APInt *Offset = nullptr;
1062 if (match(V: LHS, P: m_CombineOr(Ps: m_V, Ps: m_AddLike(L: m_V, R: m_APInt(Res&: Offset))))) {
1063 ConstantRange LHSRange = ConstantRange::makeAllowedICmpRegion(Pred, Other: *C);
1064 if (Offset)
1065 LHSRange = LHSRange.sub(Other: *Offset);
1066 Known = Known.unionWith(RHS: LHSRange.toKnownBits());
1067 }
1068 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
1069 // X & Y u> C -> X u> C && Y u> C
1070 // X nuw- Y u> C -> X u> C
1071 if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value())) ||
1072 match(V: LHS, P: m_NUWSub(L: m_V, R: m_Value())))
1073 Known.One.setHighBits(
1074 (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
1075 }
1076 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
1077 // X | Y u< C -> X u< C && Y u< C
1078 // X nuw+ Y u< C -> X u< C && Y u< C
1079 if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value())) ||
1080 match(V: LHS, P: m_c_NUWAdd(L: m_V, R: m_Value()))) {
1081 Known.Zero.setHighBits(
1082 (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
1083 }
1084 }
1085 } break;
1086 }
1087}
1088
1089static void computeKnownBitsFromICmpCond(const Value *V, ICmpInst *Cmp,
1090 KnownBits &Known,
1091 const SimplifyQuery &SQ, bool Invert) {
1092 ICmpInst::Predicate Pred =
1093 Invert ? Cmp->getInversePredicate() : Cmp->getPredicate();
1094 Value *LHS = Cmp->getOperand(i_nocapture: 0);
1095 Value *RHS = Cmp->getOperand(i_nocapture: 1);
1096
1097 // Handle icmp pred (trunc V), C
1098 if (match(V: LHS, P: m_Trunc(Op: m_Specific(V)))) {
1099 KnownBits DstKnown(LHS->getType()->getScalarSizeInBits());
1100 computeKnownBitsFromCmp(V: LHS, Pred, LHS, RHS, Known&: DstKnown, Q: SQ);
1101 if (cast<TruncInst>(Val: LHS)->hasNoUnsignedWrap())
1102 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1103 else
1104 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1105 return;
1106 }
1107
1108 computeKnownBitsFromCmp(V, Pred, LHS, RHS, Known, Q: SQ);
1109}
1110
1111static void computeKnownBitsFromCond(const Value *V, Value *Cond,
1112 KnownBits &Known, const SimplifyQuery &SQ,
1113 bool Invert, unsigned Depth) {
1114 Value *A, *B;
1115 if (Depth < MaxAnalysisRecursionDepth &&
1116 match(V: Cond, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
1117 KnownBits Known2(Known.getBitWidth());
1118 KnownBits Known3(Known.getBitWidth());
1119 computeKnownBitsFromCond(V, Cond: A, Known&: Known2, SQ, Invert, Depth: Depth + 1);
1120 computeKnownBitsFromCond(V, Cond: B, Known&: Known3, SQ, Invert, Depth: Depth + 1);
1121 if (Invert ? match(V: Cond, P: m_LogicalOr(L: m_Value(), R: m_Value()))
1122 : match(V: Cond, P: m_LogicalAnd(L: m_Value(), R: m_Value())))
1123 Known2 = Known2.unionWith(RHS: Known3);
1124 else
1125 Known2 = Known2.intersectWith(RHS: Known3);
1126 Known = Known.unionWith(RHS: Known2);
1127 return;
1128 }
1129
1130 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
1131 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ, Invert);
1132 return;
1133 }
1134
1135 if (match(V: Cond, P: m_Trunc(Op: m_Specific(V)))) {
1136 KnownBits DstKnown(1);
1137 if (Invert) {
1138 DstKnown.setAllZero();
1139 } else {
1140 DstKnown.setAllOnes();
1141 }
1142 if (cast<TruncInst>(Val: Cond)->hasNoUnsignedWrap()) {
1143 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1144 return;
1145 }
1146 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1147 return;
1148 }
1149
1150 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A))))
1151 computeKnownBitsFromCond(V, Cond: A, Known, SQ, Invert: !Invert, Depth: Depth + 1);
1152}
1153
1154void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
1155 const SimplifyQuery &Q, unsigned Depth) {
1156 // Handle injected condition.
1157 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
1158 computeKnownBitsFromCond(V, Cond: Q.CC->Cond, Known, SQ: Q, Invert: Q.CC->Invert, Depth);
1159
1160 if (!Q.CtxI)
1161 return;
1162
1163 if (Q.DC && Q.DT) {
1164 // Handle dominating conditions.
1165 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
1166 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
1167 if (Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()))
1168 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1169 /*Invert*/ false, Depth);
1170
1171 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
1172 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()))
1173 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1174 /*Invert*/ true, Depth);
1175 }
1176
1177 if (Known.hasConflict())
1178 Known.resetAll();
1179 }
1180
1181 if (!Q.AC)
1182 return;
1183
1184 unsigned BitWidth = Known.getBitWidth();
1185
1186 // Note that the patterns below need to be kept in sync with the code
1187 // in AssumptionCache::updateAffectedValues.
1188
1189 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
1190 if (!Elem.Assume)
1191 continue;
1192
1193 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
1194 assert(I->getParent()->getParent() == Q.CtxI->getParent()->getParent() &&
1195 "Got assumption for the wrong function!");
1196
1197 if (Elem.Index != AssumptionCache::ExprResultIdx) {
1198 auto OBU = I->getOperandBundleAt(Index: Elem.Index);
1199 switch (getBundleAttrFromOBU(OBU)) {
1200 case BundleAttr::Align: {
1201 auto [Ptr, _, _2, Alignment, Offset] = getAssumeAlignInfo(OBU);
1202 if (Ptr == V && Alignment && Offset && isPowerOf2_64(Value: *Alignment) &&
1203 isValidAssumeForContext(I, Q)) {
1204 Known.Zero |= (*Alignment - 1) & ~*Offset;
1205 Known.One |= (*Alignment - 1) & *Offset;
1206 }
1207 } break;
1208
1209 case BundleAttr::Range: {
1210 auto [Val, Range] = getAssumeRangeInfo(OBU);
1211 if (Val == V && isValidAssumeForContext(I, Q))
1212 Known = Known.unionWith(RHS: Range.toKnownBits());
1213 } break;
1214
1215 default:
1216 break;
1217 }
1218 continue;
1219 }
1220
1221 // Warning: This loop can end up being somewhat performance sensitive.
1222 // We're running this loop for once for each value queried resulting in a
1223 // runtime of ~O(#assumes * #values).
1224
1225 Value *Arg = I->getArgOperand(i: 0);
1226
1227 if (Arg == V && isValidAssumeForContext(I, Q)) {
1228 assert(BitWidth == 1 && "assume operand is not i1?");
1229 (void)BitWidth;
1230 Known.setAllOnes();
1231 return;
1232 }
1233 if (match(V: Arg, P: m_Not(V: m_Specific(V))) &&
1234 isValidAssumeForContext(I, Q)) {
1235 assert(BitWidth == 1 && "assume operand is not i1?");
1236 (void)BitWidth;
1237 Known.setAllZero();
1238 return;
1239 }
1240 auto *Trunc = dyn_cast<TruncInst>(Val: Arg);
1241 if (Trunc && Trunc->getOperand(i_nocapture: 0) == V &&
1242 isValidAssumeForContext(I, Q)) {
1243 if (Trunc->hasNoUnsignedWrap()) {
1244 Known = KnownBits::makeConstant(C: APInt(BitWidth, 1));
1245 return;
1246 }
1247 Known.One.setBit(0);
1248 return;
1249 }
1250
1251 // The remaining tests are all recursive, so bail out if we hit the limit.
1252 if (Depth == MaxAnalysisRecursionDepth)
1253 continue;
1254
1255 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
1256 if (!Cmp)
1257 continue;
1258
1259 if (!isValidAssumeForContext(I, Q))
1260 continue;
1261
1262 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ: Q, /*Invert=*/false);
1263 }
1264
1265 // Conflicting assumption: Undefined behavior will occur on this execution
1266 // path.
1267 if (Known.hasConflict())
1268 Known.resetAll();
1269}
1270
1271/// Compute known bits from a shift operator, including those with a
1272/// non-constant shift amount. Known is the output of this function. Known2 is a
1273/// pre-allocated temporary with the same bit width as Known and on return
1274/// contains the known bit of the shift value source. KF is an
1275/// operator-specific function that, given the known-bits and a shift amount,
1276/// compute the implied known-bits of the shift operator's result respectively
1277/// for that shift amount. The results from calling KF are conservatively
1278/// combined for all permitted shift amounts.
1279static void computeKnownBitsFromShiftOperator(
1280 const Operator *I, const APInt &DemandedElts, KnownBits &Known,
1281 KnownBits &Known2, const SimplifyQuery &Q, unsigned Depth,
1282 function_ref<KnownBits(const KnownBits &, const KnownBits &, bool)> KF) {
1283 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1284 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1285 // To limit compile-time impact, only query isKnownNonZero() if we know at
1286 // least something about the shift amount.
1287 bool ShAmtNonZero =
1288 Known.isNonZero() ||
1289 (Known.getMaxValue().ult(RHS: Known.getBitWidth()) &&
1290 isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1));
1291 Known = KF(Known2, Known, ShAmtNonZero);
1292}
1293
1294static KnownBits
1295getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
1296 const KnownBits &KnownLHS, const KnownBits &KnownRHS,
1297 const SimplifyQuery &Q, unsigned Depth) {
1298 unsigned BitWidth = KnownLHS.getBitWidth();
1299 KnownBits KnownOut(BitWidth);
1300 bool IsAnd = false;
1301 bool HasKnownOne = !KnownLHS.One.isZero() || !KnownRHS.One.isZero();
1302 Value *X = nullptr, *Y = nullptr;
1303
1304 switch (I->getOpcode()) {
1305 case Instruction::And:
1306 KnownOut = KnownLHS & KnownRHS;
1307 IsAnd = true;
1308 // and(x, -x) is common idioms that will clear all but lowest set
1309 // bit. If we have a single known bit in x, we can clear all bits
1310 // above it.
1311 // TODO: instcombine often reassociates independent `and` which can hide
1312 // this pattern. Try to match and(x, and(-x, y)) / and(and(x, y), -x).
1313 if (HasKnownOne && match(V: I, P: m_c_And(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X))))) {
1314 // -(-x) == x so using whichever (LHS/RHS) gets us a better result.
1315 if (KnownLHS.countMaxTrailingZeros() <= KnownRHS.countMaxTrailingZeros())
1316 KnownOut = KnownLHS.blsi();
1317 else
1318 KnownOut = KnownRHS.blsi();
1319 }
1320 break;
1321 case Instruction::Or:
1322 KnownOut = KnownLHS | KnownRHS;
1323 break;
1324 case Instruction::Xor:
1325 KnownOut = KnownLHS ^ KnownRHS;
1326 // xor(x, x-1) is common idioms that will clear all but lowest set
1327 // bit. If we have a single known bit in x, we can clear all bits
1328 // above it.
1329 // TODO: xor(x, x-1) is often rewritting as xor(x, x-C) where C !=
1330 // -1 but for the purpose of demanded bits (xor(x, x-C) &
1331 // Demanded) == (xor(x, x-1) & Demanded). Extend the xor pattern
1332 // to use arbitrary C if xor(x, x-C) as the same as xor(x, x-1).
1333 if (HasKnownOne &&
1334 match(V: I, P: m_c_Xor(L: m_Value(V&: X), R: m_Add(L: m_Deferred(V: X), R: m_AllOnes())))) {
1335 const KnownBits &XBits = I->getOperand(i: 0) == X ? KnownLHS : KnownRHS;
1336 KnownOut = XBits.blsmsk();
1337 }
1338 break;
1339 default:
1340 llvm_unreachable("Invalid Op used in 'analyzeKnownBitsFromAndXorOr'");
1341 }
1342
1343 // and(x, add (x, -1)) is a common idiom that always clears the low bit;
1344 // xor/or(x, add (x, -1)) is an idiom that will always set the low bit.
1345 // here we handle the more general case of adding any odd number by
1346 // matching the form and/xor/or(x, add(x, y)) where y is odd.
1347 // TODO: This could be generalized to clearing any bit set in y where the
1348 // following bit is known to be unset in y.
1349 if (!KnownOut.Zero[0] && !KnownOut.One[0] &&
1350 (match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_c_Add(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1351 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1352 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Value(V&: Y), R: m_Deferred(V: X)))))) {
1353 KnownBits KnownY(BitWidth);
1354 computeKnownBits(V: Y, DemandedElts, Known&: KnownY, Q, Depth: Depth + 1);
1355 if (KnownY.countMinTrailingOnes() > 0) {
1356 if (IsAnd)
1357 KnownOut.Zero.setBit(0);
1358 else
1359 KnownOut.One.setBit(0);
1360 }
1361 }
1362 return KnownOut;
1363}
1364
1365static KnownBits computeKnownBitsForHorizontalOperation(
1366 const Operator *I, const APInt &DemandedElts, const SimplifyQuery &Q,
1367 unsigned Depth,
1368 const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
1369 KnownBitsFunc) {
1370 APInt DemandedEltsLHS, DemandedEltsRHS;
1371 getHorizDemandedEltsForFirstOperand(VectorBitWidth: Q.DL.getTypeSizeInBits(Ty: I->getType()),
1372 DemandedElts, DemandedLHS&: DemandedEltsLHS,
1373 DemandedRHS&: DemandedEltsRHS);
1374
1375 const auto ComputeForSingleOpFunc =
1376 [Depth, &Q, KnownBitsFunc](const Value *Op, APInt &DemandedEltsOp) {
1377 return KnownBitsFunc(
1378 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp, Q, Depth: Depth + 1),
1379 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp << 1, Q, Depth: Depth + 1));
1380 };
1381
1382 if (DemandedEltsRHS.isZero())
1383 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS);
1384 if (DemandedEltsLHS.isZero())
1385 return ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS);
1386
1387 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS)
1388 .intersectWith(RHS: ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS));
1389}
1390
1391// Public so this can be used in `SimplifyDemandedUseBits`.
1392KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
1393 const KnownBits &KnownLHS,
1394 const KnownBits &KnownRHS,
1395 const SimplifyQuery &SQ,
1396 unsigned Depth) {
1397 auto *FVTy = dyn_cast<FixedVectorType>(Val: I->getType());
1398 APInt DemandedElts =
1399 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
1400
1401 return getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS, KnownRHS, Q: SQ,
1402 Depth);
1403}
1404
1405ConstantRange llvm::getVScaleRange(const Function *F, unsigned BitWidth) {
1406 Attribute Attr = F->getFnAttribute(Kind: Attribute::VScaleRange);
1407 // Without vscale_range, we only know that vscale is non-zero.
1408 if (!Attr.isValid())
1409 return ConstantRange(APInt(BitWidth, 1), APInt::getZero(numBits: BitWidth));
1410
1411 unsigned AttrMin = Attr.getVScaleRangeMin();
1412 // Minimum is larger than vscale width, result is always poison.
1413 if ((unsigned)llvm::bit_width(Value: AttrMin) > BitWidth)
1414 return ConstantRange::getEmpty(BitWidth);
1415
1416 APInt Min(BitWidth, AttrMin);
1417 std::optional<unsigned> AttrMax = Attr.getVScaleRangeMax();
1418 if (!AttrMax || (unsigned)llvm::bit_width(Value: *AttrMax) > BitWidth)
1419 return ConstantRange(Min, APInt::getZero(numBits: BitWidth));
1420
1421 return ConstantRange(Min, APInt(BitWidth, *AttrMax) + 1);
1422}
1423
1424/// Return true if \p II reads a register named "vlenb". On RISC-V this is the
1425/// VLENB CSR, which holds VLEN/8: a non-zero power of two bounded by the
1426/// target's VLEN range. Callers must ensure the target is RISC-V.
1427static bool isReadVLENB(const IntrinsicInst &II) {
1428 auto *MAV = dyn_cast<MetadataAsValue>(Val: II.getArgOperand(i: 0));
1429 if (!MAV)
1430 return false;
1431 auto *MD = dyn_cast<MDNode>(Val: MAV->getMetadata());
1432 if (!MD || MD->getNumOperands() != 1)
1433 return false;
1434 auto *RegName = dyn_cast<MDString>(Val: MD->getOperand(I: 0));
1435 return RegName && RegName->getString() == "vlenb";
1436}
1437
1438/// Return the value range of a RISC-V vlenb CSR read. RVV requires VLEN to be a
1439/// power of two in [32, 65536] (Zvl32b is the smallest vector extension), so
1440/// VLENB = VLEN/8 is in [4, 8192]. This architectural bound is independent of
1441/// any function attribute and stays sound for Zvl32b, whose VLEN (32) is not
1442/// representable as an integer vscale (VLEN / RVVBitsPerBlock). A vscale_range
1443/// attribute, when present, pins the subtarget's VLEN in units of
1444/// RVVBitsPerBlock (64 bits) and so gives a tighter VLENB = vscale *
1445/// RVVBytesPerBlock.
1446static ConstantRange getRISCVVLENBRange(const IntrinsicInst &II,
1447 unsigned Width) {
1448 // Architectural bounds: VLEN in [32, 65536] => VLENB in [4, 8192].
1449 ConstantRange Range(APInt(Width, 32 / 8), APInt(Width, 65536 / 8) + 1);
1450
1451 const Function *F = II.getFunction();
1452 if (F->getFnAttribute(Kind: Attribute::VScaleRange).isValid()) {
1453 ConstantRange VScale = getVScaleRange(F, BitWidth: Width);
1454 Range = Range.intersectWith(
1455 CR: VScale.multiply(Other: ConstantRange(APInt(Width, RISCV::RVVBytesPerBlock))));
1456 }
1457 return Range;
1458}
1459
1460void llvm::adjustKnownBitsForSelectArm(KnownBits &Known, Value *Cond,
1461 Value *Arm, bool Invert,
1462 const SimplifyQuery &Q, unsigned Depth) {
1463 // If we have a constant arm, we are done.
1464 if (Known.isConstant())
1465 return;
1466
1467 // See what condition implies about the bits of the select arm.
1468 KnownBits CondRes(Known.getBitWidth());
1469 computeKnownBitsFromCond(V: Arm, Cond, Known&: CondRes, SQ: Q, Invert, Depth: Depth + 1);
1470 // If we don't get any information from the condition, no reason to
1471 // proceed.
1472 if (CondRes.isUnknown())
1473 return;
1474
1475 // We can have conflict if the condition is dead. I.e if we have
1476 // (x | 64) < 32 ? (x | 64) : y
1477 // we will have conflict at bit 6 from the condition/the `or`.
1478 // In that case just return. Its not particularly important
1479 // what we do, as this select is going to be simplified soon.
1480 CondRes = CondRes.unionWith(RHS: Known);
1481 if (CondRes.hasConflict())
1482 return;
1483
1484 // Finally make sure the information we found is valid. This is relatively
1485 // expensive so it's left for the very end.
1486 if (!isGuaranteedNotToBeUndef(V: Arm, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1))
1487 return;
1488
1489 // Finally, we know we get information from the condition and its valid,
1490 // so return it.
1491 Known = std::move(CondRes);
1492}
1493
1494// Match a signed min+max clamp pattern like smax(smin(In, CHigh), CLow).
1495// Returns the input and lower/upper bounds.
1496static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
1497 const APInt *&CLow, const APInt *&CHigh) {
1498 assert(isa<Operator>(Select) &&
1499 cast<Operator>(Select)->getOpcode() == Instruction::Select &&
1500 "Input should be a Select!");
1501
1502 const Value *LHS = nullptr, *RHS = nullptr;
1503 SelectPatternFlavor SPF = matchSelectPattern(V: Select, LHS, RHS).Flavor;
1504 if (SPF != SPF_SMAX && SPF != SPF_SMIN)
1505 return false;
1506
1507 if (!match(V: RHS, P: m_APInt(Res&: CLow)))
1508 return false;
1509
1510 const Value *LHS2 = nullptr, *RHS2 = nullptr;
1511 SelectPatternFlavor SPF2 = matchSelectPattern(V: LHS, LHS&: LHS2, RHS&: RHS2).Flavor;
1512 if (getInverseMinMaxFlavor(SPF) != SPF2)
1513 return false;
1514
1515 if (!match(V: RHS2, P: m_APInt(Res&: CHigh)))
1516 return false;
1517
1518 if (SPF == SPF_SMIN)
1519 std::swap(a&: CLow, b&: CHigh);
1520
1521 In = LHS2;
1522 return CLow->sle(RHS: *CHigh);
1523}
1524
1525static bool isSignedMinMaxIntrinsicClamp(const IntrinsicInst *II,
1526 const APInt *&CLow,
1527 const APInt *&CHigh) {
1528 assert((II->getIntrinsicID() == Intrinsic::smin ||
1529 II->getIntrinsicID() == Intrinsic::smax) &&
1530 "Must be smin/smax");
1531
1532 Intrinsic::ID InverseID = getInverseMinMaxIntrinsic(MinMaxID: II->getIntrinsicID());
1533 auto *InnerII = dyn_cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
1534 if (!InnerII || InnerII->getIntrinsicID() != InverseID ||
1535 !match(V: II->getArgOperand(i: 1), P: m_APInt(Res&: CLow)) ||
1536 !match(V: InnerII->getArgOperand(i: 1), P: m_APInt(Res&: CHigh)))
1537 return false;
1538
1539 if (II->getIntrinsicID() == Intrinsic::smin)
1540 std::swap(a&: CLow, b&: CHigh);
1541 return CLow->sle(RHS: *CHigh);
1542}
1543
1544static void unionWithMinMaxIntrinsicClamp(const IntrinsicInst *II,
1545 KnownBits &Known) {
1546 const APInt *CLow, *CHigh;
1547 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
1548 Known = Known.unionWith(
1549 RHS: ConstantRange::getNonEmpty(Lower: *CLow, Upper: *CHigh + 1).toKnownBits());
1550}
1551
1552static void computeKnownBitsForRecurrenceOperands(
1553 const PHINode *P, Value *Start, Value *Step, const APInt &DemandedElts,
1554 KnownBits &KnownStart, KnownBits &KnownStep, const SimplifyQuery &Q,
1555 unsigned Depth) {
1556 // Change the context instruction to the "edge" that flows into the phi. This
1557 // is important because that is where the value is actually "evaluated" even
1558 // though it is used later somewhere else. (see also D69571).
1559 SimplifyQuery RecQ = Q.getWithoutCondContext();
1560 unsigned OpNum = P->getOperand(i_nocapture: 0) == Start ? 0 : 1;
1561
1562 RecQ.CtxI = P->getIncomingBlock(i: OpNum)->getTerminator();
1563 computeKnownBits(V: Start, DemandedElts, Known&: KnownStart, Q: RecQ, Depth: Depth + 1);
1564
1565 RecQ.CtxI = P->getIncomingBlock(i: 1 - OpNum)->getTerminator();
1566 computeKnownBits(V: Step, DemandedElts, Known&: KnownStep, Q: RecQ, Depth: Depth + 1);
1567}
1568
1569static void computeKnownBitsFromOperator(const Operator *I,
1570 const APInt &DemandedElts,
1571 KnownBits &Known,
1572 const SimplifyQuery &Q,
1573 unsigned Depth) {
1574 unsigned BitWidth = Known.getBitWidth();
1575
1576 KnownBits Known2(BitWidth);
1577 switch (I->getOpcode()) {
1578 default: break;
1579 case Instruction::Load:
1580 if (MDNode *MD =
1581 Q.IIQ.getMetadata(I: cast<LoadInst>(Val: I), KindID: LLVMContext::MD_range))
1582 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
1583 break;
1584 case Instruction::And:
1585 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1586 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1587
1588 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1589 break;
1590 case Instruction::Or:
1591 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1592 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1593
1594 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1595 break;
1596 case Instruction::Xor:
1597 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1598 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1599
1600 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1601 break;
1602 case Instruction::Mul: {
1603 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1604 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1605 computeKnownBitsMul(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1606 DemandedElts, Known, Known2, Q, Depth);
1607 break;
1608 }
1609 case Instruction::UDiv: {
1610 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1611 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1612 Known =
1613 KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1614 break;
1615 }
1616 case Instruction::SDiv: {
1617 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1618 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1619 Known =
1620 KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1621 break;
1622 }
1623 case Instruction::Select: {
1624 auto ComputeForArm = [&](Value *Arm, bool Invert) {
1625 KnownBits Res(Known.getBitWidth());
1626 computeKnownBits(V: Arm, DemandedElts, Known&: Res, Q, Depth: Depth + 1);
1627 adjustKnownBitsForSelectArm(Known&: Res, Cond: I->getOperand(i: 0), Arm, Invert, Q, Depth);
1628 return Res;
1629 };
1630 // Only known if known in both the LHS and RHS.
1631 Known =
1632 ComputeForArm(I->getOperand(i: 1), /*Invert=*/false)
1633 .intersectWith(RHS: ComputeForArm(I->getOperand(i: 2), /*Invert=*/true));
1634 break;
1635 }
1636 case Instruction::FPToSI: {
1637 // fptosi is poison if the rounded value doesn't fit in the result type,
1638 // so we can assume the conversion is well-defined and rounds towards
1639 // zero. +-Inf can never fit in an integer type, so it is always poison,
1640 // like NaN. Negative subnormals and negative zero round to 0. That
1641 // leaves negative normals as the only class that can produce a defined
1642 // negative result.
1643 KnownFPClass SrcFPClass = computeKnownFPClass(
1644 V: I->getOperand(i: 0), DemandedElts, InterestedClasses: fcNegNormal, SQ: Q, Depth: Depth + 1);
1645 if (SrcFPClass.isKnownNever(Mask: fcNegNormal))
1646 Known.makeNonNegative();
1647 break;
1648 }
1649 case Instruction::FPTrunc:
1650 case Instruction::FPExt:
1651 case Instruction::FPToUI:
1652 case Instruction::SIToFP:
1653 case Instruction::UIToFP:
1654 break; // Can't work with floating point.
1655 case Instruction::PtrToInt:
1656 case Instruction::PtrToAddr:
1657 case Instruction::IntToPtr:
1658 // Fall through and handle them the same as zext/trunc.
1659 [[fallthrough]];
1660 case Instruction::ZExt:
1661 case Instruction::Trunc: {
1662 Type *SrcTy = I->getOperand(i: 0)->getType();
1663
1664 unsigned SrcBitWidth;
1665 // Note that we handle pointer operands here because of inttoptr/ptrtoint
1666 // which fall through here.
1667 Type *ScalarTy = SrcTy->getScalarType();
1668 SrcBitWidth = ScalarTy->isPointerTy() ?
1669 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
1670 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
1671
1672 assert(SrcBitWidth && "SrcBitWidth can't be zero");
1673 Known = Known.anyextOrTrunc(BitWidth: SrcBitWidth);
1674 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1675 if (auto *Inst = dyn_cast<PossiblyNonNegInst>(Val: I);
1676 Inst && Inst->hasNonNeg() && !Known.isNegative())
1677 Known.makeNonNegative();
1678 Known = Known.zextOrTrunc(BitWidth);
1679 break;
1680 }
1681 case Instruction::BitCast: {
1682 Type *SrcTy = I->getOperand(i: 0)->getType();
1683 if (SrcTy->isIntOrPtrTy() &&
1684 // TODO: For now, not handling conversions like:
1685 // (bitcast i64 %x to <2 x i32>)
1686 !I->getType()->isVectorTy()) {
1687 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1688 break;
1689 }
1690
1691 const Value *V;
1692 // Handle bitcast from floating point to integer.
1693 if (match(V: I, P: m_ElementWiseBitCast(Op: m_Value(V))) &&
1694 V->getType()->isFPOrFPVectorTy()) {
1695 Type *FPType = V->getType()->getScalarType();
1696 KnownFPClass Result =
1697 computeKnownFPClass(V, DemandedElts, InterestedClasses: fcAllFlags, SQ: Q, Depth: Depth + 1);
1698
1699 Known = Result.toKnownBits(FltSemantics: FPType->getFltSemantics());
1700
1701 break;
1702 }
1703
1704 // Handle cast from vector integer type to scalar or vector integer.
1705 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy);
1706 if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
1707 !I->getType()->isIntOrIntVectorTy() ||
1708 isa<ScalableVectorType>(Val: I->getType()))
1709 break;
1710
1711 unsigned NumElts = DemandedElts.getBitWidth();
1712 bool IsLE = Q.DL.isLittleEndian();
1713 // Look through a cast from narrow vector elements to wider type.
1714 // Examples: v4i32 -> v2i64, v3i8 -> v24
1715 unsigned SubBitWidth = SrcVecTy->getScalarSizeInBits();
1716 if (BitWidth % SubBitWidth == 0) {
1717 // Known bits are automatically intersected across demanded elements of a
1718 // vector. So for example, if a bit is computed as known zero, it must be
1719 // zero across all demanded elements of the vector.
1720 //
1721 // For this bitcast, each demanded element of the output is sub-divided
1722 // across a set of smaller vector elements in the source vector. To get
1723 // the known bits for an entire element of the output, compute the known
1724 // bits for each sub-element sequentially. This is done by shifting the
1725 // one-set-bit demanded elements parameter across the sub-elements for
1726 // consecutive calls to computeKnownBits. We are using the demanded
1727 // elements parameter as a mask operator.
1728 //
1729 // The known bits of each sub-element are then inserted into place
1730 // (dependent on endian) to form the full result of known bits.
1731 unsigned SubScale = BitWidth / SubBitWidth;
1732 APInt SubDemandedElts = APInt::getZero(numBits: NumElts * SubScale);
1733 for (unsigned i = 0; i != NumElts; ++i) {
1734 if (DemandedElts[i])
1735 SubDemandedElts.setBit(i * SubScale);
1736 }
1737
1738 KnownBits KnownSrc(SubBitWidth);
1739 for (unsigned i = 0; i != SubScale; ++i) {
1740 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts.shl(shiftAmt: i), Known&: KnownSrc, Q,
1741 Depth: Depth + 1);
1742 unsigned ShiftElt = IsLE ? i : SubScale - 1 - i;
1743 Known.insertBits(SubBits: KnownSrc, BitPosition: ShiftElt * SubBitWidth);
1744 }
1745 }
1746 // Look through a cast from wider vector elements to narrow type.
1747 // Examples: v2i64 -> v4i32
1748 if (SubBitWidth % BitWidth == 0) {
1749 unsigned SubScale = SubBitWidth / BitWidth;
1750 KnownBits KnownSrc(SubBitWidth);
1751 APInt SubDemandedElts =
1752 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
1753 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts, Known&: KnownSrc, Q,
1754 Depth: Depth + 1);
1755
1756 Known.setAllConflict();
1757 for (unsigned i = 0; i != NumElts; ++i) {
1758 if (DemandedElts[i]) {
1759 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
1760 unsigned Offset = (Shifts % SubScale) * BitWidth;
1761 Known = Known.intersectWith(RHS: KnownSrc.extractBits(NumBits: BitWidth, BitPosition: Offset));
1762 if (Known.isUnknown())
1763 break;
1764 }
1765 }
1766 }
1767 break;
1768 }
1769 case Instruction::SExt: {
1770 // Compute the bits in the result that are not present in the input.
1771 unsigned SrcBitWidth = I->getOperand(i: 0)->getType()->getScalarSizeInBits();
1772
1773 Known = Known.trunc(BitWidth: SrcBitWidth);
1774 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1775 // If the sign bit of the input is known set or clear, then we know the
1776 // top bits of the result.
1777 Known = Known.sext(BitWidth);
1778 break;
1779 }
1780 case Instruction::Shl: {
1781 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1782 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1783 auto KF = [NUW, NSW](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1784 bool ShAmtNonZero) {
1785 return KnownBits::shl(LHS: KnownVal, RHS: KnownAmt, NUW, NSW, ShAmtNonZero);
1786 };
1787 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1788 KF);
1789 // X << X is even: X is either zero or shifts by at least one bit. Both
1790 // uses must observe the same value, so X must not be undef.
1791 if (I->getOperand(i: 0) == I->getOperand(i: 1) &&
1792 isGuaranteedNotToBeUndef(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT,
1793 Depth: Depth + 1))
1794 Known.Zero.setBit(0);
1795
1796 // Trailing zeros of a right-shifted constant never decrease.
1797 const APInt *C;
1798 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1799 Known.Zero.setLowBits(C->countr_zero());
1800
1801 // shl X, sub(Y, xor(ctlz(X, true), BitWidth-1)) shifts X so that its MSB
1802 // lands at bit Y, when BitWidth is a power of 2.
1803 const APInt *YC;
1804 Value *X = I->getOperand(i: 0);
1805 if (isPowerOf2_32(Value: BitWidth) &&
1806 match(V: I->getOperand(i: 1),
1807 P: m_Sub(L: m_APInt(Res&: YC), R: m_Xor(L: m_Ctlz(Op0: m_Specific(V: X), Op1: m_One()),
1808 R: m_SpecificInt(V: BitWidth - 1)))) &&
1809 YC->ult(RHS: BitWidth - 1)) {
1810 unsigned Y = YC->getZExtValue();
1811 Known.One.setBit(Y);
1812 Known.Zero.setBitsFrom(Y + 1);
1813 }
1814 break;
1815 }
1816 case Instruction::LShr: {
1817 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1818 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1819 bool ShAmtNonZero) {
1820 return KnownBits::lshr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1821 };
1822 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1823 KF);
1824 // Leading zeros of a left-shifted constant never decrease.
1825 const APInt *C;
1826 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1827 Known.Zero.setHighBits(C->countl_zero());
1828 break;
1829 }
1830 case Instruction::AShr: {
1831 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1832 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1833 bool ShAmtNonZero) {
1834 return KnownBits::ashr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1835 };
1836 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1837 KF);
1838 break;
1839 }
1840 case Instruction::Sub: {
1841 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1842 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1843 computeKnownBitsAddSub(Add: false, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1844 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1845 break;
1846 }
1847 case Instruction::Add: {
1848 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1849 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1850 computeKnownBitsAddSub(Add: true, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1851 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1852 break;
1853 }
1854 case Instruction::SRem:
1855 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1856 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1857 Known = KnownBits::srem(LHS: Known, RHS: Known2);
1858 break;
1859
1860 case Instruction::URem:
1861 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1862 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1863 Known = KnownBits::urem(LHS: Known, RHS: Known2);
1864 break;
1865 case Instruction::Alloca:
1866 Known.Zero.setLowBits(Log2(A: cast<AllocaInst>(Val: I)->getAlign()));
1867 break;
1868 case Instruction::GetElementPtr: {
1869 // Analyze all of the subscripts of this getelementptr instruction
1870 // to determine if we can prove known low zero bits.
1871 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1872 // Accumulate the constant indices in a separate variable
1873 // to minimize the number of calls to computeForAddSub.
1874 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: I->getType());
1875 APInt AccConstIndices(IndexWidth, 0);
1876
1877 auto AddIndexToKnown = [&](KnownBits IndexBits) {
1878 if (IndexWidth == BitWidth) {
1879 // Note that inbounds does *not* guarantee nsw for the addition, as only
1880 // the offset is signed, while the base address is unsigned.
1881 Known = KnownBits::add(LHS: Known, RHS: IndexBits);
1882 } else {
1883 // If the index width is smaller than the pointer width, only add the
1884 // value to the low bits.
1885 assert(IndexWidth < BitWidth &&
1886 "Index width can't be larger than pointer width");
1887 Known.insertBits(SubBits: KnownBits::add(LHS: Known.trunc(BitWidth: IndexWidth), RHS: IndexBits), BitPosition: 0);
1888 }
1889 };
1890
1891 gep_type_iterator GTI = gep_type_begin(GEP: I);
1892 for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
1893 // TrailZ can only become smaller, short-circuit if we hit zero.
1894 if (Known.isUnknown())
1895 break;
1896
1897 Value *Index = I->getOperand(i);
1898
1899 // Handle case when index is zero.
1900 Constant *CIndex = dyn_cast<Constant>(Val: Index);
1901 if (CIndex && CIndex->isNullValue())
1902 continue;
1903
1904 if (StructType *STy = GTI.getStructTypeOrNull()) {
1905 // Handle struct member offset arithmetic.
1906
1907 assert(CIndex &&
1908 "Access to structure field must be known at compile time");
1909
1910 if (CIndex->getType()->isVectorTy())
1911 Index = CIndex->getSplatValue();
1912
1913 unsigned Idx = cast<ConstantInt>(Val: Index)->getZExtValue();
1914 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
1915 uint64_t Offset = SL->getElementOffset(Idx);
1916 AccConstIndices += Offset;
1917 continue;
1918 }
1919
1920 // Handle array index arithmetic.
1921 Type *IndexedTy = GTI.getIndexedType();
1922 if (!IndexedTy->isSized()) {
1923 Known.resetAll();
1924 break;
1925 }
1926
1927 TypeSize Stride = GTI.getSequentialElementStride(DL: Q.DL);
1928 uint64_t StrideInBytes = Stride.getKnownMinValue();
1929 if (!Stride.isScalable()) {
1930 // Fast path for constant offset.
1931 if (auto *CI = dyn_cast<ConstantInt>(Val: Index)) {
1932 AccConstIndices +=
1933 CI->getValue().sextOrTrunc(width: IndexWidth) * StrideInBytes;
1934 continue;
1935 }
1936 }
1937
1938 KnownBits IndexBits =
1939 computeKnownBits(V: Index, Q, Depth: Depth + 1).sextOrTrunc(BitWidth: IndexWidth);
1940 KnownBits ScalingFactor(IndexWidth);
1941 // Multiply by current sizeof type.
1942 // &A[i] == A + i * sizeof(*A[i]).
1943 if (Stride.isScalable()) {
1944 // For scalable types the only thing we know about sizeof is
1945 // that this is a multiple of the minimum size.
1946 ScalingFactor.Zero.setLowBits(llvm::countr_zero(Val: StrideInBytes));
1947 } else {
1948 ScalingFactor =
1949 KnownBits::makeConstant(C: APInt(IndexWidth, StrideInBytes));
1950 }
1951 AddIndexToKnown(KnownBits::mul(LHS: IndexBits, RHS: ScalingFactor));
1952 }
1953 if (!Known.isUnknown() && !AccConstIndices.isZero())
1954 AddIndexToKnown(KnownBits::makeConstant(C: AccConstIndices));
1955 break;
1956 }
1957 case Instruction::PHI: {
1958 const PHINode *P = cast<PHINode>(Val: I);
1959 BinaryOperator *BO = nullptr;
1960 Value *Start = nullptr, *Step = nullptr;
1961 KnownBits &KnownStart = Known2;
1962 if (matchSimpleRecurrence(P, BO, Start, Step)) {
1963 // Handle the case of a simple two-predecessor recurrence PHI.
1964 // There's a lot more that could theoretically be done here, but
1965 // this is sufficient to catch some interesting cases.
1966 unsigned Opcode = BO->getOpcode();
1967
1968 switch (Opcode) {
1969 // If this is a shift recurrence, we know the bits being shifted in. We
1970 // can combine that with information about the start value of the
1971 // recurrence to conclude facts about the result. If this is a udiv
1972 // recurrence, we know that the result can never exceed either the
1973 // numerator or the start value, whichever is greater.
1974 case Instruction::LShr:
1975 case Instruction::AShr:
1976 case Instruction::Shl:
1977 case Instruction::UDiv:
1978 if (BO->getOperand(i_nocapture: 0) != I)
1979 break;
1980 [[fallthrough]];
1981
1982 // For a urem recurrence, the result can never exceed the start value. The
1983 // phi could either be the numerator or the denominator.
1984 case Instruction::URem: {
1985 // We have matched a recurrence of the form:
1986 // %iv = [R, %entry], [%iv.next, %backedge]
1987 // %iv.next = shift_op %iv, L
1988
1989 // Recurse with the phi context to avoid concern about whether facts
1990 // inferred hold at original context instruction. TODO: It may be
1991 // correct to use the original context. IF warranted, explore and
1992 // add sufficient tests to cover.
1993 SimplifyQuery RecQ = Q.getWithoutCondContext();
1994 RecQ.CtxI = P;
1995 computeKnownBits(V: Start, DemandedElts, Known&: KnownStart, Q: RecQ, Depth: Depth + 1);
1996 switch (Opcode) {
1997 case Instruction::Shl:
1998 // A shl recurrence will only increase the tailing zeros
1999 Known.Zero.setLowBits(KnownStart.countMinTrailingZeros());
2000 break;
2001 case Instruction::LShr:
2002 case Instruction::UDiv:
2003 case Instruction::URem:
2004 // lshr, udiv, and urem recurrences will preserve the leading zeros of
2005 // the start value.
2006 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
2007 break;
2008 case Instruction::AShr:
2009 // An ashr recurrence will extend the initial sign bit
2010 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
2011 Known.One.setHighBits(KnownStart.countMinLeadingOnes());
2012 break;
2013 }
2014 break;
2015 }
2016
2017 case Instruction::And: {
2018 // Bits that are zero in the start value stay zero, and bits that are
2019 // one in both the start value and the step stay one.
2020 KnownBits KnownStep(BitWidth);
2021 computeKnownBitsForRecurrenceOperands(P, Start, Step, DemandedElts,
2022 KnownStart, KnownStep, Q, Depth);
2023 Known.Zero |= KnownStart.Zero;
2024 Known.One |= KnownStart.One & KnownStep.One;
2025 break;
2026 }
2027
2028 case Instruction::Or: {
2029 // Bits that are zero in both the start value and the step stay zero,
2030 // and bits that are one in the start value stay one.
2031 KnownBits KnownStep(BitWidth);
2032 computeKnownBitsForRecurrenceOperands(P, Start, Step, DemandedElts,
2033 KnownStart, KnownStep, Q, Depth);
2034 Known.Zero |= KnownStart.Zero & KnownStep.Zero;
2035 Known.One |= KnownStart.One;
2036 break;
2037 }
2038
2039 // Check for operations that have the property that if
2040 // both their operands have low zero bits, the result
2041 // will have low zero bits.
2042 case Instruction::Add:
2043 case Instruction::Sub:
2044 case Instruction::Mul: {
2045 // Ok, we have a recurrence of the form {Start,op,Step}. Check for low
2046 // zero bits.
2047 KnownBits KnownStep(BitWidth);
2048 computeKnownBitsForRecurrenceOperands(P, Start, Step, DemandedElts,
2049 KnownStart, KnownStep, Q, Depth);
2050
2051 Known.Zero.setLowBits(std::min(a: KnownStart.countMinTrailingZeros(),
2052 b: KnownStep.countMinTrailingZeros()));
2053
2054 auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(Val: BO);
2055 if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(Op: OverflowOp))
2056 break;
2057
2058 switch (Opcode) {
2059 // If initial value of recurrence is nonnegative, and we are adding
2060 // a nonnegative number with nsw, the result can only be nonnegative
2061 // or poison value regardless of the number of times we execute the
2062 // add in phi recurrence. If initial value is negative and we are
2063 // adding a negative number with nsw, the result can only be
2064 // negative or poison value. Similar arguments apply to sub and mul.
2065 //
2066 // (add non-negative, non-negative) --> non-negative
2067 // (add negative, negative) --> negative
2068 case Instruction::Add: {
2069 if (KnownStart.isNonNegative() && KnownStep.isNonNegative())
2070 Known.makeNonNegative();
2071 else if (KnownStart.isNegative() && KnownStep.isNegative())
2072 Known.makeNegative();
2073 break;
2074 }
2075
2076 // (sub nsw non-negative, negative) --> non-negative
2077 // (sub nsw negative, non-negative) --> negative
2078 case Instruction::Sub: {
2079 if (BO->getOperand(i_nocapture: 0) != I)
2080 break;
2081 if (KnownStart.isNonNegative() && KnownStep.isNegative())
2082 Known.makeNonNegative();
2083 else if (KnownStart.isNegative() && KnownStep.isNonNegative())
2084 Known.makeNegative();
2085 break;
2086 }
2087
2088 // (mul nsw non-negative, non-negative) --> non-negative
2089 case Instruction::Mul:
2090 if (KnownStart.isNonNegative() && KnownStep.isNonNegative())
2091 Known.makeNonNegative();
2092 break;
2093
2094 default:
2095 break;
2096 }
2097 break;
2098 }
2099
2100 default:
2101 break;
2102 }
2103 } else {
2104 IntrinsicInst *II = nullptr;
2105 GEPOperator *GEP = nullptr;
2106 if (matchTwoInputRecurrence<IntrinsicInst>(PN: P, Inst&: II, Init&: Start, OtherOp&: Step)) {
2107 // %iv = [<Start>, %entry], [%iv.next, %backedge]
2108 //
2109 // %iv.next = <II>(%iv, <Step>)
2110 // or
2111 // %iv.next = <II>(<Step>, %iv)
2112 Intrinsic::ID IntrinsicID = II->getIntrinsicID();
2113 if (IntrinsicID == Intrinsic::umin || IntrinsicID == Intrinsic::umax) {
2114 KnownBits KnownStep(BitWidth);
2115 computeKnownBitsForRecurrenceOperands(
2116 P, Start, Step, DemandedElts, KnownStart, KnownStep, Q, Depth);
2117
2118 if (IntrinsicID == Intrinsic::umin) {
2119 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
2120 Known.One.setHighBits(std::min(a: KnownStart.countMinLeadingOnes(),
2121 b: KnownStep.countMinLeadingOnes()));
2122 } else {
2123 // umax
2124 Known.Zero.setHighBits(std::min(a: KnownStart.countMinLeadingZeros(),
2125 b: KnownStep.countMinLeadingZeros()));
2126 Known.One.setHighBits(KnownStart.countMinLeadingOnes());
2127 }
2128 }
2129 } else if (Known.isUnknown() && I->getType()->isPointerTy() &&
2130 matchTwoInputRecurrence<GEPOperator>(PN: P, Inst&: GEP, Init&: Start, OtherOp&: Step) &&
2131 GEP->getPointerOperand() == P) {
2132 // Handle a pointer induction variable, whose step is a getelementptr:
2133 // %p = phi ptr [ %start, %preheader ], [ %next, %latch ]
2134 // %next = getelementptr <ty>, ptr %p, <indices>
2135 // Every value %p takes is %start plus a whole number of steps, and
2136 // tz(a + b) >= min(tz(a), tz(b)), so the trailing zeros of %p - and
2137 // hence its alignment - are at least the smaller of those of %start and
2138 // of the step.
2139
2140 // Evaluate the start value on its own incoming edge.
2141 unsigned StartIdx = P->getIncomingValue(i: 0) == Start ? 0 : 1;
2142 SimplifyQuery RecQ = Q.getWithoutCondContext();
2143 RecQ.CtxI = P->getIncomingBlock(i: StartIdx)->getTerminator();
2144 computeKnownBits(V: Start, DemandedElts, Known&: KnownStart, Q: RecQ, Depth: Depth + 1);
2145 Known.Zero.setLowBits(
2146 std::min(a: KnownStart.countMinTrailingZeros(),
2147 b: Log2(A: GEP->getMaxPreservedAlignment(DL: Q.DL))));
2148 }
2149 }
2150
2151 // Unreachable blocks may have zero-operand PHI nodes.
2152 if (P->getNumIncomingValues() == 0)
2153 break;
2154
2155 // Otherwise take the unions of the known bit sets of the operands,
2156 // taking conservative care to avoid excessive recursion.
2157 if (Depth < MaxAnalysisRecursionDepth - 1 && Known.isUnknown()) {
2158 // Skip if every incoming value references to ourself.
2159 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
2160 break;
2161
2162 Known.setAllConflict();
2163 for (const Use &U : P->operands()) {
2164 Value *IncValue;
2165 const PHINode *CtxPhi;
2166 Instruction *CtxI;
2167 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CtxI, PhiOut: &CtxPhi);
2168 // Skip direct self references.
2169 if (IncValue == P)
2170 continue;
2171
2172 // Change the context instruction to the "edge" that flows into the
2173 // phi. This is important because that is where the value is actually
2174 // "evaluated" even though it is used later somewhere else. (see also
2175 // D69571).
2176 SimplifyQuery RecQ = Q.getWithoutCondContext().getWithInstruction(I: CtxI);
2177
2178 Known2 = KnownBits(BitWidth);
2179
2180 // Recurse, but cap the recursion to one level, because we don't
2181 // want to waste time spinning around in loops.
2182 // TODO: See if we can base recursion limiter on number of incoming phi
2183 // edges so we don't overly clamp analysis.
2184 computeKnownBits(V: IncValue, DemandedElts, Known&: Known2, Q: RecQ,
2185 Depth: MaxAnalysisRecursionDepth - 1);
2186
2187 // See if we can further use a conditional branch into the phi
2188 // to help us determine the range of the value.
2189 if (!Known2.isConstant()) {
2190 CmpPredicate Pred;
2191 Value *RHSValue = nullptr;
2192 BasicBlock *TrueSucc, *FalseSucc;
2193 if (match(
2194 V: RecQ.CtxI,
2195 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: IncValue), R: m_Value(V&: RHSValue)),
2196 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
2197 // Check for cases of duplicate successors.
2198 if ((TrueSucc == CtxPhi->getParent()) !=
2199 (FalseSucc == CtxPhi->getParent())) {
2200 // If we're using the false successor, invert the predicate.
2201 if (FalseSucc == CtxPhi->getParent())
2202 Pred = CmpInst::getInversePredicate(pred: Pred);
2203 // Get the knownbits implied by the incoming phi condition.
2204 KnownBits RHSKnown = computeKnownBits(
2205 V: RHSValue, DemandedElts, Q: RecQ, Depth: MaxAnalysisRecursionDepth - 1);
2206 if (!RHSKnown.isUnknown()) {
2207 ConstantRange RHSRange = ConstantRange::fromKnownBits(
2208 Known: RHSKnown, /*IsSigned=*/ICmpInst::isSigned(Pred));
2209 auto CR = ConstantRange::makeAllowedICmpRegion(Pred, Other: RHSRange);
2210 KnownBits KnownUnion = Known2.unionWith(RHS: CR.toKnownBits());
2211 // We can have conflicts here if we are analyzing deadcode (its
2212 // impossible for us reach this BB based the icmp).
2213 if (KnownUnion.hasConflict()) {
2214 // No reason to continue analyzing in a known dead region, so
2215 // just resetAll and break. This will cause us to also exit
2216 // the outer loop.
2217 Known.resetAll();
2218 break;
2219 }
2220 Known2 = KnownUnion;
2221 }
2222 }
2223 }
2224 }
2225
2226 Known = Known.intersectWith(RHS: Known2);
2227 // If all bits have been ruled out, there's no need to check
2228 // more operands.
2229 if (Known.isUnknown())
2230 break;
2231 }
2232 }
2233 break;
2234 }
2235 case Instruction::Call:
2236 case Instruction::Invoke: {
2237 // If range metadata is attached to this call, set known bits from that,
2238 // and then intersect with known bits based on other properties of the
2239 // function.
2240 if (MDNode *MD =
2241 Q.IIQ.getMetadata(I: cast<Instruction>(Val: I), KindID: LLVMContext::MD_range))
2242 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
2243
2244 const auto *CB = cast<CallBase>(Val: I);
2245
2246 if (std::optional<ConstantRange> Range = CB->getRange())
2247 Known = Known.unionWith(RHS: Range->toKnownBits());
2248
2249 if (const Value *RV = CB->getReturnedArgOperand()) {
2250 if (RV->getType() == I->getType()) {
2251 computeKnownBits(V: RV, Known&: Known2, Q, Depth: Depth + 1);
2252 Known = Known.unionWith(RHS: Known2);
2253 // If the function doesn't return properly for all input values
2254 // (e.g. unreachable exits) then there might be conflicts between the
2255 // argument value and the range metadata. Simply discard the known bits
2256 // in case of conflicts.
2257 if (Known.hasConflict())
2258 Known.resetAll();
2259 }
2260 }
2261 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
2262 switch (II->getIntrinsicID()) {
2263 default:
2264 break;
2265 case Intrinsic::abs: {
2266 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2267 bool IntMinIsPoison = match(V: II->getArgOperand(i: 1), P: m_One());
2268 Known = Known.unionWith(RHS: Known2.abs(IntMinIsPoison));
2269 break;
2270 }
2271 case Intrinsic::bitreverse:
2272 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2273 Known = Known.unionWith(RHS: Known2.reverseBits());
2274 break;
2275 case Intrinsic::bswap:
2276 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2277 Known = Known.unionWith(RHS: Known2.byteSwap());
2278 break;
2279 case Intrinsic::ctlz: {
2280 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2281 // If we have a known 1, its position is our upper bound.
2282 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2283 // If this call is poison for 0 input, the result will be less than 2^n.
2284 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2285 PossibleLZ = std::min(a: PossibleLZ, b: BitWidth - 1);
2286 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
2287 Known.Zero.setBitsFrom(LowBits);
2288 break;
2289 }
2290 case Intrinsic::cttz: {
2291 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2292 // If we have a known 1, its position is our upper bound.
2293 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2294 // If this call is poison for 0 input, the result will be less than 2^n.
2295 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2296 PossibleTZ = std::min(a: PossibleTZ, b: BitWidth - 1);
2297 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
2298 Known.Zero.setBitsFrom(LowBits);
2299 break;
2300 }
2301 case Intrinsic::ctpop: {
2302 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2303 // We can bound the space the count needs. Also, bits known to be zero
2304 // can't contribute to the population.
2305 unsigned BitsPossiblySet = Known2.countMaxPopulation();
2306 unsigned LowBits = llvm::bit_width(Value: BitsPossiblySet);
2307 Known.Zero.setBitsFrom(LowBits);
2308 // TODO: we could bound KnownOne using the lower bound on the number
2309 // of bits which might be set provided by popcnt KnownOne2.
2310 break;
2311 }
2312 case Intrinsic::fshr:
2313 case Intrinsic::fshl: {
2314 const APInt *SA;
2315 if (!match(V: I->getOperand(i: 2), P: m_APInt(Res&: SA)))
2316 break;
2317
2318 KnownBits Known3(BitWidth);
2319 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2320 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known3, Q, Depth: Depth + 1);
2321 Known = II->getIntrinsicID() == Intrinsic::fshl
2322 ? KnownBits::fshl(LHS: Known2, RHS: Known3, Amt: *SA)
2323 : KnownBits::fshr(LHS: Known2, RHS: Known3, Amt: *SA);
2324 break;
2325 }
2326 case Intrinsic::clmul:
2327 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2328 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2329 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
2330 break;
2331 case Intrinsic::pext:
2332 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2333 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2334 Known = KnownBits::pext(Val: Known, Mask: Known2);
2335 break;
2336 case Intrinsic::pdep:
2337 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2338 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2339 Known = KnownBits::pdep(Val: Known, Mask: Known2);
2340 break;
2341 case Intrinsic::smulh:
2342 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2343 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2344 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
2345 break;
2346 case Intrinsic::umulh:
2347 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2348 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2349 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
2350 break;
2351 case Intrinsic::uadd_sat:
2352 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2353 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2354 Known = KnownBits::uadd_sat(LHS: Known, RHS: Known2);
2355 break;
2356 case Intrinsic::usub_sat:
2357 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2358 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2359 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
2360 break;
2361 case Intrinsic::sadd_sat:
2362 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2363 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2364 Known = KnownBits::sadd_sat(LHS: Known, RHS: Known2);
2365 break;
2366 case Intrinsic::ssub_sat:
2367 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2368 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2369 Known = KnownBits::ssub_sat(LHS: Known, RHS: Known2);
2370 break;
2371 // Vec reverse preserves bits from input vec.
2372 case Intrinsic::vector_reverse:
2373 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: DemandedElts.reverseBits(), Known, Q,
2374 Depth: Depth + 1);
2375 break;
2376 // for min/max/and/or reduce, any bit common to each element in the
2377 // input vec is set in the output.
2378 case Intrinsic::vector_reduce_and:
2379 case Intrinsic::vector_reduce_or:
2380 case Intrinsic::vector_reduce_umax:
2381 case Intrinsic::vector_reduce_umin:
2382 case Intrinsic::vector_reduce_smax:
2383 case Intrinsic::vector_reduce_smin:
2384 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2385 break;
2386 case Intrinsic::vector_reduce_xor: {
2387 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2388 // The zeros common to all vecs are zero in the output.
2389 // If the number of elements is odd, then the common ones remain. If the
2390 // number of elements is even, then the common ones becomes zeros.
2391 auto *VecTy = cast<VectorType>(Val: I->getOperand(i: 0)->getType());
2392 // Even, so the ones become zeros.
2393 bool EvenCnt = VecTy->getElementCount().isKnownEven();
2394 if (EvenCnt)
2395 Known.Zero |= Known.One;
2396 // Maybe even element count so need to clear ones.
2397 if (VecTy->isScalableTy() || EvenCnt)
2398 Known.One.clearAllBits();
2399 break;
2400 }
2401 case Intrinsic::vector_reduce_add: {
2402 auto *VecTy = dyn_cast<FixedVectorType>(Val: I->getOperand(i: 0)->getType());
2403 if (!VecTy)
2404 break;
2405 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2406 Known = Known.reduceAdd(NumElts: VecTy->getNumElements());
2407 break;
2408 }
2409 case Intrinsic::umin:
2410 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2411 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2412 Known = KnownBits::umin(LHS: Known, RHS: Known2);
2413 break;
2414 case Intrinsic::umax:
2415 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2416 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2417 Known = KnownBits::umax(LHS: Known, RHS: Known2);
2418 break;
2419 case Intrinsic::smin:
2420 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2421 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2422 Known = KnownBits::smin(LHS: Known, RHS: Known2);
2423 unionWithMinMaxIntrinsicClamp(II, Known);
2424 break;
2425 case Intrinsic::smax:
2426 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2427 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2428 Known = KnownBits::smax(LHS: Known, RHS: Known2);
2429 unionWithMinMaxIntrinsicClamp(II, Known);
2430 break;
2431 case Intrinsic::ptrmask: {
2432 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2433
2434 const Value *Mask = I->getOperand(i: 1);
2435 Known2 = KnownBits(Mask->getType()->getScalarSizeInBits());
2436 computeKnownBits(V: Mask, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2437 // TODO: 1-extend would be more precise.
2438 Known &= Known2.anyextOrTrunc(BitWidth);
2439 break;
2440 }
2441 case Intrinsic::x86_sse42_crc32_64_64:
2442 Known.Zero.setBitsFrom(32);
2443 break;
2444 case Intrinsic::x86_ssse3_phadd_d_128:
2445 case Intrinsic::x86_ssse3_phadd_w_128:
2446 case Intrinsic::x86_avx2_phadd_d:
2447 case Intrinsic::x86_avx2_phadd_w: {
2448 Known = computeKnownBitsForHorizontalOperation(
2449 I, DemandedElts, Q, Depth,
2450 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2451 return KnownBits::add(LHS: KnownLHS, RHS: KnownRHS);
2452 });
2453 break;
2454 }
2455 case Intrinsic::x86_ssse3_phadd_sw_128:
2456 case Intrinsic::x86_avx2_phadd_sw: {
2457 Known = computeKnownBitsForHorizontalOperation(
2458 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::sadd_sat);
2459 break;
2460 }
2461 case Intrinsic::x86_ssse3_phsub_d_128:
2462 case Intrinsic::x86_ssse3_phsub_w_128:
2463 case Intrinsic::x86_avx2_phsub_d:
2464 case Intrinsic::x86_avx2_phsub_w: {
2465 Known = computeKnownBitsForHorizontalOperation(
2466 I, DemandedElts, Q, Depth,
2467 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2468 return KnownBits::sub(LHS: KnownLHS, RHS: KnownRHS);
2469 });
2470 break;
2471 }
2472 case Intrinsic::x86_ssse3_phsub_sw_128:
2473 case Intrinsic::x86_avx2_phsub_sw: {
2474 Known = computeKnownBitsForHorizontalOperation(
2475 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::ssub_sat);
2476 break;
2477 }
2478 case Intrinsic::riscv_vsetvli:
2479 case Intrinsic::riscv_vsetvlimax: {
2480 bool HasAVL = II->getIntrinsicID() == Intrinsic::riscv_vsetvli;
2481 const ConstantRange Range = getVScaleRange(F: II->getFunction(), BitWidth);
2482 uint64_t SEW = RISCVVType::decodeVSEW(
2483 VSEW: cast<ConstantInt>(Val: II->getArgOperand(i: HasAVL))->getZExtValue());
2484 RISCVVType::VLMUL VLMUL = static_cast<RISCVVType::VLMUL>(
2485 cast<ConstantInt>(Val: II->getArgOperand(i: 1 + HasAVL))->getZExtValue());
2486 uint64_t MaxVLEN =
2487 Range.getUnsignedMax().getZExtValue() * RISCV::RVVBitsPerBlock;
2488 uint64_t MaxVL = MaxVLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
2489
2490 // Result of vsetvli must be not larger than AVL.
2491 if (HasAVL)
2492 if (auto *CI = dyn_cast<ConstantInt>(Val: II->getArgOperand(i: 0)))
2493 MaxVL = std::min(a: MaxVL, b: CI->getZExtValue());
2494
2495 unsigned KnownZeroFirstBit = Log2_32(Value: MaxVL) + 1;
2496 if (BitWidth > KnownZeroFirstBit)
2497 Known.Zero.setBitsFrom(KnownZeroFirstBit);
2498 break;
2499 }
2500 case Intrinsic::amdgcn_mbcnt_hi:
2501 case Intrinsic::amdgcn_mbcnt_lo: {
2502 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
2503 // most 31 + src1.
2504 KnownBits MbcntKnown(BitWidth);
2505 MbcntKnown.Zero.setBitsFrom(
2506 II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo ? 6 : 5);
2507 computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Q, Depth: Depth + 1);
2508 Known = Known.unionWith(RHS: KnownBits::add(LHS: MbcntKnown, RHS: Known2));
2509 break;
2510 }
2511 case Intrinsic::vscale: {
2512 if (!II->getParent() || !II->getFunction())
2513 break;
2514
2515 Known = getVScaleRange(F: II->getFunction(), BitWidth).toKnownBits();
2516 break;
2517 }
2518 case Intrinsic::stepvector: {
2519 auto *VecTy = cast<VectorType>(Val: II->getType());
2520 unsigned MinNumElts = VecTy->getElementCount().getKnownMinValue();
2521 if (!isUIntN(N: BitWidth, x: MinNumElts))
2522 break;
2523
2524 bool Overflow = false;
2525 APInt MaxNumElts(BitWidth, MinNumElts);
2526 if (VecTy->isScalableTy()) {
2527 if (!II->getParent() || !II->getFunction())
2528 break;
2529 MaxNumElts = getVScaleRange(F: II->getFunction(), BitWidth)
2530 .getUnsignedMax()
2531 .umul_ov(RHS: MaxNumElts, Overflow);
2532 }
2533
2534 // Give up if the lane count could wrap. Stepvector truncates lane
2535 // indices that do not fit in the element type.
2536 if (Overflow)
2537 break;
2538
2539 Known.Zero.setHighBits((MaxNumElts - 1).countl_zero());
2540 break;
2541 }
2542 }
2543 }
2544 break;
2545 }
2546 case Instruction::ShuffleVector: {
2547 if (auto *Splat = getSplatValue(V: I)) {
2548 computeKnownBits(V: Splat, Known, Q, Depth: Depth + 1);
2549 break;
2550 }
2551
2552 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
2553 // FIXME: Do we need to handle ConstantExpr involving shufflevectors?
2554 if (!Shuf) {
2555 Known.resetAll();
2556 return;
2557 }
2558 // For undef elements, we don't know anything about the common state of
2559 // the shuffle result.
2560 APInt DemandedLHS, DemandedRHS;
2561 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS)) {
2562 Known.resetAll();
2563 return;
2564 }
2565 Known.setAllConflict();
2566 if (!!DemandedLHS) {
2567 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
2568 computeKnownBits(V: LHS, DemandedElts: DemandedLHS, Known, Q, Depth: Depth + 1);
2569 // If we don't know any bits, early out.
2570 if (Known.isUnknown())
2571 break;
2572 }
2573 if (!!DemandedRHS) {
2574 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
2575 computeKnownBits(V: RHS, DemandedElts: DemandedRHS, Known&: Known2, Q, Depth: Depth + 1);
2576 Known = Known.intersectWith(RHS: Known2);
2577 }
2578 break;
2579 }
2580 case Instruction::InsertElement: {
2581 if (isa<ScalableVectorType>(Val: I->getType())) {
2582 Known.resetAll();
2583 return;
2584 }
2585 const Value *Vec = I->getOperand(i: 0);
2586 const Value *Elt = I->getOperand(i: 1);
2587 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
2588 unsigned NumElts = DemandedElts.getBitWidth();
2589 APInt DemandedVecElts = DemandedElts;
2590 bool NeedsElt = true;
2591 // If we know the index we are inserting too, clear it from Vec check.
2592 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
2593 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
2594 NeedsElt = DemandedElts[CIdx->getZExtValue()];
2595 }
2596
2597 Known.setAllConflict();
2598 if (NeedsElt) {
2599 computeKnownBits(V: Elt, Known, Q, Depth: Depth + 1);
2600 // If we don't know any bits, early out.
2601 if (Known.isUnknown())
2602 break;
2603 }
2604
2605 if (!DemandedVecElts.isZero()) {
2606 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known&: Known2, Q, Depth: Depth + 1);
2607 Known = Known.intersectWith(RHS: Known2);
2608 }
2609 break;
2610 }
2611 case Instruction::ExtractElement: {
2612 // Look through extract element. If the index is non-constant or
2613 // out-of-range demand all elements, otherwise just the extracted element.
2614 const Value *Vec = I->getOperand(i: 0);
2615 const Value *Idx = I->getOperand(i: 1);
2616 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
2617 if (isa<ScalableVectorType>(Val: Vec->getType())) {
2618 // FIXME: there's probably *something* we can do with scalable vectors
2619 Known.resetAll();
2620 break;
2621 }
2622 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2623 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
2624 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
2625 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
2626 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known, Q, Depth: Depth + 1);
2627 break;
2628 }
2629 case Instruction::ExtractValue:
2630 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I->getOperand(i: 0))) {
2631 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: I);
2632 if (EVI->getNumIndices() != 1) break;
2633 if (EVI->getIndices()[0] == 0) {
2634 switch (II->getIntrinsicID()) {
2635 default: break;
2636 case Intrinsic::uadd_with_overflow:
2637 case Intrinsic::sadd_with_overflow:
2638 computeKnownBitsAddSub(
2639 Add: true, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2640 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2641 break;
2642 case Intrinsic::usub_with_overflow:
2643 case Intrinsic::ssub_with_overflow:
2644 computeKnownBitsAddSub(
2645 Add: false, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2646 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2647 break;
2648 case Intrinsic::umul_with_overflow:
2649 case Intrinsic::smul_with_overflow:
2650 computeKnownBitsMul(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), NSW: false,
2651 NUW: false, DemandedElts, Known, Known2, Q, Depth);
2652 break;
2653 }
2654 }
2655 }
2656 break;
2657 case Instruction::Freeze:
2658 if (isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT,
2659 Depth: Depth + 1))
2660 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2661 break;
2662 }
2663}
2664
2665/// Determine which bits of V are known to be either zero or one and return
2666/// them.
2667KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
2668 const SimplifyQuery &Q, unsigned Depth) {
2669 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2670 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
2671 return Known;
2672}
2673
2674/// Determine which bits of V are known to be either zero or one and return
2675/// them.
2676KnownBits llvm::computeKnownBits(const Value *V, const SimplifyQuery &Q,
2677 unsigned Depth) {
2678 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2679 computeKnownBits(V, Known, Q, Depth);
2680 return Known;
2681}
2682
2683/// Determine which bits of V are known to be either zero or one and return
2684/// them in the Known bit set.
2685///
2686/// NOTE: we cannot consider 'undef' to be "IsZero" here. The problem is that
2687/// we cannot optimize based on the assumption that it is zero without changing
2688/// it to be an explicit zero. If we don't change it to zero, other code could
2689/// optimized based on the contradictory assumption that it is non-zero.
2690/// Because instcombine aggressively folds operations with undef args anyway,
2691/// this won't lose us code quality.
2692///
2693/// This function is defined on values with integer type, values with pointer
2694/// type, and vectors of integers. In the case
2695/// where V is a vector, known zero, and known one values are the
2696/// same width as the vector element, and the bit is set only if it is true
2697/// for all of the demanded elements in the vector specified by DemandedElts.
2698void computeKnownBits(const Value *V, const APInt &DemandedElts,
2699 KnownBits &Known, const SimplifyQuery &Q,
2700 unsigned Depth) {
2701 if (!DemandedElts) {
2702 // No demanded elts, better to assume we don't know anything.
2703 Known.resetAll();
2704 return;
2705 }
2706
2707 assert(V && "No Value?");
2708 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2709
2710#ifndef NDEBUG
2711 Type *Ty = V->getType();
2712 unsigned BitWidth = Known.getBitWidth();
2713
2714 assert((Ty->isIntOrIntVectorTy(BitWidth) || Ty->isPtrOrPtrVectorTy()) &&
2715 "Not integer or pointer type!");
2716
2717 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
2718 assert(
2719 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
2720 "DemandedElt width should equal the fixed vector number of elements");
2721 } else {
2722 assert(DemandedElts == APInt(1, 1) &&
2723 "DemandedElt width should be 1 for scalars or scalable vectors");
2724 }
2725
2726 Type *ScalarTy = Ty->getScalarType();
2727 if (ScalarTy->isPointerTy()) {
2728 assert(BitWidth == Q.DL.getPointerTypeSizeInBits(ScalarTy) &&
2729 "V and Known should have same BitWidth");
2730 } else {
2731 assert(BitWidth == Q.DL.getTypeSizeInBits(ScalarTy) &&
2732 "V and Known should have same BitWidth");
2733 }
2734#endif
2735
2736 const APInt *C;
2737 if (match(V, P: m_APInt(Res&: C))) {
2738 // We know all of the bits for a scalar constant or a splat vector constant!
2739 Known = KnownBits::makeConstant(C: *C);
2740 return;
2741 }
2742 // Null and aggregate-zero are all-zeros.
2743 if (isa<ConstantPointerNull>(Val: V) || isa<ConstantAggregateZero>(Val: V)) {
2744 Known.setAllZero();
2745 return;
2746 }
2747 // Handle a constant vector by taking the intersection of the known bits of
2748 // each element.
2749 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: V)) {
2750 assert(!isa<ScalableVectorType>(V->getType()));
2751 // We know that CDV must be a vector of integers. Take the intersection of
2752 // each element.
2753 Known.setAllConflict();
2754 for (unsigned i = 0, e = CDV->getNumElements(); i != e; ++i) {
2755 if (!DemandedElts[i])
2756 continue;
2757 APInt Elt = CDV->getElementAsAPInt(i);
2758 Known.Zero &= ~Elt;
2759 Known.One &= Elt;
2760 }
2761 if (Known.hasConflict())
2762 Known.resetAll();
2763 return;
2764 }
2765
2766 if (const auto *CV = dyn_cast<ConstantVector>(Val: V)) {
2767 assert(!isa<ScalableVectorType>(V->getType()));
2768 // We know that CV must be a vector of integers. Take the intersection of
2769 // each element.
2770 Known.setAllConflict();
2771 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
2772 if (!DemandedElts[i])
2773 continue;
2774 Constant *Element = CV->getAggregateElement(Elt: i);
2775 if (isa<PoisonValue>(Val: Element))
2776 continue;
2777 auto *ElementCI = dyn_cast_or_null<ConstantInt>(Val: Element);
2778 if (!ElementCI) {
2779 Known.resetAll();
2780 return;
2781 }
2782 const APInt &Elt = ElementCI->getValue();
2783 Known.Zero &= ~Elt;
2784 Known.One &= Elt;
2785 }
2786 if (Known.hasConflict())
2787 Known.resetAll();
2788 return;
2789 }
2790
2791 // Start out not knowing anything.
2792 Known.resetAll();
2793
2794 // We can't imply anything about undefs.
2795 if (isa<UndefValue>(Val: V))
2796 return;
2797
2798 // There's no point in looking through other users of ConstantData for
2799 // assumptions. Confirm that we've handled them all.
2800 assert(!isa<ConstantData>(V) && "Unhandled constant data!");
2801
2802 if (const auto *A = dyn_cast<Argument>(Val: V))
2803 if (std::optional<ConstantRange> Range = A->getRange())
2804 Known = Range->toKnownBits();
2805
2806 // All recursive calls that increase depth must come after this.
2807 if (Depth == MaxAnalysisRecursionDepth)
2808 return;
2809
2810 // A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
2811 // the bits of its aliasee.
2812 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: V)) {
2813 if (!GA->isInterposable())
2814 computeKnownBits(V: GA->getAliasee(), Known, Q, Depth: Depth + 1);
2815 return;
2816 }
2817
2818 if (const Operator *I = dyn_cast<Operator>(Val: V))
2819 computeKnownBitsFromOperator(I, DemandedElts, Known, Q, Depth);
2820 else if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
2821 if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
2822 Known = CR->toKnownBits();
2823 }
2824
2825 // Aligned pointers have trailing zeros - refine Known.Zero set
2826 if (isa<PointerType>(Val: V->getType())) {
2827 Align Alignment = V->getPointerAlignment(DL: Q.DL);
2828 Known.Zero.setLowBits(Log2(A: Alignment));
2829 }
2830
2831 // computeKnownBitsFromContext strictly refines Known.
2832 // Therefore, we run them after computeKnownBitsFromOperator.
2833
2834 // Check whether we can determine known bits from context such as assumes.
2835 computeKnownBitsFromContext(V, Known, Q, Depth);
2836}
2837
2838/// Try to detect a recurrence that the value of the induction variable is
2839/// always a power of two (or zero).
2840static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
2841 SimplifyQuery &Q, unsigned Depth) {
2842 BinaryOperator *BO = nullptr;
2843 Value *Start = nullptr, *Step = nullptr;
2844 if (!matchSimpleRecurrence(P: PN, BO, Start, Step))
2845 return false;
2846
2847 // Initial value must be a power of two.
2848 for (const Use &U : PN->operands()) {
2849 if (U.get() == Start) {
2850 // Initial value comes from a different BB, need to adjust context
2851 // instruction for analysis.
2852 Q.CtxI = PN->getIncomingBlock(U)->getTerminator();
2853 if (!isKnownToBeAPowerOfTwo(V: Start, OrZero, Q, Depth))
2854 return false;
2855 }
2856 }
2857
2858 // Except for Mul, the induction variable must be on the left side of the
2859 // increment expression, otherwise its value can be arbitrary.
2860 if (BO->getOpcode() != Instruction::Mul && BO->getOperand(i_nocapture: 1) != Step)
2861 return false;
2862
2863 Q.CtxI = BO->getParent()->getTerminator();
2864 switch (BO->getOpcode()) {
2865 case Instruction::Mul:
2866 // Power of two is closed under multiplication.
2867 return (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) ||
2868 Q.IIQ.hasNoSignedWrap(Op: BO)) &&
2869 isKnownToBeAPowerOfTwo(V: Step, OrZero, Q, Depth);
2870 case Instruction::SDiv:
2871 // Start value must not be signmask for signed division, so simply being a
2872 // power of two is not sufficient, and it has to be a constant.
2873 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2874 return false;
2875 [[fallthrough]];
2876 case Instruction::UDiv:
2877 // Divisor must be a power of two.
2878 // If OrZero is false, cannot guarantee induction variable is non-zero after
2879 // division, same for Shr, unless it is exact division.
2880 return (OrZero || Q.IIQ.isExact(Op: BO)) &&
2881 isKnownToBeAPowerOfTwo(V: Step, OrZero: false, Q, Depth);
2882 case Instruction::Shl:
2883 return OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO);
2884 case Instruction::AShr:
2885 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2886 return false;
2887 [[fallthrough]];
2888 case Instruction::LShr:
2889 return OrZero || Q.IIQ.isExact(Op: BO);
2890 default:
2891 return false;
2892 }
2893}
2894
2895/// Return true if we can infer that \p V is known to be a power of 2 from
2896/// dominating condition \p Cond (e.g., ctpop(V) == 1).
2897static bool isImpliedToBeAPowerOfTwoFromCond(const Value *V, bool OrZero,
2898 const Value *Cond,
2899 bool CondIsTrue) {
2900 CmpPredicate Pred;
2901 const APInt *RHSC;
2902 if (!match(V: Cond, P: m_ICmp(Pred, L: m_Ctpop(Op0: m_Specific(V)), R: m_APInt(Res&: RHSC))))
2903 return false;
2904 if (!CondIsTrue)
2905 Pred = ICmpInst::getInversePredicate(pred: Pred);
2906 // ctpop(V) u< 2
2907 if (OrZero && Pred == ICmpInst::ICMP_ULT && *RHSC == 2)
2908 return true;
2909 // ctpop(V) == 1
2910 return Pred == ICmpInst::ICMP_EQ && *RHSC == 1;
2911}
2912
2913/// Return true if the given value is known to have exactly one
2914/// bit set when defined. For vectors return true if every element is known to
2915/// be a power of two when defined. Supports values with integer or pointer
2916/// types and vectors of integers.
2917bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero,
2918 const SimplifyQuery &Q, unsigned Depth) {
2919 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2920
2921 if (isa<Constant>(Val: V))
2922 return OrZero ? match(V, P: m_Power2OrZero()) : match(V, P: m_Power2());
2923
2924 // i1 is by definition a power of 2 or zero.
2925 if (OrZero && V->getType()->getScalarSizeInBits() == 1)
2926 return true;
2927
2928 // Try to infer from assumptions.
2929 if (Q.AC && Q.CtxI) {
2930 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
2931 if (!AssumeVH)
2932 continue;
2933 CallInst *I = cast<CallInst>(Val&: AssumeVH);
2934 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond: I->getArgOperand(i: 0),
2935 /*CondIsTrue=*/true) &&
2936 isValidAssumeForContext(I, Q))
2937 return true;
2938 }
2939 }
2940
2941 // Handle dominating conditions.
2942 if (Q.DC && Q.CtxI && Q.DT) {
2943 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
2944 Value *Cond = BI->getCondition();
2945
2946 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
2947 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2948 /*CondIsTrue=*/true) &&
2949 Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()))
2950 return true;
2951
2952 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
2953 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2954 /*CondIsTrue=*/false) &&
2955 Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()))
2956 return true;
2957 }
2958 }
2959
2960 auto *I = dyn_cast<Instruction>(Val: V);
2961 if (!I)
2962 return false;
2963
2964 if (Q.CtxI && match(V, P: m_VScale())) {
2965 const Function *F = Q.CtxI->getFunction();
2966 // The vscale_range indicates vscale is a power-of-two.
2967 return F->hasFnAttribute(Kind: Attribute::VScaleRange);
2968 }
2969
2970 // 1 << X is clearly a power of two if the one is not shifted off the end. If
2971 // it is shifted off the end then the result is undefined.
2972 if (match(V: I, P: m_Shl(L: m_One(), R: m_Value())))
2973 return true;
2974
2975 // (signmask) >>l X is clearly a power of two if the one is not shifted off
2976 // the bottom. If it is shifted off the bottom then the result is undefined.
2977 if (match(V: I, P: m_LShr(L: m_SignMask(), R: m_Value())))
2978 return true;
2979
2980 // The remaining tests are all recursive, so bail out if we hit the limit.
2981 if (Depth++ == MaxAnalysisRecursionDepth)
2982 return false;
2983
2984 switch (I->getOpcode()) {
2985 case Instruction::ZExt:
2986 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2987 case Instruction::Trunc:
2988 return OrZero && isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2989 case Instruction::Shl:
2990 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: I) || Q.IIQ.hasNoSignedWrap(Op: I))
2991 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2992 return false;
2993 case Instruction::LShr:
2994 if (OrZero || Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2995 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2996 return false;
2997 case Instruction::UDiv:
2998 if (Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2999 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
3000 return false;
3001 case Instruction::Mul:
3002 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
3003 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth) &&
3004 (OrZero || isKnownNonZero(V: I, Q, Depth));
3005 case Instruction::And:
3006 // A power of two and'd with anything is a power of two or zero.
3007 if (OrZero &&
3008 (isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), /*OrZero*/ true, Q, Depth) ||
3009 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), /*OrZero*/ true, Q, Depth)))
3010 return true;
3011 // X & (-X) is always a power of two or zero.
3012 if (match(V: I->getOperand(i: 0), P: m_Neg(V: m_Specific(V: I->getOperand(i: 1)))) ||
3013 match(V: I->getOperand(i: 1), P: m_Neg(V: m_Specific(V: I->getOperand(i: 0)))))
3014 return OrZero || isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3015 return false;
3016 case Instruction::Add: {
3017 // Adding a power-of-two or zero to the same power-of-two or zero yields
3018 // either the original power-of-two, a larger power-of-two or zero.
3019 const OverflowingBinaryOperator *VOBO = cast<OverflowingBinaryOperator>(Val: V);
3020 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO) ||
3021 Q.IIQ.hasNoSignedWrap(Op: VOBO)) {
3022 if (match(V: I->getOperand(i: 0),
3023 P: m_c_And(L: m_Specific(V: I->getOperand(i: 1)), R: m_Value())) &&
3024 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth))
3025 return true;
3026 if (match(V: I->getOperand(i: 1),
3027 P: m_c_And(L: m_Specific(V: I->getOperand(i: 0)), R: m_Value())) &&
3028 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth))
3029 return true;
3030
3031 unsigned BitWidth = V->getType()->getScalarSizeInBits();
3032 KnownBits LHSBits(BitWidth);
3033 computeKnownBits(V: I->getOperand(i: 0), Known&: LHSBits, Q, Depth);
3034
3035 KnownBits RHSBits(BitWidth);
3036 computeKnownBits(V: I->getOperand(i: 1), Known&: RHSBits, Q, Depth);
3037 // If i8 V is a power of two or zero:
3038 // ZeroBits: 1 1 1 0 1 1 1 1
3039 // ~ZeroBits: 0 0 0 1 0 0 0 0
3040 if ((~(LHSBits.Zero & RHSBits.Zero)).isPowerOf2())
3041 // If OrZero isn't set, we cannot give back a zero result.
3042 // Make sure either the LHS or RHS has a bit set.
3043 if (OrZero || RHSBits.One.getBoolValue() || LHSBits.One.getBoolValue())
3044 return true;
3045 }
3046
3047 // LShr(UINT_MAX, Y) + 1 is a power of two (if add is nuw) or zero.
3048 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO))
3049 if (match(V: I, P: m_Add(L: m_LShr(L: m_AllOnes(), R: m_Value()), R: m_One())))
3050 return true;
3051 return false;
3052 }
3053 case Instruction::Select:
3054 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
3055 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 2), OrZero, Q, Depth);
3056 case Instruction::PHI: {
3057 // A PHI node is power of two if all incoming values are power of two, or if
3058 // it is an induction variable where in each step its value is a power of
3059 // two.
3060 auto *PN = cast<PHINode>(Val: I);
3061 SimplifyQuery RecQ = Q.getWithoutCondContext();
3062
3063 // Check if it is an induction variable and always power of two.
3064 if (isPowerOfTwoRecurrence(PN, OrZero, Q&: RecQ, Depth))
3065 return true;
3066
3067 // Recursively check all incoming values. Limit recursion to 2 levels, so
3068 // that search complexity is limited to number of operands^2.
3069 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
3070 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
3071 // Value is power of 2 if it is coming from PHI node itself by induction.
3072 if (U.get() == PN)
3073 return true;
3074
3075 // Change the context instruction to the incoming block where it is
3076 // evaluated.
3077 RecQ.CtxI = PN->getIncomingBlock(U)->getTerminator();
3078 return isKnownToBeAPowerOfTwo(V: U.get(), OrZero, Q: RecQ, Depth: NewDepth);
3079 });
3080 }
3081 case Instruction::Invoke:
3082 case Instruction::Call: {
3083 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3084 switch (II->getIntrinsicID()) {
3085 case Intrinsic::umax:
3086 case Intrinsic::smax:
3087 case Intrinsic::umin:
3088 case Intrinsic::smin:
3089 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 1), OrZero, Q, Depth) &&
3090 isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
3091 // bswap/bitreverse just move around bits, but don't change any 1s/0s
3092 // thus dont change pow2/non-pow2 status.
3093 case Intrinsic::bitreverse:
3094 case Intrinsic::bswap:
3095 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
3096 case Intrinsic::fshr:
3097 case Intrinsic::fshl:
3098 // If Op0 == Op1, this is a rotate. is_pow2(rotate(x, y)) == is_pow2(x)
3099 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
3100 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
3101 break;
3102 case Intrinsic::riscv_vsetvlimax:
3103 // VLMAX is VLEN * LMUL / SEW, which is always a non-zero power of two
3104 // for any valid vtype, so it is a power of two regardless of OrZero.
3105 return true;
3106 case Intrinsic::read_register:
3107 case Intrinsic::read_volatile_register: {
3108 // The RISC-V vlenb CSR holds VLEN/8, which is always a non-zero power
3109 // of two, so it is a power of two regardless of OrZero.
3110 const Module *M = II->getModule();
3111 if (!M || !M->getTargetTriple().isRISCV())
3112 break;
3113 return isReadVLENB(II: *II);
3114 }
3115 default:
3116 break;
3117 }
3118 }
3119 return false;
3120 }
3121 default:
3122 return false;
3123 }
3124}
3125
3126/// Test whether a GEP's result is known to be non-null.
3127///
3128/// Uses properties inherent in a GEP to try to determine whether it is known
3129/// to be non-null.
3130///
3131/// Currently this routine does not support vector GEPs.
3132static bool isGEPKnownNonNull(const GEPOperator *GEP, const SimplifyQuery &Q,
3133 unsigned Depth) {
3134 const Function *F = nullptr;
3135 if (const Instruction *I = dyn_cast<Instruction>(Val: GEP))
3136 F = I->getFunction();
3137
3138 // If the gep is nuw or inbounds with invalid null pointer, then the GEP
3139 // may be null iff the base pointer is null and the offset is zero.
3140 if (!GEP->hasNoUnsignedWrap() &&
3141 !(GEP->isInBounds() &&
3142 !NullPointerIsDefined(F, AS: GEP->getPointerAddressSpace())))
3143 return false;
3144
3145 // FIXME: Support vector-GEPs.
3146 assert(GEP->getType()->isPointerTy() && "We only support plain pointer GEP");
3147
3148 // If the base pointer is non-null, we cannot walk to a null address with an
3149 // inbounds GEP in address space zero.
3150 if (isKnownNonZero(V: GEP->getPointerOperand(), Q, Depth))
3151 return true;
3152
3153 // Walk the GEP operands and see if any operand introduces a non-zero offset.
3154 // If so, then the GEP cannot produce a null pointer, as doing so would
3155 // inherently violate the inbounds contract within address space zero.
3156 for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
3157 GTI != GTE; ++GTI) {
3158 // Struct types are easy -- they must always be indexed by a constant.
3159 if (StructType *STy = GTI.getStructTypeOrNull()) {
3160 ConstantInt *OpC = cast<ConstantInt>(Val: GTI.getOperand());
3161 unsigned ElementIdx = OpC->getZExtValue();
3162 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
3163 uint64_t ElementOffset = SL->getElementOffset(Idx: ElementIdx);
3164 if (ElementOffset > 0)
3165 return true;
3166 continue;
3167 }
3168
3169 // If we have a zero-sized type, the index doesn't matter. Keep looping.
3170 if (GTI.getSequentialElementStride(DL: Q.DL).isZero())
3171 continue;
3172
3173 // Fast path the constant operand case both for efficiency and so we don't
3174 // increment Depth when just zipping down an all-constant GEP.
3175 if (ConstantInt *OpC = dyn_cast<ConstantInt>(Val: GTI.getOperand())) {
3176 if (!OpC->isZero())
3177 return true;
3178 continue;
3179 }
3180
3181 // We post-increment Depth here because while isKnownNonZero increments it
3182 // as well, when we pop back up that increment won't persist. We don't want
3183 // to recurse 10k times just because we have 10k GEP operands. We don't
3184 // bail completely out because we want to handle constant GEPs regardless
3185 // of depth.
3186 if (Depth++ >= MaxAnalysisRecursionDepth)
3187 continue;
3188
3189 if (isKnownNonZero(V: GTI.getOperand(), Q, Depth))
3190 return true;
3191 }
3192
3193 return false;
3194}
3195
3196static bool isKnownNonNullFromDominatingCondition(const Value *V,
3197 const Instruction *CtxI,
3198 const DominatorTree *DT) {
3199 assert(!isa<Constant>(V) && "Called for constant?");
3200
3201 if (!CtxI || !DT)
3202 return false;
3203
3204 unsigned NumUsesExplored = 0;
3205 for (auto &U : V->uses()) {
3206 // Avoid massive lists
3207 if (NumUsesExplored >= DomConditionsMaxUses)
3208 break;
3209 NumUsesExplored++;
3210
3211 const Instruction *UI = cast<Instruction>(Val: U.getUser());
3212 // If the value is used as an argument to a call or invoke, then argument
3213 // attributes may provide an answer about null-ness.
3214 if (V->getType()->isPointerTy()) {
3215 if (const auto *CB = dyn_cast<CallBase>(Val: UI)) {
3216 if (CB->isArgOperand(U: &U) &&
3217 CB->paramHasNonNullAttr(ArgNo: CB->getArgOperandNo(U: &U),
3218 /*AllowUndefOrPoison=*/false) &&
3219 DT->dominates(Def: CB, User: CtxI))
3220 return true;
3221 }
3222 }
3223
3224 // If the value is used as a load/store, then the pointer must be non null.
3225 if (V == getLoadStorePointerOperand(V: UI)) {
3226 if (!NullPointerIsDefined(F: UI->getFunction(),
3227 AS: V->getType()->getPointerAddressSpace()) &&
3228 DT->dominates(Def: UI, User: CtxI))
3229 return true;
3230 }
3231
3232 if ((match(V: UI, P: m_IDiv(L: m_Value(), R: m_Specific(V))) ||
3233 match(V: UI, P: m_IRem(L: m_Value(), R: m_Specific(V)))) &&
3234 isValidAssumeForContext(Inv: UI, CtxI, DT))
3235 return true;
3236
3237 // Consider only compare instructions uniquely controlling a branch
3238 Value *RHS;
3239 CmpPredicate Pred;
3240 bool NonNullIfTrue;
3241 if (match(V: UI, P: m_c_ICmp(Pred, L: m_Specific(V), R: m_Value(V&: RHS)))) {
3242 if (cmpExcludesZero(Pred, RHS))
3243 NonNullIfTrue = true;
3244 else if (cmpExcludesZero(Pred: CmpInst::getInversePredicate(pred: Pred), RHS))
3245 NonNullIfTrue = false;
3246 else
3247 continue;
3248 } else if (UI->getType()->isIntegerTy(BitWidth: 1) &&
3249 match(V: UI, P: m_Trunc(Op: m_Specific(V)))) {
3250 NonNullIfTrue = true;
3251 } else
3252 continue;
3253
3254 SmallVector<const User *, 4> WorkList;
3255 SmallPtrSet<const User *, 4> Visited;
3256 for (const auto *CmpU : UI->users()) {
3257 assert(WorkList.empty() && "Should be!");
3258 if (Visited.insert(Ptr: CmpU).second)
3259 WorkList.push_back(Elt: CmpU);
3260
3261 while (!WorkList.empty()) {
3262 auto *Curr = WorkList.pop_back_val();
3263
3264 // If a user is an AND, add all its users to the work list. We only
3265 // propagate "pred != null" condition through AND because it is only
3266 // correct to assume that all conditions of AND are met in true branch.
3267 // TODO: Support similar logic of OR and EQ predicate?
3268 if (NonNullIfTrue)
3269 if (match(V: Curr, P: m_LogicalAnd(L: m_Value(), R: m_Value()))) {
3270 for (const auto *CurrU : Curr->users())
3271 if (Visited.insert(Ptr: CurrU).second)
3272 WorkList.push_back(Elt: CurrU);
3273 continue;
3274 }
3275
3276 if (const CondBrInst *BI = dyn_cast<CondBrInst>(Val: Curr)) {
3277 BasicBlock *NonNullSuccessor =
3278 BI->getSuccessor(i: NonNullIfTrue ? 0 : 1);
3279 BasicBlockEdge Edge(BI->getParent(), NonNullSuccessor);
3280 if (DT->dominates(BBE: Edge, BB: CtxI->getParent()))
3281 return true;
3282 } else if (NonNullIfTrue && isGuard(U: Curr) &&
3283 DT->dominates(Def: cast<Instruction>(Val: Curr), User: CtxI)) {
3284 return true;
3285 }
3286 }
3287 }
3288 }
3289
3290 return false;
3291}
3292
3293/// Does the 'Range' metadata (which must be a valid MD_range operand list)
3294/// ensure that the value it's attached to is never Value? 'RangeType' is
3295/// is the type of the value described by the range.
3296static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value) {
3297 const unsigned NumRanges = Ranges->getNumOperands() / 2;
3298 assert(NumRanges >= 1);
3299 for (unsigned i = 0; i < NumRanges; ++i) {
3300 ConstantInt *Lower =
3301 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 0));
3302 ConstantInt *Upper =
3303 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 1));
3304 ConstantRange Range(Lower->getValue(), Upper->getValue());
3305 if (Range.contains(Val: Value))
3306 return false;
3307 }
3308 return true;
3309}
3310
3311/// Try to detect a recurrence that monotonically increases/decreases from a
3312/// non-zero starting value. These are common as induction variables.
3313static bool isNonZeroRecurrence(const PHINode *PN) {
3314 BinaryOperator *BO = nullptr;
3315 Value *Start = nullptr, *Step = nullptr;
3316 const APInt *StartC, *StepC;
3317 if (!matchSimpleRecurrence(P: PN, BO, Start, Step) ||
3318 !match(V: Start, P: m_APInt(Res&: StartC)) || StartC->isZero())
3319 return false;
3320
3321 switch (BO->getOpcode()) {
3322 case Instruction::Add:
3323 // Starting from non-zero and stepping away from zero can never wrap back
3324 // to zero.
3325 return BO->hasNoUnsignedWrap() ||
3326 (BO->hasNoSignedWrap() && match(V: Step, P: m_APInt(Res&: StepC)) &&
3327 StartC->isNegative() == StepC->isNegative());
3328 case Instruction::Mul:
3329 return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) &&
3330 match(V: Step, P: m_APInt(Res&: StepC)) && !StepC->isZero();
3331 case Instruction::Shl:
3332 return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap();
3333 case Instruction::AShr:
3334 case Instruction::LShr:
3335 return BO->isExact();
3336 case Instruction::Or:
3337 return true;
3338 default:
3339 return false;
3340 }
3341}
3342
3343static bool matchOpWithOpEqZero(Value *Op0, Value *Op1) {
3344 return match(V: Op0, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3345 L: m_Specific(V: Op1), R: m_Zero()))) ||
3346 match(V: Op1, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3347 L: m_Specific(V: Op0), R: m_Zero())));
3348}
3349
3350/// Return true if "addr(P) + Off" cannot be zero. The object P points to
3351/// cannot cross the unsigned address space boundary, including the address one
3352/// past its end, so an offset that stays within the object cannot wrap around
3353/// to zero. This holds even if the object starts at the zero address, which is
3354/// all that null_pointer_is_valid permits, so a non-zero offset is enough.
3355static bool isNonZeroAddrPlusOffset(const SimplifyQuery &Q, Value *X,
3356 Value *Y) {
3357 const APInt *Off;
3358 if (!match(V: Y, P: m_APInt(Res&: Off)) || Off->isZero())
3359 return false;
3360
3361 // ptrtoaddr always produces the address. ptrtoint only does so when its
3362 // result is at least as wide as the address: a narrower result is truncated
3363 // and may wrap to zero, while bits above the address can only make the
3364 // result more non-zero.
3365 Value *P;
3366 if (!match(V: X, P: m_PtrToAddr(Op: m_Value(V&: P))) &&
3367 !(match(V: X, P: m_PtrToInt(Op: m_Value(V&: P))) &&
3368 X->getType()->getScalarSizeInBits() >=
3369 Q.DL.getAddressSizeInBits(Ty: P->getType())))
3370 return false;
3371
3372 // A vector of pointers has no single object whose size can be taken, and
3373 // getPointerDereferenceableBytes() requires a plain pointer.
3374 if (!P->getType()->isPointerTy())
3375 return false;
3376
3377 bool CanBeNull;
3378 uint64_t DerefBytes = P->getPointerDereferenceableBytes(
3379 DL: Q.DL, CanBeNull, /*CanBeFreed=*/nullptr);
3380 if (CanBeNull || DerefBytes == 0 || !Off->ule(RHS: DerefBytes))
3381 return false;
3382
3383 return true;
3384}
3385
3386static bool isNonZeroAdd(const APInt &DemandedElts, const SimplifyQuery &Q,
3387 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3388 bool NUW, unsigned Depth) {
3389 if (isNonZeroAddrPlusOffset(Q, X, Y) || isNonZeroAddrPlusOffset(Q, X: Y, Y: X))
3390 return true;
3391
3392 // (X + (X != 0)) is non zero
3393 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3394 return true;
3395
3396 if (NUW)
3397 return isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3398 isKnownNonZero(V: X, DemandedElts, Q, Depth);
3399
3400 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3401 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3402
3403 // If X and Y are both non-negative (as signed values) then their sum is not
3404 // zero unless both X and Y are zero.
3405 if (XKnown.isNonNegative() && YKnown.isNonNegative())
3406 if (isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3407 isKnownNonZero(V: X, DemandedElts, Q, Depth))
3408 return true;
3409
3410 // If X and Y are both negative (as signed values) then their sum is not
3411 // zero unless both X and Y equal INT_MIN.
3412 if (XKnown.isNegative() && YKnown.isNegative()) {
3413 APInt Mask = APInt::getSignedMaxValue(numBits: BitWidth);
3414 // The sign bit of X is set. If some other bit is set then X is not equal
3415 // to INT_MIN.
3416 if (XKnown.One.intersects(RHS: Mask))
3417 return true;
3418 // The sign bit of Y is set. If some other bit is set then Y is not equal
3419 // to INT_MIN.
3420 if (YKnown.One.intersects(RHS: Mask))
3421 return true;
3422 }
3423
3424 // The sum of a non-negative number and a power of two is not zero.
3425 if (XKnown.isNonNegative() &&
3426 isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ false, Q, Depth))
3427 return true;
3428 if (YKnown.isNonNegative() &&
3429 isKnownToBeAPowerOfTwo(V: X, /*OrZero*/ false, Q, Depth))
3430 return true;
3431
3432 return KnownBits::add(LHS: XKnown, RHS: YKnown, NSW, NUW).isNonZero();
3433}
3434
3435static bool isNonZeroSub(const APInt &DemandedElts, const SimplifyQuery &Q,
3436 unsigned BitWidth, Value *X, Value *Y,
3437 unsigned Depth) {
3438 // (X - (X != 0)) is non zero
3439 // ((X != 0) - X) is non zero
3440 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3441 return true;
3442
3443 // TODO: Move this case into isKnownNonEqual().
3444 if (auto *C = dyn_cast<Constant>(Val: X))
3445 if (C->isNullValue() && isKnownNonZero(V: Y, DemandedElts, Q, Depth))
3446 return true;
3447
3448 return ::isKnownNonEqual(V1: X, V2: Y, DemandedElts, Q, Depth);
3449}
3450
3451static bool isNonZeroMul(const APInt &DemandedElts, const SimplifyQuery &Q,
3452 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3453 bool NUW, unsigned Depth) {
3454 // If X and Y are non-zero then so is X * Y as long as the multiplication
3455 // does not overflow.
3456 if (NSW || NUW)
3457 return isKnownNonZero(V: X, DemandedElts, Q, Depth) &&
3458 isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3459
3460 // If either X or Y is odd, then if the other is non-zero the result can't
3461 // be zero.
3462 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3463 if (XKnown.One[0])
3464 return isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3465
3466 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3467 if (YKnown.One[0])
3468 return XKnown.isNonZero() || isKnownNonZero(V: X, DemandedElts, Q, Depth);
3469
3470 // If there exists any subset of X (sX) and subset of Y (sY) s.t sX * sY is
3471 // non-zero, then X * Y is non-zero. We can find sX and sY by just taking
3472 // the lowest known One of X and Y. If they are non-zero, the result
3473 // must be non-zero. We can check if LSB(X) * LSB(Y) != 0 by doing
3474 // X.CountLeadingZeros + Y.CountLeadingZeros < BitWidth.
3475 return (XKnown.countMaxTrailingZeros() + YKnown.countMaxTrailingZeros()) <
3476 BitWidth;
3477}
3478
3479static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
3480 const SimplifyQuery &Q, const KnownBits &KnownVal,
3481 unsigned Depth) {
3482 auto ShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3483 switch (I->getOpcode()) {
3484 case Instruction::Shl:
3485 return Lhs.shl(ShiftAmt: Rhs);
3486 case Instruction::LShr:
3487 return Lhs.lshr(ShiftAmt: Rhs);
3488 case Instruction::AShr:
3489 return Lhs.ashr(ShiftAmt: Rhs);
3490 default:
3491 llvm_unreachable("Unknown Shift Opcode");
3492 }
3493 };
3494
3495 auto InvShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3496 switch (I->getOpcode()) {
3497 case Instruction::Shl:
3498 return Lhs.lshr(ShiftAmt: Rhs);
3499 case Instruction::LShr:
3500 case Instruction::AShr:
3501 return Lhs.shl(ShiftAmt: Rhs);
3502 default:
3503 llvm_unreachable("Unknown Shift Opcode");
3504 }
3505 };
3506
3507 if (KnownVal.isUnknown())
3508 return false;
3509
3510 KnownBits KnownCnt =
3511 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3512 APInt MaxShift = KnownCnt.getMaxValue();
3513 unsigned NumBits = KnownVal.getBitWidth();
3514 if (MaxShift.uge(RHS: NumBits))
3515 return false;
3516
3517 if (!ShiftOp(KnownVal.One, MaxShift).isZero())
3518 return true;
3519
3520 // If all of the bits shifted out are known to be zero, and Val is known
3521 // non-zero then at least one non-zero bit must remain.
3522 if (InvShiftOp(KnownVal.Zero, NumBits - MaxShift)
3523 .eq(RHS: InvShiftOp(APInt::getAllOnes(numBits: NumBits), NumBits - MaxShift)) &&
3524 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth))
3525 return true;
3526
3527 return false;
3528}
3529
3530static bool isKnownNonZeroFromOperator(const Operator *I,
3531 const APInt &DemandedElts,
3532 const SimplifyQuery &Q, unsigned Depth) {
3533 unsigned BitWidth = getBitWidth(Ty: I->getType()->getScalarType(), DL: Q.DL);
3534 switch (I->getOpcode()) {
3535 case Instruction::Alloca:
3536 // Alloca never returns null, malloc might.
3537 return I->getType()->getPointerAddressSpace() == 0;
3538 case Instruction::GetElementPtr:
3539 if (I->getType()->isPointerTy())
3540 return isGEPKnownNonNull(GEP: cast<GEPOperator>(Val: I), Q, Depth);
3541 break;
3542 case Instruction::BitCast: {
3543 // We need to be a bit careful here. We can only peek through the bitcast
3544 // if the scalar size of elements in the operand are smaller than and a
3545 // multiple of the size they are casting too. Take three cases:
3546 //
3547 // 1) Unsafe:
3548 // bitcast <2 x i16> %NonZero to <4 x i8>
3549 //
3550 // %NonZero can have 2 non-zero i16 elements, but isKnownNonZero on a
3551 // <4 x i8> requires that all 4 i8 elements be non-zero which isn't
3552 // guranteed (imagine just sign bit set in the 2 i16 elements).
3553 //
3554 // 2) Unsafe:
3555 // bitcast <4 x i3> %NonZero to <3 x i4>
3556 //
3557 // Even though the scalar size of the src (`i3`) is smaller than the
3558 // scalar size of the dst `i4`, because `i3` is not a multiple of `i4`
3559 // its possible for the `3 x i4` elements to be zero because there are
3560 // some elements in the destination that don't contain any full src
3561 // element.
3562 //
3563 // 3) Safe:
3564 // bitcast <4 x i8> %NonZero to <2 x i16>
3565 //
3566 // This is always safe as non-zero in the 4 i8 elements implies
3567 // non-zero in the combination of any two adjacent ones. Since i8 is a
3568 // multiple of i16, each i16 is guranteed to have 2 full i8 elements.
3569 // This all implies the 2 i16 elements are non-zero.
3570 Type *FromTy = I->getOperand(i: 0)->getType();
3571 if ((FromTy->isIntOrIntVectorTy() || FromTy->isPtrOrPtrVectorTy()) &&
3572 (BitWidth % getBitWidth(Ty: FromTy->getScalarType(), DL: Q.DL)) == 0)
3573 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3574 } break;
3575 case Instruction::IntToPtr:
3576 // Note that we have to take special care to avoid looking through
3577 // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
3578 // as casts that can alter the value, e.g., AddrSpaceCasts.
3579 if (!isa<ScalableVectorType>(Val: I->getType()) &&
3580 Q.DL.getTypeSizeInBits(Ty: I->getOperand(i: 0)->getType()).getFixedValue() <=
3581 Q.DL.getTypeSizeInBits(Ty: I->getType()).getFixedValue())
3582 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3583 break;
3584 case Instruction::PtrToAddr:
3585 // isKnownNonZero() for pointers refers to the address bits being non-zero,
3586 // so we can directly forward.
3587 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3588 case Instruction::PtrToInt:
3589 // For inttoptr, make sure the result size is >= the address size. If the
3590 // address is non-zero, any larger value is also non-zero.
3591 if (Q.DL.getAddressSizeInBits(Ty: I->getOperand(i: 0)->getType()) <=
3592 I->getType()->getScalarSizeInBits())
3593 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3594 break;
3595 case Instruction::Trunc:
3596 // nuw/nsw trunc preserves zero/non-zero status of input.
3597 if (auto *TI = dyn_cast<TruncInst>(Val: I))
3598 if (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap())
3599 return isKnownNonZero(V: TI->getOperand(i_nocapture: 0), DemandedElts, Q, Depth);
3600 break;
3601
3602 // Iff x - y != 0, then x ^ y != 0
3603 // Therefore we can do the same exact checks
3604 case Instruction::Xor:
3605 case Instruction::Sub:
3606 return isNonZeroSub(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3607 Y: I->getOperand(i: 1), Depth);
3608 case Instruction::Or:
3609 // (X | (X != 0)) is non zero
3610 if (matchOpWithOpEqZero(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1)))
3611 return true;
3612 // X | Y != 0 if X != Y.
3613 if (isKnownNonEqual(V1: I->getOperand(i: 0), V2: I->getOperand(i: 1), DemandedElts, Q,
3614 Depth))
3615 return true;
3616 // X | Y != 0 if X != 0 or Y != 0.
3617 return isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth) ||
3618 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3619 case Instruction::SExt:
3620 case Instruction::ZExt:
3621 // ext X != 0 if X != 0.
3622 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3623
3624 case Instruction::Shl: {
3625 // shl nsw/nuw can't remove any non-zero bits.
3626 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3627 if (Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO))
3628 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3629
3630 // shl X, Y != 0 if X is odd. Note that the value of the shift is undefined
3631 // if the lowest bit is shifted off the end.
3632 KnownBits Known(BitWidth);
3633 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth);
3634 if (Known.One[0])
3635 return true;
3636
3637 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3638 }
3639 case Instruction::LShr:
3640 case Instruction::AShr: {
3641 // shr exact can only shift out zero bits.
3642 const PossiblyExactOperator *BO = cast<PossiblyExactOperator>(Val: I);
3643 if (BO->isExact())
3644 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3645
3646 // shr X, Y != 0 if X is negative. Note that the value of the shift is not
3647 // defined if the sign bit is shifted off the end.
3648 KnownBits Known =
3649 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3650 if (Known.isNegative())
3651 return true;
3652
3653 // shr (add nuw A, B), C is non-zero if A or B has a known-one bit at
3654 // position >= C, because the sum >= max(A, B).
3655 Value *A, *B;
3656 const APInt *C;
3657 if (Depth + 1 < MaxAnalysisRecursionDepth &&
3658 match(V: I->getOperand(i: 0), P: m_NUWAdd(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3659 match(V: I->getOperand(i: 1), P: m_APInt(Res&: C)) && C->ult(RHS: BitWidth)) {
3660 KnownBits KnownA = computeKnownBits(V: A, DemandedElts, Q, Depth: Depth + 1);
3661 if (!KnownA.One.lshr(ShiftAmt: *C).isZero())
3662 return true;
3663 KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
3664 if (!KnownB.One.lshr(ShiftAmt: *C).isZero())
3665 return true;
3666 }
3667
3668 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3669 }
3670 case Instruction::UDiv:
3671 case Instruction::SDiv: {
3672 // X / Y
3673 // div exact can only produce a zero if the dividend is zero.
3674 if (cast<PossiblyExactOperator>(Val: I)->isExact())
3675 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3676
3677 KnownBits XKnown =
3678 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3679 // If X is fully unknown we won't be able to figure anything out so don't
3680 // both computing knownbits for Y.
3681 if (XKnown.isUnknown())
3682 return false;
3683
3684 KnownBits YKnown =
3685 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3686 if (I->getOpcode() == Instruction::SDiv) {
3687 // For signed division need to compare abs value of the operands.
3688 XKnown = XKnown.abs(/*IntMinIsPoison*/ false);
3689 YKnown = YKnown.abs(/*IntMinIsPoison*/ false);
3690 }
3691 // If X u>= Y then div is non zero (0/0 is UB).
3692 std::optional<bool> XUgeY = KnownBits::uge(LHS: XKnown, RHS: YKnown);
3693 // If X is total unknown or X u< Y we won't be able to prove non-zero
3694 // with compute known bits so just return early.
3695 return XUgeY && *XUgeY;
3696 }
3697 case Instruction::Add: {
3698 // X + Y.
3699
3700 // If Add has nuw wrap flag, then if either X or Y is non-zero the result is
3701 // non-zero.
3702 auto *BO = cast<OverflowingBinaryOperator>(Val: I);
3703 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3704 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3705 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3706 }
3707 case Instruction::Mul: {
3708 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3709 return isNonZeroMul(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3710 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3711 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3712 }
3713 case Instruction::Select: {
3714 // (C ? X : Y) != 0 if X != 0 and Y != 0.
3715
3716 // First check if the arm is non-zero using `isKnownNonZero`. If that fails,
3717 // then see if the select condition implies the arm is non-zero. For example
3718 // (X != 0 ? X : Y), we know the true arm is non-zero as the `X` "return" is
3719 // dominated by `X != 0`.
3720 auto SelectArmIsNonZero = [&](bool IsTrueArm) {
3721 Value *Op;
3722 Op = IsTrueArm ? I->getOperand(i: 1) : I->getOperand(i: 2);
3723 // Op is trivially non-zero.
3724 if (isKnownNonZero(V: Op, DemandedElts, Q, Depth))
3725 return true;
3726
3727 // The condition of the select dominates the true/false arm. Check if the
3728 // condition implies that a given arm is non-zero.
3729 Value *X;
3730 CmpPredicate Pred;
3731 if (!match(V: I->getOperand(i: 0), P: m_c_ICmp(Pred, L: m_Specific(V: Op), R: m_Value(V&: X))))
3732 return false;
3733
3734 if (!IsTrueArm)
3735 Pred = ICmpInst::getInversePredicate(pred: Pred);
3736
3737 return cmpExcludesZero(Pred, RHS: X);
3738 };
3739
3740 if (SelectArmIsNonZero(/* IsTrueArm */ true) &&
3741 SelectArmIsNonZero(/* IsTrueArm */ false))
3742 return true;
3743 break;
3744 }
3745 case Instruction::PHI: {
3746 auto *PN = cast<PHINode>(Val: I);
3747 if (Q.IIQ.UseInstrInfo && isNonZeroRecurrence(PN))
3748 return true;
3749
3750 // Check if all incoming values are non-zero using recursion.
3751 SimplifyQuery RecQ = Q.getWithoutCondContext();
3752 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
3753 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
3754 if (U.get() == PN)
3755 return true;
3756 RecQ.CtxI = PN->getIncomingBlock(U)->getTerminator();
3757 // Check if the branch on the phi excludes zero.
3758 CmpPredicate Pred;
3759 Value *X;
3760 BasicBlock *TrueSucc, *FalseSucc;
3761 if (match(V: RecQ.CtxI,
3762 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: U.get()), R: m_Value(V&: X)),
3763 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
3764 // Check for cases of duplicate successors.
3765 if ((TrueSucc == PN->getParent()) != (FalseSucc == PN->getParent())) {
3766 // If we're using the false successor, invert the predicate.
3767 if (FalseSucc == PN->getParent())
3768 Pred = CmpInst::getInversePredicate(pred: Pred);
3769 if (cmpExcludesZero(Pred, RHS: X))
3770 return true;
3771 }
3772 }
3773 // Finally recurse on the edge and check it directly.
3774 return isKnownNonZero(V: U.get(), DemandedElts, Q: RecQ, Depth: NewDepth);
3775 });
3776 }
3777 case Instruction::InsertElement: {
3778 if (isa<ScalableVectorType>(Val: I->getType()))
3779 break;
3780
3781 const Value *Vec = I->getOperand(i: 0);
3782 const Value *Elt = I->getOperand(i: 1);
3783 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
3784
3785 unsigned NumElts = DemandedElts.getBitWidth();
3786 APInt DemandedVecElts = DemandedElts;
3787 bool SkipElt = false;
3788 // If we know the index we are inserting too, clear it from Vec check.
3789 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
3790 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
3791 SkipElt = !DemandedElts[CIdx->getZExtValue()];
3792 }
3793
3794 // Result is zero if Elt is non-zero and rest of the demanded elts in Vec
3795 // are non-zero.
3796 return (SkipElt || isKnownNonZero(V: Elt, Q, Depth)) &&
3797 (DemandedVecElts.isZero() ||
3798 isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth));
3799 }
3800 case Instruction::ExtractElement:
3801 if (const auto *EEI = dyn_cast<ExtractElementInst>(Val: I)) {
3802 const Value *Vec = EEI->getVectorOperand();
3803 const Value *Idx = EEI->getIndexOperand();
3804 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
3805 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
3806 unsigned NumElts = VecTy->getNumElements();
3807 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
3808 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
3809 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
3810 return isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth);
3811 }
3812 }
3813 break;
3814 case Instruction::ShuffleVector: {
3815 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
3816 if (!Shuf)
3817 break;
3818 APInt DemandedLHS, DemandedRHS;
3819 // For undef elements, we don't know anything about the common state of
3820 // the shuffle result.
3821 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
3822 break;
3823 // If demanded elements for both vecs are non-zero, the shuffle is non-zero.
3824 return (DemandedRHS.isZero() ||
3825 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 1), DemandedElts: DemandedRHS, Q, Depth)) &&
3826 (DemandedLHS.isZero() ||
3827 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 0), DemandedElts: DemandedLHS, Q, Depth));
3828 }
3829 case Instruction::Freeze:
3830 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth) &&
3831 isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT,
3832 Depth);
3833 case Instruction::Load: {
3834 auto *LI = cast<LoadInst>(Val: I);
3835 // A Load tagged with nonnull or dereferenceable with null pointer undefined
3836 // is never null.
3837 if (auto *PtrT = dyn_cast<PointerType>(Val: I->getType())) {
3838 if (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_nonnull) ||
3839 (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_dereferenceable) &&
3840 !NullPointerIsDefined(F: LI->getFunction(), AS: PtrT->getAddressSpace())))
3841 return true;
3842 } else if (MDNode *Ranges = Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_range)) {
3843 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3844 }
3845
3846 // No need to fall through to computeKnownBits as range metadata is already
3847 // handled in isKnownNonZero.
3848 return false;
3849 }
3850 case Instruction::ExtractValue: {
3851 const WithOverflowInst *WO;
3852 if (match(V: I, P: m_ExtractValue<0>(V: m_WithOverflowInst(I&: WO)))) {
3853 switch (WO->getBinaryOp()) {
3854 default:
3855 break;
3856 case Instruction::Add:
3857 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3858 Y: WO->getArgOperand(i: 1),
3859 /*NSW=*/false,
3860 /*NUW=*/false, Depth);
3861 case Instruction::Sub:
3862 return isNonZeroSub(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3863 Y: WO->getArgOperand(i: 1), Depth);
3864 case Instruction::Mul:
3865 return isNonZeroMul(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3866 Y: WO->getArgOperand(i: 1),
3867 /*NSW=*/false, /*NUW=*/false, Depth);
3868 break;
3869 }
3870 }
3871 break;
3872 }
3873 case Instruction::Call:
3874 case Instruction::Invoke: {
3875 const auto *Call = cast<CallBase>(Val: I);
3876 if (I->getType()->isPointerTy()) {
3877 if (Call->isReturnNonNull())
3878 return true;
3879 if (const auto *RP = getArgumentAliasingToReturnedPointer(
3880 Call, /*MustPreserveOffset=*/true))
3881 return isKnownNonZero(V: RP, Q, Depth);
3882 } else {
3883 if (MDNode *Ranges = Q.IIQ.getMetadata(I: Call, KindID: LLVMContext::MD_range))
3884 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3885 if (std::optional<ConstantRange> Range = Call->getRange()) {
3886 const APInt ZeroValue(Range->getBitWidth(), 0);
3887 if (!Range->contains(Val: ZeroValue))
3888 return true;
3889 }
3890 if (const Value *RV = Call->getReturnedArgOperand())
3891 if (RV->getType() == I->getType() && isKnownNonZero(V: RV, Q, Depth))
3892 return true;
3893 }
3894
3895 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3896 switch (II->getIntrinsicID()) {
3897 case Intrinsic::sshl_sat:
3898 case Intrinsic::ushl_sat:
3899 case Intrinsic::abs:
3900 case Intrinsic::bitreverse:
3901 case Intrinsic::bswap:
3902 case Intrinsic::ctpop:
3903 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3904 // NB: We don't do usub_sat here as in any case we can prove its
3905 // non-zero, we will fold it to `sub nuw` in InstCombine.
3906 case Intrinsic::ssub_sat:
3907 // For most types, if x != y then ssub.sat x, y != 0. But
3908 // ssub.sat.i1 0, -1 = 0, because 1 saturates to 0. This means
3909 // isNonZeroSub will do the wrong thing for ssub.sat.i1.
3910 if (BitWidth == 1)
3911 return false;
3912 return isNonZeroSub(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3913 Y: II->getArgOperand(i: 1), Depth);
3914 case Intrinsic::sadd_sat:
3915 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3916 Y: II->getArgOperand(i: 1),
3917 /*NSW=*/true, /* NUW=*/false, Depth);
3918 // Vec reverse preserves zero/non-zero status from input vec.
3919 case Intrinsic::vector_reverse:
3920 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
3921 Q, Depth);
3922 // umin/smin/smax/smin/or of all non-zero elements is always non-zero.
3923 case Intrinsic::vector_reduce_or:
3924 case Intrinsic::vector_reduce_umax:
3925 case Intrinsic::vector_reduce_umin:
3926 case Intrinsic::vector_reduce_smax:
3927 case Intrinsic::vector_reduce_smin:
3928 return isKnownNonZero(V: II->getArgOperand(i: 0), Q, Depth);
3929 case Intrinsic::umax:
3930 case Intrinsic::uadd_sat:
3931 // umax(X, (X != 0)) is non zero
3932 // X +usat (X != 0) is non zero
3933 if (matchOpWithOpEqZero(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1)))
3934 return true;
3935
3936 return isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth) ||
3937 isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3938 case Intrinsic::smax: {
3939 // If either arg is strictly positive the result is non-zero. Otherwise
3940 // the result is non-zero if both ops are non-zero.
3941 auto IsNonZero = [&](Value *Op, std::optional<bool> &OpNonZero,
3942 const KnownBits &OpKnown) {
3943 if (!OpNonZero.has_value())
3944 OpNonZero = OpKnown.isNonZero() ||
3945 isKnownNonZero(V: Op, DemandedElts, Q, Depth);
3946 return *OpNonZero;
3947 };
3948 // Avoid re-computing isKnownNonZero.
3949 std::optional<bool> Op0NonZero, Op1NonZero;
3950 KnownBits Op1Known =
3951 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3952 if (Op1Known.isNonNegative() &&
3953 IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known))
3954 return true;
3955 KnownBits Op0Known =
3956 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3957 if (Op0Known.isNonNegative() &&
3958 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known))
3959 return true;
3960 return IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known) &&
3961 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known);
3962 }
3963 case Intrinsic::smin: {
3964 // If either arg is negative the result is non-zero. Otherwise
3965 // the result is non-zero if both ops are non-zero.
3966 KnownBits Op1Known =
3967 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3968 if (Op1Known.isNegative())
3969 return true;
3970 KnownBits Op0Known =
3971 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3972 if (Op0Known.isNegative())
3973 return true;
3974
3975 if (Op1Known.isNonZero() && Op0Known.isNonZero())
3976 return true;
3977 }
3978 [[fallthrough]];
3979 case Intrinsic::umin:
3980 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth) &&
3981 isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3982 case Intrinsic::cttz:
3983 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3984 .Zero[0];
3985 case Intrinsic::ctlz:
3986 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3987 .isNonNegative();
3988 case Intrinsic::fshr:
3989 case Intrinsic::fshl:
3990 // If Op0 == Op1, this is a rotate. rotate(x, y) != 0 iff x != 0.
3991 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
3992 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3993 break;
3994 case Intrinsic::vscale:
3995 return true;
3996 case Intrinsic::experimental_get_vector_length:
3997 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3998 default:
3999 break;
4000 }
4001 break;
4002 }
4003
4004 return false;
4005 }
4006 }
4007
4008 KnownBits Known(BitWidth);
4009 computeKnownBits(V: I, DemandedElts, Known, Q, Depth);
4010 return Known.One != 0;
4011}
4012
4013/// Return true if the given value is known to be non-zero when defined. For
4014/// vectors, return true if every demanded element is known to be non-zero when
4015/// defined. For pointers, if the context instruction and dominator tree are
4016/// specified, perform context-sensitive analysis and return true if the
4017/// pointer couldn't possibly be null at the specified instruction.
4018/// Supports values with integer or pointer type and vectors of integers.
4019bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
4020 const SimplifyQuery &Q, unsigned Depth) {
4021 Type *Ty = V->getType();
4022
4023#ifndef NDEBUG
4024 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
4025
4026 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
4027 assert(
4028 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
4029 "DemandedElt width should equal the fixed vector number of elements");
4030 } else {
4031 assert(DemandedElts == APInt(1, 1) &&
4032 "DemandedElt width should be 1 for scalars");
4033 }
4034#endif
4035
4036 if (auto *C = dyn_cast<Constant>(Val: V)) {
4037 if (C->isNullValue())
4038 return false;
4039 if (isa<ConstantInt>(Val: C))
4040 // Must be non-zero due to null test above.
4041 return true;
4042
4043 // For constant vectors, check that all elements are poison or known
4044 // non-zero to determine that the whole vector is known non-zero.
4045 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Ty)) {
4046 for (unsigned i = 0, e = VecTy->getNumElements(); i != e; ++i) {
4047 if (!DemandedElts[i])
4048 continue;
4049 Constant *Elt = C->getAggregateElement(Elt: i);
4050 if (!Elt || Elt->isNullValue())
4051 return false;
4052 if (!isa<PoisonValue>(Val: Elt) && !isa<ConstantInt>(Val: Elt))
4053 return false;
4054 }
4055 return true;
4056 }
4057
4058 // Constant ptrauth can be null, iff the base pointer can be.
4059 if (auto *CPA = dyn_cast<ConstantPtrAuth>(Val: V))
4060 return isKnownNonZero(V: CPA->getPointer(), DemandedElts, Q, Depth);
4061
4062 // A global variable is non null unless extern weak or an absolute symbol
4063 // reference. An alias is also excluded, as it may resolve to a weak
4064 // undefined symbol and thus to the null address. Where null is a valid
4065 // address an allocation may start there, so nothing can be assumed; that
4066 // covers both null_pointer_is_valid and address spaces other than zero.
4067 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
4068 if (!NullPointerIsDefined(F: Q.getFunction(),
4069 AS: GV->getType()->getAddressSpace()) &&
4070 !GV->isAbsoluteSymbolRef() && !GV->hasExternalWeakLinkage() &&
4071 !isa<GlobalAlias>(Val: GV))
4072 return true;
4073 }
4074
4075 // For constant expressions, fall through to the Operator code below.
4076 if (!isa<ConstantExpr>(Val: V))
4077 return false;
4078 }
4079
4080 if (const auto *A = dyn_cast<Argument>(Val: V))
4081 if (std::optional<ConstantRange> Range = A->getRange()) {
4082 const APInt ZeroValue(Range->getBitWidth(), 0);
4083 if (!Range->contains(Val: ZeroValue))
4084 return true;
4085 }
4086
4087 if (!isa<Constant>(Val: V) && isKnownNonZeroFromAssume(V, Q))
4088 return true;
4089
4090 // Some of the tests below are recursive, so bail out if we hit the limit.
4091 if (Depth++ >= MaxAnalysisRecursionDepth)
4092 return false;
4093
4094 // Check for pointer simplifications.
4095
4096 if (PointerType *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
4097 // A byval, inalloca may not be null in a non-default addres space. A
4098 // nonnull argument is assumed never 0.
4099 if (const Argument *A = dyn_cast<Argument>(Val: V)) {
4100 if (((A->hasPassPointeeByValueCopyAttr() &&
4101 !NullPointerIsDefined(F: A->getParent(), AS: PtrTy->getAddressSpace())) ||
4102 A->hasNonNullAttr()))
4103 return true;
4104 }
4105 }
4106
4107 if (const auto *I = dyn_cast<Operator>(Val: V))
4108 if (isKnownNonZeroFromOperator(I, DemandedElts, Q, Depth))
4109 return true;
4110
4111 if (!isa<Constant>(Val: V) &&
4112 isKnownNonNullFromDominatingCondition(V, CtxI: Q.CtxI, DT: Q.DT))
4113 return true;
4114
4115 if (const Value *Stripped = stripNullTest(V))
4116 return isKnownNonZero(V: Stripped, DemandedElts, Q, Depth);
4117
4118 return false;
4119}
4120
4121bool llvm::isKnownNonZero(const Value *V, const SimplifyQuery &Q,
4122 unsigned Depth) {
4123 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
4124 APInt DemandedElts =
4125 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
4126 return ::isKnownNonZero(V, DemandedElts, Q, Depth);
4127}
4128
4129/// If the pair of operators are the same invertible function, return the
4130/// the operands of the function corresponding to each input. Otherwise,
4131/// return std::nullopt. An invertible function is one that is 1-to-1 and maps
4132/// every input value to exactly one output value. This is equivalent to
4133/// saying that Op1 and Op2 are equal exactly when the specified pair of
4134/// operands are equal, (except that Op1 and Op2 may be poison more often.)
4135static std::optional<std::pair<Value*, Value*>>
4136getInvertibleOperands(const Operator *Op1,
4137 const Operator *Op2) {
4138 if (Op1->getOpcode() != Op2->getOpcode())
4139 return std::nullopt;
4140
4141 auto getOperands = [&](unsigned OpNum) -> auto {
4142 return std::make_pair(x: Op1->getOperand(i: OpNum), y: Op2->getOperand(i: OpNum));
4143 };
4144
4145 switch (Op1->getOpcode()) {
4146 default:
4147 break;
4148 case Instruction::Or:
4149 if (!cast<PossiblyDisjointInst>(Val: Op1)->isDisjoint() ||
4150 !cast<PossiblyDisjointInst>(Val: Op2)->isDisjoint())
4151 break;
4152 [[fallthrough]];
4153 case Instruction::Xor:
4154 case Instruction::Add: {
4155 Value *Other;
4156 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 0)), R: m_Value(V&: Other))))
4157 return std::make_pair(x: Op1->getOperand(i: 1), y&: Other);
4158 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 1)), R: m_Value(V&: Other))))
4159 return std::make_pair(x: Op1->getOperand(i: 0), y&: Other);
4160 break;
4161 }
4162 case Instruction::Sub:
4163 if (Op1->getOperand(i: 0) == Op2->getOperand(i: 0))
4164 return getOperands(1);
4165 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
4166 return getOperands(0);
4167 break;
4168 case Instruction::Mul: {
4169 // invertible if A * B == (A * B) mod 2^N where A, and B are integers
4170 // and N is the bitwdith. The nsw case is non-obvious, but proven by
4171 // alive2: https://alive2.llvm.org/ce/z/Z6D5qK
4172 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
4173 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
4174 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
4175 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
4176 break;
4177
4178 // Assume operand order has been canonicalized
4179 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1) &&
4180 isa<ConstantInt>(Val: Op1->getOperand(i: 1)) &&
4181 !cast<ConstantInt>(Val: Op1->getOperand(i: 1))->isZero())
4182 return getOperands(0);
4183 break;
4184 }
4185 case Instruction::Shl: {
4186 // Same as multiplies, with the difference that we don't need to check
4187 // for a non-zero multiply. Shifts always multiply by non-zero.
4188 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
4189 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
4190 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
4191 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
4192 break;
4193
4194 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
4195 return getOperands(0);
4196 break;
4197 }
4198 case Instruction::AShr:
4199 case Instruction::LShr: {
4200 auto *PEO1 = cast<PossiblyExactOperator>(Val: Op1);
4201 auto *PEO2 = cast<PossiblyExactOperator>(Val: Op2);
4202 if (!PEO1->isExact() || !PEO2->isExact())
4203 break;
4204
4205 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
4206 return getOperands(0);
4207 break;
4208 }
4209 case Instruction::SExt:
4210 case Instruction::ZExt:
4211 if (Op1->getOperand(i: 0)->getType() == Op2->getOperand(i: 0)->getType())
4212 return getOperands(0);
4213 break;
4214 case Instruction::PHI: {
4215 const PHINode *PN1 = cast<PHINode>(Val: Op1);
4216 const PHINode *PN2 = cast<PHINode>(Val: Op2);
4217
4218 // If PN1 and PN2 are both recurrences, can we prove the entire recurrences
4219 // are a single invertible function of the start values? Note that repeated
4220 // application of an invertible function is also invertible
4221 BinaryOperator *BO1 = nullptr;
4222 Value *Start1 = nullptr, *Step1 = nullptr;
4223 BinaryOperator *BO2 = nullptr;
4224 Value *Start2 = nullptr, *Step2 = nullptr;
4225 if (PN1->getParent() != PN2->getParent() ||
4226 !matchSimpleRecurrence(P: PN1, BO&: BO1, Start&: Start1, Step&: Step1) ||
4227 !matchSimpleRecurrence(P: PN2, BO&: BO2, Start&: Start2, Step&: Step2))
4228 break;
4229
4230 auto Values = getInvertibleOperands(Op1: cast<Operator>(Val: BO1),
4231 Op2: cast<Operator>(Val: BO2));
4232 if (!Values)
4233 break;
4234
4235 // We have to be careful of mutually defined recurrences here. Ex:
4236 // * X_i = X_(i-1) OP Y_(i-1), and Y_i = X_(i-1) OP V
4237 // * X_i = Y_i = X_(i-1) OP Y_(i-1)
4238 // The invertibility of these is complicated, and not worth reasoning
4239 // about (yet?).
4240 if (Values->first != PN1 || Values->second != PN2)
4241 break;
4242
4243 return std::make_pair(x&: Start1, y&: Start2);
4244 }
4245 }
4246 return std::nullopt;
4247}
4248
4249/// Return true if V1 == (binop V2, X), where X is known non-zero.
4250/// Only handle a small subset of binops where (binop V2, X) with non-zero X
4251/// implies V2 != V1.
4252static bool isModifyingBinopOfNonZero(const Value *V1, const Value *V2,
4253 const APInt &DemandedElts,
4254 const SimplifyQuery &Q, unsigned Depth) {
4255 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V1);
4256 if (!BO)
4257 return false;
4258 switch (BO->getOpcode()) {
4259 default:
4260 break;
4261 case Instruction::Or:
4262 if (!cast<PossiblyDisjointInst>(Val: V1)->isDisjoint())
4263 break;
4264 [[fallthrough]];
4265 case Instruction::Xor:
4266 case Instruction::Add:
4267 Value *Op = nullptr;
4268 if (V2 == BO->getOperand(i_nocapture: 0))
4269 Op = BO->getOperand(i_nocapture: 1);
4270 else if (V2 == BO->getOperand(i_nocapture: 1))
4271 Op = BO->getOperand(i_nocapture: 0);
4272 else
4273 return false;
4274 return isKnownNonZero(V: Op, DemandedElts, Q, Depth: Depth + 1);
4275 }
4276 return false;
4277}
4278
4279/// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
4280/// the multiplication is nuw or nsw.
4281static bool isNonEqualMul(const Value *V1, const Value *V2,
4282 const APInt &DemandedElts, const SimplifyQuery &Q,
4283 unsigned Depth) {
4284 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
4285 const APInt *C;
4286 return match(V: OBO, P: m_Mul(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
4287 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
4288 !C->isZero() && !C->isOne() &&
4289 isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
4290 }
4291 return false;
4292}
4293
4294/// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
4295/// the shift is nuw or nsw.
4296static bool isNonEqualShl(const Value *V1, const Value *V2,
4297 const APInt &DemandedElts, const SimplifyQuery &Q,
4298 unsigned Depth) {
4299 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
4300 const APInt *C;
4301 return match(V: OBO, P: m_Shl(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
4302 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
4303 !C->isZero() && isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
4304 }
4305 return false;
4306}
4307
4308static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
4309 const APInt &DemandedElts, const SimplifyQuery &Q,
4310 unsigned Depth) {
4311 // Check two PHIs are in same block.
4312 if (PN1->getParent() != PN2->getParent())
4313 return false;
4314
4315 SmallPtrSet<const BasicBlock *, 8> VisitedBBs;
4316 bool UsedFullRecursion = false;
4317 for (const BasicBlock *IncomBB : PN1->blocks()) {
4318 if (!VisitedBBs.insert(Ptr: IncomBB).second)
4319 continue; // Don't reprocess blocks that we have dealt with already.
4320 const Value *IV1 = PN1->getIncomingValueForBlock(BB: IncomBB);
4321 const Value *IV2 = PN2->getIncomingValueForBlock(BB: IncomBB);
4322 const APInt *C1, *C2;
4323 if (match(V: IV1, P: m_APInt(Res&: C1)) && match(V: IV2, P: m_APInt(Res&: C2)) && *C1 != *C2)
4324 continue;
4325
4326 // Only one pair of phi operands is allowed for full recursion.
4327 if (UsedFullRecursion)
4328 return false;
4329
4330 SimplifyQuery RecQ = Q.getWithoutCondContext();
4331 RecQ.CtxI = IncomBB->getTerminator();
4332 if (!isKnownNonEqual(V1: IV1, V2: IV2, DemandedElts, Q: RecQ, Depth: Depth + 1))
4333 return false;
4334 UsedFullRecursion = true;
4335 }
4336 return true;
4337}
4338
4339static bool isNonEqualSelect(const Value *V1, const Value *V2,
4340 const APInt &DemandedElts, const SimplifyQuery &Q,
4341 unsigned Depth) {
4342 const SelectInst *SI1 = dyn_cast<SelectInst>(Val: V1);
4343 if (!SI1)
4344 return false;
4345
4346 if (const SelectInst *SI2 = dyn_cast<SelectInst>(Val: V2)) {
4347 const Value *Cond1 = SI1->getCondition();
4348 const Value *Cond2 = SI2->getCondition();
4349 if (Cond1 == Cond2)
4350 return isKnownNonEqual(V1: SI1->getTrueValue(), V2: SI2->getTrueValue(),
4351 DemandedElts, Q, Depth: Depth + 1) &&
4352 isKnownNonEqual(V1: SI1->getFalseValue(), V2: SI2->getFalseValue(),
4353 DemandedElts, Q, Depth: Depth + 1);
4354 }
4355 return isKnownNonEqual(V1: SI1->getTrueValue(), V2, DemandedElts, Q, Depth: Depth + 1) &&
4356 isKnownNonEqual(V1: SI1->getFalseValue(), V2, DemandedElts, Q, Depth: Depth + 1);
4357}
4358
4359// Check to see if A is both a GEP and is the incoming value for a PHI in the
4360// loop, and B is either a ptr or another GEP. If the PHI has 2 incoming values,
4361// one of them being the recursive GEP A and the other a ptr at same base and at
4362// the same/higher offset than B we are only incrementing the pointer further in
4363// loop if offset of recursive GEP is greater than 0.
4364static bool isNonEqualPointersWithRecursiveGEP(const Value *A, const Value *B,
4365 const SimplifyQuery &Q) {
4366 if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
4367 return false;
4368
4369 auto *GEPA = dyn_cast<GEPOperator>(Val: A);
4370 if (!GEPA || GEPA->getNumIndices() != 1 || !isa<Constant>(Val: GEPA->idx_begin()))
4371 return false;
4372
4373 // Handle 2 incoming PHI values with one being a recursive GEP.
4374 auto *PN = dyn_cast<PHINode>(Val: GEPA->getPointerOperand());
4375 if (!PN || PN->getNumIncomingValues() != 2)
4376 return false;
4377
4378 // Search for the recursive GEP as an incoming operand, and record that as
4379 // Step.
4380 Value *Start = nullptr;
4381 Value *Step = const_cast<Value *>(A);
4382 if (PN->getIncomingValue(i: 0) == Step)
4383 Start = PN->getIncomingValue(i: 1);
4384 else if (PN->getIncomingValue(i: 1) == Step)
4385 Start = PN->getIncomingValue(i: 0);
4386 else
4387 return false;
4388
4389 // Other incoming node base should match the B base.
4390 // StartOffset >= OffsetB && StepOffset > 0?
4391 // StartOffset <= OffsetB && StepOffset < 0?
4392 // Is non-equal if above are true.
4393 // We use stripAndAccumulateInBoundsConstantOffsets to restrict the
4394 // optimisation to inbounds GEPs only.
4395 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: Start->getType());
4396 APInt StartOffset(IndexWidth, 0);
4397 Start = Start->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StartOffset);
4398 APInt StepOffset(IndexWidth, 0);
4399 Step = Step->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StepOffset);
4400
4401 // Check if Base Pointer of Step matches the PHI.
4402 if (Step != PN)
4403 return false;
4404 APInt OffsetB(IndexWidth, 0);
4405 B = B->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: OffsetB);
4406 return Start == B &&
4407 ((StartOffset.sge(RHS: OffsetB) && StepOffset.isStrictlyPositive()) ||
4408 (StartOffset.sle(RHS: OffsetB) && StepOffset.isNegative()));
4409}
4410
4411static bool isKnownNonEqualFromContext(const Value *V1, const Value *V2,
4412 const SimplifyQuery &Q, unsigned Depth) {
4413 if (!Q.CtxI)
4414 return false;
4415
4416 // Try to infer NonEqual based on information from dominating conditions.
4417 if (Q.DC && Q.DT) {
4418 auto IsKnownNonEqualFromDominatingCondition = [&](const Value *V) {
4419 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4420 Value *Cond = BI->getCondition();
4421 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4422 if (Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()) &&
4423 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4424 /*LHSIsTrue=*/true, Depth)
4425 .value_or(u: false))
4426 return true;
4427
4428 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4429 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()) &&
4430 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4431 /*LHSIsTrue=*/false, Depth)
4432 .value_or(u: false))
4433 return true;
4434 }
4435
4436 return false;
4437 };
4438
4439 if (IsKnownNonEqualFromDominatingCondition(V1) ||
4440 IsKnownNonEqualFromDominatingCondition(V2))
4441 return true;
4442 }
4443
4444 if (!Q.AC)
4445 return false;
4446
4447 // Try to infer NonEqual based on information from assumptions.
4448 for (auto &AssumeVH : Q.AC->assumptionsFor(V: V1)) {
4449 if (!AssumeVH)
4450 continue;
4451 CallInst *I = cast<CallInst>(Val&: AssumeVH);
4452
4453 assert(I->getFunction() == Q.CtxI->getFunction() &&
4454 "Got assumption for the wrong function!");
4455 assert(I->getIntrinsicID() == Intrinsic::assume &&
4456 "must be an assume intrinsic");
4457
4458 if (isImpliedCondition(LHS: I->getArgOperand(i: 0), RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4459 /*LHSIsTrue=*/true, Depth)
4460 .value_or(u: false) &&
4461 isValidAssumeForContext(I, Q))
4462 return true;
4463 }
4464
4465 return false;
4466}
4467
4468static bool isNonEqualURem(const Value *X, const Value *Rem,
4469 const SimplifyQuery &Q) {
4470 const Value *Y;
4471 if (!match(V: Rem, P: m_URem(L: m_Specific(V: X), R: m_Value(V&: Y))))
4472 return false;
4473
4474 // For a defined urem, X != X urem Y exactly when X u>= Y.
4475 // isTruePredicate does not handle UGE, so use the equivalent Y u<= X.
4476 if (isTruePredicate(Pred: ICmpInst::ICMP_ULE, LHS: Y, RHS: X))
4477 return true;
4478
4479 std::optional<bool> Implied =
4480 isImpliedByDomCondition(Pred: ICmpInst::ICMP_UGE, LHS: X, RHS: Y, ContextI: Q.CtxI, DL: Q.DL);
4481 return Implied && *Implied;
4482}
4483
4484/// Return true if it is known that V1 != V2.
4485static bool isKnownNonEqual(const Value *V1, const Value *V2,
4486 const APInt &DemandedElts, const SimplifyQuery &Q,
4487 unsigned Depth) {
4488 if (V1 == V2)
4489 return false;
4490 if (V1->getType() != V2->getType())
4491 // We can't look through casts yet.
4492 return false;
4493
4494 if (Depth >= MaxAnalysisRecursionDepth)
4495 return false;
4496
4497 // See if we can recurse through (exactly one of) our operands. This
4498 // requires our operation be 1-to-1 and map every input value to exactly
4499 // one output value. Such an operation is invertible.
4500 auto *O1 = dyn_cast<Operator>(Val: V1);
4501 auto *O2 = dyn_cast<Operator>(Val: V2);
4502 if (O1 && O2 && O1->getOpcode() == O2->getOpcode()) {
4503 if (auto Values = getInvertibleOperands(Op1: O1, Op2: O2))
4504 return isKnownNonEqual(V1: Values->first, V2: Values->second, DemandedElts, Q,
4505 Depth: Depth + 1);
4506
4507 if (const PHINode *PN1 = dyn_cast<PHINode>(Val: V1)) {
4508 const PHINode *PN2 = cast<PHINode>(Val: V2);
4509 // FIXME: This is missing a generalization to handle the case where one is
4510 // a PHI and another one isn't.
4511 if (isNonEqualPHIs(PN1, PN2, DemandedElts, Q, Depth))
4512 return true;
4513 };
4514 }
4515
4516 if (isModifyingBinopOfNonZero(V1, V2, DemandedElts, Q, Depth) ||
4517 isModifyingBinopOfNonZero(V1: V2, V2: V1, DemandedElts, Q, Depth))
4518 return true;
4519
4520 if (isNonEqualMul(V1, V2, DemandedElts, Q, Depth) ||
4521 isNonEqualMul(V1: V2, V2: V1, DemandedElts, Q, Depth))
4522 return true;
4523
4524 if (isNonEqualShl(V1, V2, DemandedElts, Q, Depth) ||
4525 isNonEqualShl(V1: V2, V2: V1, DemandedElts, Q, Depth))
4526 return true;
4527
4528 if (V1->getType()->isIntOrIntVectorTy()) {
4529 // Are any known bits in V1 contradictory to known bits in V2? If V1
4530 // has a known zero where V2 has a known one, they must not be equal.
4531 KnownBits Known1 = computeKnownBits(V: V1, DemandedElts, Q, Depth);
4532 if (!Known1.isUnknown()) {
4533 KnownBits Known2 = computeKnownBits(V: V2, DemandedElts, Q, Depth);
4534 if (Known1.Zero.intersects(RHS: Known2.One) ||
4535 Known2.Zero.intersects(RHS: Known1.One))
4536 return true;
4537 }
4538 }
4539
4540 if (isNonEqualSelect(V1, V2, DemandedElts, Q, Depth) ||
4541 isNonEqualSelect(V1: V2, V2: V1, DemandedElts, Q, Depth))
4542 return true;
4543
4544 if (isNonEqualPointersWithRecursiveGEP(A: V1, B: V2, Q) ||
4545 isNonEqualPointersWithRecursiveGEP(A: V2, B: V1, Q))
4546 return true;
4547
4548 Value *A, *B;
4549 // PtrToInts are NonEqual if their Ptrs are NonEqual.
4550 // Check PtrToInt type matches the pointer size.
4551 if (match(V: V1, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: A))) &&
4552 match(V: V2, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: B))))
4553 return isKnownNonEqual(V1: A, V2: B, DemandedElts, Q, Depth: Depth + 1);
4554
4555 if (isNonEqualURem(X: V1, Rem: V2, Q) || isNonEqualURem(X: V2, Rem: V1, Q))
4556 return true;
4557
4558 if (isKnownNonEqualFromContext(V1, V2, Q, Depth))
4559 return true;
4560
4561 return false;
4562}
4563
4564/// For vector constants, loop over the elements and find the constant with the
4565/// minimum number of sign bits. Return 0 if the value is not a vector constant
4566/// or if any element was not analyzed; otherwise, return the count for the
4567/// element with the minimum number of sign bits.
4568static unsigned computeNumSignBitsVectorConstant(const Value *V,
4569 const APInt &DemandedElts,
4570 unsigned TyBits) {
4571 const auto *CV = dyn_cast<Constant>(Val: V);
4572 if (!CV || !isa<FixedVectorType>(Val: CV->getType()))
4573 return 0;
4574
4575 unsigned MinSignBits = TyBits;
4576 unsigned NumElts = cast<FixedVectorType>(Val: CV->getType())->getNumElements();
4577 for (unsigned i = 0; i != NumElts; ++i) {
4578 if (!DemandedElts[i])
4579 continue;
4580 // If we find a non-ConstantInt, bail out.
4581 auto *Elt = dyn_cast_or_null<ConstantInt>(Val: CV->getAggregateElement(Elt: i));
4582 if (!Elt)
4583 return 0;
4584
4585 MinSignBits = std::min(a: MinSignBits, b: Elt->getValue().getNumSignBits());
4586 }
4587
4588 return MinSignBits;
4589}
4590
4591static unsigned ComputeNumSignBitsImpl(const Value *V,
4592 const APInt &DemandedElts,
4593 const SimplifyQuery &Q, unsigned Depth);
4594
4595static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
4596 const SimplifyQuery &Q, unsigned Depth) {
4597 unsigned Result = ComputeNumSignBitsImpl(V, DemandedElts, Q, Depth);
4598 assert(Result > 0 && "At least one sign bit needs to be present!");
4599 return Result;
4600}
4601
4602/// Return the number of times the sign bit of the register is replicated into
4603/// the other bits. We know that at least 1 bit is always equal to the sign bit
4604/// (itself), but other cases can give us information. For example, immediately
4605/// after an "ashr X, 2", we know that the top 3 bits are all equal to each
4606/// other, so we return 3. For vectors, return the number of sign bits for the
4607/// vector element with the minimum number of known sign bits of the demanded
4608/// elements in the vector specified by DemandedElts.
4609static unsigned ComputeNumSignBitsImpl(const Value *V,
4610 const APInt &DemandedElts,
4611 const SimplifyQuery &Q, unsigned Depth) {
4612 Type *Ty = V->getType();
4613#ifndef NDEBUG
4614 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
4615
4616 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
4617 assert(
4618 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
4619 "DemandedElt width should equal the fixed vector number of elements");
4620 } else {
4621 assert(DemandedElts == APInt(1, 1) &&
4622 "DemandedElt width should be 1 for scalars");
4623 }
4624#endif
4625
4626 // We return the minimum number of sign bits that are guaranteed to be present
4627 // in V, so for undef we have to conservatively return 1. We don't have the
4628 // same behavior for poison though -- that's a FIXME today.
4629
4630 Type *ScalarTy = Ty->getScalarType();
4631 unsigned TyBits = ScalarTy->isPointerTy() ?
4632 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
4633 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
4634
4635 unsigned Tmp, Tmp2;
4636 unsigned FirstAnswer = 1;
4637
4638 // Note that ConstantInt is handled by the general computeKnownBits case
4639 // below.
4640
4641 if (Depth == MaxAnalysisRecursionDepth)
4642 return 1;
4643
4644 if (auto *U = dyn_cast<Operator>(Val: V)) {
4645 switch (Operator::getOpcode(V)) {
4646 default: break;
4647 case Instruction::BitCast: {
4648 Value *Src = U->getOperand(i: 0);
4649 Type *SrcTy = Src->getType();
4650
4651 // Skip if the source type is not an integer or integer vector type
4652 // This ensures we only process integer-like types
4653 if (!SrcTy->isIntOrIntVectorTy())
4654 break;
4655
4656 unsigned SrcBits = SrcTy->getScalarSizeInBits();
4657
4658 // Bitcast 'large element' scalar/vector to 'small element' vector.
4659 if ((SrcBits % TyBits) != 0)
4660 break;
4661
4662 // Only proceed if the destination type is a fixed-size vector
4663 if (isa<FixedVectorType>(Val: Ty)) {
4664 // Fast case - sign splat can be simply split across the small elements.
4665 // This works for both vector and scalar sources
4666 Tmp = ComputeNumSignBits(V: Src, Q, Depth: Depth + 1);
4667 if (Tmp == SrcBits)
4668 return TyBits;
4669 }
4670 break;
4671 }
4672 case Instruction::SExt:
4673 Tmp = TyBits - U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4674 return ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1) +
4675 Tmp;
4676
4677 case Instruction::SDiv: {
4678 const APInt *Denominator;
4679 // sdiv X, C -> adds log(C) sign bits.
4680 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4681
4682 // Ignore non-positive denominator.
4683 if (!Denominator->isStrictlyPositive())
4684 break;
4685
4686 // Calculate the incoming numerator bits.
4687 unsigned NumBits =
4688 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4689
4690 // Add floor(log(C)) bits to the numerator bits.
4691 return std::min(a: TyBits, b: NumBits + Denominator->logBase2());
4692 }
4693 break;
4694 }
4695
4696 case Instruction::SRem: {
4697 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4698
4699 const APInt *Denominator;
4700 // srem X, C -> we know that the result is within [-C+1,C) when C is a
4701 // positive constant. This let us put a lower bound on the number of sign
4702 // bits.
4703 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4704
4705 // Ignore non-positive denominator.
4706 if (Denominator->isStrictlyPositive()) {
4707 // Calculate the leading sign bit constraints by examining the
4708 // denominator. Given that the denominator is positive, there are two
4709 // cases:
4710 //
4711 // 1. The numerator is positive. The result range is [0,C) and
4712 // [0,C) u< (1 << ceilLogBase2(C)).
4713 //
4714 // 2. The numerator is negative. Then the result range is (-C,0] and
4715 // integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
4716 //
4717 // Thus a lower bound on the number of sign bits is `TyBits -
4718 // ceilLogBase2(C)`.
4719
4720 unsigned ResBits = TyBits - Denominator->ceilLogBase2();
4721 Tmp = std::max(a: Tmp, b: ResBits);
4722 }
4723 }
4724 return Tmp;
4725 }
4726
4727 case Instruction::AShr: {
4728 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4729 // ashr X, C -> adds C sign bits. Vectors too.
4730 const APInt *ShAmt;
4731 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4732 if (ShAmt->uge(RHS: TyBits))
4733 break; // Bad shift.
4734 unsigned ShAmtLimited = ShAmt->getZExtValue();
4735 Tmp += ShAmtLimited;
4736 if (Tmp > TyBits) Tmp = TyBits;
4737 }
4738 return Tmp;
4739 }
4740 case Instruction::Shl: {
4741 const APInt *ShAmt;
4742 Value *X = nullptr;
4743 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4744 // shl destroys sign bits.
4745 if (ShAmt->uge(RHS: TyBits))
4746 break; // Bad shift.
4747 // We can look through a zext (more or less treating it as a sext) if
4748 // all extended bits are shifted out.
4749 if (match(V: U->getOperand(i: 0), P: m_ZExt(Op: m_Value(V&: X))) &&
4750 ShAmt->uge(RHS: TyBits - X->getType()->getScalarSizeInBits())) {
4751 Tmp = ComputeNumSignBits(V: X, DemandedElts, Q, Depth: Depth + 1);
4752 Tmp += TyBits - X->getType()->getScalarSizeInBits();
4753 } else
4754 Tmp =
4755 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4756 if (ShAmt->uge(RHS: Tmp))
4757 break; // Shifted all sign bits out.
4758 Tmp2 = ShAmt->getZExtValue();
4759 return Tmp - Tmp2;
4760 }
4761 break;
4762 }
4763 case Instruction::And:
4764 case Instruction::Or:
4765 case Instruction::Xor: // NOT is handled here.
4766 // Logical binary ops preserve the number of sign bits at the worst.
4767 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4768 if (Tmp != 1) {
4769 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4770 FirstAnswer = std::min(a: Tmp, b: Tmp2);
4771 // We computed what we know about the sign bits as our first
4772 // answer. Now proceed to the generic code that uses
4773 // computeKnownBits, and pick whichever answer is better.
4774 }
4775 break;
4776
4777 case Instruction::Select: {
4778 // If we have a clamp pattern, we know that the number of sign bits will
4779 // be the minimum of the clamp min/max range.
4780 const Value *X;
4781 const APInt *CLow, *CHigh;
4782 if (isSignedMinMaxClamp(Select: U, In&: X, CLow, CHigh))
4783 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4784
4785 Tmp = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4786 if (Tmp == 1)
4787 break;
4788 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 2), DemandedElts, Q, Depth: Depth + 1);
4789 return std::min(a: Tmp, b: Tmp2);
4790 }
4791
4792 case Instruction::Add:
4793 // Add can have at most one carry bit. Thus we know that the output
4794 // is, at worst, one more bit than the inputs.
4795 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4796 if (Tmp == 1) break;
4797
4798 // Special case decrementing a value (ADD X, -1):
4799 if (const auto *CRHS = dyn_cast<Constant>(Val: U->getOperand(i: 1)))
4800 if (CRHS->isAllOnesValue()) {
4801 KnownBits Known(TyBits);
4802 computeKnownBits(V: U->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
4803
4804 // If the input is known to be 0 or 1, the output is 0/-1, which is
4805 // all sign bits set.
4806 if ((Known.Zero | 1).isAllOnes())
4807 return TyBits;
4808
4809 // If we are subtracting one from a positive number, there is no carry
4810 // out of the result.
4811 if (Known.isNonNegative())
4812 return Tmp;
4813 }
4814
4815 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4816 if (Tmp2 == 1)
4817 break;
4818 return std::min(a: Tmp, b: Tmp2) - 1;
4819
4820 case Instruction::Sub:
4821 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4822 if (Tmp2 == 1)
4823 break;
4824
4825 // Handle NEG.
4826 if (const auto *CLHS = dyn_cast<Constant>(Val: U->getOperand(i: 0)))
4827 if (CLHS->isNullValue()) {
4828 KnownBits Known(TyBits);
4829 computeKnownBits(V: U->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
4830 // If the input is known to be 0 or 1, the output is 0/-1, which is
4831 // all sign bits set.
4832 if ((Known.Zero | 1).isAllOnes())
4833 return TyBits;
4834
4835 // If the input is known to be positive (the sign bit is known clear),
4836 // the output of the NEG has the same number of sign bits as the
4837 // input.
4838 if (Known.isNonNegative())
4839 return Tmp2;
4840
4841 // Otherwise, we treat this like a SUB.
4842 }
4843
4844 // Sub can have at most one carry bit. Thus we know that the output
4845 // is, at worst, one more bit than the inputs.
4846 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4847 if (Tmp == 1)
4848 break;
4849 return std::min(a: Tmp, b: Tmp2) - 1;
4850
4851 case Instruction::Mul: {
4852 // The output of the Mul can be at most twice the valid bits in the
4853 // inputs.
4854 unsigned SignBitsOp0 =
4855 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4856 if (SignBitsOp0 == 1)
4857 break;
4858 unsigned SignBitsOp1 =
4859 ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4860 if (SignBitsOp1 == 1)
4861 break;
4862 unsigned OutValidBits =
4863 (TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
4864 return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
4865 }
4866
4867 case Instruction::PHI: {
4868 const PHINode *PN = cast<PHINode>(Val: U);
4869 unsigned NumIncomingValues = PN->getNumIncomingValues();
4870 // Don't analyze large in-degree PHIs.
4871 if (NumIncomingValues > 4) break;
4872 // Unreachable blocks may have zero-operand PHI nodes.
4873 if (NumIncomingValues == 0) break;
4874
4875 // Take the minimum of all incoming values. This can't infinitely loop
4876 // because of our depth threshold.
4877 SimplifyQuery RecQ = Q.getWithoutCondContext();
4878 Tmp = TyBits;
4879 for (unsigned i = 0, e = NumIncomingValues; i != e; ++i) {
4880 if (Tmp == 1) return Tmp;
4881 RecQ.CtxI = PN->getIncomingBlock(i)->getTerminator();
4882 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(V: PN->getIncomingValue(i),
4883 DemandedElts, Q: RecQ, Depth: Depth + 1));
4884 }
4885 return Tmp;
4886 }
4887
4888 case Instruction::Trunc: {
4889 // If the input contained enough sign bits that some remain after the
4890 // truncation, then we can make use of that. Otherwise we don't know
4891 // anything.
4892 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4893 unsigned OperandTyBits = U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4894 if (Tmp > (OperandTyBits - TyBits))
4895 return Tmp - (OperandTyBits - TyBits);
4896
4897 return 1;
4898 }
4899
4900 case Instruction::ExtractElement:
4901 // Look through extract element. At the moment we keep this simple and
4902 // skip tracking the specific element. But at least we might find
4903 // information valid for all elements of the vector (for example if vector
4904 // is sign extended, shifted, etc).
4905 return ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4906
4907 case Instruction::ShuffleVector: {
4908 // Collect the minimum number of sign bits that are shared by every vector
4909 // element referenced by the shuffle.
4910 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: U);
4911 if (!Shuf) {
4912 // FIXME: Add support for shufflevector constant expressions.
4913 return 1;
4914 }
4915 APInt DemandedLHS, DemandedRHS;
4916 // For undef elements, we don't know anything about the common state of
4917 // the shuffle result.
4918 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
4919 return 1;
4920 Tmp = std::numeric_limits<unsigned>::max();
4921 if (!!DemandedLHS) {
4922 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
4923 Tmp = ComputeNumSignBits(V: LHS, DemandedElts: DemandedLHS, Q, Depth: Depth + 1);
4924 }
4925 // If we don't know anything, early out and try computeKnownBits
4926 // fall-back.
4927 if (Tmp == 1)
4928 break;
4929 if (!!DemandedRHS) {
4930 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
4931 Tmp2 = ComputeNumSignBits(V: RHS, DemandedElts: DemandedRHS, Q, Depth: Depth + 1);
4932 Tmp = std::min(a: Tmp, b: Tmp2);
4933 }
4934 // If we don't know anything, early out and try computeKnownBits
4935 // fall-back.
4936 if (Tmp == 1)
4937 break;
4938 assert(Tmp <= TyBits && "Failed to determine minimum sign bits");
4939 return Tmp;
4940 }
4941 case Instruction::Call: {
4942 if (const auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
4943 switch (II->getIntrinsicID()) {
4944 default:
4945 break;
4946 case Intrinsic::abs:
4947 Tmp =
4948 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4949 if (Tmp == 1)
4950 break;
4951
4952 // Absolute value reduces number of sign bits by at most 1.
4953 return Tmp - 1;
4954 case Intrinsic::smin:
4955 case Intrinsic::smax: {
4956 const APInt *CLow, *CHigh;
4957 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
4958 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4959 }
4960 }
4961 }
4962 }
4963 }
4964 }
4965
4966 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4967 // use this information.
4968
4969 // If we can examine all elements of a vector constant successfully, we're
4970 // done (we can't do any better than that). If not, keep trying.
4971 if (unsigned VecSignBits =
4972 computeNumSignBitsVectorConstant(V, DemandedElts, TyBits))
4973 return VecSignBits;
4974
4975 KnownBits Known(TyBits);
4976 computeKnownBits(V, DemandedElts, Known, Q, Depth);
4977
4978 // If we know that the sign bit is either zero or one, determine the number of
4979 // identical bits in the top of the input value.
4980 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
4981}
4982
4983Intrinsic::ID llvm::getIntrinsicForCallSite(const CallBase &CB,
4984 const TargetLibraryInfo *TLI) {
4985 const Function *F = CB.getCalledFunction();
4986 if (!F)
4987 return Intrinsic::not_intrinsic;
4988
4989 if (F->isIntrinsic())
4990 return F->getIntrinsicID();
4991
4992 // We are going to infer semantics of a library function based on mapping it
4993 // to an LLVM intrinsic. Check that the library function is available from
4994 // this callbase and in this environment.
4995 if (F->hasLocalLinkage() || !TLI || !CB.onlyReadsMemory())
4996 return Intrinsic::not_intrinsic;
4997
4998 LibFunc Func = TLI->getLibFunc(CB);
4999 if (Func == NotLibFunc)
5000 return Intrinsic::not_intrinsic;
5001
5002 switch (Func) {
5003 default:
5004 break;
5005 case LibFunc_sin:
5006 case LibFunc_sinf:
5007 case LibFunc_sinl:
5008 return Intrinsic::sin;
5009 case LibFunc_cos:
5010 case LibFunc_cosf:
5011 case LibFunc_cosl:
5012 return Intrinsic::cos;
5013 case LibFunc_tan:
5014 case LibFunc_tanf:
5015 case LibFunc_tanl:
5016 return Intrinsic::tan;
5017 case LibFunc_asin:
5018 case LibFunc_asinf:
5019 case LibFunc_asinl:
5020 return Intrinsic::asin;
5021 case LibFunc_acos:
5022 case LibFunc_acosf:
5023 case LibFunc_acosl:
5024 return Intrinsic::acos;
5025 case LibFunc_atan:
5026 case LibFunc_atanf:
5027 case LibFunc_atanl:
5028 return Intrinsic::atan;
5029 case LibFunc_atan2:
5030 case LibFunc_atan2f:
5031 case LibFunc_atan2l:
5032 return Intrinsic::atan2;
5033 case LibFunc_sinh:
5034 case LibFunc_sinhf:
5035 case LibFunc_sinhl:
5036 return Intrinsic::sinh;
5037 case LibFunc_cosh:
5038 case LibFunc_coshf:
5039 case LibFunc_coshl:
5040 return Intrinsic::cosh;
5041 case LibFunc_tanh:
5042 case LibFunc_tanhf:
5043 case LibFunc_tanhl:
5044 return Intrinsic::tanh;
5045 case LibFunc_exp:
5046 case LibFunc_expf:
5047 case LibFunc_expl:
5048 return Intrinsic::exp;
5049 case LibFunc_exp2:
5050 case LibFunc_exp2f:
5051 case LibFunc_exp2l:
5052 return Intrinsic::exp2;
5053 case LibFunc_exp10:
5054 case LibFunc_exp10f:
5055 case LibFunc_exp10l:
5056 return Intrinsic::exp10;
5057 case LibFunc_log:
5058 case LibFunc_logf:
5059 case LibFunc_logl:
5060 return Intrinsic::log;
5061 case LibFunc_log10:
5062 case LibFunc_log10f:
5063 case LibFunc_log10l:
5064 return Intrinsic::log10;
5065 case LibFunc_log2:
5066 case LibFunc_log2f:
5067 case LibFunc_log2l:
5068 return Intrinsic::log2;
5069 case LibFunc_fabs:
5070 case LibFunc_fabsf:
5071 case LibFunc_fabsl:
5072 return Intrinsic::fabs;
5073 case LibFunc_fmin:
5074 case LibFunc_fminf:
5075 case LibFunc_fminl:
5076 return Intrinsic::minnum;
5077 case LibFunc_fmax:
5078 case LibFunc_fmaxf:
5079 case LibFunc_fmaxl:
5080 return Intrinsic::maxnum;
5081 case LibFunc_copysign:
5082 case LibFunc_copysignf:
5083 case LibFunc_copysignl:
5084 return Intrinsic::copysign;
5085 case LibFunc_floor:
5086 case LibFunc_floorf:
5087 case LibFunc_floorl:
5088 return Intrinsic::floor;
5089 case LibFunc_ceil:
5090 case LibFunc_ceilf:
5091 case LibFunc_ceill:
5092 return Intrinsic::ceil;
5093 case LibFunc_trunc:
5094 case LibFunc_truncf:
5095 case LibFunc_truncl:
5096 return Intrinsic::trunc;
5097 case LibFunc_rint:
5098 case LibFunc_rintf:
5099 case LibFunc_rintl:
5100 return Intrinsic::rint;
5101 case LibFunc_nearbyint:
5102 case LibFunc_nearbyintf:
5103 case LibFunc_nearbyintl:
5104 return Intrinsic::nearbyint;
5105 case LibFunc_round:
5106 case LibFunc_roundf:
5107 case LibFunc_roundl:
5108 return Intrinsic::round;
5109 case LibFunc_roundeven:
5110 case LibFunc_roundevenf:
5111 case LibFunc_roundevenl:
5112 return Intrinsic::roundeven;
5113 case LibFunc_pow:
5114 case LibFunc_powf:
5115 case LibFunc_powl:
5116 return Intrinsic::pow;
5117 case LibFunc_sqrt:
5118 case LibFunc_sqrtf:
5119 case LibFunc_sqrtl:
5120 return Intrinsic::sqrt;
5121 }
5122
5123 return Intrinsic::not_intrinsic;
5124}
5125
5126/// Given an exploded icmp instruction, return true if the comparison only
5127/// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if
5128/// the result of the comparison is true when the input value is signed.
5129bool llvm::isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
5130 bool &TrueIfSigned) {
5131 switch (Pred) {
5132 case ICmpInst::ICMP_SLT: // True if LHS s< 0
5133 TrueIfSigned = true;
5134 return RHS.isZero();
5135 case ICmpInst::ICMP_SLE: // True if LHS s<= -1
5136 TrueIfSigned = true;
5137 return RHS.isAllOnes();
5138 case ICmpInst::ICMP_SGT: // True if LHS s> -1
5139 TrueIfSigned = false;
5140 return RHS.isAllOnes();
5141 case ICmpInst::ICMP_SGE: // True if LHS s>= 0
5142 TrueIfSigned = false;
5143 return RHS.isZero();
5144 case ICmpInst::ICMP_UGT:
5145 // True if LHS u> RHS and RHS == sign-bit-mask - 1
5146 TrueIfSigned = true;
5147 return RHS.isMaxSignedValue();
5148 case ICmpInst::ICMP_UGE:
5149 // True if LHS u>= RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
5150 TrueIfSigned = true;
5151 return RHS.isMinSignedValue();
5152 case ICmpInst::ICMP_ULT:
5153 // True if LHS u< RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
5154 TrueIfSigned = false;
5155 return RHS.isMinSignedValue();
5156 case ICmpInst::ICMP_ULE:
5157 // True if LHS u<= RHS and RHS == sign-bit-mask - 1
5158 TrueIfSigned = false;
5159 return RHS.isMaxSignedValue();
5160 default:
5161 return false;
5162 }
5163}
5164
5165static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
5166 bool CondIsTrue,
5167 const Instruction *CtxI,
5168 KnownFPClass &KnownFromContext,
5169 unsigned Depth = 0) {
5170 Value *A, *B;
5171 if (Depth < MaxAnalysisRecursionDepth &&
5172 (CondIsTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B)))
5173 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))))) {
5174 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue, CtxI, KnownFromContext,
5175 Depth: Depth + 1);
5176 computeKnownFPClassFromCond(V, Cond: B, CondIsTrue, CtxI, KnownFromContext,
5177 Depth: Depth + 1);
5178 return;
5179 }
5180 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A)))) {
5181 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue: !CondIsTrue, CtxI, KnownFromContext,
5182 Depth: Depth + 1);
5183 return;
5184 }
5185 CmpPredicate Pred;
5186 Value *LHS;
5187 uint64_t ClassVal = 0;
5188 const APFloat *CRHS;
5189 const APInt *RHS;
5190 if (match(V: Cond, P: m_FCmp(Pred, L: m_Value(V&: LHS), R: m_APFloat(Res&: CRHS)))) {
5191 auto [CmpVal, MaskIfTrue, MaskIfFalse] = fcmpImpliesClass(
5192 Pred, F: *cast<Instruction>(Val: Cond)->getParent()->getParent(), LHS, ConstRHS: *CRHS,
5193 LookThroughSrc: LHS != V);
5194 if (CmpVal == V)
5195 KnownFromContext.knownNot(RuleOut: ~(CondIsTrue ? MaskIfTrue : MaskIfFalse));
5196 } else if (match(V: Cond, P: m_Intrinsic<Intrinsic::is_fpclass>(
5197 Ops: m_Specific(V), Ops: m_ConstantInt(V&: ClassVal)))) {
5198 FPClassTest Mask = static_cast<FPClassTest>(ClassVal);
5199 KnownFromContext.knownNot(RuleOut: CondIsTrue ? ~Mask : Mask);
5200 } else if (match(V: Cond, P: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Specific(V)),
5201 R: m_APInt(Res&: RHS)))) {
5202 bool TrueIfSigned;
5203 if (!isSignBitCheck(Pred, RHS: *RHS, TrueIfSigned))
5204 return;
5205 if (TrueIfSigned == CondIsTrue)
5206 KnownFromContext.signBitMustBeOne();
5207 else
5208 KnownFromContext.signBitMustBeZero();
5209 }
5210}
5211
5212/// Compute the minimum and maximum values (inclusive) for the exponent of \p V,
5213/// assuming it is not nan. Returns {min, max, max-assuming-nonzero}. A value
5214/// frexp(0) = 0, so the tighter max-assuming-nonzero bound is only usable when
5215/// \p V is known not to be a logical zero (e.g., for fabs(x) < 0.25, the non-0
5216/// exponent range is [-149, -2], but the 0 edge case is above this range).
5217static std::tuple<int, int, int>
5218computeKnownExponentRangeFromContext(const Value *V, const SimplifyQuery &Q) {
5219 if (!Q.CtxI || !Q.DC || !Q.DT)
5220 return {APFloat::IEK_NaN, APFloat::IEK_Inf, APFloat::IEK_Inf};
5221
5222 // Intersect the bounds implied by every dominating condition, keeping the
5223 // tightest maximum. A value may participate in multiple compares
5224 // (e.g. fabs(x) < 2.0 and fabs(x) < 1.0), and the tighter one wins.
5225 int MaxExp = APFloat::IEK_Inf;
5226 int MaxExpNonZero = APFloat::IEK_Inf;
5227
5228 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
5229 CmpPredicate Pred;
5230 const APFloat *LimitC;
5231 if (!match(V: BI->getCondition(),
5232 P: m_FCmp(Pred, L: m_FAbs(Op0: m_Specific(V)), R: m_Finite(V&: LimitC))))
5233 continue;
5234
5235 if (Pred == FCmpInst::FCMP_ORD || Pred == FCmpInst::FCMP_UNO ||
5236 Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE)
5237 continue;
5238
5239 // If fabs(x) <= K, implies the exponent min exp range.
5240 // if fabs(x) >= K, swap the successor
5241 bool IsLessEqual =
5242 Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE ||
5243 Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE ||
5244 Pred == FCmpInst::FCMP_OEQ || Pred == FCmpInst::FCMP_UEQ;
5245
5246 bool KnownStrictlyLess =
5247 Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_ULT ||
5248 Pred == FCmpInst::FCMP_OGE || Pred == FCmpInst::FCMP_UGE;
5249
5250 BasicBlockEdge Edge1(BI->getParent(),
5251 BI->getSuccessor(i: IsLessEqual ? 0 : 1));
5252 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent())) {
5253 // frexp returns an exponent one greater than ilogb.
5254 int Exp = ilogb(Arg: *LimitC) + 1;
5255
5256 // A strict bound fabs(V) < 2^n forces ilogb(V) <= n - 1, so the max frexp
5257 // exponent drops by one when K is exact power of two.
5258 if (KnownStrictlyLess && LimitC->getExactLog2Abs() != INT_MIN)
5259 --Exp;
5260
5261 // frexp(0) = 0, which the bound above (assuming a normal nonzero value)
5262 // may exclude.
5263
5264 // TODO: Figure out lower bound to detect no-underflow.
5265 MaxExpNonZero = std::min(a: MaxExpNonZero, b: Exp);
5266 MaxExp = std::min(a: MaxExp, b: std::max(a: Exp, b: 0));
5267 }
5268 }
5269
5270 return {APFloat::IEK_NaN, MaxExp, MaxExpNonZero};
5271}
5272
5273static KnownFPClass computeKnownFPClassFromContext(const Value *V,
5274 const SimplifyQuery &Q) {
5275 KnownFPClass KnownFromContext;
5276
5277 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
5278 computeKnownFPClassFromCond(V, Cond: Q.CC->Cond, CondIsTrue: !Q.CC->Invert, CtxI: Q.CtxI,
5279 KnownFromContext);
5280
5281 if (!Q.CtxI)
5282 return KnownFromContext;
5283
5284 if (Q.DC && Q.DT) {
5285 // Handle dominating conditions.
5286 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
5287 Value *Cond = BI->getCondition();
5288
5289 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
5290 if (Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()))
5291 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, CtxI: Q.CtxI,
5292 KnownFromContext);
5293
5294 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
5295 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()))
5296 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, CtxI: Q.CtxI,
5297 KnownFromContext);
5298 }
5299 }
5300
5301 if (!Q.AC)
5302 return KnownFromContext;
5303
5304 // Try to restrict the floating-point classes based on information from
5305 // assumptions.
5306 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
5307 if (!AssumeVH)
5308 continue;
5309 CallInst *I = cast<CallInst>(Val&: AssumeVH);
5310
5311 assert(I->getFunction() == Q.CtxI->getParent()->getParent() &&
5312 "Got assumption for the wrong function!");
5313 assert(I->getIntrinsicID() == Intrinsic::assume &&
5314 "must be an assume intrinsic");
5315
5316 if (!isValidAssumeForContext(I, Q))
5317 continue;
5318
5319 computeKnownFPClassFromCond(V, Cond: I->getArgOperand(i: 0),
5320 /*CondIsTrue=*/true, CtxI: Q.CtxI, KnownFromContext);
5321 }
5322
5323 return KnownFromContext;
5324}
5325
5326void llvm::adjustKnownFPClassForSelectArm(KnownFPClass &Known, Value *Cond,
5327 Value *Arm, bool Invert,
5328 const SimplifyQuery &SQ,
5329 unsigned Depth) {
5330
5331 KnownFPClass KnownSrc;
5332 computeKnownFPClassFromCond(V: Arm, Cond,
5333 /*CondIsTrue=*/!Invert, CtxI: SQ.CtxI, KnownFromContext&: KnownSrc,
5334 Depth: Depth + 1);
5335 KnownSrc = KnownSrc.unionWith(RHS: Known);
5336 if (KnownSrc.isUnknown())
5337 return;
5338
5339 if (isGuaranteedNotToBeUndef(V: Arm, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT, Depth: Depth + 1))
5340 Known = KnownSrc;
5341}
5342
5343void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5344 FPClassTest InterestedClasses, KnownFPClass &Known,
5345 const SimplifyQuery &Q, unsigned Depth);
5346
5347static void computeKnownFPClass(const Value *V, KnownFPClass &Known,
5348 FPClassTest InterestedClasses,
5349 const SimplifyQuery &Q, unsigned Depth) {
5350 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5351 APInt DemandedElts =
5352 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
5353 computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Q, Depth);
5354}
5355
5356static void computeKnownFPClassForFPTrunc(const Operator *Op,
5357 const APInt &DemandedElts,
5358 FPClassTest InterestedClasses,
5359 KnownFPClass &Known,
5360 const SimplifyQuery &Q,
5361 unsigned Depth) {
5362 if ((InterestedClasses &
5363 (KnownFPClass::OrderedLessThanZeroMask | fcNan)) == fcNone)
5364 return;
5365
5366 KnownFPClass KnownSrc;
5367 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5368 Known&: KnownSrc, Q, Depth: Depth + 1);
5369
5370 const fltSemantics &DstTy = Op->getType()->getScalarType()->getFltSemantics();
5371 const fltSemantics &SrcTy =
5372 Op->getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
5373
5374 const auto *I = dyn_cast<Instruction>(Val: Op);
5375 const Function *F = I ? I->getFunction() : nullptr;
5376 DenormalMode DstMode =
5377 F ? F->getDenormalMode(FPType: DstTy) : DenormalMode::getDynamic();
5378 DenormalMode SrcMode =
5379 F ? F->getDenormalMode(FPType: SrcTy) : DenormalMode::getDynamic();
5380
5381 Known = KnownFPClass::fptrunc(KnownSrc, DstTy, SrcTy, DstMode, SrcMode);
5382}
5383
5384static void
5385computeKnownFPClassForFAddSub(const Operator *Op, const APInt &DemandedElts,
5386 FPClassTest InterestedClasses,
5387 KnownFPClass &Known, const SimplifyQuery &Q,
5388 unsigned Depth, bool IsAdd, bool IsStrict) {
5389 const Value *LHS = Op->getOperand(i: 0);
5390 const Value *RHS = Op->getOperand(i: 1);
5391
5392 const RoundingMode RM =
5393 IsStrict ? cast<ConstrainedFPIntrinsic>(Val: Op)->getRoundingMode().value_or(
5394 u: RoundingMode::Dynamic)
5395 : RoundingMode::NearestTiesToEven;
5396
5397 KnownFPClass KnownLHS, KnownRHS;
5398 bool WantNegative =
5399 IsAdd &&
5400 (InterestedClasses & KnownFPClass::OrderedLessThanZeroMask) != fcNone;
5401 bool WantNaN = (InterestedClasses & fcNan) != fcNone;
5402 bool WantNegZero = (InterestedClasses & fcNegZero) != fcNone;
5403 bool WantSubnormal = (InterestedClasses & fcSubnormal) != fcNone;
5404
5405 if (!WantNaN && !WantNegative && !WantNegZero && !WantSubnormal)
5406 return;
5407
5408 FPClassTest InterestedSrcs = InterestedClasses;
5409 if (WantNegative)
5410 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5411 if (InterestedClasses & fcNan)
5412 InterestedSrcs |= fcInf;
5413 computeKnownFPClass(V: RHS, DemandedElts, InterestedClasses: InterestedSrcs, Known&: KnownRHS, Q,
5414 Depth: Depth + 1);
5415
5416 // Special case fadd x, x, which is the canonical form of fmul x, 2.
5417 bool Self = LHS == RHS &&
5418 isGuaranteedNotToBeUndef(V: LHS, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1);
5419 if (Self)
5420 KnownLHS = KnownRHS;
5421
5422 if ((WantNaN && KnownRHS.isKnownNeverNaN()) ||
5423 (WantNegative && KnownRHS.cannotBeOrderedLessThanZero()) || WantNegZero ||
5424 WantSubnormal || !IsAdd) {
5425
5426 // FIXME: Context function should always be passed in separately
5427 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5428 const fltSemantics &FltSem =
5429 Op->getType()->getScalarType()->getFltSemantics();
5430 DenormalMode Mode =
5431 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5432
5433 if (Self && IsAdd) {
5434 Known = KnownFPClass::fadd_self(Src: KnownLHS, RM, Mode);
5435 } else {
5436 // RHS is canonically cheaper to compute. Skip inspecting the LHS if
5437 // there's no point.
5438
5439 if (!Self) {
5440 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: InterestedSrcs, Known&: KnownLHS, Q,
5441 Depth: Depth + 1);
5442 }
5443
5444 if (const APFloat *CRHS; IsAdd && match(V: RHS, P: m_APFloat(Res&: CRHS))) {
5445 Known = KnownFPClass::fadd(LHS: KnownLHS, RHS: *CRHS, RM, Mode);
5446 return;
5447 }
5448
5449 Known = IsAdd ? KnownFPClass::fadd(LHS: KnownLHS, RHS: KnownRHS, RM, Mode)
5450 : KnownFPClass::fsub(LHS: KnownLHS, RHS: KnownRHS, RM, Mode);
5451 }
5452 }
5453}
5454
5455static void computeKnownFPClassForFRem(const Operator *Op,
5456 const APInt &DemandedElts,
5457 FPClassTest InterestedClasses,
5458 KnownFPClass &Known,
5459 const SimplifyQuery &Q, unsigned Depth) {
5460 const Value *LHS = Op->getOperand(i: 0);
5461 const Value *RHS = Op->getOperand(i: 1);
5462
5463 FPClassTest InterestedLHS = fcNone;
5464 FPClassTest InterestedRHS = fcNone;
5465
5466 // NaN is also generated for frem(Inf, x) and frem(x, 0.0).
5467 if (InterestedClasses & fcNan) {
5468 InterestedLHS |= fcNan | fcInf;
5469 InterestedRHS |= fcNan | fcZero | fcSubnormal;
5470 }
5471
5472 // The sign for frem is the same as the first operand.
5473 if (InterestedClasses & (fcPosNormal | fcPosSubnormal))
5474 InterestedLHS |= fcPosNormal | fcPosSubnormal;
5475 if (InterestedClasses & (fcNegNormal | fcNegSubnormal))
5476 InterestedLHS |= fcNegNormal | fcNegSubnormal;
5477
5478 // A negative zero result requires a negative finite first operand.
5479 if (InterestedClasses & fcNegZero)
5480 InterestedLHS |= fcNegFinite;
5481
5482 // A positive zero result can additionally come from a negative finite
5483 // result being flushed to positive zero.
5484 if (InterestedClasses & fcPosZero)
5485 InterestedLHS |= fcPosFinite | fcNegNormal | fcNegSubnormal;
5486
5487 const Function *F = cast<Instruction>(Val: Op)->getFunction();
5488 DenormalMode Mode =
5489 F ? F->getDenormalMode(FPType: Op->getType()->getScalarType()->getFltSemantics())
5490 : DenormalMode::getDynamic();
5491
5492 if (LHS == RHS && isGuaranteedNotToBeUndef(V: LHS, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT)) {
5493 // X % X is always exactly [+-]0.0 or a NaN.
5494 FPClassTest InterestedSrcs = InterestedLHS | InterestedRHS;
5495 KnownFPClass KnownSrc;
5496 if (InterestedSrcs != fcNone)
5497 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: InterestedSrcs, Known&: KnownSrc, Q,
5498 Depth: Depth + 1);
5499 Known = KnownFPClass::frem_self(Src: KnownSrc, Mode);
5500 return;
5501 }
5502
5503 KnownFPClass KnownLHS;
5504 if (InterestedLHS != fcNone)
5505 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: InterestedLHS, Known&: KnownLHS, Q,
5506 Depth: Depth + 1);
5507
5508 KnownFPClass KnownRHS;
5509 // RHS is only useful for refining NaN classes.
5510 if (InterestedRHS != fcNone && KnownLHS.isKnownNever(Mask: fcSNan))
5511 computeKnownFPClass(V: RHS, DemandedElts, InterestedClasses: InterestedRHS, Known&: KnownRHS, Q,
5512 Depth: Depth + 1);
5513
5514 Known = KnownFPClass::frem(LHS: KnownLHS, RHS: KnownRHS, Mode);
5515}
5516
5517static constexpr KnownFPClass::MinMaxKind getMinMaxKind(Intrinsic::ID IID) {
5518 switch (IID) {
5519 case Intrinsic::minimum:
5520 return KnownFPClass::MinMaxKind::minimum;
5521 case Intrinsic::maximum:
5522 return KnownFPClass::MinMaxKind::maximum;
5523 case Intrinsic::minimumnum:
5524 return KnownFPClass::MinMaxKind::minimumnum;
5525 case Intrinsic::maximumnum:
5526 return KnownFPClass::MinMaxKind::maximumnum;
5527 case Intrinsic::minnum:
5528 return KnownFPClass::MinMaxKind::minnum;
5529 case Intrinsic::maxnum:
5530 return KnownFPClass::MinMaxKind::maxnum;
5531 default:
5532 llvm_unreachable("not a floating-point min-max intrinsic");
5533 }
5534}
5535
5536/// \return true if this is a floating point value that is known to have a
5537/// magnitude smaller than 1. i.e., fabs(X) <= 1.0 or is nan.
5538static bool isAbsoluteValueULEOne(const Value *V) {
5539 // TODO: Handle frexp
5540 // TODO: Other rounding intrinsics?
5541 // TODO: Try computeKnownExponentRangeFromContext
5542
5543 // fabs(x - floor(x)) <= 1
5544 const Value *SubFloorX;
5545 if (match(V, P: m_FSub(L: m_Value(V&: SubFloorX),
5546 R: m_Intrinsic<Intrinsic::floor>(Ops: m_Deferred(V: SubFloorX)))))
5547 return true;
5548
5549 return match(V, P: m_Intrinsic<Intrinsic::amdgcn_trig_preop>(Ops: m_Value())) ||
5550 match(V, P: m_Intrinsic<Intrinsic::amdgcn_fract>(Ops: m_Value()));
5551}
5552
5553void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5554 FPClassTest InterestedClasses, KnownFPClass &Known,
5555 const SimplifyQuery &Q, unsigned Depth) {
5556 assert(Known.isUnknown() && "should not be called with known information");
5557
5558 if (!DemandedElts) {
5559 // No demanded elts, better to assume we don't know anything.
5560 Known.resetAll();
5561 return;
5562 }
5563
5564 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
5565
5566 if (auto *CFP = dyn_cast<ConstantFP>(Val: V)) {
5567 Known = KnownFPClass(CFP->getValueAPF());
5568 return;
5569 }
5570
5571 if (isa<ConstantAggregateZero>(Val: V)) {
5572 Known.setKnownFPClasses(fcPosZero);
5573 return;
5574 }
5575
5576 if (isa<PoisonValue>(Val: V)) {
5577 Known = KnownFPClass::getEmpty();
5578 return;
5579 }
5580
5581 // Try to handle fixed width vector constants
5582 auto *VFVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5583 const Constant *CV = dyn_cast<Constant>(Val: V);
5584 if (VFVTy && CV) {
5585 Known = KnownFPClass::getEmpty();
5586
5587 // For vectors, verify that each element is not NaN.
5588 unsigned NumElts = VFVTy->getNumElements();
5589 for (unsigned i = 0; i != NumElts; ++i) {
5590 if (!DemandedElts[i])
5591 continue;
5592
5593 Constant *Elt = CV->getAggregateElement(Elt: i);
5594 if (!Elt) {
5595 Known = KnownFPClass();
5596 return;
5597 }
5598 if (isa<PoisonValue>(Val: Elt))
5599 continue;
5600 auto *CElt = dyn_cast<ConstantFP>(Val: Elt);
5601 if (!CElt) {
5602 Known = KnownFPClass();
5603 return;
5604 }
5605
5606 Known |= KnownFPClass(CElt->getValueAPF());
5607 }
5608 return;
5609 }
5610
5611 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) {
5612 Known = KnownFPClass::getEmpty();
5613 for (size_t I = 0, E = CDS->getNumElements(); I != E; ++I)
5614 Known |= KnownFPClass(CDS->getElementAsAPFloat(i: I));
5615 return;
5616 }
5617
5618 if (const auto *CA = dyn_cast<ConstantAggregate>(Val: V)) {
5619 // TODO: Handle complex aggregates
5620 Known = KnownFPClass::getEmpty();
5621 for (const Use &Op : CA->operands()) {
5622 auto *CFP = dyn_cast<ConstantFP>(Val: Op.get());
5623 if (!CFP) {
5624 Known = KnownFPClass();
5625 return;
5626 }
5627
5628 Known |= KnownFPClass(CFP->getValueAPF());
5629 }
5630
5631 return;
5632 }
5633
5634 FPClassTest KnownNotFromFlags = fcNone;
5635 if (const auto *CB = dyn_cast<CallBase>(Val: V))
5636 KnownNotFromFlags |= CB->getRetNoFPClass();
5637 else if (const auto *Arg = dyn_cast<Argument>(Val: V))
5638 KnownNotFromFlags |= Arg->getNoFPClass();
5639
5640 const Operator *Op = dyn_cast<Operator>(Val: V);
5641 if (const FPMathOperator *FPOp = dyn_cast_or_null<FPMathOperator>(Val: Op)) {
5642 if (FPOp->hasNoNaNs())
5643 KnownNotFromFlags |= fcNan;
5644 if (FPOp->hasNoInfs())
5645 KnownNotFromFlags |= fcInf;
5646 }
5647
5648 KnownFPClass AssumedClasses = computeKnownFPClassFromContext(V, Q);
5649 KnownNotFromFlags |= ~AssumedClasses.getKnownFPClasses();
5650
5651 // We no longer need to find out about these bits from inputs if we can
5652 // assume this from flags/attributes.
5653 InterestedClasses &= ~KnownNotFromFlags;
5654
5655 llvm::scope_exit ClearClassesFromFlags([=, &Known] {
5656 Known.knownNot(RuleOut: KnownNotFromFlags);
5657 if (!Known.getSignBit() && AssumedClasses.getSignBit()) {
5658 if (*AssumedClasses.getSignBit())
5659 Known.signBitMustBeOne();
5660 else
5661 Known.signBitMustBeZero();
5662 }
5663 });
5664
5665 if (!Op)
5666 return;
5667
5668 // All recursive calls that increase depth must come after this.
5669 if (Depth == MaxAnalysisRecursionDepth)
5670 return;
5671
5672 const unsigned Opc = Op->getOpcode();
5673 switch (Opc) {
5674 case Instruction::FNeg: {
5675 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5676 Known, Q, Depth: Depth + 1);
5677 Known.fneg();
5678 break;
5679 }
5680 case Instruction::Select: {
5681 auto ComputeForArm = [&](Value *Arm, bool Invert) {
5682 KnownFPClass Res;
5683 computeKnownFPClass(V: Arm, DemandedElts, InterestedClasses, Known&: Res, Q,
5684 Depth: Depth + 1);
5685 adjustKnownFPClassForSelectArm(Known&: Res, Cond: Op->getOperand(i: 0), Arm, Invert, SQ: Q,
5686 Depth);
5687 return Res;
5688 };
5689 // Only known if known in both the LHS and RHS.
5690 Known =
5691 ComputeForArm(Op->getOperand(i: 1), /*Invert=*/false)
5692 .intersectWith(RHS: ComputeForArm(Op->getOperand(i: 2), /*Invert=*/true));
5693 break;
5694 }
5695 case Instruction::Load: {
5696 const MDNode *NoFPClass =
5697 cast<LoadInst>(Val: Op)->getMetadata(KindID: LLVMContext::MD_nofpclass);
5698 if (!NoFPClass)
5699 break;
5700
5701 ConstantInt *MaskVal =
5702 mdconst::extract<ConstantInt>(MD: NoFPClass->getOperand(I: 0));
5703 Known.knownNot(RuleOut: static_cast<FPClassTest>(MaskVal->getZExtValue()));
5704 break;
5705 }
5706 case Instruction::Call: {
5707 const CallInst *II = cast<CallInst>(Val: Op);
5708 const Intrinsic::ID IID = II->getIntrinsicID();
5709 switch (IID) {
5710 case Intrinsic::fabs: {
5711 if ((InterestedClasses & (fcNan | fcPositive)) != fcNone) {
5712 // If we only care about the sign bit we don't need to inspect the
5713 // operand.
5714 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5715 InterestedClasses, Known, Q, Depth: Depth + 1);
5716 }
5717
5718 Known.fabs();
5719 break;
5720 }
5721 case Intrinsic::copysign: {
5722 KnownFPClass KnownSign;
5723
5724 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5725 Known, Q, Depth: Depth + 1);
5726 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5727 Known&: KnownSign, Q, Depth: Depth + 1);
5728 Known.copysign(Sign: KnownSign);
5729 break;
5730 }
5731 case Intrinsic::fma:
5732 case Intrinsic::fmuladd: {
5733 if ((InterestedClasses & (fcNegative | fcNan)) == fcNone)
5734 break;
5735
5736 const bool IsFMA = IID == Intrinsic::fma;
5737
5738 // FIXME: This should check isGuaranteedNotToBeUndef
5739 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) {
5740 FPClassTest InterestedSquared = InterestedClasses;
5741 FPClassTest InterestedAddend = InterestedClasses;
5742 // Only fma(+inf, +inf, -inf) and fma(-inf, -inf, -inf) produce NaN when
5743 // squaring.
5744 if (InterestedClasses & fcQNan) {
5745 InterestedSquared |= fcSNan | fcInf;
5746 InterestedAddend |= fcSNan | fcNegInf;
5747 }
5748
5749 KnownFPClass KnownSrc, KnownAddend;
5750 computeKnownFPClass(V: II->getArgOperand(i: 2), DemandedElts,
5751 InterestedClasses: InterestedAddend, Known&: KnownAddend, Q, Depth: Depth + 1);
5752 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5753 InterestedClasses: InterestedSquared, Known&: KnownSrc, Q, Depth: Depth + 1);
5754
5755 const Function *F = II->getFunction();
5756 const fltSemantics &FltSem =
5757 II->getType()->getScalarType()->getFltSemantics();
5758 DenormalMode Mode =
5759 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5760
5761 if ((KnownNotFromFlags & fcNan) == fcNan) {
5762 KnownSrc.knownNot(RuleOut: fcNan);
5763 KnownAddend.knownNot(RuleOut: fcNan);
5764 }
5765
5766 if ((KnownNotFromFlags & fcInf) == fcInf) {
5767 KnownSrc.knownNot(RuleOut: fcInf);
5768 KnownAddend.knownNot(RuleOut: fcInf);
5769 }
5770
5771 Known = IsFMA
5772 ? KnownFPClass::fma_square(Squared: KnownSrc, Addend: KnownAddend, Mode)
5773 : KnownFPClass::fmuladd_square(Squared: KnownSrc, Addend: KnownAddend, Mode);
5774 break;
5775 }
5776
5777 KnownFPClass KnownSrc[3];
5778 for (int I = 0; I != 3; ++I) {
5779 computeKnownFPClass(V: II->getArgOperand(i: I), DemandedElts,
5780 InterestedClasses, Known&: KnownSrc[I], Q, Depth: Depth + 1);
5781 if (KnownSrc[I].isUnknown())
5782 return;
5783
5784 if ((KnownNotFromFlags & fcNan) == fcNan)
5785 KnownSrc[I].knownNot(RuleOut: fcNan);
5786 if ((KnownNotFromFlags & fcInf) == fcInf)
5787 KnownSrc[I].knownNot(RuleOut: fcInf);
5788 }
5789
5790 const Function *F = II->getFunction();
5791 const fltSemantics &FltSem =
5792 II->getType()->getScalarType()->getFltSemantics();
5793 DenormalMode Mode =
5794 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5795 Known =
5796 IsFMA ? KnownFPClass::fma(LHS: KnownSrc[0], RHS: KnownSrc[1], Addend: KnownSrc[2], Mode)
5797 : KnownFPClass::fmuladd(LHS: KnownSrc[0], RHS: KnownSrc[1], Addend: KnownSrc[2],
5798 Mode);
5799 break;
5800 }
5801 case Intrinsic::experimental_constrained_fadd:
5802 case Intrinsic::experimental_constrained_fsub: {
5803 computeKnownFPClassForFAddSub(
5804 Op, DemandedElts, InterestedClasses, Known, Q, Depth,
5805 /*IsAdd=*/IID == Intrinsic::experimental_constrained_fadd,
5806 /*IsStrict=*/true);
5807 break;
5808 }
5809 case Intrinsic::experimental_constrained_frem: {
5810 computeKnownFPClassForFRem(Op, DemandedElts, InterestedClasses, Known, Q,
5811 Depth);
5812 break;
5813 }
5814 case Intrinsic::sqrt:
5815 case Intrinsic::experimental_constrained_sqrt: {
5816 KnownFPClass KnownSrc;
5817 FPClassTest InterestedSrcs = InterestedClasses;
5818 if (InterestedClasses & fcQNan)
5819 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask | fcSNan;
5820 // Subnormals may flush to zero.
5821 if (InterestedClasses & fcZero)
5822 InterestedSrcs |= fcSubnormal;
5823
5824 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5825 Known&: KnownSrc, Q, Depth: Depth + 1);
5826
5827 DenormalMode Mode = DenormalMode::getDynamic();
5828
5829 bool HasNSZ = Q.IIQ.hasNoSignedZeros(Op: II);
5830 if (!HasNSZ) {
5831 const Function *F = II->getFunction();
5832 const fltSemantics &FltSem =
5833 II->getType()->getScalarType()->getFltSemantics();
5834 Mode = F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5835 }
5836
5837 Known = KnownFPClass::sqrt(Src: KnownSrc, Mode);
5838 if (HasNSZ)
5839 Known.knownNot(RuleOut: fcNegZero);
5840
5841 break;
5842 }
5843 case Intrinsic::sin: {
5844 KnownFPClass KnownSrc;
5845 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5846 Known&: KnownSrc, Q, Depth: Depth + 1);
5847 Known = KnownFPClass::sin(Src: KnownSrc);
5848 break;
5849 }
5850 case Intrinsic::cos: {
5851 KnownFPClass KnownSrc;
5852 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5853 Known&: KnownSrc, Q, Depth: Depth + 1);
5854 Known = KnownFPClass::cos(Src: KnownSrc);
5855 break;
5856 }
5857 case Intrinsic::tan: {
5858 KnownFPClass KnownSrc;
5859 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5860 Known&: KnownSrc, Q, Depth: Depth + 1);
5861 Known = KnownFPClass::tan(Src: KnownSrc);
5862 break;
5863 }
5864 case Intrinsic::sinh: {
5865 KnownFPClass KnownSrc;
5866 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5867 Known&: KnownSrc, Q, Depth: Depth + 1);
5868 Known = KnownFPClass::sinh(Src: KnownSrc);
5869 break;
5870 }
5871 case Intrinsic::cosh: {
5872 KnownFPClass KnownSrc;
5873 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5874 Known&: KnownSrc, Q, Depth: Depth + 1);
5875 Known = KnownFPClass::cosh(Src: KnownSrc);
5876 break;
5877 }
5878 case Intrinsic::tanh: {
5879 KnownFPClass KnownSrc;
5880 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5881 Known&: KnownSrc, Q, Depth: Depth + 1);
5882 Known = KnownFPClass::tanh(Src: KnownSrc);
5883 break;
5884 }
5885 case Intrinsic::asin: {
5886 KnownFPClass KnownSrc;
5887 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5888 Known&: KnownSrc, Q, Depth: Depth + 1);
5889 Known = KnownFPClass::asin(Src: KnownSrc);
5890 break;
5891 }
5892 case Intrinsic::acos: {
5893 KnownFPClass KnownSrc;
5894 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5895 Known&: KnownSrc, Q, Depth: Depth + 1);
5896 Known = KnownFPClass::acos(Src: KnownSrc);
5897 break;
5898 }
5899 case Intrinsic::atan: {
5900 KnownFPClass KnownSrc;
5901 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5902 Known&: KnownSrc, Q, Depth: Depth + 1);
5903 Known = KnownFPClass::atan(Src: KnownSrc);
5904 break;
5905 }
5906 case Intrinsic::atan2: {
5907 FPClassTest InterestedY = InterestedClasses;
5908 FPClassTest InterestedX = InterestedClasses;
5909
5910 // We can rule out negative values if y cannot have a negative value.
5911 if ((InterestedClasses & fcNegFinite) != fcNone)
5912 InterestedY |= fcNegative;
5913
5914 // We can rule out positive values if y cannot have a positive value.
5915 if ((InterestedClasses & fcPosFinite) != fcNone)
5916 InterestedY |= fcPositive | fcNegSubnormal;
5917
5918 // We can rule out zero and subnormal if x cannot have a positive value.
5919 if ((InterestedClasses & (fcZero | fcSubnormal)) != fcNone)
5920 InterestedX |= fcPositive | fcNegSubnormal;
5921
5922 KnownFPClass KnownY, KnownX;
5923 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedY,
5924 Known&: KnownY, Q, Depth: Depth + 1);
5925 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses: InterestedX,
5926 Known&: KnownX, Q, Depth: Depth + 1);
5927
5928 const Function *F = II->getFunction();
5929 DenormalMode Mode =
5930 F ? F->getDenormalMode(
5931 FPType: II->getType()->getScalarType()->getFltSemantics())
5932 : DenormalMode::getDynamic();
5933 Known = KnownFPClass::atan2(LHS: KnownY, RHS: KnownX, Mode);
5934 break;
5935 }
5936 case Intrinsic::maxnum:
5937 case Intrinsic::minnum:
5938 case Intrinsic::minimum:
5939 case Intrinsic::maximum:
5940 case Intrinsic::minimumnum:
5941 case Intrinsic::maximumnum: {
5942 KnownFPClass KnownLHS, KnownRHS;
5943 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5944 Known&: KnownLHS, Q, Depth: Depth + 1);
5945 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5946 Known&: KnownRHS, Q, Depth: Depth + 1);
5947
5948 const Function *F = II->getFunction();
5949
5950 DenormalMode Mode =
5951 F ? F->getDenormalMode(
5952 FPType: II->getType()->getScalarType()->getFltSemantics())
5953 : DenormalMode::getDynamic();
5954
5955 Known = KnownFPClass::minMaxLike(LHS: KnownLHS, RHS: KnownRHS, Kind: getMinMaxKind(IID),
5956 Mode);
5957 break;
5958 }
5959 case Intrinsic::canonicalize: {
5960 KnownFPClass KnownSrc;
5961 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5962 Known&: KnownSrc, Q, Depth: Depth + 1);
5963
5964 const Function *F = II->getFunction();
5965 const fltSemantics &Flt =
5966 II->getType()->getScalarType()->getFltSemantics();
5967 DenormalMode Mode =
5968 F ? F->getDenormalMode(FPType: Flt) : DenormalMode::getDynamic();
5969 Known = KnownFPClass::canonicalize(Src: KnownSrc, Flt, Mode);
5970 break;
5971 }
5972 case Intrinsic::vector_reduce_fmax:
5973 case Intrinsic::vector_reduce_fmin:
5974 case Intrinsic::vector_reduce_fmaximum:
5975 case Intrinsic::vector_reduce_fminimum:
5976 case Intrinsic::vector_reduce_fmaximumnum:
5977 case Intrinsic::vector_reduce_fminimumnum: {
5978 // reduce min/max will choose an element from one of the vector elements,
5979 // so we can infer and class information that is common to all elements.
5980 Known = computeKnownFPClass(V: II->getArgOperand(i: 0), FMF: II->getFastMathFlags(),
5981 InterestedClasses, SQ: Q, Depth: Depth + 1);
5982 // Can only propagate sign if output is never NaN.
5983 if (!Known.isKnownNeverNaN())
5984 Known.setSignBit(std::nullopt);
5985 break;
5986 }
5987 // reverse preserves all characteristics of the input vec's element.
5988 case Intrinsic::vector_reverse:
5989 Known = computeKnownFPClass(
5990 V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
5991 FMF: II->getFastMathFlags(), InterestedClasses, SQ: Q, Depth: Depth + 1);
5992 break;
5993 case Intrinsic::trunc:
5994 case Intrinsic::floor:
5995 case Intrinsic::ceil:
5996 case Intrinsic::rint:
5997 case Intrinsic::nearbyint:
5998 case Intrinsic::round:
5999 case Intrinsic::roundeven: {
6000 KnownFPClass KnownSrc;
6001 FPClassTest InterestedSrcs = InterestedClasses;
6002
6003 // Negative round ups towards zero produce negative zero.
6004 if (InterestedSrcs & fcNegFinite)
6005 InterestedSrcs |= fcNegFinite;
6006
6007 // Negative subnormals may flush to positive zero.
6008 if (InterestedSrcs & fcPosFinite)
6009 InterestedSrcs |= fcPosFinite | fcNegSubnormal;
6010
6011 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
6012 Known&: KnownSrc, Q, Depth: Depth + 1);
6013
6014 const Function *F = II->getFunction();
6015 DenormalMode Mode =
6016 F ? F->getDenormalMode(
6017 FPType: II->getType()->getScalarType()->getFltSemantics())
6018 : DenormalMode::getDynamic();
6019 const bool IsMultiUnitFPType =
6020 V->getType()->getScalarType()->isMultiUnitFPType();
6021
6022 switch (IID) {
6023 case Intrinsic::trunc:
6024 Known = KnownFPClass::trunc(Src: KnownSrc, Mode);
6025 break;
6026 case Intrinsic::floor:
6027 Known = KnownFPClass::floor(Src: KnownSrc, IsMultiUnitFPType, Mode);
6028 break;
6029 case Intrinsic::ceil:
6030 Known = KnownFPClass::ceil(Src: KnownSrc, IsMultiUnitFPType, Mode);
6031 break;
6032 default:
6033 Known =
6034 KnownFPClass::roundToIntegral(Src: KnownSrc, IsMultiUnitFPType, Mode);
6035 break;
6036 }
6037 break;
6038 }
6039 case Intrinsic::exp:
6040 case Intrinsic::exp2:
6041 case Intrinsic::exp10:
6042 case Intrinsic::amdgcn_exp2: {
6043 KnownFPClass KnownSrc;
6044 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
6045 Known&: KnownSrc, Q, Depth: Depth + 1);
6046
6047 Known = KnownFPClass::exp(Src: KnownSrc);
6048
6049 Type *EltTy = II->getType()->getScalarType();
6050 if (IID == Intrinsic::amdgcn_exp2 && EltTy->isFloatTy())
6051 Known.knownNot(RuleOut: fcSubnormal);
6052
6053 break;
6054 }
6055 case Intrinsic::fptrunc_round: {
6056 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
6057 Q, Depth);
6058 break;
6059 }
6060 case Intrinsic::log:
6061 case Intrinsic::log10:
6062 case Intrinsic::log2:
6063 case Intrinsic::experimental_constrained_log:
6064 case Intrinsic::experimental_constrained_log10:
6065 case Intrinsic::experimental_constrained_log2:
6066 case Intrinsic::amdgcn_log: {
6067 FPClassTest InterestedSrcs = fcNone;
6068
6069 // log(negative) produces NaN.
6070 if ((InterestedClasses & fcNan) != fcNone)
6071 InterestedSrcs |= fcNan | fcNegative;
6072
6073 // log(logical-zero) produces negative infinity.
6074 if ((InterestedClasses & fcNegInf) != fcNone)
6075 InterestedSrcs |= fcZero | fcSubnormal;
6076
6077 // log(x) < -0.0 if x < +1.0
6078 if ((InterestedClasses & fcNegNormal) != fcNone)
6079 InterestedSrcs |= fcPosSubnormal | fcPosNormal;
6080
6081 // log(x) >= +0.0 if x >= +1.0
6082 if ((InterestedClasses & (fcPosZero | fcPosNormal)) != fcNone)
6083 InterestedSrcs |= fcPosNormal;
6084
6085 // log(x) is positive infinity iff x is positive infinity.
6086 if ((InterestedClasses & fcPosInf) != fcNone)
6087 InterestedSrcs |= fcPosInf;
6088
6089 KnownFPClass KnownSrc;
6090 if (InterestedSrcs != fcNone)
6091 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
6092 Known&: KnownSrc, Q, Depth: Depth + 1);
6093 const Function *F = II->getFunction();
6094 DenormalMode Mode =
6095 F ? F->getDenormalMode(
6096 FPType: II->getType()->getScalarType()->getFltSemantics())
6097 : DenormalMode::getDynamic();
6098 const bool IsMultiUnitFPType =
6099 V->getType()->getScalarType()->isMultiUnitFPType();
6100 Known = KnownFPClass::log(Src: KnownSrc, Mode, IsMultiUnitFPType);
6101 break;
6102 }
6103 case Intrinsic::pow: {
6104 const bool WantNaN = (InterestedClasses & fcNan) != fcNone;
6105 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
6106 if (!WantNaN && !WantNegative)
6107 break;
6108
6109 FPClassTest InterestedLHS = fcNone;
6110 FPClassTest InterestedRHS = fcNone;
6111 if (WantNaN) {
6112 // pow may return NaN if one of the arguments is NaN. NaN may also be
6113 // produced from a negative, non-zero finite base and a non-integer
6114 // exponent.
6115 InterestedLHS |= fcNan | fcNegNormal | fcNegSubnormal;
6116 InterestedRHS |= fcNan;
6117 }
6118 if (WantNegative) {
6119 // A negative value is returned when a negative base is raised to an odd
6120 // integer power. Only normal values can be odd integers.
6121 InterestedLHS |= fcNegative;
6122 InterestedRHS |= fcNormal;
6123 }
6124
6125 KnownFPClass KnownLHS;
6126 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedLHS,
6127 Known&: KnownLHS, Q, Depth: Depth + 1);
6128
6129 // If the LHS is unknown, then querying the RHS is only useful for rare
6130 // edge cases.
6131 if (KnownLHS.isUnknown())
6132 break;
6133
6134 KnownFPClass KnownRHS;
6135 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses: InterestedRHS,
6136 Known&: KnownRHS, Q, Depth: Depth + 1);
6137 Known = KnownFPClass::pow(LHS: KnownLHS, RHS: KnownRHS);
6138 break;
6139 }
6140 case Intrinsic::powi: {
6141 if ((InterestedClasses & (fcNan | fcInf | fcNegative)) == fcNone)
6142 break;
6143
6144 // The exponent is always a scalar, even when raising a vector to a power.
6145 const Value *Exp = II->getArgOperand(i: 1);
6146 unsigned BitWidth = Exp->getType()->getIntegerBitWidth();
6147 KnownBits ExponentKnownBits(BitWidth);
6148 computeKnownBits(V: Exp, DemandedElts: APInt(1, 1), Known&: ExponentKnownBits, Q, Depth: Depth + 1);
6149
6150 FPClassTest InterestedSrcs = fcNone;
6151 if (InterestedClasses & fcNan)
6152 InterestedSrcs |= fcNan;
6153 if (!ExponentKnownBits.isZero()) {
6154 if (InterestedClasses & fcInf)
6155 InterestedSrcs |= fcFinite | fcInf;
6156 if ((InterestedClasses & fcNegative) && !ExponentKnownBits.isEven())
6157 InterestedSrcs |= fcNegative;
6158 }
6159
6160 KnownFPClass KnownSrc;
6161 if (InterestedSrcs != fcNone)
6162 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
6163 Known&: KnownSrc, Q, Depth: Depth + 1);
6164
6165 Known = KnownFPClass::powi(Src: KnownSrc, N: ExponentKnownBits);
6166 break;
6167 }
6168 case Intrinsic::ldexp: {
6169 KnownFPClass KnownSrc;
6170 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
6171 Known&: KnownSrc, Q, Depth: Depth + 1);
6172 // Can refine inf/zero handling based on the exponent operand.
6173 const FPClassTest ExpInfoMask = fcZero | fcSubnormal | fcInf;
6174
6175 const Value *ExpArg = II->getArgOperand(i: 1);
6176 ConstantRange ExpKnownRange =
6177 ((KnownSrc.getKnownFPClasses() & ExpInfoMask) != fcNone)
6178 ? computeConstantRange(V: ExpArg, /*ForSigned=*/true, SQ: Q, Depth: Depth + 1)
6179 : ConstantRange::getFull(
6180 BitWidth: ExpArg->getType()->getScalarSizeInBits());
6181
6182 const fltSemantics &Flt =
6183 II->getType()->getScalarType()->getFltSemantics();
6184
6185 const Function *F = II->getFunction();
6186 DenormalMode Mode =
6187 F ? F->getDenormalMode(FPType: Flt) : DenormalMode::getDynamic();
6188
6189 Known = KnownFPClass::ldexp(Src: KnownSrc, ConstantRangeMin: ExpKnownRange.getSignedMin(),
6190 ConstantRangeMax: ExpKnownRange.getSignedMax(), Flt, Mode);
6191 break;
6192 }
6193 case Intrinsic::arithmetic_fence: {
6194 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
6195 Known, Q, Depth: Depth + 1);
6196 break;
6197 }
6198 case Intrinsic::experimental_constrained_sitofp:
6199 case Intrinsic::experimental_constrained_uitofp:
6200 // Cannot produce nan
6201 Known.knownNot(RuleOut: fcNan);
6202
6203 // sitofp and uitofp turn into +0.0 for zero.
6204 Known.knownNot(RuleOut: fcNegZero);
6205
6206 // Integers cannot be subnormal
6207 Known.knownNot(RuleOut: fcSubnormal);
6208
6209 if (IID == Intrinsic::experimental_constrained_uitofp)
6210 Known.signBitMustBeZero();
6211
6212 // TODO: Copy inf handling from instructions
6213 break;
6214
6215 case Intrinsic::amdgcn_fract: {
6216 Known.knownNot(RuleOut: fcInf);
6217
6218 if (InterestedClasses & fcNan) {
6219 KnownFPClass KnownSrc;
6220 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
6221 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
6222
6223 if (KnownSrc.isKnownNeverInfOrNaN())
6224 Known.knownNot(RuleOut: fcNan);
6225 else if (KnownSrc.isKnownNever(Mask: fcSNan))
6226 Known.knownNot(RuleOut: fcSNan);
6227 }
6228
6229 break;
6230 }
6231 case Intrinsic::amdgcn_rcp: {
6232 KnownFPClass KnownSrc;
6233 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
6234 Known&: KnownSrc, Q, Depth: Depth + 1);
6235
6236 Known.propagateNonNaN(Src: KnownSrc);
6237
6238 Type *EltTy = II->getType()->getScalarType();
6239
6240 // f32 denormal always flushed.
6241 if (EltTy->isFloatTy()) {
6242 Known.knownNot(RuleOut: fcSubnormal);
6243 KnownSrc.knownNot(RuleOut: fcSubnormal);
6244 }
6245
6246 if (KnownSrc.isKnownNever(Mask: fcNegative))
6247 Known.knownNot(RuleOut: fcNegative);
6248 if (KnownSrc.isKnownNever(Mask: fcPositive))
6249 Known.knownNot(RuleOut: fcPositive);
6250
6251 if (const Function *F = II->getFunction()) {
6252 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
6253 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
6254 Known.knownNot(RuleOut: fcPosInf);
6255 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
6256 Known.knownNot(RuleOut: fcNegInf);
6257 }
6258
6259 break;
6260 }
6261 case Intrinsic::amdgcn_rsq: {
6262 KnownFPClass KnownSrc;
6263 // The only negative value that can be returned is -inf for -0 inputs.
6264 Known.knownNot(RuleOut: fcNegZero | fcNegSubnormal | fcNegNormal);
6265
6266 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
6267 Known&: KnownSrc, Q, Depth: Depth + 1);
6268
6269 // Negative -> nan
6270 if (KnownSrc.isKnownNeverNaN() && KnownSrc.cannotBeOrderedLessThanZero())
6271 Known.knownNot(RuleOut: fcNan);
6272 else if (KnownSrc.isKnownNever(Mask: fcSNan))
6273 Known.knownNot(RuleOut: fcSNan);
6274
6275 // +inf -> +0
6276 if (KnownSrc.isKnownNeverPosInfinity())
6277 Known.knownNot(RuleOut: fcPosZero);
6278
6279 Type *EltTy = II->getType()->getScalarType();
6280
6281 // f32 denormal always flushed.
6282 if (EltTy->isFloatTy())
6283 Known.knownNot(RuleOut: fcPosSubnormal);
6284
6285 if (const Function *F = II->getFunction()) {
6286 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
6287
6288 // -0 -> -inf
6289 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
6290 Known.knownNot(RuleOut: fcNegInf);
6291
6292 // +0 -> +inf
6293 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
6294 Known.knownNot(RuleOut: fcPosInf);
6295 }
6296
6297 break;
6298 }
6299 case Intrinsic::amdgcn_trig_preop: {
6300 // Always returns a value [0, 1)
6301 Known.knownNot(RuleOut: fcNan | fcInf | fcNegative);
6302 break;
6303 }
6304 case Intrinsic::convert_from_arbitrary_fp: {
6305 auto *MD = cast<MetadataAsValue>(Val: II->getArgOperand(i: 1))->getMetadata();
6306 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
6307
6308 const fltSemantics *SrcSemantics =
6309 APFloat::getArbitraryFPSemantics(Format: FormatStr);
6310 if (!SrcSemantics)
6311 break;
6312
6313 const fltSemantics DstSemantics =
6314 II->getType()->getScalarType()->getFltSemantics();
6315
6316 if (!APFloat::semanticsHasNaN(*SrcSemantics))
6317 Known.knownNot(RuleOut: fcNan);
6318
6319 // fcInf can only be cleared if the source format has no Inf encoding
6320 // and the dst max exp can accommodate src max exp.
6321 if (!APFloat::semanticsHasInf(*SrcSemantics) &&
6322 APFloat::semanticsMaxExponent(*SrcSemantics) <=
6323 APFloat::semanticsMaxExponent(DstSemantics))
6324 Known.knownNot(RuleOut: fcInf);
6325
6326 // Check and clear all neg flags for formats that do not have signed
6327 // representation.
6328 if (!APFloat::semanticsHasSignedRepr(*SrcSemantics))
6329 Known.knownNot(RuleOut: fcNegative);
6330
6331 // Check if format has no zero at all (Float8E8M0FNU), or no negative
6332 // zero.
6333 if (!APFloat::semanticsHasZero(*SrcSemantics))
6334 Known.knownNot(RuleOut: fcZero);
6335 else if (SrcSemantics->nanEncoding == fltNanEncoding::NegativeZero)
6336 Known.knownNot(RuleOut: fcNegZero);
6337
6338 // If src lands normally in dest, the result can never be subnormal.
6339 if (APFloat::isRepresentableAsNormalIn(Src: *SrcSemantics, Dst: DstSemantics))
6340 Known.knownNot(RuleOut: fcSubnormal);
6341 break;
6342 }
6343 default:
6344 break;
6345 }
6346
6347 break;
6348 }
6349 case Instruction::FAdd:
6350 case Instruction::FSub: {
6351 computeKnownFPClassForFAddSub(Op, DemandedElts, InterestedClasses, Known, Q,
6352 Depth, /*IsAdd=*/Opc == Instruction::FAdd,
6353 /*IsStrict=*/false);
6354 break;
6355 }
6356 case Instruction::FMul: {
6357 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6358 DenormalMode Mode =
6359 F ? F->getDenormalMode(
6360 FPType: Op->getType()->getScalarType()->getFltSemantics())
6361 : DenormalMode::getDynamic();
6362
6363 Value *LHS = Op->getOperand(i: 0);
6364 Value *RHS = Op->getOperand(i: 1);
6365 // X * X is always non-negative or a NaN.
6366 // FIXME: Should check isGuaranteedNotToBeUndef
6367 if (LHS == RHS) {
6368 KnownFPClass KnownSrc;
6369 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownSrc, Q,
6370 Depth: Depth + 1);
6371 Known = KnownFPClass::square(Src: KnownSrc, Mode);
6372 break;
6373 }
6374
6375 KnownFPClass KnownLHS, KnownRHS;
6376
6377 const APFloat *CRHS;
6378 if (match(V: RHS, P: m_APFloat(Res&: CRHS))) {
6379 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
6380 Depth: Depth + 1);
6381 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: *CRHS, Mode);
6382 } else {
6383 computeKnownFPClass(V: RHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS, Q,
6384 Depth: Depth + 1);
6385 // TODO: Improve accuracy in unfused FMA pattern. We can prove an
6386 // additional not-nan if the addend is known-not negative infinity if the
6387 // multiply is known-not infinity.
6388
6389 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
6390 Depth: Depth + 1);
6391 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: KnownRHS, Mode);
6392 }
6393
6394 /// Propgate no-infs if the other source is known smaller than one, such
6395 /// that this cannot introduce overflow.
6396 if (KnownLHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: RHS))
6397 Known.knownNot(RuleOut: fcInf);
6398 else if (KnownRHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: LHS))
6399 Known.knownNot(RuleOut: fcInf);
6400
6401 break;
6402 }
6403 case Instruction::FDiv: {
6404 const bool WantNan = (InterestedClasses & fcNan) != fcNone;
6405
6406 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6407 const fltSemantics &FltSem =
6408 Op->getType()->getScalarType()->getFltSemantics();
6409 DenormalMode Mode =
6410 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6411
6412 if (Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
6413 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT)) {
6414 // X / X is always exactly 1.0 or a NaN.
6415 Known.setKnownFPClasses(fcNan | fcPosNormal);
6416
6417 if (!WantNan)
6418 break;
6419
6420 KnownFPClass KnownSrc;
6421 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts,
6422 InterestedClasses: fcNan | fcInf | fcZero | fcSubnormal, Known&: KnownSrc, Q,
6423 Depth: Depth + 1);
6424
6425 Known = KnownFPClass::fdiv_self(Src: KnownSrc, Mode);
6426 break;
6427 }
6428
6429 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
6430 const bool WantPositive = (InterestedClasses & fcPositive) != fcNone;
6431 if (!WantNan && !WantNegative && !WantPositive)
6432 break;
6433
6434 KnownFPClass KnownLHS, KnownRHS;
6435 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS,
6436 Q, Depth: Depth + 1);
6437
6438 bool KnowSomethingUseful =
6439 KnownRHS.isKnownNever(Mask: fcSNan) ||
6440 KnownRHS.isKnownNever(Mask: fcNegNormal | fcNegSubnormal) ||
6441 KnownRHS.isKnownNever(Mask: fcPosNormal | fcPosSubnormal);
6442
6443 if (KnowSomethingUseful)
6444 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS,
6445 Q, Depth: Depth + 1);
6446
6447 Known = KnownFPClass::fdiv(LHS: KnownLHS, RHS: KnownRHS, Mode);
6448 break;
6449 }
6450 case Instruction::FRem: {
6451 computeKnownFPClassForFRem(Op, DemandedElts, InterestedClasses, Known, Q,
6452 Depth);
6453 break;
6454 }
6455 case Instruction::FPExt: {
6456 KnownFPClass KnownSrc;
6457 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
6458 Known&: KnownSrc, Q, Depth: Depth + 1);
6459
6460 const fltSemantics &DstTy =
6461 Op->getType()->getScalarType()->getFltSemantics();
6462 const fltSemantics &SrcTy =
6463 Op->getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
6464
6465 const auto *I = dyn_cast<Instruction>(Val: Op);
6466 const Function *F = I ? I->getFunction() : nullptr;
6467 DenormalMode DstMode =
6468 F ? F->getDenormalMode(FPType: DstTy) : DenormalMode::getDynamic();
6469 DenormalMode SrcMode =
6470 F ? F->getDenormalMode(FPType: SrcTy) : DenormalMode::getDynamic();
6471
6472 Known = KnownFPClass::fpext(KnownSrc, DstTy, SrcTy, DstMode, SrcMode);
6473 break;
6474 }
6475 case Instruction::FPTrunc: {
6476 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known, Q,
6477 Depth);
6478 break;
6479 }
6480 case Instruction::SIToFP:
6481 case Instruction::UIToFP: {
6482 // Cannot produce nan
6483 Known.knownNot(RuleOut: fcNan);
6484
6485 // Integers cannot be subnormal
6486 Known.knownNot(RuleOut: fcSubnormal);
6487
6488 // sitofp and uitofp turn into +0.0 for zero.
6489 Known.knownNot(RuleOut: fcNegZero);
6490
6491 // UIToFP is always non-negative regardless of known bits.
6492 if (Op->getOpcode() == Instruction::UIToFP)
6493 Known.signBitMustBeZero();
6494
6495 // Only compute known bits if we can learn something useful from them.
6496 if (!(InterestedClasses & (fcPosZero | fcNormal | fcInf)))
6497 break;
6498
6499 KnownBits IntKnown =
6500 computeKnownBits(V: Op->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
6501
6502 // If the integer is non-zero, the result cannot be +0.0
6503 if (IntKnown.isNonZero())
6504 Known.knownNot(RuleOut: fcPosZero);
6505
6506 if (Op->getOpcode() == Instruction::SIToFP) {
6507 // If the signed integer is known non-negative, the result is
6508 // non-negative. If the signed integer is known negative, the result is
6509 // negative.
6510 if (IntKnown.isNonNegative()) {
6511 Known.signBitMustBeZero();
6512 } else if (IntKnown.isNegative()) {
6513 Known.signBitMustBeOne();
6514 }
6515 }
6516
6517 // Guard kept for ilogb()
6518 if (InterestedClasses & fcInf) {
6519 // Get width of largest magnitude integer known.
6520 // This still works for a signed minimum value because the largest FP
6521 // value is scaled by some fraction close to 2.0 (1.0 + 0.xxxx).
6522 int IntSize = IntKnown.getBitWidth();
6523 if (Op->getOpcode() == Instruction::UIToFP)
6524 IntSize -= IntKnown.countMinLeadingZeros();
6525 else if (Op->getOpcode() == Instruction::SIToFP)
6526 IntSize -= IntKnown.countMinSignBits();
6527
6528 // If the exponent of the largest finite FP value can hold the largest
6529 // integer, the result of the cast must be finite.
6530 Type *FPTy = Op->getType()->getScalarType();
6531 if (ilogb(Arg: APFloat::getLargest(Sem: FPTy->getFltSemantics())) >= IntSize)
6532 Known.knownNot(RuleOut: fcInf);
6533 }
6534
6535 break;
6536 }
6537 case Instruction::ExtractElement: {
6538 // Look through extract element. If the index is non-constant or
6539 // out-of-range demand all elements, otherwise just the extracted element.
6540 const Value *Vec = Op->getOperand(i: 0);
6541
6542 APInt DemandedVecElts;
6543 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
6544 unsigned NumElts = VecTy->getNumElements();
6545 DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
6546 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1));
6547 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
6548 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
6549 } else {
6550 DemandedVecElts = APInt(1, 1);
6551 }
6552
6553 return computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known,
6554 Q, Depth: Depth + 1);
6555 }
6556 case Instruction::InsertElement: {
6557 if (isa<ScalableVectorType>(Val: Op->getType()))
6558 return;
6559
6560 const Value *Vec = Op->getOperand(i: 0);
6561 const Value *Elt = Op->getOperand(i: 1);
6562 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 2));
6563 unsigned NumElts = DemandedElts.getBitWidth();
6564 APInt DemandedVecElts = DemandedElts;
6565 bool NeedsElt = true;
6566 // If we know the index we are inserting to, clear it from Vec check.
6567 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
6568 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
6569 NeedsElt = DemandedElts[CIdx->getZExtValue()];
6570 }
6571
6572 // Do we demand the inserted element?
6573 if (NeedsElt) {
6574 computeKnownFPClass(V: Elt, Known, InterestedClasses, Q, Depth: Depth + 1);
6575 // If we don't know any bits, early out.
6576 if (Known.isUnknown())
6577 break;
6578 } else {
6579 Known = KnownFPClass::getEmpty();
6580 }
6581
6582 // Do we need anymore elements from Vec?
6583 if (!DemandedVecElts.isZero()) {
6584 KnownFPClass Known2;
6585 computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known&: Known2, Q,
6586 Depth: Depth + 1);
6587 Known |= Known2;
6588 }
6589
6590 break;
6591 }
6592 case Instruction::ShuffleVector: {
6593 // Handle vector splat idiom
6594 if (Value *Splat = getSplatValue(V)) {
6595 computeKnownFPClass(V: Splat, Known, InterestedClasses, Q, Depth: Depth + 1);
6596 break;
6597 }
6598
6599 // For undef elements, we don't know anything about the common state of
6600 // the shuffle result.
6601 APInt DemandedLHS, DemandedRHS;
6602 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: Op);
6603 if (!Shuf || !getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
6604 return;
6605
6606 if (!!DemandedLHS) {
6607 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
6608 computeKnownFPClass(V: LHS, DemandedElts: DemandedLHS, InterestedClasses, Known, Q,
6609 Depth: Depth + 1);
6610
6611 // If we don't know any bits, early out.
6612 if (Known.isUnknown())
6613 break;
6614 } else {
6615 Known = KnownFPClass::getEmpty();
6616 }
6617
6618 if (!!DemandedRHS) {
6619 KnownFPClass Known2;
6620 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
6621 computeKnownFPClass(V: RHS, DemandedElts: DemandedRHS, InterestedClasses, Known&: Known2, Q,
6622 Depth: Depth + 1);
6623 Known |= Known2;
6624 }
6625
6626 break;
6627 }
6628 case Instruction::ExtractValue: {
6629 const ExtractValueInst *Extract = cast<ExtractValueInst>(Val: Op);
6630 ArrayRef<unsigned> Indices = Extract->getIndices();
6631 const Value *Src = Extract->getAggregateOperand();
6632 if (isa<StructType>(Val: Src->getType()) && Indices.size() == 1 &&
6633 Indices[0] == 0) {
6634 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Src)) {
6635 switch (II->getIntrinsicID()) {
6636 case Intrinsic::frexp: {
6637 FPClassTest InterestedSrcs = InterestedClasses;
6638
6639 // Positive subnormals and negative subnormals could become positive
6640 // zero.
6641 if (InterestedClasses & fcPosZero)
6642 InterestedSrcs |= fcSubnormal;
6643
6644 // Negative subnormals could become negative zero.
6645 if (InterestedClasses & fcNegZero)
6646 InterestedSrcs |= fcNegSubnormal;
6647
6648 if (InterestedClasses & fcPosNormal)
6649 InterestedSrcs |= fcPosSubnormal;
6650
6651 if (InterestedClasses & fcNegNormal)
6652 InterestedSrcs |= fcNegSubnormal;
6653
6654 KnownFPClass KnownSrc;
6655 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
6656 InterestedClasses: InterestedSrcs, Known&: KnownSrc, Q, Depth: Depth + 1);
6657
6658 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6659 const fltSemantics &FltSem =
6660 Op->getType()->getScalarType()->getFltSemantics();
6661
6662 DenormalMode Mode =
6663 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6664 Known = KnownFPClass::frexp_mant(Src: KnownSrc, Mode);
6665 return;
6666 }
6667 default:
6668 break;
6669 }
6670 }
6671 }
6672
6673 computeKnownFPClass(V: Src, DemandedElts, InterestedClasses, Known, Q,
6674 Depth: Depth + 1);
6675 break;
6676 }
6677 case Instruction::PHI: {
6678 const PHINode *P = cast<PHINode>(Val: Op);
6679 // Unreachable blocks may have zero-operand PHI nodes.
6680 if (P->getNumIncomingValues() == 0)
6681 break;
6682
6683 // Otherwise take the unions of the known bit sets of the operands,
6684 // taking conservative care to avoid excessive recursion.
6685 const unsigned PhiRecursionLimit = MaxAnalysisRecursionDepth - 2;
6686
6687 if (Depth < PhiRecursionLimit) {
6688 // Skip if every incoming value references to ourself.
6689 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
6690 break;
6691
6692 bool First = true;
6693
6694 for (const Use &U : P->operands()) {
6695 Value *IncValue;
6696 Instruction *CtxI;
6697 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CtxI);
6698 // Skip direct self references.
6699 if (IncValue == P)
6700 continue;
6701
6702 KnownFPClass KnownSrc;
6703 // Recurse, but cap the recursion to two levels, because we don't want
6704 // to waste time spinning around in loops. We need at least depth 2 to
6705 // detect known sign bits.
6706 computeKnownFPClass(V: IncValue, DemandedElts, InterestedClasses, Known&: KnownSrc,
6707 Q: Q.getWithoutCondContext().getWithInstruction(I: CtxI),
6708 Depth: PhiRecursionLimit);
6709
6710 if (First) {
6711 Known = KnownSrc;
6712 First = false;
6713 } else {
6714 Known |= KnownSrc;
6715 }
6716
6717 if (Known.getKnownFPClasses() == fcAllFlags)
6718 break;
6719 }
6720 }
6721
6722 // Look for the case of a for loop which has a positive
6723 // initial value and is incremented by a squared value.
6724 // This will propagate sign information out of such loops.
6725 if (P->getNumIncomingValues() != 2 || Known.cannotBeOrderedLessThanZero())
6726 break;
6727 for (unsigned I = 0; I < 2; I++) {
6728 Value *RecurValue = P->getIncomingValue(i: 1 - I);
6729 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: RecurValue);
6730 if (!II)
6731 continue;
6732 Value *R, *L, *Init;
6733 PHINode *PN;
6734 if (matchSimpleTernaryIntrinsicRecurrence(I: II, P&: PN, Init, OtherOp0&: L, OtherOp1&: R) &&
6735 PN == P) {
6736 switch (II->getIntrinsicID()) {
6737 case Intrinsic::fma:
6738 case Intrinsic::fmuladd: {
6739 KnownFPClass KnownStart;
6740 computeKnownFPClass(V: Init, DemandedElts, InterestedClasses, Known&: KnownStart,
6741 Q, Depth: Depth + 1);
6742 if (KnownStart.cannotBeOrderedLessThanZero() && L == R &&
6743 isGuaranteedNotToBeUndef(V: L, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1))
6744 Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
6745 break;
6746 }
6747 }
6748 }
6749 }
6750 break;
6751 }
6752 case Instruction::BitCast: {
6753 const Value *Src;
6754 if (!match(V: Op, P: m_ElementWiseBitCast(Op: m_Value(V&: Src))) ||
6755 !Src->getType()->isIntOrIntVectorTy())
6756 break;
6757
6758 const Type *Ty = Op->getType();
6759
6760 Value *CastLHS, *CastRHS;
6761
6762 // Match bitcast(umax(bitcast(a), bitcast(b)))
6763 if (match(V: Src, P: m_c_MaxOrMin(L: m_BitCast(Op: m_Value(V&: CastLHS)),
6764 R: m_BitCast(Op: m_Value(V&: CastRHS)))) &&
6765 CastLHS->getType() == Ty && CastRHS->getType() == Ty) {
6766 KnownFPClass KnownLHS, KnownRHS;
6767 computeKnownFPClass(V: CastRHS, DemandedElts, InterestedClasses, Known&: KnownRHS, Q,
6768 Depth: Depth + 1);
6769 if (!KnownRHS.isUnknown()) {
6770 computeKnownFPClass(V: CastLHS, DemandedElts, InterestedClasses, Known&: KnownLHS,
6771 Q, Depth: Depth + 1);
6772 Known = KnownLHS | KnownRHS;
6773 }
6774
6775 return;
6776 }
6777
6778 const Type *EltTy = Ty->getScalarType();
6779 KnownBits Bits(EltTy->getPrimitiveSizeInBits());
6780 computeKnownBits(V: Src, DemandedElts, Known&: Bits, Q, Depth: Depth + 1);
6781
6782 Known = KnownFPClass::bitcast(FltSemantics: EltTy->getFltSemantics(), Bits);
6783 break;
6784 }
6785 default:
6786 break;
6787 }
6788}
6789
6790KnownFPClass llvm::computeKnownFPClass(const Value *V,
6791 const APInt &DemandedElts,
6792 FPClassTest InterestedClasses,
6793 const SimplifyQuery &SQ,
6794 unsigned Depth) {
6795 KnownFPClass KnownClasses;
6796 ::computeKnownFPClass(V, DemandedElts, InterestedClasses, Known&: KnownClasses, Q: SQ,
6797 Depth);
6798 return KnownClasses;
6799}
6800
6801KnownFPClass llvm::computeKnownFPClass(const Value *V,
6802 FPClassTest InterestedClasses,
6803 const SimplifyQuery &SQ,
6804 unsigned Depth) {
6805 KnownFPClass Known;
6806 ::computeKnownFPClass(V, Known, InterestedClasses, Q: SQ, Depth);
6807 return Known;
6808}
6809
6810KnownFPClass llvm::computeKnownFPClass(
6811 const Value *V, const DataLayout &DL, FPClassTest InterestedClasses,
6812 const TargetLibraryInfo *TLI, AssumptionCache *AC, const Instruction *CtxI,
6813 const DominatorTree *DT, bool UseInstrInfo, unsigned Depth) {
6814 return computeKnownFPClass(V, InterestedClasses,
6815 SQ: SimplifyQuery(DL, TLI, DT, AC, CtxI, UseInstrInfo),
6816 Depth);
6817}
6818
6819KnownFPClass
6820llvm::computeKnownFPClass(const Value *V, const APInt &DemandedElts,
6821 FastMathFlags FMF, FPClassTest InterestedClasses,
6822 const SimplifyQuery &SQ, unsigned Depth) {
6823 if (FMF.noNaNs())
6824 InterestedClasses &= ~fcNan;
6825 if (FMF.noInfs())
6826 InterestedClasses &= ~fcInf;
6827
6828 KnownFPClass Result =
6829 computeKnownFPClass(V, DemandedElts, InterestedClasses, SQ, Depth);
6830
6831 if (FMF.noNaNs())
6832 Result.setKnownFPClasses(Result.getKnownFPClasses() & ~fcNan);
6833 if (FMF.noInfs())
6834 Result.setKnownFPClasses(Result.getKnownFPClasses() & ~fcInf);
6835 return Result;
6836}
6837
6838KnownFPClass llvm::computeKnownFPClass(const Value *V, FastMathFlags FMF,
6839 FPClassTest InterestedClasses,
6840 const SimplifyQuery &SQ,
6841 unsigned Depth) {
6842 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
6843 APInt DemandedElts =
6844 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
6845 return computeKnownFPClass(V, DemandedElts, FMF, InterestedClasses, SQ,
6846 Depth);
6847}
6848
6849bool llvm::cannotBeNegativeZero(const Value *V, const SimplifyQuery &SQ,
6850 unsigned Depth) {
6851 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNegZero, SQ, Depth);
6852 return Known.isKnownNever(Mask: fcNegZero);
6853}
6854
6855bool llvm::cannotBeOrderedLessThanZero(const Value *V, const SimplifyQuery &SQ,
6856 unsigned Depth) {
6857 KnownFPClass Known =
6858 computeKnownFPClass(V, InterestedClasses: KnownFPClass::OrderedLessThanZeroMask, SQ, Depth);
6859 return Known.cannotBeOrderedLessThanZero();
6860}
6861
6862bool llvm::isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ,
6863 unsigned Depth) {
6864 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf, SQ, Depth);
6865 return Known.isKnownNeverInfinity();
6866}
6867
6868/// Return true if the floating-point value can never contain a NaN or infinity.
6869bool llvm::isKnownNeverInfOrNaN(const Value *V, const SimplifyQuery &SQ,
6870 unsigned Depth) {
6871 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf | fcNan, SQ, Depth);
6872 return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity();
6873}
6874
6875/// Return true if the floating-point scalar value is not a NaN or if the
6876/// floating-point vector value has no NaN elements. Return false if a value
6877/// could ever be NaN.
6878bool llvm::isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ,
6879 unsigned Depth) {
6880 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNan, SQ, Depth);
6881 return Known.isKnownNeverNaN();
6882}
6883
6884/// Return false if we can prove that the specified FP value's sign bit is 0.
6885/// Return true if we can prove that the specified FP value's sign bit is 1.
6886/// Otherwise return std::nullopt.
6887std::optional<bool> llvm::computeKnownFPSignBit(const Value *V,
6888 const SimplifyQuery &SQ,
6889 unsigned Depth) {
6890 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcAllFlags, SQ, Depth);
6891 return Known.getSignBit();
6892}
6893
6894bool llvm::canIgnoreSignBitOfZero(const Use &U) {
6895 auto *User = cast<Instruction>(Val: U.getUser());
6896 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6897 if (FPOp->hasNoSignedZeros())
6898 return true;
6899 }
6900
6901 switch (User->getOpcode()) {
6902 case Instruction::FPToSI:
6903 case Instruction::FPToUI:
6904 return true;
6905 case Instruction::FRem:
6906 // frem(x, +-0.0) == NaN, and more generally frem(x, y) == frem(x, -y).
6907 return U.getOperandNo() == 1;
6908 case Instruction::FCmp:
6909 // fcmp treats both positive and negative zero as equal.
6910 return true;
6911 case Instruction::Call:
6912 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6913 switch (II->getIntrinsicID()) {
6914 case Intrinsic::fabs:
6915 case Intrinsic::exp:
6916 case Intrinsic::exp2:
6917 case Intrinsic::exp10:
6918 case Intrinsic::log:
6919 case Intrinsic::log2:
6920 case Intrinsic::log10:
6921 case Intrinsic::cos:
6922 case Intrinsic::acos:
6923 case Intrinsic::cosh:
6924 case Intrinsic::fptosi_sat:
6925 case Intrinsic::fptoui_sat:
6926 case Intrinsic::lround:
6927 case Intrinsic::llround:
6928 case Intrinsic::lrint:
6929 case Intrinsic::llrint:
6930 return true;
6931 case Intrinsic::copysign:
6932 return U.getOperandNo() == 0;
6933 case Intrinsic::pow:
6934 // pow(x, +-0.0) == +1.0 or NaN
6935 return U.getOperandNo() == 1;
6936 case Intrinsic::is_fpclass: {
6937 auto Test =
6938 static_cast<FPClassTest>(
6939 cast<ConstantInt>(Val: II->getArgOperand(i: 1))->getZExtValue()) &
6940 FPClassTest::fcZero;
6941 return Test == FPClassTest::fcZero || Test == FPClassTest::fcNone;
6942 }
6943 default:
6944 return false;
6945 }
6946 }
6947 return false;
6948 default:
6949 return false;
6950 }
6951}
6952
6953bool llvm::canIgnoreSignBitOfNaN(const Use &U) {
6954 auto *User = cast<Instruction>(Val: U.getUser());
6955 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6956 if (FPOp->hasNoNaNs())
6957 return true;
6958 }
6959
6960 switch (User->getOpcode()) {
6961 case Instruction::FPToSI:
6962 case Instruction::FPToUI:
6963 return true;
6964 // Proper FP math operations ignore the sign bit of NaN.
6965 case Instruction::FAdd:
6966 case Instruction::FSub:
6967 case Instruction::FMul:
6968 case Instruction::FDiv:
6969 case Instruction::FRem:
6970 case Instruction::FPTrunc:
6971 case Instruction::FPExt:
6972 case Instruction::FCmp:
6973 return true;
6974 // Bitwise FP operations should preserve the sign bit of NaN.
6975 case Instruction::FNeg:
6976 case Instruction::Select:
6977 case Instruction::PHI:
6978 return false;
6979 case Instruction::Ret:
6980 return User->getFunction()->getAttributes().getRetNoFPClass() &
6981 FPClassTest::fcNan;
6982 case Instruction::Call:
6983 case Instruction::Invoke: {
6984 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6985 switch (II->getIntrinsicID()) {
6986 case Intrinsic::fabs:
6987 return true;
6988 case Intrinsic::copysign:
6989 return U.getOperandNo() == 0;
6990 // Other proper FP math intrinsics ignore the sign bit of NaN.
6991 case Intrinsic::maxnum:
6992 case Intrinsic::minnum:
6993 case Intrinsic::maximum:
6994 case Intrinsic::minimum:
6995 case Intrinsic::maximumnum:
6996 case Intrinsic::minimumnum:
6997 case Intrinsic::canonicalize:
6998 case Intrinsic::fma:
6999 case Intrinsic::fmuladd:
7000 case Intrinsic::sqrt:
7001 case Intrinsic::pow:
7002 case Intrinsic::powi:
7003 case Intrinsic::fptoui_sat:
7004 case Intrinsic::fptosi_sat:
7005 case Intrinsic::is_fpclass:
7006 return true;
7007 default:
7008 return false;
7009 }
7010 }
7011
7012 FPClassTest NoFPClass =
7013 cast<CallBase>(Val: User)->getParamNoFPClass(i: U.getOperandNo());
7014 return NoFPClass & FPClassTest::fcNan;
7015 }
7016 default:
7017 return false;
7018 }
7019}
7020
7021bool llvm::isKnownIntegral(const Value *V, const SimplifyQuery &SQ,
7022 FastMathFlags FMF) {
7023 if (isa<PoisonValue>(Val: V))
7024 return true;
7025 if (isa<UndefValue>(Val: V))
7026 return false;
7027
7028 if (match(V, P: m_CheckedFp(CheckFn: [](const APFloat &Val) { return Val.isInteger(); })))
7029 return true;
7030
7031 const Instruction *I = dyn_cast<Instruction>(Val: V);
7032 if (!I)
7033 return false;
7034
7035 switch (I->getOpcode()) {
7036 case Instruction::SIToFP:
7037 case Instruction::UIToFP:
7038 // TODO: Could check nofpclass(inf) on incoming argument
7039 if (FMF.noInfs())
7040 return true;
7041
7042 // Need to check int size cannot produce infinity, which computeKnownFPClass
7043 // knows how to do already.
7044 return isKnownNeverInfinity(V: I, SQ);
7045 case Instruction::Call: {
7046 const CallInst *CI = cast<CallInst>(Val: I);
7047 switch (CI->getIntrinsicID()) {
7048 case Intrinsic::trunc:
7049 case Intrinsic::floor:
7050 case Intrinsic::ceil:
7051 case Intrinsic::rint:
7052 case Intrinsic::nearbyint:
7053 case Intrinsic::round:
7054 case Intrinsic::roundeven:
7055 return (FMF.noInfs() && FMF.noNaNs()) || isKnownNeverInfOrNaN(V: I, SQ);
7056 default:
7057 break;
7058 }
7059
7060 break;
7061 }
7062 default:
7063 break;
7064 }
7065
7066 return false;
7067}
7068
7069Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
7070
7071 // All byte-wide stores are splatable, even of arbitrary variables.
7072 if (V->getType()->isIntegerTy(BitWidth: 8))
7073 return V;
7074
7075 LLVMContext &Ctx = V->getContext();
7076
7077 // Undef don't care.
7078 auto *UndefInt8 = UndefValue::get(T: Type::getInt8Ty(C&: Ctx));
7079 if (isa<UndefValue>(Val: V))
7080 return UndefInt8;
7081
7082 // Return poison for zero-sized type.
7083 if (DL.getTypeStoreSize(Ty: V->getType()).isZero())
7084 return PoisonValue::get(T: Type::getInt8Ty(C&: Ctx));
7085
7086 Constant *C = dyn_cast<Constant>(Val: V);
7087 if (!C) {
7088 // Conceptually, we could handle things like:
7089 // %a = zext i8 %X to i16
7090 // %b = shl i16 %a, 8
7091 // %c = or i16 %a, %b
7092 // but until there is an example that actually needs this, it doesn't seem
7093 // worth worrying about.
7094 return nullptr;
7095 }
7096
7097 // Handle 'null' ConstantArrayZero etc.
7098 if (C->isNullValue())
7099 return Constant::getNullValue(Ty: Type::getInt8Ty(C&: Ctx));
7100
7101 // Constant floating-point values can be handled as integer values if the
7102 // corresponding integer value is "byteable". An important case is 0.0.
7103 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
7104 Type *ScalarTy = CFP->getType()->getScalarType();
7105 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
7106 return isBytewiseValue(
7107 V: ConstantInt::get(Context&: Ctx, V: CFP->getValue().bitcastToAPInt()), DL);
7108
7109 // Don't handle long double formats, which have strange constraints.
7110 return nullptr;
7111 }
7112
7113 // We can handle constant integers that are multiple of 8 bits.
7114 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
7115 if (CI->getBitWidth() % 8 == 0) {
7116 if (!CI->getValue().isSplat(SplatSizeInBits: 8))
7117 return nullptr;
7118 return ConstantInt::get(Context&: Ctx, V: CI->getValue().trunc(width: 8));
7119 }
7120 }
7121
7122 if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
7123 if (CE->getOpcode() == Instruction::IntToPtr) {
7124 if (auto *PtrTy = dyn_cast<PointerType>(Val: CE->getType())) {
7125 unsigned BitWidth = DL.getPointerSizeInBits(AS: PtrTy->getAddressSpace());
7126 if (Constant *Op = ConstantFoldIntegerCast(
7127 C: CE->getOperand(i_nocapture: 0), DestTy: Type::getIntNTy(C&: Ctx, N: BitWidth), IsSigned: false, DL))
7128 return isBytewiseValue(V: Op, DL);
7129 }
7130 }
7131 }
7132
7133 auto Merge = [&](Value *LHS, Value *RHS) -> Value * {
7134 if (LHS == RHS)
7135 return LHS;
7136 if (!LHS || !RHS)
7137 return nullptr;
7138 if (LHS == UndefInt8)
7139 return RHS;
7140 if (RHS == UndefInt8)
7141 return LHS;
7142 return nullptr;
7143 };
7144
7145 if (ConstantDataSequential *CA = dyn_cast<ConstantDataSequential>(Val: C)) {
7146 Value *Val = UndefInt8;
7147 for (uint64_t I = 0, E = CA->getNumElements(); I != E; ++I)
7148 if (!(Val = Merge(Val, isBytewiseValue(V: CA->getElementAsConstant(i: I), DL))))
7149 return nullptr;
7150 return Val;
7151 }
7152
7153 if (isa<ConstantAggregate>(Val: C)) {
7154 Value *Val = UndefInt8;
7155 for (Value *Op : C->operands())
7156 if (!(Val = Merge(Val, isBytewiseValue(V: Op, DL))))
7157 return nullptr;
7158 return Val;
7159 }
7160
7161 // Don't try to handle the handful of other constants.
7162 return nullptr;
7163}
7164
7165// This is the recursive version of BuildSubAggregate. It takes a few different
7166// arguments. Idxs is the index within the nested struct From that we are
7167// looking at now (which is of type IndexedType). IdxSkip is the number of
7168// indices from Idxs that should be left out when inserting into the resulting
7169// struct. To is the result struct built so far, new insertvalue instructions
7170// build on that.
7171static Value *BuildSubAggregate(Value *From, Value *To, Type *IndexedType,
7172 SmallVectorImpl<unsigned> &Idxs,
7173 unsigned IdxSkip,
7174 BasicBlock::iterator InsertBefore) {
7175 StructType *STy = dyn_cast<StructType>(Val: IndexedType);
7176 if (STy) {
7177 // Save the original To argument so we can modify it
7178 Value *OrigTo = To;
7179 // General case, the type indexed by Idxs is a struct
7180 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
7181 // Process each struct element recursively
7182 Idxs.push_back(Elt: i);
7183 Value *PrevTo = To;
7184 To = BuildSubAggregate(From, To, IndexedType: STy->getElementType(N: i), Idxs, IdxSkip,
7185 InsertBefore);
7186 Idxs.pop_back();
7187 if (!To) {
7188 // Couldn't find any inserted value for this index? Cleanup
7189 while (PrevTo != OrigTo) {
7190 InsertValueInst* Del = cast<InsertValueInst>(Val: PrevTo);
7191 PrevTo = Del->getAggregateOperand();
7192 Del->eraseFromParent();
7193 }
7194 // Stop processing elements
7195 break;
7196 }
7197 }
7198 // If we successfully found a value for each of our subaggregates
7199 if (To)
7200 return To;
7201 }
7202 // Base case, the type indexed by SourceIdxs is not a struct, or not all of
7203 // the struct's elements had a value that was inserted directly. In the latter
7204 // case, perhaps we can't determine each of the subelements individually, but
7205 // we might be able to find the complete struct somewhere.
7206
7207 // Find the value that is at that particular spot
7208 Value *V = FindInsertedValue(V: From, idx_range: Idxs);
7209
7210 if (!V)
7211 return nullptr;
7212
7213 // Insert the value in the new (sub) aggregate
7214 return InsertValueInst::Create(Agg: To, Val: V, Idxs: ArrayRef(Idxs).slice(N: IdxSkip), NameStr: "tmp",
7215 InsertBefore);
7216}
7217
7218// This helper takes a nested struct and extracts a part of it (which is again a
7219// struct) into a new value. For example, given the struct:
7220// { a, { b, { c, d }, e } }
7221// and the indices "1, 1" this returns
7222// { c, d }.
7223//
7224// It does this by inserting an insertvalue for each element in the resulting
7225// struct, as opposed to just inserting a single struct. This will only work if
7226// each of the elements of the substruct are known (ie, inserted into From by an
7227// insertvalue instruction somewhere).
7228//
7229// All inserted insertvalue instructions are inserted before InsertBefore
7230static Value *BuildSubAggregate(Value *From, ArrayRef<unsigned> idx_range,
7231 BasicBlock::iterator InsertBefore) {
7232 Type *IndexedType = ExtractValueInst::getIndexedType(Agg: From->getType(),
7233 Idxs: idx_range);
7234 Value *To = PoisonValue::get(T: IndexedType);
7235 SmallVector<unsigned, 10> Idxs(idx_range);
7236 unsigned IdxSkip = Idxs.size();
7237
7238 return BuildSubAggregate(From, To, IndexedType, Idxs, IdxSkip, InsertBefore);
7239}
7240
7241/// Given an aggregate and a sequence of indices, see if the scalar value
7242/// indexed is already around as a register, for example if it was inserted
7243/// directly into the aggregate.
7244///
7245/// If InsertBefore is not null, this function will duplicate (modified)
7246/// insertvalues when a part of a nested struct is extracted.
7247Value *
7248llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
7249 std::optional<BasicBlock::iterator> InsertBefore) {
7250 // Nothing to index? Just return V then (this is useful at the end of our
7251 // recursion).
7252 if (idx_range.empty())
7253 return V;
7254 // We have indices, so V should have an indexable type.
7255 assert((V->getType()->isStructTy() || V->getType()->isArrayTy()) &&
7256 "Not looking at a struct or array?");
7257 assert(ExtractValueInst::getIndexedType(V->getType(), idx_range) &&
7258 "Invalid indices for type?");
7259
7260 if (Constant *C = dyn_cast<Constant>(Val: V)) {
7261 C = C->getAggregateElement(Elt: idx_range[0]);
7262 if (!C) return nullptr;
7263 return FindInsertedValue(V: C, idx_range: idx_range.slice(N: 1), InsertBefore);
7264 }
7265
7266 if (InsertValueInst *I = dyn_cast<InsertValueInst>(Val: V)) {
7267 // Loop the indices for the insertvalue instruction in parallel with the
7268 // requested indices
7269 const unsigned *req_idx = idx_range.begin();
7270 for (const unsigned *i = I->idx_begin(), *e = I->idx_end();
7271 i != e; ++i, ++req_idx) {
7272 if (req_idx == idx_range.end()) {
7273 // We can't handle this without inserting insertvalues
7274 if (!InsertBefore)
7275 return nullptr;
7276
7277 // The requested index identifies a part of a nested aggregate. Handle
7278 // this specially. For example,
7279 // %A = insertvalue { i32, {i32, i32 } } undef, i32 10, 1, 0
7280 // %B = insertvalue { i32, {i32, i32 } } %A, i32 11, 1, 1
7281 // %C = extractvalue {i32, { i32, i32 } } %B, 1
7282 // This can be changed into
7283 // %A = insertvalue {i32, i32 } undef, i32 10, 0
7284 // %C = insertvalue {i32, i32 } %A, i32 11, 1
7285 // which allows the unused 0,0 element from the nested struct to be
7286 // removed.
7287 return BuildSubAggregate(From: V, idx_range: ArrayRef(idx_range.begin(), req_idx),
7288 InsertBefore: *InsertBefore);
7289 }
7290
7291 // This insert value inserts something else than what we are looking for.
7292 // See if the (aggregate) value inserted into has the value we are
7293 // looking for, then.
7294 if (*req_idx != *i)
7295 return FindInsertedValue(V: I->getAggregateOperand(), idx_range,
7296 InsertBefore);
7297 }
7298 // If we end up here, the indices of the insertvalue match with those
7299 // requested (though possibly only partially). Now we recursively look at
7300 // the inserted value, passing any remaining indices.
7301 return FindInsertedValue(V: I->getInsertedValueOperand(),
7302 idx_range: ArrayRef(req_idx, idx_range.end()), InsertBefore);
7303 }
7304
7305 if (ExtractValueInst *I = dyn_cast<ExtractValueInst>(Val: V)) {
7306 // If we're extracting a value from an aggregate that was extracted from
7307 // something else, we can extract from that something else directly instead.
7308 // However, we will need to chain I's indices with the requested indices.
7309
7310 // Calculate the number of indices required
7311 unsigned size = I->getNumIndices() + idx_range.size();
7312 // Allocate some space to put the new indices in
7313 SmallVector<unsigned, 5> Idxs;
7314 Idxs.reserve(N: size);
7315 // Add indices from the extract value instruction
7316 Idxs.append(in_start: I->idx_begin(), in_end: I->idx_end());
7317
7318 // Add requested indices
7319 Idxs.append(in_start: idx_range.begin(), in_end: idx_range.end());
7320
7321 assert(Idxs.size() == size
7322 && "Number of indices added not correct?");
7323
7324 return FindInsertedValue(V: I->getAggregateOperand(), idx_range: Idxs, InsertBefore);
7325 }
7326 // Otherwise, we don't know (such as, extracting from a function return value
7327 // or load instruction)
7328 return nullptr;
7329}
7330
7331// If V refers to an initialized global constant, set Slice either to
7332// its initializer if the size of its elements equals ElementSize, or,
7333// for ElementSize == 8, to its representation as an array of unsiged
7334// char. Return true on success.
7335// Offset is in the unit "nr of ElementSize sized elements".
7336bool llvm::getConstantDataArrayInfo(const Value *V,
7337 ConstantDataArraySlice &Slice,
7338 unsigned ElementSize, uint64_t Offset) {
7339 assert(V && "V should not be null.");
7340 assert((ElementSize % 8) == 0 &&
7341 "ElementSize expected to be a multiple of the size of a byte.");
7342 unsigned ElementSizeInBytes = ElementSize / 8;
7343
7344 // Drill down into the pointer expression V, ignoring any intervening
7345 // casts, and determine the identity of the object it references along
7346 // with the cumulative byte offset into it.
7347 const GlobalVariable *GV =
7348 dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V));
7349 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
7350 // Fail if V is not based on constant global object.
7351 return false;
7352
7353 const DataLayout &DL = GV->getDataLayout();
7354 APInt Off(DL.getIndexTypeSizeInBits(Ty: V->getType()), 0);
7355
7356 if (GV != V->stripAndAccumulateConstantOffsets(DL, Offset&: Off,
7357 /*AllowNonInbounds*/ true))
7358 // Fail if a constant offset could not be determined.
7359 return false;
7360
7361 uint64_t StartIdx = Off.getLimitedValue();
7362 if (StartIdx == UINT64_MAX)
7363 // Fail if the constant offset is excessive.
7364 return false;
7365
7366 // Off/StartIdx is in the unit of bytes. So we need to convert to number of
7367 // elements. Simply bail out if that isn't possible.
7368 if ((StartIdx % ElementSizeInBytes) != 0)
7369 return false;
7370
7371 Offset += StartIdx / ElementSizeInBytes;
7372 ConstantDataArray *Array = nullptr;
7373 ArrayType *ArrayTy = nullptr;
7374
7375 if (GV->getInitializer()->isNullValue()) {
7376 Type *GVTy = GV->getValueType();
7377 uint64_t SizeInBytes = DL.getTypeStoreSize(Ty: GVTy).getFixedValue();
7378 uint64_t Length = SizeInBytes / ElementSizeInBytes;
7379
7380 Slice.Array = nullptr;
7381 Slice.Offset = 0;
7382 // Return an empty Slice for undersized constants to let callers
7383 // transform even undefined library calls into simpler, well-defined
7384 // expressions. This is preferable to making the calls although it
7385 // prevents sanitizers from detecting such calls.
7386 Slice.Length = Length < Offset ? 0 : Length - Offset;
7387 return true;
7388 }
7389
7390 auto *Init = const_cast<Constant *>(GV->getInitializer());
7391 if (auto *ArrayInit = dyn_cast<ConstantDataArray>(Val: Init)) {
7392 Type *InitElTy = ArrayInit->getElementType();
7393 if (InitElTy->isIntegerTy(BitWidth: ElementSize)) {
7394 // If Init is an initializer for an array of the expected type
7395 // and size, use it as is.
7396 Array = ArrayInit;
7397 ArrayTy = ArrayInit->getType();
7398 }
7399 }
7400
7401 if (!Array) {
7402 if (ElementSize != 8)
7403 // TODO: Handle conversions to larger integral types.
7404 return false;
7405
7406 // Otherwise extract the portion of the initializer starting
7407 // at Offset as an array of bytes, and reset Offset.
7408 Init = ReadByteArrayFromGlobal(GV, Offset);
7409 if (!Init)
7410 return false;
7411
7412 Offset = 0;
7413 Array = dyn_cast<ConstantDataArray>(Val: Init);
7414 ArrayTy = dyn_cast<ArrayType>(Val: Init->getType());
7415 }
7416
7417 uint64_t NumElts = ArrayTy->getArrayNumElements();
7418 if (Offset > NumElts)
7419 return false;
7420
7421 Slice.Array = Array;
7422 Slice.Offset = Offset;
7423 Slice.Length = NumElts - Offset;
7424 return true;
7425}
7426
7427/// Extract bytes from the initializer of the constant array V, which need
7428/// not be a nul-terminated string. On success, store the bytes in Str and
7429/// return true. When TrimAtNul is set, Str will contain only the bytes up
7430/// to but not including the first nul. Return false on failure.
7431bool llvm::getConstantStringInfo(const Value *V, StringRef &Str,
7432 bool TrimAtNul) {
7433 ConstantDataArraySlice Slice;
7434 if (!getConstantDataArrayInfo(V, Slice, ElementSize: 8))
7435 return false;
7436
7437 if (Slice.Array == nullptr) {
7438 if (TrimAtNul) {
7439 // Return a nul-terminated string even for an empty Slice. This is
7440 // safe because all existing SimplifyLibcalls callers require string
7441 // arguments and the behavior of the functions they fold is undefined
7442 // otherwise. Folding the calls this way is preferable to making
7443 // the undefined library calls, even though it prevents sanitizers
7444 // from reporting such calls.
7445 Str = StringRef();
7446 return true;
7447 }
7448 if (Slice.Length == 1) {
7449 Str = StringRef("", 1);
7450 return true;
7451 }
7452 // We cannot instantiate a StringRef as we do not have an appropriate string
7453 // of 0s at hand.
7454 return false;
7455 }
7456
7457 // Start out with the entire array in the StringRef.
7458 Str = Slice.Array->getAsString();
7459 // Skip over 'offset' bytes.
7460 Str = Str.substr(Start: Slice.Offset);
7461
7462 if (TrimAtNul) {
7463 // Trim off the \0 and anything after it. If the array is not nul
7464 // terminated, we just return the whole end of string. The client may know
7465 // some other way that the string is length-bound.
7466 Str = Str.substr(Start: 0, N: Str.find(C: '\0'));
7467 }
7468 return true;
7469}
7470
7471// These next two are very similar to the above, but also look through PHI
7472// nodes.
7473// TODO: See if we can integrate these two together.
7474
7475/// If we can compute the length of the string pointed to by
7476/// the specified pointer, return 'len+1'. If we can't, return 0.
7477static uint64_t GetStringLengthH(const Value *V,
7478 SmallPtrSetImpl<const PHINode*> &PHIs,
7479 unsigned CharSize) {
7480 // Look through noop bitcast instructions.
7481 V = V->stripPointerCasts();
7482
7483 // If this is a PHI node, there are two cases: either we have already seen it
7484 // or we haven't.
7485 if (const PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7486 if (!PHIs.insert(Ptr: PN).second)
7487 return ~0ULL; // already in the set.
7488
7489 // If it was new, see if all the input strings are the same length.
7490 uint64_t LenSoFar = ~0ULL;
7491 for (Value *IncValue : PN->incoming_values()) {
7492 uint64_t Len = GetStringLengthH(V: IncValue, PHIs, CharSize);
7493 if (Len == 0) return 0; // Unknown length -> unknown.
7494
7495 if (Len == ~0ULL) continue;
7496
7497 if (Len != LenSoFar && LenSoFar != ~0ULL)
7498 return 0; // Disagree -> unknown.
7499 LenSoFar = Len;
7500 }
7501
7502 // Success, all agree.
7503 return LenSoFar;
7504 }
7505
7506 // strlen(select(c,x,y)) -> strlen(x) ^ strlen(y)
7507 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: V)) {
7508 uint64_t Len1 = GetStringLengthH(V: SI->getTrueValue(), PHIs, CharSize);
7509 if (Len1 == 0) return 0;
7510 uint64_t Len2 = GetStringLengthH(V: SI->getFalseValue(), PHIs, CharSize);
7511 if (Len2 == 0) return 0;
7512 if (Len1 == ~0ULL) return Len2;
7513 if (Len2 == ~0ULL) return Len1;
7514 if (Len1 != Len2) return 0;
7515 return Len1;
7516 }
7517
7518 // Otherwise, see if we can read the string.
7519 ConstantDataArraySlice Slice;
7520 if (!getConstantDataArrayInfo(V, Slice, ElementSize: CharSize))
7521 return 0;
7522
7523 if (Slice.Array == nullptr)
7524 // Zeroinitializer (including an empty one).
7525 return 1;
7526
7527 // Search for the first nul character. Return a conservative result even
7528 // when there is no nul. This is safe since otherwise the string function
7529 // being folded such as strlen is undefined, and can be preferable to
7530 // making the undefined library call.
7531 unsigned NullIndex = 0;
7532 for (unsigned E = Slice.Length; NullIndex < E; ++NullIndex) {
7533 if (Slice.Array->getElementAsInteger(i: Slice.Offset + NullIndex) == 0)
7534 break;
7535 }
7536
7537 return NullIndex + 1;
7538}
7539
7540/// If we can compute the length of the string pointed to by
7541/// the specified pointer, return 'len+1'. If we can't, return 0.
7542uint64_t llvm::GetStringLength(const Value *V, unsigned CharSize) {
7543 if (!V->getType()->isPointerTy())
7544 return 0;
7545
7546 SmallPtrSet<const PHINode*, 32> PHIs;
7547 uint64_t Len = GetStringLengthH(V, PHIs, CharSize);
7548 // If Len is ~0ULL, we had an infinite phi cycle: this is dead code, so return
7549 // an empty string as a length.
7550 return Len == ~0ULL ? 1 : Len;
7551}
7552
7553const Value *
7554llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call,
7555 bool MustPreserveOffset,
7556 bool MustPreserveProvenance) {
7557 assert(Call &&
7558 "getArgumentAliasingToReturnedPointer only works on nonnull calls");
7559 if (const Value *RV = Call->getReturnedArgOperand())
7560 return RV;
7561 // This can be used only as a aliasing property.
7562 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
7563 Call, MustPreserveOffset, MustPreserveProvenance))
7564 return Call->getArgOperand(i: 0);
7565 return nullptr;
7566}
7567
7568bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
7569 const CallBase *Call, bool MustPreserveOffset,
7570 bool MustPreserveProvenance) {
7571 switch (Call->getIntrinsicID()) {
7572 case Intrinsic::launder_invariant_group:
7573 case Intrinsic::aarch64_irg:
7574 case Intrinsic::aarch64_tagp:
7575 // The amdgcn_make_buffer_rsrc function does not alter the address of the
7576 // input pointer (and thus preserves the byte offset, which is the property
7577 // the MustPreserveOffset flag selects). However, it will not necessarily
7578 // map ptr addrspace(N) null to ptr addrspace(8) null, aka the "null
7579 // descriptor", which has "all loads return 0, all stores are dropped"
7580 // semantics. Given the context of this intrinsic list, no one should be
7581 // relying on such a strict bit-exact null mapping (and, at time of
7582 // writing, they are not), but we document this fact out of an abundance
7583 // of caution.
7584 case Intrinsic::amdgcn_make_buffer_rsrc:
7585 return !MustPreserveProvenance;
7586 case Intrinsic::ptrmask:
7587 return !MustPreserveOffset;
7588 case Intrinsic::threadlocal_address:
7589 // The underlying variable changes with thread ID. The Thread ID may change
7590 // at coroutine suspend points.
7591 return !Call->getParent()->getParent()->isPresplitCoroutine();
7592 default:
7593 return false;
7594 }
7595}
7596
7597/// \p PN defines a loop-variant pointer to an object. Check if the
7598/// previous iteration of the loop was referring to the same object as \p PN.
7599static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
7600 const LoopInfo *LI) {
7601 // Find the loop-defined value.
7602 Loop *L = LI->getLoopFor(BB: PN->getParent());
7603 if (PN->getNumIncomingValues() != 2)
7604 return true;
7605
7606 // Find the value from previous iteration.
7607 auto *PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 0));
7608 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
7609 PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 1));
7610 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
7611 return true;
7612
7613 // If a new pointer is loaded in the loop, the pointer references a different
7614 // object in every iteration. E.g.:
7615 // for (i)
7616 // int *p = a[i];
7617 // ...
7618 if (auto *Load = dyn_cast<LoadInst>(Val: PrevValue))
7619 if (!L->isLoopInvariant(V: Load->getPointerOperand()))
7620 return false;
7621 return true;
7622}
7623
7624const Value *llvm::getUnderlyingObject(const Value *V, unsigned MaxLookup,
7625 bool MustPreserveProvenance) {
7626 for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
7627 if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
7628 const Value *PtrOp = GEP->getPointerOperand();
7629 if (!PtrOp->getType()->isPointerTy()) // Only handle scalar pointer base.
7630 return V;
7631 V = PtrOp;
7632 } else if (Operator::getOpcode(V) == Instruction::BitCast ||
7633 Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
7634 Value *NewV = cast<Operator>(Val: V)->getOperand(i: 0);
7635 if (!NewV->getType()->isPointerTy())
7636 return V;
7637 V = NewV;
7638 } else if (auto *GA = dyn_cast<GlobalAlias>(Val: V)) {
7639 if (GA->isInterposable())
7640 return V;
7641 V = GA->getAliasee();
7642 } else {
7643 if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
7644 // Look through single-arg phi nodes created by LCSSA.
7645 if (PHI->getNumIncomingValues() == 1) {
7646 V = PHI->getIncomingValue(i: 0);
7647 continue;
7648 }
7649 } else if (auto *Call = dyn_cast<CallBase>(Val: V)) {
7650 // CaptureTracking can know about special capturing properties of some
7651 // intrinsics like launder.invariant.group, that can't be expressed with
7652 // the attributes, but have properties like returning aliasing pointer.
7653 // Because some analysis may assume that nocaptured pointer is not
7654 // returned from some special intrinsic (because function would have to
7655 // be marked with returns attribute), it is crucial to use this function
7656 // because it should be in sync with CaptureTracking. Not using it may
7657 // cause weird miscompilations where 2 aliasing pointers are assumed to
7658 // noalias.
7659 if (auto *RP = getArgumentAliasingToReturnedPointer(
7660 Call, /*MustPreserveOffset=*/false, MustPreserveProvenance)) {
7661 V = RP;
7662 continue;
7663 }
7664 }
7665
7666 return V;
7667 }
7668 assert(V->getType()->isPointerTy() && "Unexpected operand type!");
7669 }
7670 return V;
7671}
7672
7673void llvm::getUnderlyingObjects(const Value *V,
7674 SmallVectorImpl<const Value *> &Objects,
7675 const LoopInfo *LI, unsigned MaxLookup) {
7676 SmallPtrSet<const Value *, 4> Visited;
7677 SmallVector<const Value *, 4> Worklist;
7678 Worklist.push_back(Elt: V);
7679 do {
7680 const Value *P = Worklist.pop_back_val();
7681 P = getUnderlyingObject(V: P, MaxLookup);
7682
7683 if (!Visited.insert(Ptr: P).second)
7684 continue;
7685
7686 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7687 Worklist.push_back(Elt: SI->getTrueValue());
7688 Worklist.push_back(Elt: SI->getFalseValue());
7689 continue;
7690 }
7691
7692 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7693 // If this PHI changes the underlying object in every iteration of the
7694 // loop, don't look through it. Consider:
7695 // int **A;
7696 // for (i) {
7697 // Prev = Curr; // Prev = PHI (Prev_0, Curr)
7698 // Curr = A[i];
7699 // *Prev, *Curr;
7700 //
7701 // Prev is tracking Curr one iteration behind so they refer to different
7702 // underlying objects.
7703 if (!LI || !LI->isLoopHeader(BB: PN->getParent()) ||
7704 isSameUnderlyingObjectInLoop(PN, LI))
7705 append_range(C&: Worklist, R: PN->incoming_values());
7706 else
7707 Objects.push_back(Elt: P);
7708 continue;
7709 }
7710
7711 Objects.push_back(Elt: P);
7712 } while (!Worklist.empty());
7713}
7714
7715const Value *llvm::getUnderlyingObjectAggressive(const Value *V,
7716 bool MustPreserveProvenance) {
7717 const unsigned MaxVisited = 8;
7718
7719 SmallPtrSet<const Value *, 8> Visited;
7720 SmallVector<const Value *, 8> Worklist;
7721 Worklist.push_back(Elt: V);
7722 const Value *Object = nullptr;
7723 // Used as fallback if we can't find a common underlying object through
7724 // recursion.
7725 bool First = true;
7726 const Value *FirstObject =
7727 getUnderlyingObject(V, MaxLookup: MaxLookupSearchDepth, MustPreserveProvenance);
7728 do {
7729 const Value *P = Worklist.pop_back_val();
7730 P = First ? FirstObject
7731 : getUnderlyingObject(V: P, MaxLookup: MaxLookupSearchDepth,
7732 MustPreserveProvenance);
7733 First = false;
7734
7735 if (!Visited.insert(Ptr: P).second)
7736 continue;
7737
7738 if (Visited.size() == MaxVisited)
7739 return FirstObject;
7740
7741 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7742 Worklist.push_back(Elt: SI->getTrueValue());
7743 Worklist.push_back(Elt: SI->getFalseValue());
7744 continue;
7745 }
7746
7747 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7748 append_range(C&: Worklist, R: PN->incoming_values());
7749 continue;
7750 }
7751
7752 if (!Object)
7753 Object = P;
7754 else if (Object != P)
7755 return FirstObject;
7756 } while (!Worklist.empty());
7757
7758 return Object ? Object : FirstObject;
7759}
7760
7761/// This is the function that does the work of looking through basic
7762/// ptrtoint+arithmetic+inttoptr sequences.
7763static const Value *getUnderlyingObjectFromInt(const Value *V) {
7764 do {
7765 if (const Operator *U = dyn_cast<Operator>(Val: V)) {
7766 // If we find a ptrtoint, we can transfer control back to the
7767 // regular getUnderlyingObjectFromInt.
7768 if (U->getOpcode() == Instruction::PtrToInt)
7769 return U->getOperand(i: 0);
7770 // If we find an add of a constant, a multiplied value, or a phi, it's
7771 // likely that the other operand will lead us to the base
7772 // object. We don't have to worry about the case where the
7773 // object address is somehow being computed by the multiply,
7774 // because our callers only care when the result is an
7775 // identifiable object.
7776 if (U->getOpcode() != Instruction::Add ||
7777 (!isa<ConstantInt>(Val: U->getOperand(i: 1)) &&
7778 Operator::getOpcode(V: U->getOperand(i: 1)) != Instruction::Mul &&
7779 !isa<PHINode>(Val: U->getOperand(i: 1))))
7780 return V;
7781 V = U->getOperand(i: 0);
7782 } else {
7783 return V;
7784 }
7785 assert(V->getType()->isIntegerTy() && "Unexpected operand type!");
7786 } while (true);
7787}
7788
7789/// This is a wrapper around getUnderlyingObjects and adds support for basic
7790/// ptrtoint+arithmetic+inttoptr sequences.
7791/// It returns false if unidentified object is found in getUnderlyingObjects.
7792bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
7793 SmallVectorImpl<Value *> &Objects) {
7794 SmallPtrSet<const Value *, 16> Visited;
7795 SmallVector<const Value *, 4> Working(1, V);
7796 bool AllObjectsIdentified = true;
7797 do {
7798 V = Working.pop_back_val();
7799
7800 SmallVector<const Value *, 4> Objs;
7801 getUnderlyingObjects(V, Objects&: Objs);
7802
7803 for (const Value *V : Objs) {
7804 if (!Visited.insert(Ptr: V).second)
7805 continue;
7806 if (Operator::getOpcode(V) == Instruction::IntToPtr) {
7807 const Value *O =
7808 getUnderlyingObjectFromInt(V: cast<User>(Val: V)->getOperand(i: 0));
7809 if (O->getType()->isPointerTy()) {
7810 Working.push_back(Elt: O);
7811 continue;
7812 }
7813 }
7814 AllObjectsIdentified &= isIdentifiedObject(V);
7815 Objects.push_back(Elt: const_cast<Value *>(V));
7816 }
7817 } while (!Working.empty());
7818 return AllObjectsIdentified;
7819}
7820
7821AllocaInst *llvm::findAllocaForValue(Value *V, bool OffsetZero) {
7822 AllocaInst *Result = nullptr;
7823 SmallPtrSet<Value *, 4> Visited;
7824 SmallVector<Value *, 4> Worklist;
7825
7826 auto AddWork = [&](Value *V) {
7827 if (Visited.insert(Ptr: V).second)
7828 Worklist.push_back(Elt: V);
7829 };
7830
7831 AddWork(V);
7832 do {
7833 V = Worklist.pop_back_val();
7834 assert(Visited.count(V));
7835
7836 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
7837 if (Result && Result != AI)
7838 return nullptr;
7839 Result = AI;
7840 } else if (CastInst *CI = dyn_cast<CastInst>(Val: V)) {
7841 AddWork(CI->getOperand(i_nocapture: 0));
7842 } else if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7843 for (Value *IncValue : PN->incoming_values())
7844 AddWork(IncValue);
7845 } else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
7846 AddWork(SI->getTrueValue());
7847 AddWork(SI->getFalseValue());
7848 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: V)) {
7849 if (OffsetZero && !GEP->hasAllZeroIndices())
7850 return nullptr;
7851 AddWork(GEP->getPointerOperand());
7852 } else if (CallBase *CB = dyn_cast<CallBase>(Val: V)) {
7853 Value *Returned = CB->getReturnedArgOperand();
7854 if (Returned)
7855 AddWork(Returned);
7856 else
7857 return nullptr;
7858 } else {
7859 return nullptr;
7860 }
7861 } while (!Worklist.empty());
7862
7863 return Result;
7864}
7865
7866bool llvm::isNotCrossLaneOperation(const Instruction *I) {
7867 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
7868 return isTriviallyVectorizable(ID: II->getIntrinsicID());
7869 auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: I);
7870 return (!Shuffle || Shuffle->isSelect()) &&
7871 !isa<CallBase, BitCastInst, ExtractElementInst>(Val: I);
7872}
7873
7874bool llvm::isSafeToSpeculativelyExecute(
7875 const Instruction *Inst, const Instruction *CtxI, AssumptionCache *AC,
7876 const DominatorTree *DT, const TargetLibraryInfo *TLI, bool UseVariableInfo,
7877 bool IgnoreUBImplyingAttrs) {
7878 return isSafeToSpeculativelyExecuteWithOpcode(Opcode: Inst->getOpcode(), Inst, CtxI,
7879 AC, DT, TLI, UseVariableInfo,
7880 IgnoreUBImplyingAttrs);
7881}
7882
7883bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
7884 unsigned Opcode, const Instruction *Inst, const Instruction *CtxI,
7885 AssumptionCache *AC, const DominatorTree *DT, const TargetLibraryInfo *TLI,
7886 bool UseVariableInfo, bool IgnoreUBImplyingAttrs) {
7887#ifndef NDEBUG
7888 if (Inst->getOpcode() != Opcode) {
7889 // Check that the operands are actually compatible with the Opcode override.
7890 auto hasEqualReturnAndLeadingOperandTypes =
7891 [](const Instruction *Inst, unsigned NumLeadingOperands) {
7892 if (Inst->getNumOperands() < NumLeadingOperands)
7893 return false;
7894 const Type *ExpectedType = Inst->getType();
7895 for (unsigned ItOp = 0; ItOp < NumLeadingOperands; ++ItOp)
7896 if (Inst->getOperand(ItOp)->getType() != ExpectedType)
7897 return false;
7898 return true;
7899 };
7900 assert(!Instruction::isBinaryOp(Opcode) ||
7901 hasEqualReturnAndLeadingOperandTypes(Inst, 2));
7902 assert(!Instruction::isUnaryOp(Opcode) ||
7903 hasEqualReturnAndLeadingOperandTypes(Inst, 1));
7904 }
7905#endif
7906
7907 switch (Opcode) {
7908 default:
7909 return true;
7910 case Instruction::UDiv:
7911 case Instruction::URem: {
7912 // x / y is undefined if y == 0.
7913 const APInt *V;
7914 if (match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: V)))
7915 return *V != 0;
7916 return false;
7917 }
7918 case Instruction::SDiv:
7919 case Instruction::SRem: {
7920 // x / y is undefined if y == 0 or x == INT_MIN and y == -1
7921 const APInt *Numerator, *Denominator;
7922 if (!match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: Denominator)))
7923 return false;
7924 // We cannot hoist this division if the denominator is 0.
7925 if (*Denominator == 0)
7926 return false;
7927 // It's safe to hoist if the denominator is not 0 or -1.
7928 if (!Denominator->isAllOnes())
7929 return true;
7930 // At this point we know that the denominator is -1. It is safe to hoist as
7931 // long we know that the numerator is not INT_MIN.
7932 if (match(V: Inst->getOperand(i: 0), P: m_APInt(Res&: Numerator)))
7933 return !Numerator->isMinSignedValue();
7934 // The numerator *might* be MinSignedValue.
7935 return false;
7936 }
7937 case Instruction::Load: {
7938 if (!UseVariableInfo)
7939 return false;
7940
7941 const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst);
7942 if (!LI)
7943 return false;
7944 if (mustSuppressSpeculation(LI: *LI))
7945 return false;
7946 const DataLayout &DL = LI->getDataLayout();
7947 return isDereferenceableAndAlignedPointer(
7948 V: LI->getPointerOperand(), Ty: LI->getType(), Alignment: LI->getAlign(),
7949 Q: SimplifyQuery(DL, TLI, DT, AC, CtxI));
7950 }
7951 case Instruction::Call: {
7952 auto *CI = dyn_cast<const CallInst>(Val: Inst);
7953 if (!CI)
7954 return false;
7955 const Function *Callee = CI->getCalledFunction();
7956
7957 // The called function could have undefined behavior or side-effects, even
7958 // if marked readnone nounwind.
7959 if (!Callee || !Callee->isSpeculatable())
7960 return false;
7961 // Since the operands may be changed after hoisting, undefined behavior may
7962 // be triggered by some UB-implying attributes.
7963 return IgnoreUBImplyingAttrs || !CI->hasUBImplyingAttrs();
7964 }
7965 case Instruction::VAArg:
7966 case Instruction::Alloca:
7967 case Instruction::Invoke:
7968 case Instruction::CallBr:
7969 case Instruction::PHI:
7970 case Instruction::Store:
7971 case Instruction::Ret:
7972 case Instruction::UncondBr:
7973 case Instruction::CondBr:
7974 case Instruction::IndirectBr:
7975 case Instruction::Switch:
7976 case Instruction::Unreachable:
7977 case Instruction::Fence:
7978 case Instruction::AtomicRMW:
7979 case Instruction::AtomicCmpXchg:
7980 case Instruction::LandingPad:
7981 case Instruction::Resume:
7982 case Instruction::CatchSwitch:
7983 case Instruction::CatchPad:
7984 case Instruction::CatchRet:
7985 case Instruction::CleanupPad:
7986 case Instruction::CleanupRet:
7987 return false; // Misc instructions which have effects
7988 }
7989}
7990
7991bool llvm::mayHaveNonDefUseDependency(const Instruction &I) {
7992 if (I.mayReadOrWriteMemory())
7993 // Memory dependency possible
7994 return true;
7995 if (!isSafeToSpeculativelyExecute(Inst: &I))
7996 // Can't move above a maythrow call or infinite loop. Or if an
7997 // inalloca alloca, above a stacksave call.
7998 return true;
7999 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8000 // 1) Can't reorder two inf-loop calls, even if readonly
8001 // 2) Also can't reorder an inf-loop call below a instruction which isn't
8002 // safe to speculative execute. (Inverse of above)
8003 return true;
8004 return false;
8005}
8006
8007/// Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
8008static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) {
8009 switch (OR) {
8010 case ConstantRange::OverflowResult::MayOverflow:
8011 return OverflowResult::MayOverflow;
8012 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
8013 return OverflowResult::AlwaysOverflowsLow;
8014 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
8015 return OverflowResult::AlwaysOverflowsHigh;
8016 case ConstantRange::OverflowResult::NeverOverflows:
8017 return OverflowResult::NeverOverflows;
8018 }
8019 llvm_unreachable("Unknown OverflowResult");
8020}
8021
8022/// Combine constant ranges from computeConstantRange() and computeKnownBits().
8023ConstantRange
8024llvm::computeConstantRangeIncludingKnownBits(const WithCache<const Value *> &V,
8025 bool ForSigned,
8026 const SimplifyQuery &SQ) {
8027 ConstantRange CR1 =
8028 ConstantRange::fromKnownBits(Known: V.getKnownBits(Q: SQ), IsSigned: ForSigned);
8029 ConstantRange CR2 = computeConstantRange(V, ForSigned, SQ);
8030 ConstantRange::PreferredRangeType RangeType =
8031 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
8032 return CR1.intersectWith(CR: CR2, Type: RangeType);
8033}
8034
8035OverflowResult llvm::computeOverflowForUnsignedMul(const Value *LHS,
8036 const Value *RHS,
8037 const SimplifyQuery &SQ,
8038 bool IsNSW) {
8039 ConstantRange LHSRange =
8040 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
8041 ConstantRange RHSRange =
8042 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
8043
8044 // mul nsw of two non-negative numbers is also nuw.
8045 if (IsNSW && LHSRange.isAllNonNegative() && RHSRange.isAllNonNegative())
8046 return OverflowResult::NeverOverflows;
8047
8048 return mapOverflowResult(OR: LHSRange.unsignedMulMayOverflow(Other: RHSRange));
8049}
8050
8051OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
8052 const Value *RHS,
8053 const SimplifyQuery &SQ) {
8054 // Multiplying n * m significant bits yields a result of n + m significant
8055 // bits. If the total number of significant bits does not exceed the
8056 // result bit width (minus 1), there is no overflow.
8057 // This means if we have enough leading sign bits in the operands
8058 // we can guarantee that the result does not overflow.
8059 // Ref: "Hacker's Delight" by Henry Warren
8060 unsigned BitWidth = LHS->getType()->getScalarSizeInBits();
8061
8062 // Note that underestimating the number of sign bits gives a more
8063 // conservative answer.
8064 unsigned SignBits =
8065 ::ComputeNumSignBits(V: LHS, Q: SQ) + ::ComputeNumSignBits(V: RHS, Q: SQ);
8066
8067 // First handle the easy case: if we have enough sign bits there's
8068 // definitely no overflow.
8069 if (SignBits > BitWidth + 1)
8070 return OverflowResult::NeverOverflows;
8071
8072 // There are two ambiguous cases where there can be no overflow:
8073 // SignBits == BitWidth + 1 and
8074 // SignBits == BitWidth
8075 // The second case is difficult to check, therefore we only handle the
8076 // first case.
8077 if (SignBits == BitWidth + 1) {
8078 // It overflows only when both arguments are negative and the true
8079 // product is exactly the minimum negative number.
8080 // E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
8081 // For simplicity we just check if at least one side is not negative.
8082 KnownBits LHSKnown = computeKnownBits(V: LHS, Q: SQ);
8083 KnownBits RHSKnown = computeKnownBits(V: RHS, Q: SQ);
8084 if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
8085 return OverflowResult::NeverOverflows;
8086 }
8087 return OverflowResult::MayOverflow;
8088}
8089
8090OverflowResult
8091llvm::computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
8092 const WithCache<const Value *> &RHS,
8093 const SimplifyQuery &SQ) {
8094 ConstantRange LHSRange =
8095 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
8096 ConstantRange RHSRange =
8097 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
8098 return mapOverflowResult(OR: LHSRange.unsignedAddMayOverflow(Other: RHSRange));
8099}
8100
8101static OverflowResult
8102computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
8103 const WithCache<const Value *> &RHS,
8104 const AddOperator *Add, const SimplifyQuery &SQ) {
8105 if (Add && Add->hasNoSignedWrap()) {
8106 return OverflowResult::NeverOverflows;
8107 }
8108
8109 // If LHS and RHS each have at least two sign bits, the addition will look
8110 // like
8111 //
8112 // XX..... +
8113 // YY.....
8114 //
8115 // If the carry into the most significant position is 0, X and Y can't both
8116 // be 1 and therefore the carry out of the addition is also 0.
8117 //
8118 // If the carry into the most significant position is 1, X and Y can't both
8119 // be 0 and therefore the carry out of the addition is also 1.
8120 //
8121 // Since the carry into the most significant position is always equal to
8122 // the carry out of the addition, there is no signed overflow.
8123 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
8124 return OverflowResult::NeverOverflows;
8125
8126 ConstantRange LHSRange =
8127 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
8128 ConstantRange RHSRange =
8129 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
8130 OverflowResult OR =
8131 mapOverflowResult(OR: LHSRange.signedAddMayOverflow(Other: RHSRange));
8132 if (OR != OverflowResult::MayOverflow)
8133 return OR;
8134
8135 // The remaining code needs Add to be available. Early returns if not so.
8136 if (!Add)
8137 return OverflowResult::MayOverflow;
8138
8139 // If the sign of Add is the same as at least one of the operands, this add
8140 // CANNOT overflow. If this can be determined from the known bits of the
8141 // operands the above signedAddMayOverflow() check will have already done so.
8142 // The only other way to improve on the known bits is from an assumption, so
8143 // call computeKnownBitsFromContext() directly.
8144 bool LHSOrRHSKnownNonNegative =
8145 (LHSRange.isAllNonNegative() || RHSRange.isAllNonNegative());
8146 bool LHSOrRHSKnownNegative =
8147 (LHSRange.isAllNegative() || RHSRange.isAllNegative());
8148 if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) {
8149 KnownBits AddKnown(LHSRange.getBitWidth());
8150 computeKnownBitsFromContext(V: Add, Known&: AddKnown, Q: SQ);
8151 if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) ||
8152 (AddKnown.isNegative() && LHSOrRHSKnownNegative))
8153 return OverflowResult::NeverOverflows;
8154 }
8155
8156 return OverflowResult::MayOverflow;
8157}
8158
8159OverflowResult llvm::computeOverflowForUnsignedSub(const Value *LHS,
8160 const Value *RHS,
8161 const SimplifyQuery &SQ) {
8162 // X - (X % ?)
8163 // The remainder of a value can't have greater magnitude than itself,
8164 // so the subtraction can't overflow.
8165
8166 // X - (X -nuw ?)
8167 // In the minimal case, this would simplify to "?", so there's no subtract
8168 // at all. But if this analysis is used to peek through casts, for example,
8169 // then determining no-overflow may allow other transforms.
8170
8171 // TODO: There are other patterns like this.
8172 // See simplifyICmpWithBinOpOnLHS() for candidates.
8173 if (match(V: RHS, P: m_URem(L: m_Specific(V: LHS), R: m_Value())) ||
8174 match(V: RHS, P: m_NUWSub(L: m_Specific(V: LHS), R: m_Value())))
8175 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT))
8176 return OverflowResult::NeverOverflows;
8177
8178 if (auto C = isImpliedByDomCondition(Pred: CmpInst::ICMP_UGE, LHS, RHS, ContextI: SQ.CtxI,
8179 DL: SQ.DL)) {
8180 if (*C)
8181 return OverflowResult::NeverOverflows;
8182 return OverflowResult::AlwaysOverflowsLow;
8183 }
8184
8185 ConstantRange LHSRange =
8186 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
8187 ConstantRange RHSRange =
8188 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
8189 return mapOverflowResult(OR: LHSRange.unsignedSubMayOverflow(Other: RHSRange));
8190}
8191
8192OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS,
8193 const Value *RHS,
8194 const SimplifyQuery &SQ) {
8195 // X - (X % ?)
8196 // The remainder of a value can't have greater magnitude than itself,
8197 // so the subtraction can't overflow.
8198
8199 // X - (X -nsw ?)
8200 // In the minimal case, this would simplify to "?", so there's no subtract
8201 // at all. But if this analysis is used to peek through casts, for example,
8202 // then determining no-overflow may allow other transforms.
8203 if (match(V: RHS, P: m_SRem(L: m_Specific(V: LHS), R: m_Value())) ||
8204 match(V: RHS, P: m_NSWSub(L: m_Specific(V: LHS), R: m_Value())))
8205 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT))
8206 return OverflowResult::NeverOverflows;
8207
8208 // If LHS and RHS each have at least two sign bits, the subtraction
8209 // cannot overflow.
8210 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
8211 return OverflowResult::NeverOverflows;
8212
8213 ConstantRange LHSRange =
8214 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
8215 ConstantRange RHSRange =
8216 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
8217 return mapOverflowResult(OR: LHSRange.signedSubMayOverflow(Other: RHSRange));
8218}
8219
8220bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO,
8221 const DominatorTree &DT) {
8222 SmallVector<const CondBrInst *, 2> GuardingBranches;
8223 SmallVector<const ExtractValueInst *, 2> Results;
8224
8225 for (const User *U : WO->users()) {
8226 if (const auto *EVI = dyn_cast<ExtractValueInst>(Val: U)) {
8227 assert(EVI->getNumIndices() == 1 && "Obvious from CI's type");
8228
8229 if (EVI->getIndices()[0] == 0)
8230 Results.push_back(Elt: EVI);
8231 else {
8232 assert(EVI->getIndices()[0] == 1 && "Obvious from CI's type");
8233
8234 for (const auto *U : EVI->users())
8235 if (const auto *B = dyn_cast<CondBrInst>(Val: U))
8236 GuardingBranches.push_back(Elt: B);
8237 }
8238 } else {
8239 // We are using the aggregate directly in a way we don't want to analyze
8240 // here (storing it to a global, say).
8241 return false;
8242 }
8243 }
8244
8245 auto AllUsesGuardedByBranch = [&](const CondBrInst *BI) {
8246 BasicBlockEdge NoWrapEdge(BI->getParent(), BI->getSuccessor(i: 1));
8247
8248 // Check if all users of the add are provably no-wrap.
8249 for (const auto *Result : Results) {
8250 // If the extractvalue itself is not executed on overflow, the we don't
8251 // need to check each use separately, since domination is transitive.
8252 if (DT.dominates(BBE: NoWrapEdge, BB: Result->getParent()))
8253 continue;
8254
8255 for (const auto &RU : Result->uses())
8256 if (!DT.dominates(BBE: NoWrapEdge, U: RU))
8257 return false;
8258 }
8259
8260 return true;
8261 };
8262
8263 return llvm::any_of(Range&: GuardingBranches, P: AllUsesGuardedByBranch);
8264}
8265
8266/// Shifts return poison if shiftwidth is larger than the bitwidth.
8267static bool shiftAmountKnownInRange(const Value *ShiftAmount) {
8268 auto *C = dyn_cast<Constant>(Val: ShiftAmount);
8269 if (!C)
8270 return false;
8271
8272 // Shifts return poison if shiftwidth is larger than the bitwidth.
8273 SmallVector<const Constant *, 4> ShiftAmounts;
8274 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: C->getType())) {
8275 unsigned NumElts = FVTy->getNumElements();
8276 for (unsigned i = 0; i < NumElts; ++i)
8277 ShiftAmounts.push_back(Elt: C->getAggregateElement(Elt: i));
8278 } else if (isa<ScalableVectorType>(Val: C->getType()))
8279 return false; // Can't tell, just return false to be safe
8280 else
8281 ShiftAmounts.push_back(Elt: C);
8282
8283 bool Safe = llvm::all_of(Range&: ShiftAmounts, P: [](const Constant *C) {
8284 auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
8285 return CI && CI->getValue().ult(RHS: C->getType()->getIntegerBitWidth());
8286 });
8287
8288 return Safe;
8289}
8290
8291static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
8292 bool ConsiderFlagsAndMetadata) {
8293 const auto *CB = dyn_cast<CallBase>(Val: Op);
8294 bool CBHasNoUndef = CB && CB->hasRetAttr(Kind: Attribute::NoUndef);
8295 if (ConsiderFlagsAndMetadata && includesPoison(Kind) &&
8296 Op->hasPoisonGeneratingAnnotations() && !CBHasNoUndef)
8297 return true;
8298
8299 unsigned Opcode = Op->getOpcode();
8300
8301 // Check whether opcode is a poison/undef-generating operation
8302 switch (Opcode) {
8303 case Instruction::Shl:
8304 case Instruction::AShr:
8305 case Instruction::LShr:
8306 return includesPoison(Kind) && !shiftAmountKnownInRange(ShiftAmount: Op->getOperand(i: 1));
8307 case Instruction::FPToSI:
8308 case Instruction::FPToUI:
8309 // fptosi/ui yields poison if the resulting value does not fit in the
8310 // destination type.
8311 return true;
8312 case Instruction::Call:
8313 if (auto *II = dyn_cast<IntrinsicInst>(Val: Op)) {
8314 switch (II->getIntrinsicID()) {
8315 // NOTE: Use IntrNoCreateUndefOrPoison when possible.
8316 case Intrinsic::ctlz:
8317 case Intrinsic::cttz:
8318 case Intrinsic::abs:
8319 // We're not considering flags so it is safe to just return false.
8320 return false;
8321 case Intrinsic::sshl_sat:
8322 case Intrinsic::ushl_sat:
8323 if (!includesPoison(Kind) ||
8324 shiftAmountKnownInRange(ShiftAmount: II->getArgOperand(i: 1)))
8325 return false;
8326 break;
8327 }
8328 }
8329 [[fallthrough]];
8330 case Instruction::CallBr:
8331 case Instruction::Invoke: {
8332 return !CBHasNoUndef && !CB->hasFnAttr(Kind: Attribute::NoCreateUndefOrPoison);
8333 }
8334 case Instruction::InsertElement:
8335 case Instruction::ExtractElement: {
8336 // If index exceeds the length of the vector, it returns poison
8337 auto *VTy = cast<VectorType>(Val: Op->getOperand(i: 0)->getType());
8338 unsigned IdxOp = Op->getOpcode() == Instruction::InsertElement ? 2 : 1;
8339 auto *Idx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: IdxOp));
8340 if (includesPoison(Kind))
8341 return !Idx ||
8342 Idx->getValue().uge(RHS: VTy->getElementCount().getKnownMinValue());
8343 return false;
8344 }
8345 case Instruction::ShuffleVector: {
8346 ArrayRef<int> Mask = isa<ConstantExpr>(Val: Op)
8347 ? cast<ConstantExpr>(Val: Op)->getShuffleMask()
8348 : cast<ShuffleVectorInst>(Val: Op)->getShuffleMask();
8349 return includesPoison(Kind) && is_contained(Range&: Mask, Element: PoisonMaskElem);
8350 }
8351 case Instruction::FNeg:
8352 case Instruction::PHI:
8353 case Instruction::Select:
8354 case Instruction::ExtractValue:
8355 case Instruction::InsertValue:
8356 case Instruction::Freeze:
8357 case Instruction::ICmp:
8358 case Instruction::FCmp:
8359 case Instruction::GetElementPtr:
8360 return false;
8361 case Instruction::AddrSpaceCast:
8362 return true;
8363 default: {
8364 const auto *CE = dyn_cast<ConstantExpr>(Val: Op);
8365 if (isa<CastInst>(Val: Op) || (CE && CE->isCast()))
8366 return false;
8367 else if (Instruction::isBinaryOp(Opcode))
8368 return false;
8369 // Be conservative and return true.
8370 return true;
8371 }
8372 }
8373}
8374
8375bool llvm::canCreateUndefOrPoison(const Operator *Op,
8376 bool ConsiderFlagsAndMetadata) {
8377 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::UndefOrPoison,
8378 ConsiderFlagsAndMetadata);
8379}
8380
8381bool llvm::canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata) {
8382 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::PoisonOnly,
8383 ConsiderFlagsAndMetadata);
8384}
8385
8386static bool directlyImpliesPoison(const Value *ValAssumedPoison, const Value *V,
8387 unsigned Depth) {
8388 if (ValAssumedPoison == V)
8389 return true;
8390
8391 const unsigned MaxDepth = 2;
8392 if (Depth >= MaxDepth)
8393 return false;
8394
8395 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
8396 if (any_of(Range: I->operands(), P: [=](const Use &Op) {
8397 return propagatesPoison(PoisonOp: Op) &&
8398 directlyImpliesPoison(ValAssumedPoison, V: Op, Depth: Depth + 1);
8399 }))
8400 return true;
8401
8402 // V = extractvalue V0, idx
8403 // V2 = extractvalue V0, idx2
8404 // V0's elements are all poison or not. (e.g., add_with_overflow)
8405 const WithOverflowInst *II;
8406 if (match(V: I, P: m_ExtractValue(V: m_WithOverflowInst(I&: II))) &&
8407 (match(V: ValAssumedPoison, P: m_ExtractValue(V: m_Specific(V: II))) ||
8408 llvm::is_contained(Range: II->args(), Element: ValAssumedPoison)))
8409 return true;
8410 }
8411 return false;
8412}
8413
8414static bool impliesPoison(const Value *ValAssumedPoison, const Value *V,
8415 unsigned Depth) {
8416 if (isGuaranteedNotToBePoison(V: ValAssumedPoison))
8417 return true;
8418
8419 if (directlyImpliesPoison(ValAssumedPoison, V, /* Depth */ 0))
8420 return true;
8421
8422 const unsigned MaxDepth = 2;
8423 if (Depth >= MaxDepth)
8424 return false;
8425
8426 const auto *I = dyn_cast<Instruction>(Val: ValAssumedPoison);
8427 if (I && !canCreatePoison(Op: cast<Operator>(Val: I))) {
8428 return all_of(Range: I->operands(), P: [=](const Value *Op) {
8429 return impliesPoison(ValAssumedPoison: Op, V, Depth: Depth + 1);
8430 });
8431 }
8432 return false;
8433}
8434
8435bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
8436 return ::impliesPoison(ValAssumedPoison, V, /* Depth */ 0);
8437}
8438
8439static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
8440
8441static bool isGuaranteedNotToBeUndefOrPoison(
8442 const Value *V, AssumptionCache *AC, const Instruction *CtxI,
8443 const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
8444 if (Depth >= MaxAnalysisRecursionDepth)
8445 return false;
8446
8447 if (isa<MetadataAsValue>(Val: V))
8448 return false;
8449
8450 if (const auto *A = dyn_cast<Argument>(Val: V)) {
8451 if (A->hasAttribute(Kind: Attribute::NoUndef) ||
8452 A->hasAttribute(Kind: Attribute::Dereferenceable) ||
8453 A->hasAttribute(Kind: Attribute::DereferenceableOrNull))
8454 return true;
8455 }
8456
8457 if (auto *C = dyn_cast<Constant>(Val: V)) {
8458 if (isa<PoisonValue>(Val: C))
8459 return !includesPoison(Kind);
8460
8461 if (isa<UndefValue>(Val: C))
8462 return !includesUndef(Kind);
8463
8464 if (isa<ConstantInt>(Val: C) || isa<GlobalVariable>(Val: C) || isa<ConstantFP>(Val: C) ||
8465 isa<ConstantPointerNull>(Val: C) || isa<Function>(Val: C) || isa<ConstantData>(Val: C))
8466 return true;
8467
8468 if (C->getType()->isVectorTy() || C->getType()->isAggregateType()) {
8469 if (isa<ConstantExpr>(Val: C)) {
8470 // Scalable vectors can use a ConstantExpr to build a splat.
8471 if (Constant *SplatC = C->getSplatValue())
8472 if (isa<ConstantInt>(Val: SplatC) || isa<ConstantFP>(Val: SplatC))
8473 return true;
8474 } else if (auto *CA = dyn_cast<ConstantAggregate>(Val: C)) {
8475 return all_of(Range: CA->operands(), P: [&](const Use &Op) {
8476 return isGuaranteedNotToBeUndefOrPoison(V: Op.get(), AC, CtxI, DT,
8477 Depth: Depth + 1, Kind);
8478 });
8479 }
8480 }
8481 }
8482
8483 // Strip cast operations from a pointer value.
8484 // Note that stripPointerCastsSameRepresentation can strip off getelementptr
8485 // inbounds with zero offset. To guarantee that the result isn't poison, the
8486 // stripped pointer is checked as it has to be pointing into an allocated
8487 // object or be null `null` to ensure `inbounds` getelement pointers with a
8488 // zero offset could not produce poison.
8489 // It can strip off addrspacecast that do not change bit representation as
8490 // well. We believe that such addrspacecast is equivalent to no-op.
8491 auto *StrippedV = V->stripPointerCastsSameRepresentation();
8492 if (isa<AllocaInst>(Val: StrippedV) || isa<GlobalVariable>(Val: StrippedV) ||
8493 isa<Function>(Val: StrippedV) || isa<ConstantPointerNull>(Val: StrippedV))
8494 return true;
8495
8496 auto OpCheck = [&](const Value *V) {
8497 return isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth: Depth + 1, Kind);
8498 };
8499
8500 if (auto *Opr = dyn_cast<Operator>(Val: V)) {
8501 // If the value is a freeze instruction, then it can never
8502 // be undef or poison.
8503 if (isa<FreezeInst>(Val: V))
8504 return true;
8505
8506 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
8507 if (CB->hasRetAttr(Kind: Attribute::NoUndef) ||
8508 CB->hasRetAttr(Kind: Attribute::Dereferenceable) ||
8509 CB->hasRetAttr(Kind: Attribute::DereferenceableOrNull))
8510 return true;
8511 }
8512
8513 if (!::canCreateUndefOrPoison(Op: Opr, Kind,
8514 /*ConsiderFlagsAndMetadata=*/true)) {
8515 if (const auto *PN = dyn_cast<PHINode>(Val: V)) {
8516 unsigned Num = PN->getNumIncomingValues();
8517 bool IsWellDefined = true;
8518 for (unsigned i = 0; i < Num; ++i) {
8519 if (PN == PN->getIncomingValue(i))
8520 continue;
8521 auto *TI = PN->getIncomingBlock(i)->getTerminator();
8522 if (!isGuaranteedNotToBeUndefOrPoison(V: PN->getIncomingValue(i), AC, CtxI: TI,
8523 DT, Depth: Depth + 1, Kind)) {
8524 IsWellDefined = false;
8525 break;
8526 }
8527 }
8528 if (IsWellDefined)
8529 return true;
8530 } else if (auto *Splat = isa<ShuffleVectorInst>(Val: Opr) ? getSplatValue(V: Opr)
8531 : nullptr) {
8532 // For splats we only need to check the value being splatted.
8533 if (OpCheck(Splat))
8534 return true;
8535 } else if (all_of(Range: Opr->operands(), P: OpCheck))
8536 return true;
8537 }
8538 }
8539
8540 if (auto *I = dyn_cast<LoadInst>(Val: V))
8541 if (I->hasMetadata(KindID: LLVMContext::MD_noundef) ||
8542 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable) ||
8543 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable_or_null))
8544 return true;
8545
8546 if (programUndefinedIfUndefOrPoison(V, PoisonOnly: !includesUndef(Kind)))
8547 return true;
8548
8549 // CtxI may be null or a cloned instruction.
8550 if (!CtxI || !CtxI->getParent() || !DT)
8551 return false;
8552
8553 auto *DNode = DT->getNode(BB: CtxI->getParent());
8554 if (!DNode)
8555 // Unreachable block
8556 return false;
8557
8558 // If V is used as a branch condition before reaching CtxI, V cannot be
8559 // undef or poison.
8560 // br V, BB1, BB2
8561 // BB1:
8562 // CtxI ; V cannot be undef or poison here
8563 auto *Dominator = DNode->getIDom();
8564 // This check is purely for compile time reasons: we can skip the IDom walk
8565 // if what we are checking for includes undef and the value is not an integer.
8566 if (!includesUndef(Kind) || V->getType()->isIntegerTy())
8567 while (Dominator) {
8568 auto *TI = Dominator->getBlock()->getTerminatorOrNull();
8569
8570 Value *Cond = nullptr;
8571 if (auto BI = dyn_cast_or_null<CondBrInst>(Val: TI)) {
8572 Cond = BI->getCondition();
8573 } else if (auto SI = dyn_cast_or_null<SwitchInst>(Val: TI)) {
8574 Cond = SI->getCondition();
8575 }
8576
8577 if (Cond) {
8578 if (Cond == V)
8579 return true;
8580 else if (!includesUndef(Kind) && isa<Operator>(Val: Cond)) {
8581 // For poison, we can analyze further
8582 auto *Opr = cast<Operator>(Val: Cond);
8583 if (any_of(Range: Opr->operands(), P: [V](const Use &U) {
8584 return V == U && propagatesPoison(PoisonOp: U);
8585 }))
8586 return true;
8587 }
8588 }
8589
8590 Dominator = Dominator->getIDom();
8591 }
8592
8593 if (AC && getKnowledgeValidInContext(V, AttrKinds: {Attribute::NoUndef}, AC&: *AC, CtxI, DT))
8594 return true;
8595
8596 return false;
8597}
8598
8599bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC,
8600 const Instruction *CtxI,
8601 const DominatorTree *DT,
8602 unsigned Depth) {
8603 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8604 Kind: UndefPoisonKind::UndefOrPoison);
8605}
8606
8607bool llvm::isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC,
8608 const Instruction *CtxI,
8609 const DominatorTree *DT, unsigned Depth) {
8610 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8611 Kind: UndefPoisonKind::PoisonOnly);
8612}
8613
8614bool llvm::isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC,
8615 const Instruction *CtxI,
8616 const DominatorTree *DT, unsigned Depth) {
8617 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8618 Kind: UndefPoisonKind::UndefOnly);
8619}
8620
8621/// Return true if undefined behavior would provably be executed on the path to
8622/// OnPathTo if Root produced a posion result. Note that this doesn't say
8623/// anything about whether OnPathTo is actually executed or whether Root is
8624/// actually poison. This can be used to assess whether a new use of Root can
8625/// be added at a location which is control equivalent with OnPathTo (such as
8626/// immediately before it) without introducing UB which didn't previously
8627/// exist. Note that a false result conveys no information.
8628bool llvm::mustExecuteUBIfPoisonOnPathTo(Instruction *Root,
8629 Instruction *OnPathTo,
8630 DominatorTree *DT) {
8631 // Basic approach is to assume Root is poison, propagate poison forward
8632 // through all users we can easily track, and then check whether any of those
8633 // users are provable UB and must execute before out exiting block might
8634 // exit.
8635
8636 // The set of all recursive users we've visited (which are assumed to all be
8637 // poison because of said visit)
8638 SmallPtrSet<const Value *, 16> KnownPoison;
8639 SmallVector<const Instruction*, 16> Worklist;
8640 Worklist.push_back(Elt: Root);
8641 while (!Worklist.empty()) {
8642 const Instruction *I = Worklist.pop_back_val();
8643
8644 // If we know this must trigger UB on a path leading our target.
8645 if (mustTriggerUB(I, KnownPoison) && DT->dominates(Def: I, User: OnPathTo))
8646 return true;
8647
8648 // If we can't analyze propagation through this instruction, just skip it
8649 // and transitive users. Safe as false is a conservative result.
8650 if (I != Root && !any_of(Range: I->operands(), P: [&KnownPoison](const Use &U) {
8651 return KnownPoison.contains(Ptr: U) && propagatesPoison(PoisonOp: U);
8652 }))
8653 continue;
8654
8655 if (KnownPoison.insert(Ptr: I).second)
8656 for (const User *User : I->users())
8657 Worklist.push_back(Elt: cast<Instruction>(Val: User));
8658 }
8659
8660 // Might be non-UB, or might have a path we couldn't prove must execute on
8661 // way to exiting bb.
8662 return false;
8663}
8664
8665OverflowResult llvm::computeOverflowForSignedAdd(const AddOperator *Add,
8666 const SimplifyQuery &SQ) {
8667 return ::computeOverflowForSignedAdd(LHS: Add->getOperand(i_nocapture: 0), RHS: Add->getOperand(i_nocapture: 1),
8668 Add, SQ);
8669}
8670
8671OverflowResult
8672llvm::computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
8673 const WithCache<const Value *> &RHS,
8674 const SimplifyQuery &SQ) {
8675 return ::computeOverflowForSignedAdd(LHS, RHS, Add: nullptr, SQ);
8676}
8677
8678bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
8679 // Note: An atomic operation isn't guaranteed to return in a reasonable amount
8680 // of time because it's possible for another thread to interfere with it for an
8681 // arbitrary length of time, but programs aren't allowed to rely on that.
8682
8683 // If there is no successor, then execution can't transfer to it.
8684 if (isa<ReturnInst>(Val: I))
8685 return false;
8686 if (isa<UnreachableInst>(Val: I))
8687 return false;
8688
8689 // Note: Do not add new checks here; instead, change Instruction::mayThrow or
8690 // Instruction::willReturn.
8691 //
8692 // FIXME: Move this check into Instruction::willReturn.
8693 if (isa<CatchPadInst>(Val: I)) {
8694 switch (classifyEHPersonality(Pers: I->getFunction()->getPersonalityFn())) {
8695 default:
8696 // A catchpad may invoke exception object constructors and such, which
8697 // in some languages can be arbitrary code, so be conservative by default.
8698 return false;
8699 case EHPersonality::CoreCLR:
8700 // For CoreCLR, it just involves a type test.
8701 return true;
8702 }
8703 }
8704
8705 // An instruction that returns without throwing must transfer control flow
8706 // to a successor.
8707 return !I->mayThrow() && I->willReturn();
8708}
8709
8710bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) {
8711 // TODO: This is slightly conservative for invoke instruction since exiting
8712 // via an exception *is* normal control for them.
8713 for (const Instruction &I : *BB)
8714 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8715 return false;
8716 return true;
8717}
8718
8719bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8720 BasicBlock::const_iterator Begin, BasicBlock::const_iterator End,
8721 unsigned ScanLimit) {
8722 return isGuaranteedToTransferExecutionToSuccessor(Range: make_range(x: Begin, y: End),
8723 ScanLimit);
8724}
8725
8726bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8727 iterator_range<BasicBlock::const_iterator> Range, unsigned ScanLimit) {
8728 assert(ScanLimit && "scan limit must be non-zero");
8729 for (const Instruction &I : Range) {
8730 if (--ScanLimit == 0)
8731 return false;
8732 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8733 return false;
8734 }
8735 return true;
8736}
8737
8738bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
8739 const Loop *L) {
8740 // The loop header is guaranteed to be executed for every iteration.
8741 //
8742 // FIXME: Relax this constraint to cover all basic blocks that are
8743 // guaranteed to be executed at every iteration.
8744 if (I->getParent() != L->getHeader()) return false;
8745
8746 for (const Instruction &LI : *L->getHeader()) {
8747 if (&LI == I) return true;
8748 if (!isGuaranteedToTransferExecutionToSuccessor(I: &LI)) return false;
8749 }
8750 llvm_unreachable("Instruction not contained in its own parent basic block.");
8751}
8752
8753bool llvm::intrinsicPropagatesPoison(Intrinsic::ID IID) {
8754 switch (IID) {
8755 // TODO: Add more intrinsics.
8756 case Intrinsic::sadd_with_overflow:
8757 case Intrinsic::ssub_with_overflow:
8758 case Intrinsic::smul_with_overflow:
8759 case Intrinsic::uadd_with_overflow:
8760 case Intrinsic::usub_with_overflow:
8761 case Intrinsic::umul_with_overflow:
8762 // If an input is a vector containing a poison element, the
8763 // two output vectors (calculated results, overflow bits)'
8764 // corresponding lanes are poison.
8765 return true;
8766 case Intrinsic::ctpop:
8767 case Intrinsic::ctlz:
8768 case Intrinsic::cttz:
8769 case Intrinsic::abs:
8770 case Intrinsic::smax:
8771 case Intrinsic::smin:
8772 case Intrinsic::umax:
8773 case Intrinsic::umin:
8774 case Intrinsic::scmp:
8775 case Intrinsic::smulh:
8776 case Intrinsic::umulh:
8777 case Intrinsic::is_fpclass:
8778 case Intrinsic::ptrmask:
8779 case Intrinsic::ucmp:
8780 case Intrinsic::bitreverse:
8781 case Intrinsic::bswap:
8782 case Intrinsic::sadd_sat:
8783 case Intrinsic::ssub_sat:
8784 case Intrinsic::sshl_sat:
8785 case Intrinsic::uadd_sat:
8786 case Intrinsic::usub_sat:
8787 case Intrinsic::ushl_sat:
8788 case Intrinsic::smul_fix:
8789 case Intrinsic::smul_fix_sat:
8790 case Intrinsic::umul_fix:
8791 case Intrinsic::umul_fix_sat:
8792 case Intrinsic::pow:
8793 case Intrinsic::experimental_constrained_pow:
8794 case Intrinsic::powi:
8795 case Intrinsic::experimental_constrained_powi:
8796 case Intrinsic::sin:
8797 case Intrinsic::experimental_constrained_sin:
8798 case Intrinsic::sinh:
8799 case Intrinsic::experimental_constrained_sinh:
8800 case Intrinsic::cos:
8801 case Intrinsic::experimental_constrained_cos:
8802 case Intrinsic::cosh:
8803 case Intrinsic::experimental_constrained_cosh:
8804 case Intrinsic::sincos:
8805 case Intrinsic::sincospi:
8806 case Intrinsic::tan:
8807 case Intrinsic::experimental_constrained_tan:
8808 case Intrinsic::tanh:
8809 case Intrinsic::experimental_constrained_tanh:
8810 case Intrinsic::asin:
8811 case Intrinsic::experimental_constrained_asin:
8812 case Intrinsic::acos:
8813 case Intrinsic::experimental_constrained_acos:
8814 case Intrinsic::atan:
8815 case Intrinsic::experimental_constrained_atan:
8816 case Intrinsic::atan2:
8817 case Intrinsic::experimental_constrained_atan2:
8818 case Intrinsic::canonicalize:
8819 case Intrinsic::sqrt:
8820 case Intrinsic::experimental_constrained_sqrt:
8821 case Intrinsic::fma:
8822 case Intrinsic::experimental_constrained_fma:
8823 case Intrinsic::fmuladd:
8824 case Intrinsic::experimental_constrained_fmuladd:
8825 case Intrinsic::exp:
8826 case Intrinsic::experimental_constrained_exp:
8827 case Intrinsic::exp2:
8828 case Intrinsic::experimental_constrained_exp2:
8829 case Intrinsic::exp10:
8830 case Intrinsic::log:
8831 case Intrinsic::experimental_constrained_log:
8832 case Intrinsic::log2:
8833 case Intrinsic::experimental_constrained_log2:
8834 case Intrinsic::log10:
8835 case Intrinsic::experimental_constrained_log10:
8836 case Intrinsic::modf:
8837 case Intrinsic::floor:
8838 case Intrinsic::experimental_constrained_floor:
8839 case Intrinsic::ceil:
8840 case Intrinsic::experimental_constrained_ceil:
8841 case Intrinsic::trunc:
8842 case Intrinsic::experimental_constrained_trunc:
8843 case Intrinsic::rint:
8844 case Intrinsic::experimental_constrained_rint:
8845 case Intrinsic::nearbyint:
8846 case Intrinsic::experimental_constrained_nearbyint:
8847 case Intrinsic::round:
8848 case Intrinsic::experimental_constrained_round:
8849 case Intrinsic::roundeven:
8850 case Intrinsic::experimental_constrained_roundeven:
8851 case Intrinsic::lrint:
8852 case Intrinsic::experimental_constrained_lrint:
8853 case Intrinsic::llrint:
8854 case Intrinsic::experimental_constrained_llrint:
8855 case Intrinsic::lround:
8856 case Intrinsic::experimental_constrained_lround:
8857 case Intrinsic::llround:
8858 case Intrinsic::experimental_constrained_llround:
8859 case Intrinsic::fshl:
8860 case Intrinsic::fshr:
8861 case Intrinsic::frexp:
8862 case Intrinsic::ldexp:
8863 case Intrinsic::experimental_constrained_ldexp:
8864 case Intrinsic::get_active_lane_mask:
8865 case Intrinsic::mask_beforefirst:
8866 return true;
8867 default:
8868 return false;
8869 }
8870}
8871
8872bool llvm::propagatesPoison(const Use &PoisonOp) {
8873 const Operator *I = cast<Operator>(Val: PoisonOp.getUser());
8874 switch (I->getOpcode()) {
8875 case Instruction::Freeze:
8876 case Instruction::PHI:
8877 case Instruction::Invoke:
8878 return false;
8879 case Instruction::Select:
8880 return PoisonOp.getOperandNo() == 0;
8881 case Instruction::Call:
8882 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
8883 return intrinsicPropagatesPoison(IID: II->getIntrinsicID());
8884 return false;
8885 case Instruction::ICmp:
8886 case Instruction::FCmp:
8887 case Instruction::GetElementPtr:
8888 return true;
8889 default:
8890 if (isa<BinaryOperator>(Val: I) || isa<UnaryOperator>(Val: I) || isa<CastInst>(Val: I))
8891 return true;
8892
8893 // Be conservative and return false.
8894 return false;
8895 }
8896}
8897
8898/// Enumerates all operands of \p I that are guaranteed to not be undef or
8899/// poison. If the callback \p Handle returns true, stop processing and return
8900/// true. Otherwise, return false.
8901template <typename CallableT>
8902static bool handleGuaranteedWellDefinedOps(const Instruction *I,
8903 const CallableT &Handle) {
8904 switch (I->getOpcode()) {
8905 case Instruction::Store:
8906 if (Handle(cast<StoreInst>(Val: I)->getPointerOperand()))
8907 return true;
8908 break;
8909
8910 case Instruction::Load:
8911 if (Handle(cast<LoadInst>(Val: I)->getPointerOperand()))
8912 return true;
8913 break;
8914
8915 // Since dereferenceable attribute imply noundef, atomic operations
8916 // also implicitly have noundef pointers too
8917 case Instruction::AtomicCmpXchg:
8918 if (Handle(cast<AtomicCmpXchgInst>(Val: I)->getPointerOperand()))
8919 return true;
8920 break;
8921
8922 case Instruction::AtomicRMW:
8923 if (Handle(cast<AtomicRMWInst>(Val: I)->getPointerOperand()))
8924 return true;
8925 break;
8926
8927 case Instruction::Call:
8928 case Instruction::Invoke: {
8929 const CallBase *CB = cast<CallBase>(Val: I);
8930 if (CB->isIndirectCall() && Handle(CB->getCalledOperand()))
8931 return true;
8932 for (unsigned i = 0; i < CB->arg_size(); ++i)
8933 if ((CB->paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef) ||
8934 CB->paramHasAttr(ArgNo: i, Kind: Attribute::Dereferenceable) ||
8935 CB->paramHasAttr(ArgNo: i, Kind: Attribute::DereferenceableOrNull)) &&
8936 Handle(CB->getArgOperand(i)))
8937 return true;
8938 break;
8939 }
8940 case Instruction::Ret:
8941 if (I->getFunction()->hasRetAttribute(Kind: Attribute::NoUndef) &&
8942 Handle(I->getOperand(i: 0)))
8943 return true;
8944 break;
8945 case Instruction::Switch:
8946 if (Handle(cast<SwitchInst>(Val: I)->getCondition()))
8947 return true;
8948 break;
8949 case Instruction::CondBr:
8950 if (Handle(cast<CondBrInst>(Val: I)->getCondition()))
8951 return true;
8952 break;
8953 default:
8954 break;
8955 }
8956
8957 return false;
8958}
8959
8960/// Enumerates all operands of \p I that are guaranteed to not be poison.
8961template <typename CallableT>
8962static bool handleGuaranteedNonPoisonOps(const Instruction *I,
8963 const CallableT &Handle) {
8964 if (handleGuaranteedWellDefinedOps(I, Handle))
8965 return true;
8966 switch (I->getOpcode()) {
8967 // Divisors of these operations are allowed to be partially undef.
8968 case Instruction::UDiv:
8969 case Instruction::SDiv:
8970 case Instruction::URem:
8971 case Instruction::SRem:
8972 return Handle(I->getOperand(i: 1));
8973 default:
8974 return false;
8975 }
8976}
8977
8978bool llvm::mustTriggerUB(const Instruction *I,
8979 const SmallPtrSetImpl<const Value *> &KnownPoison) {
8980 return handleGuaranteedNonPoisonOps(
8981 I, Handle: [&](const Value *V) { return KnownPoison.count(Ptr: V); });
8982}
8983
8984static bool programUndefinedIfUndefOrPoison(const Value *V,
8985 bool PoisonOnly) {
8986 // We currently only look for uses of values within the same basic
8987 // block, as that makes it easier to guarantee that the uses will be
8988 // executed given that Inst is executed.
8989 //
8990 // FIXME: Expand this to consider uses beyond the same basic block. To do
8991 // this, look out for the distinction between post-dominance and strong
8992 // post-dominance.
8993 const BasicBlock *BB = nullptr;
8994 BasicBlock::const_iterator Begin;
8995 if (const auto *Inst = dyn_cast<Instruction>(Val: V)) {
8996 BB = Inst->getParent();
8997 Begin = Inst->getIterator();
8998 Begin++;
8999 } else if (const auto *Arg = dyn_cast<Argument>(Val: V)) {
9000 if (Arg->getParent()->isDeclaration())
9001 return false;
9002 BB = &Arg->getParent()->getEntryBlock();
9003 Begin = BB->begin();
9004 } else {
9005 return false;
9006 }
9007
9008 // Limit number of instructions we look at, to avoid scanning through large
9009 // blocks. The current limit is chosen arbitrarily.
9010 unsigned ScanLimit = 32;
9011 BasicBlock::const_iterator End = BB->end();
9012
9013 if (!PoisonOnly) {
9014 // Since undef does not propagate eagerly, be conservative & just check
9015 // whether a value is directly passed to an instruction that must take
9016 // well-defined operands.
9017
9018 for (const auto &I : make_range(x: Begin, y: End)) {
9019 if (--ScanLimit == 0)
9020 break;
9021
9022 if (handleGuaranteedWellDefinedOps(I: &I, Handle: [V](const Value *WellDefinedOp) {
9023 return WellDefinedOp == V;
9024 }))
9025 return true;
9026
9027 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
9028 break;
9029 }
9030 return false;
9031 }
9032
9033 // Set of instructions that we have proved will yield poison if Inst
9034 // does.
9035 SmallPtrSet<const Value *, 16> YieldsPoison;
9036 SmallPtrSet<const BasicBlock *, 4> Visited;
9037
9038 YieldsPoison.insert(Ptr: V);
9039 Visited.insert(Ptr: BB);
9040
9041 while (true) {
9042 for (const auto &I : make_range(x: Begin, y: End)) {
9043 if (--ScanLimit == 0)
9044 return false;
9045 if (mustTriggerUB(I: &I, KnownPoison: YieldsPoison))
9046 return true;
9047 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
9048 return false;
9049
9050 // If an operand is poison and propagates it, mark I as yielding poison.
9051 for (const Use &Op : I.operands()) {
9052 if (YieldsPoison.count(Ptr: Op) && propagatesPoison(PoisonOp: Op)) {
9053 YieldsPoison.insert(Ptr: &I);
9054 break;
9055 }
9056 }
9057
9058 // Special handling for select, which returns poison if its operand 0 is
9059 // poison (handled in the loop above) *or* if both its true/false operands
9060 // are poison (handled here).
9061 if (I.getOpcode() == Instruction::Select &&
9062 YieldsPoison.count(Ptr: I.getOperand(i: 1)) &&
9063 YieldsPoison.count(Ptr: I.getOperand(i: 2))) {
9064 YieldsPoison.insert(Ptr: &I);
9065 }
9066 }
9067
9068 BB = BB->getSingleSuccessor();
9069 if (!BB || !Visited.insert(Ptr: BB).second)
9070 break;
9071
9072 Begin = BB->getFirstNonPHIIt();
9073 End = BB->end();
9074 }
9075 return false;
9076}
9077
9078bool llvm::programUndefinedIfUndefOrPoison(const Instruction *Inst) {
9079 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: false);
9080}
9081
9082bool llvm::programUndefinedIfPoison(const Instruction *Inst) {
9083 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: true);
9084}
9085
9086static bool isKnownNonNaN(const Value *V, FastMathFlags FMF) {
9087 if (FMF.noNaNs())
9088 return true;
9089
9090 if (auto *C = dyn_cast<ConstantFP>(Val: V))
9091 return !C->isNaN();
9092
9093 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
9094 if (!C->getElementType()->isFloatingPointTy())
9095 return false;
9096 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
9097 if (C->getElementAsAPFloat(i: I).isNaN())
9098 return false;
9099 }
9100 return true;
9101 }
9102
9103 if (isa<ConstantAggregateZero>(Val: V))
9104 return true;
9105
9106 return false;
9107}
9108
9109static bool isKnownNonZero(const Value *V) {
9110 if (auto *C = dyn_cast<ConstantFP>(Val: V))
9111 return !C->isZero();
9112
9113 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
9114 if (!C->getElementType()->isFloatingPointTy())
9115 return false;
9116 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
9117 if (C->getElementAsAPFloat(i: I).isZero())
9118 return false;
9119 }
9120 return true;
9121 }
9122
9123 return false;
9124}
9125
9126/// Match clamp pattern for float types without care about NaNs or signed zeros.
9127/// Given non-min/max outer cmp/select from the clamp pattern this
9128/// function recognizes if it can be substitued by a "canonical" min/max
9129/// pattern.
9130static SelectPatternResult matchFastFloatClamp(CmpInst::Predicate Pred,
9131 Value *CmpLHS, Value *CmpRHS,
9132 Value *TrueVal, Value *FalseVal,
9133 Value *&LHS, Value *&RHS) {
9134 // Try to match
9135 // X < C1 ? C1 : Min(X, C2) --> Max(C1, Min(X, C2))
9136 // X > C1 ? C1 : Max(X, C2) --> Min(C1, Max(X, C2))
9137 // and return description of the outer Max/Min.
9138
9139 // First, check if select has inverse order:
9140 if (CmpRHS == FalseVal) {
9141 std::swap(a&: TrueVal, b&: FalseVal);
9142 Pred = CmpInst::getInversePredicate(pred: Pred);
9143 }
9144
9145 // Assume success now. If there's no match, callers should not use these anyway.
9146 LHS = TrueVal;
9147 RHS = FalseVal;
9148
9149 const APFloat *FC1;
9150 if (CmpRHS != TrueVal || !match(V: CmpRHS, P: m_APFloat(Res&: FC1)) || !FC1->isFinite())
9151 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9152
9153 const APFloat *FC2;
9154 switch (Pred) {
9155 case CmpInst::FCMP_OLT:
9156 case CmpInst::FCMP_OLE:
9157 case CmpInst::FCMP_ULT:
9158 case CmpInst::FCMP_ULE:
9159 if (match(V: FalseVal, P: m_OrdOrUnordFMin(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
9160 *FC1 < *FC2)
9161 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
9162 if (match(V: FalseVal, P: m_FMinNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
9163 *FC1 < *FC2)
9164 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
9165 break;
9166 case CmpInst::FCMP_OGT:
9167 case CmpInst::FCMP_OGE:
9168 case CmpInst::FCMP_UGT:
9169 case CmpInst::FCMP_UGE:
9170 if (match(V: FalseVal, P: m_OrdOrUnordFMax(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
9171 *FC1 > *FC2)
9172 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
9173 if (match(V: FalseVal, P: m_FMaxNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
9174 *FC1 > *FC2)
9175 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
9176 break;
9177 default:
9178 break;
9179 }
9180
9181 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9182}
9183
9184/// Recognize variations of:
9185/// CLAMP(v,l,h) ==> ((v) < (l) ? (l) : ((v) > (h) ? (h) : (v)))
9186static SelectPatternResult matchClamp(CmpInst::Predicate Pred,
9187 Value *CmpLHS, Value *CmpRHS,
9188 Value *TrueVal, Value *FalseVal) {
9189 // Swap the select operands and predicate to match the patterns below.
9190 if (CmpRHS != TrueVal) {
9191 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
9192 std::swap(a&: TrueVal, b&: FalseVal);
9193 }
9194 const APInt *C1;
9195 if (CmpRHS == TrueVal && match(V: CmpRHS, P: m_APInt(Res&: C1))) {
9196 const APInt *C2;
9197 // (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
9198 if (match(V: FalseVal, P: m_SMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
9199 C1->slt(RHS: *C2) && Pred == CmpInst::ICMP_SLT)
9200 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9201
9202 // (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
9203 if (match(V: FalseVal, P: m_SMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
9204 C1->sgt(RHS: *C2) && Pred == CmpInst::ICMP_SGT)
9205 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9206
9207 // (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
9208 if (match(V: FalseVal, P: m_UMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
9209 C1->ult(RHS: *C2) && Pred == CmpInst::ICMP_ULT)
9210 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9211
9212 // (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
9213 if (match(V: FalseVal, P: m_UMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
9214 C1->ugt(RHS: *C2) && Pred == CmpInst::ICMP_UGT)
9215 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9216 }
9217 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9218}
9219
9220/// If the input value is the result of a 'not' op, constant integer, or vector
9221/// splat of a constant integer, return the bitwise-not source value.
9222/// TODO: This could be extended to handle non-splat vector integer constants.
9223static Value *getNotValue(Value *V) {
9224 Value *NotV;
9225 if (match(V, P: m_Not(V: m_Value(V&: NotV))))
9226 return NotV;
9227
9228 const APInt *C;
9229 if (match(V, P: m_APInt(Res&: C)))
9230 return ConstantInt::get(Ty: V->getType(), V: ~(*C));
9231
9232 return nullptr;
9233}
9234
9235/// Match non-obvious integer minimum and maximum sequences.
9236static SelectPatternResult matchMinMax(CmpInst::Predicate Pred, Value *CmpLHS,
9237 Value *CmpRHS, Value *TrueVal,
9238 Value *FalseVal, Value *&LHS,
9239 Value *&RHS) {
9240 // Assume success. If there's no match, callers should not use these anyway.
9241 LHS = TrueVal;
9242 RHS = FalseVal;
9243
9244 SelectPatternResult SPR = matchClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal);
9245 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
9246 return SPR;
9247
9248 // Look through 'not' ops to find disguised min/max.
9249 // (X > Y) ? ~X : ~Y ==> (~X < ~Y) ? ~X : ~Y ==> MIN(~X, ~Y)
9250 // (X < Y) ? ~X : ~Y ==> (~X > ~Y) ? ~X : ~Y ==> MAX(~X, ~Y)
9251 if (CmpLHS == getNotValue(V: TrueVal) && CmpRHS == getNotValue(V: FalseVal)) {
9252 switch (Pred) {
9253 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9254 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9255 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9256 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9257 default: break;
9258 }
9259 }
9260
9261 // (X > Y) ? ~Y : ~X ==> (~X < ~Y) ? ~Y : ~X ==> MAX(~Y, ~X)
9262 // (X < Y) ? ~Y : ~X ==> (~X > ~Y) ? ~Y : ~X ==> MIN(~Y, ~X)
9263 if (CmpLHS == getNotValue(V: FalseVal) && CmpRHS == getNotValue(V: TrueVal)) {
9264 switch (Pred) {
9265 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9266 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9267 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9268 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9269 default: break;
9270 }
9271 }
9272
9273 if (Pred != CmpInst::ICMP_SGT && Pred != CmpInst::ICMP_SLT)
9274 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9275
9276 const APInt *C1;
9277 if (!match(V: CmpRHS, P: m_APInt(Res&: C1)))
9278 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9279
9280 // An unsigned min/max can be written with a signed compare.
9281 const APInt *C2;
9282 if ((CmpLHS == TrueVal && match(V: FalseVal, P: m_APInt(Res&: C2))) ||
9283 (CmpLHS == FalseVal && match(V: TrueVal, P: m_APInt(Res&: C2)))) {
9284 // Is the sign bit set?
9285 // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
9286 // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
9287 if (Pred == CmpInst::ICMP_SLT && C1->isZero() && C2->isMaxSignedValue())
9288 return {.Flavor: CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9289
9290 // Is the sign bit clear?
9291 // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
9292 // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
9293 if (Pred == CmpInst::ICMP_SGT && C1->isAllOnes() && C2->isMinSignedValue())
9294 return {.Flavor: CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9295 }
9296
9297 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9298}
9299
9300bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW,
9301 bool AllowPoison) {
9302 assert(X && Y && "Invalid operand");
9303
9304 auto IsNegationOf = [&](const Value *X, const Value *Y) {
9305 if (!match(V: X, P: m_Neg(V: m_Specific(V: Y))))
9306 return false;
9307
9308 auto *BO = cast<BinaryOperator>(Val: X);
9309 if (NeedNSW && !BO->hasNoSignedWrap())
9310 return false;
9311
9312 auto *Zero = cast<Constant>(Val: BO->getOperand(i_nocapture: 0));
9313 if (!AllowPoison && !Zero->isNullValue())
9314 return false;
9315
9316 return true;
9317 };
9318
9319 // X = -Y or Y = -X
9320 if (IsNegationOf(X, Y) || IsNegationOf(Y, X))
9321 return true;
9322
9323 // X = sub (A, B), Y = sub (B, A) || X = sub nsw (A, B), Y = sub nsw (B, A)
9324 Value *A, *B;
9325 return (!NeedNSW && (match(V: X, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9326 match(V: Y, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))))) ||
9327 (NeedNSW && (match(V: X, P: m_NSWSub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9328 match(V: Y, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A)))));
9329}
9330
9331bool llvm::isKnownInversion(const Value *X, const Value *Y) {
9332 // Handle X = icmp pred A, B, Y = icmp pred A, C.
9333 Value *A, *B, *C;
9334 CmpPredicate Pred1, Pred2;
9335 if (!match(V: X, P: m_ICmp(Pred&: Pred1, L: m_Value(V&: A), R: m_Value(V&: B))) ||
9336 !match(V: Y, P: m_c_ICmp(Pred&: Pred2, L: m_Specific(V: A), R: m_Value(V&: C))))
9337 return false;
9338
9339 // They must both have samesign flag or not.
9340 if (Pred1.hasSameSign() != Pred2.hasSameSign())
9341 return false;
9342
9343 if (B == C)
9344 return Pred1 == ICmpInst::getInversePredicate(pred: Pred2);
9345
9346 // Try to infer the relationship from constant ranges.
9347 const APInt *RHSC1, *RHSC2;
9348 if (!match(V: B, P: m_APInt(Res&: RHSC1)) || !match(V: C, P: m_APInt(Res&: RHSC2)))
9349 return false;
9350
9351 // Sign bits of two RHSCs should match.
9352 if (Pred1.hasSameSign() && RHSC1->isNonNegative() != RHSC2->isNonNegative())
9353 return false;
9354
9355 const auto CR1 = ConstantRange::makeExactICmpRegion(Pred: Pred1, Other: *RHSC1);
9356 const auto CR2 = ConstantRange::makeExactICmpRegion(Pred: Pred2, Other: *RHSC2);
9357
9358 return CR1.inverse() == CR2;
9359}
9360
9361SelectPatternResult llvm::getSelectPattern(CmpInst::Predicate Pred,
9362 SelectPatternNaNBehavior NaNBehavior,
9363 bool Ordered) {
9364 switch (Pred) {
9365 default:
9366 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false}; // Equality.
9367 case ICmpInst::ICMP_UGT:
9368 case ICmpInst::ICMP_UGE:
9369 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9370 case ICmpInst::ICMP_SGT:
9371 case ICmpInst::ICMP_SGE:
9372 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9373 case ICmpInst::ICMP_ULT:
9374 case ICmpInst::ICMP_ULE:
9375 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9376 case ICmpInst::ICMP_SLT:
9377 case ICmpInst::ICMP_SLE:
9378 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9379 case FCmpInst::FCMP_UGT:
9380 case FCmpInst::FCMP_UGE:
9381 case FCmpInst::FCMP_OGT:
9382 case FCmpInst::FCMP_OGE:
9383 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
9384 case FCmpInst::FCMP_ULT:
9385 case FCmpInst::FCMP_ULE:
9386 case FCmpInst::FCMP_OLT:
9387 case FCmpInst::FCMP_OLE:
9388 return {.Flavor: SPF_FMINNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
9389 }
9390}
9391
9392std::optional<std::pair<CmpPredicate, Constant *>>
9393llvm::getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred, Constant *C) {
9394 assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
9395 "Only for relational integer predicates.");
9396 if (isa<UndefValue>(Val: C))
9397 return std::nullopt;
9398
9399 Type *Type = C->getType();
9400 bool IsSigned = ICmpInst::isSigned(Pred);
9401
9402 CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred);
9403 bool WillIncrement =
9404 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
9405
9406 // Check if the constant operand can be safely incremented/decremented
9407 // without overflowing/underflowing.
9408 auto ConstantIsOk = [Pred, WillIncrement, IsSigned](ConstantInt *C) {
9409 if (WillIncrement ? C->isMaxValue(IsSigned) : C->isMinValue(IsSigned))
9410 return false;
9411
9412 if (!Pred.hasSameSign())
9413 return true;
9414
9415 // Crossing the corresponding boundary in the other ordering changes the
9416 // sign bit, and therefore changes the poison domain.
9417 return WillIncrement ? !C->isMaxValue(IsSigned: !IsSigned)
9418 : !C->isMinValue(IsSigned: !IsSigned);
9419 };
9420
9421 Constant *SafeReplacementConstant = nullptr;
9422 if (auto *CI = dyn_cast<ConstantInt>(Val: C)) {
9423 // Bail out if the constant can't be safely incremented/decremented.
9424 if (!ConstantIsOk(CI))
9425 return std::nullopt;
9426 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Type)) {
9427 unsigned NumElts = FVTy->getNumElements();
9428 for (unsigned i = 0; i != NumElts; ++i) {
9429 Constant *Elt = C->getAggregateElement(Elt: i);
9430 if (!Elt)
9431 return std::nullopt;
9432
9433 if (isa<UndefValue>(Val: Elt))
9434 continue;
9435
9436 // Bail out if we can't determine if this constant is min/max or if we
9437 // know that this constant is min/max.
9438 auto *CI = dyn_cast<ConstantInt>(Val: Elt);
9439 if (!CI || !ConstantIsOk(CI))
9440 return std::nullopt;
9441
9442 if (!SafeReplacementConstant)
9443 SafeReplacementConstant = CI;
9444 }
9445 } else if (isa<VectorType>(Val: C->getType())) {
9446 // Handle scalable splat
9447 Value *SplatC = C->getSplatValue();
9448 auto *CI = dyn_cast_or_null<ConstantInt>(Val: SplatC);
9449 // Bail out if the constant can't be safely incremented/decremented.
9450 if (!CI || !ConstantIsOk(CI))
9451 return std::nullopt;
9452 } else {
9453 // ConstantExpr?
9454 return std::nullopt;
9455 }
9456
9457 // It may not be safe to change a compare predicate in the presence of
9458 // undefined elements, so replace those elements with the first safe constant
9459 // that we found.
9460 // TODO: in case of poison, it is safe; let's replace undefs only.
9461 if (C->containsUndefOrPoisonElement()) {
9462 assert(SafeReplacementConstant && "Replacement constant not set");
9463 C = Constant::replaceUndefsWith(C, Replacement: SafeReplacementConstant);
9464 }
9465
9466 CmpPredicate NewPred(CmpInst::getFlippedStrictnessPredicate(pred: Pred),
9467 Pred.hasSameSign());
9468
9469 // Increment or decrement the constant.
9470 Constant *OneOrNegOne = ConstantInt::get(Ty: Type, V: WillIncrement ? 1 : -1, IsSigned: true);
9471 Constant *NewC = ConstantExpr::getAdd(C1: C, C2: OneOrNegOne);
9472
9473 return std::make_pair(x&: NewPred, y&: NewC);
9474}
9475
9476static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
9477 FastMathFlags FMF, Value *CmpLHS,
9478 Value *CmpRHS, Value *TrueVal,
9479 Value *FalseVal, Value *&LHS,
9480 Value *&RHS) {
9481 if (CmpInst::isFPPredicate(P: Pred)) {
9482 // IEEE-754 ignores the sign of 0.0 in comparisons. So if the select has one
9483 // 0.0 operand, set the compare's 0.0 operands to that same value for the
9484 // purpose of identifying min/max. Disregard vector constants with undefined
9485 // elements because those can not be back-propagated for analysis.
9486 Value *OutputZeroVal = nullptr;
9487 if (match(V: TrueVal, P: m_AnyZeroFP()) && !match(V: FalseVal, P: m_AnyZeroFP()) &&
9488 !cast<Constant>(Val: TrueVal)->containsUndefOrPoisonElement())
9489 OutputZeroVal = TrueVal;
9490 else if (match(V: FalseVal, P: m_AnyZeroFP()) && !match(V: TrueVal, P: m_AnyZeroFP()) &&
9491 !cast<Constant>(Val: FalseVal)->containsUndefOrPoisonElement())
9492 OutputZeroVal = FalseVal;
9493
9494 if (OutputZeroVal) {
9495 if (match(V: CmpLHS, P: m_AnyZeroFP()) && CmpLHS != OutputZeroVal)
9496 CmpLHS = OutputZeroVal;
9497 if (match(V: CmpRHS, P: m_AnyZeroFP()) && CmpRHS != OutputZeroVal)
9498 CmpRHS = OutputZeroVal;
9499 }
9500 }
9501
9502 LHS = CmpLHS;
9503 RHS = CmpRHS;
9504
9505 // Signed zero may return inconsistent results between implementations.
9506 // (0.0 <= -0.0) ? 0.0 : -0.0 // Returns 0.0
9507 // minNum(0.0, -0.0) // May return -0.0 or 0.0 (IEEE 754-2008 5.3.1)
9508 // Therefore, we behave conservatively and only proceed if at least one of the
9509 // operands is known to not be zero or if we don't care about signed zero.
9510 if (CmpInst::isFPPredicate(P: Pred)) {
9511 if (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9512 !isKnownNonZero(V: CmpRHS))
9513 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9514 }
9515
9516 SelectPatternNaNBehavior NaNBehavior = SPNB_NA;
9517 bool Ordered = false;
9518
9519 // When given one NaN and one non-NaN input:
9520 // - maxnum/minnum (C99 fmaxf()/fminf()) return the non-NaN input.
9521 // - A simple C99 (a < b ? a : b) construction will return 'b' (as the
9522 // ordered comparison fails), which could be NaN or non-NaN.
9523 // so here we discover exactly what NaN behavior is required/accepted.
9524 if (CmpInst::isFPPredicate(P: Pred)) {
9525 bool LHSSafe = isKnownNonNaN(V: CmpLHS, FMF);
9526 bool RHSSafe = isKnownNonNaN(V: CmpRHS, FMF);
9527
9528 if (LHSSafe && RHSSafe) {
9529 // Both operands are known non-NaN.
9530 NaNBehavior = SPNB_RETURNS_ANY;
9531 Ordered = CmpInst::isOrdered(predicate: Pred);
9532 } else if (CmpInst::isOrdered(predicate: Pred)) {
9533 // An ordered comparison will return false when given a NaN, so it
9534 // returns the RHS.
9535 Ordered = true;
9536 if (LHSSafe)
9537 // LHS is non-NaN, so if RHS is NaN then NaN will be returned.
9538 NaNBehavior = SPNB_RETURNS_NAN;
9539 else if (RHSSafe)
9540 NaNBehavior = SPNB_RETURNS_OTHER;
9541 else
9542 // Completely unsafe.
9543 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9544 } else {
9545 Ordered = false;
9546 // An unordered comparison will return true when given a NaN, so it
9547 // returns the LHS.
9548 if (LHSSafe)
9549 // LHS is non-NaN, so if RHS is NaN then non-NaN will be returned.
9550 NaNBehavior = SPNB_RETURNS_OTHER;
9551 else if (RHSSafe)
9552 NaNBehavior = SPNB_RETURNS_NAN;
9553 else
9554 // Completely unsafe.
9555 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9556 }
9557 }
9558
9559 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
9560 std::swap(a&: CmpLHS, b&: CmpRHS);
9561 Pred = CmpInst::getSwappedPredicate(pred: Pred);
9562 if (NaNBehavior == SPNB_RETURNS_NAN)
9563 NaNBehavior = SPNB_RETURNS_OTHER;
9564 else if (NaNBehavior == SPNB_RETURNS_OTHER)
9565 NaNBehavior = SPNB_RETURNS_NAN;
9566 Ordered = !Ordered;
9567 }
9568
9569 // ([if]cmp X, Y) ? X : Y
9570 if (TrueVal == CmpLHS && FalseVal == CmpRHS)
9571 return getSelectPattern(Pred, NaNBehavior, Ordered);
9572
9573 if (isKnownNegation(X: TrueVal, Y: FalseVal)) {
9574 // Sign-extending LHS does not change its sign, so TrueVal/FalseVal can
9575 // match against either LHS or sign-preserving operations on LHS, like
9576 // sext(LHS), or binary ops that do not wrap in signed sense.
9577 auto CmpLHSOrSExt =
9578 m_CombineOr(Ps: m_Specific(V: CmpLHS), Ps: m_SExt(Op: m_Specific(V: CmpLHS)));
9579 auto MaybeSExtOrMulCmpLHS =
9580 m_CombineOr(Ps: CmpLHSOrSExt, Ps: m_NSWMul(L: CmpLHSOrSExt, R: m_StrictlyPositive()),
9581 Ps: m_NSWShl(L: CmpLHSOrSExt, R: m_Value()));
9582 auto ZeroOrAllOnes = m_CombineOr(Ps: m_ZeroInt(), Ps: m_AllOnes());
9583 auto ZeroOrOne = m_CombineOr(Ps: m_ZeroInt(), Ps: m_One());
9584 if (match(V: TrueVal, P: MaybeSExtOrMulCmpLHS)) {
9585 // Set the return values. If the compare uses the negated value (-X >s 0),
9586 // swap the return values because the negated value is always 'RHS'.
9587 LHS = TrueVal;
9588 RHS = FalseVal;
9589 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: FalseVal))))
9590 std::swap(a&: LHS, b&: RHS);
9591
9592 // (X >s 0) ? X : -X or (X >s -1) ? X : -X --> ABS(X)
9593 // (-X >s 0) ? -X : X or (-X >s -1) ? -X : X --> ABS(X)
9594 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9595 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9596
9597 // (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X)
9598 if (Pred == ICmpInst::ICMP_SGE && match(V: CmpRHS, P: ZeroOrOne))
9599 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9600
9601 // (X <s 0) ? X : -X or (X <s 1) ? X : -X --> NABS(X)
9602 // (-X <s 0) ? -X : X or (-X <s 1) ? -X : X --> NABS(X)
9603 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9604 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9605 } else if (match(V: FalseVal, P: MaybeSExtOrMulCmpLHS)) {
9606 // Set the return values. If the compare uses the negated value (-X >s 0),
9607 // swap the return values because the negated value is always 'RHS'.
9608 LHS = FalseVal;
9609 RHS = TrueVal;
9610 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: TrueVal))))
9611 std::swap(a&: LHS, b&: RHS);
9612
9613 // (X >s 0) ? -X : X or (X >s -1) ? -X : X --> NABS(X)
9614 // (-X >s 0) ? X : -X or (-X >s -1) ? X : -X --> NABS(X)
9615 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9616 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9617
9618 // (X <s 0) ? -X : X or (X <s 1) ? -X : X --> ABS(X)
9619 // (-X <s 0) ? X : -X or (-X <s 1) ? X : -X --> ABS(X)
9620 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9621 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9622 }
9623 }
9624
9625 if (CmpInst::isIntPredicate(P: Pred))
9626 return matchMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
9627
9628 // According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar
9629 // may return either -0.0 or 0.0, so fcmp/select pair has stricter
9630 // semantics than minNum. Be conservative in such case.
9631 if (NaNBehavior != SPNB_RETURNS_ANY ||
9632 (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9633 !isKnownNonZero(V: CmpRHS)))
9634 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9635
9636 return matchFastFloatClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
9637}
9638
9639static Value *lookThroughCastConst(CmpInst *CmpI, Type *SrcTy, Constant *C,
9640 Instruction::CastOps *CastOp) {
9641 const DataLayout &DL = CmpI->getDataLayout();
9642
9643 Constant *CastedTo = nullptr;
9644 switch (*CastOp) {
9645 case Instruction::ZExt:
9646 if (CmpI->isUnsigned())
9647 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy);
9648 break;
9649 case Instruction::SExt:
9650 if (CmpI->isSigned())
9651 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy, OnlyIfReduced: true);
9652 break;
9653 case Instruction::Trunc:
9654 Constant *CmpConst;
9655 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_Constant(C&: CmpConst)) &&
9656 CmpConst->getType() == SrcTy) {
9657 // Here we have the following case:
9658 //
9659 // %cond = cmp iN %x, CmpConst
9660 // %tr = trunc iN %x to iK
9661 // %narrowsel = select i1 %cond, iK %t, iK C
9662 //
9663 // We can always move trunc after select operation:
9664 //
9665 // %cond = cmp iN %x, CmpConst
9666 // %widesel = select i1 %cond, iN %x, iN CmpConst
9667 // %tr = trunc iN %widesel to iK
9668 //
9669 // Note that C could be extended in any way because we don't care about
9670 // upper bits after truncation. It can't be abs pattern, because it would
9671 // look like:
9672 //
9673 // select i1 %cond, x, -x.
9674 //
9675 // So only min/max pattern could be matched. Such match requires widened C
9676 // == CmpConst. That is why set widened C = CmpConst, condition trunc
9677 // CmpConst == C is checked below.
9678 CastedTo = CmpConst;
9679 } else {
9680 unsigned ExtOp = CmpI->isSigned() ? Instruction::SExt : Instruction::ZExt;
9681 CastedTo = ConstantFoldCastOperand(Opcode: ExtOp, C, DestTy: SrcTy, DL);
9682 }
9683 break;
9684 case Instruction::FPTrunc:
9685 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPExt, C, DestTy: SrcTy, DL);
9686 break;
9687 case Instruction::FPExt:
9688 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPTrunc, C, DestTy: SrcTy, DL);
9689 break;
9690 case Instruction::FPToUI:
9691 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::UIToFP, C, DestTy: SrcTy, DL);
9692 break;
9693 case Instruction::FPToSI:
9694 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::SIToFP, C, DestTy: SrcTy, DL);
9695 break;
9696 case Instruction::UIToFP:
9697 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToUI, C, DestTy: SrcTy, DL);
9698 break;
9699 case Instruction::SIToFP:
9700 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToSI, C, DestTy: SrcTy, DL);
9701 break;
9702 default:
9703 break;
9704 }
9705
9706 if (!CastedTo)
9707 return nullptr;
9708
9709 // Make sure the cast doesn't lose any information.
9710 Constant *CastedBack =
9711 ConstantFoldCastOperand(Opcode: *CastOp, C: CastedTo, DestTy: C->getType(), DL);
9712 if (CastedBack && CastedBack != C)
9713 return nullptr;
9714
9715 return CastedTo;
9716}
9717
9718/// Helps to match a select pattern in case of a type mismatch.
9719///
9720/// The function processes the case when type of true and false values of a
9721/// select instruction differs from type of the cmp instruction operands because
9722/// of a cast instruction. The function checks if it is legal to move the cast
9723/// operation after "select". If yes, it returns the new second value of
9724/// "select" (with the assumption that cast is moved):
9725/// 1. As operand of cast instruction when both values of "select" are same cast
9726/// instructions.
9727/// 2. As restored constant (by applying reverse cast operation) when the first
9728/// value of the "select" is a cast operation and the second value is a
9729/// constant. It is implemented in lookThroughCastConst().
9730/// 3. As one operand is cast instruction and the other is not. The operands in
9731/// sel(cmp) are in different type integer.
9732/// NOTE: We return only the new second value because the first value could be
9733/// accessed as operand of cast instruction.
9734static Value *lookThroughCast(CmpInst *CmpI, Value *V1, Value *V2,
9735 Instruction::CastOps *CastOp) {
9736 auto *Cast1 = dyn_cast<CastInst>(Val: V1);
9737 if (!Cast1)
9738 return nullptr;
9739
9740 *CastOp = Cast1->getOpcode();
9741 Type *SrcTy = Cast1->getSrcTy();
9742 if (auto *Cast2 = dyn_cast<CastInst>(Val: V2)) {
9743 // If V1 and V2 are both the same cast from the same type, look through V1.
9744 if (*CastOp == Cast2->getOpcode() && SrcTy == Cast2->getSrcTy())
9745 return Cast2->getOperand(i_nocapture: 0);
9746 return nullptr;
9747 }
9748
9749 auto *C = dyn_cast<Constant>(Val: V2);
9750 if (C)
9751 return lookThroughCastConst(CmpI, SrcTy, C, CastOp);
9752
9753 Value *CastedTo = nullptr;
9754 if (*CastOp == Instruction::Trunc) {
9755 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_ZExtOrSExt(Op: m_Specific(V: V2)))) {
9756 // Here we have the following case:
9757 // %y_ext = sext iK %y to iN
9758 // %cond = cmp iN %x, %y_ext
9759 // %tr = trunc iN %x to iK
9760 // %narrowsel = select i1 %cond, iK %tr, iK %y
9761 //
9762 // We can always move trunc after select operation:
9763 // %y_ext = sext iK %y to iN
9764 // %cond = cmp iN %x, %y_ext
9765 // %widesel = select i1 %cond, iN %x, iN %y_ext
9766 // %tr = trunc iN %widesel to iK
9767 assert(V2->getType() == Cast1->getType() &&
9768 "V2 and Cast1 should be the same type.");
9769 CastedTo = CmpI->getOperand(i_nocapture: 1);
9770 }
9771 }
9772
9773 return CastedTo;
9774}
9775SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
9776 Instruction::CastOps *CastOp) {
9777 SelectInst *SI = dyn_cast<SelectInst>(Val: V);
9778 if (!SI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9779
9780 CmpInst *CmpI = dyn_cast<CmpInst>(Val: SI->getCondition());
9781 if (!CmpI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9782
9783 Value *TrueVal = SI->getTrueValue();
9784 Value *FalseVal = SI->getFalseValue();
9785
9786 return llvm::matchDecomposedSelectPattern(
9787 CmpI, TrueVal, FalseVal, LHS, RHS, FMF: SI->getFastMathFlagsOrNone(), CastOp);
9788}
9789
9790SelectPatternResult llvm::matchDecomposedSelectPattern(
9791 CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
9792 FastMathFlags FMF, Instruction::CastOps *CastOp) {
9793 CmpInst::Predicate Pred = CmpI->getPredicate();
9794 Value *CmpLHS = CmpI->getOperand(i_nocapture: 0);
9795 Value *CmpRHS = CmpI->getOperand(i_nocapture: 1);
9796 if (isa<FPMathOperator>(Val: CmpI) && CmpI->hasNoNaNs())
9797 FMF.setNoNaNs();
9798
9799 // Bail out early.
9800 if (CmpI->isEquality())
9801 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9802
9803 // Deal with type mismatches.
9804 if (CastOp && CmpLHS->getType() != TrueVal->getType()) {
9805 if (Value *C = lookThroughCast(CmpI, V1: TrueVal, V2: FalseVal, CastOp)) {
9806 // If this is a potential fmin/fmax with a cast to integer, then ignore
9807 // -0.0 because there is no corresponding integer value.
9808 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9809 FMF.setNoSignedZeros();
9810 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9811 TrueVal: cast<CastInst>(Val: TrueVal)->getOperand(i_nocapture: 0), FalseVal: C,
9812 LHS, RHS);
9813 }
9814 if (Value *C = lookThroughCast(CmpI, V1: FalseVal, V2: TrueVal, CastOp)) {
9815 // If this is a potential fmin/fmax with a cast to integer, then ignore
9816 // -0.0 because there is no corresponding integer value.
9817 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9818 FMF.setNoSignedZeros();
9819 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal: C,
9820 FalseVal: cast<CastInst>(Val: FalseVal)->getOperand(i_nocapture: 0), LHS,
9821 RHS);
9822 }
9823 }
9824 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS,
9825 RHS);
9826}
9827
9828CmpInst::Predicate llvm::getMinMaxPred(SelectPatternFlavor SPF, bool Ordered) {
9829 if (SPF == SPF_SMIN) return ICmpInst::ICMP_SLT;
9830 if (SPF == SPF_UMIN) return ICmpInst::ICMP_ULT;
9831 if (SPF == SPF_SMAX) return ICmpInst::ICMP_SGT;
9832 if (SPF == SPF_UMAX) return ICmpInst::ICMP_UGT;
9833 if (SPF == SPF_FMINNUM)
9834 return Ordered ? FCmpInst::FCMP_OLT : FCmpInst::FCMP_ULT;
9835 if (SPF == SPF_FMAXNUM)
9836 return Ordered ? FCmpInst::FCMP_OGT : FCmpInst::FCMP_UGT;
9837 llvm_unreachable("unhandled!");
9838}
9839
9840Intrinsic::ID llvm::getMinMaxIntrinsic(SelectPatternFlavor SPF) {
9841 switch (SPF) {
9842 case SelectPatternFlavor::SPF_UMIN:
9843 return Intrinsic::umin;
9844 case SelectPatternFlavor::SPF_UMAX:
9845 return Intrinsic::umax;
9846 case SelectPatternFlavor::SPF_SMIN:
9847 return Intrinsic::smin;
9848 case SelectPatternFlavor::SPF_SMAX:
9849 return Intrinsic::smax;
9850 default:
9851 llvm_unreachable("Unexpected SPF");
9852 }
9853}
9854
9855SelectPatternFlavor llvm::getInverseMinMaxFlavor(SelectPatternFlavor SPF) {
9856 if (SPF == SPF_SMIN) return SPF_SMAX;
9857 if (SPF == SPF_UMIN) return SPF_UMAX;
9858 if (SPF == SPF_SMAX) return SPF_SMIN;
9859 if (SPF == SPF_UMAX) return SPF_UMIN;
9860 llvm_unreachable("unhandled!");
9861}
9862
9863Intrinsic::ID llvm::getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID) {
9864 switch (MinMaxID) {
9865 case Intrinsic::smax: return Intrinsic::smin;
9866 case Intrinsic::smin: return Intrinsic::smax;
9867 case Intrinsic::umax: return Intrinsic::umin;
9868 case Intrinsic::umin: return Intrinsic::umax;
9869 // Please note that next four intrinsics may produce the same result for
9870 // original and inverted case even if X != Y due to NaN is handled specially.
9871 case Intrinsic::maximum: return Intrinsic::minimum;
9872 case Intrinsic::minimum: return Intrinsic::maximum;
9873 case Intrinsic::maxnum: return Intrinsic::minnum;
9874 case Intrinsic::minnum: return Intrinsic::maxnum;
9875 case Intrinsic::maximumnum:
9876 return Intrinsic::minimumnum;
9877 case Intrinsic::minimumnum:
9878 return Intrinsic::maximumnum;
9879 default: llvm_unreachable("Unexpected intrinsic");
9880 }
9881}
9882
9883APInt llvm::getMinMaxLimit(SelectPatternFlavor SPF, unsigned BitWidth) {
9884 switch (SPF) {
9885 case SPF_SMAX: return APInt::getSignedMaxValue(numBits: BitWidth);
9886 case SPF_SMIN: return APInt::getSignedMinValue(numBits: BitWidth);
9887 case SPF_UMAX: return APInt::getMaxValue(numBits: BitWidth);
9888 case SPF_UMIN: return APInt::getMinValue(numBits: BitWidth);
9889 default: llvm_unreachable("Unexpected flavor");
9890 }
9891}
9892
9893std::pair<Intrinsic::ID, bool>
9894llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
9895 // Check if VL contains select instructions that can be folded into a min/max
9896 // vector intrinsic and return the intrinsic if it is possible.
9897 // TODO: Support floating point min/max.
9898 bool AllCmpSingleUse = true;
9899 SelectPatternResult SelectPattern;
9900 SelectPattern.Flavor = SPF_UNKNOWN;
9901 if (all_of(Range&: VL, P: [&SelectPattern, &AllCmpSingleUse](Value *I) {
9902 Value *LHS, *RHS;
9903 auto CurrentPattern = matchSelectPattern(V: I, LHS, RHS);
9904 if (!SelectPatternResult::isMinOrMax(SPF: CurrentPattern.Flavor))
9905 return false;
9906 if (SelectPattern.Flavor != SPF_UNKNOWN &&
9907 SelectPattern.Flavor != CurrentPattern.Flavor)
9908 return false;
9909 SelectPattern = CurrentPattern;
9910 AllCmpSingleUse &=
9911 match(V: I, P: m_Select(C: m_OneUse(SubPattern: m_Value()), L: m_Value(), R: m_Value()));
9912 return true;
9913 })) {
9914 switch (SelectPattern.Flavor) {
9915 case SPF_SMIN:
9916 return {Intrinsic::smin, AllCmpSingleUse};
9917 case SPF_UMIN:
9918 return {Intrinsic::umin, AllCmpSingleUse};
9919 case SPF_SMAX:
9920 return {Intrinsic::smax, AllCmpSingleUse};
9921 case SPF_UMAX:
9922 return {Intrinsic::umax, AllCmpSingleUse};
9923 case SPF_FMAXNUM:
9924 return {Intrinsic::maxnum, AllCmpSingleUse};
9925 case SPF_FMINNUM:
9926 return {Intrinsic::minnum, AllCmpSingleUse};
9927 default:
9928 llvm_unreachable("unexpected select pattern flavor");
9929 }
9930 }
9931 return {Intrinsic::not_intrinsic, false};
9932}
9933
9934template <typename InstTy>
9935static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
9936 Value *&Init, Value *&OtherOp) {
9937 // Handle the case of a simple two-predecessor recurrence PHI.
9938 // There's a lot more that could theoretically be done here, but
9939 // this is sufficient to catch some interesting cases.
9940 // TODO: Expand list -- gep, uadd.sat etc.
9941 if (PN->getNumIncomingValues() != 2)
9942 return false;
9943
9944 for (unsigned I = 0; I != 2; ++I) {
9945 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9946 Operation && Operation->getNumOperands() >= 2) {
9947 Value *LHS = Operation->getOperand(0);
9948 Value *RHS = Operation->getOperand(1);
9949 if (LHS != PN && RHS != PN)
9950 continue;
9951
9952 Inst = Operation;
9953 Init = PN->getIncomingValue(i: !I);
9954 OtherOp = (LHS == PN) ? RHS : LHS;
9955 return true;
9956 }
9957 }
9958 return false;
9959}
9960
9961template <typename InstTy>
9962static bool matchThreeInputRecurrence(const PHINode *PN, InstTy *&Inst,
9963 Value *&Init, Value *&OtherOp0,
9964 Value *&OtherOp1) {
9965 if (PN->getNumIncomingValues() != 2)
9966 return false;
9967
9968 for (unsigned I = 0; I != 2; ++I) {
9969 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9970 Operation && Operation->getNumOperands() >= 3) {
9971 Value *Op0 = Operation->getOperand(0);
9972 Value *Op1 = Operation->getOperand(1);
9973 Value *Op2 = Operation->getOperand(2);
9974
9975 if (Op0 != PN && Op1 != PN && Op2 != PN)
9976 continue;
9977
9978 Inst = Operation;
9979 Init = PN->getIncomingValue(i: !I);
9980 if (Op0 == PN) {
9981 OtherOp0 = Op1;
9982 OtherOp1 = Op2;
9983 } else if (Op1 == PN) {
9984 OtherOp0 = Op0;
9985 OtherOp1 = Op2;
9986 } else {
9987 OtherOp0 = Op0;
9988 OtherOp1 = Op1;
9989 }
9990 return true;
9991 }
9992 }
9993 return false;
9994}
9995bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
9996 Value *&Start, Value *&Step) {
9997 // We try to match a recurrence of the form:
9998 // %iv = [Start, %entry], [%iv.next, %backedge]
9999 // %iv.next = binop %iv, Step
10000 // Or:
10001 // %iv = [Start, %entry], [%iv.next, %backedge]
10002 // %iv.next = binop Step, %iv
10003 return matchTwoInputRecurrence(PN: P, Inst&: BO, Init&: Start, OtherOp&: Step);
10004}
10005
10006bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
10007 Value *&Start, Value *&Step) {
10008 BinaryOperator *BO = nullptr;
10009 return match(V: I, P: m_c_BinOp(L: m_Phi(PN&: P), R: m_Value())) &&
10010 matchSimpleRecurrence(P, BO, Start, Step) && BO == I;
10011}
10012
10013bool llvm::matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I,
10014 PHINode *&P, Value *&Init,
10015 Value *&OtherOp) {
10016 // Binary intrinsics only supported for now.
10017 if (I->arg_size() != 2 || I->getType() != I->getArgOperand(i: 0)->getType() ||
10018 I->getType() != I->getArgOperand(i: 1)->getType())
10019 return false;
10020
10021 IntrinsicInst *II = nullptr;
10022 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
10023 if (!P)
10024 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
10025
10026 return P && matchTwoInputRecurrence(PN: P, Inst&: II, Init, OtherOp) && II == I;
10027}
10028
10029bool llvm::matchSimpleTernaryIntrinsicRecurrence(const IntrinsicInst *I,
10030 PHINode *&P, Value *&Init,
10031 Value *&OtherOp0,
10032 Value *&OtherOp1) {
10033 if (I->arg_size() != 3 || I->getType() != I->getArgOperand(i: 0)->getType() ||
10034 I->getType() != I->getArgOperand(i: 1)->getType() ||
10035 I->getType() != I->getArgOperand(i: 2)->getType())
10036 return false;
10037 IntrinsicInst *II = nullptr;
10038 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
10039 if (!P) {
10040 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
10041 if (!P)
10042 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 2));
10043 }
10044 return P && matchThreeInputRecurrence(PN: P, Inst&: II, Init, OtherOp0, OtherOp1) &&
10045 II == I;
10046}
10047
10048/// Return true if "icmp Pred LHS RHS" is always true.
10049static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
10050 const Value *RHS) {
10051 if (ICmpInst::isTrueWhenEqual(predicate: Pred) && LHS == RHS)
10052 return true;
10053
10054 switch (Pred) {
10055 default:
10056 return false;
10057
10058 case CmpInst::ICMP_SLE: {
10059 const APInt *C;
10060
10061 // LHS s<= LHS +_{nsw} C if C >= 0
10062 // LHS s<= LHS | C if C >= 0
10063 if (match(V: RHS, P: m_NSWAdd(L: m_Specific(V: LHS), R: m_APInt(Res&: C))) ||
10064 match(V: RHS, P: m_Or(L: m_Specific(V: LHS), R: m_APInt(Res&: C))))
10065 return !C->isNegative();
10066
10067 // LHS s<= smax(LHS, V) for any V
10068 if (match(V: RHS, P: m_c_SMax(L: m_Specific(V: LHS), R: m_Value())))
10069 return true;
10070
10071 // smin(RHS, V) s<= RHS for any V
10072 if (match(V: LHS, P: m_c_SMin(L: m_Specific(V: RHS), R: m_Value())))
10073 return true;
10074
10075 // Match A to (X +_{nsw} CA) and B to (X +_{nsw} CB)
10076 const Value *X;
10077 const APInt *CLHS, *CRHS;
10078 if (match(V: LHS, P: m_NSWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
10079 match(V: RHS, P: m_NSWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
10080 return CLHS->sle(RHS: *CRHS);
10081
10082 return false;
10083 }
10084
10085 case CmpInst::ICMP_ULE: {
10086 // LHS u<= LHS +_{nuw} V for any V
10087 if (match(V: RHS, P: m_c_Add(L: m_Specific(V: LHS), R: m_Value())) &&
10088 cast<OverflowingBinaryOperator>(Val: RHS)->hasNoUnsignedWrap())
10089 return true;
10090
10091 // LHS u<= LHS | V for any V
10092 if (match(V: RHS, P: m_c_Or(L: m_Specific(V: LHS), R: m_Value())))
10093 return true;
10094
10095 // LHS u<= umax(LHS, V) for any V
10096 if (match(V: RHS, P: m_c_UMax(L: m_Specific(V: LHS), R: m_Value())))
10097 return true;
10098
10099 // RHS >> V u<= RHS for any V
10100 if (match(V: LHS, P: m_LShr(L: m_Specific(V: RHS), R: m_Value())))
10101 return true;
10102
10103 // RHS u/ C_ugt_1 u<= RHS
10104 const APInt *C;
10105 if (match(V: LHS, P: m_UDiv(L: m_Specific(V: RHS), R: m_APInt(Res&: C))) && C->ugt(RHS: 1))
10106 return true;
10107
10108 // RHS & V u<= RHS for any V
10109 if (match(V: LHS, P: m_c_And(L: m_Specific(V: RHS), R: m_Value())))
10110 return true;
10111
10112 // umin(RHS, V) u<= RHS for any V
10113 if (match(V: LHS, P: m_c_UMin(L: m_Specific(V: RHS), R: m_Value())))
10114 return true;
10115
10116 // Match A to (X +_{nuw} CA) and B to (X +_{nuw} CB)
10117 const Value *X;
10118 const APInt *CLHS, *CRHS;
10119 if (match(V: LHS, P: m_NUWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
10120 match(V: RHS, P: m_NUWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
10121 return CLHS->ule(RHS: *CRHS);
10122
10123 return false;
10124 }
10125 }
10126}
10127
10128/// Return true if "icmp Pred BLHS BRHS" is true whenever "icmp Pred
10129/// ALHS ARHS" is true. Otherwise, return std::nullopt.
10130static std::optional<bool>
10131isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS,
10132 const Value *ARHS, const Value *BLHS, const Value *BRHS) {
10133 switch (Pred) {
10134 default:
10135 return std::nullopt;
10136
10137 case CmpInst::ICMP_SLT:
10138 case CmpInst::ICMP_SLE:
10139 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BLHS, RHS: ALHS) &&
10140 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ARHS, RHS: BRHS))
10141 return true;
10142 return std::nullopt;
10143
10144 case CmpInst::ICMP_SGT:
10145 case CmpInst::ICMP_SGE:
10146 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ALHS, RHS: BLHS) &&
10147 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BRHS, RHS: ARHS))
10148 return true;
10149 return std::nullopt;
10150
10151 case CmpInst::ICMP_ULT:
10152 case CmpInst::ICMP_ULE:
10153 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BLHS, RHS: ALHS) &&
10154 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ARHS, RHS: BRHS))
10155 return true;
10156 return std::nullopt;
10157
10158 case CmpInst::ICMP_UGT:
10159 case CmpInst::ICMP_UGE:
10160 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ALHS, RHS: BLHS) &&
10161 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BRHS, RHS: ARHS))
10162 return true;
10163 return std::nullopt;
10164 }
10165}
10166
10167/// Return true if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is true.
10168/// Return false if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is false.
10169/// Otherwise, return std::nullopt if we can't infer anything.
10170static std::optional<bool>
10171isImpliedCondCommonOperandWithCR(CmpPredicate LPred, const ConstantRange &LCR,
10172 CmpPredicate RPred, const ConstantRange &RCR) {
10173 auto CRImpliesPred = [&](ConstantRange CR,
10174 CmpInst::Predicate Pred) -> std::optional<bool> {
10175 // If all true values for lhs and true for rhs, lhs implies rhs
10176 if (CR.icmp(Pred, Other: RCR))
10177 return true;
10178
10179 // If there is no overlap, lhs implies not rhs
10180 if (CR.icmp(Pred: CmpInst::getInversePredicate(pred: Pred), Other: RCR))
10181 return false;
10182
10183 return std::nullopt;
10184 };
10185 if (auto Res = CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
10186 RPred))
10187 return Res;
10188 if (LPred.hasSameSign() ^ RPred.hasSameSign()) {
10189 LPred = LPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: LPred)
10190 : LPred.dropSameSign();
10191 RPred = RPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: RPred)
10192 : RPred.dropSameSign();
10193 return CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
10194 RPred);
10195 }
10196 return std::nullopt;
10197}
10198
10199/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
10200/// is true. Return false if LHS implies RHS is false. Otherwise, return
10201/// std::nullopt if we can't infer anything.
10202static std::optional<bool>
10203isImpliedCondICmps(CmpPredicate LPred, const Value *L0, const Value *L1,
10204 CmpPredicate RPred, const Value *R0, const Value *R1,
10205 const DataLayout &DL, bool LHSIsTrue) {
10206 // The rest of the logic assumes the LHS condition is true. If that's not the
10207 // case, invert the predicate to make it so.
10208 if (!LHSIsTrue)
10209 LPred = ICmpInst::getInverseCmpPredicate(Pred: LPred);
10210
10211 // We can have non-canonical operands, so try to normalize any common operand
10212 // to L0/R0.
10213 if (L0 == R1) {
10214 std::swap(a&: R0, b&: R1);
10215 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10216 }
10217 if (R0 == L1) {
10218 std::swap(a&: L0, b&: L1);
10219 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
10220 }
10221 if (L1 == R1) {
10222 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
10223 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
10224 std::swap(a&: L0, b&: L1);
10225 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
10226 std::swap(a&: R0, b&: R1);
10227 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10228 }
10229 }
10230
10231 // See if we can infer anything if operand-0 matches and we have at least one
10232 // constant.
10233 const APInt *Unused;
10234 if (L0 == R0 && (match(V: L1, P: m_APInt(Res&: Unused)) || match(V: R1, P: m_APInt(Res&: Unused)))) {
10235 // Potential TODO: We could also further use the constant range of L0/R0 to
10236 // further constraint the constant ranges. At the moment this leads to
10237 // several regressions related to not transforming `multi_use(A + C0) eq/ne
10238 // C1` (see discussion: D58633).
10239 SimplifyQuery SQ(DL);
10240 ConstantRange LCR = computeConstantRange(V: L1, ForSigned: ICmpInst::isSigned(Pred: LPred), SQ,
10241 Depth: MaxAnalysisRecursionDepth - 1);
10242 ConstantRange RCR = computeConstantRange(V: R1, ForSigned: ICmpInst::isSigned(Pred: RPred), SQ,
10243 Depth: MaxAnalysisRecursionDepth - 1);
10244
10245 // Even if L1/R1 are not both constant, we can still sometimes deduce
10246 // relationship from a single constant. For example X u> Y implies X != 0.
10247 if (auto R = isImpliedCondCommonOperandWithCR(LPred, LCR, RPred, RCR))
10248 return R;
10249 // If both L1/R1 were exact constant ranges and we didn't get anything
10250 // here, we won't be able to deduce this.
10251 if (match(V: L1, P: m_APInt(Res&: Unused)) && match(V: R1, P: m_APInt(Res&: Unused)))
10252 return std::nullopt;
10253 }
10254
10255 // Can we infer anything when the two compares have matching operands?
10256 if (L0 == R0 && L1 == R1)
10257 return ICmpInst::isImpliedByMatchingCmp(Pred1: LPred, Pred2: RPred);
10258
10259 // It only really makes sense in the context of signed comparison for "X - Y
10260 // must be positive if X >= Y and no overflow".
10261 // Take SGT as an example: L0:x > L1:y and C >= 0
10262 // ==> R0:(x -nsw y) < R1:(-C) is false
10263 CmpInst::Predicate SignedLPred = LPred.getPreferredSignedPredicate();
10264 if ((SignedLPred == ICmpInst::ICMP_SGT ||
10265 SignedLPred == ICmpInst::ICMP_SGE) &&
10266 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
10267 if (match(V: R1, P: m_NonPositive()) &&
10268 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == false)
10269 return false;
10270 }
10271
10272 // Take SLT as an example: L0:x < L1:y and C <= 0
10273 // ==> R0:(x -nsw y) < R1:(-C) is true
10274 if ((SignedLPred == ICmpInst::ICMP_SLT ||
10275 SignedLPred == ICmpInst::ICMP_SLE) &&
10276 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
10277 if (match(V: R1, P: m_NonNegative()) &&
10278 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == true)
10279 return true;
10280 }
10281
10282 // (a - b) pred C -> a != b, if 0 pred C is false
10283 // (ptrtoint(a) - ptrtoint(b)) pred C -> a != b, if 0 pred C is false
10284 Value *A, *B;
10285 if (ICmpInst::isEquality(P: RPred) && cmpExcludesZero(Pred: LPred, RHS: L1) &&
10286 match(V: L0, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
10287 ((A == R0 && B == R1) || (A == R1 && B == R0) ||
10288 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0))) &&
10289 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1)))) ||
10290 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1))) &&
10291 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0)))))) {
10292 return RPred.dropSameSign() == ICmpInst::ICMP_NE;
10293 }
10294
10295 // L0 = R0 = L1 + R1, L0 >=u L1 implies R0 >=u R1, L0 <u L1 implies R0 <u R1
10296 if (L0 == R0 &&
10297 (LPred == ICmpInst::ICMP_ULT || LPred == ICmpInst::ICMP_UGE) &&
10298 (RPred == ICmpInst::ICMP_ULT || RPred == ICmpInst::ICMP_UGE) &&
10299 match(V: L0, P: m_c_Add(L: m_Specific(V: L1), R: m_Specific(V: R1))))
10300 return CmpPredicate::getMatching(A: LPred, B: RPred).has_value();
10301
10302 if (auto P = CmpPredicate::getMatching(A: LPred, B: RPred))
10303 return isImpliedCondOperands(Pred: *P, ALHS: L0, ARHS: L1, BLHS: R0, BRHS: R1);
10304
10305 // L0 u< C sets limits to L0's bits which may imply (L0 & Mask) pred RC
10306 // Example: L0 u< 13 => (L0 & 16) == 0
10307 const APInt *LC, *RC, *MaskC;
10308 if (match(V: L1, P: m_APInt(Res&: LC)) && match(V: R1, P: m_APInt(Res&: RC)) &&
10309 match(V: R0, P: m_And(L: m_Specific(V: L0), R: m_APInt(Res&: MaskC)))) {
10310 ConstantRange LCRange = ConstantRange::makeExactICmpRegion(Pred: LPred, Other: *LC);
10311 ConstantRange MaskedCRange = LCRange.binaryAnd(Other: *MaskC);
10312 if (MaskedCRange.icmp(Pred: RPred, Other: ConstantRange(*RC)))
10313 return true;
10314 if (MaskedCRange.icmp(Pred: ICmpInst::getInversePredicate(pred: RPred),
10315 Other: ConstantRange(*RC)))
10316 return false;
10317 }
10318
10319 return std::nullopt;
10320}
10321
10322/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
10323/// is true. Return false if LHS implies RHS is false. Otherwise, return
10324/// std::nullopt if we can't infer anything.
10325static std::optional<bool>
10326isImpliedCondFCmps(FCmpInst::Predicate LPred, const Value *L0, const Value *L1,
10327 FCmpInst::Predicate RPred, const Value *R0, const Value *R1,
10328 const DataLayout &DL, bool LHSIsTrue) {
10329 // The rest of the logic assumes the LHS condition is true. If that's not the
10330 // case, invert the predicate to make it so.
10331 if (!LHSIsTrue)
10332 LPred = FCmpInst::getInversePredicate(pred: LPred);
10333
10334 // We can have non-canonical operands, so try to normalize any common operand
10335 // to L0/R0.
10336 if (L0 == R1) {
10337 std::swap(a&: R0, b&: R1);
10338 RPred = FCmpInst::getSwappedPredicate(pred: RPred);
10339 }
10340 if (R0 == L1) {
10341 std::swap(a&: L0, b&: L1);
10342 LPred = FCmpInst::getSwappedPredicate(pred: LPred);
10343 }
10344 if (L1 == R1) {
10345 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
10346 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
10347 std::swap(a&: L0, b&: L1);
10348 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
10349 std::swap(a&: R0, b&: R1);
10350 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10351 }
10352 }
10353
10354 // Can we infer anything when the two compares have matching operands?
10355 if (L0 == R0 && L1 == R1) {
10356 if ((LPred & RPred) == LPred)
10357 return true;
10358 if ((LPred & ~RPred) == LPred)
10359 return false;
10360 }
10361
10362 // See if we can infer anything if operand-0 matches and we have at least one
10363 // constant.
10364 const APFloat *L1C, *R1C;
10365 if (L0 == R0 && match(V: L1, P: m_APFloat(Res&: L1C)) && match(V: R1, P: m_APFloat(Res&: R1C))) {
10366 if (std::optional<ConstantFPRange> DomCR =
10367 ConstantFPRange::makeExactFCmpRegion(Pred: LPred, Other: *L1C)) {
10368 if (std::optional<ConstantFPRange> ImpliedCR =
10369 ConstantFPRange::makeExactFCmpRegion(Pred: RPred, Other: *R1C)) {
10370 if (ImpliedCR->contains(CR: *DomCR))
10371 return true;
10372 }
10373 if (std::optional<ConstantFPRange> ImpliedCR =
10374 ConstantFPRange::makeExactFCmpRegion(
10375 Pred: FCmpInst::getInversePredicate(pred: RPred), Other: *R1C)) {
10376 if (ImpliedCR->contains(CR: *DomCR))
10377 return false;
10378 }
10379 }
10380 }
10381
10382 return std::nullopt;
10383}
10384
10385/// Return true if LHS implies RHS is true. Return false if LHS implies RHS is
10386/// false. Otherwise, return std::nullopt if we can't infer anything. We
10387/// expect the RHS to be an icmp and the LHS to be an 'and', 'or', or a 'select'
10388/// instruction.
10389static std::optional<bool>
10390isImpliedCondAndOr(const Instruction *LHS, CmpPredicate RHSPred,
10391 const Value *RHSOp0, const Value *RHSOp1,
10392 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
10393 // The LHS must be an 'or', 'and', or a 'select' instruction.
10394 assert((LHS->getOpcode() == Instruction::And ||
10395 LHS->getOpcode() == Instruction::Or ||
10396 LHS->getOpcode() == Instruction::Select) &&
10397 "Expected LHS to be 'and', 'or', or 'select'.");
10398
10399 assert(Depth <= MaxAnalysisRecursionDepth && "Hit recursion limit");
10400
10401 // If the result of an 'or' is false, then we know both legs of the 'or' are
10402 // false. Similarly, if the result of an 'and' is true, then we know both
10403 // legs of the 'and' are true.
10404 const Value *ALHS, *ARHS;
10405 if ((!LHSIsTrue && match(V: LHS, P: m_LogicalOr(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS)))) ||
10406 (LHSIsTrue && match(V: LHS, P: m_LogicalAnd(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS))))) {
10407 // FIXME: Make this non-recursion.
10408 if (std::optional<bool> Implication = isImpliedCondition(
10409 LHS: ALHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
10410 return Implication;
10411 if (std::optional<bool> Implication = isImpliedCondition(
10412 LHS: ARHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
10413 return Implication;
10414 return std::nullopt;
10415 }
10416 return std::nullopt;
10417}
10418
10419std::optional<bool>
10420llvm::isImpliedCondition(const Value *LHS, CmpPredicate RHSPred,
10421 const Value *RHSOp0, const Value *RHSOp1,
10422 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
10423 // Bail out when we hit the limit.
10424 if (Depth == MaxAnalysisRecursionDepth)
10425 return std::nullopt;
10426
10427 // A mismatch occurs when we compare a scalar cmp to a vector cmp, for
10428 // example.
10429 if (RHSOp0->getType()->isVectorTy() != LHS->getType()->isVectorTy())
10430 return std::nullopt;
10431
10432 assert(LHS->getType()->isIntOrIntVectorTy(1) &&
10433 "Expected integer type only!");
10434
10435 // Match not
10436 if (match(V: LHS, P: m_Not(V: m_Value(V&: LHS))))
10437 LHSIsTrue = !LHSIsTrue;
10438
10439 // Both LHS and RHS are icmps.
10440 if (RHSOp0->getType()->getScalarType()->isIntOrPtrTy()) {
10441 CmpPredicate LHSPred;
10442 Value *LHSOp0, *LHSOp1;
10443 if (match(V: LHS, P: m_ICmpLike(Pred&: LHSPred, L: m_Value(V&: LHSOp0), R: m_Value(V&: LHSOp1))))
10444 return isImpliedCondICmps(LPred: LHSPred, L0: LHSOp0, L1: LHSOp1, RPred: RHSPred, R0: RHSOp0,
10445 R1: RHSOp1, DL, LHSIsTrue);
10446 } else {
10447 assert(RHSOp0->getType()->isFPOrFPVectorTy() &&
10448 "Expected floating point type only!");
10449 if (const auto *LHSCmp = dyn_cast<FCmpInst>(Val: LHS))
10450 return isImpliedCondFCmps(LPred: LHSCmp->getPredicate(), L0: LHSCmp->getOperand(i_nocapture: 0),
10451 L1: LHSCmp->getOperand(i_nocapture: 1), RPred: RHSPred, R0: RHSOp0, R1: RHSOp1,
10452 DL, LHSIsTrue);
10453 }
10454
10455 /// The LHS should be an 'or', 'and', or a 'select' instruction. We expect
10456 /// the RHS to be an icmp.
10457 /// FIXME: Add support for and/or/select on the RHS.
10458 if (const Instruction *LHSI = dyn_cast<Instruction>(Val: LHS)) {
10459 if ((LHSI->getOpcode() == Instruction::And ||
10460 LHSI->getOpcode() == Instruction::Or ||
10461 LHSI->getOpcode() == Instruction::Select))
10462 return isImpliedCondAndOr(LHS: LHSI, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue,
10463 Depth);
10464 }
10465 return std::nullopt;
10466}
10467
10468std::optional<bool> llvm::isImpliedCondition(const Value *LHS, const Value *RHS,
10469 const DataLayout &DL,
10470 bool LHSIsTrue, unsigned Depth) {
10471 // LHS ==> RHS by definition
10472 if (LHS == RHS)
10473 return LHSIsTrue;
10474
10475 // Match not
10476 bool InvertRHS = false;
10477 if (match(V: RHS, P: m_Not(V: m_Value(V&: RHS)))) {
10478 if (LHS == RHS)
10479 return !LHSIsTrue;
10480 InvertRHS = true;
10481 }
10482
10483 CmpPredicate RHSPred;
10484 Value *RHSOp0, *RHSOp1;
10485 if (match(V: RHS, P: m_ICmpLike(Pred&: RHSPred, L: m_Value(V&: RHSOp0), R: m_Value(V&: RHSOp1)))) {
10486 if (auto Implied = isImpliedCondition(LHS, RHSPred, RHSOp0, RHSOp1, DL,
10487 LHSIsTrue, Depth))
10488 return InvertRHS ? !*Implied : *Implied;
10489 return std::nullopt;
10490 }
10491 if (const FCmpInst *RHSCmp = dyn_cast<FCmpInst>(Val: RHS)) {
10492 if (auto Implied = isImpliedCondition(
10493 LHS, RHSPred: RHSCmp->getPredicate(), RHSOp0: RHSCmp->getOperand(i_nocapture: 0),
10494 RHSOp1: RHSCmp->getOperand(i_nocapture: 1), DL, LHSIsTrue, Depth))
10495 return InvertRHS ? !*Implied : *Implied;
10496 return std::nullopt;
10497 }
10498
10499 if (Depth == MaxAnalysisRecursionDepth)
10500 return std::nullopt;
10501
10502 // LHS ==> (RHS1 || RHS2) if LHS ==> RHS1 or LHS ==> RHS2
10503 // LHS ==> !(RHS1 && RHS2) if LHS ==> !RHS1 or LHS ==> !RHS2
10504 const Value *RHS1, *RHS2;
10505 if (match(V: RHS, P: m_LogicalOr(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
10506 if (std::optional<bool> Imp =
10507 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
10508 if (*Imp == true)
10509 return !InvertRHS;
10510 if (std::optional<bool> Imp =
10511 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
10512 if (*Imp == true)
10513 return !InvertRHS;
10514 }
10515 if (match(V: RHS, P: m_LogicalAnd(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
10516 if (std::optional<bool> Imp =
10517 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
10518 if (*Imp == false)
10519 return InvertRHS;
10520 if (std::optional<bool> Imp =
10521 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
10522 if (*Imp == false)
10523 return InvertRHS;
10524 }
10525
10526 return std::nullopt;
10527}
10528
10529// Returns a pair (Condition, ConditionIsTrue), where Condition is a branch
10530// condition dominating ContextI or nullptr, if no condition is found.
10531static std::pair<Value *, bool>
10532getDomPredecessorCondition(const Instruction *ContextI) {
10533 if (!ContextI || !ContextI->getParent())
10534 return {nullptr, false};
10535
10536 // TODO: This is a poor/cheap way to determine dominance. Should we use a
10537 // dominator tree (eg, from a SimplifyQuery) instead?
10538 const BasicBlock *ContextBB = ContextI->getParent();
10539 const BasicBlock *PredBB = ContextBB->getSinglePredecessor();
10540 if (!PredBB)
10541 return {nullptr, false};
10542
10543 // We need a conditional branch in the predecessor.
10544 Value *PredCond;
10545 BasicBlock *TrueBB, *FalseBB;
10546 if (!match(V: PredBB->getTerminator(), P: m_Br(C: m_Value(V&: PredCond), T&: TrueBB, F&: FalseBB)))
10547 return {nullptr, false};
10548
10549 // The branch should get simplified. Don't bother simplifying this condition.
10550 if (TrueBB == FalseBB)
10551 return {nullptr, false};
10552
10553 assert((TrueBB == ContextBB || FalseBB == ContextBB) &&
10554 "Predecessor block does not point to successor?");
10555
10556 // Is this condition implied by the predecessor condition?
10557 return {PredCond, TrueBB == ContextBB};
10558}
10559
10560std::optional<bool> llvm::isImpliedByDomCondition(const Value *Cond,
10561 const Instruction *ContextI,
10562 const DataLayout &DL) {
10563 assert(Cond->getType()->isIntOrIntVectorTy(1) && "Condition must be bool");
10564 auto PredCond = getDomPredecessorCondition(ContextI);
10565 if (PredCond.first)
10566 return isImpliedCondition(LHS: PredCond.first, RHS: Cond, DL, LHSIsTrue: PredCond.second);
10567 return std::nullopt;
10568}
10569
10570std::optional<bool> llvm::isImpliedByDomCondition(CmpPredicate Pred,
10571 const Value *LHS,
10572 const Value *RHS,
10573 const Instruction *ContextI,
10574 const DataLayout &DL) {
10575 auto PredCond = getDomPredecessorCondition(ContextI);
10576 if (PredCond.first)
10577 return isImpliedCondition(LHS: PredCond.first, RHSPred: Pred, RHSOp0: LHS, RHSOp1: RHS, DL,
10578 LHSIsTrue: PredCond.second);
10579 return std::nullopt;
10580}
10581
10582static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
10583 APInt &Upper, const InstrInfoQuery &IIQ,
10584 bool PreferSignedRange) {
10585 unsigned Width = Lower.getBitWidth();
10586 const APInt *C;
10587 switch (BO.getOpcode()) {
10588 case Instruction::Sub:
10589 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10590 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10591 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10592
10593 // If the caller expects a signed compare, then try to use a signed range.
10594 // Otherwise if both no-wraps are set, use the unsigned range because it
10595 // is never larger than the signed range. Example:
10596 // "sub nuw nsw i8 -2, x" is unsigned [0, 254] vs. signed [-128, 126].
10597 // "sub nuw nsw i8 2, x" is unsigned [0, 2] vs. signed [-125, 127].
10598 if (PreferSignedRange && HasNSW && HasNUW)
10599 HasNUW = false;
10600
10601 if (HasNUW) {
10602 // 'sub nuw c, x' produces [0, C].
10603 Upper = *C + 1;
10604 } else if (HasNSW) {
10605 if (C->isNegative()) {
10606 // 'sub nsw -C, x' produces [SINT_MIN, -C - SINT_MIN].
10607 Lower = APInt::getSignedMinValue(numBits: Width);
10608 Upper = *C - APInt::getSignedMaxValue(numBits: Width);
10609 } else {
10610 // Note that sub 0, INT_MIN is not NSW. It techically is a signed wrap
10611 // 'sub nsw C, x' produces [C - SINT_MAX, SINT_MAX].
10612 Lower = *C - APInt::getSignedMaxValue(numBits: Width);
10613 Upper = APInt::getSignedMinValue(numBits: Width);
10614 }
10615 }
10616 }
10617 break;
10618 case Instruction::Add:
10619 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10620 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10621 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10622
10623 // If the caller expects a signed compare, then try to use a signed
10624 // range. Otherwise if both no-wraps are set, use the unsigned range
10625 // because it is never larger than the signed range. Example: "add nuw
10626 // nsw i8 X, -2" is unsigned [254,255] vs. signed [-128, 125].
10627 if (PreferSignedRange && HasNSW && HasNUW)
10628 HasNUW = false;
10629
10630 if (HasNUW) {
10631 // 'add nuw x, C' produces [C, UINT_MAX].
10632 Lower = *C;
10633 } else if (HasNSW) {
10634 if (C->isNegative()) {
10635 // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C].
10636 Lower = APInt::getSignedMinValue(numBits: Width);
10637 Upper = APInt::getSignedMaxValue(numBits: Width) + *C + 1;
10638 } else {
10639 // 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX].
10640 Lower = APInt::getSignedMinValue(numBits: Width) + *C;
10641 Upper = APInt::getSignedMaxValue(numBits: Width) + 1;
10642 }
10643 }
10644 }
10645 break;
10646
10647 case Instruction::And:
10648 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10649 // 'and x, C' produces [0, C].
10650 Upper = *C + 1;
10651 // X & -X is a power of two or zero. So we can cap the value at max power of
10652 // two.
10653 if (match(V: BO.getOperand(i_nocapture: 0), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 1)))) ||
10654 match(V: BO.getOperand(i_nocapture: 1), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 0)))))
10655 Upper = APInt::getSignedMinValue(numBits: Width) + 1;
10656 break;
10657
10658 case Instruction::Or:
10659 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10660 // 'or x, C' produces [C, UINT_MAX].
10661 Lower = *C;
10662 break;
10663
10664 case Instruction::AShr:
10665 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10666 // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C].
10667 Lower = APInt::getSignedMinValue(numBits: Width).ashr(ShiftAmt: *C);
10668 Upper = APInt::getSignedMaxValue(numBits: Width).ashr(ShiftAmt: *C) + 1;
10669 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10670 unsigned ShiftAmount = Width - 1;
10671 if (!C->isZero() && IIQ.isExact(Op: &BO))
10672 ShiftAmount = C->countr_zero();
10673 if (C->isNegative()) {
10674 // 'ashr C, x' produces [C, C >> (Width-1)]
10675 Lower = *C;
10676 Upper = C->ashr(ShiftAmt: ShiftAmount) + 1;
10677 } else {
10678 // 'ashr C, x' produces [C >> (Width-1), C]
10679 Lower = C->ashr(ShiftAmt: ShiftAmount);
10680 Upper = *C + 1;
10681 }
10682 }
10683 break;
10684
10685 case Instruction::LShr:
10686 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10687 // 'lshr x, C' produces [0, UINT_MAX >> C].
10688 Upper = APInt::getAllOnes(numBits: Width).lshr(ShiftAmt: *C) + 1;
10689 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10690 // 'lshr C, x' produces [C >> (Width-1), C].
10691 unsigned ShiftAmount = Width - 1;
10692 if (!C->isZero() && IIQ.isExact(Op: &BO))
10693 ShiftAmount = C->countr_zero();
10694 Lower = C->lshr(shiftAmt: ShiftAmount);
10695 Upper = *C + 1;
10696 }
10697 break;
10698
10699 case Instruction::Shl:
10700 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10701 if (IIQ.hasNoUnsignedWrap(Op: &BO)) {
10702 // 'shl nuw C, x' produces [C, C << CLZ(C)]
10703 Lower = *C;
10704 Upper = Lower.shl(shiftAmt: Lower.countl_zero()) + 1;
10705 } else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw?
10706 if (C->isNegative()) {
10707 // 'shl nsw C, x' produces [C << CLO(C)-1, C]
10708 unsigned ShiftAmount = C->countl_one() - 1;
10709 Lower = C->shl(shiftAmt: ShiftAmount);
10710 Upper = *C + 1;
10711 } else {
10712 // 'shl nsw C, x' produces [C, C << CLZ(C)-1]
10713 unsigned ShiftAmount = C->countl_zero() - 1;
10714 Lower = *C;
10715 Upper = C->shl(shiftAmt: ShiftAmount) + 1;
10716 }
10717 } else {
10718 // If lowbit is set, value can never be zero.
10719 if ((*C)[0])
10720 Lower = APInt::getOneBitSet(numBits: Width, BitNo: 0);
10721 // If we are shifting a constant the largest it can be is if the longest
10722 // sequence of consecutive ones is shifted to the highbits (breaking
10723 // ties for which sequence is higher). At the moment we take a liberal
10724 // upper bound on this by just popcounting the constant.
10725 // TODO: There may be a bitwise trick for it longest/highest
10726 // consecutative sequence of ones (naive method is O(Width) loop).
10727 Upper = APInt::getHighBitsSet(numBits: Width, hiBitsSet: C->popcount()) + 1;
10728 }
10729 } else if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10730 Upper = APInt::getBitsSetFrom(numBits: Width, loBit: C->getZExtValue()) + 1;
10731 }
10732 break;
10733
10734 case Instruction::SDiv:
10735 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10736 APInt IntMin = APInt::getSignedMinValue(numBits: Width);
10737 APInt IntMax = APInt::getSignedMaxValue(numBits: Width);
10738 if (C->isAllOnes()) {
10739 // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX]
10740 // where C != -1 and C != 0 and C != 1
10741 Lower = IntMin + 1;
10742 Upper = IntMax + 1;
10743 } else if (C->countl_zero() < Width - 1) {
10744 // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C]
10745 // where C != -1 and C != 0 and C != 1
10746 Lower = IntMin.sdiv(RHS: *C);
10747 Upper = IntMax.sdiv(RHS: *C);
10748 if (Lower.sgt(RHS: Upper))
10749 std::swap(a&: Lower, b&: Upper);
10750 Upper = Upper + 1;
10751 assert(Upper != Lower && "Upper part of range has wrapped!");
10752 }
10753 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10754 if (C->isMinSignedValue()) {
10755 // 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2].
10756 Lower = *C;
10757 Upper = Lower.lshr(shiftAmt: 1) + 1;
10758 } else {
10759 // 'sdiv C, x' produces [-|C|, |C|].
10760 Upper = C->abs() + 1;
10761 Lower = (-Upper) + 1;
10762 }
10763 }
10764 break;
10765
10766 case Instruction::UDiv:
10767 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10768 // 'udiv x, C' produces [0, UINT_MAX / C].
10769 Upper = APInt::getMaxValue(numBits: Width).udiv(RHS: *C) + 1;
10770 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10771 // 'udiv C, x' produces [0, C].
10772 Upper = *C + 1;
10773 }
10774 break;
10775
10776 case Instruction::SRem:
10777 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10778 // 'srem x, C' produces (-|C|, |C|).
10779 Upper = C->abs();
10780 Lower = (-Upper) + 1;
10781 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10782 if (C->isNegative()) {
10783 // 'srem -|C|, x' produces [-|C|, 0].
10784 Upper = 1;
10785 Lower = *C;
10786 } else {
10787 // 'srem |C|, x' produces [0, |C|].
10788 Upper = *C + 1;
10789 }
10790 }
10791 break;
10792
10793 case Instruction::URem:
10794 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10795 // 'urem x, C' produces [0, C).
10796 Upper = *C;
10797 else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10798 // 'urem C, x' produces [0, C].
10799 Upper = *C + 1;
10800 break;
10801
10802 default:
10803 break;
10804 }
10805}
10806
10807static ConstantRange getRangeForIntrinsic(const IntrinsicInst &II,
10808 bool UseInstrInfo) {
10809 unsigned Width = II.getType()->getScalarSizeInBits();
10810 const APInt *C;
10811 switch (II.getIntrinsicID()) {
10812 case Intrinsic::ctlz:
10813 case Intrinsic::cttz: {
10814 APInt Upper(Width, Width);
10815 if (!UseInstrInfo || !match(V: II.getArgOperand(i: 1), P: m_One()))
10816 Upper += 1;
10817 // Maximum of set/clear bits is the bit width.
10818 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper);
10819 }
10820 case Intrinsic::ctpop:
10821 // Maximum of set/clear bits is the bit width.
10822 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10823 Upper: APInt(Width, Width) + 1);
10824 case Intrinsic::uadd_sat:
10825 // uadd.sat(x, C) produces [C, UINT_MAX].
10826 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10827 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10828 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10829 break;
10830 case Intrinsic::sadd_sat:
10831 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10832 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10833 if (C->isNegative())
10834 // sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)].
10835 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10836 Upper: APInt::getSignedMaxValue(numBits: Width) + *C +
10837 1);
10838
10839 // sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX].
10840 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) + *C,
10841 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10842 }
10843 break;
10844 case Intrinsic::usub_sat:
10845 // usub.sat(C, x) produces [0, C].
10846 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10847 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10848
10849 // usub.sat(x, C) produces [0, UINT_MAX - C].
10850 if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10851 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10852 Upper: APInt::getMaxValue(numBits: Width) - *C + 1);
10853 break;
10854 case Intrinsic::ssub_sat:
10855 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10856 if (C->isNegative())
10857 // ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)].
10858 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10859 Upper: *C - APInt::getSignedMinValue(numBits: Width) +
10860 1);
10861
10862 // ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX].
10863 return ConstantRange::getNonEmpty(Lower: *C - APInt::getSignedMaxValue(numBits: Width),
10864 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10865 } else if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10866 if (C->isNegative())
10867 // ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]:
10868 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) - *C,
10869 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10870
10871 // ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C].
10872 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10873 Upper: APInt::getSignedMaxValue(numBits: Width) - *C +
10874 1);
10875 }
10876 break;
10877 case Intrinsic::ushl_sat:
10878 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10879 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10880 break;
10881 case Intrinsic::sshl_sat:
10882 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10883 if (C->isNegative())
10884 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10885 Upper: *C + 1);
10886 return ConstantRange::getNonEmpty(Lower: *C,
10887 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10888 }
10889 break;
10890 case Intrinsic::umin:
10891 case Intrinsic::umax:
10892 case Intrinsic::smin:
10893 case Intrinsic::smax:
10894 if (!match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) &&
10895 !match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10896 break;
10897
10898 switch (II.getIntrinsicID()) {
10899 case Intrinsic::umin:
10900 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10901 case Intrinsic::umax:
10902 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10903 case Intrinsic::smin:
10904 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10905 Upper: *C + 1);
10906 case Intrinsic::smax:
10907 return ConstantRange::getNonEmpty(Lower: *C,
10908 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10909 default:
10910 llvm_unreachable("Must be min/max intrinsic");
10911 }
10912 break;
10913 case Intrinsic::abs:
10914 // If abs of SIGNED_MIN is poison, then the result is [0..SIGNED_MAX],
10915 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10916 if (match(V: II.getOperand(i_nocapture: 1), P: m_One()))
10917 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10918 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10919
10920 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10921 Upper: APInt::getSignedMinValue(numBits: Width) + 1);
10922 case Intrinsic::vscale:
10923 if (!II.getParent() || !II.getFunction())
10924 break;
10925 return getVScaleRange(F: II.getFunction(), BitWidth: Width);
10926 case Intrinsic::read_register:
10927 case Intrinsic::read_volatile_register: {
10928 const Module *M = II.getModule();
10929 if (!M || !M->getTargetTriple().isRISCV())
10930 break;
10931 if (II.getFunction() && isReadVLENB(II))
10932 return getRISCVVLENBRange(II, Width);
10933 break;
10934 }
10935 default:
10936 break;
10937 }
10938
10939 return ConstantRange::getFull(BitWidth: Width);
10940}
10941
10942static ConstantRange getRangeForSelectPattern(const SelectInst &SI,
10943 const InstrInfoQuery &IIQ) {
10944 unsigned BitWidth = SI.getType()->getScalarSizeInBits();
10945 const Value *LHS = nullptr, *RHS = nullptr;
10946 SelectPatternResult R = matchSelectPattern(V: &SI, LHS, RHS);
10947 if (R.Flavor == SPF_UNKNOWN)
10948 return ConstantRange::getFull(BitWidth);
10949
10950 if (R.Flavor == SelectPatternFlavor::SPF_ABS) {
10951 // If the negation part of the abs (in RHS) has the NSW flag,
10952 // then the result of abs(X) is [0..SIGNED_MAX],
10953 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10954 if (match(V: RHS, P: m_Neg(V: m_Specific(V: LHS))) &&
10955 IIQ.hasNoSignedWrap(Op: cast<Instruction>(Val: RHS)))
10956 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10957 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10958
10959 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10960 Upper: APInt::getSignedMinValue(numBits: BitWidth) + 1);
10961 }
10962
10963 if (R.Flavor == SelectPatternFlavor::SPF_NABS) {
10964 // The result of -abs(X) is <= 0.
10965 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10966 Upper: APInt(BitWidth, 1));
10967 }
10968
10969 const APInt *C;
10970 if (!match(V: LHS, P: m_APInt(Res&: C)) && !match(V: RHS, P: m_APInt(Res&: C)))
10971 return ConstantRange::getFull(BitWidth);
10972
10973 switch (R.Flavor) {
10974 case SPF_UMIN:
10975 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth), Upper: *C + 1);
10976 case SPF_UMAX:
10977 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: BitWidth));
10978 case SPF_SMIN:
10979 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10980 Upper: *C + 1);
10981 case SPF_SMAX:
10982 return ConstantRange::getNonEmpty(Lower: *C,
10983 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10984 default:
10985 return ConstantRange::getFull(BitWidth);
10986 }
10987}
10988
10989static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) {
10990 // The maximum representable value of a half is 65504. For floats the maximum
10991 // value is 3.4e38 which requires roughly 129 bits.
10992 unsigned BitWidth = I->getType()->getScalarSizeInBits();
10993 if (!I->getOperand(i: 0)->getType()->getScalarType()->isHalfTy())
10994 return;
10995 if (isa<FPToSIInst>(Val: I) && BitWidth >= 17) {
10996 Lower = APInt(BitWidth, -65504, true);
10997 Upper = APInt(BitWidth, 65505);
10998 }
10999
11000 if (isa<FPToUIInst>(Val: I) && BitWidth >= 16) {
11001 // For a fptoui the lower limit is left as 0.
11002 Upper = APInt(BitWidth, 65505);
11003 }
11004}
11005
11006ConstantRange llvm::computeConstantRange(const Value *V, bool ForSigned,
11007 const SimplifyQuery &SQ,
11008 unsigned Depth) {
11009 assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction");
11010
11011 if (Depth == MaxAnalysisRecursionDepth)
11012 return ConstantRange::getFull(BitWidth: V->getType()->getScalarSizeInBits());
11013
11014 if (auto *C = dyn_cast<Constant>(Val: V))
11015 return C->toConstantRange();
11016
11017 unsigned BitWidth = V->getType()->getScalarSizeInBits();
11018 ConstantRange CR = ConstantRange::getFull(BitWidth);
11019 if (auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
11020 APInt Lower = APInt(BitWidth, 0);
11021 APInt Upper = APInt(BitWidth, 0);
11022 // TODO: Return ConstantRange.
11023 setLimitsForBinOp(BO: *BO, Lower, Upper, IIQ: SQ.IIQ, PreferSignedRange: ForSigned);
11024 CR = ConstantRange::getNonEmpty(Lower, Upper);
11025 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
11026 CR = getRangeForIntrinsic(II: *II, UseInstrInfo: SQ.IIQ.UseInstrInfo);
11027 else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
11028 ConstantRange CRTrue =
11029 computeConstantRange(V: SI->getTrueValue(), ForSigned, SQ, Depth: Depth + 1);
11030 ConstantRange CRFalse =
11031 computeConstantRange(V: SI->getFalseValue(), ForSigned, SQ, Depth: Depth + 1);
11032 CR = CRTrue.unionWith(CR: CRFalse, Type: ForSigned ? ConstantRange::Signed
11033 : ConstantRange::Unsigned);
11034 CR = CR.intersectWith(CR: getRangeForSelectPattern(SI: *SI, IIQ: SQ.IIQ));
11035 } else if (auto *TI = dyn_cast<TruncInst>(Val: V)) {
11036 ConstantRange SrcCR =
11037 computeConstantRange(V: TI->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
11038 CR = SrcCR.truncate(BitWidth);
11039 } else if (auto *ZExt = dyn_cast<ZExtInst>(Val: V)) {
11040 ConstantRange SrcCR =
11041 computeConstantRange(V: ZExt->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
11042 CR = SrcCR.zeroExtend(BitWidth);
11043 } else if (auto *SExt = dyn_cast<SExtInst>(Val: V)) {
11044 ConstantRange SrcCR =
11045 computeConstantRange(V: SExt->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
11046 CR = SrcCR.signExtend(BitWidth);
11047 } else if (isa<FPToUIInst>(Val: V) || isa<FPToSIInst>(Val: V)) {
11048 APInt Lower = APInt(BitWidth, 0);
11049 APInt Upper = APInt(BitWidth, 0);
11050 // TODO: Return ConstantRange.
11051 setLimitForFPToI(I: cast<Instruction>(Val: V), Lower, Upper);
11052 CR = ConstantRange::getNonEmpty(Lower, Upper);
11053 } else if (const auto *A = dyn_cast<Argument>(Val: V))
11054 if (std::optional<ConstantRange> Range = A->getRange())
11055 CR = *Range;
11056
11057 if (auto *I = dyn_cast<Instruction>(Val: V)) {
11058 if (auto *Range = SQ.IIQ.getMetadata(I, KindID: LLVMContext::MD_range))
11059 CR = CR.intersectWith(CR: getConstantRangeFromMetadata(RangeMD: *Range));
11060
11061 Value *FrexpSrc;
11062 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
11063 if (std::optional<ConstantRange> Range = CB->getRange())
11064 CR = CR.intersectWith(CR: *Range);
11065 } else if (match(V: I, P: m_ExtractValue<1>(V: m_Intrinsic<Intrinsic::frexp>(
11066 Ops: m_Value(V&: FrexpSrc))))) {
11067 const fltSemantics &FltSem =
11068 FrexpSrc->getType()->getScalarType()->getFltSemantics();
11069 // It should be possible to implement this for any type, but this logic
11070 // only computes the range assuming standard subnormal handling.
11071 if (APFloat::isIEEELikeFP(FltSem)) {
11072 KnownFPClass KnownSrc = computeKnownFPClass(
11073 V: FrexpSrc, InterestedClasses: fcSubnormal | fcZero | fcNan | fcInf, SQ, Depth: Depth + 1);
11074
11075 // The exponent of frexp(NaN) and frexp(Inf) is unspecified. Only
11076 // constrain its range when the source can be neither.
11077 if (KnownSrc.isKnownNeverInfOrNaN()) {
11078 int MinExp = APFloat::semanticsMinExponent(FltSem) + 1;
11079
11080 // Offset to find the true minimum exponent value for a denormal.
11081 if (!KnownSrc.isKnownNeverSubnormal())
11082 MinExp -= (APFloat::semanticsPrecision(FltSem) - 1);
11083
11084 int MaxExp = APFloat::semanticsMaxExponent(FltSem) + 1;
11085
11086 auto [AdjustedMin, AdjustedMax, AdjustedMaxNonZero] =
11087 computeKnownExponentRangeFromContext(V: FrexpSrc, Q: SQ);
11088
11089 DenormalMode Mode = I->getFunction()->getDenormalMode(FPType: FltSem);
11090 bool NeverLogicalZero = KnownSrc.isKnownNeverLogicalZero(Mode);
11091
11092 MinExp = std::max(a: AdjustedMin, b: MinExp);
11093 MaxExp = std::min(a: NeverLogicalZero ? AdjustedMaxNonZero : AdjustedMax,
11094 b: MaxExp);
11095
11096 CR = ConstantRange::getNonEmpty(
11097 Lower: APInt(BitWidth, static_cast<int64_t>(MinExp), /*isSigned=*/true),
11098 Upper: APInt(BitWidth, static_cast<int64_t>(MaxExp) + 1,
11099 /*isSigned=*/true));
11100 }
11101 }
11102 }
11103 }
11104
11105 if (SQ.CtxI && SQ.AC) {
11106 // Try to restrict the range based on information from assumptions.
11107 for (auto &AssumeVH : SQ.AC->assumptionsFor(V)) {
11108 if (!AssumeVH)
11109 continue;
11110 CallInst *I = cast<CallInst>(Val&: AssumeVH);
11111 assert(I->getParent()->getParent() == SQ.CtxI->getParent()->getParent() &&
11112 "Got assumption for the wrong function!");
11113 assert(I->getIntrinsicID() == Intrinsic::assume &&
11114 "must be an assume intrinsic");
11115
11116 if (AssumeVH.Index != AssumptionCache::ExprResultIdx) {
11117 auto OBU = I->getOperandBundleAt(Index: AssumeVH.Index);
11118 if (getBundleAttrFromOBU(OBU) == BundleAttr::Range) {
11119 auto [Val, Range] = getAssumeRangeInfo(OBU);
11120 if (Val == V && isValidAssumeForContext(I, Q: SQ))
11121 CR = CR.intersectWith(CR: Range);
11122 }
11123 continue;
11124 }
11125
11126 if (!isValidAssumeForContext(I, Q: SQ))
11127 continue;
11128 Value *Arg = I->getArgOperand(i: 0);
11129 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
11130 // Currently we just use information from comparisons.
11131 if (!Cmp || Cmp->getOperand(i_nocapture: 0) != V)
11132 continue;
11133 // TODO: Set "ForSigned" parameter via Cmp->isSigned()?
11134 ConstantRange RHS =
11135 computeConstantRange(V: Cmp->getOperand(i_nocapture: 1), /*ForSigned=*/false,
11136 SQ: SQ.getWithInstruction(I), Depth: Depth + 1);
11137 CR = CR.intersectWith(
11138 CR: ConstantRange::makeAllowedICmpRegion(Pred: Cmp->getCmpPredicate(), Other: RHS));
11139 }
11140 }
11141
11142 return CR;
11143}
11144
11145static void
11146addValueAffectedByCondition(Value *V,
11147 function_ref<void(Value *)> InsertAffected) {
11148 assert(V != nullptr);
11149 if (isa<Argument>(Val: V) || isa<GlobalValue>(Val: V)) {
11150 InsertAffected(V);
11151 } else if (auto *I = dyn_cast<Instruction>(Val: V)) {
11152 InsertAffected(V);
11153
11154 // Peek through unary operators to find the source of the condition.
11155 Value *Op;
11156 if (match(V: I, P: m_CombineOr(Ps: m_PtrToIntOrAddr(Op: m_Value(V&: Op)),
11157 Ps: m_Trunc(Op: m_Value(V&: Op))))) {
11158 if (isa<Instruction>(Val: Op) || isa<Argument>(Val: Op))
11159 InsertAffected(Op);
11160 }
11161 }
11162}
11163
11164void llvm::findValuesAffectedByCondition(
11165 Value *Cond, bool IsAssume, function_ref<void(Value *)> InsertAffected) {
11166 auto AddAffected = [&InsertAffected](Value *V) {
11167 addValueAffectedByCondition(V, InsertAffected);
11168 };
11169
11170 auto AddCmpOperands = [&AddAffected, IsAssume](Value *LHS, Value *RHS) {
11171 if (IsAssume) {
11172 AddAffected(LHS);
11173 AddAffected(RHS);
11174 } else if (match(V: RHS, P: m_Constant()))
11175 AddAffected(LHS);
11176 };
11177
11178 SmallVector<Value *, 8> Worklist;
11179 SmallPtrSet<Value *, 8> Visited;
11180 Worklist.push_back(Elt: Cond);
11181 while (!Worklist.empty()) {
11182 Value *V = Worklist.pop_back_val();
11183 if (!Visited.insert(Ptr: V).second)
11184 continue;
11185
11186 CmpPredicate Pred;
11187 Value *A, *B, *X;
11188
11189 if (IsAssume) {
11190 AddAffected(V);
11191 if (match(V, P: m_Not(V: m_Value(V&: X))))
11192 AddAffected(X);
11193 }
11194
11195 if (match(V, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
11196 // assume(A && B) is split to -> assume(A); assume(B);
11197 // assume(!(A || B)) is split to -> assume(!A); assume(!B);
11198 // Finally, assume(A || B) / assume(!(A && B)) generally don't provide
11199 // enough information to be worth handling (intersection of information as
11200 // opposed to union).
11201 if (!IsAssume) {
11202 Worklist.push_back(Elt: A);
11203 Worklist.push_back(Elt: B);
11204 }
11205 } else if (match(V, P: m_ICmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
11206 bool HasRHSC = match(V: B, P: m_ConstantInt());
11207 if (ICmpInst::isEquality(P: Pred)) {
11208 AddAffected(A);
11209 if (IsAssume)
11210 AddAffected(B);
11211 if (HasRHSC) {
11212 Value *Y;
11213 // (X << C) or (X >>_s C) or (X >>_u C).
11214 if (match(V: A, P: m_Shift(L: m_Value(V&: X), R: m_ConstantInt())))
11215 AddAffected(X);
11216 // (X & C) or (X | C).
11217 else if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
11218 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
11219 AddAffected(X);
11220 AddAffected(Y);
11221 }
11222 // X - Y
11223 else if (match(V: A, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
11224 AddAffected(X);
11225 AddAffected(Y);
11226 }
11227 }
11228 } else {
11229 AddCmpOperands(A, B);
11230 if (HasRHSC) {
11231 Value *Y;
11232 // Handle (A + C1) u< C2, which is the canonical form of
11233 // A > C3 && A < C4.
11234 if (match(V: A, P: m_AddLike(L: m_Value(V&: X), R: m_ConstantInt())))
11235 AddAffected(X);
11236 // X - Y pred C, which implies X != Y if 0 pred C is false.
11237 else if (match(V: A, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
11238 AddAffected(X);
11239 AddAffected(Y);
11240 }
11241
11242 if (ICmpInst::isUnsigned(Pred)) {
11243 // X & Y u> C -> X >u C && Y >u C
11244 // X | Y u< C -> X u< C && Y u< C
11245 // X nuw+ Y u< C -> X u< C && Y u< C
11246 if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
11247 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
11248 match(V: A, P: m_NUWAdd(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
11249 AddAffected(X);
11250 AddAffected(Y);
11251 }
11252 }
11253 }
11254
11255 // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported
11256 // by computeKnownFPClass().
11257 if (match(V: A, P: m_ElementWiseBitCast(Op: m_Value(V&: X)))) {
11258 if (Pred == ICmpInst::ICMP_SLT && match(V: B, P: m_Zero()))
11259 InsertAffected(X);
11260 else if (Pred == ICmpInst::ICMP_SGT && match(V: B, P: m_AllOnes()))
11261 InsertAffected(X);
11262 }
11263 }
11264
11265 auto AddNuwSquareOperand = [&AddAffected](Value *Op) {
11266 Value *SquareOp = nullptr;
11267 if (match(V: Op, P: m_NUWMul(L: m_Value(V&: SquareOp), R: m_Deferred(V: SquareOp))))
11268 AddAffected(SquareOp);
11269 };
11270 AddNuwSquareOperand(A);
11271 AddNuwSquareOperand(B);
11272
11273 if (HasRHSC && match(V: A, P: m_Ctpop(Op0: m_Value(V&: X))))
11274 AddAffected(X);
11275 } else if (match(V, P: m_FCmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
11276 AddCmpOperands(A, B);
11277
11278 // fcmp fneg(x), y
11279 // fcmp fabs(x), y
11280 // fcmp fneg(fabs(x)), y
11281 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
11282 if (match(V: A, P: m_FNegOrFSubNegZero(X: m_Value(V&: A))))
11283 AddAffected(A);
11284 if (match(V: A, P: m_FAbs(Op0: m_Value(V&: A))))
11285 AddAffected(A);
11286
11287 } else if (match(V, P: m_Intrinsic<Intrinsic::is_fpclass>(Ops: m_Value(V&: A),
11288 Ops: m_Value()))) {
11289 // Handle patterns that computeKnownFPClass() support.
11290 AddAffected(A);
11291 } else if (!IsAssume && match(V, P: m_Trunc(Op: m_Value(V&: X)))) {
11292 // Assume is checked here as X is already added above for assumes in
11293 // addValueAffectedByCondition
11294 AddAffected(X);
11295 } else if (!IsAssume && match(V, P: m_Not(V: m_Value(V&: X)))) {
11296 // Assume is checked here to avoid issues with ephemeral values
11297 Worklist.push_back(Elt: X);
11298 }
11299 }
11300}
11301
11302const Value *llvm::stripNullTest(const Value *V) {
11303 // (X >> C) or/add (X & mask(C) != 0)
11304 if (const auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
11305 if (BO->getOpcode() == Instruction::Add ||
11306 BO->getOpcode() == Instruction::Or) {
11307 const Value *X;
11308 const APInt *C1, *C2;
11309 if (match(V: BO, P: m_c_BinOp(L: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: C1)),
11310 R: m_ZExt(Op: m_SpecificICmp(
11311 MatchPred: ICmpInst::ICMP_NE,
11312 L: m_And(L: m_Deferred(V: X), R: m_LowBitMask(V&: C2)),
11313 R: m_Zero())))) &&
11314 C2->popcount() == C1->getZExtValue())
11315 return X;
11316 }
11317 }
11318
11319 // select(cond, X, -X)
11320 {
11321 const Value *X;
11322 if (match(V, P: m_c_Select(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X)))))
11323 return X;
11324 }
11325
11326 return nullptr;
11327}
11328
11329Value *llvm::stripNullTest(Value *V) {
11330 return const_cast<Value *>(stripNullTest(V: const_cast<const Value *>(V)));
11331}
11332
11333bool llvm::collectPossibleValues(const Value *V,
11334 SmallPtrSetImpl<const Constant *> &Constants,
11335 unsigned MaxCount, bool AllowUndefOrPoison) {
11336 SmallPtrSet<const Instruction *, 8> Visited;
11337 SmallVector<const Instruction *, 8> Worklist;
11338 auto Push = [&](const Value *V) -> bool {
11339 Constant *C;
11340 if (match(V: const_cast<Value *>(V), P: m_ImmConstant(C))) {
11341 if (!AllowUndefOrPoison && !isGuaranteedNotToBeUndefOrPoison(V: C))
11342 return false;
11343 // Check existence first to avoid unnecessary allocations.
11344 if (Constants.contains(Ptr: C))
11345 return true;
11346 if (Constants.size() == MaxCount)
11347 return false;
11348 Constants.insert(Ptr: C);
11349 return true;
11350 }
11351
11352 if (auto *Inst = dyn_cast<Instruction>(Val: V)) {
11353 if (Visited.insert(Ptr: Inst).second)
11354 Worklist.push_back(Elt: Inst);
11355 return true;
11356 }
11357 return false;
11358 };
11359 if (!Push(V))
11360 return false;
11361 while (!Worklist.empty()) {
11362 const Instruction *CurInst = Worklist.pop_back_val();
11363 switch (CurInst->getOpcode()) {
11364 case Instruction::Select:
11365 if (!Push(CurInst->getOperand(i: 1)))
11366 return false;
11367 if (!Push(CurInst->getOperand(i: 2)))
11368 return false;
11369 break;
11370 case Instruction::PHI:
11371 for (Value *IncomingValue : cast<PHINode>(Val: CurInst)->incoming_values()) {
11372 // Fast path for recurrence PHI.
11373 if (IncomingValue == CurInst)
11374 continue;
11375 if (!Push(IncomingValue))
11376 return false;
11377 }
11378 break;
11379 default:
11380 return false;
11381 }
11382 }
11383 return true;
11384}
11385