1//===- ValueTracking.cpp - Walk computations to compute properties --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains routines that help analyze properties that chains of
10// computations have.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ValueTracking.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/FloatingPointMode.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/ADT/SmallPtrSet.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/ADT/iterator_range.h"
24#include "llvm/Analysis/AliasAnalysis.h"
25#include "llvm/Analysis/AssumeBundleQueries.h"
26#include "llvm/Analysis/AssumptionCache.h"
27#include "llvm/Analysis/ConstantFolding.h"
28#include "llvm/Analysis/DomConditionCache.h"
29#include "llvm/Analysis/FloatingPointPredicateUtils.h"
30#include "llvm/Analysis/GuardUtils.h"
31#include "llvm/Analysis/InstructionSimplify.h"
32#include "llvm/Analysis/Loads.h"
33#include "llvm/Analysis/LoopInfo.h"
34#include "llvm/Analysis/TargetLibraryInfo.h"
35#include "llvm/Analysis/VectorUtils.h"
36#include "llvm/Analysis/WithCache.h"
37#include "llvm/IR/Argument.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/BasicBlock.h"
40#include "llvm/IR/BundleAttributes.h"
41#include "llvm/IR/Constant.h"
42#include "llvm/IR/ConstantFPRange.h"
43#include "llvm/IR/ConstantRange.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DerivedTypes.h"
46#include "llvm/IR/DiagnosticInfo.h"
47#include "llvm/IR/Dominators.h"
48#include "llvm/IR/EHPersonalities.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GetElementPtrTypeIterator.h"
51#include "llvm/IR/GlobalAlias.h"
52#include "llvm/IR/GlobalValue.h"
53#include "llvm/IR/GlobalVariable.h"
54#include "llvm/IR/InstrTypes.h"
55#include "llvm/IR/Instruction.h"
56#include "llvm/IR/Instructions.h"
57#include "llvm/IR/IntrinsicInst.h"
58#include "llvm/IR/Intrinsics.h"
59#include "llvm/IR/IntrinsicsAArch64.h"
60#include "llvm/IR/IntrinsicsAMDGPU.h"
61#include "llvm/IR/IntrinsicsRISCV.h"
62#include "llvm/IR/IntrinsicsX86.h"
63#include "llvm/IR/LLVMContext.h"
64#include "llvm/IR/Metadata.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/Operator.h"
67#include "llvm/IR/PatternMatch.h"
68#include "llvm/IR/Type.h"
69#include "llvm/IR/User.h"
70#include "llvm/IR/Value.h"
71#include "llvm/Support/Casting.h"
72#include "llvm/Support/CommandLine.h"
73#include "llvm/Support/Compiler.h"
74#include "llvm/Support/ErrorHandling.h"
75#include "llvm/Support/KnownBits.h"
76#include "llvm/Support/KnownFPClass.h"
77#include "llvm/Support/MathExtras.h"
78#include "llvm/Support/UndefPoison.h"
79#include "llvm/TargetParser/RISCVTargetParser.h"
80#include <algorithm>
81#include <cassert>
82#include <cstdint>
83#include <optional>
84#include <utility>
85
86using namespace llvm;
87using namespace llvm::PatternMatch;
88
89// Controls the number of uses of the value searched for possible
90// dominating comparisons.
91static cl::opt<unsigned> DomConditionsMaxUses("dom-conditions-max-uses",
92 cl::Hidden, cl::init(Val: 20));
93
94/// Maximum number of instructions to check between assume and context
95/// instruction.
96static constexpr unsigned MaxInstrsToCheckForFree = 32;
97
98template <typename InstTy>
99static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
100 Value *&Init, Value *&OtherOp);
101
102/// Returns the bitwidth of the given scalar or pointer type. For vector types,
103/// returns the element type's bitwidth.
104static unsigned getBitWidth(Type *Ty, const DataLayout &DL) {
105 if (unsigned BitWidth = Ty->getScalarSizeInBits())
106 return BitWidth;
107
108 return DL.getPointerTypeSizeInBits(Ty);
109}
110
111// Given the provided Value and, potentially, a context instruction, return
112// the preferred context instruction (if any).
113static const Instruction *safeCxtI(const Value *V, const Instruction *CxtI) {
114 // If we've been provided with a context instruction, then use that (provided
115 // it has been inserted).
116 if (CxtI && CxtI->getParent())
117 return CxtI;
118
119 // If the value is really an already-inserted instruction, then use that.
120 CxtI = dyn_cast<Instruction>(Val: V);
121 if (CxtI && CxtI->getParent())
122 return CxtI;
123
124 return nullptr;
125}
126
127static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
128 const APInt &DemandedElts,
129 APInt &DemandedLHS, APInt &DemandedRHS) {
130 if (isa<ScalableVectorType>(Val: Shuf->getType())) {
131 assert(DemandedElts == APInt(1,1));
132 DemandedLHS = DemandedRHS = DemandedElts;
133 return true;
134 }
135
136 int NumElts =
137 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
138 return llvm::getShuffleDemandedElts(SrcWidth: NumElts, Mask: Shuf->getShuffleMask(),
139 DemandedElts, DemandedLHS, DemandedRHS);
140}
141
142static void computeKnownBits(const Value *V, const APInt &DemandedElts,
143 KnownBits &Known, const SimplifyQuery &Q,
144 unsigned Depth);
145
146void llvm::computeKnownBits(const Value *V, KnownBits &Known,
147 const SimplifyQuery &Q, unsigned Depth) {
148 // Since the number of lanes in a scalable vector is unknown at compile time,
149 // we track one bit which is implicitly broadcast to all lanes. This means
150 // that all lanes in a scalable vector are considered demanded.
151 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
152 APInt DemandedElts =
153 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
154 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
155}
156
157void llvm::computeKnownBits(const Value *V, KnownBits &Known,
158 const DataLayout &DL, AssumptionCache *AC,
159 const Instruction *CxtI, const DominatorTree *DT,
160 bool UseInstrInfo, unsigned Depth) {
161 computeKnownBits(V, Known,
162 Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo),
163 Depth);
164}
165
166KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
167 AssumptionCache *AC, const Instruction *CxtI,
168 const DominatorTree *DT, bool UseInstrInfo,
169 unsigned Depth) {
170 return computeKnownBits(
171 V, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
172}
173
174KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
175 const DataLayout &DL, AssumptionCache *AC,
176 const Instruction *CxtI,
177 const DominatorTree *DT, bool UseInstrInfo,
178 unsigned Depth) {
179 return computeKnownBits(
180 V, DemandedElts,
181 Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
182}
183
184static NoCommonBitsSetResult
185haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
186 const SimplifyQuery &SQ) {
187 // Look for an inverted mask: (X & ~M) op (Y & M).
188 {
189 Value *M;
190 if (match(V: LHS, P: m_c_And(L: m_Not(V: m_Value(V&: M)), R: m_Value())) &&
191 match(V: RHS, P: m_c_And(L: m_Specific(V: M), R: m_Value())))
192 return isGuaranteedNotToBeUndef(V: M, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT)
193 ? NoCommonBitsSetResult::Known
194 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
195 }
196
197 // X op (Y & ~X)
198 if (match(V: RHS, P: m_c_And(L: m_Not(V: m_Specific(V: LHS)), R: m_Value())))
199 return isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT)
200 ? NoCommonBitsSetResult::Known
201 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
202
203 // X op ((X & Y) ^ Y) -- this is the canonical form of the previous pattern
204 // for constant Y.
205 Value *Y;
206 if (match(V: RHS,
207 P: m_c_Xor(L: m_c_And(L: m_Specific(V: LHS), R: m_Value(V&: Y)), R: m_Deferred(V: Y)))) {
208 bool IsNoUndef = isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT) &&
209 isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT);
210 return IsNoUndef ? NoCommonBitsSetResult::Known
211 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
212 }
213
214 // Peek through extends to find a 'not' of the other side:
215 // (ext Y) op ext(~Y)
216 if (match(V: LHS, P: m_ZExtOrSExt(Op: m_Value(V&: Y))) &&
217 match(V: RHS, P: m_ZExtOrSExt(Op: m_Not(V: m_Specific(V: Y)))))
218 return isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT)
219 ? NoCommonBitsSetResult::Known
220 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
221
222 // Look for: (A & B) op ~(A | B)
223 {
224 Value *A, *B;
225 if (match(V: LHS, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
226 match(V: RHS, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))) {
227 bool IsNoUndef = isGuaranteedNotToBeUndef(V: A, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT) &&
228 isGuaranteedNotToBeUndef(V: B, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT);
229 return IsNoUndef ? NoCommonBitsSetResult::Known
230 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
231 }
232 }
233
234 // Look for: (X << V) op (Y >> (BitWidth - V))
235 // or (X >> V) op (Y << (BitWidth - V))
236 {
237 const Value *V;
238 const APInt *R;
239 if (((match(V: RHS, P: m_Shl(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
240 match(V: LHS, P: m_LShr(L: m_Value(), R: m_Specific(V)))) ||
241 (match(V: RHS, P: m_LShr(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
242 match(V: LHS, P: m_Shl(L: m_Value(), R: m_Specific(V))))) &&
243 R->uge(RHS: LHS->getType()->getScalarSizeInBits()))
244 return NoCommonBitsSetResult::Known;
245 }
246
247 return NoCommonBitsSetResult::Unknown;
248}
249
250NoCommonBitsSetResult
251llvm::getNoCommonBitsSetResult(const WithCache<const Value *> &LHSCache,
252 const WithCache<const Value *> &RHSCache,
253 const SimplifyQuery &SQ) {
254 const Value *LHS = LHSCache.getValue();
255 const Value *RHS = RHSCache.getValue();
256
257 assert(LHS->getType() == RHS->getType() &&
258 "LHS and RHS should have the same type");
259 assert(LHS->getType()->isIntOrIntVectorTy() &&
260 "LHS and RHS should be integers");
261
262 NoCommonBitsSetResult Result = haveNoCommonBitsSetSpecialCases(LHS, RHS, SQ);
263 if (Result == NoCommonBitsSetResult::Known)
264 return NoCommonBitsSetResult::Known;
265
266 NoCommonBitsSetResult CommuteResult =
267 haveNoCommonBitsSetSpecialCases(LHS: RHS, RHS: LHS, SQ);
268 if (CommuteResult == NoCommonBitsSetResult::Known)
269 return NoCommonBitsSetResult::Known;
270
271 if (KnownBits::haveNoCommonBitsSet(LHS: LHSCache.getKnownBits(Q: SQ),
272 RHS: RHSCache.getKnownBits(Q: SQ)))
273 return NoCommonBitsSetResult::Known;
274
275 if (Result == NoCommonBitsSetResult::OnlyIfUndefIgnored ||
276 CommuteResult == NoCommonBitsSetResult::OnlyIfUndefIgnored)
277 return NoCommonBitsSetResult::OnlyIfUndefIgnored;
278
279 return NoCommonBitsSetResult::Unknown;
280}
281
282bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
283 const WithCache<const Value *> &RHSCache,
284 const SimplifyQuery &SQ) {
285 NoCommonBitsSetResult Result =
286 getNoCommonBitsSetResult(LHSCache, RHSCache, SQ);
287 return Result == NoCommonBitsSetResult::Known;
288}
289
290bool llvm::isOnlyUsedInZeroComparison(const Instruction *I) {
291 return !I->user_empty() &&
292 all_of(Range: I->users(), P: match_fn(P: m_ICmp(L: m_Value(), R: m_Zero())));
293}
294
295bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) {
296 return !I->user_empty() && all_of(Range: I->users(), P: [](const User *U) {
297 CmpPredicate P;
298 return match(V: U, P: m_ICmp(Pred&: P, L: m_Value(), R: m_Zero())) && ICmpInst::isEquality(P);
299 });
300}
301
302bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
303 bool OrZero, AssumptionCache *AC,
304 const Instruction *CxtI,
305 const DominatorTree *DT, bool UseInstrInfo,
306 unsigned Depth) {
307 return ::isKnownToBeAPowerOfTwo(
308 V, OrZero, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo),
309 Depth);
310}
311
312static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
313 const SimplifyQuery &Q, unsigned Depth);
314
315bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
316 unsigned Depth) {
317 return computeKnownBits(V, Q: SQ, Depth).isNonNegative();
318}
319
320bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
321 unsigned Depth) {
322 if (auto *CI = dyn_cast<ConstantInt>(Val: V))
323 return CI->getValue().isStrictlyPositive();
324
325 // If `isKnownNonNegative` ever becomes more sophisticated, make sure to keep
326 // this updated.
327 KnownBits Known = computeKnownBits(V, Q: SQ, Depth);
328 return Known.isNonNegative() &&
329 (Known.isNonZero() || isKnownNonZero(V, Q: SQ, Depth));
330}
331
332bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ,
333 unsigned Depth) {
334 return computeKnownBits(V, Q: SQ, Depth).isNegative();
335}
336
337static bool isKnownNonEqual(const Value *V1, const Value *V2,
338 const APInt &DemandedElts, const SimplifyQuery &Q,
339 unsigned Depth);
340
341static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
342 const Value *RHS);
343
344bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
345 const SimplifyQuery &Q, unsigned Depth) {
346 // We don't support looking through casts.
347 if (V1 == V2 || V1->getType() != V2->getType())
348 return false;
349 auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType());
350 APInt DemandedElts =
351 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
352 return ::isKnownNonEqual(V1, V2, DemandedElts, Q, Depth);
353}
354
355bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
356 const SimplifyQuery &SQ, unsigned Depth) {
357 KnownBits Known(Mask.getBitWidth());
358 computeKnownBits(V, Known, Q: SQ, Depth);
359 return Mask.isSubsetOf(RHS: Known.Zero);
360}
361
362static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
363 const SimplifyQuery &Q, unsigned Depth);
364
365static unsigned ComputeNumSignBits(const Value *V, const SimplifyQuery &Q,
366 unsigned Depth = 0) {
367 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
368 APInt DemandedElts =
369 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
370 return ComputeNumSignBits(V, DemandedElts, Q, Depth);
371}
372
373unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL,
374 AssumptionCache *AC, const Instruction *CxtI,
375 const DominatorTree *DT, bool UseInstrInfo,
376 unsigned Depth) {
377 return ::ComputeNumSignBits(
378 V, Q: SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo), Depth);
379}
380
381unsigned llvm::ComputeMaxSignificantBits(const Value *V, const DataLayout &DL,
382 AssumptionCache *AC,
383 const Instruction *CxtI,
384 const DominatorTree *DT,
385 unsigned Depth) {
386 unsigned SignBits = ComputeNumSignBits(V, DL, AC, CxtI, DT, UseInstrInfo: Depth);
387 return V->getType()->getScalarSizeInBits() - SignBits + 1;
388}
389
390/// Try to detect the lerp pattern: a * (b - c) + c * d
391/// where a >= 0, b >= 0, c >= 0, d >= 0, and b >= c.
392///
393/// In that particular case, we can use the following chain of reasoning:
394///
395/// a * (b - c) + c * d <= a' * (b - c) + a' * c = a' * b where a' = max(a, d)
396///
397/// Since that is true for arbitrary a, b, c and d within our constraints, we
398/// can conclude that:
399///
400/// max(a * (b - c) + c * d) <= max(max(a), max(d)) * max(b) = U
401///
402/// Considering that any result of the lerp would be less or equal to U, it
403/// would have at least the number of leading 0s as in U.
404///
405/// While being quite a specific situation, it is fairly common in computer
406/// graphics in the shape of alpha blending.
407///
408/// Modifies given KnownOut in-place with the inferred information.
409static void computeKnownBitsFromLerpPattern(const Value *Op0, const Value *Op1,
410 const APInt &DemandedElts,
411 KnownBits &KnownOut,
412 const SimplifyQuery &Q,
413 unsigned Depth) {
414
415 Type *Ty = Op0->getType();
416 const unsigned BitWidth = Ty->getScalarSizeInBits();
417
418 // Only handle scalar types for now
419 if (Ty->isVectorTy())
420 return;
421
422 // Try to match: a * (b - c) + c * d.
423 // When a == 1 => A == nullptr, the same applies to d/D as well.
424 const Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
425 const Instruction *SubBC = nullptr;
426
427 const auto MatchSubBC = [&]() {
428 // (b - c) can have two forms that interest us:
429 //
430 // 1. sub nuw %b, %c
431 // 2. xor %c, %b
432 //
433 // For the first case, nuw flag guarantees our requirement b >= c.
434 //
435 // The second case might happen when the analysis can infer that b is a mask
436 // for c and we can transform sub operation into xor (that is usually true
437 // for constant b's). Even though xor is symmetrical, canonicalization
438 // ensures that the constant will be the RHS. We have additional checks
439 // later on to ensure that this xor operation is equivalent to subtraction.
440 return m_Instruction(I&: SubBC, P: m_CombineOr(Ps: m_NUWSub(L: m_Value(V&: B), R: m_Value(V&: C)),
441 Ps: m_Xor(L: m_Value(V&: C), R: m_Value(V&: B))));
442 };
443
444 const auto MatchASubBC = [&]() {
445 // Cases:
446 // - a * (b - c)
447 // - (b - c) * a
448 // - (b - c) <- a implicitly equals 1
449 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: A), R: MatchSubBC()), Ps: MatchSubBC());
450 };
451
452 const auto MatchCD = [&]() {
453 // Cases:
454 // - d * c
455 // - c * d
456 // - c <- d implicitly equals 1
457 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: D), R: m_Specific(V: C)), Ps: m_Specific(V: C));
458 };
459
460 const auto Match = [&](const Value *LHS, const Value *RHS) {
461 // We do use m_Specific(C) in MatchCD, so we have to make sure that
462 // it's bound to anything and match(LHS, MatchASubBC()) absolutely
463 // has to evaluate first and return true.
464 //
465 // If Match returns true, it is guaranteed that B != nullptr, C != nullptr.
466 return match(V: LHS, P: MatchASubBC()) && match(V: RHS, P: MatchCD());
467 };
468
469 if (!Match(Op0, Op1) && !Match(Op1, Op0))
470 return;
471
472 const auto ComputeKnownBitsOrOne = [&](const Value *V) {
473 // For some of the values we use the convention of leaving
474 // it nullptr to signify an implicit constant 1.
475 return V ? computeKnownBits(V, DemandedElts, Q, Depth: Depth + 1)
476 : KnownBits::makeConstant(C: APInt(BitWidth, 1));
477 };
478
479 // Check that all operands are non-negative
480 const KnownBits KnownA = ComputeKnownBitsOrOne(A);
481 if (!KnownA.isNonNegative())
482 return;
483
484 const KnownBits KnownD = ComputeKnownBitsOrOne(D);
485 if (!KnownD.isNonNegative())
486 return;
487
488 const KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
489 if (!KnownB.isNonNegative())
490 return;
491
492 const KnownBits KnownC = computeKnownBits(V: C, DemandedElts, Q, Depth: Depth + 1);
493 if (!KnownC.isNonNegative())
494 return;
495
496 // If we matched subtraction as xor, we need to actually check that xor
497 // is semantically equivalent to subtraction.
498 //
499 // For that to be true, b has to be a mask for c or that b's known
500 // ones cover all known and possible ones of c.
501 if (SubBC->getOpcode() == Instruction::Xor &&
502 !KnownC.getMaxValue().isSubsetOf(RHS: KnownB.getMinValue()))
503 return;
504
505 const APInt MaxA = KnownA.getMaxValue();
506 const APInt MaxD = KnownD.getMaxValue();
507 const APInt MaxAD = APIntOps::umax(A: MaxA, B: MaxD);
508 const APInt MaxB = KnownB.getMaxValue();
509
510 // We can't infer leading zeros info if the upper-bound estimate wraps.
511 bool Overflow;
512 const APInt UpperBound = MaxAD.umul_ov(RHS: MaxB, Overflow);
513
514 if (Overflow)
515 return;
516
517 // If we know that x <= y and both are positive than x has at least the same
518 // number of leading zeros as y.
519 const unsigned MinimumNumberOfLeadingZeros = UpperBound.countl_zero();
520 KnownOut.Zero.setHighBits(MinimumNumberOfLeadingZeros);
521}
522
523static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
524 bool NSW, bool NUW,
525 const APInt &DemandedElts,
526 KnownBits &KnownOut, KnownBits &Known2,
527 const SimplifyQuery &Q, unsigned Depth) {
528 computeKnownBits(V: Op1, DemandedElts, Known&: KnownOut, Q, Depth: Depth + 1);
529
530 // If one operand is unknown and we have no nowrap information,
531 // the result will be unknown independently of the second operand.
532 if (KnownOut.isUnknown() && !NSW && !NUW)
533 return;
534
535 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
536 KnownOut = KnownBits::computeForAddSub(Add, NSW, NUW, LHS: Known2, RHS: KnownOut);
537
538 if (!Add && NSW && !KnownOut.isNonNegative() &&
539 (isImpliedByDomCondition(Pred: ICmpInst::ICMP_SLE, LHS: Op1, RHS: Op0, ContextI: Q.CxtI, DL: Q.DL)
540 .value_or(u: false) ||
541 match(V: Op1, P: m_c_SMin(L: m_Specific(V: Op0), R: m_Value()))))
542 KnownOut.makeNonNegative();
543
544 if (Add)
545 // Try to match lerp pattern and combine results
546 computeKnownBitsFromLerpPattern(Op0, Op1, DemandedElts, KnownOut, Q, Depth);
547}
548
549static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
550 bool NUW, const APInt &DemandedElts,
551 KnownBits &Known, KnownBits &Known2,
552 const SimplifyQuery &Q, unsigned Depth) {
553 computeKnownBits(V: Op1, DemandedElts, Known, Q, Depth: Depth + 1);
554 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
555
556 bool isKnownNegative = false;
557 bool isKnownNonNegative = false;
558 // If the multiplication is known not to overflow, compute the sign bit.
559 if (NSW) {
560 if (Op0 == Op1) {
561 // The product of a number with itself is non-negative.
562 isKnownNonNegative = true;
563 } else {
564 bool isKnownNonNegativeOp1 = Known.isNonNegative();
565 bool isKnownNonNegativeOp0 = Known2.isNonNegative();
566 bool isKnownNegativeOp1 = Known.isNegative();
567 bool isKnownNegativeOp0 = Known2.isNegative();
568 // The product of two numbers with the same sign is non-negative.
569 isKnownNonNegative = (isKnownNegativeOp1 && isKnownNegativeOp0) ||
570 (isKnownNonNegativeOp1 && isKnownNonNegativeOp0);
571 if (!isKnownNonNegative && NUW) {
572 // mul nuw nsw with a factor > 1 is non-negative.
573 KnownBits One = KnownBits::makeConstant(C: APInt(Known.getBitWidth(), 1));
574 isKnownNonNegative = KnownBits::sgt(LHS: Known, RHS: One).value_or(u: false) ||
575 KnownBits::sgt(LHS: Known2, RHS: One).value_or(u: false);
576 }
577
578 // The product of a negative number and a non-negative number is either
579 // negative or zero.
580 if (!isKnownNonNegative)
581 isKnownNegative =
582 (isKnownNegativeOp1 && isKnownNonNegativeOp0 &&
583 Known2.isNonZero()) ||
584 (isKnownNegativeOp0 && isKnownNonNegativeOp1 && Known.isNonZero());
585 }
586 }
587
588 bool SelfMultiply = Op0 == Op1;
589 if (SelfMultiply)
590 SelfMultiply &=
591 isGuaranteedNotToBeUndef(V: Op0, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1);
592 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
593
594 if (SelfMultiply) {
595 unsigned SignBits = ComputeNumSignBits(V: Op0, DemandedElts, Q, Depth: Depth + 1);
596 unsigned TyBits = Op0->getType()->getScalarSizeInBits();
597 unsigned OutValidBits = 2 * (TyBits - SignBits + 1);
598
599 if (OutValidBits < TyBits) {
600 APInt KnownZeroMask =
601 APInt::getHighBitsSet(numBits: TyBits, hiBitsSet: TyBits - OutValidBits + 1);
602 Known.Zero |= KnownZeroMask;
603 }
604 }
605
606 // Only make use of no-wrap flags if we failed to compute the sign bit
607 // directly. This matters if the multiplication always overflows, in
608 // which case we prefer to follow the result of the direct computation,
609 // though as the program is invoking undefined behaviour we can choose
610 // whatever we like here.
611 if (isKnownNonNegative && !Known.isNegative())
612 Known.makeNonNegative();
613 else if (isKnownNegative && !Known.isNonNegative())
614 Known.makeNegative();
615}
616
617void llvm::computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
618 KnownBits &Known) {
619 unsigned BitWidth = Known.getBitWidth();
620 unsigned NumRanges = Ranges.getNumOperands() / 2;
621 assert(NumRanges >= 1);
622
623 Known.setAllConflict();
624
625 for (unsigned i = 0; i < NumRanges; ++i) {
626 ConstantInt *Lower =
627 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 0));
628 ConstantInt *Upper =
629 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 1));
630 ConstantRange Range(Lower->getValue(), Upper->getValue());
631 // BitWidth must equal the Ranges BitWidth for the correct number of high
632 // bits to be set.
633 assert(BitWidth == Range.getBitWidth() &&
634 "Known bit width must match range bit width!");
635
636 // The first CommonPrefixBits of all values in Range are equal.
637 unsigned CommonPrefixBits =
638 (Range.getUnsignedMax() ^ Range.getUnsignedMin()).countl_zero();
639 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: CommonPrefixBits);
640 APInt UnsignedMax = Range.getUnsignedMax().zextOrTrunc(width: BitWidth);
641 Known.One &= UnsignedMax & Mask;
642 Known.Zero &= ~UnsignedMax & Mask;
643 }
644}
645
646static bool isEphemeralValueOf(const Instruction *I, const Value *E) {
647 // The instruction defining an assumption's condition itself is always
648 // considered ephemeral to that assumption (even if it has other
649 // non-ephemeral users). See r246696's test case for an example.
650 if (is_contained(Range: I->operands(), Element: E))
651 return true;
652
653 const auto *EI = dyn_cast<Instruction>(Val: E);
654 if (!EI)
655 return false;
656
657 if (EI == I)
658 return true;
659
660 SmallPtrSet<const Instruction *, 16> Visited;
661 SmallVector<const Instruction *, 16> WorkList;
662 Visited.insert(Ptr: EI);
663 WorkList.push_back(Elt: EI);
664 bool ReachesI = false;
665 while (!WorkList.empty()) {
666 const Instruction *V = WorkList.pop_back_val();
667 for (const User *U : V->users()) {
668 const auto *UI = cast<Instruction>(Val: U);
669 if (UI == I) {
670 ReachesI = true;
671 continue;
672 }
673 if (UI->mayHaveSideEffects() || UI->isTerminator())
674 return false;
675 if (Visited.insert(Ptr: UI).second)
676 WorkList.push_back(Elt: UI);
677 }
678 }
679 return ReachesI;
680}
681
682// Is this an intrinsic that cannot be speculated but also cannot trap?
683bool llvm::isAssumeLikeIntrinsic(const Instruction *I) {
684 if (const IntrinsicInst *CI = dyn_cast<IntrinsicInst>(Val: I))
685 return CI->isAssumeLikeIntrinsic();
686
687 return false;
688}
689
690bool llvm::isValidAssumeForContext(const Instruction *Inv,
691 const Instruction *CxtI,
692 const DominatorTree *DT,
693 bool AllowEphemerals) {
694 // There are two restrictions on the use of an assume:
695 // 1. The assume must dominate the context (or the control flow must
696 // reach the assume whenever it reaches the context).
697 // 2. The context must not be in the assume's set of ephemeral values
698 // (otherwise we will use the assume to prove that the condition
699 // feeding the assume is trivially true, thus causing the removal of
700 // the assume).
701
702 if (Inv->getParent() == CxtI->getParent()) {
703 // If Inv and CtxI are in the same block, check if the assume (Inv) is first
704 // in the BB.
705 if (Inv->comesBefore(Other: CxtI))
706 return true;
707
708 // Don't let an assume affect itself - this would cause the problems
709 // `isEphemeralValueOf` is trying to prevent, and it would also make
710 // the loop below go out of bounds.
711 if (!AllowEphemerals && Inv == CxtI)
712 return false;
713
714 // The context comes first, but they're both in the same block.
715 // Make sure there is nothing in between that might interrupt
716 // the control flow, not even CxtI itself.
717 // We limit the scan distance between the assume and its context instruction
718 // to avoid a compile-time explosion. This limit is chosen arbitrarily, so
719 // it can be adjusted if needed (could be turned into a cl::opt).
720 auto Range = make_range(x: CxtI->getIterator(), y: Inv->getIterator());
721 if (!isGuaranteedToTransferExecutionToSuccessor(Range, ScanLimit: 15))
722 return false;
723
724 return AllowEphemerals || !isEphemeralValueOf(I: Inv, E: CxtI);
725 }
726
727 // Inv and CxtI are in different blocks.
728 if (DT) {
729 if (DT->dominates(Def: Inv, User: CxtI))
730 return true;
731 } else if (Inv->getParent() == CxtI->getParent()->getSinglePredecessor() ||
732 Inv->getParent()->isEntryBlock()) {
733 // We don't have a DT, but this trivially dominates.
734 return true;
735 }
736
737 return false;
738}
739
740bool llvm::willNotFreeBetween(const Instruction *Assume,
741 const Instruction *CtxI) {
742 // Helper to check if there are any calls in the range that may free memory.
743 unsigned NumChecked = 0;
744 auto hasNoFreeInRange = [&NumChecked](auto Range) {
745 for (const Instruction &I : Range) {
746 if (NumChecked++ > MaxInstrsToCheckForFree)
747 return false;
748
749 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
750 if (!CB->hasFnAttr(Kind: Attribute::NoFree))
751 return false;
752 } else if (I.maySynchronize())
753 return false;
754 }
755 return true;
756 };
757
758 const BasicBlock *CtxBB = CtxI->getParent();
759 const BasicBlock *AssumeBB = Assume->getParent();
760 BasicBlock::const_iterator CtxIter = CtxI->getIterator();
761 if (CtxBB == AssumeBB) {
762 // Same block case: check that Assume comes before CtxI.
763 if (Assume != CtxI && !Assume->comesBefore(Other: CtxI))
764 return false;
765 return hasNoFreeInRange(make_range(x: Assume->getIterator(), y: CtxIter));
766 }
767
768 // Handle chain of single-predecessor blocks.
769 const BasicBlock *CurBB = CtxBB;
770 while (true) {
771 if (CurBB == AssumeBB)
772 return hasNoFreeInRange(
773 make_range(x: Assume->getIterator(), y: AssumeBB->end()));
774
775 const BasicBlock *PredBB = CurBB->getSinglePredecessor();
776 if (!PredBB)
777 return false;
778
779 if (!hasNoFreeInRange(make_range(x: CurBB->begin(),
780 y: CurBB == CtxBB ? CtxIter : CurBB->end())))
781 return false;
782 CurBB = PredBB;
783 }
784}
785
786// TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but
787// we still have enough information about `RHS` to conclude non-zero. For
788// example Pred=EQ, RHS=isKnownNonZero. cmpExcludesZero is called in loops
789// so the extra compile time may not be worth it, but possibly a second API
790// should be created for use outside of loops.
791static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) {
792 // v u> y implies v != 0.
793 if (Pred == ICmpInst::ICMP_UGT)
794 return true;
795
796 // Special-case v != 0 to also handle v != null.
797 if (Pred == ICmpInst::ICMP_NE)
798 return match(V: RHS, P: m_Zero());
799
800 // All other predicates - rely on generic ConstantRange handling.
801 const APInt *C;
802 auto Zero = APInt::getZero(numBits: RHS->getType()->getScalarSizeInBits());
803 if (match(V: RHS, P: m_APInt(Res&: C))) {
804 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
805 return !TrueValues.contains(Val: Zero);
806 }
807
808 auto *VC = dyn_cast<ConstantDataVector>(Val: RHS);
809 if (VC == nullptr)
810 return false;
811
812 for (unsigned ElemIdx = 0, NElem = VC->getNumElements(); ElemIdx < NElem;
813 ++ElemIdx) {
814 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(
815 Pred, Other: VC->getElementAsAPInt(i: ElemIdx));
816 if (TrueValues.contains(Val: Zero))
817 return false;
818 }
819 return true;
820}
821
822static void breakSelfRecursivePHI(const Use *U, const PHINode *PHI,
823 Value *&ValOut, Instruction *&CtxIOut,
824 const PHINode **PhiOut = nullptr) {
825 ValOut = U->get();
826 if (ValOut == PHI)
827 return;
828 CtxIOut = PHI->getIncomingBlock(U: *U)->getTerminator();
829 if (PhiOut)
830 *PhiOut = PHI;
831 Value *V;
832 // If the Use is a select of this phi, compute analysis on other arm to break
833 // recursion.
834 // TODO: Min/Max
835 if (match(V: ValOut, P: m_Select(C: m_Value(), L: m_Specific(V: PHI), R: m_Value(V))) ||
836 match(V: ValOut, P: m_Select(C: m_Value(), L: m_Value(V), R: m_Specific(V: PHI))))
837 ValOut = V;
838
839 // Same for select, if this phi is 2-operand phi, compute analysis on other
840 // incoming value to break recursion.
841 // TODO: We could handle any number of incoming edges as long as we only have
842 // two unique values.
843 if (auto *IncPhi = dyn_cast<PHINode>(Val: ValOut);
844 IncPhi && IncPhi->getNumIncomingValues() == 2) {
845 for (int Idx = 0; Idx < 2; ++Idx) {
846 if (IncPhi->getIncomingValue(i: Idx) == PHI) {
847 ValOut = IncPhi->getIncomingValue(i: 1 - Idx);
848 if (PhiOut)
849 *PhiOut = IncPhi;
850 CtxIOut = IncPhi->getIncomingBlock(i: 1 - Idx)->getTerminator();
851 break;
852 }
853 }
854 }
855}
856
857static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) {
858 // Use of assumptions is context-sensitive. If we don't have a context, we
859 // cannot use them!
860 if (!Q.AC || !Q.CxtI)
861 return false;
862
863 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
864 if (!Elem.Assume)
865 continue;
866
867 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
868 assert(I->getFunction() == Q.CxtI->getFunction() &&
869 "Got assumption for the wrong function!");
870
871 if (Elem.Index != AssumptionCache::ExprResultIdx) {
872 if (assumeBundleImpliesNonNull(Val: V, Context: Q.CxtI->getFunction(),
873 OBU: I->getOperandBundleAt(Index: Elem.Index)) &&
874 isValidAssumeForContext(I, Q))
875 return true;
876 continue;
877 }
878
879 // Warning: This loop can end up being somewhat performance sensitive.
880 // We're running this loop for once for each value queried resulting in a
881 // runtime of ~O(#assumes * #values).
882
883 Value *RHS;
884 CmpPredicate Pred;
885 auto m_V = m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToInt(Op: m_Specific(V)));
886 if (!match(V: I->getArgOperand(i: 0), P: m_c_ICmp(Pred, L: m_V, R: m_Value(V&: RHS))))
887 continue;
888
889 if (cmpExcludesZero(Pred, RHS) && isValidAssumeForContext(I, Q))
890 return true;
891 }
892
893 return false;
894}
895
896static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
897 Value *LHS, Value *RHS, KnownBits &Known,
898 const SimplifyQuery &Q) {
899 if (RHS->getType()->isPointerTy()) {
900 // Handle comparison of pointer to null explicitly, as it will not be
901 // covered by the m_APInt() logic below.
902 if (LHS == V && match(V: RHS, P: m_Zero())) {
903 switch (Pred) {
904 case ICmpInst::ICMP_EQ:
905 Known.setAllZero();
906 break;
907 case ICmpInst::ICMP_SGE:
908 case ICmpInst::ICMP_SGT:
909 Known.makeNonNegative();
910 break;
911 case ICmpInst::ICMP_SLT:
912 Known.makeNegative();
913 break;
914 default:
915 break;
916 }
917 }
918 return;
919 }
920
921 unsigned BitWidth = Known.getBitWidth();
922 auto m_V =
923 m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToIntSameSize(DL: Q.DL, Op: m_Specific(V)));
924
925 Value *Y;
926 const APInt *Mask, *C;
927 if (!match(V: RHS, P: m_APInt(Res&: C)))
928 return;
929
930 uint64_t ShAmt;
931 switch (Pred) {
932 case ICmpInst::ICMP_EQ:
933 // assume(V = C)
934 if (match(V: LHS, P: m_V)) {
935 Known = Known.unionWith(RHS: KnownBits::makeConstant(C: *C));
936 // assume(V & Mask = C)
937 } else if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value(V&: Y)))) {
938 // For one bits in Mask, we can propagate bits from C to V.
939 Known.One |= *C;
940 if (match(V: Y, P: m_APInt(Res&: Mask)))
941 Known.Zero |= ~*C & *Mask;
942 // assume(V | Mask = C)
943 } else if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value(V&: Y)))) {
944 // For zero bits in Mask, we can propagate bits from C to V.
945 Known.Zero |= ~*C;
946 if (match(V: Y, P: m_APInt(Res&: Mask)))
947 Known.One |= *C & ~*Mask;
948 // assume(V << ShAmt = C)
949 } else if (match(V: LHS, P: m_Shl(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
950 ShAmt < BitWidth) {
951 // For those bits in C that are known, we can propagate them to known
952 // bits in V shifted to the right by ShAmt.
953 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
954 RHSKnown >>= ShAmt;
955 Known = Known.unionWith(RHS: RHSKnown);
956 // assume(V >> ShAmt = C)
957 } else if (match(V: LHS, P: m_Shr(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
958 ShAmt < BitWidth) {
959 // For those bits in RHS that are known, we can propagate them to known
960 // bits in V shifted to the right by C.
961 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
962 RHSKnown <<= ShAmt;
963 Known = Known.unionWith(RHS: RHSKnown);
964 }
965 break;
966 case ICmpInst::ICMP_NE: {
967 // assume (V & B != 0) where B is a power of 2
968 const APInt *BPow2;
969 if (C->isZero() && match(V: LHS, P: m_And(L: m_V, R: m_Power2(V&: BPow2))))
970 Known.One |= *BPow2;
971 break;
972 }
973 default: {
974 const APInt *Offset = nullptr;
975 if (match(V: LHS, P: m_CombineOr(Ps: m_V, Ps: m_AddLike(L: m_V, R: m_APInt(Res&: Offset))))) {
976 ConstantRange LHSRange = ConstantRange::makeAllowedICmpRegion(Pred, Other: *C);
977 if (Offset)
978 LHSRange = LHSRange.sub(Other: *Offset);
979 Known = Known.unionWith(RHS: LHSRange.toKnownBits());
980 }
981 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
982 // X & Y u> C -> X u> C && Y u> C
983 // X nuw- Y u> C -> X u> C
984 if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value())) ||
985 match(V: LHS, P: m_NUWSub(L: m_V, R: m_Value())))
986 Known.One.setHighBits(
987 (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
988 }
989 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
990 // X | Y u< C -> X u< C && Y u< C
991 // X nuw+ Y u< C -> X u< C && Y u< C
992 if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value())) ||
993 match(V: LHS, P: m_c_NUWAdd(L: m_V, R: m_Value()))) {
994 Known.Zero.setHighBits(
995 (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
996 }
997 }
998 } break;
999 }
1000}
1001
1002static void computeKnownBitsFromICmpCond(const Value *V, ICmpInst *Cmp,
1003 KnownBits &Known,
1004 const SimplifyQuery &SQ, bool Invert) {
1005 ICmpInst::Predicate Pred =
1006 Invert ? Cmp->getInversePredicate() : Cmp->getPredicate();
1007 Value *LHS = Cmp->getOperand(i_nocapture: 0);
1008 Value *RHS = Cmp->getOperand(i_nocapture: 1);
1009
1010 // Handle icmp pred (trunc V), C
1011 if (match(V: LHS, P: m_Trunc(Op: m_Specific(V)))) {
1012 KnownBits DstKnown(LHS->getType()->getScalarSizeInBits());
1013 computeKnownBitsFromCmp(V: LHS, Pred, LHS, RHS, Known&: DstKnown, Q: SQ);
1014 if (cast<TruncInst>(Val: LHS)->hasNoUnsignedWrap())
1015 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1016 else
1017 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1018 return;
1019 }
1020
1021 computeKnownBitsFromCmp(V, Pred, LHS, RHS, Known, Q: SQ);
1022}
1023
1024static void computeKnownBitsFromCond(const Value *V, Value *Cond,
1025 KnownBits &Known, const SimplifyQuery &SQ,
1026 bool Invert, unsigned Depth) {
1027 Value *A, *B;
1028 if (Depth < MaxAnalysisRecursionDepth &&
1029 match(V: Cond, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
1030 KnownBits Known2(Known.getBitWidth());
1031 KnownBits Known3(Known.getBitWidth());
1032 computeKnownBitsFromCond(V, Cond: A, Known&: Known2, SQ, Invert, Depth: Depth + 1);
1033 computeKnownBitsFromCond(V, Cond: B, Known&: Known3, SQ, Invert, Depth: Depth + 1);
1034 if (Invert ? match(V: Cond, P: m_LogicalOr(L: m_Value(), R: m_Value()))
1035 : match(V: Cond, P: m_LogicalAnd(L: m_Value(), R: m_Value())))
1036 Known2 = Known2.unionWith(RHS: Known3);
1037 else
1038 Known2 = Known2.intersectWith(RHS: Known3);
1039 Known = Known.unionWith(RHS: Known2);
1040 return;
1041 }
1042
1043 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
1044 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ, Invert);
1045 return;
1046 }
1047
1048 if (match(V: Cond, P: m_Trunc(Op: m_Specific(V)))) {
1049 KnownBits DstKnown(1);
1050 if (Invert) {
1051 DstKnown.setAllZero();
1052 } else {
1053 DstKnown.setAllOnes();
1054 }
1055 if (cast<TruncInst>(Val: Cond)->hasNoUnsignedWrap()) {
1056 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1057 return;
1058 }
1059 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1060 return;
1061 }
1062
1063 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A))))
1064 computeKnownBitsFromCond(V, Cond: A, Known, SQ, Invert: !Invert, Depth: Depth + 1);
1065}
1066
1067void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
1068 const SimplifyQuery &Q, unsigned Depth) {
1069 // Handle injected condition.
1070 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
1071 computeKnownBitsFromCond(V, Cond: Q.CC->Cond, Known, SQ: Q, Invert: Q.CC->Invert, Depth);
1072
1073 if (!Q.CxtI)
1074 return;
1075
1076 if (Q.DC && Q.DT) {
1077 // Handle dominating conditions.
1078 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
1079 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
1080 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
1081 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1082 /*Invert*/ false, Depth);
1083
1084 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
1085 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
1086 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1087 /*Invert*/ true, Depth);
1088 }
1089
1090 if (Known.hasConflict())
1091 Known.resetAll();
1092 }
1093
1094 if (!Q.AC)
1095 return;
1096
1097 unsigned BitWidth = Known.getBitWidth();
1098
1099 // Note that the patterns below need to be kept in sync with the code
1100 // in AssumptionCache::updateAffectedValues.
1101
1102 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
1103 if (!Elem.Assume)
1104 continue;
1105
1106 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
1107 assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() &&
1108 "Got assumption for the wrong function!");
1109
1110 if (Elem.Index != AssumptionCache::ExprResultIdx) {
1111 if (auto OBU = I->getOperandBundleAt(Index: Elem.Index);
1112 getBundleAttrFromOBU(OBU) == BundleAttr::Align) {
1113 auto [Ptr, _, _2, Alignment, Offset] = getAssumeAlignInfo(OBU);
1114 if (Ptr == V && Alignment && Offset && isPowerOf2_64(Value: *Alignment) &&
1115 isValidAssumeForContext(I, Q)) {
1116 Known.Zero |= (*Alignment - 1) & ~*Offset;
1117 Known.One |= (*Alignment - 1) & *Offset;
1118 }
1119 }
1120 continue;
1121 }
1122
1123 // Warning: This loop can end up being somewhat performance sensitive.
1124 // We're running this loop for once for each value queried resulting in a
1125 // runtime of ~O(#assumes * #values).
1126
1127 Value *Arg = I->getArgOperand(i: 0);
1128
1129 if (Arg == V && isValidAssumeForContext(I, Q)) {
1130 assert(BitWidth == 1 && "assume operand is not i1?");
1131 (void)BitWidth;
1132 Known.setAllOnes();
1133 return;
1134 }
1135 if (match(V: Arg, P: m_Not(V: m_Specific(V))) &&
1136 isValidAssumeForContext(I, Q)) {
1137 assert(BitWidth == 1 && "assume operand is not i1?");
1138 (void)BitWidth;
1139 Known.setAllZero();
1140 return;
1141 }
1142 auto *Trunc = dyn_cast<TruncInst>(Val: Arg);
1143 if (Trunc && Trunc->getOperand(i_nocapture: 0) == V &&
1144 isValidAssumeForContext(I, Q)) {
1145 if (Trunc->hasNoUnsignedWrap()) {
1146 Known = KnownBits::makeConstant(C: APInt(BitWidth, 1));
1147 return;
1148 }
1149 Known.One.setBit(0);
1150 return;
1151 }
1152
1153 // The remaining tests are all recursive, so bail out if we hit the limit.
1154 if (Depth == MaxAnalysisRecursionDepth)
1155 continue;
1156
1157 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
1158 if (!Cmp)
1159 continue;
1160
1161 if (!isValidAssumeForContext(I, Q))
1162 continue;
1163
1164 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ: Q, /*Invert=*/false);
1165 }
1166
1167 // Conflicting assumption: Undefined behavior will occur on this execution
1168 // path.
1169 if (Known.hasConflict())
1170 Known.resetAll();
1171}
1172
1173/// Compute known bits from a shift operator, including those with a
1174/// non-constant shift amount. Known is the output of this function. Known2 is a
1175/// pre-allocated temporary with the same bit width as Known and on return
1176/// contains the known bit of the shift value source. KF is an
1177/// operator-specific function that, given the known-bits and a shift amount,
1178/// compute the implied known-bits of the shift operator's result respectively
1179/// for that shift amount. The results from calling KF are conservatively
1180/// combined for all permitted shift amounts.
1181static void computeKnownBitsFromShiftOperator(
1182 const Operator *I, const APInt &DemandedElts, KnownBits &Known,
1183 KnownBits &Known2, const SimplifyQuery &Q, unsigned Depth,
1184 function_ref<KnownBits(const KnownBits &, const KnownBits &, bool)> KF) {
1185 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1186 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1187 // To limit compile-time impact, only query isKnownNonZero() if we know at
1188 // least something about the shift amount.
1189 bool ShAmtNonZero =
1190 Known.isNonZero() ||
1191 (Known.getMaxValue().ult(RHS: Known.getBitWidth()) &&
1192 isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1));
1193 Known = KF(Known2, Known, ShAmtNonZero);
1194}
1195
1196static KnownBits
1197getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
1198 const KnownBits &KnownLHS, const KnownBits &KnownRHS,
1199 const SimplifyQuery &Q, unsigned Depth) {
1200 unsigned BitWidth = KnownLHS.getBitWidth();
1201 KnownBits KnownOut(BitWidth);
1202 bool IsAnd = false;
1203 bool HasKnownOne = !KnownLHS.One.isZero() || !KnownRHS.One.isZero();
1204 Value *X = nullptr, *Y = nullptr;
1205
1206 switch (I->getOpcode()) {
1207 case Instruction::And:
1208 KnownOut = KnownLHS & KnownRHS;
1209 IsAnd = true;
1210 // and(x, -x) is common idioms that will clear all but lowest set
1211 // bit. If we have a single known bit in x, we can clear all bits
1212 // above it.
1213 // TODO: instcombine often reassociates independent `and` which can hide
1214 // this pattern. Try to match and(x, and(-x, y)) / and(and(x, y), -x).
1215 if (HasKnownOne && match(V: I, P: m_c_And(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X))))) {
1216 // -(-x) == x so using whichever (LHS/RHS) gets us a better result.
1217 if (KnownLHS.countMaxTrailingZeros() <= KnownRHS.countMaxTrailingZeros())
1218 KnownOut = KnownLHS.blsi();
1219 else
1220 KnownOut = KnownRHS.blsi();
1221 }
1222 break;
1223 case Instruction::Or:
1224 KnownOut = KnownLHS | KnownRHS;
1225 break;
1226 case Instruction::Xor:
1227 KnownOut = KnownLHS ^ KnownRHS;
1228 // xor(x, x-1) is common idioms that will clear all but lowest set
1229 // bit. If we have a single known bit in x, we can clear all bits
1230 // above it.
1231 // TODO: xor(x, x-1) is often rewritting as xor(x, x-C) where C !=
1232 // -1 but for the purpose of demanded bits (xor(x, x-C) &
1233 // Demanded) == (xor(x, x-1) & Demanded). Extend the xor pattern
1234 // to use arbitrary C if xor(x, x-C) as the same as xor(x, x-1).
1235 if (HasKnownOne &&
1236 match(V: I, P: m_c_Xor(L: m_Value(V&: X), R: m_Add(L: m_Deferred(V: X), R: m_AllOnes())))) {
1237 const KnownBits &XBits = I->getOperand(i: 0) == X ? KnownLHS : KnownRHS;
1238 KnownOut = XBits.blsmsk();
1239 }
1240 break;
1241 default:
1242 llvm_unreachable("Invalid Op used in 'analyzeKnownBitsFromAndXorOr'");
1243 }
1244
1245 // and(x, add (x, -1)) is a common idiom that always clears the low bit;
1246 // xor/or(x, add (x, -1)) is an idiom that will always set the low bit.
1247 // here we handle the more general case of adding any odd number by
1248 // matching the form and/xor/or(x, add(x, y)) where y is odd.
1249 // TODO: This could be generalized to clearing any bit set in y where the
1250 // following bit is known to be unset in y.
1251 if (!KnownOut.Zero[0] && !KnownOut.One[0] &&
1252 (match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_c_Add(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1253 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1254 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Value(V&: Y), R: m_Deferred(V: X)))))) {
1255 KnownBits KnownY(BitWidth);
1256 computeKnownBits(V: Y, DemandedElts, Known&: KnownY, Q, Depth: Depth + 1);
1257 if (KnownY.countMinTrailingOnes() > 0) {
1258 if (IsAnd)
1259 KnownOut.Zero.setBit(0);
1260 else
1261 KnownOut.One.setBit(0);
1262 }
1263 }
1264 return KnownOut;
1265}
1266
1267static KnownBits computeKnownBitsForHorizontalOperation(
1268 const Operator *I, const APInt &DemandedElts, const SimplifyQuery &Q,
1269 unsigned Depth,
1270 const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
1271 KnownBitsFunc) {
1272 APInt DemandedEltsLHS, DemandedEltsRHS;
1273 getHorizDemandedEltsForFirstOperand(VectorBitWidth: Q.DL.getTypeSizeInBits(Ty: I->getType()),
1274 DemandedElts, DemandedLHS&: DemandedEltsLHS,
1275 DemandedRHS&: DemandedEltsRHS);
1276
1277 const auto ComputeForSingleOpFunc =
1278 [Depth, &Q, KnownBitsFunc](const Value *Op, APInt &DemandedEltsOp) {
1279 return KnownBitsFunc(
1280 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp, Q, Depth: Depth + 1),
1281 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp << 1, Q, Depth: Depth + 1));
1282 };
1283
1284 if (DemandedEltsRHS.isZero())
1285 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS);
1286 if (DemandedEltsLHS.isZero())
1287 return ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS);
1288
1289 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS)
1290 .intersectWith(RHS: ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS));
1291}
1292
1293// Public so this can be used in `SimplifyDemandedUseBits`.
1294KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
1295 const KnownBits &KnownLHS,
1296 const KnownBits &KnownRHS,
1297 const SimplifyQuery &SQ,
1298 unsigned Depth) {
1299 auto *FVTy = dyn_cast<FixedVectorType>(Val: I->getType());
1300 APInt DemandedElts =
1301 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
1302
1303 return getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS, KnownRHS, Q: SQ,
1304 Depth);
1305}
1306
1307ConstantRange llvm::getVScaleRange(const Function *F, unsigned BitWidth) {
1308 Attribute Attr = F->getFnAttribute(Kind: Attribute::VScaleRange);
1309 // Without vscale_range, we only know that vscale is non-zero.
1310 if (!Attr.isValid())
1311 return ConstantRange(APInt(BitWidth, 1), APInt::getZero(numBits: BitWidth));
1312
1313 unsigned AttrMin = Attr.getVScaleRangeMin();
1314 // Minimum is larger than vscale width, result is always poison.
1315 if ((unsigned)llvm::bit_width(Value: AttrMin) > BitWidth)
1316 return ConstantRange::getEmpty(BitWidth);
1317
1318 APInt Min(BitWidth, AttrMin);
1319 std::optional<unsigned> AttrMax = Attr.getVScaleRangeMax();
1320 if (!AttrMax || (unsigned)llvm::bit_width(Value: *AttrMax) > BitWidth)
1321 return ConstantRange(Min, APInt::getZero(numBits: BitWidth));
1322
1323 return ConstantRange(Min, APInt(BitWidth, *AttrMax) + 1);
1324}
1325
1326/// Return true if \p II reads a register named "vlenb". On RISC-V this is the
1327/// VLENB CSR, which holds VLEN/8: a non-zero power of two bounded by the
1328/// target's VLEN range. Callers must ensure the target is RISC-V.
1329static bool isReadVLENB(const IntrinsicInst &II) {
1330 auto *MAV = dyn_cast<MetadataAsValue>(Val: II.getArgOperand(i: 0));
1331 if (!MAV)
1332 return false;
1333 auto *MD = dyn_cast<MDNode>(Val: MAV->getMetadata());
1334 if (!MD || MD->getNumOperands() != 1)
1335 return false;
1336 auto *RegName = dyn_cast<MDString>(Val: MD->getOperand(I: 0));
1337 return RegName && RegName->getString() == "vlenb";
1338}
1339
1340/// Return the value range of a RISC-V vlenb CSR read. RVV requires VLEN to be a
1341/// power of two in [32, 65536] (Zvl32b is the smallest vector extension), so
1342/// VLENB = VLEN/8 is in [4, 8192]. This architectural bound is independent of
1343/// any function attribute and stays sound for Zvl32b, whose VLEN (32) is not
1344/// representable as an integer vscale (VLEN / RVVBitsPerBlock). A vscale_range
1345/// attribute, when present, pins the subtarget's VLEN in units of
1346/// RVVBitsPerBlock (64 bits) and so gives a tighter VLENB = vscale *
1347/// RVVBytesPerBlock.
1348static ConstantRange getRISCVVLENBRange(const IntrinsicInst &II,
1349 unsigned Width) {
1350 // Architectural bounds: VLEN in [32, 65536] => VLENB in [4, 8192].
1351 ConstantRange Range(APInt(Width, 32 / 8), APInt(Width, 65536 / 8) + 1);
1352
1353 const Function *F = II.getFunction();
1354 if (F->getFnAttribute(Kind: Attribute::VScaleRange).isValid()) {
1355 ConstantRange VScale = getVScaleRange(F, BitWidth: Width);
1356 Range = Range.intersectWith(
1357 CR: VScale.multiply(Other: ConstantRange(APInt(Width, RISCV::RVVBytesPerBlock))));
1358 }
1359 return Range;
1360}
1361
1362void llvm::adjustKnownBitsForSelectArm(KnownBits &Known, Value *Cond,
1363 Value *Arm, bool Invert,
1364 const SimplifyQuery &Q, unsigned Depth) {
1365 // If we have a constant arm, we are done.
1366 if (Known.isConstant())
1367 return;
1368
1369 // See what condition implies about the bits of the select arm.
1370 KnownBits CondRes(Known.getBitWidth());
1371 computeKnownBitsFromCond(V: Arm, Cond, Known&: CondRes, SQ: Q, Invert, Depth: Depth + 1);
1372 // If we don't get any information from the condition, no reason to
1373 // proceed.
1374 if (CondRes.isUnknown())
1375 return;
1376
1377 // We can have conflict if the condition is dead. I.e if we have
1378 // (x | 64) < 32 ? (x | 64) : y
1379 // we will have conflict at bit 6 from the condition/the `or`.
1380 // In that case just return. Its not particularly important
1381 // what we do, as this select is going to be simplified soon.
1382 CondRes = CondRes.unionWith(RHS: Known);
1383 if (CondRes.hasConflict())
1384 return;
1385
1386 // Finally make sure the information we found is valid. This is relatively
1387 // expensive so it's left for the very end.
1388 if (!isGuaranteedNotToBeUndef(V: Arm, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1))
1389 return;
1390
1391 // Finally, we know we get information from the condition and its valid,
1392 // so return it.
1393 Known = std::move(CondRes);
1394}
1395
1396// Match a signed min+max clamp pattern like smax(smin(In, CHigh), CLow).
1397// Returns the input and lower/upper bounds.
1398static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
1399 const APInt *&CLow, const APInt *&CHigh) {
1400 assert(isa<Operator>(Select) &&
1401 cast<Operator>(Select)->getOpcode() == Instruction::Select &&
1402 "Input should be a Select!");
1403
1404 const Value *LHS = nullptr, *RHS = nullptr;
1405 SelectPatternFlavor SPF = matchSelectPattern(V: Select, LHS, RHS).Flavor;
1406 if (SPF != SPF_SMAX && SPF != SPF_SMIN)
1407 return false;
1408
1409 if (!match(V: RHS, P: m_APInt(Res&: CLow)))
1410 return false;
1411
1412 const Value *LHS2 = nullptr, *RHS2 = nullptr;
1413 SelectPatternFlavor SPF2 = matchSelectPattern(V: LHS, LHS&: LHS2, RHS&: RHS2).Flavor;
1414 if (getInverseMinMaxFlavor(SPF) != SPF2)
1415 return false;
1416
1417 if (!match(V: RHS2, P: m_APInt(Res&: CHigh)))
1418 return false;
1419
1420 if (SPF == SPF_SMIN)
1421 std::swap(a&: CLow, b&: CHigh);
1422
1423 In = LHS2;
1424 return CLow->sle(RHS: *CHigh);
1425}
1426
1427static bool isSignedMinMaxIntrinsicClamp(const IntrinsicInst *II,
1428 const APInt *&CLow,
1429 const APInt *&CHigh) {
1430 assert((II->getIntrinsicID() == Intrinsic::smin ||
1431 II->getIntrinsicID() == Intrinsic::smax) &&
1432 "Must be smin/smax");
1433
1434 Intrinsic::ID InverseID = getInverseMinMaxIntrinsic(MinMaxID: II->getIntrinsicID());
1435 auto *InnerII = dyn_cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
1436 if (!InnerII || InnerII->getIntrinsicID() != InverseID ||
1437 !match(V: II->getArgOperand(i: 1), P: m_APInt(Res&: CLow)) ||
1438 !match(V: InnerII->getArgOperand(i: 1), P: m_APInt(Res&: CHigh)))
1439 return false;
1440
1441 if (II->getIntrinsicID() == Intrinsic::smin)
1442 std::swap(a&: CLow, b&: CHigh);
1443 return CLow->sle(RHS: *CHigh);
1444}
1445
1446static void unionWithMinMaxIntrinsicClamp(const IntrinsicInst *II,
1447 KnownBits &Known) {
1448 const APInt *CLow, *CHigh;
1449 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
1450 Known = Known.unionWith(
1451 RHS: ConstantRange::getNonEmpty(Lower: *CLow, Upper: *CHigh + 1).toKnownBits());
1452}
1453
1454static void computeKnownBitsForRecurrenceOperands(
1455 const PHINode *P, Value *Start, Value *Step, const APInt &DemandedElts,
1456 KnownBits &KnownStart, KnownBits &KnownStep, const SimplifyQuery &Q,
1457 unsigned Depth) {
1458 // Change the context instruction to the "edge" that flows into the phi. This
1459 // is important because that is where the value is actually "evaluated" even
1460 // though it is used later somewhere else. (see also D69571).
1461 SimplifyQuery RecQ = Q.getWithoutCondContext();
1462 unsigned OpNum = P->getOperand(i_nocapture: 0) == Start ? 0 : 1;
1463
1464 RecQ.CxtI = P->getIncomingBlock(i: OpNum)->getTerminator();
1465 computeKnownBits(V: Start, DemandedElts, Known&: KnownStart, Q: RecQ, Depth: Depth + 1);
1466
1467 RecQ.CxtI = P->getIncomingBlock(i: 1 - OpNum)->getTerminator();
1468 computeKnownBits(V: Step, DemandedElts, Known&: KnownStep, Q: RecQ, Depth: Depth + 1);
1469}
1470
1471static void computeKnownBitsFromOperator(const Operator *I,
1472 const APInt &DemandedElts,
1473 KnownBits &Known,
1474 const SimplifyQuery &Q,
1475 unsigned Depth) {
1476 unsigned BitWidth = Known.getBitWidth();
1477
1478 KnownBits Known2(BitWidth);
1479 switch (I->getOpcode()) {
1480 default: break;
1481 case Instruction::Load:
1482 if (MDNode *MD =
1483 Q.IIQ.getMetadata(I: cast<LoadInst>(Val: I), KindID: LLVMContext::MD_range))
1484 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
1485 break;
1486 case Instruction::And:
1487 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1488 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1489
1490 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1491 break;
1492 case Instruction::Or:
1493 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1494 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1495
1496 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1497 break;
1498 case Instruction::Xor:
1499 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1500 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1501
1502 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1503 break;
1504 case Instruction::Mul: {
1505 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1506 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1507 computeKnownBitsMul(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1508 DemandedElts, Known, Known2, Q, Depth);
1509 break;
1510 }
1511 case Instruction::UDiv: {
1512 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1513 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1514 Known =
1515 KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1516 break;
1517 }
1518 case Instruction::SDiv: {
1519 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1520 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1521 Known =
1522 KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1523 break;
1524 }
1525 case Instruction::Select: {
1526 auto ComputeForArm = [&](Value *Arm, bool Invert) {
1527 KnownBits Res(Known.getBitWidth());
1528 computeKnownBits(V: Arm, DemandedElts, Known&: Res, Q, Depth: Depth + 1);
1529 adjustKnownBitsForSelectArm(Known&: Res, Cond: I->getOperand(i: 0), Arm, Invert, Q, Depth);
1530 return Res;
1531 };
1532 // Only known if known in both the LHS and RHS.
1533 Known =
1534 ComputeForArm(I->getOperand(i: 1), /*Invert=*/false)
1535 .intersectWith(RHS: ComputeForArm(I->getOperand(i: 2), /*Invert=*/true));
1536 break;
1537 }
1538 case Instruction::FPToSI: {
1539 // fptosi is poison if the rounded value doesn't fit in the result type,
1540 // so we can assume the conversion is well-defined and rounds towards
1541 // zero. +-Inf can never fit in an integer type, so it is always poison,
1542 // like NaN. Negative subnormals and negative zero round to 0. That
1543 // leaves negative normals as the only class that can produce a defined
1544 // negative result.
1545 KnownFPClass SrcFPClass = computeKnownFPClass(
1546 V: I->getOperand(i: 0), DemandedElts, InterestedClasses: fcNegNormal, SQ: Q, Depth: Depth + 1);
1547 if (SrcFPClass.isKnownNever(Mask: fcNegNormal))
1548 Known.makeNonNegative();
1549 break;
1550 }
1551 case Instruction::FPTrunc:
1552 case Instruction::FPExt:
1553 case Instruction::FPToUI:
1554 case Instruction::SIToFP:
1555 case Instruction::UIToFP:
1556 break; // Can't work with floating point.
1557 case Instruction::PtrToInt:
1558 case Instruction::PtrToAddr:
1559 case Instruction::IntToPtr:
1560 // Fall through and handle them the same as zext/trunc.
1561 [[fallthrough]];
1562 case Instruction::ZExt:
1563 case Instruction::Trunc: {
1564 Type *SrcTy = I->getOperand(i: 0)->getType();
1565
1566 unsigned SrcBitWidth;
1567 // Note that we handle pointer operands here because of inttoptr/ptrtoint
1568 // which fall through here.
1569 Type *ScalarTy = SrcTy->getScalarType();
1570 SrcBitWidth = ScalarTy->isPointerTy() ?
1571 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
1572 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
1573
1574 assert(SrcBitWidth && "SrcBitWidth can't be zero");
1575 Known = Known.anyextOrTrunc(BitWidth: SrcBitWidth);
1576 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1577 if (auto *Inst = dyn_cast<PossiblyNonNegInst>(Val: I);
1578 Inst && Inst->hasNonNeg() && !Known.isNegative())
1579 Known.makeNonNegative();
1580 Known = Known.zextOrTrunc(BitWidth);
1581 break;
1582 }
1583 case Instruction::BitCast: {
1584 Type *SrcTy = I->getOperand(i: 0)->getType();
1585 if (SrcTy->isIntOrPtrTy() &&
1586 // TODO: For now, not handling conversions like:
1587 // (bitcast i64 %x to <2 x i32>)
1588 !I->getType()->isVectorTy()) {
1589 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1590 break;
1591 }
1592
1593 const Value *V;
1594 // Handle bitcast from floating point to integer.
1595 if (match(V: I, P: m_ElementWiseBitCast(Op: m_Value(V))) &&
1596 V->getType()->isFPOrFPVectorTy()) {
1597 Type *FPType = V->getType()->getScalarType();
1598 KnownFPClass Result =
1599 computeKnownFPClass(V, DemandedElts, InterestedClasses: fcAllFlags, SQ: Q, Depth: Depth + 1);
1600
1601 Known = Result.toKnownBits(FltSemantics: FPType->getFltSemantics());
1602
1603 break;
1604 }
1605
1606 // Handle cast from vector integer type to scalar or vector integer.
1607 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy);
1608 if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
1609 !I->getType()->isIntOrIntVectorTy() ||
1610 isa<ScalableVectorType>(Val: I->getType()))
1611 break;
1612
1613 unsigned NumElts = DemandedElts.getBitWidth();
1614 bool IsLE = Q.DL.isLittleEndian();
1615 // Look through a cast from narrow vector elements to wider type.
1616 // Examples: v4i32 -> v2i64, v3i8 -> v24
1617 unsigned SubBitWidth = SrcVecTy->getScalarSizeInBits();
1618 if (BitWidth % SubBitWidth == 0) {
1619 // Known bits are automatically intersected across demanded elements of a
1620 // vector. So for example, if a bit is computed as known zero, it must be
1621 // zero across all demanded elements of the vector.
1622 //
1623 // For this bitcast, each demanded element of the output is sub-divided
1624 // across a set of smaller vector elements in the source vector. To get
1625 // the known bits for an entire element of the output, compute the known
1626 // bits for each sub-element sequentially. This is done by shifting the
1627 // one-set-bit demanded elements parameter across the sub-elements for
1628 // consecutive calls to computeKnownBits. We are using the demanded
1629 // elements parameter as a mask operator.
1630 //
1631 // The known bits of each sub-element are then inserted into place
1632 // (dependent on endian) to form the full result of known bits.
1633 unsigned SubScale = BitWidth / SubBitWidth;
1634 APInt SubDemandedElts = APInt::getZero(numBits: NumElts * SubScale);
1635 for (unsigned i = 0; i != NumElts; ++i) {
1636 if (DemandedElts[i])
1637 SubDemandedElts.setBit(i * SubScale);
1638 }
1639
1640 KnownBits KnownSrc(SubBitWidth);
1641 for (unsigned i = 0; i != SubScale; ++i) {
1642 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts.shl(shiftAmt: i), Known&: KnownSrc, Q,
1643 Depth: Depth + 1);
1644 unsigned ShiftElt = IsLE ? i : SubScale - 1 - i;
1645 Known.insertBits(SubBits: KnownSrc, BitPosition: ShiftElt * SubBitWidth);
1646 }
1647 }
1648 // Look through a cast from wider vector elements to narrow type.
1649 // Examples: v2i64 -> v4i32
1650 if (SubBitWidth % BitWidth == 0) {
1651 unsigned SubScale = SubBitWidth / BitWidth;
1652 KnownBits KnownSrc(SubBitWidth);
1653 APInt SubDemandedElts =
1654 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
1655 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts, Known&: KnownSrc, Q,
1656 Depth: Depth + 1);
1657
1658 Known.setAllConflict();
1659 for (unsigned i = 0; i != NumElts; ++i) {
1660 if (DemandedElts[i]) {
1661 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
1662 unsigned Offset = (Shifts % SubScale) * BitWidth;
1663 Known = Known.intersectWith(RHS: KnownSrc.extractBits(NumBits: BitWidth, BitPosition: Offset));
1664 if (Known.isUnknown())
1665 break;
1666 }
1667 }
1668 }
1669 break;
1670 }
1671 case Instruction::SExt: {
1672 // Compute the bits in the result that are not present in the input.
1673 unsigned SrcBitWidth = I->getOperand(i: 0)->getType()->getScalarSizeInBits();
1674
1675 Known = Known.trunc(BitWidth: SrcBitWidth);
1676 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1677 // If the sign bit of the input is known set or clear, then we know the
1678 // top bits of the result.
1679 Known = Known.sext(BitWidth);
1680 break;
1681 }
1682 case Instruction::Shl: {
1683 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1684 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1685 auto KF = [NUW, NSW](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1686 bool ShAmtNonZero) {
1687 return KnownBits::shl(LHS: KnownVal, RHS: KnownAmt, NUW, NSW, ShAmtNonZero);
1688 };
1689 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1690 KF);
1691 // Trailing zeros of a right-shifted constant never decrease.
1692 const APInt *C;
1693 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1694 Known.Zero.setLowBits(C->countr_zero());
1695
1696 // shl X, sub(Y, xor(ctlz(X, true), BitWidth-1)) shifts X so that its MSB
1697 // lands at bit Y, when BitWidth is a power of 2.
1698 const APInt *YC;
1699 Value *X = I->getOperand(i: 0);
1700 if (isPowerOf2_32(Value: BitWidth) &&
1701 match(V: I->getOperand(i: 1),
1702 P: m_Sub(L: m_APInt(Res&: YC), R: m_Xor(L: m_Ctlz(Op0: m_Specific(V: X), Op1: m_One()),
1703 R: m_SpecificInt(V: BitWidth - 1)))) &&
1704 YC->ult(RHS: BitWidth - 1)) {
1705 unsigned Y = YC->getZExtValue();
1706 Known.One.setBit(Y);
1707 Known.Zero.setBitsFrom(Y + 1);
1708 }
1709 break;
1710 }
1711 case Instruction::LShr: {
1712 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1713 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1714 bool ShAmtNonZero) {
1715 return KnownBits::lshr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1716 };
1717 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1718 KF);
1719 // Leading zeros of a left-shifted constant never decrease.
1720 const APInt *C;
1721 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1722 Known.Zero.setHighBits(C->countl_zero());
1723 break;
1724 }
1725 case Instruction::AShr: {
1726 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1727 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1728 bool ShAmtNonZero) {
1729 return KnownBits::ashr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1730 };
1731 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1732 KF);
1733 break;
1734 }
1735 case Instruction::Sub: {
1736 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1737 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1738 computeKnownBitsAddSub(Add: false, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1739 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1740 break;
1741 }
1742 case Instruction::Add: {
1743 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1744 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1745 computeKnownBitsAddSub(Add: true, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1746 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1747 break;
1748 }
1749 case Instruction::SRem:
1750 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1751 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1752 Known = KnownBits::srem(LHS: Known, RHS: Known2);
1753 break;
1754
1755 case Instruction::URem:
1756 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1757 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1758 Known = KnownBits::urem(LHS: Known, RHS: Known2);
1759 break;
1760 case Instruction::Alloca:
1761 Known.Zero.setLowBits(Log2(A: cast<AllocaInst>(Val: I)->getAlign()));
1762 break;
1763 case Instruction::GetElementPtr: {
1764 // Analyze all of the subscripts of this getelementptr instruction
1765 // to determine if we can prove known low zero bits.
1766 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1767 // Accumulate the constant indices in a separate variable
1768 // to minimize the number of calls to computeForAddSub.
1769 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: I->getType());
1770 APInt AccConstIndices(IndexWidth, 0);
1771
1772 auto AddIndexToKnown = [&](KnownBits IndexBits) {
1773 if (IndexWidth == BitWidth) {
1774 // Note that inbounds does *not* guarantee nsw for the addition, as only
1775 // the offset is signed, while the base address is unsigned.
1776 Known = KnownBits::add(LHS: Known, RHS: IndexBits);
1777 } else {
1778 // If the index width is smaller than the pointer width, only add the
1779 // value to the low bits.
1780 assert(IndexWidth < BitWidth &&
1781 "Index width can't be larger than pointer width");
1782 Known.insertBits(SubBits: KnownBits::add(LHS: Known.trunc(BitWidth: IndexWidth), RHS: IndexBits), BitPosition: 0);
1783 }
1784 };
1785
1786 gep_type_iterator GTI = gep_type_begin(GEP: I);
1787 for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
1788 // TrailZ can only become smaller, short-circuit if we hit zero.
1789 if (Known.isUnknown())
1790 break;
1791
1792 Value *Index = I->getOperand(i);
1793
1794 // Handle case when index is zero.
1795 Constant *CIndex = dyn_cast<Constant>(Val: Index);
1796 if (CIndex && CIndex->isNullValue())
1797 continue;
1798
1799 if (StructType *STy = GTI.getStructTypeOrNull()) {
1800 // Handle struct member offset arithmetic.
1801
1802 assert(CIndex &&
1803 "Access to structure field must be known at compile time");
1804
1805 if (CIndex->getType()->isVectorTy())
1806 Index = CIndex->getSplatValue();
1807
1808 unsigned Idx = cast<ConstantInt>(Val: Index)->getZExtValue();
1809 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
1810 uint64_t Offset = SL->getElementOffset(Idx);
1811 AccConstIndices += Offset;
1812 continue;
1813 }
1814
1815 // Handle array index arithmetic.
1816 Type *IndexedTy = GTI.getIndexedType();
1817 if (!IndexedTy->isSized()) {
1818 Known.resetAll();
1819 break;
1820 }
1821
1822 TypeSize Stride = GTI.getSequentialElementStride(DL: Q.DL);
1823 uint64_t StrideInBytes = Stride.getKnownMinValue();
1824 if (!Stride.isScalable()) {
1825 // Fast path for constant offset.
1826 if (auto *CI = dyn_cast<ConstantInt>(Val: Index)) {
1827 AccConstIndices +=
1828 CI->getValue().sextOrTrunc(width: IndexWidth) * StrideInBytes;
1829 continue;
1830 }
1831 }
1832
1833 KnownBits IndexBits =
1834 computeKnownBits(V: Index, Q, Depth: Depth + 1).sextOrTrunc(BitWidth: IndexWidth);
1835 KnownBits ScalingFactor(IndexWidth);
1836 // Multiply by current sizeof type.
1837 // &A[i] == A + i * sizeof(*A[i]).
1838 if (Stride.isScalable()) {
1839 // For scalable types the only thing we know about sizeof is
1840 // that this is a multiple of the minimum size.
1841 ScalingFactor.Zero.setLowBits(llvm::countr_zero(Val: StrideInBytes));
1842 } else {
1843 ScalingFactor =
1844 KnownBits::makeConstant(C: APInt(IndexWidth, StrideInBytes));
1845 }
1846 AddIndexToKnown(KnownBits::mul(LHS: IndexBits, RHS: ScalingFactor));
1847 }
1848 if (!Known.isUnknown() && !AccConstIndices.isZero())
1849 AddIndexToKnown(KnownBits::makeConstant(C: AccConstIndices));
1850 break;
1851 }
1852 case Instruction::PHI: {
1853 const PHINode *P = cast<PHINode>(Val: I);
1854 BinaryOperator *BO = nullptr;
1855 Value *Start = nullptr, *Step = nullptr;
1856 KnownBits &KnownStart = Known2;
1857 if (matchSimpleRecurrence(P, BO, Start, Step)) {
1858 // Handle the case of a simple two-predecessor recurrence PHI.
1859 // There's a lot more that could theoretically be done here, but
1860 // this is sufficient to catch some interesting cases.
1861 unsigned Opcode = BO->getOpcode();
1862
1863 switch (Opcode) {
1864 // If this is a shift recurrence, we know the bits being shifted in. We
1865 // can combine that with information about the start value of the
1866 // recurrence to conclude facts about the result. If this is a udiv
1867 // recurrence, we know that the result can never exceed either the
1868 // numerator or the start value, whichever is greater.
1869 case Instruction::LShr:
1870 case Instruction::AShr:
1871 case Instruction::Shl:
1872 case Instruction::UDiv:
1873 if (BO->getOperand(i_nocapture: 0) != I)
1874 break;
1875 [[fallthrough]];
1876
1877 // For a urem recurrence, the result can never exceed the start value. The
1878 // phi could either be the numerator or the denominator.
1879 case Instruction::URem: {
1880 // We have matched a recurrence of the form:
1881 // %iv = [R, %entry], [%iv.next, %backedge]
1882 // %iv.next = shift_op %iv, L
1883
1884 // Recurse with the phi context to avoid concern about whether facts
1885 // inferred hold at original context instruction. TODO: It may be
1886 // correct to use the original context. IF warranted, explore and
1887 // add sufficient tests to cover.
1888 SimplifyQuery RecQ = Q.getWithoutCondContext();
1889 RecQ.CxtI = P;
1890 computeKnownBits(V: Start, DemandedElts, Known&: KnownStart, Q: RecQ, Depth: Depth + 1);
1891 switch (Opcode) {
1892 case Instruction::Shl:
1893 // A shl recurrence will only increase the tailing zeros
1894 Known.Zero.setLowBits(KnownStart.countMinTrailingZeros());
1895 break;
1896 case Instruction::LShr:
1897 case Instruction::UDiv:
1898 case Instruction::URem:
1899 // lshr, udiv, and urem recurrences will preserve the leading zeros of
1900 // the start value.
1901 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
1902 break;
1903 case Instruction::AShr:
1904 // An ashr recurrence will extend the initial sign bit
1905 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
1906 Known.One.setHighBits(KnownStart.countMinLeadingOnes());
1907 break;
1908 }
1909 break;
1910 }
1911
1912 // Check for operations that have the property that if
1913 // both their operands have low zero bits, the result
1914 // will have low zero bits.
1915 case Instruction::Add:
1916 case Instruction::Sub:
1917 case Instruction::And:
1918 case Instruction::Or:
1919 case Instruction::Mul: {
1920 // Ok, we have a recurrence of the form {Start,op,Step}. Check for low
1921 // zero bits.
1922 KnownBits KnownStep(BitWidth);
1923 computeKnownBitsForRecurrenceOperands(P, Start, Step, DemandedElts,
1924 KnownStart, KnownStep, Q, Depth);
1925
1926 Known.Zero.setLowBits(std::min(a: KnownStart.countMinTrailingZeros(),
1927 b: KnownStep.countMinTrailingZeros()));
1928
1929 auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(Val: BO);
1930 if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(Op: OverflowOp))
1931 break;
1932
1933 switch (Opcode) {
1934 // If initial value of recurrence is nonnegative, and we are adding
1935 // a nonnegative number with nsw, the result can only be nonnegative
1936 // or poison value regardless of the number of times we execute the
1937 // add in phi recurrence. If initial value is negative and we are
1938 // adding a negative number with nsw, the result can only be
1939 // negative or poison value. Similar arguments apply to sub and mul.
1940 //
1941 // (add non-negative, non-negative) --> non-negative
1942 // (add negative, negative) --> negative
1943 case Instruction::Add: {
1944 if (KnownStart.isNonNegative() && KnownStep.isNonNegative())
1945 Known.makeNonNegative();
1946 else if (KnownStart.isNegative() && KnownStep.isNegative())
1947 Known.makeNegative();
1948 break;
1949 }
1950
1951 // (sub nsw non-negative, negative) --> non-negative
1952 // (sub nsw negative, non-negative) --> negative
1953 case Instruction::Sub: {
1954 if (BO->getOperand(i_nocapture: 0) != I)
1955 break;
1956 if (KnownStart.isNonNegative() && KnownStep.isNegative())
1957 Known.makeNonNegative();
1958 else if (KnownStart.isNegative() && KnownStep.isNonNegative())
1959 Known.makeNegative();
1960 break;
1961 }
1962
1963 // (mul nsw non-negative, non-negative) --> non-negative
1964 case Instruction::Mul:
1965 if (KnownStart.isNonNegative() && KnownStep.isNonNegative())
1966 Known.makeNonNegative();
1967 break;
1968
1969 default:
1970 break;
1971 }
1972 break;
1973 }
1974
1975 default:
1976 break;
1977 }
1978 } else {
1979 IntrinsicInst *II = nullptr;
1980 if (matchTwoInputRecurrence<IntrinsicInst>(PN: P, Inst&: II, Init&: Start, OtherOp&: Step)) {
1981 // %iv = [<Start>, %entry], [%iv.next, %backedge]
1982 //
1983 // %iv.next = <II>(%iv, <Step>)
1984 // or
1985 // %iv.next = <II>(<Step>, %iv)
1986 Intrinsic::ID IntrinsicID = II->getIntrinsicID();
1987 if (IntrinsicID == Intrinsic::umin || IntrinsicID == Intrinsic::umax) {
1988 KnownBits KnownStep(BitWidth);
1989 computeKnownBitsForRecurrenceOperands(
1990 P, Start, Step, DemandedElts, KnownStart, KnownStep, Q, Depth);
1991
1992 if (IntrinsicID == Intrinsic::umin) {
1993 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
1994 Known.One.setHighBits(std::min(a: KnownStart.countMinLeadingOnes(),
1995 b: KnownStep.countMinLeadingOnes()));
1996 } else {
1997 // umax
1998 Known.Zero.setHighBits(std::min(a: KnownStart.countMinLeadingZeros(),
1999 b: KnownStep.countMinLeadingZeros()));
2000 Known.One.setHighBits(KnownStart.countMinLeadingOnes());
2001 }
2002 }
2003 }
2004 }
2005
2006 // Unreachable blocks may have zero-operand PHI nodes.
2007 if (P->getNumIncomingValues() == 0)
2008 break;
2009
2010 // Otherwise take the unions of the known bit sets of the operands,
2011 // taking conservative care to avoid excessive recursion.
2012 if (Depth < MaxAnalysisRecursionDepth - 1 && Known.isUnknown()) {
2013 // Skip if every incoming value references to ourself.
2014 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
2015 break;
2016
2017 Known.setAllConflict();
2018 for (const Use &U : P->operands()) {
2019 Value *IncValue;
2020 const PHINode *CxtPhi;
2021 Instruction *CxtI;
2022 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CxtI, PhiOut: &CxtPhi);
2023 // Skip direct self references.
2024 if (IncValue == P)
2025 continue;
2026
2027 // Change the context instruction to the "edge" that flows into the
2028 // phi. This is important because that is where the value is actually
2029 // "evaluated" even though it is used later somewhere else. (see also
2030 // D69571).
2031 SimplifyQuery RecQ = Q.getWithoutCondContext().getWithInstruction(I: CxtI);
2032
2033 Known2 = KnownBits(BitWidth);
2034
2035 // Recurse, but cap the recursion to one level, because we don't
2036 // want to waste time spinning around in loops.
2037 // TODO: See if we can base recursion limiter on number of incoming phi
2038 // edges so we don't overly clamp analysis.
2039 computeKnownBits(V: IncValue, DemandedElts, Known&: Known2, Q: RecQ,
2040 Depth: MaxAnalysisRecursionDepth - 1);
2041
2042 // See if we can further use a conditional branch into the phi
2043 // to help us determine the range of the value.
2044 if (!Known2.isConstant()) {
2045 CmpPredicate Pred;
2046 const APInt *RHSC;
2047 BasicBlock *TrueSucc, *FalseSucc;
2048 // TODO: Use RHS Value and compute range from its known bits.
2049 if (match(V: RecQ.CxtI,
2050 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: IncValue), R: m_APInt(Res&: RHSC)),
2051 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
2052 // Check for cases of duplicate successors.
2053 if ((TrueSucc == CxtPhi->getParent()) !=
2054 (FalseSucc == CxtPhi->getParent())) {
2055 // If we're using the false successor, invert the predicate.
2056 if (FalseSucc == CxtPhi->getParent())
2057 Pred = CmpInst::getInversePredicate(pred: Pred);
2058 // Get the knownbits implied by the incoming phi condition.
2059 auto CR = ConstantRange::makeExactICmpRegion(Pred, Other: *RHSC);
2060 KnownBits KnownUnion = Known2.unionWith(RHS: CR.toKnownBits());
2061 // We can have conflicts here if we are analyzing deadcode (its
2062 // impossible for us reach this BB based the icmp).
2063 if (KnownUnion.hasConflict()) {
2064 // No reason to continue analyzing in a known dead region, so
2065 // just resetAll and break. This will cause us to also exit the
2066 // outer loop.
2067 Known.resetAll();
2068 break;
2069 }
2070 Known2 = KnownUnion;
2071 }
2072 }
2073 }
2074
2075 Known = Known.intersectWith(RHS: Known2);
2076 // If all bits have been ruled out, there's no need to check
2077 // more operands.
2078 if (Known.isUnknown())
2079 break;
2080 }
2081 }
2082 break;
2083 }
2084 case Instruction::Call:
2085 case Instruction::Invoke: {
2086 // If range metadata is attached to this call, set known bits from that,
2087 // and then intersect with known bits based on other properties of the
2088 // function.
2089 if (MDNode *MD =
2090 Q.IIQ.getMetadata(I: cast<Instruction>(Val: I), KindID: LLVMContext::MD_range))
2091 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
2092
2093 const auto *CB = cast<CallBase>(Val: I);
2094
2095 if (std::optional<ConstantRange> Range = CB->getRange())
2096 Known = Known.unionWith(RHS: Range->toKnownBits());
2097
2098 if (const Value *RV = CB->getReturnedArgOperand()) {
2099 if (RV->getType() == I->getType()) {
2100 computeKnownBits(V: RV, Known&: Known2, Q, Depth: Depth + 1);
2101 Known = Known.unionWith(RHS: Known2);
2102 // If the function doesn't return properly for all input values
2103 // (e.g. unreachable exits) then there might be conflicts between the
2104 // argument value and the range metadata. Simply discard the known bits
2105 // in case of conflicts.
2106 if (Known.hasConflict())
2107 Known.resetAll();
2108 }
2109 }
2110 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
2111 switch (II->getIntrinsicID()) {
2112 default:
2113 break;
2114 case Intrinsic::abs: {
2115 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2116 bool IntMinIsPoison = match(V: II->getArgOperand(i: 1), P: m_One());
2117 Known = Known.unionWith(RHS: Known2.abs(IntMinIsPoison));
2118 break;
2119 }
2120 case Intrinsic::bitreverse:
2121 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2122 Known = Known.unionWith(RHS: Known2.reverseBits());
2123 break;
2124 case Intrinsic::bswap:
2125 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2126 Known = Known.unionWith(RHS: Known2.byteSwap());
2127 break;
2128 case Intrinsic::ctlz: {
2129 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2130 // If we have a known 1, its position is our upper bound.
2131 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2132 // If this call is poison for 0 input, the result will be less than 2^n.
2133 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2134 PossibleLZ = std::min(a: PossibleLZ, b: BitWidth - 1);
2135 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
2136 Known.Zero.setBitsFrom(LowBits);
2137 break;
2138 }
2139 case Intrinsic::cttz: {
2140 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2141 // If we have a known 1, its position is our upper bound.
2142 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2143 // If this call is poison for 0 input, the result will be less than 2^n.
2144 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2145 PossibleTZ = std::min(a: PossibleTZ, b: BitWidth - 1);
2146 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
2147 Known.Zero.setBitsFrom(LowBits);
2148 break;
2149 }
2150 case Intrinsic::ctpop: {
2151 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2152 // We can bound the space the count needs. Also, bits known to be zero
2153 // can't contribute to the population.
2154 unsigned BitsPossiblySet = Known2.countMaxPopulation();
2155 unsigned LowBits = llvm::bit_width(Value: BitsPossiblySet);
2156 Known.Zero.setBitsFrom(LowBits);
2157 // TODO: we could bound KnownOne using the lower bound on the number
2158 // of bits which might be set provided by popcnt KnownOne2.
2159 break;
2160 }
2161 case Intrinsic::fshr:
2162 case Intrinsic::fshl: {
2163 const APInt *SA;
2164 if (!match(V: I->getOperand(i: 2), P: m_APInt(Res&: SA)))
2165 break;
2166
2167 KnownBits Known3(BitWidth);
2168 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2169 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known3, Q, Depth: Depth + 1);
2170 Known = II->getIntrinsicID() == Intrinsic::fshl
2171 ? KnownBits::fshl(LHS: Known2, RHS: Known3, Amt: *SA)
2172 : KnownBits::fshr(LHS: Known2, RHS: Known3, Amt: *SA);
2173 break;
2174 }
2175 case Intrinsic::clmul:
2176 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2177 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2178 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
2179 break;
2180 case Intrinsic::pext:
2181 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2182 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2183 Known = KnownBits::pext(Val: Known, Mask: Known2);
2184 break;
2185 case Intrinsic::pdep:
2186 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2187 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2188 Known = KnownBits::pdep(Val: Known, Mask: Known2);
2189 break;
2190 case Intrinsic::smulh:
2191 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2192 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2193 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
2194 break;
2195 case Intrinsic::umulh:
2196 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2197 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2198 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
2199 break;
2200 case Intrinsic::uadd_sat:
2201 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2202 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2203 Known = KnownBits::uadd_sat(LHS: Known, RHS: Known2);
2204 break;
2205 case Intrinsic::usub_sat:
2206 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2207 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2208 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
2209 break;
2210 case Intrinsic::sadd_sat:
2211 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2212 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2213 Known = KnownBits::sadd_sat(LHS: Known, RHS: Known2);
2214 break;
2215 case Intrinsic::ssub_sat:
2216 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2217 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2218 Known = KnownBits::ssub_sat(LHS: Known, RHS: Known2);
2219 break;
2220 // Vec reverse preserves bits from input vec.
2221 case Intrinsic::vector_reverse:
2222 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: DemandedElts.reverseBits(), Known, Q,
2223 Depth: Depth + 1);
2224 break;
2225 // for min/max/and/or reduce, any bit common to each element in the
2226 // input vec is set in the output.
2227 case Intrinsic::vector_reduce_and:
2228 case Intrinsic::vector_reduce_or:
2229 case Intrinsic::vector_reduce_umax:
2230 case Intrinsic::vector_reduce_umin:
2231 case Intrinsic::vector_reduce_smax:
2232 case Intrinsic::vector_reduce_smin:
2233 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2234 break;
2235 case Intrinsic::vector_reduce_xor: {
2236 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2237 // The zeros common to all vecs are zero in the output.
2238 // If the number of elements is odd, then the common ones remain. If the
2239 // number of elements is even, then the common ones becomes zeros.
2240 auto *VecTy = cast<VectorType>(Val: I->getOperand(i: 0)->getType());
2241 // Even, so the ones become zeros.
2242 bool EvenCnt = VecTy->getElementCount().isKnownEven();
2243 if (EvenCnt)
2244 Known.Zero |= Known.One;
2245 // Maybe even element count so need to clear ones.
2246 if (VecTy->isScalableTy() || EvenCnt)
2247 Known.One.clearAllBits();
2248 break;
2249 }
2250 case Intrinsic::vector_reduce_add: {
2251 auto *VecTy = dyn_cast<FixedVectorType>(Val: I->getOperand(i: 0)->getType());
2252 if (!VecTy)
2253 break;
2254 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2255 Known = Known.reduceAdd(NumElts: VecTy->getNumElements());
2256 break;
2257 }
2258 case Intrinsic::umin:
2259 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2260 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2261 Known = KnownBits::umin(LHS: Known, RHS: Known2);
2262 break;
2263 case Intrinsic::umax:
2264 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2265 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2266 Known = KnownBits::umax(LHS: Known, RHS: Known2);
2267 break;
2268 case Intrinsic::smin:
2269 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2270 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2271 Known = KnownBits::smin(LHS: Known, RHS: Known2);
2272 unionWithMinMaxIntrinsicClamp(II, Known);
2273 break;
2274 case Intrinsic::smax:
2275 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2276 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2277 Known = KnownBits::smax(LHS: Known, RHS: Known2);
2278 unionWithMinMaxIntrinsicClamp(II, Known);
2279 break;
2280 case Intrinsic::ptrmask: {
2281 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2282
2283 const Value *Mask = I->getOperand(i: 1);
2284 Known2 = KnownBits(Mask->getType()->getScalarSizeInBits());
2285 computeKnownBits(V: Mask, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2286 // TODO: 1-extend would be more precise.
2287 Known &= Known2.anyextOrTrunc(BitWidth);
2288 break;
2289 }
2290 case Intrinsic::x86_sse2_pmulh_w:
2291 case Intrinsic::x86_avx2_pmulh_w:
2292 case Intrinsic::x86_avx512_pmulh_w_512:
2293 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2294 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2295 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
2296 break;
2297 case Intrinsic::x86_sse2_pmulhu_w:
2298 case Intrinsic::x86_avx2_pmulhu_w:
2299 case Intrinsic::x86_avx512_pmulhu_w_512:
2300 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2301 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2302 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
2303 break;
2304 case Intrinsic::x86_sse42_crc32_64_64:
2305 Known.Zero.setBitsFrom(32);
2306 break;
2307 case Intrinsic::x86_ssse3_phadd_d_128:
2308 case Intrinsic::x86_ssse3_phadd_w_128:
2309 case Intrinsic::x86_avx2_phadd_d:
2310 case Intrinsic::x86_avx2_phadd_w: {
2311 Known = computeKnownBitsForHorizontalOperation(
2312 I, DemandedElts, Q, Depth,
2313 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2314 return KnownBits::add(LHS: KnownLHS, RHS: KnownRHS);
2315 });
2316 break;
2317 }
2318 case Intrinsic::x86_ssse3_phadd_sw_128:
2319 case Intrinsic::x86_avx2_phadd_sw: {
2320 Known = computeKnownBitsForHorizontalOperation(
2321 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::sadd_sat);
2322 break;
2323 }
2324 case Intrinsic::x86_ssse3_phsub_d_128:
2325 case Intrinsic::x86_ssse3_phsub_w_128:
2326 case Intrinsic::x86_avx2_phsub_d:
2327 case Intrinsic::x86_avx2_phsub_w: {
2328 Known = computeKnownBitsForHorizontalOperation(
2329 I, DemandedElts, Q, Depth,
2330 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2331 return KnownBits::sub(LHS: KnownLHS, RHS: KnownRHS);
2332 });
2333 break;
2334 }
2335 case Intrinsic::x86_ssse3_phsub_sw_128:
2336 case Intrinsic::x86_avx2_phsub_sw: {
2337 Known = computeKnownBitsForHorizontalOperation(
2338 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::ssub_sat);
2339 break;
2340 }
2341 case Intrinsic::riscv_vsetvli:
2342 case Intrinsic::riscv_vsetvlimax: {
2343 bool HasAVL = II->getIntrinsicID() == Intrinsic::riscv_vsetvli;
2344 const ConstantRange Range = getVScaleRange(F: II->getFunction(), BitWidth);
2345 uint64_t SEW = RISCVVType::decodeVSEW(
2346 VSEW: cast<ConstantInt>(Val: II->getArgOperand(i: HasAVL))->getZExtValue());
2347 RISCVVType::VLMUL VLMUL = static_cast<RISCVVType::VLMUL>(
2348 cast<ConstantInt>(Val: II->getArgOperand(i: 1 + HasAVL))->getZExtValue());
2349 uint64_t MaxVLEN =
2350 Range.getUnsignedMax().getZExtValue() * RISCV::RVVBitsPerBlock;
2351 uint64_t MaxVL = MaxVLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
2352
2353 // Result of vsetvli must be not larger than AVL.
2354 if (HasAVL)
2355 if (auto *CI = dyn_cast<ConstantInt>(Val: II->getArgOperand(i: 0)))
2356 MaxVL = std::min(a: MaxVL, b: CI->getZExtValue());
2357
2358 unsigned KnownZeroFirstBit = Log2_32(Value: MaxVL) + 1;
2359 if (BitWidth > KnownZeroFirstBit)
2360 Known.Zero.setBitsFrom(KnownZeroFirstBit);
2361 break;
2362 }
2363 case Intrinsic::amdgcn_mbcnt_hi:
2364 case Intrinsic::amdgcn_mbcnt_lo: {
2365 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
2366 // most 31 + src1.
2367 Known.Zero.setBitsFrom(
2368 II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo ? 6 : 5);
2369 computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Q, Depth: Depth + 1);
2370 Known = KnownBits::add(LHS: Known, RHS: Known2);
2371 break;
2372 }
2373 case Intrinsic::vscale: {
2374 if (!II->getParent() || !II->getFunction())
2375 break;
2376
2377 Known = getVScaleRange(F: II->getFunction(), BitWidth).toKnownBits();
2378 break;
2379 }
2380 case Intrinsic::stepvector: {
2381 auto *VecTy = cast<VectorType>(Val: II->getType());
2382 unsigned MinNumElts = VecTy->getElementCount().getKnownMinValue();
2383 if (!isUIntN(N: BitWidth, x: MinNumElts))
2384 break;
2385
2386 bool Overflow = false;
2387 APInt MaxNumElts(BitWidth, MinNumElts);
2388 if (VecTy->isScalableTy()) {
2389 if (!II->getParent() || !II->getFunction())
2390 break;
2391 MaxNumElts = getVScaleRange(F: II->getFunction(), BitWidth)
2392 .getUnsignedMax()
2393 .umul_ov(RHS: MaxNumElts, Overflow);
2394 }
2395
2396 // Give up if the lane count could wrap. Stepvector truncates lane
2397 // indices that do not fit in the element type.
2398 if (Overflow)
2399 break;
2400
2401 Known.Zero.setHighBits((MaxNumElts - 1).countl_zero());
2402 break;
2403 }
2404 }
2405 }
2406 break;
2407 }
2408 case Instruction::ShuffleVector: {
2409 if (auto *Splat = getSplatValue(V: I)) {
2410 computeKnownBits(V: Splat, Known, Q, Depth: Depth + 1);
2411 break;
2412 }
2413
2414 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
2415 // FIXME: Do we need to handle ConstantExpr involving shufflevectors?
2416 if (!Shuf) {
2417 Known.resetAll();
2418 return;
2419 }
2420 // For undef elements, we don't know anything about the common state of
2421 // the shuffle result.
2422 APInt DemandedLHS, DemandedRHS;
2423 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS)) {
2424 Known.resetAll();
2425 return;
2426 }
2427 Known.setAllConflict();
2428 if (!!DemandedLHS) {
2429 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
2430 computeKnownBits(V: LHS, DemandedElts: DemandedLHS, Known, Q, Depth: Depth + 1);
2431 // If we don't know any bits, early out.
2432 if (Known.isUnknown())
2433 break;
2434 }
2435 if (!!DemandedRHS) {
2436 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
2437 computeKnownBits(V: RHS, DemandedElts: DemandedRHS, Known&: Known2, Q, Depth: Depth + 1);
2438 Known = Known.intersectWith(RHS: Known2);
2439 }
2440 break;
2441 }
2442 case Instruction::InsertElement: {
2443 if (isa<ScalableVectorType>(Val: I->getType())) {
2444 Known.resetAll();
2445 return;
2446 }
2447 const Value *Vec = I->getOperand(i: 0);
2448 const Value *Elt = I->getOperand(i: 1);
2449 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
2450 unsigned NumElts = DemandedElts.getBitWidth();
2451 APInt DemandedVecElts = DemandedElts;
2452 bool NeedsElt = true;
2453 // If we know the index we are inserting too, clear it from Vec check.
2454 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
2455 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
2456 NeedsElt = DemandedElts[CIdx->getZExtValue()];
2457 }
2458
2459 Known.setAllConflict();
2460 if (NeedsElt) {
2461 computeKnownBits(V: Elt, Known, Q, Depth: Depth + 1);
2462 // If we don't know any bits, early out.
2463 if (Known.isUnknown())
2464 break;
2465 }
2466
2467 if (!DemandedVecElts.isZero()) {
2468 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known&: Known2, Q, Depth: Depth + 1);
2469 Known = Known.intersectWith(RHS: Known2);
2470 }
2471 break;
2472 }
2473 case Instruction::ExtractElement: {
2474 // Look through extract element. If the index is non-constant or
2475 // out-of-range demand all elements, otherwise just the extracted element.
2476 const Value *Vec = I->getOperand(i: 0);
2477 const Value *Idx = I->getOperand(i: 1);
2478 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
2479 if (isa<ScalableVectorType>(Val: Vec->getType())) {
2480 // FIXME: there's probably *something* we can do with scalable vectors
2481 Known.resetAll();
2482 break;
2483 }
2484 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2485 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
2486 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
2487 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
2488 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known, Q, Depth: Depth + 1);
2489 break;
2490 }
2491 case Instruction::ExtractValue:
2492 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I->getOperand(i: 0))) {
2493 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: I);
2494 if (EVI->getNumIndices() != 1) break;
2495 if (EVI->getIndices()[0] == 0) {
2496 switch (II->getIntrinsicID()) {
2497 default: break;
2498 case Intrinsic::uadd_with_overflow:
2499 case Intrinsic::sadd_with_overflow:
2500 computeKnownBitsAddSub(
2501 Add: true, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2502 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2503 break;
2504 case Intrinsic::usub_with_overflow:
2505 case Intrinsic::ssub_with_overflow:
2506 computeKnownBitsAddSub(
2507 Add: false, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2508 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2509 break;
2510 case Intrinsic::umul_with_overflow:
2511 case Intrinsic::smul_with_overflow:
2512 computeKnownBitsMul(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), NSW: false,
2513 NUW: false, DemandedElts, Known, Known2, Q, Depth);
2514 break;
2515 }
2516 }
2517 }
2518 break;
2519 case Instruction::Freeze:
2520 if (isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
2521 Depth: Depth + 1))
2522 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2523 break;
2524 }
2525}
2526
2527/// Determine which bits of V are known to be either zero or one and return
2528/// them.
2529KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
2530 const SimplifyQuery &Q, unsigned Depth) {
2531 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2532 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
2533 return Known;
2534}
2535
2536/// Determine which bits of V are known to be either zero or one and return
2537/// them.
2538KnownBits llvm::computeKnownBits(const Value *V, const SimplifyQuery &Q,
2539 unsigned Depth) {
2540 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2541 computeKnownBits(V, Known, Q, Depth);
2542 return Known;
2543}
2544
2545/// Determine which bits of V are known to be either zero or one and return
2546/// them in the Known bit set.
2547///
2548/// NOTE: we cannot consider 'undef' to be "IsZero" here. The problem is that
2549/// we cannot optimize based on the assumption that it is zero without changing
2550/// it to be an explicit zero. If we don't change it to zero, other code could
2551/// optimized based on the contradictory assumption that it is non-zero.
2552/// Because instcombine aggressively folds operations with undef args anyway,
2553/// this won't lose us code quality.
2554///
2555/// This function is defined on values with integer type, values with pointer
2556/// type, and vectors of integers. In the case
2557/// where V is a vector, known zero, and known one values are the
2558/// same width as the vector element, and the bit is set only if it is true
2559/// for all of the demanded elements in the vector specified by DemandedElts.
2560void computeKnownBits(const Value *V, const APInt &DemandedElts,
2561 KnownBits &Known, const SimplifyQuery &Q,
2562 unsigned Depth) {
2563 if (!DemandedElts) {
2564 // No demanded elts, better to assume we don't know anything.
2565 Known.resetAll();
2566 return;
2567 }
2568
2569 assert(V && "No Value?");
2570 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2571
2572#ifndef NDEBUG
2573 Type *Ty = V->getType();
2574 unsigned BitWidth = Known.getBitWidth();
2575
2576 assert((Ty->isIntOrIntVectorTy(BitWidth) || Ty->isPtrOrPtrVectorTy()) &&
2577 "Not integer or pointer type!");
2578
2579 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
2580 assert(
2581 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
2582 "DemandedElt width should equal the fixed vector number of elements");
2583 } else {
2584 assert(DemandedElts == APInt(1, 1) &&
2585 "DemandedElt width should be 1 for scalars or scalable vectors");
2586 }
2587
2588 Type *ScalarTy = Ty->getScalarType();
2589 if (ScalarTy->isPointerTy()) {
2590 assert(BitWidth == Q.DL.getPointerTypeSizeInBits(ScalarTy) &&
2591 "V and Known should have same BitWidth");
2592 } else {
2593 assert(BitWidth == Q.DL.getTypeSizeInBits(ScalarTy) &&
2594 "V and Known should have same BitWidth");
2595 }
2596#endif
2597
2598 const APInt *C;
2599 if (match(V, P: m_APInt(Res&: C))) {
2600 // We know all of the bits for a scalar constant or a splat vector constant!
2601 Known = KnownBits::makeConstant(C: *C);
2602 return;
2603 }
2604 // Null and aggregate-zero are all-zeros.
2605 if (isa<ConstantPointerNull>(Val: V) || isa<ConstantAggregateZero>(Val: V)) {
2606 Known.setAllZero();
2607 return;
2608 }
2609 // Handle a constant vector by taking the intersection of the known bits of
2610 // each element.
2611 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: V)) {
2612 assert(!isa<ScalableVectorType>(V->getType()));
2613 // We know that CDV must be a vector of integers. Take the intersection of
2614 // each element.
2615 Known.setAllConflict();
2616 for (unsigned i = 0, e = CDV->getNumElements(); i != e; ++i) {
2617 if (!DemandedElts[i])
2618 continue;
2619 APInt Elt = CDV->getElementAsAPInt(i);
2620 Known.Zero &= ~Elt;
2621 Known.One &= Elt;
2622 }
2623 if (Known.hasConflict())
2624 Known.resetAll();
2625 return;
2626 }
2627
2628 if (const auto *CV = dyn_cast<ConstantVector>(Val: V)) {
2629 assert(!isa<ScalableVectorType>(V->getType()));
2630 // We know that CV must be a vector of integers. Take the intersection of
2631 // each element.
2632 Known.setAllConflict();
2633 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
2634 if (!DemandedElts[i])
2635 continue;
2636 Constant *Element = CV->getAggregateElement(Elt: i);
2637 if (isa<PoisonValue>(Val: Element))
2638 continue;
2639 auto *ElementCI = dyn_cast_or_null<ConstantInt>(Val: Element);
2640 if (!ElementCI) {
2641 Known.resetAll();
2642 return;
2643 }
2644 const APInt &Elt = ElementCI->getValue();
2645 Known.Zero &= ~Elt;
2646 Known.One &= Elt;
2647 }
2648 if (Known.hasConflict())
2649 Known.resetAll();
2650 return;
2651 }
2652
2653 // Start out not knowing anything.
2654 Known.resetAll();
2655
2656 // We can't imply anything about undefs.
2657 if (isa<UndefValue>(Val: V))
2658 return;
2659
2660 // There's no point in looking through other users of ConstantData for
2661 // assumptions. Confirm that we've handled them all.
2662 assert(!isa<ConstantData>(V) && "Unhandled constant data!");
2663
2664 if (const auto *A = dyn_cast<Argument>(Val: V))
2665 if (std::optional<ConstantRange> Range = A->getRange())
2666 Known = Range->toKnownBits();
2667
2668 // All recursive calls that increase depth must come after this.
2669 if (Depth == MaxAnalysisRecursionDepth)
2670 return;
2671
2672 // A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
2673 // the bits of its aliasee.
2674 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: V)) {
2675 if (!GA->isInterposable())
2676 computeKnownBits(V: GA->getAliasee(), Known, Q, Depth: Depth + 1);
2677 return;
2678 }
2679
2680 if (const Operator *I = dyn_cast<Operator>(Val: V))
2681 computeKnownBitsFromOperator(I, DemandedElts, Known, Q, Depth);
2682 else if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
2683 if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
2684 Known = CR->toKnownBits();
2685 }
2686
2687 // Aligned pointers have trailing zeros - refine Known.Zero set
2688 if (isa<PointerType>(Val: V->getType())) {
2689 Align Alignment = V->getPointerAlignment(DL: Q.DL);
2690 Known.Zero.setLowBits(Log2(A: Alignment));
2691 }
2692
2693 // computeKnownBitsFromContext strictly refines Known.
2694 // Therefore, we run them after computeKnownBitsFromOperator.
2695
2696 // Check whether we can determine known bits from context such as assumes.
2697 computeKnownBitsFromContext(V, Known, Q, Depth);
2698}
2699
2700/// Try to detect a recurrence that the value of the induction variable is
2701/// always a power of two (or zero).
2702static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
2703 SimplifyQuery &Q, unsigned Depth) {
2704 BinaryOperator *BO = nullptr;
2705 Value *Start = nullptr, *Step = nullptr;
2706 if (!matchSimpleRecurrence(P: PN, BO, Start, Step))
2707 return false;
2708
2709 // Initial value must be a power of two.
2710 for (const Use &U : PN->operands()) {
2711 if (U.get() == Start) {
2712 // Initial value comes from a different BB, need to adjust context
2713 // instruction for analysis.
2714 Q.CxtI = PN->getIncomingBlock(U)->getTerminator();
2715 if (!isKnownToBeAPowerOfTwo(V: Start, OrZero, Q, Depth))
2716 return false;
2717 }
2718 }
2719
2720 // Except for Mul, the induction variable must be on the left side of the
2721 // increment expression, otherwise its value can be arbitrary.
2722 if (BO->getOpcode() != Instruction::Mul && BO->getOperand(i_nocapture: 1) != Step)
2723 return false;
2724
2725 Q.CxtI = BO->getParent()->getTerminator();
2726 switch (BO->getOpcode()) {
2727 case Instruction::Mul:
2728 // Power of two is closed under multiplication.
2729 return (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) ||
2730 Q.IIQ.hasNoSignedWrap(Op: BO)) &&
2731 isKnownToBeAPowerOfTwo(V: Step, OrZero, Q, Depth);
2732 case Instruction::SDiv:
2733 // Start value must not be signmask for signed division, so simply being a
2734 // power of two is not sufficient, and it has to be a constant.
2735 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2736 return false;
2737 [[fallthrough]];
2738 case Instruction::UDiv:
2739 // Divisor must be a power of two.
2740 // If OrZero is false, cannot guarantee induction variable is non-zero after
2741 // division, same for Shr, unless it is exact division.
2742 return (OrZero || Q.IIQ.isExact(Op: BO)) &&
2743 isKnownToBeAPowerOfTwo(V: Step, OrZero: false, Q, Depth);
2744 case Instruction::Shl:
2745 return OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO);
2746 case Instruction::AShr:
2747 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2748 return false;
2749 [[fallthrough]];
2750 case Instruction::LShr:
2751 return OrZero || Q.IIQ.isExact(Op: BO);
2752 default:
2753 return false;
2754 }
2755}
2756
2757/// Return true if we can infer that \p V is known to be a power of 2 from
2758/// dominating condition \p Cond (e.g., ctpop(V) == 1).
2759static bool isImpliedToBeAPowerOfTwoFromCond(const Value *V, bool OrZero,
2760 const Value *Cond,
2761 bool CondIsTrue) {
2762 CmpPredicate Pred;
2763 const APInt *RHSC;
2764 if (!match(V: Cond, P: m_ICmp(Pred, L: m_Ctpop(Op0: m_Specific(V)), R: m_APInt(Res&: RHSC))))
2765 return false;
2766 if (!CondIsTrue)
2767 Pred = ICmpInst::getInversePredicate(pred: Pred);
2768 // ctpop(V) u< 2
2769 if (OrZero && Pred == ICmpInst::ICMP_ULT && *RHSC == 2)
2770 return true;
2771 // ctpop(V) == 1
2772 return Pred == ICmpInst::ICMP_EQ && *RHSC == 1;
2773}
2774
2775/// Return true if the given value is known to have exactly one
2776/// bit set when defined. For vectors return true if every element is known to
2777/// be a power of two when defined. Supports values with integer or pointer
2778/// types and vectors of integers.
2779bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero,
2780 const SimplifyQuery &Q, unsigned Depth) {
2781 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2782
2783 if (isa<Constant>(Val: V))
2784 return OrZero ? match(V, P: m_Power2OrZero()) : match(V, P: m_Power2());
2785
2786 // i1 is by definition a power of 2 or zero.
2787 if (OrZero && V->getType()->getScalarSizeInBits() == 1)
2788 return true;
2789
2790 // Try to infer from assumptions.
2791 if (Q.AC && Q.CxtI) {
2792 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
2793 if (!AssumeVH)
2794 continue;
2795 CallInst *I = cast<CallInst>(Val&: AssumeVH);
2796 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond: I->getArgOperand(i: 0),
2797 /*CondIsTrue=*/true) &&
2798 isValidAssumeForContext(I, Q))
2799 return true;
2800 }
2801 }
2802
2803 // Handle dominating conditions.
2804 if (Q.DC && Q.CxtI && Q.DT) {
2805 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
2806 Value *Cond = BI->getCondition();
2807
2808 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
2809 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2810 /*CondIsTrue=*/true) &&
2811 Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
2812 return true;
2813
2814 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
2815 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2816 /*CondIsTrue=*/false) &&
2817 Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
2818 return true;
2819 }
2820 }
2821
2822 auto *I = dyn_cast<Instruction>(Val: V);
2823 if (!I)
2824 return false;
2825
2826 if (Q.CxtI && match(V, P: m_VScale())) {
2827 const Function *F = Q.CxtI->getFunction();
2828 // The vscale_range indicates vscale is a power-of-two.
2829 return F->hasFnAttribute(Kind: Attribute::VScaleRange);
2830 }
2831
2832 // 1 << X is clearly a power of two if the one is not shifted off the end. If
2833 // it is shifted off the end then the result is undefined.
2834 if (match(V: I, P: m_Shl(L: m_One(), R: m_Value())))
2835 return true;
2836
2837 // (signmask) >>l X is clearly a power of two if the one is not shifted off
2838 // the bottom. If it is shifted off the bottom then the result is undefined.
2839 if (match(V: I, P: m_LShr(L: m_SignMask(), R: m_Value())))
2840 return true;
2841
2842 // The remaining tests are all recursive, so bail out if we hit the limit.
2843 if (Depth++ == MaxAnalysisRecursionDepth)
2844 return false;
2845
2846 switch (I->getOpcode()) {
2847 case Instruction::ZExt:
2848 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2849 case Instruction::Trunc:
2850 return OrZero && isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2851 case Instruction::Shl:
2852 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: I) || Q.IIQ.hasNoSignedWrap(Op: I))
2853 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2854 return false;
2855 case Instruction::LShr:
2856 if (OrZero || Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2857 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2858 return false;
2859 case Instruction::UDiv:
2860 if (Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2861 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2862 return false;
2863 case Instruction::Mul:
2864 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2865 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth) &&
2866 (OrZero || isKnownNonZero(V: I, Q, Depth));
2867 case Instruction::And:
2868 // A power of two and'd with anything is a power of two or zero.
2869 if (OrZero &&
2870 (isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), /*OrZero*/ true, Q, Depth) ||
2871 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), /*OrZero*/ true, Q, Depth)))
2872 return true;
2873 // X & (-X) is always a power of two or zero.
2874 if (match(V: I->getOperand(i: 0), P: m_Neg(V: m_Specific(V: I->getOperand(i: 1)))) ||
2875 match(V: I->getOperand(i: 1), P: m_Neg(V: m_Specific(V: I->getOperand(i: 0)))))
2876 return OrZero || isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
2877 return false;
2878 case Instruction::Add: {
2879 // Adding a power-of-two or zero to the same power-of-two or zero yields
2880 // either the original power-of-two, a larger power-of-two or zero.
2881 const OverflowingBinaryOperator *VOBO = cast<OverflowingBinaryOperator>(Val: V);
2882 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO) ||
2883 Q.IIQ.hasNoSignedWrap(Op: VOBO)) {
2884 if (match(V: I->getOperand(i: 0),
2885 P: m_c_And(L: m_Specific(V: I->getOperand(i: 1)), R: m_Value())) &&
2886 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth))
2887 return true;
2888 if (match(V: I->getOperand(i: 1),
2889 P: m_c_And(L: m_Specific(V: I->getOperand(i: 0)), R: m_Value())) &&
2890 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth))
2891 return true;
2892
2893 unsigned BitWidth = V->getType()->getScalarSizeInBits();
2894 KnownBits LHSBits(BitWidth);
2895 computeKnownBits(V: I->getOperand(i: 0), Known&: LHSBits, Q, Depth);
2896
2897 KnownBits RHSBits(BitWidth);
2898 computeKnownBits(V: I->getOperand(i: 1), Known&: RHSBits, Q, Depth);
2899 // If i8 V is a power of two or zero:
2900 // ZeroBits: 1 1 1 0 1 1 1 1
2901 // ~ZeroBits: 0 0 0 1 0 0 0 0
2902 if ((~(LHSBits.Zero & RHSBits.Zero)).isPowerOf2())
2903 // If OrZero isn't set, we cannot give back a zero result.
2904 // Make sure either the LHS or RHS has a bit set.
2905 if (OrZero || RHSBits.One.getBoolValue() || LHSBits.One.getBoolValue())
2906 return true;
2907 }
2908
2909 // LShr(UINT_MAX, Y) + 1 is a power of two (if add is nuw) or zero.
2910 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO))
2911 if (match(V: I, P: m_Add(L: m_LShr(L: m_AllOnes(), R: m_Value()), R: m_One())))
2912 return true;
2913 return false;
2914 }
2915 case Instruction::Select:
2916 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2917 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 2), OrZero, Q, Depth);
2918 case Instruction::PHI: {
2919 // A PHI node is power of two if all incoming values are power of two, or if
2920 // it is an induction variable where in each step its value is a power of
2921 // two.
2922 auto *PN = cast<PHINode>(Val: I);
2923 SimplifyQuery RecQ = Q.getWithoutCondContext();
2924
2925 // Check if it is an induction variable and always power of two.
2926 if (isPowerOfTwoRecurrence(PN, OrZero, Q&: RecQ, Depth))
2927 return true;
2928
2929 // Recursively check all incoming values. Limit recursion to 2 levels, so
2930 // that search complexity is limited to number of operands^2.
2931 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
2932 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
2933 // Value is power of 2 if it is coming from PHI node itself by induction.
2934 if (U.get() == PN)
2935 return true;
2936
2937 // Change the context instruction to the incoming block where it is
2938 // evaluated.
2939 RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
2940 return isKnownToBeAPowerOfTwo(V: U.get(), OrZero, Q: RecQ, Depth: NewDepth);
2941 });
2942 }
2943 case Instruction::Invoke:
2944 case Instruction::Call: {
2945 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2946 switch (II->getIntrinsicID()) {
2947 case Intrinsic::umax:
2948 case Intrinsic::smax:
2949 case Intrinsic::umin:
2950 case Intrinsic::smin:
2951 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 1), OrZero, Q, Depth) &&
2952 isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2953 // bswap/bitreverse just move around bits, but don't change any 1s/0s
2954 // thus dont change pow2/non-pow2 status.
2955 case Intrinsic::bitreverse:
2956 case Intrinsic::bswap:
2957 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2958 case Intrinsic::fshr:
2959 case Intrinsic::fshl:
2960 // If Op0 == Op1, this is a rotate. is_pow2(rotate(x, y)) == is_pow2(x)
2961 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
2962 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2963 break;
2964 case Intrinsic::riscv_vsetvlimax:
2965 // VLMAX is VLEN * LMUL / SEW, which is always a non-zero power of two
2966 // for any valid vtype, so it is a power of two regardless of OrZero.
2967 return true;
2968 case Intrinsic::read_register:
2969 case Intrinsic::read_volatile_register: {
2970 // The RISC-V vlenb CSR holds VLEN/8, which is always a non-zero power
2971 // of two, so it is a power of two regardless of OrZero.
2972 const Module *M = II->getModule();
2973 if (!M || !M->getTargetTriple().isRISCV())
2974 break;
2975 return isReadVLENB(II: *II);
2976 }
2977 default:
2978 break;
2979 }
2980 }
2981 return false;
2982 }
2983 default:
2984 return false;
2985 }
2986}
2987
2988/// Test whether a GEP's result is known to be non-null.
2989///
2990/// Uses properties inherent in a GEP to try to determine whether it is known
2991/// to be non-null.
2992///
2993/// Currently this routine does not support vector GEPs.
2994static bool isGEPKnownNonNull(const GEPOperator *GEP, const SimplifyQuery &Q,
2995 unsigned Depth) {
2996 const Function *F = nullptr;
2997 if (const Instruction *I = dyn_cast<Instruction>(Val: GEP))
2998 F = I->getFunction();
2999
3000 // If the gep is nuw or inbounds with invalid null pointer, then the GEP
3001 // may be null iff the base pointer is null and the offset is zero.
3002 if (!GEP->hasNoUnsignedWrap() &&
3003 !(GEP->isInBounds() &&
3004 !NullPointerIsDefined(F, AS: GEP->getPointerAddressSpace())))
3005 return false;
3006
3007 // FIXME: Support vector-GEPs.
3008 assert(GEP->getType()->isPointerTy() && "We only support plain pointer GEP");
3009
3010 // If the base pointer is non-null, we cannot walk to a null address with an
3011 // inbounds GEP in address space zero.
3012 if (isKnownNonZero(V: GEP->getPointerOperand(), Q, Depth))
3013 return true;
3014
3015 // Walk the GEP operands and see if any operand introduces a non-zero offset.
3016 // If so, then the GEP cannot produce a null pointer, as doing so would
3017 // inherently violate the inbounds contract within address space zero.
3018 for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
3019 GTI != GTE; ++GTI) {
3020 // Struct types are easy -- they must always be indexed by a constant.
3021 if (StructType *STy = GTI.getStructTypeOrNull()) {
3022 ConstantInt *OpC = cast<ConstantInt>(Val: GTI.getOperand());
3023 unsigned ElementIdx = OpC->getZExtValue();
3024 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
3025 uint64_t ElementOffset = SL->getElementOffset(Idx: ElementIdx);
3026 if (ElementOffset > 0)
3027 return true;
3028 continue;
3029 }
3030
3031 // If we have a zero-sized type, the index doesn't matter. Keep looping.
3032 if (GTI.getSequentialElementStride(DL: Q.DL).isZero())
3033 continue;
3034
3035 // Fast path the constant operand case both for efficiency and so we don't
3036 // increment Depth when just zipping down an all-constant GEP.
3037 if (ConstantInt *OpC = dyn_cast<ConstantInt>(Val: GTI.getOperand())) {
3038 if (!OpC->isZero())
3039 return true;
3040 continue;
3041 }
3042
3043 // We post-increment Depth here because while isKnownNonZero increments it
3044 // as well, when we pop back up that increment won't persist. We don't want
3045 // to recurse 10k times just because we have 10k GEP operands. We don't
3046 // bail completely out because we want to handle constant GEPs regardless
3047 // of depth.
3048 if (Depth++ >= MaxAnalysisRecursionDepth)
3049 continue;
3050
3051 if (isKnownNonZero(V: GTI.getOperand(), Q, Depth))
3052 return true;
3053 }
3054
3055 return false;
3056}
3057
3058static bool isKnownNonNullFromDominatingCondition(const Value *V,
3059 const Instruction *CtxI,
3060 const DominatorTree *DT) {
3061 assert(!isa<Constant>(V) && "Called for constant?");
3062
3063 if (!CtxI || !DT)
3064 return false;
3065
3066 unsigned NumUsesExplored = 0;
3067 for (auto &U : V->uses()) {
3068 // Avoid massive lists
3069 if (NumUsesExplored >= DomConditionsMaxUses)
3070 break;
3071 NumUsesExplored++;
3072
3073 const Instruction *UI = cast<Instruction>(Val: U.getUser());
3074 // If the value is used as an argument to a call or invoke, then argument
3075 // attributes may provide an answer about null-ness.
3076 if (V->getType()->isPointerTy()) {
3077 if (const auto *CB = dyn_cast<CallBase>(Val: UI)) {
3078 if (CB->isArgOperand(U: &U) &&
3079 CB->paramHasNonNullAttr(ArgNo: CB->getArgOperandNo(U: &U),
3080 /*AllowUndefOrPoison=*/false) &&
3081 DT->dominates(Def: CB, User: CtxI))
3082 return true;
3083 }
3084 }
3085
3086 // If the value is used as a load/store, then the pointer must be non null.
3087 if (V == getLoadStorePointerOperand(V: UI)) {
3088 if (!NullPointerIsDefined(F: UI->getFunction(),
3089 AS: V->getType()->getPointerAddressSpace()) &&
3090 DT->dominates(Def: UI, User: CtxI))
3091 return true;
3092 }
3093
3094 if ((match(V: UI, P: m_IDiv(L: m_Value(), R: m_Specific(V))) ||
3095 match(V: UI, P: m_IRem(L: m_Value(), R: m_Specific(V)))) &&
3096 isValidAssumeForContext(Inv: UI, CxtI: CtxI, DT))
3097 return true;
3098
3099 // Consider only compare instructions uniquely controlling a branch
3100 Value *RHS;
3101 CmpPredicate Pred;
3102 if (!match(V: UI, P: m_c_ICmp(Pred, L: m_Specific(V), R: m_Value(V&: RHS))))
3103 continue;
3104
3105 bool NonNullIfTrue;
3106 if (cmpExcludesZero(Pred, RHS))
3107 NonNullIfTrue = true;
3108 else if (cmpExcludesZero(Pred: CmpInst::getInversePredicate(pred: Pred), RHS))
3109 NonNullIfTrue = false;
3110 else
3111 continue;
3112
3113 SmallVector<const User *, 4> WorkList;
3114 SmallPtrSet<const User *, 4> Visited;
3115 for (const auto *CmpU : UI->users()) {
3116 assert(WorkList.empty() && "Should be!");
3117 if (Visited.insert(Ptr: CmpU).second)
3118 WorkList.push_back(Elt: CmpU);
3119
3120 while (!WorkList.empty()) {
3121 auto *Curr = WorkList.pop_back_val();
3122
3123 // If a user is an AND, add all its users to the work list. We only
3124 // propagate "pred != null" condition through AND because it is only
3125 // correct to assume that all conditions of AND are met in true branch.
3126 // TODO: Support similar logic of OR and EQ predicate?
3127 if (NonNullIfTrue)
3128 if (match(V: Curr, P: m_LogicalAnd(L: m_Value(), R: m_Value()))) {
3129 for (const auto *CurrU : Curr->users())
3130 if (Visited.insert(Ptr: CurrU).second)
3131 WorkList.push_back(Elt: CurrU);
3132 continue;
3133 }
3134
3135 if (const CondBrInst *BI = dyn_cast<CondBrInst>(Val: Curr)) {
3136 BasicBlock *NonNullSuccessor =
3137 BI->getSuccessor(i: NonNullIfTrue ? 0 : 1);
3138 BasicBlockEdge Edge(BI->getParent(), NonNullSuccessor);
3139 if (DT->dominates(BBE: Edge, BB: CtxI->getParent()))
3140 return true;
3141 } else if (NonNullIfTrue && isGuard(U: Curr) &&
3142 DT->dominates(Def: cast<Instruction>(Val: Curr), User: CtxI)) {
3143 return true;
3144 }
3145 }
3146 }
3147 }
3148
3149 return false;
3150}
3151
3152/// Does the 'Range' metadata (which must be a valid MD_range operand list)
3153/// ensure that the value it's attached to is never Value? 'RangeType' is
3154/// is the type of the value described by the range.
3155static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value) {
3156 const unsigned NumRanges = Ranges->getNumOperands() / 2;
3157 assert(NumRanges >= 1);
3158 for (unsigned i = 0; i < NumRanges; ++i) {
3159 ConstantInt *Lower =
3160 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 0));
3161 ConstantInt *Upper =
3162 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 1));
3163 ConstantRange Range(Lower->getValue(), Upper->getValue());
3164 if (Range.contains(Val: Value))
3165 return false;
3166 }
3167 return true;
3168}
3169
3170/// Try to detect a recurrence that monotonically increases/decreases from a
3171/// non-zero starting value. These are common as induction variables.
3172static bool isNonZeroRecurrence(const PHINode *PN) {
3173 BinaryOperator *BO = nullptr;
3174 Value *Start = nullptr, *Step = nullptr;
3175 const APInt *StartC, *StepC;
3176 if (!matchSimpleRecurrence(P: PN, BO, Start, Step) ||
3177 !match(V: Start, P: m_APInt(Res&: StartC)) || StartC->isZero())
3178 return false;
3179
3180 switch (BO->getOpcode()) {
3181 case Instruction::Add:
3182 // Starting from non-zero and stepping away from zero can never wrap back
3183 // to zero.
3184 return BO->hasNoUnsignedWrap() ||
3185 (BO->hasNoSignedWrap() && match(V: Step, P: m_APInt(Res&: StepC)) &&
3186 StartC->isNegative() == StepC->isNegative());
3187 case Instruction::Mul:
3188 return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) &&
3189 match(V: Step, P: m_APInt(Res&: StepC)) && !StepC->isZero();
3190 case Instruction::Shl:
3191 return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap();
3192 case Instruction::AShr:
3193 case Instruction::LShr:
3194 return BO->isExact();
3195 default:
3196 return false;
3197 }
3198}
3199
3200static bool matchOpWithOpEqZero(Value *Op0, Value *Op1) {
3201 return match(V: Op0, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3202 L: m_Specific(V: Op1), R: m_Zero()))) ||
3203 match(V: Op1, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3204 L: m_Specific(V: Op0), R: m_Zero())));
3205}
3206
3207static bool isNonZeroAdd(const APInt &DemandedElts, const SimplifyQuery &Q,
3208 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3209 bool NUW, unsigned Depth) {
3210 // (X + (X != 0)) is non zero
3211 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3212 return true;
3213
3214 if (NUW)
3215 return isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3216 isKnownNonZero(V: X, DemandedElts, Q, Depth);
3217
3218 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3219 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3220
3221 // If X and Y are both non-negative (as signed values) then their sum is not
3222 // zero unless both X and Y are zero.
3223 if (XKnown.isNonNegative() && YKnown.isNonNegative())
3224 if (isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3225 isKnownNonZero(V: X, DemandedElts, Q, Depth))
3226 return true;
3227
3228 // If X and Y are both negative (as signed values) then their sum is not
3229 // zero unless both X and Y equal INT_MIN.
3230 if (XKnown.isNegative() && YKnown.isNegative()) {
3231 APInt Mask = APInt::getSignedMaxValue(numBits: BitWidth);
3232 // The sign bit of X is set. If some other bit is set then X is not equal
3233 // to INT_MIN.
3234 if (XKnown.One.intersects(RHS: Mask))
3235 return true;
3236 // The sign bit of Y is set. If some other bit is set then Y is not equal
3237 // to INT_MIN.
3238 if (YKnown.One.intersects(RHS: Mask))
3239 return true;
3240 }
3241
3242 // The sum of a non-negative number and a power of two is not zero.
3243 if (XKnown.isNonNegative() &&
3244 isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ false, Q, Depth))
3245 return true;
3246 if (YKnown.isNonNegative() &&
3247 isKnownToBeAPowerOfTwo(V: X, /*OrZero*/ false, Q, Depth))
3248 return true;
3249
3250 return KnownBits::add(LHS: XKnown, RHS: YKnown, NSW, NUW).isNonZero();
3251}
3252
3253static bool isNonZeroSub(const APInt &DemandedElts, const SimplifyQuery &Q,
3254 unsigned BitWidth, Value *X, Value *Y,
3255 unsigned Depth) {
3256 // (X - (X != 0)) is non zero
3257 // ((X != 0) - X) is non zero
3258 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3259 return true;
3260
3261 // TODO: Move this case into isKnownNonEqual().
3262 if (auto *C = dyn_cast<Constant>(Val: X))
3263 if (C->isNullValue() && isKnownNonZero(V: Y, DemandedElts, Q, Depth))
3264 return true;
3265
3266 return ::isKnownNonEqual(V1: X, V2: Y, DemandedElts, Q, Depth);
3267}
3268
3269static bool isNonZeroMul(const APInt &DemandedElts, const SimplifyQuery &Q,
3270 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3271 bool NUW, unsigned Depth) {
3272 // If X and Y are non-zero then so is X * Y as long as the multiplication
3273 // does not overflow.
3274 if (NSW || NUW)
3275 return isKnownNonZero(V: X, DemandedElts, Q, Depth) &&
3276 isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3277
3278 // If either X or Y is odd, then if the other is non-zero the result can't
3279 // be zero.
3280 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3281 if (XKnown.One[0])
3282 return isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3283
3284 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3285 if (YKnown.One[0])
3286 return XKnown.isNonZero() || isKnownNonZero(V: X, DemandedElts, Q, Depth);
3287
3288 // If there exists any subset of X (sX) and subset of Y (sY) s.t sX * sY is
3289 // non-zero, then X * Y is non-zero. We can find sX and sY by just taking
3290 // the lowest known One of X and Y. If they are non-zero, the result
3291 // must be non-zero. We can check if LSB(X) * LSB(Y) != 0 by doing
3292 // X.CountLeadingZeros + Y.CountLeadingZeros < BitWidth.
3293 return (XKnown.countMaxTrailingZeros() + YKnown.countMaxTrailingZeros()) <
3294 BitWidth;
3295}
3296
3297static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
3298 const SimplifyQuery &Q, const KnownBits &KnownVal,
3299 unsigned Depth) {
3300 auto ShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3301 switch (I->getOpcode()) {
3302 case Instruction::Shl:
3303 return Lhs.shl(ShiftAmt: Rhs);
3304 case Instruction::LShr:
3305 return Lhs.lshr(ShiftAmt: Rhs);
3306 case Instruction::AShr:
3307 return Lhs.ashr(ShiftAmt: Rhs);
3308 default:
3309 llvm_unreachable("Unknown Shift Opcode");
3310 }
3311 };
3312
3313 auto InvShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3314 switch (I->getOpcode()) {
3315 case Instruction::Shl:
3316 return Lhs.lshr(ShiftAmt: Rhs);
3317 case Instruction::LShr:
3318 case Instruction::AShr:
3319 return Lhs.shl(ShiftAmt: Rhs);
3320 default:
3321 llvm_unreachable("Unknown Shift Opcode");
3322 }
3323 };
3324
3325 if (KnownVal.isUnknown())
3326 return false;
3327
3328 KnownBits KnownCnt =
3329 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3330 APInt MaxShift = KnownCnt.getMaxValue();
3331 unsigned NumBits = KnownVal.getBitWidth();
3332 if (MaxShift.uge(RHS: NumBits))
3333 return false;
3334
3335 if (!ShiftOp(KnownVal.One, MaxShift).isZero())
3336 return true;
3337
3338 // If all of the bits shifted out are known to be zero, and Val is known
3339 // non-zero then at least one non-zero bit must remain.
3340 if (InvShiftOp(KnownVal.Zero, NumBits - MaxShift)
3341 .eq(RHS: InvShiftOp(APInt::getAllOnes(numBits: NumBits), NumBits - MaxShift)) &&
3342 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth))
3343 return true;
3344
3345 return false;
3346}
3347
3348static bool isKnownNonZeroFromOperator(const Operator *I,
3349 const APInt &DemandedElts,
3350 const SimplifyQuery &Q, unsigned Depth) {
3351 unsigned BitWidth = getBitWidth(Ty: I->getType()->getScalarType(), DL: Q.DL);
3352 switch (I->getOpcode()) {
3353 case Instruction::Alloca:
3354 // Alloca never returns null, malloc might.
3355 return I->getType()->getPointerAddressSpace() == 0;
3356 case Instruction::GetElementPtr:
3357 if (I->getType()->isPointerTy())
3358 return isGEPKnownNonNull(GEP: cast<GEPOperator>(Val: I), Q, Depth);
3359 break;
3360 case Instruction::BitCast: {
3361 // We need to be a bit careful here. We can only peek through the bitcast
3362 // if the scalar size of elements in the operand are smaller than and a
3363 // multiple of the size they are casting too. Take three cases:
3364 //
3365 // 1) Unsafe:
3366 // bitcast <2 x i16> %NonZero to <4 x i8>
3367 //
3368 // %NonZero can have 2 non-zero i16 elements, but isKnownNonZero on a
3369 // <4 x i8> requires that all 4 i8 elements be non-zero which isn't
3370 // guranteed (imagine just sign bit set in the 2 i16 elements).
3371 //
3372 // 2) Unsafe:
3373 // bitcast <4 x i3> %NonZero to <3 x i4>
3374 //
3375 // Even though the scalar size of the src (`i3`) is smaller than the
3376 // scalar size of the dst `i4`, because `i3` is not a multiple of `i4`
3377 // its possible for the `3 x i4` elements to be zero because there are
3378 // some elements in the destination that don't contain any full src
3379 // element.
3380 //
3381 // 3) Safe:
3382 // bitcast <4 x i8> %NonZero to <2 x i16>
3383 //
3384 // This is always safe as non-zero in the 4 i8 elements implies
3385 // non-zero in the combination of any two adjacent ones. Since i8 is a
3386 // multiple of i16, each i16 is guranteed to have 2 full i8 elements.
3387 // This all implies the 2 i16 elements are non-zero.
3388 Type *FromTy = I->getOperand(i: 0)->getType();
3389 if ((FromTy->isIntOrIntVectorTy() || FromTy->isPtrOrPtrVectorTy()) &&
3390 (BitWidth % getBitWidth(Ty: FromTy->getScalarType(), DL: Q.DL)) == 0)
3391 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3392 } break;
3393 case Instruction::IntToPtr:
3394 // Note that we have to take special care to avoid looking through
3395 // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
3396 // as casts that can alter the value, e.g., AddrSpaceCasts.
3397 if (!isa<ScalableVectorType>(Val: I->getType()) &&
3398 Q.DL.getTypeSizeInBits(Ty: I->getOperand(i: 0)->getType()).getFixedValue() <=
3399 Q.DL.getTypeSizeInBits(Ty: I->getType()).getFixedValue())
3400 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3401 break;
3402 case Instruction::PtrToAddr:
3403 // isKnownNonZero() for pointers refers to the address bits being non-zero,
3404 // so we can directly forward.
3405 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3406 case Instruction::PtrToInt:
3407 // For inttoptr, make sure the result size is >= the address size. If the
3408 // address is non-zero, any larger value is also non-zero.
3409 if (Q.DL.getAddressSizeInBits(Ty: I->getOperand(i: 0)->getType()) <=
3410 I->getType()->getScalarSizeInBits())
3411 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3412 break;
3413 case Instruction::Trunc:
3414 // nuw/nsw trunc preserves zero/non-zero status of input.
3415 if (auto *TI = dyn_cast<TruncInst>(Val: I))
3416 if (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap())
3417 return isKnownNonZero(V: TI->getOperand(i_nocapture: 0), DemandedElts, Q, Depth);
3418 break;
3419
3420 // Iff x - y != 0, then x ^ y != 0
3421 // Therefore we can do the same exact checks
3422 case Instruction::Xor:
3423 case Instruction::Sub:
3424 return isNonZeroSub(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3425 Y: I->getOperand(i: 1), Depth);
3426 case Instruction::Or:
3427 // (X | (X != 0)) is non zero
3428 if (matchOpWithOpEqZero(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1)))
3429 return true;
3430 // X | Y != 0 if X != Y.
3431 if (isKnownNonEqual(V1: I->getOperand(i: 0), V2: I->getOperand(i: 1), DemandedElts, Q,
3432 Depth))
3433 return true;
3434 // X | Y != 0 if X != 0 or Y != 0.
3435 return isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth) ||
3436 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3437 case Instruction::SExt:
3438 case Instruction::ZExt:
3439 // ext X != 0 if X != 0.
3440 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3441
3442 case Instruction::Shl: {
3443 // shl nsw/nuw can't remove any non-zero bits.
3444 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3445 if (Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO))
3446 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3447
3448 // shl X, Y != 0 if X is odd. Note that the value of the shift is undefined
3449 // if the lowest bit is shifted off the end.
3450 KnownBits Known(BitWidth);
3451 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth);
3452 if (Known.One[0])
3453 return true;
3454
3455 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3456 }
3457 case Instruction::LShr:
3458 case Instruction::AShr: {
3459 // shr exact can only shift out zero bits.
3460 const PossiblyExactOperator *BO = cast<PossiblyExactOperator>(Val: I);
3461 if (BO->isExact())
3462 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3463
3464 // shr X, Y != 0 if X is negative. Note that the value of the shift is not
3465 // defined if the sign bit is shifted off the end.
3466 KnownBits Known =
3467 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3468 if (Known.isNegative())
3469 return true;
3470
3471 // shr (add nuw A, B), C is non-zero if A or B has a known-one bit at
3472 // position >= C, because the sum >= max(A, B).
3473 Value *A, *B;
3474 const APInt *C;
3475 if (Depth + 1 < MaxAnalysisRecursionDepth &&
3476 match(V: I->getOperand(i: 0), P: m_NUWAdd(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3477 match(V: I->getOperand(i: 1), P: m_APInt(Res&: C)) && C->ult(RHS: BitWidth)) {
3478 KnownBits KnownA = computeKnownBits(V: A, DemandedElts, Q, Depth: Depth + 1);
3479 if (!KnownA.One.lshr(ShiftAmt: *C).isZero())
3480 return true;
3481 KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
3482 if (!KnownB.One.lshr(ShiftAmt: *C).isZero())
3483 return true;
3484 }
3485
3486 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3487 }
3488 case Instruction::UDiv:
3489 case Instruction::SDiv: {
3490 // X / Y
3491 // div exact can only produce a zero if the dividend is zero.
3492 if (cast<PossiblyExactOperator>(Val: I)->isExact())
3493 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3494
3495 KnownBits XKnown =
3496 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3497 // If X is fully unknown we won't be able to figure anything out so don't
3498 // both computing knownbits for Y.
3499 if (XKnown.isUnknown())
3500 return false;
3501
3502 KnownBits YKnown =
3503 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3504 if (I->getOpcode() == Instruction::SDiv) {
3505 // For signed division need to compare abs value of the operands.
3506 XKnown = XKnown.abs(/*IntMinIsPoison*/ false);
3507 YKnown = YKnown.abs(/*IntMinIsPoison*/ false);
3508 }
3509 // If X u>= Y then div is non zero (0/0 is UB).
3510 std::optional<bool> XUgeY = KnownBits::uge(LHS: XKnown, RHS: YKnown);
3511 // If X is total unknown or X u< Y we won't be able to prove non-zero
3512 // with compute known bits so just return early.
3513 return XUgeY && *XUgeY;
3514 }
3515 case Instruction::Add: {
3516 // X + Y.
3517
3518 // If Add has nuw wrap flag, then if either X or Y is non-zero the result is
3519 // non-zero.
3520 auto *BO = cast<OverflowingBinaryOperator>(Val: I);
3521 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3522 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3523 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3524 }
3525 case Instruction::Mul: {
3526 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3527 return isNonZeroMul(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3528 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3529 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3530 }
3531 case Instruction::Select: {
3532 // (C ? X : Y) != 0 if X != 0 and Y != 0.
3533
3534 // First check if the arm is non-zero using `isKnownNonZero`. If that fails,
3535 // then see if the select condition implies the arm is non-zero. For example
3536 // (X != 0 ? X : Y), we know the true arm is non-zero as the `X` "return" is
3537 // dominated by `X != 0`.
3538 auto SelectArmIsNonZero = [&](bool IsTrueArm) {
3539 Value *Op;
3540 Op = IsTrueArm ? I->getOperand(i: 1) : I->getOperand(i: 2);
3541 // Op is trivially non-zero.
3542 if (isKnownNonZero(V: Op, DemandedElts, Q, Depth))
3543 return true;
3544
3545 // The condition of the select dominates the true/false arm. Check if the
3546 // condition implies that a given arm is non-zero.
3547 Value *X;
3548 CmpPredicate Pred;
3549 if (!match(V: I->getOperand(i: 0), P: m_c_ICmp(Pred, L: m_Specific(V: Op), R: m_Value(V&: X))))
3550 return false;
3551
3552 if (!IsTrueArm)
3553 Pred = ICmpInst::getInversePredicate(pred: Pred);
3554
3555 return cmpExcludesZero(Pred, RHS: X);
3556 };
3557
3558 if (SelectArmIsNonZero(/* IsTrueArm */ true) &&
3559 SelectArmIsNonZero(/* IsTrueArm */ false))
3560 return true;
3561 break;
3562 }
3563 case Instruction::PHI: {
3564 auto *PN = cast<PHINode>(Val: I);
3565 if (Q.IIQ.UseInstrInfo && isNonZeroRecurrence(PN))
3566 return true;
3567
3568 // Check if all incoming values are non-zero using recursion.
3569 SimplifyQuery RecQ = Q.getWithoutCondContext();
3570 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
3571 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
3572 if (U.get() == PN)
3573 return true;
3574 RecQ.CxtI = PN->getIncomingBlock(U)->getTerminator();
3575 // Check if the branch on the phi excludes zero.
3576 CmpPredicate Pred;
3577 Value *X;
3578 BasicBlock *TrueSucc, *FalseSucc;
3579 if (match(V: RecQ.CxtI,
3580 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: U.get()), R: m_Value(V&: X)),
3581 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
3582 // Check for cases of duplicate successors.
3583 if ((TrueSucc == PN->getParent()) != (FalseSucc == PN->getParent())) {
3584 // If we're using the false successor, invert the predicate.
3585 if (FalseSucc == PN->getParent())
3586 Pred = CmpInst::getInversePredicate(pred: Pred);
3587 if (cmpExcludesZero(Pred, RHS: X))
3588 return true;
3589 }
3590 }
3591 // Finally recurse on the edge and check it directly.
3592 return isKnownNonZero(V: U.get(), DemandedElts, Q: RecQ, Depth: NewDepth);
3593 });
3594 }
3595 case Instruction::InsertElement: {
3596 if (isa<ScalableVectorType>(Val: I->getType()))
3597 break;
3598
3599 const Value *Vec = I->getOperand(i: 0);
3600 const Value *Elt = I->getOperand(i: 1);
3601 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
3602
3603 unsigned NumElts = DemandedElts.getBitWidth();
3604 APInt DemandedVecElts = DemandedElts;
3605 bool SkipElt = false;
3606 // If we know the index we are inserting too, clear it from Vec check.
3607 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
3608 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
3609 SkipElt = !DemandedElts[CIdx->getZExtValue()];
3610 }
3611
3612 // Result is zero if Elt is non-zero and rest of the demanded elts in Vec
3613 // are non-zero.
3614 return (SkipElt || isKnownNonZero(V: Elt, Q, Depth)) &&
3615 (DemandedVecElts.isZero() ||
3616 isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth));
3617 }
3618 case Instruction::ExtractElement:
3619 if (const auto *EEI = dyn_cast<ExtractElementInst>(Val: I)) {
3620 const Value *Vec = EEI->getVectorOperand();
3621 const Value *Idx = EEI->getIndexOperand();
3622 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
3623 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
3624 unsigned NumElts = VecTy->getNumElements();
3625 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
3626 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
3627 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
3628 return isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth);
3629 }
3630 }
3631 break;
3632 case Instruction::ShuffleVector: {
3633 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
3634 if (!Shuf)
3635 break;
3636 APInt DemandedLHS, DemandedRHS;
3637 // For undef elements, we don't know anything about the common state of
3638 // the shuffle result.
3639 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
3640 break;
3641 // If demanded elements for both vecs are non-zero, the shuffle is non-zero.
3642 return (DemandedRHS.isZero() ||
3643 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 1), DemandedElts: DemandedRHS, Q, Depth)) &&
3644 (DemandedLHS.isZero() ||
3645 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 0), DemandedElts: DemandedLHS, Q, Depth));
3646 }
3647 case Instruction::Freeze:
3648 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth) &&
3649 isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
3650 Depth);
3651 case Instruction::Load: {
3652 auto *LI = cast<LoadInst>(Val: I);
3653 // A Load tagged with nonnull or dereferenceable with null pointer undefined
3654 // is never null.
3655 if (auto *PtrT = dyn_cast<PointerType>(Val: I->getType())) {
3656 if (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_nonnull) ||
3657 (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_dereferenceable) &&
3658 !NullPointerIsDefined(F: LI->getFunction(), AS: PtrT->getAddressSpace())))
3659 return true;
3660 } else if (MDNode *Ranges = Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_range)) {
3661 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3662 }
3663
3664 // No need to fall through to computeKnownBits as range metadata is already
3665 // handled in isKnownNonZero.
3666 return false;
3667 }
3668 case Instruction::ExtractValue: {
3669 const WithOverflowInst *WO;
3670 if (match(V: I, P: m_ExtractValue<0>(V: m_WithOverflowInst(I&: WO)))) {
3671 switch (WO->getBinaryOp()) {
3672 default:
3673 break;
3674 case Instruction::Add:
3675 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3676 Y: WO->getArgOperand(i: 1),
3677 /*NSW=*/false,
3678 /*NUW=*/false, Depth);
3679 case Instruction::Sub:
3680 return isNonZeroSub(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3681 Y: WO->getArgOperand(i: 1), Depth);
3682 case Instruction::Mul:
3683 return isNonZeroMul(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3684 Y: WO->getArgOperand(i: 1),
3685 /*NSW=*/false, /*NUW=*/false, Depth);
3686 break;
3687 }
3688 }
3689 break;
3690 }
3691 case Instruction::Call:
3692 case Instruction::Invoke: {
3693 const auto *Call = cast<CallBase>(Val: I);
3694 if (I->getType()->isPointerTy()) {
3695 if (Call->isReturnNonNull())
3696 return true;
3697 if (const auto *RP = getArgumentAliasingToReturnedPointer(
3698 Call, /*MustPreserveOffset=*/true))
3699 return isKnownNonZero(V: RP, Q, Depth);
3700 } else {
3701 if (MDNode *Ranges = Q.IIQ.getMetadata(I: Call, KindID: LLVMContext::MD_range))
3702 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3703 if (std::optional<ConstantRange> Range = Call->getRange()) {
3704 const APInt ZeroValue(Range->getBitWidth(), 0);
3705 if (!Range->contains(Val: ZeroValue))
3706 return true;
3707 }
3708 if (const Value *RV = Call->getReturnedArgOperand())
3709 if (RV->getType() == I->getType() && isKnownNonZero(V: RV, Q, Depth))
3710 return true;
3711 }
3712
3713 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3714 switch (II->getIntrinsicID()) {
3715 case Intrinsic::sshl_sat:
3716 case Intrinsic::ushl_sat:
3717 case Intrinsic::abs:
3718 case Intrinsic::bitreverse:
3719 case Intrinsic::bswap:
3720 case Intrinsic::ctpop:
3721 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3722 // NB: We don't do usub_sat here as in any case we can prove its
3723 // non-zero, we will fold it to `sub nuw` in InstCombine.
3724 case Intrinsic::ssub_sat:
3725 // For most types, if x != y then ssub.sat x, y != 0. But
3726 // ssub.sat.i1 0, -1 = 0, because 1 saturates to 0. This means
3727 // isNonZeroSub will do the wrong thing for ssub.sat.i1.
3728 if (BitWidth == 1)
3729 return false;
3730 return isNonZeroSub(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3731 Y: II->getArgOperand(i: 1), Depth);
3732 case Intrinsic::sadd_sat:
3733 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3734 Y: II->getArgOperand(i: 1),
3735 /*NSW=*/true, /* NUW=*/false, Depth);
3736 // Vec reverse preserves zero/non-zero status from input vec.
3737 case Intrinsic::vector_reverse:
3738 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
3739 Q, Depth);
3740 // umin/smin/smax/smin/or of all non-zero elements is always non-zero.
3741 case Intrinsic::vector_reduce_or:
3742 case Intrinsic::vector_reduce_umax:
3743 case Intrinsic::vector_reduce_umin:
3744 case Intrinsic::vector_reduce_smax:
3745 case Intrinsic::vector_reduce_smin:
3746 return isKnownNonZero(V: II->getArgOperand(i: 0), Q, Depth);
3747 case Intrinsic::umax:
3748 case Intrinsic::uadd_sat:
3749 // umax(X, (X != 0)) is non zero
3750 // X +usat (X != 0) is non zero
3751 if (matchOpWithOpEqZero(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1)))
3752 return true;
3753
3754 return isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth) ||
3755 isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3756 case Intrinsic::smax: {
3757 // If either arg is strictly positive the result is non-zero. Otherwise
3758 // the result is non-zero if both ops are non-zero.
3759 auto IsNonZero = [&](Value *Op, std::optional<bool> &OpNonZero,
3760 const KnownBits &OpKnown) {
3761 if (!OpNonZero.has_value())
3762 OpNonZero = OpKnown.isNonZero() ||
3763 isKnownNonZero(V: Op, DemandedElts, Q, Depth);
3764 return *OpNonZero;
3765 };
3766 // Avoid re-computing isKnownNonZero.
3767 std::optional<bool> Op0NonZero, Op1NonZero;
3768 KnownBits Op1Known =
3769 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3770 if (Op1Known.isNonNegative() &&
3771 IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known))
3772 return true;
3773 KnownBits Op0Known =
3774 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3775 if (Op0Known.isNonNegative() &&
3776 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known))
3777 return true;
3778 return IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known) &&
3779 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known);
3780 }
3781 case Intrinsic::smin: {
3782 // If either arg is negative the result is non-zero. Otherwise
3783 // the result is non-zero if both ops are non-zero.
3784 KnownBits Op1Known =
3785 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3786 if (Op1Known.isNegative())
3787 return true;
3788 KnownBits Op0Known =
3789 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3790 if (Op0Known.isNegative())
3791 return true;
3792
3793 if (Op1Known.isNonZero() && Op0Known.isNonZero())
3794 return true;
3795 }
3796 [[fallthrough]];
3797 case Intrinsic::umin:
3798 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth) &&
3799 isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3800 case Intrinsic::cttz:
3801 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3802 .Zero[0];
3803 case Intrinsic::ctlz:
3804 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3805 .isNonNegative();
3806 case Intrinsic::fshr:
3807 case Intrinsic::fshl:
3808 // If Op0 == Op1, this is a rotate. rotate(x, y) != 0 iff x != 0.
3809 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
3810 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3811 break;
3812 case Intrinsic::vscale:
3813 return true;
3814 case Intrinsic::experimental_get_vector_length:
3815 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3816 default:
3817 break;
3818 }
3819 break;
3820 }
3821
3822 return false;
3823 }
3824 }
3825
3826 KnownBits Known(BitWidth);
3827 computeKnownBits(V: I, DemandedElts, Known, Q, Depth);
3828 return Known.One != 0;
3829}
3830
3831/// Return true if the given value is known to be non-zero when defined. For
3832/// vectors, return true if every demanded element is known to be non-zero when
3833/// defined. For pointers, if the context instruction and dominator tree are
3834/// specified, perform context-sensitive analysis and return true if the
3835/// pointer couldn't possibly be null at the specified instruction.
3836/// Supports values with integer or pointer type and vectors of integers.
3837bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
3838 const SimplifyQuery &Q, unsigned Depth) {
3839 Type *Ty = V->getType();
3840
3841#ifndef NDEBUG
3842 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
3843
3844 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
3845 assert(
3846 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
3847 "DemandedElt width should equal the fixed vector number of elements");
3848 } else {
3849 assert(DemandedElts == APInt(1, 1) &&
3850 "DemandedElt width should be 1 for scalars");
3851 }
3852#endif
3853
3854 if (auto *C = dyn_cast<Constant>(Val: V)) {
3855 if (C->isNullValue())
3856 return false;
3857 if (isa<ConstantInt>(Val: C))
3858 // Must be non-zero due to null test above.
3859 return true;
3860
3861 // For constant vectors, check that all elements are poison or known
3862 // non-zero to determine that the whole vector is known non-zero.
3863 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Ty)) {
3864 for (unsigned i = 0, e = VecTy->getNumElements(); i != e; ++i) {
3865 if (!DemandedElts[i])
3866 continue;
3867 Constant *Elt = C->getAggregateElement(Elt: i);
3868 if (!Elt || Elt->isNullValue())
3869 return false;
3870 if (!isa<PoisonValue>(Val: Elt) && !isa<ConstantInt>(Val: Elt))
3871 return false;
3872 }
3873 return true;
3874 }
3875
3876 // Constant ptrauth can be null, iff the base pointer can be.
3877 if (auto *CPA = dyn_cast<ConstantPtrAuth>(Val: V))
3878 return isKnownNonZero(V: CPA->getPointer(), DemandedElts, Q, Depth);
3879
3880 // A global variable in address space 0 is non null unless extern weak
3881 // or an absolute symbol reference. Other address spaces may have null as a
3882 // valid address for a global, so we can't assume anything.
3883 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
3884 if (!GV->isAbsoluteSymbolRef() && !GV->hasExternalWeakLinkage() &&
3885 GV->getType()->getAddressSpace() == 0)
3886 return true;
3887 }
3888
3889 // For constant expressions, fall through to the Operator code below.
3890 if (!isa<ConstantExpr>(Val: V))
3891 return false;
3892 }
3893
3894 if (const auto *A = dyn_cast<Argument>(Val: V))
3895 if (std::optional<ConstantRange> Range = A->getRange()) {
3896 const APInt ZeroValue(Range->getBitWidth(), 0);
3897 if (!Range->contains(Val: ZeroValue))
3898 return true;
3899 }
3900
3901 if (!isa<Constant>(Val: V) && isKnownNonZeroFromAssume(V, Q))
3902 return true;
3903
3904 // Some of the tests below are recursive, so bail out if we hit the limit.
3905 if (Depth++ >= MaxAnalysisRecursionDepth)
3906 return false;
3907
3908 // Check for pointer simplifications.
3909
3910 if (PointerType *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
3911 // A byval, inalloca may not be null in a non-default addres space. A
3912 // nonnull argument is assumed never 0.
3913 if (const Argument *A = dyn_cast<Argument>(Val: V)) {
3914 if (((A->hasPassPointeeByValueCopyAttr() &&
3915 !NullPointerIsDefined(F: A->getParent(), AS: PtrTy->getAddressSpace())) ||
3916 A->hasNonNullAttr()))
3917 return true;
3918 }
3919 }
3920
3921 if (const auto *I = dyn_cast<Operator>(Val: V))
3922 if (isKnownNonZeroFromOperator(I, DemandedElts, Q, Depth))
3923 return true;
3924
3925 if (!isa<Constant>(Val: V) &&
3926 isKnownNonNullFromDominatingCondition(V, CtxI: Q.CxtI, DT: Q.DT))
3927 return true;
3928
3929 if (const Value *Stripped = stripNullTest(V))
3930 return isKnownNonZero(V: Stripped, DemandedElts, Q, Depth);
3931
3932 return false;
3933}
3934
3935bool llvm::isKnownNonZero(const Value *V, const SimplifyQuery &Q,
3936 unsigned Depth) {
3937 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
3938 APInt DemandedElts =
3939 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
3940 return ::isKnownNonZero(V, DemandedElts, Q, Depth);
3941}
3942
3943/// If the pair of operators are the same invertible function, return the
3944/// the operands of the function corresponding to each input. Otherwise,
3945/// return std::nullopt. An invertible function is one that is 1-to-1 and maps
3946/// every input value to exactly one output value. This is equivalent to
3947/// saying that Op1 and Op2 are equal exactly when the specified pair of
3948/// operands are equal, (except that Op1 and Op2 may be poison more often.)
3949static std::optional<std::pair<Value*, Value*>>
3950getInvertibleOperands(const Operator *Op1,
3951 const Operator *Op2) {
3952 if (Op1->getOpcode() != Op2->getOpcode())
3953 return std::nullopt;
3954
3955 auto getOperands = [&](unsigned OpNum) -> auto {
3956 return std::make_pair(x: Op1->getOperand(i: OpNum), y: Op2->getOperand(i: OpNum));
3957 };
3958
3959 switch (Op1->getOpcode()) {
3960 default:
3961 break;
3962 case Instruction::Or:
3963 if (!cast<PossiblyDisjointInst>(Val: Op1)->isDisjoint() ||
3964 !cast<PossiblyDisjointInst>(Val: Op2)->isDisjoint())
3965 break;
3966 [[fallthrough]];
3967 case Instruction::Xor:
3968 case Instruction::Add: {
3969 Value *Other;
3970 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 0)), R: m_Value(V&: Other))))
3971 return std::make_pair(x: Op1->getOperand(i: 1), y&: Other);
3972 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 1)), R: m_Value(V&: Other))))
3973 return std::make_pair(x: Op1->getOperand(i: 0), y&: Other);
3974 break;
3975 }
3976 case Instruction::Sub:
3977 if (Op1->getOperand(i: 0) == Op2->getOperand(i: 0))
3978 return getOperands(1);
3979 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3980 return getOperands(0);
3981 break;
3982 case Instruction::Mul: {
3983 // invertible if A * B == (A * B) mod 2^N where A, and B are integers
3984 // and N is the bitwdith. The nsw case is non-obvious, but proven by
3985 // alive2: https://alive2.llvm.org/ce/z/Z6D5qK
3986 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
3987 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
3988 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
3989 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
3990 break;
3991
3992 // Assume operand order has been canonicalized
3993 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1) &&
3994 isa<ConstantInt>(Val: Op1->getOperand(i: 1)) &&
3995 !cast<ConstantInt>(Val: Op1->getOperand(i: 1))->isZero())
3996 return getOperands(0);
3997 break;
3998 }
3999 case Instruction::Shl: {
4000 // Same as multiplies, with the difference that we don't need to check
4001 // for a non-zero multiply. Shifts always multiply by non-zero.
4002 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
4003 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
4004 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
4005 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
4006 break;
4007
4008 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
4009 return getOperands(0);
4010 break;
4011 }
4012 case Instruction::AShr:
4013 case Instruction::LShr: {
4014 auto *PEO1 = cast<PossiblyExactOperator>(Val: Op1);
4015 auto *PEO2 = cast<PossiblyExactOperator>(Val: Op2);
4016 if (!PEO1->isExact() || !PEO2->isExact())
4017 break;
4018
4019 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
4020 return getOperands(0);
4021 break;
4022 }
4023 case Instruction::SExt:
4024 case Instruction::ZExt:
4025 if (Op1->getOperand(i: 0)->getType() == Op2->getOperand(i: 0)->getType())
4026 return getOperands(0);
4027 break;
4028 case Instruction::PHI: {
4029 const PHINode *PN1 = cast<PHINode>(Val: Op1);
4030 const PHINode *PN2 = cast<PHINode>(Val: Op2);
4031
4032 // If PN1 and PN2 are both recurrences, can we prove the entire recurrences
4033 // are a single invertible function of the start values? Note that repeated
4034 // application of an invertible function is also invertible
4035 BinaryOperator *BO1 = nullptr;
4036 Value *Start1 = nullptr, *Step1 = nullptr;
4037 BinaryOperator *BO2 = nullptr;
4038 Value *Start2 = nullptr, *Step2 = nullptr;
4039 if (PN1->getParent() != PN2->getParent() ||
4040 !matchSimpleRecurrence(P: PN1, BO&: BO1, Start&: Start1, Step&: Step1) ||
4041 !matchSimpleRecurrence(P: PN2, BO&: BO2, Start&: Start2, Step&: Step2))
4042 break;
4043
4044 auto Values = getInvertibleOperands(Op1: cast<Operator>(Val: BO1),
4045 Op2: cast<Operator>(Val: BO2));
4046 if (!Values)
4047 break;
4048
4049 // We have to be careful of mutually defined recurrences here. Ex:
4050 // * X_i = X_(i-1) OP Y_(i-1), and Y_i = X_(i-1) OP V
4051 // * X_i = Y_i = X_(i-1) OP Y_(i-1)
4052 // The invertibility of these is complicated, and not worth reasoning
4053 // about (yet?).
4054 if (Values->first != PN1 || Values->second != PN2)
4055 break;
4056
4057 return std::make_pair(x&: Start1, y&: Start2);
4058 }
4059 }
4060 return std::nullopt;
4061}
4062
4063/// Return true if V1 == (binop V2, X), where X is known non-zero.
4064/// Only handle a small subset of binops where (binop V2, X) with non-zero X
4065/// implies V2 != V1.
4066static bool isModifyingBinopOfNonZero(const Value *V1, const Value *V2,
4067 const APInt &DemandedElts,
4068 const SimplifyQuery &Q, unsigned Depth) {
4069 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V1);
4070 if (!BO)
4071 return false;
4072 switch (BO->getOpcode()) {
4073 default:
4074 break;
4075 case Instruction::Or:
4076 if (!cast<PossiblyDisjointInst>(Val: V1)->isDisjoint())
4077 break;
4078 [[fallthrough]];
4079 case Instruction::Xor:
4080 case Instruction::Add:
4081 Value *Op = nullptr;
4082 if (V2 == BO->getOperand(i_nocapture: 0))
4083 Op = BO->getOperand(i_nocapture: 1);
4084 else if (V2 == BO->getOperand(i_nocapture: 1))
4085 Op = BO->getOperand(i_nocapture: 0);
4086 else
4087 return false;
4088 return isKnownNonZero(V: Op, DemandedElts, Q, Depth: Depth + 1);
4089 }
4090 return false;
4091}
4092
4093/// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
4094/// the multiplication is nuw or nsw.
4095static bool isNonEqualMul(const Value *V1, const Value *V2,
4096 const APInt &DemandedElts, const SimplifyQuery &Q,
4097 unsigned Depth) {
4098 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
4099 const APInt *C;
4100 return match(V: OBO, P: m_Mul(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
4101 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
4102 !C->isZero() && !C->isOne() &&
4103 isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
4104 }
4105 return false;
4106}
4107
4108/// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
4109/// the shift is nuw or nsw.
4110static bool isNonEqualShl(const Value *V1, const Value *V2,
4111 const APInt &DemandedElts, const SimplifyQuery &Q,
4112 unsigned Depth) {
4113 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
4114 const APInt *C;
4115 return match(V: OBO, P: m_Shl(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
4116 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
4117 !C->isZero() && isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
4118 }
4119 return false;
4120}
4121
4122static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
4123 const APInt &DemandedElts, const SimplifyQuery &Q,
4124 unsigned Depth) {
4125 // Check two PHIs are in same block.
4126 if (PN1->getParent() != PN2->getParent())
4127 return false;
4128
4129 SmallPtrSet<const BasicBlock *, 8> VisitedBBs;
4130 bool UsedFullRecursion = false;
4131 for (const BasicBlock *IncomBB : PN1->blocks()) {
4132 if (!VisitedBBs.insert(Ptr: IncomBB).second)
4133 continue; // Don't reprocess blocks that we have dealt with already.
4134 const Value *IV1 = PN1->getIncomingValueForBlock(BB: IncomBB);
4135 const Value *IV2 = PN2->getIncomingValueForBlock(BB: IncomBB);
4136 const APInt *C1, *C2;
4137 if (match(V: IV1, P: m_APInt(Res&: C1)) && match(V: IV2, P: m_APInt(Res&: C2)) && *C1 != *C2)
4138 continue;
4139
4140 // Only one pair of phi operands is allowed for full recursion.
4141 if (UsedFullRecursion)
4142 return false;
4143
4144 SimplifyQuery RecQ = Q.getWithoutCondContext();
4145 RecQ.CxtI = IncomBB->getTerminator();
4146 if (!isKnownNonEqual(V1: IV1, V2: IV2, DemandedElts, Q: RecQ, Depth: Depth + 1))
4147 return false;
4148 UsedFullRecursion = true;
4149 }
4150 return true;
4151}
4152
4153static bool isNonEqualSelect(const Value *V1, const Value *V2,
4154 const APInt &DemandedElts, const SimplifyQuery &Q,
4155 unsigned Depth) {
4156 const SelectInst *SI1 = dyn_cast<SelectInst>(Val: V1);
4157 if (!SI1)
4158 return false;
4159
4160 if (const SelectInst *SI2 = dyn_cast<SelectInst>(Val: V2)) {
4161 const Value *Cond1 = SI1->getCondition();
4162 const Value *Cond2 = SI2->getCondition();
4163 if (Cond1 == Cond2)
4164 return isKnownNonEqual(V1: SI1->getTrueValue(), V2: SI2->getTrueValue(),
4165 DemandedElts, Q, Depth: Depth + 1) &&
4166 isKnownNonEqual(V1: SI1->getFalseValue(), V2: SI2->getFalseValue(),
4167 DemandedElts, Q, Depth: Depth + 1);
4168 }
4169 return isKnownNonEqual(V1: SI1->getTrueValue(), V2, DemandedElts, Q, Depth: Depth + 1) &&
4170 isKnownNonEqual(V1: SI1->getFalseValue(), V2, DemandedElts, Q, Depth: Depth + 1);
4171}
4172
4173// Check to see if A is both a GEP and is the incoming value for a PHI in the
4174// loop, and B is either a ptr or another GEP. If the PHI has 2 incoming values,
4175// one of them being the recursive GEP A and the other a ptr at same base and at
4176// the same/higher offset than B we are only incrementing the pointer further in
4177// loop if offset of recursive GEP is greater than 0.
4178static bool isNonEqualPointersWithRecursiveGEP(const Value *A, const Value *B,
4179 const SimplifyQuery &Q) {
4180 if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
4181 return false;
4182
4183 auto *GEPA = dyn_cast<GEPOperator>(Val: A);
4184 if (!GEPA || GEPA->getNumIndices() != 1 || !isa<Constant>(Val: GEPA->idx_begin()))
4185 return false;
4186
4187 // Handle 2 incoming PHI values with one being a recursive GEP.
4188 auto *PN = dyn_cast<PHINode>(Val: GEPA->getPointerOperand());
4189 if (!PN || PN->getNumIncomingValues() != 2)
4190 return false;
4191
4192 // Search for the recursive GEP as an incoming operand, and record that as
4193 // Step.
4194 Value *Start = nullptr;
4195 Value *Step = const_cast<Value *>(A);
4196 if (PN->getIncomingValue(i: 0) == Step)
4197 Start = PN->getIncomingValue(i: 1);
4198 else if (PN->getIncomingValue(i: 1) == Step)
4199 Start = PN->getIncomingValue(i: 0);
4200 else
4201 return false;
4202
4203 // Other incoming node base should match the B base.
4204 // StartOffset >= OffsetB && StepOffset > 0?
4205 // StartOffset <= OffsetB && StepOffset < 0?
4206 // Is non-equal if above are true.
4207 // We use stripAndAccumulateInBoundsConstantOffsets to restrict the
4208 // optimisation to inbounds GEPs only.
4209 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: Start->getType());
4210 APInt StartOffset(IndexWidth, 0);
4211 Start = Start->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StartOffset);
4212 APInt StepOffset(IndexWidth, 0);
4213 Step = Step->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StepOffset);
4214
4215 // Check if Base Pointer of Step matches the PHI.
4216 if (Step != PN)
4217 return false;
4218 APInt OffsetB(IndexWidth, 0);
4219 B = B->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: OffsetB);
4220 return Start == B &&
4221 ((StartOffset.sge(RHS: OffsetB) && StepOffset.isStrictlyPositive()) ||
4222 (StartOffset.sle(RHS: OffsetB) && StepOffset.isNegative()));
4223}
4224
4225static bool isKnownNonEqualFromContext(const Value *V1, const Value *V2,
4226 const SimplifyQuery &Q, unsigned Depth) {
4227 if (!Q.CxtI)
4228 return false;
4229
4230 // Try to infer NonEqual based on information from dominating conditions.
4231 if (Q.DC && Q.DT) {
4232 auto IsKnownNonEqualFromDominatingCondition = [&](const Value *V) {
4233 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4234 Value *Cond = BI->getCondition();
4235 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4236 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()) &&
4237 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4238 /*LHSIsTrue=*/true, Depth)
4239 .value_or(u: false))
4240 return true;
4241
4242 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4243 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()) &&
4244 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4245 /*LHSIsTrue=*/false, Depth)
4246 .value_or(u: false))
4247 return true;
4248 }
4249
4250 return false;
4251 };
4252
4253 if (IsKnownNonEqualFromDominatingCondition(V1) ||
4254 IsKnownNonEqualFromDominatingCondition(V2))
4255 return true;
4256 }
4257
4258 if (!Q.AC)
4259 return false;
4260
4261 // Try to infer NonEqual based on information from assumptions.
4262 for (auto &AssumeVH : Q.AC->assumptionsFor(V: V1)) {
4263 if (!AssumeVH)
4264 continue;
4265 CallInst *I = cast<CallInst>(Val&: AssumeVH);
4266
4267 assert(I->getFunction() == Q.CxtI->getFunction() &&
4268 "Got assumption for the wrong function!");
4269 assert(I->getIntrinsicID() == Intrinsic::assume &&
4270 "must be an assume intrinsic");
4271
4272 if (isImpliedCondition(LHS: I->getArgOperand(i: 0), RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4273 /*LHSIsTrue=*/true, Depth)
4274 .value_or(u: false) &&
4275 isValidAssumeForContext(I, Q))
4276 return true;
4277 }
4278
4279 return false;
4280}
4281
4282static bool isNonEqualURem(const Value *X, const Value *Rem,
4283 const SimplifyQuery &Q) {
4284 const Value *Y;
4285 if (!match(V: Rem, P: m_URem(L: m_Specific(V: X), R: m_Value(V&: Y))))
4286 return false;
4287
4288 // For a defined urem, X != X urem Y exactly when X u>= Y.
4289 // isTruePredicate does not handle UGE, so use the equivalent Y u<= X.
4290 if (isTruePredicate(Pred: ICmpInst::ICMP_ULE, LHS: Y, RHS: X))
4291 return true;
4292
4293 std::optional<bool> Implied =
4294 isImpliedByDomCondition(Pred: ICmpInst::ICMP_UGE, LHS: X, RHS: Y, ContextI: Q.CxtI, DL: Q.DL);
4295 return Implied && *Implied;
4296}
4297
4298/// Return true if it is known that V1 != V2.
4299static bool isKnownNonEqual(const Value *V1, const Value *V2,
4300 const APInt &DemandedElts, const SimplifyQuery &Q,
4301 unsigned Depth) {
4302 if (V1 == V2)
4303 return false;
4304 if (V1->getType() != V2->getType())
4305 // We can't look through casts yet.
4306 return false;
4307
4308 if (Depth >= MaxAnalysisRecursionDepth)
4309 return false;
4310
4311 // See if we can recurse through (exactly one of) our operands. This
4312 // requires our operation be 1-to-1 and map every input value to exactly
4313 // one output value. Such an operation is invertible.
4314 auto *O1 = dyn_cast<Operator>(Val: V1);
4315 auto *O2 = dyn_cast<Operator>(Val: V2);
4316 if (O1 && O2 && O1->getOpcode() == O2->getOpcode()) {
4317 if (auto Values = getInvertibleOperands(Op1: O1, Op2: O2))
4318 return isKnownNonEqual(V1: Values->first, V2: Values->second, DemandedElts, Q,
4319 Depth: Depth + 1);
4320
4321 if (const PHINode *PN1 = dyn_cast<PHINode>(Val: V1)) {
4322 const PHINode *PN2 = cast<PHINode>(Val: V2);
4323 // FIXME: This is missing a generalization to handle the case where one is
4324 // a PHI and another one isn't.
4325 if (isNonEqualPHIs(PN1, PN2, DemandedElts, Q, Depth))
4326 return true;
4327 };
4328 }
4329
4330 if (isModifyingBinopOfNonZero(V1, V2, DemandedElts, Q, Depth) ||
4331 isModifyingBinopOfNonZero(V1: V2, V2: V1, DemandedElts, Q, Depth))
4332 return true;
4333
4334 if (isNonEqualMul(V1, V2, DemandedElts, Q, Depth) ||
4335 isNonEqualMul(V1: V2, V2: V1, DemandedElts, Q, Depth))
4336 return true;
4337
4338 if (isNonEqualShl(V1, V2, DemandedElts, Q, Depth) ||
4339 isNonEqualShl(V1: V2, V2: V1, DemandedElts, Q, Depth))
4340 return true;
4341
4342 if (V1->getType()->isIntOrIntVectorTy()) {
4343 // Are any known bits in V1 contradictory to known bits in V2? If V1
4344 // has a known zero where V2 has a known one, they must not be equal.
4345 KnownBits Known1 = computeKnownBits(V: V1, DemandedElts, Q, Depth);
4346 if (!Known1.isUnknown()) {
4347 KnownBits Known2 = computeKnownBits(V: V2, DemandedElts, Q, Depth);
4348 if (Known1.Zero.intersects(RHS: Known2.One) ||
4349 Known2.Zero.intersects(RHS: Known1.One))
4350 return true;
4351 }
4352 }
4353
4354 if (isNonEqualSelect(V1, V2, DemandedElts, Q, Depth) ||
4355 isNonEqualSelect(V1: V2, V2: V1, DemandedElts, Q, Depth))
4356 return true;
4357
4358 if (isNonEqualPointersWithRecursiveGEP(A: V1, B: V2, Q) ||
4359 isNonEqualPointersWithRecursiveGEP(A: V2, B: V1, Q))
4360 return true;
4361
4362 Value *A, *B;
4363 // PtrToInts are NonEqual if their Ptrs are NonEqual.
4364 // Check PtrToInt type matches the pointer size.
4365 if (match(V: V1, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: A))) &&
4366 match(V: V2, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: B))))
4367 return isKnownNonEqual(V1: A, V2: B, DemandedElts, Q, Depth: Depth + 1);
4368
4369 if (isNonEqualURem(X: V1, Rem: V2, Q) || isNonEqualURem(X: V2, Rem: V1, Q))
4370 return true;
4371
4372 if (isKnownNonEqualFromContext(V1, V2, Q, Depth))
4373 return true;
4374
4375 return false;
4376}
4377
4378/// For vector constants, loop over the elements and find the constant with the
4379/// minimum number of sign bits. Return 0 if the value is not a vector constant
4380/// or if any element was not analyzed; otherwise, return the count for the
4381/// element with the minimum number of sign bits.
4382static unsigned computeNumSignBitsVectorConstant(const Value *V,
4383 const APInt &DemandedElts,
4384 unsigned TyBits) {
4385 const auto *CV = dyn_cast<Constant>(Val: V);
4386 if (!CV || !isa<FixedVectorType>(Val: CV->getType()))
4387 return 0;
4388
4389 unsigned MinSignBits = TyBits;
4390 unsigned NumElts = cast<FixedVectorType>(Val: CV->getType())->getNumElements();
4391 for (unsigned i = 0; i != NumElts; ++i) {
4392 if (!DemandedElts[i])
4393 continue;
4394 // If we find a non-ConstantInt, bail out.
4395 auto *Elt = dyn_cast_or_null<ConstantInt>(Val: CV->getAggregateElement(Elt: i));
4396 if (!Elt)
4397 return 0;
4398
4399 MinSignBits = std::min(a: MinSignBits, b: Elt->getValue().getNumSignBits());
4400 }
4401
4402 return MinSignBits;
4403}
4404
4405static unsigned ComputeNumSignBitsImpl(const Value *V,
4406 const APInt &DemandedElts,
4407 const SimplifyQuery &Q, unsigned Depth);
4408
4409static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
4410 const SimplifyQuery &Q, unsigned Depth) {
4411 unsigned Result = ComputeNumSignBitsImpl(V, DemandedElts, Q, Depth);
4412 assert(Result > 0 && "At least one sign bit needs to be present!");
4413 return Result;
4414}
4415
4416/// Return the number of times the sign bit of the register is replicated into
4417/// the other bits. We know that at least 1 bit is always equal to the sign bit
4418/// (itself), but other cases can give us information. For example, immediately
4419/// after an "ashr X, 2", we know that the top 3 bits are all equal to each
4420/// other, so we return 3. For vectors, return the number of sign bits for the
4421/// vector element with the minimum number of known sign bits of the demanded
4422/// elements in the vector specified by DemandedElts.
4423static unsigned ComputeNumSignBitsImpl(const Value *V,
4424 const APInt &DemandedElts,
4425 const SimplifyQuery &Q, unsigned Depth) {
4426 Type *Ty = V->getType();
4427#ifndef NDEBUG
4428 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
4429
4430 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
4431 assert(
4432 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
4433 "DemandedElt width should equal the fixed vector number of elements");
4434 } else {
4435 assert(DemandedElts == APInt(1, 1) &&
4436 "DemandedElt width should be 1 for scalars");
4437 }
4438#endif
4439
4440 // We return the minimum number of sign bits that are guaranteed to be present
4441 // in V, so for undef we have to conservatively return 1. We don't have the
4442 // same behavior for poison though -- that's a FIXME today.
4443
4444 Type *ScalarTy = Ty->getScalarType();
4445 unsigned TyBits = ScalarTy->isPointerTy() ?
4446 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
4447 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
4448
4449 unsigned Tmp, Tmp2;
4450 unsigned FirstAnswer = 1;
4451
4452 // Note that ConstantInt is handled by the general computeKnownBits case
4453 // below.
4454
4455 if (Depth == MaxAnalysisRecursionDepth)
4456 return 1;
4457
4458 if (auto *U = dyn_cast<Operator>(Val: V)) {
4459 switch (Operator::getOpcode(V)) {
4460 default: break;
4461 case Instruction::BitCast: {
4462 Value *Src = U->getOperand(i: 0);
4463 Type *SrcTy = Src->getType();
4464
4465 // Skip if the source type is not an integer or integer vector type
4466 // This ensures we only process integer-like types
4467 if (!SrcTy->isIntOrIntVectorTy())
4468 break;
4469
4470 unsigned SrcBits = SrcTy->getScalarSizeInBits();
4471
4472 // Bitcast 'large element' scalar/vector to 'small element' vector.
4473 if ((SrcBits % TyBits) != 0)
4474 break;
4475
4476 // Only proceed if the destination type is a fixed-size vector
4477 if (isa<FixedVectorType>(Val: Ty)) {
4478 // Fast case - sign splat can be simply split across the small elements.
4479 // This works for both vector and scalar sources
4480 Tmp = ComputeNumSignBits(V: Src, Q, Depth: Depth + 1);
4481 if (Tmp == SrcBits)
4482 return TyBits;
4483 }
4484 break;
4485 }
4486 case Instruction::SExt:
4487 Tmp = TyBits - U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4488 return ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1) +
4489 Tmp;
4490
4491 case Instruction::SDiv: {
4492 const APInt *Denominator;
4493 // sdiv X, C -> adds log(C) sign bits.
4494 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4495
4496 // Ignore non-positive denominator.
4497 if (!Denominator->isStrictlyPositive())
4498 break;
4499
4500 // Calculate the incoming numerator bits.
4501 unsigned NumBits =
4502 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4503
4504 // Add floor(log(C)) bits to the numerator bits.
4505 return std::min(a: TyBits, b: NumBits + Denominator->logBase2());
4506 }
4507 break;
4508 }
4509
4510 case Instruction::SRem: {
4511 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4512
4513 const APInt *Denominator;
4514 // srem X, C -> we know that the result is within [-C+1,C) when C is a
4515 // positive constant. This let us put a lower bound on the number of sign
4516 // bits.
4517 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4518
4519 // Ignore non-positive denominator.
4520 if (Denominator->isStrictlyPositive()) {
4521 // Calculate the leading sign bit constraints by examining the
4522 // denominator. Given that the denominator is positive, there are two
4523 // cases:
4524 //
4525 // 1. The numerator is positive. The result range is [0,C) and
4526 // [0,C) u< (1 << ceilLogBase2(C)).
4527 //
4528 // 2. The numerator is negative. Then the result range is (-C,0] and
4529 // integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
4530 //
4531 // Thus a lower bound on the number of sign bits is `TyBits -
4532 // ceilLogBase2(C)`.
4533
4534 unsigned ResBits = TyBits - Denominator->ceilLogBase2();
4535 Tmp = std::max(a: Tmp, b: ResBits);
4536 }
4537 }
4538 return Tmp;
4539 }
4540
4541 case Instruction::AShr: {
4542 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4543 // ashr X, C -> adds C sign bits. Vectors too.
4544 const APInt *ShAmt;
4545 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4546 if (ShAmt->uge(RHS: TyBits))
4547 break; // Bad shift.
4548 unsigned ShAmtLimited = ShAmt->getZExtValue();
4549 Tmp += ShAmtLimited;
4550 if (Tmp > TyBits) Tmp = TyBits;
4551 }
4552 return Tmp;
4553 }
4554 case Instruction::Shl: {
4555 const APInt *ShAmt;
4556 Value *X = nullptr;
4557 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4558 // shl destroys sign bits.
4559 if (ShAmt->uge(RHS: TyBits))
4560 break; // Bad shift.
4561 // We can look through a zext (more or less treating it as a sext) if
4562 // all extended bits are shifted out.
4563 if (match(V: U->getOperand(i: 0), P: m_ZExt(Op: m_Value(V&: X))) &&
4564 ShAmt->uge(RHS: TyBits - X->getType()->getScalarSizeInBits())) {
4565 Tmp = ComputeNumSignBits(V: X, DemandedElts, Q, Depth: Depth + 1);
4566 Tmp += TyBits - X->getType()->getScalarSizeInBits();
4567 } else
4568 Tmp =
4569 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4570 if (ShAmt->uge(RHS: Tmp))
4571 break; // Shifted all sign bits out.
4572 Tmp2 = ShAmt->getZExtValue();
4573 return Tmp - Tmp2;
4574 }
4575 break;
4576 }
4577 case Instruction::And:
4578 case Instruction::Or:
4579 case Instruction::Xor: // NOT is handled here.
4580 // Logical binary ops preserve the number of sign bits at the worst.
4581 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4582 if (Tmp != 1) {
4583 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4584 FirstAnswer = std::min(a: Tmp, b: Tmp2);
4585 // We computed what we know about the sign bits as our first
4586 // answer. Now proceed to the generic code that uses
4587 // computeKnownBits, and pick whichever answer is better.
4588 }
4589 break;
4590
4591 case Instruction::Select: {
4592 // If we have a clamp pattern, we know that the number of sign bits will
4593 // be the minimum of the clamp min/max range.
4594 const Value *X;
4595 const APInt *CLow, *CHigh;
4596 if (isSignedMinMaxClamp(Select: U, In&: X, CLow, CHigh))
4597 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4598
4599 Tmp = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4600 if (Tmp == 1)
4601 break;
4602 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 2), DemandedElts, Q, Depth: Depth + 1);
4603 return std::min(a: Tmp, b: Tmp2);
4604 }
4605
4606 case Instruction::Add:
4607 // Add can have at most one carry bit. Thus we know that the output
4608 // is, at worst, one more bit than the inputs.
4609 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4610 if (Tmp == 1) break;
4611
4612 // Special case decrementing a value (ADD X, -1):
4613 if (const auto *CRHS = dyn_cast<Constant>(Val: U->getOperand(i: 1)))
4614 if (CRHS->isAllOnesValue()) {
4615 KnownBits Known(TyBits);
4616 computeKnownBits(V: U->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
4617
4618 // If the input is known to be 0 or 1, the output is 0/-1, which is
4619 // all sign bits set.
4620 if ((Known.Zero | 1).isAllOnes())
4621 return TyBits;
4622
4623 // If we are subtracting one from a positive number, there is no carry
4624 // out of the result.
4625 if (Known.isNonNegative())
4626 return Tmp;
4627 }
4628
4629 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4630 if (Tmp2 == 1)
4631 break;
4632 return std::min(a: Tmp, b: Tmp2) - 1;
4633
4634 case Instruction::Sub:
4635 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4636 if (Tmp2 == 1)
4637 break;
4638
4639 // Handle NEG.
4640 if (const auto *CLHS = dyn_cast<Constant>(Val: U->getOperand(i: 0)))
4641 if (CLHS->isNullValue()) {
4642 KnownBits Known(TyBits);
4643 computeKnownBits(V: U->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
4644 // If the input is known to be 0 or 1, the output is 0/-1, which is
4645 // all sign bits set.
4646 if ((Known.Zero | 1).isAllOnes())
4647 return TyBits;
4648
4649 // If the input is known to be positive (the sign bit is known clear),
4650 // the output of the NEG has the same number of sign bits as the
4651 // input.
4652 if (Known.isNonNegative())
4653 return Tmp2;
4654
4655 // Otherwise, we treat this like a SUB.
4656 }
4657
4658 // Sub can have at most one carry bit. Thus we know that the output
4659 // is, at worst, one more bit than the inputs.
4660 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4661 if (Tmp == 1)
4662 break;
4663 return std::min(a: Tmp, b: Tmp2) - 1;
4664
4665 case Instruction::Mul: {
4666 // The output of the Mul can be at most twice the valid bits in the
4667 // inputs.
4668 unsigned SignBitsOp0 =
4669 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4670 if (SignBitsOp0 == 1)
4671 break;
4672 unsigned SignBitsOp1 =
4673 ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4674 if (SignBitsOp1 == 1)
4675 break;
4676 unsigned OutValidBits =
4677 (TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
4678 return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
4679 }
4680
4681 case Instruction::PHI: {
4682 const PHINode *PN = cast<PHINode>(Val: U);
4683 unsigned NumIncomingValues = PN->getNumIncomingValues();
4684 // Don't analyze large in-degree PHIs.
4685 if (NumIncomingValues > 4) break;
4686 // Unreachable blocks may have zero-operand PHI nodes.
4687 if (NumIncomingValues == 0) break;
4688
4689 // Take the minimum of all incoming values. This can't infinitely loop
4690 // because of our depth threshold.
4691 SimplifyQuery RecQ = Q.getWithoutCondContext();
4692 Tmp = TyBits;
4693 for (unsigned i = 0, e = NumIncomingValues; i != e; ++i) {
4694 if (Tmp == 1) return Tmp;
4695 RecQ.CxtI = PN->getIncomingBlock(i)->getTerminator();
4696 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(V: PN->getIncomingValue(i),
4697 DemandedElts, Q: RecQ, Depth: Depth + 1));
4698 }
4699 return Tmp;
4700 }
4701
4702 case Instruction::Trunc: {
4703 // If the input contained enough sign bits that some remain after the
4704 // truncation, then we can make use of that. Otherwise we don't know
4705 // anything.
4706 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4707 unsigned OperandTyBits = U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4708 if (Tmp > (OperandTyBits - TyBits))
4709 return Tmp - (OperandTyBits - TyBits);
4710
4711 return 1;
4712 }
4713
4714 case Instruction::ExtractElement:
4715 // Look through extract element. At the moment we keep this simple and
4716 // skip tracking the specific element. But at least we might find
4717 // information valid for all elements of the vector (for example if vector
4718 // is sign extended, shifted, etc).
4719 return ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4720
4721 case Instruction::ShuffleVector: {
4722 // Collect the minimum number of sign bits that are shared by every vector
4723 // element referenced by the shuffle.
4724 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: U);
4725 if (!Shuf) {
4726 // FIXME: Add support for shufflevector constant expressions.
4727 return 1;
4728 }
4729 APInt DemandedLHS, DemandedRHS;
4730 // For undef elements, we don't know anything about the common state of
4731 // the shuffle result.
4732 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
4733 return 1;
4734 Tmp = std::numeric_limits<unsigned>::max();
4735 if (!!DemandedLHS) {
4736 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
4737 Tmp = ComputeNumSignBits(V: LHS, DemandedElts: DemandedLHS, Q, Depth: Depth + 1);
4738 }
4739 // If we don't know anything, early out and try computeKnownBits
4740 // fall-back.
4741 if (Tmp == 1)
4742 break;
4743 if (!!DemandedRHS) {
4744 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
4745 Tmp2 = ComputeNumSignBits(V: RHS, DemandedElts: DemandedRHS, Q, Depth: Depth + 1);
4746 Tmp = std::min(a: Tmp, b: Tmp2);
4747 }
4748 // If we don't know anything, early out and try computeKnownBits
4749 // fall-back.
4750 if (Tmp == 1)
4751 break;
4752 assert(Tmp <= TyBits && "Failed to determine minimum sign bits");
4753 return Tmp;
4754 }
4755 case Instruction::Call: {
4756 if (const auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
4757 switch (II->getIntrinsicID()) {
4758 default:
4759 break;
4760 case Intrinsic::abs:
4761 Tmp =
4762 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4763 if (Tmp == 1)
4764 break;
4765
4766 // Absolute value reduces number of sign bits by at most 1.
4767 return Tmp - 1;
4768 case Intrinsic::smin:
4769 case Intrinsic::smax: {
4770 const APInt *CLow, *CHigh;
4771 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
4772 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4773 }
4774 }
4775 }
4776 }
4777 }
4778 }
4779
4780 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4781 // use this information.
4782
4783 // If we can examine all elements of a vector constant successfully, we're
4784 // done (we can't do any better than that). If not, keep trying.
4785 if (unsigned VecSignBits =
4786 computeNumSignBitsVectorConstant(V, DemandedElts, TyBits))
4787 return VecSignBits;
4788
4789 KnownBits Known(TyBits);
4790 computeKnownBits(V, DemandedElts, Known, Q, Depth);
4791
4792 // If we know that the sign bit is either zero or one, determine the number of
4793 // identical bits in the top of the input value.
4794 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
4795}
4796
4797Intrinsic::ID llvm::getIntrinsicForCallSite(const CallBase &CB,
4798 const TargetLibraryInfo *TLI) {
4799 const Function *F = CB.getCalledFunction();
4800 if (!F)
4801 return Intrinsic::not_intrinsic;
4802
4803 if (F->isIntrinsic())
4804 return F->getIntrinsicID();
4805
4806 // We are going to infer semantics of a library function based on mapping it
4807 // to an LLVM intrinsic. Check that the library function is available from
4808 // this callbase and in this environment.
4809 if (F->hasLocalLinkage() || !TLI || !CB.onlyReadsMemory())
4810 return Intrinsic::not_intrinsic;
4811
4812 LibFunc Func = TLI->getLibFunc(CB);
4813 if (Func == NotLibFunc)
4814 return Intrinsic::not_intrinsic;
4815
4816 switch (Func) {
4817 default:
4818 break;
4819 case LibFunc_sin:
4820 case LibFunc_sinf:
4821 case LibFunc_sinl:
4822 return Intrinsic::sin;
4823 case LibFunc_cos:
4824 case LibFunc_cosf:
4825 case LibFunc_cosl:
4826 return Intrinsic::cos;
4827 case LibFunc_tan:
4828 case LibFunc_tanf:
4829 case LibFunc_tanl:
4830 return Intrinsic::tan;
4831 case LibFunc_asin:
4832 case LibFunc_asinf:
4833 case LibFunc_asinl:
4834 return Intrinsic::asin;
4835 case LibFunc_acos:
4836 case LibFunc_acosf:
4837 case LibFunc_acosl:
4838 return Intrinsic::acos;
4839 case LibFunc_atan:
4840 case LibFunc_atanf:
4841 case LibFunc_atanl:
4842 return Intrinsic::atan;
4843 case LibFunc_atan2:
4844 case LibFunc_atan2f:
4845 case LibFunc_atan2l:
4846 return Intrinsic::atan2;
4847 case LibFunc_sinh:
4848 case LibFunc_sinhf:
4849 case LibFunc_sinhl:
4850 return Intrinsic::sinh;
4851 case LibFunc_cosh:
4852 case LibFunc_coshf:
4853 case LibFunc_coshl:
4854 return Intrinsic::cosh;
4855 case LibFunc_tanh:
4856 case LibFunc_tanhf:
4857 case LibFunc_tanhl:
4858 return Intrinsic::tanh;
4859 case LibFunc_exp:
4860 case LibFunc_expf:
4861 case LibFunc_expl:
4862 return Intrinsic::exp;
4863 case LibFunc_exp2:
4864 case LibFunc_exp2f:
4865 case LibFunc_exp2l:
4866 return Intrinsic::exp2;
4867 case LibFunc_exp10:
4868 case LibFunc_exp10f:
4869 case LibFunc_exp10l:
4870 return Intrinsic::exp10;
4871 case LibFunc_log:
4872 case LibFunc_logf:
4873 case LibFunc_logl:
4874 return Intrinsic::log;
4875 case LibFunc_log10:
4876 case LibFunc_log10f:
4877 case LibFunc_log10l:
4878 return Intrinsic::log10;
4879 case LibFunc_log2:
4880 case LibFunc_log2f:
4881 case LibFunc_log2l:
4882 return Intrinsic::log2;
4883 case LibFunc_fabs:
4884 case LibFunc_fabsf:
4885 case LibFunc_fabsl:
4886 return Intrinsic::fabs;
4887 case LibFunc_fmin:
4888 case LibFunc_fminf:
4889 case LibFunc_fminl:
4890 return Intrinsic::minnum;
4891 case LibFunc_fmax:
4892 case LibFunc_fmaxf:
4893 case LibFunc_fmaxl:
4894 return Intrinsic::maxnum;
4895 case LibFunc_copysign:
4896 case LibFunc_copysignf:
4897 case LibFunc_copysignl:
4898 return Intrinsic::copysign;
4899 case LibFunc_floor:
4900 case LibFunc_floorf:
4901 case LibFunc_floorl:
4902 return Intrinsic::floor;
4903 case LibFunc_ceil:
4904 case LibFunc_ceilf:
4905 case LibFunc_ceill:
4906 return Intrinsic::ceil;
4907 case LibFunc_trunc:
4908 case LibFunc_truncf:
4909 case LibFunc_truncl:
4910 return Intrinsic::trunc;
4911 case LibFunc_rint:
4912 case LibFunc_rintf:
4913 case LibFunc_rintl:
4914 return Intrinsic::rint;
4915 case LibFunc_nearbyint:
4916 case LibFunc_nearbyintf:
4917 case LibFunc_nearbyintl:
4918 return Intrinsic::nearbyint;
4919 case LibFunc_round:
4920 case LibFunc_roundf:
4921 case LibFunc_roundl:
4922 return Intrinsic::round;
4923 case LibFunc_roundeven:
4924 case LibFunc_roundevenf:
4925 case LibFunc_roundevenl:
4926 return Intrinsic::roundeven;
4927 case LibFunc_pow:
4928 case LibFunc_powf:
4929 case LibFunc_powl:
4930 return Intrinsic::pow;
4931 case LibFunc_sqrt:
4932 case LibFunc_sqrtf:
4933 case LibFunc_sqrtl:
4934 return Intrinsic::sqrt;
4935 }
4936
4937 return Intrinsic::not_intrinsic;
4938}
4939
4940/// Given an exploded icmp instruction, return true if the comparison only
4941/// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if
4942/// the result of the comparison is true when the input value is signed.
4943bool llvm::isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
4944 bool &TrueIfSigned) {
4945 switch (Pred) {
4946 case ICmpInst::ICMP_SLT: // True if LHS s< 0
4947 TrueIfSigned = true;
4948 return RHS.isZero();
4949 case ICmpInst::ICMP_SLE: // True if LHS s<= -1
4950 TrueIfSigned = true;
4951 return RHS.isAllOnes();
4952 case ICmpInst::ICMP_SGT: // True if LHS s> -1
4953 TrueIfSigned = false;
4954 return RHS.isAllOnes();
4955 case ICmpInst::ICMP_SGE: // True if LHS s>= 0
4956 TrueIfSigned = false;
4957 return RHS.isZero();
4958 case ICmpInst::ICMP_UGT:
4959 // True if LHS u> RHS and RHS == sign-bit-mask - 1
4960 TrueIfSigned = true;
4961 return RHS.isMaxSignedValue();
4962 case ICmpInst::ICMP_UGE:
4963 // True if LHS u>= RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
4964 TrueIfSigned = true;
4965 return RHS.isMinSignedValue();
4966 case ICmpInst::ICMP_ULT:
4967 // True if LHS u< RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
4968 TrueIfSigned = false;
4969 return RHS.isMinSignedValue();
4970 case ICmpInst::ICMP_ULE:
4971 // True if LHS u<= RHS and RHS == sign-bit-mask - 1
4972 TrueIfSigned = false;
4973 return RHS.isMaxSignedValue();
4974 default:
4975 return false;
4976 }
4977}
4978
4979static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
4980 bool CondIsTrue,
4981 const Instruction *CxtI,
4982 KnownFPClass &KnownFromContext,
4983 unsigned Depth = 0) {
4984 Value *A, *B;
4985 if (Depth < MaxAnalysisRecursionDepth &&
4986 (CondIsTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B)))
4987 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))))) {
4988 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue, CxtI, KnownFromContext,
4989 Depth: Depth + 1);
4990 computeKnownFPClassFromCond(V, Cond: B, CondIsTrue, CxtI, KnownFromContext,
4991 Depth: Depth + 1);
4992 return;
4993 }
4994 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A)))) {
4995 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue: !CondIsTrue, CxtI, KnownFromContext,
4996 Depth: Depth + 1);
4997 return;
4998 }
4999 CmpPredicate Pred;
5000 Value *LHS;
5001 uint64_t ClassVal = 0;
5002 const APFloat *CRHS;
5003 const APInt *RHS;
5004 if (match(V: Cond, P: m_FCmp(Pred, L: m_Value(V&: LHS), R: m_APFloat(Res&: CRHS)))) {
5005 auto [CmpVal, MaskIfTrue, MaskIfFalse] = fcmpImpliesClass(
5006 Pred, F: *cast<Instruction>(Val: Cond)->getParent()->getParent(), LHS, ConstRHS: *CRHS,
5007 LookThroughSrc: LHS != V);
5008 if (CmpVal == V)
5009 KnownFromContext.knownNot(RuleOut: ~(CondIsTrue ? MaskIfTrue : MaskIfFalse));
5010 } else if (match(V: Cond, P: m_Intrinsic<Intrinsic::is_fpclass>(
5011 Ops: m_Specific(V), Ops: m_ConstantInt(V&: ClassVal)))) {
5012 FPClassTest Mask = static_cast<FPClassTest>(ClassVal);
5013 KnownFromContext.knownNot(RuleOut: CondIsTrue ? ~Mask : Mask);
5014 } else if (match(V: Cond, P: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Specific(V)),
5015 R: m_APInt(Res&: RHS)))) {
5016 bool TrueIfSigned;
5017 if (!isSignBitCheck(Pred, RHS: *RHS, TrueIfSigned))
5018 return;
5019 if (TrueIfSigned == CondIsTrue)
5020 KnownFromContext.signBitMustBeOne();
5021 else
5022 KnownFromContext.signBitMustBeZero();
5023 }
5024}
5025
5026/// Compute the minimum and maximum values (inclusive) for the exponent of \p V,
5027/// assuming it is not nan. Returns {min, max, max-assuming-nonzero}. A value
5028/// frexp(0) = 0, so the tighter max-assuming-nonzero bound is only usable when
5029/// \p V is known not to be a logical zero (e.g., for fabs(x) < 0.25, the non-0
5030/// exponent range is [-149, -2], but the 0 edge case is above this range).
5031static std::tuple<int, int, int>
5032computeKnownExponentRangeFromContext(const Value *V, const SimplifyQuery &Q) {
5033 if (!Q.CxtI || !Q.DC || !Q.DT)
5034 return {APFloat::IEK_NaN, APFloat::IEK_Inf, APFloat::IEK_Inf};
5035
5036 // Intersect the bounds implied by every dominating condition, keeping the
5037 // tightest maximum. A value may participate in multiple compares
5038 // (e.g. fabs(x) < 2.0 and fabs(x) < 1.0), and the tighter one wins.
5039 int MaxExp = APFloat::IEK_Inf;
5040 int MaxExpNonZero = APFloat::IEK_Inf;
5041
5042 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
5043 CmpPredicate Pred;
5044 const APFloat *LimitC;
5045 if (!match(V: BI->getCondition(),
5046 P: m_FCmp(Pred, L: m_FAbs(Op0: m_Specific(V)), R: m_Finite(V&: LimitC))))
5047 continue;
5048
5049 if (Pred == FCmpInst::FCMP_ORD || Pred == FCmpInst::FCMP_UNO ||
5050 Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE)
5051 continue;
5052
5053 // If fabs(x) <= K, implies the exponent min exp range.
5054 // if fabs(x) >= K, swap the successor
5055 bool IsLessEqual =
5056 Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE ||
5057 Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE ||
5058 Pred == FCmpInst::FCMP_OEQ || Pred == FCmpInst::FCMP_UEQ;
5059
5060 bool KnownStrictlyLess =
5061 Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_ULT ||
5062 Pred == FCmpInst::FCMP_OGE || Pred == FCmpInst::FCMP_UGE;
5063
5064 BasicBlockEdge Edge1(BI->getParent(),
5065 BI->getSuccessor(i: IsLessEqual ? 0 : 1));
5066 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent())) {
5067 // frexp returns an exponent one greater than ilogb.
5068 int Exp = ilogb(Arg: *LimitC) + 1;
5069
5070 // A strict bound fabs(V) < 2^n forces ilogb(V) <= n - 1, so the max frexp
5071 // exponent drops by one when K is exact power of two.
5072 if (KnownStrictlyLess && LimitC->getExactLog2Abs() != INT_MIN)
5073 --Exp;
5074
5075 // frexp(0) = 0, which the bound above (assuming a normal nonzero value)
5076 // may exclude.
5077
5078 // TODO: Figure out lower bound to detect no-underflow.
5079 MaxExpNonZero = std::min(a: MaxExpNonZero, b: Exp);
5080 MaxExp = std::min(a: MaxExp, b: std::max(a: Exp, b: 0));
5081 }
5082 }
5083
5084 return {APFloat::IEK_NaN, MaxExp, MaxExpNonZero};
5085}
5086
5087static KnownFPClass computeKnownFPClassFromContext(const Value *V,
5088 const SimplifyQuery &Q) {
5089 KnownFPClass KnownFromContext;
5090
5091 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
5092 computeKnownFPClassFromCond(V, Cond: Q.CC->Cond, CondIsTrue: !Q.CC->Invert, CxtI: Q.CxtI,
5093 KnownFromContext);
5094
5095 if (!Q.CxtI)
5096 return KnownFromContext;
5097
5098 if (Q.DC && Q.DT) {
5099 // Handle dominating conditions.
5100 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
5101 Value *Cond = BI->getCondition();
5102
5103 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
5104 if (Q.DT->dominates(BBE: Edge0, BB: Q.CxtI->getParent()))
5105 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, CxtI: Q.CxtI,
5106 KnownFromContext);
5107
5108 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
5109 if (Q.DT->dominates(BBE: Edge1, BB: Q.CxtI->getParent()))
5110 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, CxtI: Q.CxtI,
5111 KnownFromContext);
5112 }
5113 }
5114
5115 if (!Q.AC)
5116 return KnownFromContext;
5117
5118 // Try to restrict the floating-point classes based on information from
5119 // assumptions.
5120 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
5121 if (!AssumeVH)
5122 continue;
5123 CallInst *I = cast<CallInst>(Val&: AssumeVH);
5124
5125 assert(I->getFunction() == Q.CxtI->getParent()->getParent() &&
5126 "Got assumption for the wrong function!");
5127 assert(I->getIntrinsicID() == Intrinsic::assume &&
5128 "must be an assume intrinsic");
5129
5130 if (!isValidAssumeForContext(I, Q))
5131 continue;
5132
5133 computeKnownFPClassFromCond(V, Cond: I->getArgOperand(i: 0),
5134 /*CondIsTrue=*/true, CxtI: Q.CxtI, KnownFromContext);
5135 }
5136
5137 return KnownFromContext;
5138}
5139
5140void llvm::adjustKnownFPClassForSelectArm(KnownFPClass &Known, Value *Cond,
5141 Value *Arm, bool Invert,
5142 const SimplifyQuery &SQ,
5143 unsigned Depth) {
5144
5145 KnownFPClass KnownSrc;
5146 computeKnownFPClassFromCond(V: Arm, Cond,
5147 /*CondIsTrue=*/!Invert, CxtI: SQ.CxtI, KnownFromContext&: KnownSrc,
5148 Depth: Depth + 1);
5149 KnownSrc = KnownSrc.unionWith(RHS: Known);
5150 if (KnownSrc.isUnknown())
5151 return;
5152
5153 if (isGuaranteedNotToBeUndef(V: Arm, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT, Depth: Depth + 1))
5154 Known = KnownSrc;
5155}
5156
5157void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5158 FPClassTest InterestedClasses, KnownFPClass &Known,
5159 const SimplifyQuery &Q, unsigned Depth);
5160
5161static void computeKnownFPClass(const Value *V, KnownFPClass &Known,
5162 FPClassTest InterestedClasses,
5163 const SimplifyQuery &Q, unsigned Depth) {
5164 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5165 APInt DemandedElts =
5166 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
5167 computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Q, Depth);
5168}
5169
5170static void computeKnownFPClassForFPTrunc(const Operator *Op,
5171 const APInt &DemandedElts,
5172 FPClassTest InterestedClasses,
5173 KnownFPClass &Known,
5174 const SimplifyQuery &Q,
5175 unsigned Depth) {
5176 if ((InterestedClasses &
5177 (KnownFPClass::OrderedLessThanZeroMask | fcNan)) == fcNone)
5178 return;
5179
5180 KnownFPClass KnownSrc;
5181 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5182 Known&: KnownSrc, Q, Depth: Depth + 1);
5183 Known = KnownFPClass::fptrunc(KnownSrc);
5184}
5185
5186static constexpr KnownFPClass::MinMaxKind getMinMaxKind(Intrinsic::ID IID) {
5187 switch (IID) {
5188 case Intrinsic::minimum:
5189 return KnownFPClass::MinMaxKind::minimum;
5190 case Intrinsic::maximum:
5191 return KnownFPClass::MinMaxKind::maximum;
5192 case Intrinsic::minimumnum:
5193 return KnownFPClass::MinMaxKind::minimumnum;
5194 case Intrinsic::maximumnum:
5195 return KnownFPClass::MinMaxKind::maximumnum;
5196 case Intrinsic::minnum:
5197 return KnownFPClass::MinMaxKind::minnum;
5198 case Intrinsic::maxnum:
5199 return KnownFPClass::MinMaxKind::maxnum;
5200 default:
5201 llvm_unreachable("not a floating-point min-max intrinsic");
5202 }
5203}
5204
5205/// \return true if this is a floating point value that is known to have a
5206/// magnitude smaller than 1. i.e., fabs(X) <= 1.0 or is nan.
5207static bool isAbsoluteValueULEOne(const Value *V) {
5208 // TODO: Handle frexp
5209 // TODO: Other rounding intrinsics?
5210 // TODO: Try computeKnownExponentRangeFromContext
5211
5212 // fabs(x - floor(x)) <= 1
5213 const Value *SubFloorX;
5214 if (match(V, P: m_FSub(L: m_Value(V&: SubFloorX),
5215 R: m_Intrinsic<Intrinsic::floor>(Ops: m_Deferred(V: SubFloorX)))))
5216 return true;
5217
5218 return match(V, P: m_Intrinsic<Intrinsic::amdgcn_trig_preop>(Ops: m_Value())) ||
5219 match(V, P: m_Intrinsic<Intrinsic::amdgcn_fract>(Ops: m_Value()));
5220}
5221
5222void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5223 FPClassTest InterestedClasses, KnownFPClass &Known,
5224 const SimplifyQuery &Q, unsigned Depth) {
5225 assert(Known.isUnknown() && "should not be called with known information");
5226
5227 if (!DemandedElts) {
5228 // No demanded elts, better to assume we don't know anything.
5229 Known.resetAll();
5230 return;
5231 }
5232
5233 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
5234
5235 if (auto *CFP = dyn_cast<ConstantFP>(Val: V)) {
5236 Known = KnownFPClass(CFP->getValueAPF());
5237 return;
5238 }
5239
5240 if (isa<ConstantAggregateZero>(Val: V)) {
5241 Known.setKnownFPClasses(fcPosZero);
5242 Known.setSignBit(false);
5243 return;
5244 }
5245
5246 if (isa<PoisonValue>(Val: V)) {
5247 Known.setKnownFPClasses(fcNone);
5248 Known.setSignBit(false);
5249 return;
5250 }
5251
5252 // Try to handle fixed width vector constants
5253 auto *VFVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5254 const Constant *CV = dyn_cast<Constant>(Val: V);
5255 if (VFVTy && CV) {
5256 Known.setKnownFPClasses(fcNone);
5257 bool SignBitAllZero = true;
5258 bool SignBitAllOne = true;
5259
5260 // For vectors, verify that each element is not NaN.
5261 unsigned NumElts = VFVTy->getNumElements();
5262 for (unsigned i = 0; i != NumElts; ++i) {
5263 if (!DemandedElts[i])
5264 continue;
5265
5266 Constant *Elt = CV->getAggregateElement(Elt: i);
5267 if (!Elt) {
5268 Known = KnownFPClass();
5269 return;
5270 }
5271 if (isa<PoisonValue>(Val: Elt))
5272 continue;
5273 auto *CElt = dyn_cast<ConstantFP>(Val: Elt);
5274 if (!CElt) {
5275 Known = KnownFPClass();
5276 return;
5277 }
5278
5279 const APFloat &C = CElt->getValueAPF();
5280 Known.setKnownFPClasses(Known.getKnownFPClasses() | C.classify());
5281 if (C.isNegative())
5282 SignBitAllZero = false;
5283 else
5284 SignBitAllOne = false;
5285 }
5286 if (SignBitAllOne != SignBitAllZero)
5287 Known.setSignBit(SignBitAllOne);
5288 return;
5289 }
5290
5291 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) {
5292 Known.setKnownFPClasses(fcNone);
5293 for (size_t I = 0, E = CDS->getNumElements(); I != E; ++I)
5294 Known |= CDS->getElementAsAPFloat(i: I).classify();
5295 return;
5296 }
5297
5298 if (const auto *CA = dyn_cast<ConstantAggregate>(Val: V)) {
5299 // TODO: Handle complex aggregates
5300 Known.setKnownFPClasses(fcNone);
5301 for (const Use &Op : CA->operands()) {
5302 auto *CFP = dyn_cast<ConstantFP>(Val: Op.get());
5303 if (!CFP) {
5304 Known = KnownFPClass();
5305 return;
5306 }
5307
5308 Known |= CFP->getValueAPF().classify();
5309 }
5310
5311 return;
5312 }
5313
5314 FPClassTest KnownNotFromFlags = fcNone;
5315 if (const auto *CB = dyn_cast<CallBase>(Val: V))
5316 KnownNotFromFlags |= CB->getRetNoFPClass();
5317 else if (const auto *Arg = dyn_cast<Argument>(Val: V))
5318 KnownNotFromFlags |= Arg->getNoFPClass();
5319
5320 const Operator *Op = dyn_cast<Operator>(Val: V);
5321 if (const FPMathOperator *FPOp = dyn_cast_or_null<FPMathOperator>(Val: Op)) {
5322 if (FPOp->hasNoNaNs())
5323 KnownNotFromFlags |= fcNan;
5324 if (FPOp->hasNoInfs())
5325 KnownNotFromFlags |= fcInf;
5326 }
5327
5328 KnownFPClass AssumedClasses = computeKnownFPClassFromContext(V, Q);
5329 KnownNotFromFlags |= ~AssumedClasses.getKnownFPClasses();
5330
5331 // We no longer need to find out about these bits from inputs if we can
5332 // assume this from flags/attributes.
5333 InterestedClasses &= ~KnownNotFromFlags;
5334
5335 llvm::scope_exit ClearClassesFromFlags([=, &Known] {
5336 Known.knownNot(RuleOut: KnownNotFromFlags);
5337 if (!Known.getSignBit() && AssumedClasses.getSignBit()) {
5338 if (*AssumedClasses.getSignBit())
5339 Known.signBitMustBeOne();
5340 else
5341 Known.signBitMustBeZero();
5342 }
5343 });
5344
5345 if (!Op)
5346 return;
5347
5348 // All recursive calls that increase depth must come after this.
5349 if (Depth == MaxAnalysisRecursionDepth)
5350 return;
5351
5352 const unsigned Opc = Op->getOpcode();
5353 switch (Opc) {
5354 case Instruction::FNeg: {
5355 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5356 Known, Q, Depth: Depth + 1);
5357 Known.fneg();
5358 break;
5359 }
5360 case Instruction::Select: {
5361 auto ComputeForArm = [&](Value *Arm, bool Invert) {
5362 KnownFPClass Res;
5363 computeKnownFPClass(V: Arm, DemandedElts, InterestedClasses, Known&: Res, Q,
5364 Depth: Depth + 1);
5365 adjustKnownFPClassForSelectArm(Known&: Res, Cond: Op->getOperand(i: 0), Arm, Invert, SQ: Q,
5366 Depth);
5367 return Res;
5368 };
5369 // Only known if known in both the LHS and RHS.
5370 Known =
5371 ComputeForArm(Op->getOperand(i: 1), /*Invert=*/false)
5372 .intersectWith(RHS: ComputeForArm(Op->getOperand(i: 2), /*Invert=*/true));
5373 break;
5374 }
5375 case Instruction::Load: {
5376 const MDNode *NoFPClass =
5377 cast<LoadInst>(Val: Op)->getMetadata(KindID: LLVMContext::MD_nofpclass);
5378 if (!NoFPClass)
5379 break;
5380
5381 ConstantInt *MaskVal =
5382 mdconst::extract<ConstantInt>(MD: NoFPClass->getOperand(I: 0));
5383 Known.knownNot(RuleOut: static_cast<FPClassTest>(MaskVal->getZExtValue()));
5384 break;
5385 }
5386 case Instruction::Call: {
5387 const CallInst *II = cast<CallInst>(Val: Op);
5388 const Intrinsic::ID IID = II->getIntrinsicID();
5389 switch (IID) {
5390 case Intrinsic::fabs: {
5391 if ((InterestedClasses & (fcNan | fcPositive)) != fcNone) {
5392 // If we only care about the sign bit we don't need to inspect the
5393 // operand.
5394 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5395 InterestedClasses, Known, Q, Depth: Depth + 1);
5396 }
5397
5398 Known.fabs();
5399 break;
5400 }
5401 case Intrinsic::copysign: {
5402 KnownFPClass KnownSign;
5403
5404 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5405 Known, Q, Depth: Depth + 1);
5406 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5407 Known&: KnownSign, Q, Depth: Depth + 1);
5408 Known.copysign(Sign: KnownSign);
5409 break;
5410 }
5411 case Intrinsic::fma:
5412 case Intrinsic::fmuladd: {
5413 if ((InterestedClasses & fcNegative) == fcNone)
5414 break;
5415
5416 // FIXME: This should check isGuaranteedNotToBeUndef
5417 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) {
5418 KnownFPClass KnownSrc, KnownAddend;
5419 computeKnownFPClass(V: II->getArgOperand(i: 2), DemandedElts,
5420 InterestedClasses, Known&: KnownAddend, Q, Depth: Depth + 1);
5421 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5422 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5423
5424 const Function *F = II->getFunction();
5425 const fltSemantics &FltSem =
5426 II->getType()->getScalarType()->getFltSemantics();
5427 DenormalMode Mode =
5428 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5429
5430 if (KnownNotFromFlags & fcNan) {
5431 KnownSrc.knownNot(RuleOut: fcNan);
5432 KnownAddend.knownNot(RuleOut: fcNan);
5433 }
5434
5435 if (KnownNotFromFlags & fcInf) {
5436 KnownSrc.knownNot(RuleOut: fcInf);
5437 KnownAddend.knownNot(RuleOut: fcInf);
5438 }
5439
5440 Known = KnownFPClass::fma_square(Squared: KnownSrc, Addend: KnownAddend, Mode);
5441 break;
5442 }
5443
5444 KnownFPClass KnownSrc[3];
5445 for (int I = 0; I != 3; ++I) {
5446 computeKnownFPClass(V: II->getArgOperand(i: I), DemandedElts,
5447 InterestedClasses, Known&: KnownSrc[I], Q, Depth: Depth + 1);
5448 if (KnownSrc[I].isUnknown())
5449 return;
5450
5451 if (KnownNotFromFlags & fcNan)
5452 KnownSrc[I].knownNot(RuleOut: fcNan);
5453 if (KnownNotFromFlags & fcInf)
5454 KnownSrc[I].knownNot(RuleOut: fcInf);
5455 }
5456
5457 const Function *F = II->getFunction();
5458 const fltSemantics &FltSem =
5459 II->getType()->getScalarType()->getFltSemantics();
5460 DenormalMode Mode =
5461 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5462 Known = KnownFPClass::fma(LHS: KnownSrc[0], RHS: KnownSrc[1], Addend: KnownSrc[2], Mode);
5463 break;
5464 }
5465 case Intrinsic::sqrt:
5466 case Intrinsic::experimental_constrained_sqrt: {
5467 KnownFPClass KnownSrc;
5468 FPClassTest InterestedSrcs = InterestedClasses;
5469 if (InterestedClasses & fcNan)
5470 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5471
5472 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5473 Known&: KnownSrc, Q, Depth: Depth + 1);
5474
5475 DenormalMode Mode = DenormalMode::getDynamic();
5476
5477 bool HasNSZ = Q.IIQ.hasNoSignedZeros(Op: II);
5478 if (!HasNSZ) {
5479 const Function *F = II->getFunction();
5480 const fltSemantics &FltSem =
5481 II->getType()->getScalarType()->getFltSemantics();
5482 Mode = F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5483 }
5484
5485 Known = KnownFPClass::sqrt(Src: KnownSrc, Mode);
5486 if (HasNSZ)
5487 Known.knownNot(RuleOut: fcNegZero);
5488
5489 break;
5490 }
5491 case Intrinsic::sin: {
5492 KnownFPClass KnownSrc;
5493 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5494 Known&: KnownSrc, Q, Depth: Depth + 1);
5495 Known = KnownFPClass::sin(Src: KnownSrc);
5496 break;
5497 }
5498 case Intrinsic::cos: {
5499 KnownFPClass KnownSrc;
5500 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5501 Known&: KnownSrc, Q, Depth: Depth + 1);
5502 Known = KnownFPClass::cos(Src: KnownSrc);
5503 break;
5504 }
5505 case Intrinsic::tan: {
5506 KnownFPClass KnownSrc;
5507 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5508 Known&: KnownSrc, Q, Depth: Depth + 1);
5509 Known = KnownFPClass::tan(Src: KnownSrc);
5510 break;
5511 }
5512 case Intrinsic::sinh: {
5513 KnownFPClass KnownSrc;
5514 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5515 Known&: KnownSrc, Q, Depth: Depth + 1);
5516 Known = KnownFPClass::sinh(Src: KnownSrc);
5517 break;
5518 }
5519 case Intrinsic::cosh: {
5520 KnownFPClass KnownSrc;
5521 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5522 Known&: KnownSrc, Q, Depth: Depth + 1);
5523 Known = KnownFPClass::cosh(Src: KnownSrc);
5524 break;
5525 }
5526 case Intrinsic::tanh: {
5527 KnownFPClass KnownSrc;
5528 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5529 Known&: KnownSrc, Q, Depth: Depth + 1);
5530 Known = KnownFPClass::tanh(Src: KnownSrc);
5531 break;
5532 }
5533 case Intrinsic::asin: {
5534 KnownFPClass KnownSrc;
5535 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5536 Known&: KnownSrc, Q, Depth: Depth + 1);
5537 Known = KnownFPClass::asin(Src: KnownSrc);
5538 break;
5539 }
5540 case Intrinsic::acos: {
5541 KnownFPClass KnownSrc;
5542 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5543 Known&: KnownSrc, Q, Depth: Depth + 1);
5544 Known = KnownFPClass::acos(Src: KnownSrc);
5545 break;
5546 }
5547 case Intrinsic::atan: {
5548 KnownFPClass KnownSrc;
5549 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5550 Known&: KnownSrc, Q, Depth: Depth + 1);
5551 Known = KnownFPClass::atan(Src: KnownSrc);
5552 break;
5553 }
5554 case Intrinsic::atan2: {
5555 FPClassTest InterestedY = InterestedClasses;
5556 FPClassTest InterestedX = InterestedClasses;
5557
5558 // We can rule out negative values if y cannot have a negative value.
5559 if ((InterestedClasses & fcNegFinite) != fcNone)
5560 InterestedY |= fcNegative;
5561
5562 // We can rule out positive values if y cannot have a positive value.
5563 if ((InterestedClasses & fcPosFinite) != fcNone)
5564 InterestedY |= fcPositive | fcNegSubnormal;
5565
5566 // We can rule out zero and subnormal if x cannot have a positive value.
5567 if ((InterestedClasses & (fcZero | fcSubnormal)) != fcNone)
5568 InterestedX |= fcPositive | fcNegSubnormal;
5569
5570 KnownFPClass KnownY, KnownX;
5571 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedY,
5572 Known&: KnownY, Q, Depth: Depth + 1);
5573 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses: InterestedX,
5574 Known&: KnownX, Q, Depth: Depth + 1);
5575
5576 const Function *F = II->getFunction();
5577 DenormalMode Mode =
5578 F ? F->getDenormalMode(
5579 FPType: II->getType()->getScalarType()->getFltSemantics())
5580 : DenormalMode::getDynamic();
5581 Known = KnownFPClass::atan2(LHS: KnownY, RHS: KnownX, Mode);
5582 break;
5583 }
5584 case Intrinsic::maxnum:
5585 case Intrinsic::minnum:
5586 case Intrinsic::minimum:
5587 case Intrinsic::maximum:
5588 case Intrinsic::minimumnum:
5589 case Intrinsic::maximumnum: {
5590 KnownFPClass KnownLHS, KnownRHS;
5591 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5592 Known&: KnownLHS, Q, Depth: Depth + 1);
5593 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5594 Known&: KnownRHS, Q, Depth: Depth + 1);
5595
5596 const Function *F = II->getFunction();
5597
5598 DenormalMode Mode =
5599 F ? F->getDenormalMode(
5600 FPType: II->getType()->getScalarType()->getFltSemantics())
5601 : DenormalMode::getDynamic();
5602
5603 Known = KnownFPClass::minMaxLike(LHS: KnownLHS, RHS: KnownRHS, Kind: getMinMaxKind(IID),
5604 DenormMode: Mode);
5605 break;
5606 }
5607 case Intrinsic::canonicalize: {
5608 KnownFPClass KnownSrc;
5609 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5610 Known&: KnownSrc, Q, Depth: Depth + 1);
5611
5612 const Function *F = II->getFunction();
5613 DenormalMode DenormMode =
5614 F ? F->getDenormalMode(
5615 FPType: II->getType()->getScalarType()->getFltSemantics())
5616 : DenormalMode::getDynamic();
5617 Known = KnownFPClass::canonicalize(Src: KnownSrc, DenormMode);
5618 break;
5619 }
5620 case Intrinsic::vector_reduce_fmax:
5621 case Intrinsic::vector_reduce_fmin:
5622 case Intrinsic::vector_reduce_fmaximum:
5623 case Intrinsic::vector_reduce_fminimum:
5624 case Intrinsic::vector_reduce_fmaximumnum:
5625 case Intrinsic::vector_reduce_fminimumnum: {
5626 // reduce min/max will choose an element from one of the vector elements,
5627 // so we can infer and class information that is common to all elements.
5628 Known = computeKnownFPClass(V: II->getArgOperand(i: 0), FMF: II->getFastMathFlags(),
5629 InterestedClasses, SQ: Q, Depth: Depth + 1);
5630 // Can only propagate sign if output is never NaN.
5631 if (!Known.isKnownNeverNaN())
5632 Known.setSignBit(std::nullopt);
5633 break;
5634 }
5635 // reverse preserves all characteristics of the input vec's element.
5636 case Intrinsic::vector_reverse:
5637 Known = computeKnownFPClass(
5638 V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
5639 FMF: II->getFastMathFlags(), InterestedClasses, SQ: Q, Depth: Depth + 1);
5640 break;
5641 case Intrinsic::trunc:
5642 case Intrinsic::floor:
5643 case Intrinsic::ceil:
5644 case Intrinsic::rint:
5645 case Intrinsic::nearbyint:
5646 case Intrinsic::round:
5647 case Intrinsic::roundeven: {
5648 KnownFPClass KnownSrc;
5649 FPClassTest InterestedSrcs = InterestedClasses;
5650 if (InterestedSrcs & fcPosFinite)
5651 InterestedSrcs |= fcPosFinite;
5652 if (InterestedSrcs & fcNegFinite)
5653 InterestedSrcs |= fcNegFinite;
5654 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5655 Known&: KnownSrc, Q, Depth: Depth + 1);
5656
5657 Known = KnownFPClass::roundToIntegral(
5658 Src: KnownSrc, IsTrunc: IID == Intrinsic::trunc,
5659 IsMultiUnitFPType: V->getType()->getScalarType()->isMultiUnitFPType());
5660 break;
5661 }
5662 case Intrinsic::exp:
5663 case Intrinsic::exp2:
5664 case Intrinsic::exp10:
5665 case Intrinsic::amdgcn_exp2: {
5666 KnownFPClass KnownSrc;
5667 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5668 Known&: KnownSrc, Q, Depth: Depth + 1);
5669
5670 Known = KnownFPClass::exp(Src: KnownSrc);
5671
5672 Type *EltTy = II->getType()->getScalarType();
5673 if (IID == Intrinsic::amdgcn_exp2 && EltTy->isFloatTy())
5674 Known.knownNot(RuleOut: fcSubnormal);
5675
5676 break;
5677 }
5678 case Intrinsic::fptrunc_round: {
5679 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
5680 Q, Depth);
5681 break;
5682 }
5683 case Intrinsic::log:
5684 case Intrinsic::log10:
5685 case Intrinsic::log2:
5686 case Intrinsic::experimental_constrained_log:
5687 case Intrinsic::experimental_constrained_log10:
5688 case Intrinsic::experimental_constrained_log2:
5689 case Intrinsic::amdgcn_log: {
5690 FPClassTest InterestedSrcs = fcNone;
5691
5692 // log(negative) produces NaN.
5693 if ((InterestedClasses & fcNan) != fcNone)
5694 InterestedSrcs |= fcNan | fcNegative;
5695
5696 // log(logical-zero) produces negative infinity.
5697 if ((InterestedClasses & fcNegInf) != fcNone)
5698 InterestedSrcs |= fcZero | fcSubnormal;
5699
5700 // log(x) < -0.0 if x < +1.0
5701 if ((InterestedClasses & fcNegNormal) != fcNone)
5702 InterestedSrcs |= fcPosSubnormal | fcPosNormal;
5703
5704 // log(x) >= +0.0 if x >= +1.0
5705 if ((InterestedClasses & (fcPosZero | fcPosNormal)) != fcNone)
5706 InterestedSrcs |= fcPosNormal;
5707
5708 // log(x) is positive infinity iff x is positive infinity.
5709 if ((InterestedClasses & fcPosInf) != fcNone)
5710 InterestedSrcs |= fcPosInf;
5711
5712 KnownFPClass KnownSrc;
5713 if (InterestedSrcs != fcNone)
5714 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5715 Known&: KnownSrc, Q, Depth: Depth + 1);
5716 const Function *F = II->getFunction();
5717 DenormalMode Mode =
5718 F ? F->getDenormalMode(
5719 FPType: II->getType()->getScalarType()->getFltSemantics())
5720 : DenormalMode::getDynamic();
5721 Known = KnownFPClass::log(Src: KnownSrc, Mode);
5722 break;
5723 }
5724 case Intrinsic::pow: {
5725 const bool WantNaN = (InterestedClasses & fcNan) != fcNone;
5726 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
5727 if (!WantNaN && !WantNegative)
5728 break;
5729
5730 FPClassTest InterestedLHS = fcNone;
5731 FPClassTest InterestedRHS = fcNone;
5732 if (WantNaN) {
5733 // pow may return NaN if one of the arguments is NaN. NaN may also be
5734 // produced from a negative, non-zero finite base and a non-integer
5735 // exponent.
5736 InterestedLHS |= fcNan | fcNegNormal | fcNegSubnormal;
5737 InterestedRHS |= fcNan;
5738 }
5739 if (WantNegative) {
5740 // A negative value is returned when a negative base is raised to an odd
5741 // integer power. Only normal values can be odd integers.
5742 InterestedLHS |= fcNegative;
5743 InterestedRHS |= fcNormal;
5744 }
5745
5746 KnownFPClass KnownLHS;
5747 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedLHS,
5748 Known&: KnownLHS, Q, Depth: Depth + 1);
5749
5750 // If the LHS is unknown, then querying the RHS is only useful for rare
5751 // edge cases.
5752 if (KnownLHS.isUnknown())
5753 break;
5754
5755 KnownFPClass KnownRHS;
5756 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses: InterestedRHS,
5757 Known&: KnownRHS, Q, Depth: Depth + 1);
5758 Known = KnownFPClass::pow(LHS: KnownLHS, RHS: KnownRHS);
5759 break;
5760 }
5761 case Intrinsic::powi: {
5762 if ((InterestedClasses & (fcNan | fcInf | fcNegative)) == fcNone)
5763 break;
5764
5765 // The exponent is always a scalar, even when raising a vector to a power.
5766 const Value *Exp = II->getArgOperand(i: 1);
5767 unsigned BitWidth = Exp->getType()->getIntegerBitWidth();
5768 KnownBits ExponentKnownBits(BitWidth);
5769 computeKnownBits(V: Exp, DemandedElts: APInt(1, 1), Known&: ExponentKnownBits, Q, Depth: Depth + 1);
5770
5771 FPClassTest InterestedSrcs = fcNone;
5772 if (InterestedClasses & fcNan)
5773 InterestedSrcs |= fcNan;
5774 if (!ExponentKnownBits.isZero()) {
5775 if (InterestedClasses & fcInf)
5776 InterestedSrcs |= fcFinite | fcInf;
5777 if ((InterestedClasses & fcNegative) && !ExponentKnownBits.isEven())
5778 InterestedSrcs |= fcNegative;
5779 }
5780
5781 KnownFPClass KnownSrc;
5782 if (InterestedSrcs != fcNone)
5783 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5784 Known&: KnownSrc, Q, Depth: Depth + 1);
5785
5786 Known = KnownFPClass::powi(Src: KnownSrc, N: ExponentKnownBits);
5787 break;
5788 }
5789 case Intrinsic::ldexp: {
5790 KnownFPClass KnownSrc;
5791 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5792 Known&: KnownSrc, Q, Depth: Depth + 1);
5793 // Can refine inf/zero handling based on the exponent operand.
5794 const FPClassTest ExpInfoMask = fcZero | fcSubnormal | fcInf;
5795
5796 const Value *ExpArg = II->getArgOperand(i: 1);
5797 ConstantRange ExpKnownRange =
5798 ((KnownSrc.getKnownFPClasses() & ExpInfoMask) != fcNone)
5799 ? computeConstantRange(V: ExpArg, /*ForSigned=*/true, SQ: Q, Depth: Depth + 1)
5800 : ConstantRange::getFull(
5801 BitWidth: ExpArg->getType()->getScalarSizeInBits());
5802
5803 const fltSemantics &Flt =
5804 II->getType()->getScalarType()->getFltSemantics();
5805
5806 const Function *F = II->getFunction();
5807 DenormalMode Mode =
5808 F ? F->getDenormalMode(FPType: Flt) : DenormalMode::getDynamic();
5809
5810 Known = KnownFPClass::ldexp(Src: KnownSrc, ConstantRangeMin: ExpKnownRange.getSignedMin(),
5811 ConstantRangeMax: ExpKnownRange.getSignedMax(), Flt, Mode);
5812 break;
5813 }
5814 case Intrinsic::arithmetic_fence: {
5815 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5816 Known, Q, Depth: Depth + 1);
5817 break;
5818 }
5819 case Intrinsic::experimental_constrained_sitofp:
5820 case Intrinsic::experimental_constrained_uitofp:
5821 // Cannot produce nan
5822 Known.knownNot(RuleOut: fcNan);
5823
5824 // sitofp and uitofp turn into +0.0 for zero.
5825 Known.knownNot(RuleOut: fcNegZero);
5826
5827 // Integers cannot be subnormal
5828 Known.knownNot(RuleOut: fcSubnormal);
5829
5830 if (IID == Intrinsic::experimental_constrained_uitofp)
5831 Known.signBitMustBeZero();
5832
5833 // TODO: Copy inf handling from instructions
5834 break;
5835
5836 case Intrinsic::amdgcn_fract: {
5837 Known.knownNot(RuleOut: fcInf);
5838
5839 if (InterestedClasses & fcNan) {
5840 KnownFPClass KnownSrc;
5841 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5842 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5843
5844 if (KnownSrc.isKnownNeverInfOrNaN())
5845 Known.knownNot(RuleOut: fcNan);
5846 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5847 Known.knownNot(RuleOut: fcSNan);
5848 }
5849
5850 break;
5851 }
5852 case Intrinsic::amdgcn_rcp: {
5853 KnownFPClass KnownSrc;
5854 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5855 Known&: KnownSrc, Q, Depth: Depth + 1);
5856
5857 Known.propagateNonNaN(Src: KnownSrc);
5858
5859 Type *EltTy = II->getType()->getScalarType();
5860
5861 // f32 denormal always flushed.
5862 if (EltTy->isFloatTy()) {
5863 Known.knownNot(RuleOut: fcSubnormal);
5864 KnownSrc.knownNot(RuleOut: fcSubnormal);
5865 }
5866
5867 if (KnownSrc.isKnownNever(Mask: fcNegative))
5868 Known.knownNot(RuleOut: fcNegative);
5869 if (KnownSrc.isKnownNever(Mask: fcPositive))
5870 Known.knownNot(RuleOut: fcPositive);
5871
5872 if (const Function *F = II->getFunction()) {
5873 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5874 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5875 Known.knownNot(RuleOut: fcPosInf);
5876 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5877 Known.knownNot(RuleOut: fcNegInf);
5878 }
5879
5880 break;
5881 }
5882 case Intrinsic::amdgcn_rsq: {
5883 KnownFPClass KnownSrc;
5884 // The only negative value that can be returned is -inf for -0 inputs.
5885 Known.knownNot(RuleOut: fcNegZero | fcNegSubnormal | fcNegNormal);
5886
5887 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5888 Known&: KnownSrc, Q, Depth: Depth + 1);
5889
5890 // Negative -> nan
5891 if (KnownSrc.isKnownNeverNaN() && KnownSrc.cannotBeOrderedLessThanZero())
5892 Known.knownNot(RuleOut: fcNan);
5893 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5894 Known.knownNot(RuleOut: fcSNan);
5895
5896 // +inf -> +0
5897 if (KnownSrc.isKnownNeverPosInfinity())
5898 Known.knownNot(RuleOut: fcPosZero);
5899
5900 Type *EltTy = II->getType()->getScalarType();
5901
5902 // f32 denormal always flushed.
5903 if (EltTy->isFloatTy())
5904 Known.knownNot(RuleOut: fcPosSubnormal);
5905
5906 if (const Function *F = II->getFunction()) {
5907 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5908
5909 // -0 -> -inf
5910 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5911 Known.knownNot(RuleOut: fcNegInf);
5912
5913 // +0 -> +inf
5914 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5915 Known.knownNot(RuleOut: fcPosInf);
5916 }
5917
5918 break;
5919 }
5920 case Intrinsic::amdgcn_trig_preop: {
5921 // Always returns a value [0, 1)
5922 Known.knownNot(RuleOut: fcNan | fcInf | fcNegative);
5923 break;
5924 }
5925 case Intrinsic::convert_from_arbitrary_fp: {
5926 auto *MD = cast<MetadataAsValue>(Val: II->getArgOperand(i: 1))->getMetadata();
5927 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
5928
5929 const fltSemantics *SrcSemantics =
5930 APFloat::getArbitraryFPSemantics(Format: FormatStr);
5931 if (!SrcSemantics)
5932 break;
5933
5934 const fltSemantics DstSemantics =
5935 II->getType()->getScalarType()->getFltSemantics();
5936
5937 if (!APFloat::semanticsHasNaN(*SrcSemantics))
5938 Known.knownNot(RuleOut: fcNan);
5939
5940 // fcInf can only be cleared if the source format has no Inf encoding
5941 // and the dst max exp can accommodate src max exp.
5942 if (!APFloat::semanticsHasInf(*SrcSemantics) &&
5943 APFloat::semanticsMaxExponent(*SrcSemantics) <=
5944 APFloat::semanticsMaxExponent(DstSemantics))
5945 Known.knownNot(RuleOut: fcInf);
5946
5947 // Check and clear all neg flags for formats that do not have signed
5948 // representation.
5949 if (!APFloat::semanticsHasSignedRepr(*SrcSemantics))
5950 Known.knownNot(RuleOut: fcNegative);
5951
5952 // Check if format has no zero at all (Float8E8M0FNU), or no negative
5953 // zero.
5954 if (!APFloat::semanticsHasZero(*SrcSemantics))
5955 Known.knownNot(RuleOut: fcZero);
5956 else if (SrcSemantics->nanEncoding == fltNanEncoding::NegativeZero)
5957 Known.knownNot(RuleOut: fcNegZero);
5958
5959 // If src lands normally in dest, the result can never be subnormal.
5960 if (APFloat::isRepresentableAsNormalIn(Src: *SrcSemantics, Dst: DstSemantics))
5961 Known.knownNot(RuleOut: fcSubnormal);
5962 break;
5963 }
5964 default:
5965 break;
5966 }
5967
5968 break;
5969 }
5970 case Instruction::FAdd:
5971 case Instruction::FSub: {
5972 KnownFPClass KnownLHS, KnownRHS;
5973 bool WantNegative =
5974 Op->getOpcode() == Instruction::FAdd &&
5975 (InterestedClasses & KnownFPClass::OrderedLessThanZeroMask) != fcNone;
5976 bool WantNaN = (InterestedClasses & fcNan) != fcNone;
5977 bool WantNegZero = (InterestedClasses & fcNegZero) != fcNone;
5978
5979 if (!WantNaN && !WantNegative && !WantNegZero)
5980 break;
5981
5982 FPClassTest InterestedSrcs = InterestedClasses;
5983 if (WantNegative)
5984 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5985 if (InterestedClasses & fcNan)
5986 InterestedSrcs |= fcInf;
5987 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: InterestedSrcs,
5988 Known&: KnownRHS, Q, Depth: Depth + 1);
5989
5990 // Special case fadd x, x, which is the canonical form of fmul x, 2.
5991 bool Self = Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
5992 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT,
5993 Depth: Depth + 1);
5994 if (Self)
5995 KnownLHS = KnownRHS;
5996
5997 if ((WantNaN && KnownRHS.isKnownNeverNaN()) ||
5998 (WantNegative && KnownRHS.cannotBeOrderedLessThanZero()) ||
5999 WantNegZero || Opc == Instruction::FSub) {
6000
6001 // FIXME: Context function should always be passed in separately
6002 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6003 const fltSemantics &FltSem =
6004 Op->getType()->getScalarType()->getFltSemantics();
6005 DenormalMode Mode =
6006 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6007
6008 if (Self && Opc == Instruction::FAdd) {
6009 Known = KnownFPClass::fadd_self(Src: KnownLHS, Mode);
6010 } else {
6011 // RHS is canonically cheaper to compute. Skip inspecting the LHS if
6012 // there's no point.
6013
6014 if (!Self) {
6015 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
6016 Known&: KnownLHS, Q, Depth: Depth + 1);
6017 }
6018
6019 Known = Opc == Instruction::FAdd
6020 ? KnownFPClass::fadd(LHS: KnownLHS, RHS: KnownRHS, Mode)
6021 : KnownFPClass::fsub(LHS: KnownLHS, RHS: KnownRHS, Mode);
6022 }
6023 }
6024
6025 break;
6026 }
6027 case Instruction::FMul: {
6028 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6029 DenormalMode Mode =
6030 F ? F->getDenormalMode(
6031 FPType: Op->getType()->getScalarType()->getFltSemantics())
6032 : DenormalMode::getDynamic();
6033
6034 Value *LHS = Op->getOperand(i: 0);
6035 Value *RHS = Op->getOperand(i: 1);
6036 // X * X is always non-negative or a NaN.
6037 // FIXME: Should check isGuaranteedNotToBeUndef
6038 if (LHS == RHS) {
6039 KnownFPClass KnownSrc;
6040 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownSrc, Q,
6041 Depth: Depth + 1);
6042 Known = KnownFPClass::square(Src: KnownSrc, Mode);
6043 break;
6044 }
6045
6046 KnownFPClass KnownLHS, KnownRHS;
6047
6048 const APFloat *CRHS;
6049 if (match(V: RHS, P: m_APFloat(Res&: CRHS))) {
6050 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
6051 Depth: Depth + 1);
6052 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: *CRHS, Mode);
6053 } else {
6054 computeKnownFPClass(V: RHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS, Q,
6055 Depth: Depth + 1);
6056 // TODO: Improve accuracy in unfused FMA pattern. We can prove an
6057 // additional not-nan if the addend is known-not negative infinity if the
6058 // multiply is known-not infinity.
6059
6060 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
6061 Depth: Depth + 1);
6062 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: KnownRHS, Mode);
6063 }
6064
6065 /// Propgate no-infs if the other source is known smaller than one, such
6066 /// that this cannot introduce overflow.
6067 if (KnownLHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: RHS))
6068 Known.knownNot(RuleOut: fcInf);
6069 else if (KnownRHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: LHS))
6070 Known.knownNot(RuleOut: fcInf);
6071
6072 break;
6073 }
6074 case Instruction::FDiv: {
6075 const bool WantNan = (InterestedClasses & fcNan) != fcNone;
6076
6077 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6078 const fltSemantics &FltSem =
6079 Op->getType()->getScalarType()->getFltSemantics();
6080 DenormalMode Mode =
6081 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6082
6083 if (Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
6084 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT)) {
6085 // X / X is always exactly 1.0 or a NaN.
6086 Known.setKnownFPClasses(fcNan | fcPosNormal);
6087
6088 if (!WantNan)
6089 break;
6090
6091 KnownFPClass KnownSrc;
6092 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts,
6093 InterestedClasses: fcNan | fcInf | fcZero | fcSubnormal, Known&: KnownSrc, Q,
6094 Depth: Depth + 1);
6095
6096 Known = KnownFPClass::fdiv_self(Src: KnownSrc, Mode);
6097 break;
6098 }
6099
6100 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
6101 const bool WantPositive = (InterestedClasses & fcPositive) != fcNone;
6102 if (!WantNan && !WantNegative && !WantPositive)
6103 break;
6104
6105 KnownFPClass KnownLHS, KnownRHS;
6106 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS,
6107 Q, Depth: Depth + 1);
6108
6109 bool KnowSomethingUseful =
6110 KnownRHS.isKnownNeverNaN() ||
6111 KnownRHS.isKnownNever(Mask: fcNegNormal | fcNegSubnormal) ||
6112 KnownRHS.isKnownNever(Mask: fcPosNormal | fcPosSubnormal);
6113
6114 if (KnowSomethingUseful)
6115 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS,
6116 Q, Depth: Depth + 1);
6117
6118 Known = KnownFPClass::fdiv(LHS: KnownLHS, RHS: KnownRHS, Mode);
6119 break;
6120 }
6121 case Instruction::FRem: {
6122 const bool WantNan = (InterestedClasses & fcNan) != fcNone;
6123
6124 Known.knownNot(RuleOut: fcInf);
6125
6126 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6127 DenormalMode Mode =
6128 F ? F->getDenormalMode(
6129 FPType: Op->getType()->getScalarType()->getFltSemantics())
6130 : DenormalMode::getDynamic();
6131
6132 if (Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
6133 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT)) {
6134 // X % X is always exactly [+-]0.0 or a NaN.
6135 Known.setKnownFPClasses(fcNan | fcZero);
6136
6137 if (!WantNan)
6138 break;
6139
6140 KnownFPClass KnownSrc;
6141 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts,
6142 InterestedClasses: fcNan | fcInf | fcZero | fcSubnormal, Known&: KnownSrc, Q,
6143 Depth: Depth + 1);
6144
6145 Known = KnownFPClass::frem_self(Src: KnownSrc, Mode);
6146 break;
6147 }
6148
6149 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
6150 const bool WantPositive = (InterestedClasses & fcPositive) != fcNone;
6151 if (!WantNan && !WantNegative && !WantPositive)
6152 break;
6153
6154 KnownFPClass KnownLHS, KnownRHS;
6155 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts,
6156 InterestedClasses: fcNan | fcInf | fcZero | fcNegative, Known&: KnownRHS, Q,
6157 Depth: Depth + 1);
6158
6159 bool KnowSomethingUseful = KnownRHS.isKnownNeverNaN() ||
6160 KnownRHS.isKnownNever(Mask: fcNegative) ||
6161 KnownRHS.isKnownNever(Mask: fcPositive);
6162
6163 if (KnowSomethingUseful || WantPositive)
6164 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS,
6165 Q, Depth: Depth + 1);
6166
6167 Known = KnownFPClass::frem(LHS: KnownLHS, RHS: KnownRHS, Mode);
6168
6169 break;
6170 }
6171 case Instruction::FPExt: {
6172 KnownFPClass KnownSrc;
6173 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
6174 Known&: KnownSrc, Q, Depth: Depth + 1);
6175
6176 const fltSemantics &DstTy =
6177 Op->getType()->getScalarType()->getFltSemantics();
6178 const fltSemantics &SrcTy =
6179 Op->getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
6180
6181 Known = KnownFPClass::fpext(KnownSrc, DstTy, SrcTy);
6182 break;
6183 }
6184 case Instruction::FPTrunc: {
6185 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known, Q,
6186 Depth);
6187 break;
6188 }
6189 case Instruction::SIToFP:
6190 case Instruction::UIToFP: {
6191 // Cannot produce nan
6192 Known.knownNot(RuleOut: fcNan);
6193
6194 // Integers cannot be subnormal
6195 Known.knownNot(RuleOut: fcSubnormal);
6196
6197 // sitofp and uitofp turn into +0.0 for zero.
6198 Known.knownNot(RuleOut: fcNegZero);
6199
6200 // UIToFP is always non-negative regardless of known bits.
6201 if (Op->getOpcode() == Instruction::UIToFP)
6202 Known.signBitMustBeZero();
6203
6204 // Only compute known bits if we can learn something useful from them.
6205 if (!(InterestedClasses & (fcPosZero | fcNormal | fcInf)))
6206 break;
6207
6208 KnownBits IntKnown =
6209 computeKnownBits(V: Op->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
6210
6211 // If the integer is non-zero, the result cannot be +0.0
6212 if (IntKnown.isNonZero())
6213 Known.knownNot(RuleOut: fcPosZero);
6214
6215 if (Op->getOpcode() == Instruction::SIToFP) {
6216 // If the signed integer is known non-negative, the result is
6217 // non-negative. If the signed integer is known negative, the result is
6218 // negative.
6219 if (IntKnown.isNonNegative()) {
6220 Known.signBitMustBeZero();
6221 } else if (IntKnown.isNegative()) {
6222 Known.signBitMustBeOne();
6223 }
6224 }
6225
6226 // Guard kept for ilogb()
6227 if (InterestedClasses & fcInf) {
6228 // Get width of largest magnitude integer known.
6229 // This still works for a signed minimum value because the largest FP
6230 // value is scaled by some fraction close to 2.0 (1.0 + 0.xxxx).
6231 int IntSize = IntKnown.getBitWidth();
6232 if (Op->getOpcode() == Instruction::UIToFP)
6233 IntSize -= IntKnown.countMinLeadingZeros();
6234 else if (Op->getOpcode() == Instruction::SIToFP)
6235 IntSize -= IntKnown.countMinSignBits();
6236
6237 // If the exponent of the largest finite FP value can hold the largest
6238 // integer, the result of the cast must be finite.
6239 Type *FPTy = Op->getType()->getScalarType();
6240 if (ilogb(Arg: APFloat::getLargest(Sem: FPTy->getFltSemantics())) >= IntSize)
6241 Known.knownNot(RuleOut: fcInf);
6242 }
6243
6244 break;
6245 }
6246 case Instruction::ExtractElement: {
6247 // Look through extract element. If the index is non-constant or
6248 // out-of-range demand all elements, otherwise just the extracted element.
6249 const Value *Vec = Op->getOperand(i: 0);
6250
6251 APInt DemandedVecElts;
6252 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
6253 unsigned NumElts = VecTy->getNumElements();
6254 DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
6255 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1));
6256 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
6257 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
6258 } else {
6259 DemandedVecElts = APInt(1, 1);
6260 }
6261
6262 return computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known,
6263 Q, Depth: Depth + 1);
6264 }
6265 case Instruction::InsertElement: {
6266 if (isa<ScalableVectorType>(Val: Op->getType()))
6267 return;
6268
6269 const Value *Vec = Op->getOperand(i: 0);
6270 const Value *Elt = Op->getOperand(i: 1);
6271 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 2));
6272 unsigned NumElts = DemandedElts.getBitWidth();
6273 APInt DemandedVecElts = DemandedElts;
6274 bool NeedsElt = true;
6275 // If we know the index we are inserting to, clear it from Vec check.
6276 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
6277 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
6278 NeedsElt = DemandedElts[CIdx->getZExtValue()];
6279 }
6280
6281 // Do we demand the inserted element?
6282 if (NeedsElt) {
6283 computeKnownFPClass(V: Elt, Known, InterestedClasses, Q, Depth: Depth + 1);
6284 // If we don't know any bits, early out.
6285 if (Known.isUnknown())
6286 break;
6287 } else {
6288 Known.setKnownFPClasses(fcNone);
6289 }
6290
6291 // Do we need anymore elements from Vec?
6292 if (!DemandedVecElts.isZero()) {
6293 KnownFPClass Known2;
6294 computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known&: Known2, Q,
6295 Depth: Depth + 1);
6296 Known |= Known2;
6297 }
6298
6299 break;
6300 }
6301 case Instruction::ShuffleVector: {
6302 // Handle vector splat idiom
6303 if (Value *Splat = getSplatValue(V)) {
6304 computeKnownFPClass(V: Splat, Known, InterestedClasses, Q, Depth: Depth + 1);
6305 break;
6306 }
6307
6308 // For undef elements, we don't know anything about the common state of
6309 // the shuffle result.
6310 APInt DemandedLHS, DemandedRHS;
6311 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: Op);
6312 if (!Shuf || !getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
6313 return;
6314
6315 if (!!DemandedLHS) {
6316 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
6317 computeKnownFPClass(V: LHS, DemandedElts: DemandedLHS, InterestedClasses, Known, Q,
6318 Depth: Depth + 1);
6319
6320 // If we don't know any bits, early out.
6321 if (Known.isUnknown())
6322 break;
6323 } else {
6324 Known.setKnownFPClasses(fcNone);
6325 }
6326
6327 if (!!DemandedRHS) {
6328 KnownFPClass Known2;
6329 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
6330 computeKnownFPClass(V: RHS, DemandedElts: DemandedRHS, InterestedClasses, Known&: Known2, Q,
6331 Depth: Depth + 1);
6332 Known |= Known2;
6333 }
6334
6335 break;
6336 }
6337 case Instruction::ExtractValue: {
6338 const ExtractValueInst *Extract = cast<ExtractValueInst>(Val: Op);
6339 ArrayRef<unsigned> Indices = Extract->getIndices();
6340 const Value *Src = Extract->getAggregateOperand();
6341 if (isa<StructType>(Val: Src->getType()) && Indices.size() == 1 &&
6342 Indices[0] == 0) {
6343 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Src)) {
6344 switch (II->getIntrinsicID()) {
6345 case Intrinsic::frexp: {
6346 Known.knownNot(RuleOut: fcSubnormal);
6347
6348 KnownFPClass KnownSrc;
6349 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
6350 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
6351
6352 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6353 const fltSemantics &FltSem =
6354 Op->getType()->getScalarType()->getFltSemantics();
6355
6356 DenormalMode Mode =
6357 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6358 Known = KnownFPClass::frexp_mant(Src: KnownSrc, Mode);
6359 return;
6360 }
6361 default:
6362 break;
6363 }
6364 }
6365 }
6366
6367 computeKnownFPClass(V: Src, DemandedElts, InterestedClasses, Known, Q,
6368 Depth: Depth + 1);
6369 break;
6370 }
6371 case Instruction::PHI: {
6372 const PHINode *P = cast<PHINode>(Val: Op);
6373 // Unreachable blocks may have zero-operand PHI nodes.
6374 if (P->getNumIncomingValues() == 0)
6375 break;
6376
6377 // Otherwise take the unions of the known bit sets of the operands,
6378 // taking conservative care to avoid excessive recursion.
6379 const unsigned PhiRecursionLimit = MaxAnalysisRecursionDepth - 2;
6380
6381 if (Depth < PhiRecursionLimit) {
6382 // Skip if every incoming value references to ourself.
6383 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
6384 break;
6385
6386 bool First = true;
6387
6388 for (const Use &U : P->operands()) {
6389 Value *IncValue;
6390 Instruction *CxtI;
6391 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CxtI);
6392 // Skip direct self references.
6393 if (IncValue == P)
6394 continue;
6395
6396 KnownFPClass KnownSrc;
6397 // Recurse, but cap the recursion to two levels, because we don't want
6398 // to waste time spinning around in loops. We need at least depth 2 to
6399 // detect known sign bits.
6400 computeKnownFPClass(V: IncValue, DemandedElts, InterestedClasses, Known&: KnownSrc,
6401 Q: Q.getWithoutCondContext().getWithInstruction(I: CxtI),
6402 Depth: PhiRecursionLimit);
6403
6404 if (First) {
6405 Known = KnownSrc;
6406 First = false;
6407 } else {
6408 Known |= KnownSrc;
6409 }
6410
6411 if (Known.getKnownFPClasses() == fcAllFlags)
6412 break;
6413 }
6414 }
6415
6416 // Look for the case of a for loop which has a positive
6417 // initial value and is incremented by a squared value.
6418 // This will propagate sign information out of such loops.
6419 if (P->getNumIncomingValues() != 2 || Known.cannotBeOrderedLessThanZero())
6420 break;
6421 for (unsigned I = 0; I < 2; I++) {
6422 Value *RecurValue = P->getIncomingValue(i: 1 - I);
6423 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: RecurValue);
6424 if (!II)
6425 continue;
6426 Value *R, *L, *Init;
6427 PHINode *PN;
6428 if (matchSimpleTernaryIntrinsicRecurrence(I: II, P&: PN, Init, OtherOp0&: L, OtherOp1&: R) &&
6429 PN == P) {
6430 switch (II->getIntrinsicID()) {
6431 case Intrinsic::fma:
6432 case Intrinsic::fmuladd: {
6433 KnownFPClass KnownStart;
6434 computeKnownFPClass(V: Init, DemandedElts, InterestedClasses, Known&: KnownStart,
6435 Q, Depth: Depth + 1);
6436 if (KnownStart.cannotBeOrderedLessThanZero() && L == R &&
6437 isGuaranteedNotToBeUndef(V: L, AC: Q.AC, CtxI: Q.CxtI, DT: Q.DT, Depth: Depth + 1))
6438 Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
6439 break;
6440 }
6441 }
6442 }
6443 }
6444 break;
6445 }
6446 case Instruction::BitCast: {
6447 const Value *Src;
6448 if (!match(V: Op, P: m_ElementWiseBitCast(Op: m_Value(V&: Src))) ||
6449 !Src->getType()->isIntOrIntVectorTy())
6450 break;
6451
6452 const Type *Ty = Op->getType();
6453
6454 Value *CastLHS, *CastRHS;
6455
6456 // Match bitcast(umax(bitcast(a), bitcast(b)))
6457 if (match(V: Src, P: m_c_MaxOrMin(L: m_BitCast(Op: m_Value(V&: CastLHS)),
6458 R: m_BitCast(Op: m_Value(V&: CastRHS)))) &&
6459 CastLHS->getType() == Ty && CastRHS->getType() == Ty) {
6460 KnownFPClass KnownLHS, KnownRHS;
6461 computeKnownFPClass(V: CastRHS, DemandedElts, InterestedClasses, Known&: KnownRHS, Q,
6462 Depth: Depth + 1);
6463 if (!KnownRHS.isUnknown()) {
6464 computeKnownFPClass(V: CastLHS, DemandedElts, InterestedClasses, Known&: KnownLHS,
6465 Q, Depth: Depth + 1);
6466 Known = KnownLHS | KnownRHS;
6467 }
6468
6469 return;
6470 }
6471
6472 const Type *EltTy = Ty->getScalarType();
6473 KnownBits Bits(EltTy->getPrimitiveSizeInBits());
6474 computeKnownBits(V: Src, DemandedElts, Known&: Bits, Q, Depth: Depth + 1);
6475
6476 Known = KnownFPClass::bitcast(FltSemantics: EltTy->getFltSemantics(), Bits);
6477 break;
6478 }
6479 default:
6480 break;
6481 }
6482}
6483
6484KnownFPClass llvm::computeKnownFPClass(const Value *V,
6485 const APInt &DemandedElts,
6486 FPClassTest InterestedClasses,
6487 const SimplifyQuery &SQ,
6488 unsigned Depth) {
6489 KnownFPClass KnownClasses;
6490 ::computeKnownFPClass(V, DemandedElts, InterestedClasses, Known&: KnownClasses, Q: SQ,
6491 Depth);
6492 return KnownClasses;
6493}
6494
6495KnownFPClass llvm::computeKnownFPClass(const Value *V,
6496 FPClassTest InterestedClasses,
6497 const SimplifyQuery &SQ,
6498 unsigned Depth) {
6499 KnownFPClass Known;
6500 ::computeKnownFPClass(V, Known, InterestedClasses, Q: SQ, Depth);
6501 return Known;
6502}
6503
6504KnownFPClass llvm::computeKnownFPClass(
6505 const Value *V, const DataLayout &DL, FPClassTest InterestedClasses,
6506 const TargetLibraryInfo *TLI, AssumptionCache *AC, const Instruction *CxtI,
6507 const DominatorTree *DT, bool UseInstrInfo, unsigned Depth) {
6508 return computeKnownFPClass(V, InterestedClasses,
6509 SQ: SimplifyQuery(DL, TLI, DT, AC, CxtI, UseInstrInfo),
6510 Depth);
6511}
6512
6513KnownFPClass
6514llvm::computeKnownFPClass(const Value *V, const APInt &DemandedElts,
6515 FastMathFlags FMF, FPClassTest InterestedClasses,
6516 const SimplifyQuery &SQ, unsigned Depth) {
6517 if (FMF.noNaNs())
6518 InterestedClasses &= ~fcNan;
6519 if (FMF.noInfs())
6520 InterestedClasses &= ~fcInf;
6521
6522 KnownFPClass Result =
6523 computeKnownFPClass(V, DemandedElts, InterestedClasses, SQ, Depth);
6524
6525 if (FMF.noNaNs())
6526 Result.setKnownFPClasses(Result.getKnownFPClasses() & ~fcNan);
6527 if (FMF.noInfs())
6528 Result.setKnownFPClasses(Result.getKnownFPClasses() & ~fcInf);
6529 return Result;
6530}
6531
6532KnownFPClass llvm::computeKnownFPClass(const Value *V, FastMathFlags FMF,
6533 FPClassTest InterestedClasses,
6534 const SimplifyQuery &SQ,
6535 unsigned Depth) {
6536 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
6537 APInt DemandedElts =
6538 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
6539 return computeKnownFPClass(V, DemandedElts, FMF, InterestedClasses, SQ,
6540 Depth);
6541}
6542
6543bool llvm::cannotBeNegativeZero(const Value *V, const SimplifyQuery &SQ,
6544 unsigned Depth) {
6545 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNegZero, SQ, Depth);
6546 return Known.isKnownNeverNegZero();
6547}
6548
6549bool llvm::cannotBeOrderedLessThanZero(const Value *V, const SimplifyQuery &SQ,
6550 unsigned Depth) {
6551 KnownFPClass Known =
6552 computeKnownFPClass(V, InterestedClasses: KnownFPClass::OrderedLessThanZeroMask, SQ, Depth);
6553 return Known.cannotBeOrderedLessThanZero();
6554}
6555
6556bool llvm::isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ,
6557 unsigned Depth) {
6558 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf, SQ, Depth);
6559 return Known.isKnownNeverInfinity();
6560}
6561
6562/// Return true if the floating-point value can never contain a NaN or infinity.
6563bool llvm::isKnownNeverInfOrNaN(const Value *V, const SimplifyQuery &SQ,
6564 unsigned Depth) {
6565 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf | fcNan, SQ, Depth);
6566 return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity();
6567}
6568
6569/// Return true if the floating-point scalar value is not a NaN or if the
6570/// floating-point vector value has no NaN elements. Return false if a value
6571/// could ever be NaN.
6572bool llvm::isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ,
6573 unsigned Depth) {
6574 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNan, SQ, Depth);
6575 return Known.isKnownNeverNaN();
6576}
6577
6578/// Return false if we can prove that the specified FP value's sign bit is 0.
6579/// Return true if we can prove that the specified FP value's sign bit is 1.
6580/// Otherwise return std::nullopt.
6581std::optional<bool> llvm::computeKnownFPSignBit(const Value *V,
6582 const SimplifyQuery &SQ,
6583 unsigned Depth) {
6584 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcAllFlags, SQ, Depth);
6585 return Known.getSignBit();
6586}
6587
6588bool llvm::canIgnoreSignBitOfZero(const Use &U) {
6589 auto *User = cast<Instruction>(Val: U.getUser());
6590 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6591 if (FPOp->hasNoSignedZeros())
6592 return true;
6593 }
6594
6595 switch (User->getOpcode()) {
6596 case Instruction::FPToSI:
6597 case Instruction::FPToUI:
6598 return true;
6599 case Instruction::FCmp:
6600 // fcmp treats both positive and negative zero as equal.
6601 return true;
6602 case Instruction::Call:
6603 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6604 switch (II->getIntrinsicID()) {
6605 case Intrinsic::fabs:
6606 return true;
6607 case Intrinsic::copysign:
6608 return U.getOperandNo() == 0;
6609 case Intrinsic::is_fpclass: {
6610 auto Test =
6611 static_cast<FPClassTest>(
6612 cast<ConstantInt>(Val: II->getArgOperand(i: 1))->getZExtValue()) &
6613 FPClassTest::fcZero;
6614 return Test == FPClassTest::fcZero || Test == FPClassTest::fcNone;
6615 }
6616 default:
6617 return false;
6618 }
6619 }
6620 return false;
6621 default:
6622 return false;
6623 }
6624}
6625
6626bool llvm::canIgnoreSignBitOfNaN(const Use &U) {
6627 auto *User = cast<Instruction>(Val: U.getUser());
6628 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6629 if (FPOp->hasNoNaNs())
6630 return true;
6631 }
6632
6633 switch (User->getOpcode()) {
6634 case Instruction::FPToSI:
6635 case Instruction::FPToUI:
6636 return true;
6637 // Proper FP math operations ignore the sign bit of NaN.
6638 case Instruction::FAdd:
6639 case Instruction::FSub:
6640 case Instruction::FMul:
6641 case Instruction::FDiv:
6642 case Instruction::FRem:
6643 case Instruction::FPTrunc:
6644 case Instruction::FPExt:
6645 case Instruction::FCmp:
6646 return true;
6647 // Bitwise FP operations should preserve the sign bit of NaN.
6648 case Instruction::FNeg:
6649 case Instruction::Select:
6650 case Instruction::PHI:
6651 return false;
6652 case Instruction::Ret:
6653 return User->getFunction()->getAttributes().getRetNoFPClass() &
6654 FPClassTest::fcNan;
6655 case Instruction::Call:
6656 case Instruction::Invoke: {
6657 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6658 switch (II->getIntrinsicID()) {
6659 case Intrinsic::fabs:
6660 return true;
6661 case Intrinsic::copysign:
6662 return U.getOperandNo() == 0;
6663 // Other proper FP math intrinsics ignore the sign bit of NaN.
6664 case Intrinsic::maxnum:
6665 case Intrinsic::minnum:
6666 case Intrinsic::maximum:
6667 case Intrinsic::minimum:
6668 case Intrinsic::maximumnum:
6669 case Intrinsic::minimumnum:
6670 case Intrinsic::canonicalize:
6671 case Intrinsic::fma:
6672 case Intrinsic::fmuladd:
6673 case Intrinsic::sqrt:
6674 case Intrinsic::pow:
6675 case Intrinsic::powi:
6676 case Intrinsic::fptoui_sat:
6677 case Intrinsic::fptosi_sat:
6678 case Intrinsic::is_fpclass:
6679 return true;
6680 default:
6681 return false;
6682 }
6683 }
6684
6685 FPClassTest NoFPClass =
6686 cast<CallBase>(Val: User)->getParamNoFPClass(i: U.getOperandNo());
6687 return NoFPClass & FPClassTest::fcNan;
6688 }
6689 default:
6690 return false;
6691 }
6692}
6693
6694bool llvm::isKnownIntegral(const Value *V, const SimplifyQuery &SQ,
6695 FastMathFlags FMF) {
6696 if (isa<PoisonValue>(Val: V))
6697 return true;
6698 if (isa<UndefValue>(Val: V))
6699 return false;
6700
6701 if (match(V, P: m_CheckedFp(CheckFn: [](const APFloat &Val) { return Val.isInteger(); })))
6702 return true;
6703
6704 const Instruction *I = dyn_cast<Instruction>(Val: V);
6705 if (!I)
6706 return false;
6707
6708 switch (I->getOpcode()) {
6709 case Instruction::SIToFP:
6710 case Instruction::UIToFP:
6711 // TODO: Could check nofpclass(inf) on incoming argument
6712 if (FMF.noInfs())
6713 return true;
6714
6715 // Need to check int size cannot produce infinity, which computeKnownFPClass
6716 // knows how to do already.
6717 return isKnownNeverInfinity(V: I, SQ);
6718 case Instruction::Call: {
6719 const CallInst *CI = cast<CallInst>(Val: I);
6720 switch (CI->getIntrinsicID()) {
6721 case Intrinsic::trunc:
6722 case Intrinsic::floor:
6723 case Intrinsic::ceil:
6724 case Intrinsic::rint:
6725 case Intrinsic::nearbyint:
6726 case Intrinsic::round:
6727 case Intrinsic::roundeven:
6728 return (FMF.noInfs() && FMF.noNaNs()) || isKnownNeverInfOrNaN(V: I, SQ);
6729 default:
6730 break;
6731 }
6732
6733 break;
6734 }
6735 default:
6736 break;
6737 }
6738
6739 return false;
6740}
6741
6742Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
6743
6744 // All byte-wide stores are splatable, even of arbitrary variables.
6745 if (V->getType()->isIntegerTy(BitWidth: 8))
6746 return V;
6747
6748 LLVMContext &Ctx = V->getContext();
6749
6750 // Undef don't care.
6751 auto *UndefInt8 = UndefValue::get(T: Type::getInt8Ty(C&: Ctx));
6752 if (isa<UndefValue>(Val: V))
6753 return UndefInt8;
6754
6755 // Return poison for zero-sized type.
6756 if (DL.getTypeStoreSize(Ty: V->getType()).isZero())
6757 return PoisonValue::get(T: Type::getInt8Ty(C&: Ctx));
6758
6759 Constant *C = dyn_cast<Constant>(Val: V);
6760 if (!C) {
6761 // Conceptually, we could handle things like:
6762 // %a = zext i8 %X to i16
6763 // %b = shl i16 %a, 8
6764 // %c = or i16 %a, %b
6765 // but until there is an example that actually needs this, it doesn't seem
6766 // worth worrying about.
6767 return nullptr;
6768 }
6769
6770 // Handle 'null' ConstantArrayZero etc.
6771 if (C->isNullValue())
6772 return Constant::getNullValue(Ty: Type::getInt8Ty(C&: Ctx));
6773
6774 // Constant floating-point values can be handled as integer values if the
6775 // corresponding integer value is "byteable". An important case is 0.0.
6776 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
6777 Type *ScalarTy = CFP->getType()->getScalarType();
6778 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
6779 return isBytewiseValue(
6780 V: ConstantInt::get(Context&: Ctx, V: CFP->getValue().bitcastToAPInt()), DL);
6781
6782 // Don't handle long double formats, which have strange constraints.
6783 return nullptr;
6784 }
6785
6786 // We can handle constant integers that are multiple of 8 bits.
6787 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
6788 if (CI->getBitWidth() % 8 == 0) {
6789 if (!CI->getValue().isSplat(SplatSizeInBits: 8))
6790 return nullptr;
6791 return ConstantInt::get(Context&: Ctx, V: CI->getValue().trunc(width: 8));
6792 }
6793 }
6794
6795 if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
6796 if (CE->getOpcode() == Instruction::IntToPtr) {
6797 if (auto *PtrTy = dyn_cast<PointerType>(Val: CE->getType())) {
6798 unsigned BitWidth = DL.getPointerSizeInBits(AS: PtrTy->getAddressSpace());
6799 if (Constant *Op = ConstantFoldIntegerCast(
6800 C: CE->getOperand(i_nocapture: 0), DestTy: Type::getIntNTy(C&: Ctx, N: BitWidth), IsSigned: false, DL))
6801 return isBytewiseValue(V: Op, DL);
6802 }
6803 }
6804 }
6805
6806 auto Merge = [&](Value *LHS, Value *RHS) -> Value * {
6807 if (LHS == RHS)
6808 return LHS;
6809 if (!LHS || !RHS)
6810 return nullptr;
6811 if (LHS == UndefInt8)
6812 return RHS;
6813 if (RHS == UndefInt8)
6814 return LHS;
6815 return nullptr;
6816 };
6817
6818 if (ConstantDataSequential *CA = dyn_cast<ConstantDataSequential>(Val: C)) {
6819 Value *Val = UndefInt8;
6820 for (uint64_t I = 0, E = CA->getNumElements(); I != E; ++I)
6821 if (!(Val = Merge(Val, isBytewiseValue(V: CA->getElementAsConstant(i: I), DL))))
6822 return nullptr;
6823 return Val;
6824 }
6825
6826 if (isa<ConstantAggregate>(Val: C)) {
6827 Value *Val = UndefInt8;
6828 for (Value *Op : C->operands())
6829 if (!(Val = Merge(Val, isBytewiseValue(V: Op, DL))))
6830 return nullptr;
6831 return Val;
6832 }
6833
6834 // Don't try to handle the handful of other constants.
6835 return nullptr;
6836}
6837
6838// This is the recursive version of BuildSubAggregate. It takes a few different
6839// arguments. Idxs is the index within the nested struct From that we are
6840// looking at now (which is of type IndexedType). IdxSkip is the number of
6841// indices from Idxs that should be left out when inserting into the resulting
6842// struct. To is the result struct built so far, new insertvalue instructions
6843// build on that.
6844static Value *BuildSubAggregate(Value *From, Value *To, Type *IndexedType,
6845 SmallVectorImpl<unsigned> &Idxs,
6846 unsigned IdxSkip,
6847 BasicBlock::iterator InsertBefore) {
6848 StructType *STy = dyn_cast<StructType>(Val: IndexedType);
6849 if (STy) {
6850 // Save the original To argument so we can modify it
6851 Value *OrigTo = To;
6852 // General case, the type indexed by Idxs is a struct
6853 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6854 // Process each struct element recursively
6855 Idxs.push_back(Elt: i);
6856 Value *PrevTo = To;
6857 To = BuildSubAggregate(From, To, IndexedType: STy->getElementType(N: i), Idxs, IdxSkip,
6858 InsertBefore);
6859 Idxs.pop_back();
6860 if (!To) {
6861 // Couldn't find any inserted value for this index? Cleanup
6862 while (PrevTo != OrigTo) {
6863 InsertValueInst* Del = cast<InsertValueInst>(Val: PrevTo);
6864 PrevTo = Del->getAggregateOperand();
6865 Del->eraseFromParent();
6866 }
6867 // Stop processing elements
6868 break;
6869 }
6870 }
6871 // If we successfully found a value for each of our subaggregates
6872 if (To)
6873 return To;
6874 }
6875 // Base case, the type indexed by SourceIdxs is not a struct, or not all of
6876 // the struct's elements had a value that was inserted directly. In the latter
6877 // case, perhaps we can't determine each of the subelements individually, but
6878 // we might be able to find the complete struct somewhere.
6879
6880 // Find the value that is at that particular spot
6881 Value *V = FindInsertedValue(V: From, idx_range: Idxs);
6882
6883 if (!V)
6884 return nullptr;
6885
6886 // Insert the value in the new (sub) aggregate
6887 return InsertValueInst::Create(Agg: To, Val: V, Idxs: ArrayRef(Idxs).slice(N: IdxSkip), NameStr: "tmp",
6888 InsertBefore);
6889}
6890
6891// This helper takes a nested struct and extracts a part of it (which is again a
6892// struct) into a new value. For example, given the struct:
6893// { a, { b, { c, d }, e } }
6894// and the indices "1, 1" this returns
6895// { c, d }.
6896//
6897// It does this by inserting an insertvalue for each element in the resulting
6898// struct, as opposed to just inserting a single struct. This will only work if
6899// each of the elements of the substruct are known (ie, inserted into From by an
6900// insertvalue instruction somewhere).
6901//
6902// All inserted insertvalue instructions are inserted before InsertBefore
6903static Value *BuildSubAggregate(Value *From, ArrayRef<unsigned> idx_range,
6904 BasicBlock::iterator InsertBefore) {
6905 Type *IndexedType = ExtractValueInst::getIndexedType(Agg: From->getType(),
6906 Idxs: idx_range);
6907 Value *To = PoisonValue::get(T: IndexedType);
6908 SmallVector<unsigned, 10> Idxs(idx_range);
6909 unsigned IdxSkip = Idxs.size();
6910
6911 return BuildSubAggregate(From, To, IndexedType, Idxs, IdxSkip, InsertBefore);
6912}
6913
6914/// Given an aggregate and a sequence of indices, see if the scalar value
6915/// indexed is already around as a register, for example if it was inserted
6916/// directly into the aggregate.
6917///
6918/// If InsertBefore is not null, this function will duplicate (modified)
6919/// insertvalues when a part of a nested struct is extracted.
6920Value *
6921llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
6922 std::optional<BasicBlock::iterator> InsertBefore) {
6923 // Nothing to index? Just return V then (this is useful at the end of our
6924 // recursion).
6925 if (idx_range.empty())
6926 return V;
6927 // We have indices, so V should have an indexable type.
6928 assert((V->getType()->isStructTy() || V->getType()->isArrayTy()) &&
6929 "Not looking at a struct or array?");
6930 assert(ExtractValueInst::getIndexedType(V->getType(), idx_range) &&
6931 "Invalid indices for type?");
6932
6933 if (Constant *C = dyn_cast<Constant>(Val: V)) {
6934 C = C->getAggregateElement(Elt: idx_range[0]);
6935 if (!C) return nullptr;
6936 return FindInsertedValue(V: C, idx_range: idx_range.slice(N: 1), InsertBefore);
6937 }
6938
6939 if (InsertValueInst *I = dyn_cast<InsertValueInst>(Val: V)) {
6940 // Loop the indices for the insertvalue instruction in parallel with the
6941 // requested indices
6942 const unsigned *req_idx = idx_range.begin();
6943 for (const unsigned *i = I->idx_begin(), *e = I->idx_end();
6944 i != e; ++i, ++req_idx) {
6945 if (req_idx == idx_range.end()) {
6946 // We can't handle this without inserting insertvalues
6947 if (!InsertBefore)
6948 return nullptr;
6949
6950 // The requested index identifies a part of a nested aggregate. Handle
6951 // this specially. For example,
6952 // %A = insertvalue { i32, {i32, i32 } } undef, i32 10, 1, 0
6953 // %B = insertvalue { i32, {i32, i32 } } %A, i32 11, 1, 1
6954 // %C = extractvalue {i32, { i32, i32 } } %B, 1
6955 // This can be changed into
6956 // %A = insertvalue {i32, i32 } undef, i32 10, 0
6957 // %C = insertvalue {i32, i32 } %A, i32 11, 1
6958 // which allows the unused 0,0 element from the nested struct to be
6959 // removed.
6960 return BuildSubAggregate(From: V, idx_range: ArrayRef(idx_range.begin(), req_idx),
6961 InsertBefore: *InsertBefore);
6962 }
6963
6964 // This insert value inserts something else than what we are looking for.
6965 // See if the (aggregate) value inserted into has the value we are
6966 // looking for, then.
6967 if (*req_idx != *i)
6968 return FindInsertedValue(V: I->getAggregateOperand(), idx_range,
6969 InsertBefore);
6970 }
6971 // If we end up here, the indices of the insertvalue match with those
6972 // requested (though possibly only partially). Now we recursively look at
6973 // the inserted value, passing any remaining indices.
6974 return FindInsertedValue(V: I->getInsertedValueOperand(),
6975 idx_range: ArrayRef(req_idx, idx_range.end()), InsertBefore);
6976 }
6977
6978 if (ExtractValueInst *I = dyn_cast<ExtractValueInst>(Val: V)) {
6979 // If we're extracting a value from an aggregate that was extracted from
6980 // something else, we can extract from that something else directly instead.
6981 // However, we will need to chain I's indices with the requested indices.
6982
6983 // Calculate the number of indices required
6984 unsigned size = I->getNumIndices() + idx_range.size();
6985 // Allocate some space to put the new indices in
6986 SmallVector<unsigned, 5> Idxs;
6987 Idxs.reserve(N: size);
6988 // Add indices from the extract value instruction
6989 Idxs.append(in_start: I->idx_begin(), in_end: I->idx_end());
6990
6991 // Add requested indices
6992 Idxs.append(in_start: idx_range.begin(), in_end: idx_range.end());
6993
6994 assert(Idxs.size() == size
6995 && "Number of indices added not correct?");
6996
6997 return FindInsertedValue(V: I->getAggregateOperand(), idx_range: Idxs, InsertBefore);
6998 }
6999 // Otherwise, we don't know (such as, extracting from a function return value
7000 // or load instruction)
7001 return nullptr;
7002}
7003
7004// If V refers to an initialized global constant, set Slice either to
7005// its initializer if the size of its elements equals ElementSize, or,
7006// for ElementSize == 8, to its representation as an array of unsiged
7007// char. Return true on success.
7008// Offset is in the unit "nr of ElementSize sized elements".
7009bool llvm::getConstantDataArrayInfo(const Value *V,
7010 ConstantDataArraySlice &Slice,
7011 unsigned ElementSize, uint64_t Offset) {
7012 assert(V && "V should not be null.");
7013 assert((ElementSize % 8) == 0 &&
7014 "ElementSize expected to be a multiple of the size of a byte.");
7015 unsigned ElementSizeInBytes = ElementSize / 8;
7016
7017 // Drill down into the pointer expression V, ignoring any intervening
7018 // casts, and determine the identity of the object it references along
7019 // with the cumulative byte offset into it.
7020 const GlobalVariable *GV =
7021 dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V));
7022 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
7023 // Fail if V is not based on constant global object.
7024 return false;
7025
7026 const DataLayout &DL = GV->getDataLayout();
7027 APInt Off(DL.getIndexTypeSizeInBits(Ty: V->getType()), 0);
7028
7029 if (GV != V->stripAndAccumulateConstantOffsets(DL, Offset&: Off,
7030 /*AllowNonInbounds*/ true))
7031 // Fail if a constant offset could not be determined.
7032 return false;
7033
7034 uint64_t StartIdx = Off.getLimitedValue();
7035 if (StartIdx == UINT64_MAX)
7036 // Fail if the constant offset is excessive.
7037 return false;
7038
7039 // Off/StartIdx is in the unit of bytes. So we need to convert to number of
7040 // elements. Simply bail out if that isn't possible.
7041 if ((StartIdx % ElementSizeInBytes) != 0)
7042 return false;
7043
7044 Offset += StartIdx / ElementSizeInBytes;
7045 ConstantDataArray *Array = nullptr;
7046 ArrayType *ArrayTy = nullptr;
7047
7048 if (GV->getInitializer()->isNullValue()) {
7049 Type *GVTy = GV->getValueType();
7050 uint64_t SizeInBytes = DL.getTypeStoreSize(Ty: GVTy).getFixedValue();
7051 uint64_t Length = SizeInBytes / ElementSizeInBytes;
7052
7053 Slice.Array = nullptr;
7054 Slice.Offset = 0;
7055 // Return an empty Slice for undersized constants to let callers
7056 // transform even undefined library calls into simpler, well-defined
7057 // expressions. This is preferable to making the calls although it
7058 // prevents sanitizers from detecting such calls.
7059 Slice.Length = Length < Offset ? 0 : Length - Offset;
7060 return true;
7061 }
7062
7063 auto *Init = const_cast<Constant *>(GV->getInitializer());
7064 if (auto *ArrayInit = dyn_cast<ConstantDataArray>(Val: Init)) {
7065 Type *InitElTy = ArrayInit->getElementType();
7066 if (InitElTy->isIntegerTy(BitWidth: ElementSize)) {
7067 // If Init is an initializer for an array of the expected type
7068 // and size, use it as is.
7069 Array = ArrayInit;
7070 ArrayTy = ArrayInit->getType();
7071 }
7072 }
7073
7074 if (!Array) {
7075 if (ElementSize != 8)
7076 // TODO: Handle conversions to larger integral types.
7077 return false;
7078
7079 // Otherwise extract the portion of the initializer starting
7080 // at Offset as an array of bytes, and reset Offset.
7081 Init = ReadByteArrayFromGlobal(GV, Offset);
7082 if (!Init)
7083 return false;
7084
7085 Offset = 0;
7086 Array = dyn_cast<ConstantDataArray>(Val: Init);
7087 ArrayTy = dyn_cast<ArrayType>(Val: Init->getType());
7088 }
7089
7090 uint64_t NumElts = ArrayTy->getArrayNumElements();
7091 if (Offset > NumElts)
7092 return false;
7093
7094 Slice.Array = Array;
7095 Slice.Offset = Offset;
7096 Slice.Length = NumElts - Offset;
7097 return true;
7098}
7099
7100/// Extract bytes from the initializer of the constant array V, which need
7101/// not be a nul-terminated string. On success, store the bytes in Str and
7102/// return true. When TrimAtNul is set, Str will contain only the bytes up
7103/// to but not including the first nul. Return false on failure.
7104bool llvm::getConstantStringInfo(const Value *V, StringRef &Str,
7105 bool TrimAtNul) {
7106 ConstantDataArraySlice Slice;
7107 if (!getConstantDataArrayInfo(V, Slice, ElementSize: 8))
7108 return false;
7109
7110 if (Slice.Array == nullptr) {
7111 if (TrimAtNul) {
7112 // Return a nul-terminated string even for an empty Slice. This is
7113 // safe because all existing SimplifyLibcalls callers require string
7114 // arguments and the behavior of the functions they fold is undefined
7115 // otherwise. Folding the calls this way is preferable to making
7116 // the undefined library calls, even though it prevents sanitizers
7117 // from reporting such calls.
7118 Str = StringRef();
7119 return true;
7120 }
7121 if (Slice.Length == 1) {
7122 Str = StringRef("", 1);
7123 return true;
7124 }
7125 // We cannot instantiate a StringRef as we do not have an appropriate string
7126 // of 0s at hand.
7127 return false;
7128 }
7129
7130 // Start out with the entire array in the StringRef.
7131 Str = Slice.Array->getAsString();
7132 // Skip over 'offset' bytes.
7133 Str = Str.substr(Start: Slice.Offset);
7134
7135 if (TrimAtNul) {
7136 // Trim off the \0 and anything after it. If the array is not nul
7137 // terminated, we just return the whole end of string. The client may know
7138 // some other way that the string is length-bound.
7139 Str = Str.substr(Start: 0, N: Str.find(C: '\0'));
7140 }
7141 return true;
7142}
7143
7144// These next two are very similar to the above, but also look through PHI
7145// nodes.
7146// TODO: See if we can integrate these two together.
7147
7148/// If we can compute the length of the string pointed to by
7149/// the specified pointer, return 'len+1'. If we can't, return 0.
7150static uint64_t GetStringLengthH(const Value *V,
7151 SmallPtrSetImpl<const PHINode*> &PHIs,
7152 unsigned CharSize) {
7153 // Look through noop bitcast instructions.
7154 V = V->stripPointerCasts();
7155
7156 // If this is a PHI node, there are two cases: either we have already seen it
7157 // or we haven't.
7158 if (const PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7159 if (!PHIs.insert(Ptr: PN).second)
7160 return ~0ULL; // already in the set.
7161
7162 // If it was new, see if all the input strings are the same length.
7163 uint64_t LenSoFar = ~0ULL;
7164 for (Value *IncValue : PN->incoming_values()) {
7165 uint64_t Len = GetStringLengthH(V: IncValue, PHIs, CharSize);
7166 if (Len == 0) return 0; // Unknown length -> unknown.
7167
7168 if (Len == ~0ULL) continue;
7169
7170 if (Len != LenSoFar && LenSoFar != ~0ULL)
7171 return 0; // Disagree -> unknown.
7172 LenSoFar = Len;
7173 }
7174
7175 // Success, all agree.
7176 return LenSoFar;
7177 }
7178
7179 // strlen(select(c,x,y)) -> strlen(x) ^ strlen(y)
7180 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: V)) {
7181 uint64_t Len1 = GetStringLengthH(V: SI->getTrueValue(), PHIs, CharSize);
7182 if (Len1 == 0) return 0;
7183 uint64_t Len2 = GetStringLengthH(V: SI->getFalseValue(), PHIs, CharSize);
7184 if (Len2 == 0) return 0;
7185 if (Len1 == ~0ULL) return Len2;
7186 if (Len2 == ~0ULL) return Len1;
7187 if (Len1 != Len2) return 0;
7188 return Len1;
7189 }
7190
7191 // Otherwise, see if we can read the string.
7192 ConstantDataArraySlice Slice;
7193 if (!getConstantDataArrayInfo(V, Slice, ElementSize: CharSize))
7194 return 0;
7195
7196 if (Slice.Array == nullptr)
7197 // Zeroinitializer (including an empty one).
7198 return 1;
7199
7200 // Search for the first nul character. Return a conservative result even
7201 // when there is no nul. This is safe since otherwise the string function
7202 // being folded such as strlen is undefined, and can be preferable to
7203 // making the undefined library call.
7204 unsigned NullIndex = 0;
7205 for (unsigned E = Slice.Length; NullIndex < E; ++NullIndex) {
7206 if (Slice.Array->getElementAsInteger(i: Slice.Offset + NullIndex) == 0)
7207 break;
7208 }
7209
7210 return NullIndex + 1;
7211}
7212
7213/// If we can compute the length of the string pointed to by
7214/// the specified pointer, return 'len+1'. If we can't, return 0.
7215uint64_t llvm::GetStringLength(const Value *V, unsigned CharSize) {
7216 if (!V->getType()->isPointerTy())
7217 return 0;
7218
7219 SmallPtrSet<const PHINode*, 32> PHIs;
7220 uint64_t Len = GetStringLengthH(V, PHIs, CharSize);
7221 // If Len is ~0ULL, we had an infinite phi cycle: this is dead code, so return
7222 // an empty string as a length.
7223 return Len == ~0ULL ? 1 : Len;
7224}
7225
7226const Value *
7227llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call,
7228 bool MustPreserveOffset,
7229 bool MustPreserveProvenance) {
7230 assert(Call &&
7231 "getArgumentAliasingToReturnedPointer only works on nonnull calls");
7232 if (const Value *RV = Call->getReturnedArgOperand())
7233 return RV;
7234 // This can be used only as a aliasing property.
7235 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
7236 Call, MustPreserveOffset, MustPreserveProvenance))
7237 return Call->getArgOperand(i: 0);
7238 return nullptr;
7239}
7240
7241bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
7242 const CallBase *Call, bool MustPreserveOffset,
7243 bool MustPreserveProvenance) {
7244 switch (Call->getIntrinsicID()) {
7245 case Intrinsic::launder_invariant_group:
7246 case Intrinsic::strip_invariant_group:
7247 case Intrinsic::aarch64_irg:
7248 case Intrinsic::aarch64_tagp:
7249 // The amdgcn_make_buffer_rsrc function does not alter the address of the
7250 // input pointer (and thus preserves the byte offset, which is the property
7251 // the MustPreserveOffset flag selects). However, it will not necessarily
7252 // map ptr addrspace(N) null to ptr addrspace(8) null, aka the "null
7253 // descriptor", which has "all loads return 0, all stores are dropped"
7254 // semantics. Given the context of this intrinsic list, no one should be
7255 // relying on such a strict bit-exact null mapping (and, at time of
7256 // writing, they are not), but we document this fact out of an abundance
7257 // of caution.
7258 case Intrinsic::amdgcn_make_buffer_rsrc:
7259 return !MustPreserveProvenance;
7260 case Intrinsic::ptrmask:
7261 return !MustPreserveOffset;
7262 case Intrinsic::threadlocal_address:
7263 // The underlying variable changes with thread ID. The Thread ID may change
7264 // at coroutine suspend points.
7265 return !Call->getParent()->getParent()->isPresplitCoroutine();
7266 default:
7267 return false;
7268 }
7269}
7270
7271/// \p PN defines a loop-variant pointer to an object. Check if the
7272/// previous iteration of the loop was referring to the same object as \p PN.
7273static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
7274 const LoopInfo *LI) {
7275 // Find the loop-defined value.
7276 Loop *L = LI->getLoopFor(BB: PN->getParent());
7277 if (PN->getNumIncomingValues() != 2)
7278 return true;
7279
7280 // Find the value from previous iteration.
7281 auto *PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 0));
7282 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
7283 PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 1));
7284 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
7285 return true;
7286
7287 // If a new pointer is loaded in the loop, the pointer references a different
7288 // object in every iteration. E.g.:
7289 // for (i)
7290 // int *p = a[i];
7291 // ...
7292 if (auto *Load = dyn_cast<LoadInst>(Val: PrevValue))
7293 if (!L->isLoopInvariant(V: Load->getPointerOperand()))
7294 return false;
7295 return true;
7296}
7297
7298const Value *llvm::getUnderlyingObject(const Value *V, unsigned MaxLookup,
7299 bool MustPreserveProvenance) {
7300 for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
7301 if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
7302 const Value *PtrOp = GEP->getPointerOperand();
7303 if (!PtrOp->getType()->isPointerTy()) // Only handle scalar pointer base.
7304 return V;
7305 V = PtrOp;
7306 } else if (Operator::getOpcode(V) == Instruction::BitCast ||
7307 Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
7308 Value *NewV = cast<Operator>(Val: V)->getOperand(i: 0);
7309 if (!NewV->getType()->isPointerTy())
7310 return V;
7311 V = NewV;
7312 } else if (auto *GA = dyn_cast<GlobalAlias>(Val: V)) {
7313 if (GA->isInterposable())
7314 return V;
7315 V = GA->getAliasee();
7316 } else {
7317 if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
7318 // Look through single-arg phi nodes created by LCSSA.
7319 if (PHI->getNumIncomingValues() == 1) {
7320 V = PHI->getIncomingValue(i: 0);
7321 continue;
7322 }
7323 } else if (auto *Call = dyn_cast<CallBase>(Val: V)) {
7324 // CaptureTracking can know about special capturing properties of some
7325 // intrinsics like launder.invariant.group, that can't be expressed with
7326 // the attributes, but have properties like returning aliasing pointer.
7327 // Because some analysis may assume that nocaptured pointer is not
7328 // returned from some special intrinsic (because function would have to
7329 // be marked with returns attribute), it is crucial to use this function
7330 // because it should be in sync with CaptureTracking. Not using it may
7331 // cause weird miscompilations where 2 aliasing pointers are assumed to
7332 // noalias.
7333 if (auto *RP = getArgumentAliasingToReturnedPointer(
7334 Call, /*MustPreserveOffset=*/false, MustPreserveProvenance)) {
7335 V = RP;
7336 continue;
7337 }
7338 }
7339
7340 return V;
7341 }
7342 assert(V->getType()->isPointerTy() && "Unexpected operand type!");
7343 }
7344 return V;
7345}
7346
7347void llvm::getUnderlyingObjects(const Value *V,
7348 SmallVectorImpl<const Value *> &Objects,
7349 const LoopInfo *LI, unsigned MaxLookup) {
7350 SmallPtrSet<const Value *, 4> Visited;
7351 SmallVector<const Value *, 4> Worklist;
7352 Worklist.push_back(Elt: V);
7353 do {
7354 const Value *P = Worklist.pop_back_val();
7355 P = getUnderlyingObject(V: P, MaxLookup);
7356
7357 if (!Visited.insert(Ptr: P).second)
7358 continue;
7359
7360 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7361 Worklist.push_back(Elt: SI->getTrueValue());
7362 Worklist.push_back(Elt: SI->getFalseValue());
7363 continue;
7364 }
7365
7366 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7367 // If this PHI changes the underlying object in every iteration of the
7368 // loop, don't look through it. Consider:
7369 // int **A;
7370 // for (i) {
7371 // Prev = Curr; // Prev = PHI (Prev_0, Curr)
7372 // Curr = A[i];
7373 // *Prev, *Curr;
7374 //
7375 // Prev is tracking Curr one iteration behind so they refer to different
7376 // underlying objects.
7377 if (!LI || !LI->isLoopHeader(BB: PN->getParent()) ||
7378 isSameUnderlyingObjectInLoop(PN, LI))
7379 append_range(C&: Worklist, R: PN->incoming_values());
7380 else
7381 Objects.push_back(Elt: P);
7382 continue;
7383 }
7384
7385 Objects.push_back(Elt: P);
7386 } while (!Worklist.empty());
7387}
7388
7389const Value *llvm::getUnderlyingObjectAggressive(const Value *V,
7390 bool MustPreserveProvenance) {
7391 const unsigned MaxVisited = 8;
7392
7393 SmallPtrSet<const Value *, 8> Visited;
7394 SmallVector<const Value *, 8> Worklist;
7395 Worklist.push_back(Elt: V);
7396 const Value *Object = nullptr;
7397 // Used as fallback if we can't find a common underlying object through
7398 // recursion.
7399 bool First = true;
7400 const Value *FirstObject =
7401 getUnderlyingObject(V, MaxLookup: MaxLookupSearchDepth, MustPreserveProvenance);
7402 do {
7403 const Value *P = Worklist.pop_back_val();
7404 P = First ? FirstObject
7405 : getUnderlyingObject(V: P, MaxLookup: MaxLookupSearchDepth,
7406 MustPreserveProvenance);
7407 First = false;
7408
7409 if (!Visited.insert(Ptr: P).second)
7410 continue;
7411
7412 if (Visited.size() == MaxVisited)
7413 return FirstObject;
7414
7415 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7416 Worklist.push_back(Elt: SI->getTrueValue());
7417 Worklist.push_back(Elt: SI->getFalseValue());
7418 continue;
7419 }
7420
7421 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7422 append_range(C&: Worklist, R: PN->incoming_values());
7423 continue;
7424 }
7425
7426 if (!Object)
7427 Object = P;
7428 else if (Object != P)
7429 return FirstObject;
7430 } while (!Worklist.empty());
7431
7432 return Object ? Object : FirstObject;
7433}
7434
7435/// This is the function that does the work of looking through basic
7436/// ptrtoint+arithmetic+inttoptr sequences.
7437static const Value *getUnderlyingObjectFromInt(const Value *V) {
7438 do {
7439 if (const Operator *U = dyn_cast<Operator>(Val: V)) {
7440 // If we find a ptrtoint, we can transfer control back to the
7441 // regular getUnderlyingObjectFromInt.
7442 if (U->getOpcode() == Instruction::PtrToInt)
7443 return U->getOperand(i: 0);
7444 // If we find an add of a constant, a multiplied value, or a phi, it's
7445 // likely that the other operand will lead us to the base
7446 // object. We don't have to worry about the case where the
7447 // object address is somehow being computed by the multiply,
7448 // because our callers only care when the result is an
7449 // identifiable object.
7450 if (U->getOpcode() != Instruction::Add ||
7451 (!isa<ConstantInt>(Val: U->getOperand(i: 1)) &&
7452 Operator::getOpcode(V: U->getOperand(i: 1)) != Instruction::Mul &&
7453 !isa<PHINode>(Val: U->getOperand(i: 1))))
7454 return V;
7455 V = U->getOperand(i: 0);
7456 } else {
7457 return V;
7458 }
7459 assert(V->getType()->isIntegerTy() && "Unexpected operand type!");
7460 } while (true);
7461}
7462
7463/// This is a wrapper around getUnderlyingObjects and adds support for basic
7464/// ptrtoint+arithmetic+inttoptr sequences.
7465/// It returns false if unidentified object is found in getUnderlyingObjects.
7466bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
7467 SmallVectorImpl<Value *> &Objects) {
7468 SmallPtrSet<const Value *, 16> Visited;
7469 SmallVector<const Value *, 4> Working(1, V);
7470 do {
7471 V = Working.pop_back_val();
7472
7473 SmallVector<const Value *, 4> Objs;
7474 getUnderlyingObjects(V, Objects&: Objs);
7475
7476 for (const Value *V : Objs) {
7477 if (!Visited.insert(Ptr: V).second)
7478 continue;
7479 if (Operator::getOpcode(V) == Instruction::IntToPtr) {
7480 const Value *O =
7481 getUnderlyingObjectFromInt(V: cast<User>(Val: V)->getOperand(i: 0));
7482 if (O->getType()->isPointerTy()) {
7483 Working.push_back(Elt: O);
7484 continue;
7485 }
7486 }
7487 // If getUnderlyingObjects fails to find an identifiable object,
7488 // getUnderlyingObjectsForCodeGen also fails for safety.
7489 if (!isIdentifiedObject(V)) {
7490 Objects.clear();
7491 return false;
7492 }
7493 Objects.push_back(Elt: const_cast<Value *>(V));
7494 }
7495 } while (!Working.empty());
7496 return true;
7497}
7498
7499AllocaInst *llvm::findAllocaForValue(Value *V, bool OffsetZero) {
7500 AllocaInst *Result = nullptr;
7501 SmallPtrSet<Value *, 4> Visited;
7502 SmallVector<Value *, 4> Worklist;
7503
7504 auto AddWork = [&](Value *V) {
7505 if (Visited.insert(Ptr: V).second)
7506 Worklist.push_back(Elt: V);
7507 };
7508
7509 AddWork(V);
7510 do {
7511 V = Worklist.pop_back_val();
7512 assert(Visited.count(V));
7513
7514 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
7515 if (Result && Result != AI)
7516 return nullptr;
7517 Result = AI;
7518 } else if (CastInst *CI = dyn_cast<CastInst>(Val: V)) {
7519 AddWork(CI->getOperand(i_nocapture: 0));
7520 } else if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7521 for (Value *IncValue : PN->incoming_values())
7522 AddWork(IncValue);
7523 } else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
7524 AddWork(SI->getTrueValue());
7525 AddWork(SI->getFalseValue());
7526 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: V)) {
7527 if (OffsetZero && !GEP->hasAllZeroIndices())
7528 return nullptr;
7529 AddWork(GEP->getPointerOperand());
7530 } else if (CallBase *CB = dyn_cast<CallBase>(Val: V)) {
7531 Value *Returned = CB->getReturnedArgOperand();
7532 if (Returned)
7533 AddWork(Returned);
7534 else
7535 return nullptr;
7536 } else {
7537 return nullptr;
7538 }
7539 } while (!Worklist.empty());
7540
7541 return Result;
7542}
7543
7544static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7545 const Value *V, bool AllowLifetime, bool AllowDroppable) {
7546 for (const User *U : V->users()) {
7547 const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
7548 if (!II)
7549 return false;
7550
7551 if (AllowLifetime && II->isLifetimeStartOrEnd())
7552 continue;
7553
7554 if (AllowDroppable && II->isDroppable())
7555 continue;
7556
7557 return false;
7558 }
7559 return true;
7560}
7561
7562bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
7563 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7564 V, /* AllowLifetime */ true, /* AllowDroppable */ false);
7565}
7566bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) {
7567 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7568 V, /* AllowLifetime */ true, /* AllowDroppable */ true);
7569}
7570
7571bool llvm::isNotCrossLaneOperation(const Instruction *I) {
7572 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
7573 return isTriviallyVectorizable(ID: II->getIntrinsicID());
7574 auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: I);
7575 return (!Shuffle || Shuffle->isSelect()) &&
7576 !isa<CallBase, BitCastInst, ExtractElementInst>(Val: I);
7577}
7578
7579bool llvm::isSafeToSpeculativelyExecute(
7580 const Instruction *Inst, const Instruction *CtxI, AssumptionCache *AC,
7581 const DominatorTree *DT, const TargetLibraryInfo *TLI, bool UseVariableInfo,
7582 bool IgnoreUBImplyingAttrs) {
7583 return isSafeToSpeculativelyExecuteWithOpcode(Opcode: Inst->getOpcode(), Inst, CtxI,
7584 AC, DT, TLI, UseVariableInfo,
7585 IgnoreUBImplyingAttrs);
7586}
7587
7588bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
7589 unsigned Opcode, const Instruction *Inst, const Instruction *CtxI,
7590 AssumptionCache *AC, const DominatorTree *DT, const TargetLibraryInfo *TLI,
7591 bool UseVariableInfo, bool IgnoreUBImplyingAttrs) {
7592#ifndef NDEBUG
7593 if (Inst->getOpcode() != Opcode) {
7594 // Check that the operands are actually compatible with the Opcode override.
7595 auto hasEqualReturnAndLeadingOperandTypes =
7596 [](const Instruction *Inst, unsigned NumLeadingOperands) {
7597 if (Inst->getNumOperands() < NumLeadingOperands)
7598 return false;
7599 const Type *ExpectedType = Inst->getType();
7600 for (unsigned ItOp = 0; ItOp < NumLeadingOperands; ++ItOp)
7601 if (Inst->getOperand(ItOp)->getType() != ExpectedType)
7602 return false;
7603 return true;
7604 };
7605 assert(!Instruction::isBinaryOp(Opcode) ||
7606 hasEqualReturnAndLeadingOperandTypes(Inst, 2));
7607 assert(!Instruction::isUnaryOp(Opcode) ||
7608 hasEqualReturnAndLeadingOperandTypes(Inst, 1));
7609 }
7610#endif
7611
7612 switch (Opcode) {
7613 default:
7614 return true;
7615 case Instruction::UDiv:
7616 case Instruction::URem: {
7617 // x / y is undefined if y == 0.
7618 const APInt *V;
7619 if (match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: V)))
7620 return *V != 0;
7621 return false;
7622 }
7623 case Instruction::SDiv:
7624 case Instruction::SRem: {
7625 // x / y is undefined if y == 0 or x == INT_MIN and y == -1
7626 const APInt *Numerator, *Denominator;
7627 if (!match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: Denominator)))
7628 return false;
7629 // We cannot hoist this division if the denominator is 0.
7630 if (*Denominator == 0)
7631 return false;
7632 // It's safe to hoist if the denominator is not 0 or -1.
7633 if (!Denominator->isAllOnes())
7634 return true;
7635 // At this point we know that the denominator is -1. It is safe to hoist as
7636 // long we know that the numerator is not INT_MIN.
7637 if (match(V: Inst->getOperand(i: 0), P: m_APInt(Res&: Numerator)))
7638 return !Numerator->isMinSignedValue();
7639 // The numerator *might* be MinSignedValue.
7640 return false;
7641 }
7642 case Instruction::Load: {
7643 if (!UseVariableInfo)
7644 return false;
7645
7646 const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst);
7647 if (!LI)
7648 return false;
7649 if (mustSuppressSpeculation(LI: *LI))
7650 return false;
7651 const DataLayout &DL = LI->getDataLayout();
7652 return isDereferenceableAndAlignedPointer(
7653 V: LI->getPointerOperand(), Ty: LI->getType(), Alignment: LI->getAlign(),
7654 Q: SimplifyQuery(DL, TLI, DT, AC, CtxI));
7655 }
7656 case Instruction::Call: {
7657 auto *CI = dyn_cast<const CallInst>(Val: Inst);
7658 if (!CI)
7659 return false;
7660 const Function *Callee = CI->getCalledFunction();
7661
7662 // The called function could have undefined behavior or side-effects, even
7663 // if marked readnone nounwind.
7664 if (!Callee || !Callee->isSpeculatable())
7665 return false;
7666 // Since the operands may be changed after hoisting, undefined behavior may
7667 // be triggered by some UB-implying attributes.
7668 return IgnoreUBImplyingAttrs || !CI->hasUBImplyingAttrs();
7669 }
7670 case Instruction::VAArg:
7671 case Instruction::Alloca:
7672 case Instruction::Invoke:
7673 case Instruction::CallBr:
7674 case Instruction::PHI:
7675 case Instruction::Store:
7676 case Instruction::Ret:
7677 case Instruction::UncondBr:
7678 case Instruction::CondBr:
7679 case Instruction::IndirectBr:
7680 case Instruction::Switch:
7681 case Instruction::Unreachable:
7682 case Instruction::Fence:
7683 case Instruction::AtomicRMW:
7684 case Instruction::AtomicCmpXchg:
7685 case Instruction::LandingPad:
7686 case Instruction::Resume:
7687 case Instruction::CatchSwitch:
7688 case Instruction::CatchPad:
7689 case Instruction::CatchRet:
7690 case Instruction::CleanupPad:
7691 case Instruction::CleanupRet:
7692 return false; // Misc instructions which have effects
7693 }
7694}
7695
7696bool llvm::mayHaveNonDefUseDependency(const Instruction &I) {
7697 if (I.mayReadOrWriteMemory())
7698 // Memory dependency possible
7699 return true;
7700 if (!isSafeToSpeculativelyExecute(Inst: &I))
7701 // Can't move above a maythrow call or infinite loop. Or if an
7702 // inalloca alloca, above a stacksave call.
7703 return true;
7704 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7705 // 1) Can't reorder two inf-loop calls, even if readonly
7706 // 2) Also can't reorder an inf-loop call below a instruction which isn't
7707 // safe to speculative execute. (Inverse of above)
7708 return true;
7709 return false;
7710}
7711
7712/// Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
7713static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) {
7714 switch (OR) {
7715 case ConstantRange::OverflowResult::MayOverflow:
7716 return OverflowResult::MayOverflow;
7717 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
7718 return OverflowResult::AlwaysOverflowsLow;
7719 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
7720 return OverflowResult::AlwaysOverflowsHigh;
7721 case ConstantRange::OverflowResult::NeverOverflows:
7722 return OverflowResult::NeverOverflows;
7723 }
7724 llvm_unreachable("Unknown OverflowResult");
7725}
7726
7727/// Combine constant ranges from computeConstantRange() and computeKnownBits().
7728ConstantRange
7729llvm::computeConstantRangeIncludingKnownBits(const WithCache<const Value *> &V,
7730 bool ForSigned,
7731 const SimplifyQuery &SQ) {
7732 ConstantRange CR1 =
7733 ConstantRange::fromKnownBits(Known: V.getKnownBits(Q: SQ), IsSigned: ForSigned);
7734 ConstantRange CR2 = computeConstantRange(V, ForSigned, SQ);
7735 ConstantRange::PreferredRangeType RangeType =
7736 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
7737 return CR1.intersectWith(CR: CR2, Type: RangeType);
7738}
7739
7740OverflowResult llvm::computeOverflowForUnsignedMul(const Value *LHS,
7741 const Value *RHS,
7742 const SimplifyQuery &SQ,
7743 bool IsNSW) {
7744 ConstantRange LHSRange =
7745 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7746 ConstantRange RHSRange =
7747 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7748
7749 // mul nsw of two non-negative numbers is also nuw.
7750 if (IsNSW && LHSRange.isAllNonNegative() && RHSRange.isAllNonNegative())
7751 return OverflowResult::NeverOverflows;
7752
7753 return mapOverflowResult(OR: LHSRange.unsignedMulMayOverflow(Other: RHSRange));
7754}
7755
7756OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
7757 const Value *RHS,
7758 const SimplifyQuery &SQ) {
7759 // Multiplying n * m significant bits yields a result of n + m significant
7760 // bits. If the total number of significant bits does not exceed the
7761 // result bit width (minus 1), there is no overflow.
7762 // This means if we have enough leading sign bits in the operands
7763 // we can guarantee that the result does not overflow.
7764 // Ref: "Hacker's Delight" by Henry Warren
7765 unsigned BitWidth = LHS->getType()->getScalarSizeInBits();
7766
7767 // Note that underestimating the number of sign bits gives a more
7768 // conservative answer.
7769 unsigned SignBits =
7770 ::ComputeNumSignBits(V: LHS, Q: SQ) + ::ComputeNumSignBits(V: RHS, Q: SQ);
7771
7772 // First handle the easy case: if we have enough sign bits there's
7773 // definitely no overflow.
7774 if (SignBits > BitWidth + 1)
7775 return OverflowResult::NeverOverflows;
7776
7777 // There are two ambiguous cases where there can be no overflow:
7778 // SignBits == BitWidth + 1 and
7779 // SignBits == BitWidth
7780 // The second case is difficult to check, therefore we only handle the
7781 // first case.
7782 if (SignBits == BitWidth + 1) {
7783 // It overflows only when both arguments are negative and the true
7784 // product is exactly the minimum negative number.
7785 // E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
7786 // For simplicity we just check if at least one side is not negative.
7787 KnownBits LHSKnown = computeKnownBits(V: LHS, Q: SQ);
7788 KnownBits RHSKnown = computeKnownBits(V: RHS, Q: SQ);
7789 if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
7790 return OverflowResult::NeverOverflows;
7791 }
7792 return OverflowResult::MayOverflow;
7793}
7794
7795OverflowResult
7796llvm::computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
7797 const WithCache<const Value *> &RHS,
7798 const SimplifyQuery &SQ) {
7799 ConstantRange LHSRange =
7800 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7801 ConstantRange RHSRange =
7802 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7803 return mapOverflowResult(OR: LHSRange.unsignedAddMayOverflow(Other: RHSRange));
7804}
7805
7806static OverflowResult
7807computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
7808 const WithCache<const Value *> &RHS,
7809 const AddOperator *Add, const SimplifyQuery &SQ) {
7810 if (Add && Add->hasNoSignedWrap()) {
7811 return OverflowResult::NeverOverflows;
7812 }
7813
7814 // If LHS and RHS each have at least two sign bits, the addition will look
7815 // like
7816 //
7817 // XX..... +
7818 // YY.....
7819 //
7820 // If the carry into the most significant position is 0, X and Y can't both
7821 // be 1 and therefore the carry out of the addition is also 0.
7822 //
7823 // If the carry into the most significant position is 1, X and Y can't both
7824 // be 0 and therefore the carry out of the addition is also 1.
7825 //
7826 // Since the carry into the most significant position is always equal to
7827 // the carry out of the addition, there is no signed overflow.
7828 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7829 return OverflowResult::NeverOverflows;
7830
7831 ConstantRange LHSRange =
7832 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7833 ConstantRange RHSRange =
7834 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7835 OverflowResult OR =
7836 mapOverflowResult(OR: LHSRange.signedAddMayOverflow(Other: RHSRange));
7837 if (OR != OverflowResult::MayOverflow)
7838 return OR;
7839
7840 // The remaining code needs Add to be available. Early returns if not so.
7841 if (!Add)
7842 return OverflowResult::MayOverflow;
7843
7844 // If the sign of Add is the same as at least one of the operands, this add
7845 // CANNOT overflow. If this can be determined from the known bits of the
7846 // operands the above signedAddMayOverflow() check will have already done so.
7847 // The only other way to improve on the known bits is from an assumption, so
7848 // call computeKnownBitsFromContext() directly.
7849 bool LHSOrRHSKnownNonNegative =
7850 (LHSRange.isAllNonNegative() || RHSRange.isAllNonNegative());
7851 bool LHSOrRHSKnownNegative =
7852 (LHSRange.isAllNegative() || RHSRange.isAllNegative());
7853 if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) {
7854 KnownBits AddKnown(LHSRange.getBitWidth());
7855 computeKnownBitsFromContext(V: Add, Known&: AddKnown, Q: SQ);
7856 if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) ||
7857 (AddKnown.isNegative() && LHSOrRHSKnownNegative))
7858 return OverflowResult::NeverOverflows;
7859 }
7860
7861 return OverflowResult::MayOverflow;
7862}
7863
7864OverflowResult llvm::computeOverflowForUnsignedSub(const Value *LHS,
7865 const Value *RHS,
7866 const SimplifyQuery &SQ) {
7867 // X - (X % ?)
7868 // The remainder of a value can't have greater magnitude than itself,
7869 // so the subtraction can't overflow.
7870
7871 // X - (X -nuw ?)
7872 // In the minimal case, this would simplify to "?", so there's no subtract
7873 // at all. But if this analysis is used to peek through casts, for example,
7874 // then determining no-overflow may allow other transforms.
7875
7876 // TODO: There are other patterns like this.
7877 // See simplifyICmpWithBinOpOnLHS() for candidates.
7878 if (match(V: RHS, P: m_URem(L: m_Specific(V: LHS), R: m_Value())) ||
7879 match(V: RHS, P: m_NUWSub(L: m_Specific(V: LHS), R: m_Value())))
7880 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
7881 return OverflowResult::NeverOverflows;
7882
7883 if (auto C = isImpliedByDomCondition(Pred: CmpInst::ICMP_UGE, LHS, RHS, ContextI: SQ.CxtI,
7884 DL: SQ.DL)) {
7885 if (*C)
7886 return OverflowResult::NeverOverflows;
7887 return OverflowResult::AlwaysOverflowsLow;
7888 }
7889
7890 ConstantRange LHSRange =
7891 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7892 ConstantRange RHSRange =
7893 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7894 return mapOverflowResult(OR: LHSRange.unsignedSubMayOverflow(Other: RHSRange));
7895}
7896
7897OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS,
7898 const Value *RHS,
7899 const SimplifyQuery &SQ) {
7900 // X - (X % ?)
7901 // The remainder of a value can't have greater magnitude than itself,
7902 // so the subtraction can't overflow.
7903
7904 // X - (X -nsw ?)
7905 // In the minimal case, this would simplify to "?", so there's no subtract
7906 // at all. But if this analysis is used to peek through casts, for example,
7907 // then determining no-overflow may allow other transforms.
7908 if (match(V: RHS, P: m_SRem(L: m_Specific(V: LHS), R: m_Value())) ||
7909 match(V: RHS, P: m_NSWSub(L: m_Specific(V: LHS), R: m_Value())))
7910 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT))
7911 return OverflowResult::NeverOverflows;
7912
7913 // If LHS and RHS each have at least two sign bits, the subtraction
7914 // cannot overflow.
7915 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7916 return OverflowResult::NeverOverflows;
7917
7918 ConstantRange LHSRange =
7919 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7920 ConstantRange RHSRange =
7921 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7922 return mapOverflowResult(OR: LHSRange.signedSubMayOverflow(Other: RHSRange));
7923}
7924
7925bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO,
7926 const DominatorTree &DT) {
7927 SmallVector<const CondBrInst *, 2> GuardingBranches;
7928 SmallVector<const ExtractValueInst *, 2> Results;
7929
7930 for (const User *U : WO->users()) {
7931 if (const auto *EVI = dyn_cast<ExtractValueInst>(Val: U)) {
7932 assert(EVI->getNumIndices() == 1 && "Obvious from CI's type");
7933
7934 if (EVI->getIndices()[0] == 0)
7935 Results.push_back(Elt: EVI);
7936 else {
7937 assert(EVI->getIndices()[0] == 1 && "Obvious from CI's type");
7938
7939 for (const auto *U : EVI->users())
7940 if (const auto *B = dyn_cast<CondBrInst>(Val: U))
7941 GuardingBranches.push_back(Elt: B);
7942 }
7943 } else {
7944 // We are using the aggregate directly in a way we don't want to analyze
7945 // here (storing it to a global, say).
7946 return false;
7947 }
7948 }
7949
7950 auto AllUsesGuardedByBranch = [&](const CondBrInst *BI) {
7951 BasicBlockEdge NoWrapEdge(BI->getParent(), BI->getSuccessor(i: 1));
7952
7953 // Check if all users of the add are provably no-wrap.
7954 for (const auto *Result : Results) {
7955 // If the extractvalue itself is not executed on overflow, the we don't
7956 // need to check each use separately, since domination is transitive.
7957 if (DT.dominates(BBE: NoWrapEdge, BB: Result->getParent()))
7958 continue;
7959
7960 for (const auto &RU : Result->uses())
7961 if (!DT.dominates(BBE: NoWrapEdge, U: RU))
7962 return false;
7963 }
7964
7965 return true;
7966 };
7967
7968 return llvm::any_of(Range&: GuardingBranches, P: AllUsesGuardedByBranch);
7969}
7970
7971/// Shifts return poison if shiftwidth is larger than the bitwidth.
7972static bool shiftAmountKnownInRange(const Value *ShiftAmount) {
7973 auto *C = dyn_cast<Constant>(Val: ShiftAmount);
7974 if (!C)
7975 return false;
7976
7977 // Shifts return poison if shiftwidth is larger than the bitwidth.
7978 SmallVector<const Constant *, 4> ShiftAmounts;
7979 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: C->getType())) {
7980 unsigned NumElts = FVTy->getNumElements();
7981 for (unsigned i = 0; i < NumElts; ++i)
7982 ShiftAmounts.push_back(Elt: C->getAggregateElement(Elt: i));
7983 } else if (isa<ScalableVectorType>(Val: C->getType()))
7984 return false; // Can't tell, just return false to be safe
7985 else
7986 ShiftAmounts.push_back(Elt: C);
7987
7988 bool Safe = llvm::all_of(Range&: ShiftAmounts, P: [](const Constant *C) {
7989 auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
7990 return CI && CI->getValue().ult(RHS: C->getType()->getIntegerBitWidth());
7991 });
7992
7993 return Safe;
7994}
7995
7996static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
7997 bool ConsiderFlagsAndMetadata) {
7998
7999 if (ConsiderFlagsAndMetadata && includesPoison(Kind) &&
8000 Op->hasPoisonGeneratingAnnotations())
8001 return true;
8002
8003 unsigned Opcode = Op->getOpcode();
8004
8005 // Check whether opcode is a poison/undef-generating operation
8006 switch (Opcode) {
8007 case Instruction::Shl:
8008 case Instruction::AShr:
8009 case Instruction::LShr:
8010 return includesPoison(Kind) && !shiftAmountKnownInRange(ShiftAmount: Op->getOperand(i: 1));
8011 case Instruction::FPToSI:
8012 case Instruction::FPToUI:
8013 // fptosi/ui yields poison if the resulting value does not fit in the
8014 // destination type.
8015 return true;
8016 case Instruction::Call:
8017 if (auto *II = dyn_cast<IntrinsicInst>(Val: Op)) {
8018 switch (II->getIntrinsicID()) {
8019 // NOTE: Use IntrNoCreateUndefOrPoison when possible.
8020 case Intrinsic::ctlz:
8021 case Intrinsic::cttz:
8022 case Intrinsic::abs:
8023 // We're not considering flags so it is safe to just return false.
8024 return false;
8025 case Intrinsic::sshl_sat:
8026 case Intrinsic::ushl_sat:
8027 if (!includesPoison(Kind) ||
8028 shiftAmountKnownInRange(ShiftAmount: II->getArgOperand(i: 1)))
8029 return false;
8030 break;
8031 }
8032 }
8033 [[fallthrough]];
8034 case Instruction::CallBr:
8035 case Instruction::Invoke: {
8036 const auto *CB = cast<CallBase>(Val: Op);
8037 return !CB->hasRetAttr(Kind: Attribute::NoUndef) &&
8038 !CB->hasFnAttr(Kind: Attribute::NoCreateUndefOrPoison);
8039 }
8040 case Instruction::InsertElement:
8041 case Instruction::ExtractElement: {
8042 // If index exceeds the length of the vector, it returns poison
8043 auto *VTy = cast<VectorType>(Val: Op->getOperand(i: 0)->getType());
8044 unsigned IdxOp = Op->getOpcode() == Instruction::InsertElement ? 2 : 1;
8045 auto *Idx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: IdxOp));
8046 if (includesPoison(Kind))
8047 return !Idx ||
8048 Idx->getValue().uge(RHS: VTy->getElementCount().getKnownMinValue());
8049 return false;
8050 }
8051 case Instruction::ShuffleVector: {
8052 ArrayRef<int> Mask = isa<ConstantExpr>(Val: Op)
8053 ? cast<ConstantExpr>(Val: Op)->getShuffleMask()
8054 : cast<ShuffleVectorInst>(Val: Op)->getShuffleMask();
8055 return includesPoison(Kind) && is_contained(Range&: Mask, Element: PoisonMaskElem);
8056 }
8057 case Instruction::FNeg:
8058 case Instruction::PHI:
8059 case Instruction::Select:
8060 case Instruction::ExtractValue:
8061 case Instruction::InsertValue:
8062 case Instruction::Freeze:
8063 case Instruction::ICmp:
8064 case Instruction::FCmp:
8065 case Instruction::GetElementPtr:
8066 return false;
8067 case Instruction::AddrSpaceCast:
8068 return true;
8069 default: {
8070 const auto *CE = dyn_cast<ConstantExpr>(Val: Op);
8071 if (isa<CastInst>(Val: Op) || (CE && CE->isCast()))
8072 return false;
8073 else if (Instruction::isBinaryOp(Opcode))
8074 return false;
8075 // Be conservative and return true.
8076 return true;
8077 }
8078 }
8079}
8080
8081bool llvm::canCreateUndefOrPoison(const Operator *Op,
8082 bool ConsiderFlagsAndMetadata) {
8083 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::UndefOrPoison,
8084 ConsiderFlagsAndMetadata);
8085}
8086
8087bool llvm::canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata) {
8088 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::PoisonOnly,
8089 ConsiderFlagsAndMetadata);
8090}
8091
8092static bool directlyImpliesPoison(const Value *ValAssumedPoison, const Value *V,
8093 unsigned Depth) {
8094 if (ValAssumedPoison == V)
8095 return true;
8096
8097 const unsigned MaxDepth = 2;
8098 if (Depth >= MaxDepth)
8099 return false;
8100
8101 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
8102 if (any_of(Range: I->operands(), P: [=](const Use &Op) {
8103 return propagatesPoison(PoisonOp: Op) &&
8104 directlyImpliesPoison(ValAssumedPoison, V: Op, Depth: Depth + 1);
8105 }))
8106 return true;
8107
8108 // V = extractvalue V0, idx
8109 // V2 = extractvalue V0, idx2
8110 // V0's elements are all poison or not. (e.g., add_with_overflow)
8111 const WithOverflowInst *II;
8112 if (match(V: I, P: m_ExtractValue(V: m_WithOverflowInst(I&: II))) &&
8113 (match(V: ValAssumedPoison, P: m_ExtractValue(V: m_Specific(V: II))) ||
8114 llvm::is_contained(Range: II->args(), Element: ValAssumedPoison)))
8115 return true;
8116 }
8117 return false;
8118}
8119
8120static bool impliesPoison(const Value *ValAssumedPoison, const Value *V,
8121 unsigned Depth) {
8122 if (isGuaranteedNotToBePoison(V: ValAssumedPoison))
8123 return true;
8124
8125 if (directlyImpliesPoison(ValAssumedPoison, V, /* Depth */ 0))
8126 return true;
8127
8128 const unsigned MaxDepth = 2;
8129 if (Depth >= MaxDepth)
8130 return false;
8131
8132 const auto *I = dyn_cast<Instruction>(Val: ValAssumedPoison);
8133 if (I && !canCreatePoison(Op: cast<Operator>(Val: I))) {
8134 return all_of(Range: I->operands(), P: [=](const Value *Op) {
8135 return impliesPoison(ValAssumedPoison: Op, V, Depth: Depth + 1);
8136 });
8137 }
8138 return false;
8139}
8140
8141bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
8142 return ::impliesPoison(ValAssumedPoison, V, /* Depth */ 0);
8143}
8144
8145static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
8146
8147static bool isGuaranteedNotToBeUndefOrPoison(
8148 const Value *V, AssumptionCache *AC, const Instruction *CtxI,
8149 const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
8150 if (Depth >= MaxAnalysisRecursionDepth)
8151 return false;
8152
8153 if (isa<MetadataAsValue>(Val: V))
8154 return false;
8155
8156 if (const auto *A = dyn_cast<Argument>(Val: V)) {
8157 if (A->hasAttribute(Kind: Attribute::NoUndef) ||
8158 A->hasAttribute(Kind: Attribute::Dereferenceable) ||
8159 A->hasAttribute(Kind: Attribute::DereferenceableOrNull))
8160 return true;
8161 }
8162
8163 if (auto *C = dyn_cast<Constant>(Val: V)) {
8164 if (isa<PoisonValue>(Val: C))
8165 return !includesPoison(Kind);
8166
8167 if (isa<UndefValue>(Val: C))
8168 return !includesUndef(Kind);
8169
8170 if (isa<ConstantInt>(Val: C) || isa<GlobalVariable>(Val: C) || isa<ConstantFP>(Val: C) ||
8171 isa<ConstantPointerNull>(Val: C) || isa<Function>(Val: C))
8172 return true;
8173
8174 if (C->getType()->isVectorTy()) {
8175 if (isa<ConstantExpr>(Val: C)) {
8176 // Scalable vectors can use a ConstantExpr to build a splat.
8177 if (Constant *SplatC = C->getSplatValue())
8178 if (isa<ConstantInt>(Val: SplatC) || isa<ConstantFP>(Val: SplatC))
8179 return true;
8180 } else {
8181 if (includesUndef(Kind) && C->containsUndefElement())
8182 return false;
8183 if (includesPoison(Kind) && C->containsPoisonElement())
8184 return false;
8185 return !C->containsConstantExpression();
8186 }
8187 }
8188 }
8189
8190 // Strip cast operations from a pointer value.
8191 // Note that stripPointerCastsSameRepresentation can strip off getelementptr
8192 // inbounds with zero offset. To guarantee that the result isn't poison, the
8193 // stripped pointer is checked as it has to be pointing into an allocated
8194 // object or be null `null` to ensure `inbounds` getelement pointers with a
8195 // zero offset could not produce poison.
8196 // It can strip off addrspacecast that do not change bit representation as
8197 // well. We believe that such addrspacecast is equivalent to no-op.
8198 auto *StrippedV = V->stripPointerCastsSameRepresentation();
8199 if (isa<AllocaInst>(Val: StrippedV) || isa<GlobalVariable>(Val: StrippedV) ||
8200 isa<Function>(Val: StrippedV) || isa<ConstantPointerNull>(Val: StrippedV))
8201 return true;
8202
8203 auto OpCheck = [&](const Value *V) {
8204 return isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth: Depth + 1, Kind);
8205 };
8206
8207 if (auto *Opr = dyn_cast<Operator>(Val: V)) {
8208 // If the value is a freeze instruction, then it can never
8209 // be undef or poison.
8210 if (isa<FreezeInst>(Val: V))
8211 return true;
8212
8213 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
8214 if (CB->hasRetAttr(Kind: Attribute::NoUndef) ||
8215 CB->hasRetAttr(Kind: Attribute::Dereferenceable) ||
8216 CB->hasRetAttr(Kind: Attribute::DereferenceableOrNull))
8217 return true;
8218 }
8219
8220 if (!::canCreateUndefOrPoison(Op: Opr, Kind,
8221 /*ConsiderFlagsAndMetadata=*/true)) {
8222 if (const auto *PN = dyn_cast<PHINode>(Val: V)) {
8223 unsigned Num = PN->getNumIncomingValues();
8224 bool IsWellDefined = true;
8225 for (unsigned i = 0; i < Num; ++i) {
8226 if (PN == PN->getIncomingValue(i))
8227 continue;
8228 auto *TI = PN->getIncomingBlock(i)->getTerminator();
8229 if (!isGuaranteedNotToBeUndefOrPoison(V: PN->getIncomingValue(i), AC, CtxI: TI,
8230 DT, Depth: Depth + 1, Kind)) {
8231 IsWellDefined = false;
8232 break;
8233 }
8234 }
8235 if (IsWellDefined)
8236 return true;
8237 } else if (auto *Splat = isa<ShuffleVectorInst>(Val: Opr) ? getSplatValue(V: Opr)
8238 : nullptr) {
8239 // For splats we only need to check the value being splatted.
8240 if (OpCheck(Splat))
8241 return true;
8242 } else if (all_of(Range: Opr->operands(), P: OpCheck))
8243 return true;
8244 }
8245 }
8246
8247 if (auto *I = dyn_cast<LoadInst>(Val: V))
8248 if (I->hasMetadata(KindID: LLVMContext::MD_noundef) ||
8249 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable) ||
8250 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable_or_null))
8251 return true;
8252
8253 if (programUndefinedIfUndefOrPoison(V, PoisonOnly: !includesUndef(Kind)))
8254 return true;
8255
8256 // CxtI may be null or a cloned instruction.
8257 if (!CtxI || !CtxI->getParent() || !DT)
8258 return false;
8259
8260 auto *DNode = DT->getNode(BB: CtxI->getParent());
8261 if (!DNode)
8262 // Unreachable block
8263 return false;
8264
8265 // If V is used as a branch condition before reaching CtxI, V cannot be
8266 // undef or poison.
8267 // br V, BB1, BB2
8268 // BB1:
8269 // CtxI ; V cannot be undef or poison here
8270 auto *Dominator = DNode->getIDom();
8271 // This check is purely for compile time reasons: we can skip the IDom walk
8272 // if what we are checking for includes undef and the value is not an integer.
8273 if (!includesUndef(Kind) || V->getType()->isIntegerTy())
8274 while (Dominator) {
8275 auto *TI = Dominator->getBlock()->getTerminatorOrNull();
8276
8277 Value *Cond = nullptr;
8278 if (auto BI = dyn_cast_or_null<CondBrInst>(Val: TI)) {
8279 Cond = BI->getCondition();
8280 } else if (auto SI = dyn_cast_or_null<SwitchInst>(Val: TI)) {
8281 Cond = SI->getCondition();
8282 }
8283
8284 if (Cond) {
8285 if (Cond == V)
8286 return true;
8287 else if (!includesUndef(Kind) && isa<Operator>(Val: Cond)) {
8288 // For poison, we can analyze further
8289 auto *Opr = cast<Operator>(Val: Cond);
8290 if (any_of(Range: Opr->operands(), P: [V](const Use &U) {
8291 return V == U && propagatesPoison(PoisonOp: U);
8292 }))
8293 return true;
8294 }
8295 }
8296
8297 Dominator = Dominator->getIDom();
8298 }
8299
8300 if (AC && getKnowledgeValidInContext(V, AttrKinds: {Attribute::NoUndef}, AC&: *AC, CtxI, DT))
8301 return true;
8302
8303 return false;
8304}
8305
8306bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC,
8307 const Instruction *CtxI,
8308 const DominatorTree *DT,
8309 unsigned Depth) {
8310 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8311 Kind: UndefPoisonKind::UndefOrPoison);
8312}
8313
8314bool llvm::isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC,
8315 const Instruction *CtxI,
8316 const DominatorTree *DT, unsigned Depth) {
8317 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8318 Kind: UndefPoisonKind::PoisonOnly);
8319}
8320
8321bool llvm::isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC,
8322 const Instruction *CtxI,
8323 const DominatorTree *DT, unsigned Depth) {
8324 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8325 Kind: UndefPoisonKind::UndefOnly);
8326}
8327
8328/// Return true if undefined behavior would provably be executed on the path to
8329/// OnPathTo if Root produced a posion result. Note that this doesn't say
8330/// anything about whether OnPathTo is actually executed or whether Root is
8331/// actually poison. This can be used to assess whether a new use of Root can
8332/// be added at a location which is control equivalent with OnPathTo (such as
8333/// immediately before it) without introducing UB which didn't previously
8334/// exist. Note that a false result conveys no information.
8335bool llvm::mustExecuteUBIfPoisonOnPathTo(Instruction *Root,
8336 Instruction *OnPathTo,
8337 DominatorTree *DT) {
8338 // Basic approach is to assume Root is poison, propagate poison forward
8339 // through all users we can easily track, and then check whether any of those
8340 // users are provable UB and must execute before out exiting block might
8341 // exit.
8342
8343 // The set of all recursive users we've visited (which are assumed to all be
8344 // poison because of said visit)
8345 SmallPtrSet<const Value *, 16> KnownPoison;
8346 SmallVector<const Instruction*, 16> Worklist;
8347 Worklist.push_back(Elt: Root);
8348 while (!Worklist.empty()) {
8349 const Instruction *I = Worklist.pop_back_val();
8350
8351 // If we know this must trigger UB on a path leading our target.
8352 if (mustTriggerUB(I, KnownPoison) && DT->dominates(Def: I, User: OnPathTo))
8353 return true;
8354
8355 // If we can't analyze propagation through this instruction, just skip it
8356 // and transitive users. Safe as false is a conservative result.
8357 if (I != Root && !any_of(Range: I->operands(), P: [&KnownPoison](const Use &U) {
8358 return KnownPoison.contains(Ptr: U) && propagatesPoison(PoisonOp: U);
8359 }))
8360 continue;
8361
8362 if (KnownPoison.insert(Ptr: I).second)
8363 for (const User *User : I->users())
8364 Worklist.push_back(Elt: cast<Instruction>(Val: User));
8365 }
8366
8367 // Might be non-UB, or might have a path we couldn't prove must execute on
8368 // way to exiting bb.
8369 return false;
8370}
8371
8372OverflowResult llvm::computeOverflowForSignedAdd(const AddOperator *Add,
8373 const SimplifyQuery &SQ) {
8374 return ::computeOverflowForSignedAdd(LHS: Add->getOperand(i_nocapture: 0), RHS: Add->getOperand(i_nocapture: 1),
8375 Add, SQ);
8376}
8377
8378OverflowResult
8379llvm::computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
8380 const WithCache<const Value *> &RHS,
8381 const SimplifyQuery &SQ) {
8382 return ::computeOverflowForSignedAdd(LHS, RHS, Add: nullptr, SQ);
8383}
8384
8385bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
8386 // Note: An atomic operation isn't guaranteed to return in a reasonable amount
8387 // of time because it's possible for another thread to interfere with it for an
8388 // arbitrary length of time, but programs aren't allowed to rely on that.
8389
8390 // If there is no successor, then execution can't transfer to it.
8391 if (isa<ReturnInst>(Val: I))
8392 return false;
8393 if (isa<UnreachableInst>(Val: I))
8394 return false;
8395
8396 // Note: Do not add new checks here; instead, change Instruction::mayThrow or
8397 // Instruction::willReturn.
8398 //
8399 // FIXME: Move this check into Instruction::willReturn.
8400 if (isa<CatchPadInst>(Val: I)) {
8401 switch (classifyEHPersonality(Pers: I->getFunction()->getPersonalityFn())) {
8402 default:
8403 // A catchpad may invoke exception object constructors and such, which
8404 // in some languages can be arbitrary code, so be conservative by default.
8405 return false;
8406 case EHPersonality::CoreCLR:
8407 // For CoreCLR, it just involves a type test.
8408 return true;
8409 }
8410 }
8411
8412 // An instruction that returns without throwing must transfer control flow
8413 // to a successor.
8414 return !I->mayThrow() && I->willReturn();
8415}
8416
8417bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) {
8418 // TODO: This is slightly conservative for invoke instruction since exiting
8419 // via an exception *is* normal control for them.
8420 for (const Instruction &I : *BB)
8421 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8422 return false;
8423 return true;
8424}
8425
8426bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8427 BasicBlock::const_iterator Begin, BasicBlock::const_iterator End,
8428 unsigned ScanLimit) {
8429 return isGuaranteedToTransferExecutionToSuccessor(Range: make_range(x: Begin, y: End),
8430 ScanLimit);
8431}
8432
8433bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8434 iterator_range<BasicBlock::const_iterator> Range, unsigned ScanLimit) {
8435 assert(ScanLimit && "scan limit must be non-zero");
8436 for (const Instruction &I : Range) {
8437 if (--ScanLimit == 0)
8438 return false;
8439 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8440 return false;
8441 }
8442 return true;
8443}
8444
8445bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
8446 const Loop *L) {
8447 // The loop header is guaranteed to be executed for every iteration.
8448 //
8449 // FIXME: Relax this constraint to cover all basic blocks that are
8450 // guaranteed to be executed at every iteration.
8451 if (I->getParent() != L->getHeader()) return false;
8452
8453 for (const Instruction &LI : *L->getHeader()) {
8454 if (&LI == I) return true;
8455 if (!isGuaranteedToTransferExecutionToSuccessor(I: &LI)) return false;
8456 }
8457 llvm_unreachable("Instruction not contained in its own parent basic block.");
8458}
8459
8460bool llvm::intrinsicPropagatesPoison(Intrinsic::ID IID) {
8461 switch (IID) {
8462 // TODO: Add more intrinsics.
8463 case Intrinsic::sadd_with_overflow:
8464 case Intrinsic::ssub_with_overflow:
8465 case Intrinsic::smul_with_overflow:
8466 case Intrinsic::uadd_with_overflow:
8467 case Intrinsic::usub_with_overflow:
8468 case Intrinsic::umul_with_overflow:
8469 // If an input is a vector containing a poison element, the
8470 // two output vectors (calculated results, overflow bits)'
8471 // corresponding lanes are poison.
8472 return true;
8473 case Intrinsic::ctpop:
8474 case Intrinsic::ctlz:
8475 case Intrinsic::cttz:
8476 case Intrinsic::abs:
8477 case Intrinsic::smax:
8478 case Intrinsic::smin:
8479 case Intrinsic::umax:
8480 case Intrinsic::umin:
8481 case Intrinsic::scmp:
8482 case Intrinsic::is_fpclass:
8483 case Intrinsic::ptrmask:
8484 case Intrinsic::ucmp:
8485 case Intrinsic::bitreverse:
8486 case Intrinsic::bswap:
8487 case Intrinsic::sadd_sat:
8488 case Intrinsic::ssub_sat:
8489 case Intrinsic::sshl_sat:
8490 case Intrinsic::uadd_sat:
8491 case Intrinsic::usub_sat:
8492 case Intrinsic::ushl_sat:
8493 case Intrinsic::smul_fix:
8494 case Intrinsic::smul_fix_sat:
8495 case Intrinsic::umul_fix:
8496 case Intrinsic::umul_fix_sat:
8497 case Intrinsic::pow:
8498 case Intrinsic::powi:
8499 case Intrinsic::sin:
8500 case Intrinsic::sinh:
8501 case Intrinsic::cos:
8502 case Intrinsic::cosh:
8503 case Intrinsic::sincos:
8504 case Intrinsic::sincospi:
8505 case Intrinsic::tan:
8506 case Intrinsic::tanh:
8507 case Intrinsic::asin:
8508 case Intrinsic::acos:
8509 case Intrinsic::atan:
8510 case Intrinsic::atan2:
8511 case Intrinsic::canonicalize:
8512 case Intrinsic::sqrt:
8513 case Intrinsic::exp:
8514 case Intrinsic::exp2:
8515 case Intrinsic::exp10:
8516 case Intrinsic::log:
8517 case Intrinsic::log2:
8518 case Intrinsic::log10:
8519 case Intrinsic::modf:
8520 case Intrinsic::floor:
8521 case Intrinsic::ceil:
8522 case Intrinsic::trunc:
8523 case Intrinsic::rint:
8524 case Intrinsic::nearbyint:
8525 case Intrinsic::round:
8526 case Intrinsic::roundeven:
8527 case Intrinsic::lrint:
8528 case Intrinsic::llrint:
8529 case Intrinsic::fshl:
8530 case Intrinsic::fshr:
8531 case Intrinsic::frexp:
8532 case Intrinsic::get_active_lane_mask:
8533 return true;
8534 default:
8535 return false;
8536 }
8537}
8538
8539bool llvm::propagatesPoison(const Use &PoisonOp) {
8540 const Operator *I = cast<Operator>(Val: PoisonOp.getUser());
8541 switch (I->getOpcode()) {
8542 case Instruction::Freeze:
8543 case Instruction::PHI:
8544 case Instruction::Invoke:
8545 return false;
8546 case Instruction::Select:
8547 return PoisonOp.getOperandNo() == 0;
8548 case Instruction::Call:
8549 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
8550 return intrinsicPropagatesPoison(IID: II->getIntrinsicID());
8551 return false;
8552 case Instruction::ICmp:
8553 case Instruction::FCmp:
8554 case Instruction::GetElementPtr:
8555 return true;
8556 default:
8557 if (isa<BinaryOperator>(Val: I) || isa<UnaryOperator>(Val: I) || isa<CastInst>(Val: I))
8558 return true;
8559
8560 // Be conservative and return false.
8561 return false;
8562 }
8563}
8564
8565/// Enumerates all operands of \p I that are guaranteed to not be undef or
8566/// poison. If the callback \p Handle returns true, stop processing and return
8567/// true. Otherwise, return false.
8568template <typename CallableT>
8569static bool handleGuaranteedWellDefinedOps(const Instruction *I,
8570 const CallableT &Handle) {
8571 switch (I->getOpcode()) {
8572 case Instruction::Store:
8573 if (Handle(cast<StoreInst>(Val: I)->getPointerOperand()))
8574 return true;
8575 break;
8576
8577 case Instruction::Load:
8578 if (Handle(cast<LoadInst>(Val: I)->getPointerOperand()))
8579 return true;
8580 break;
8581
8582 // Since dereferenceable attribute imply noundef, atomic operations
8583 // also implicitly have noundef pointers too
8584 case Instruction::AtomicCmpXchg:
8585 if (Handle(cast<AtomicCmpXchgInst>(Val: I)->getPointerOperand()))
8586 return true;
8587 break;
8588
8589 case Instruction::AtomicRMW:
8590 if (Handle(cast<AtomicRMWInst>(Val: I)->getPointerOperand()))
8591 return true;
8592 break;
8593
8594 case Instruction::Call:
8595 case Instruction::Invoke: {
8596 const CallBase *CB = cast<CallBase>(Val: I);
8597 if (CB->isIndirectCall() && Handle(CB->getCalledOperand()))
8598 return true;
8599 for (unsigned i = 0; i < CB->arg_size(); ++i)
8600 if ((CB->paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef) ||
8601 CB->paramHasAttr(ArgNo: i, Kind: Attribute::Dereferenceable) ||
8602 CB->paramHasAttr(ArgNo: i, Kind: Attribute::DereferenceableOrNull)) &&
8603 Handle(CB->getArgOperand(i)))
8604 return true;
8605 break;
8606 }
8607 case Instruction::Ret:
8608 if (I->getFunction()->hasRetAttribute(Kind: Attribute::NoUndef) &&
8609 Handle(I->getOperand(i: 0)))
8610 return true;
8611 break;
8612 case Instruction::Switch:
8613 if (Handle(cast<SwitchInst>(Val: I)->getCondition()))
8614 return true;
8615 break;
8616 case Instruction::CondBr:
8617 if (Handle(cast<CondBrInst>(Val: I)->getCondition()))
8618 return true;
8619 break;
8620 default:
8621 break;
8622 }
8623
8624 return false;
8625}
8626
8627/// Enumerates all operands of \p I that are guaranteed to not be poison.
8628template <typename CallableT>
8629static bool handleGuaranteedNonPoisonOps(const Instruction *I,
8630 const CallableT &Handle) {
8631 if (handleGuaranteedWellDefinedOps(I, Handle))
8632 return true;
8633 switch (I->getOpcode()) {
8634 // Divisors of these operations are allowed to be partially undef.
8635 case Instruction::UDiv:
8636 case Instruction::SDiv:
8637 case Instruction::URem:
8638 case Instruction::SRem:
8639 return Handle(I->getOperand(i: 1));
8640 default:
8641 return false;
8642 }
8643}
8644
8645bool llvm::mustTriggerUB(const Instruction *I,
8646 const SmallPtrSetImpl<const Value *> &KnownPoison) {
8647 return handleGuaranteedNonPoisonOps(
8648 I, Handle: [&](const Value *V) { return KnownPoison.count(Ptr: V); });
8649}
8650
8651static bool programUndefinedIfUndefOrPoison(const Value *V,
8652 bool PoisonOnly) {
8653 // We currently only look for uses of values within the same basic
8654 // block, as that makes it easier to guarantee that the uses will be
8655 // executed given that Inst is executed.
8656 //
8657 // FIXME: Expand this to consider uses beyond the same basic block. To do
8658 // this, look out for the distinction between post-dominance and strong
8659 // post-dominance.
8660 const BasicBlock *BB = nullptr;
8661 BasicBlock::const_iterator Begin;
8662 if (const auto *Inst = dyn_cast<Instruction>(Val: V)) {
8663 BB = Inst->getParent();
8664 Begin = Inst->getIterator();
8665 Begin++;
8666 } else if (const auto *Arg = dyn_cast<Argument>(Val: V)) {
8667 if (Arg->getParent()->isDeclaration())
8668 return false;
8669 BB = &Arg->getParent()->getEntryBlock();
8670 Begin = BB->begin();
8671 } else {
8672 return false;
8673 }
8674
8675 // Limit number of instructions we look at, to avoid scanning through large
8676 // blocks. The current limit is chosen arbitrarily.
8677 unsigned ScanLimit = 32;
8678 BasicBlock::const_iterator End = BB->end();
8679
8680 if (!PoisonOnly) {
8681 // Since undef does not propagate eagerly, be conservative & just check
8682 // whether a value is directly passed to an instruction that must take
8683 // well-defined operands.
8684
8685 for (const auto &I : make_range(x: Begin, y: End)) {
8686 if (--ScanLimit == 0)
8687 break;
8688
8689 if (handleGuaranteedWellDefinedOps(I: &I, Handle: [V](const Value *WellDefinedOp) {
8690 return WellDefinedOp == V;
8691 }))
8692 return true;
8693
8694 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8695 break;
8696 }
8697 return false;
8698 }
8699
8700 // Set of instructions that we have proved will yield poison if Inst
8701 // does.
8702 SmallPtrSet<const Value *, 16> YieldsPoison;
8703 SmallPtrSet<const BasicBlock *, 4> Visited;
8704
8705 YieldsPoison.insert(Ptr: V);
8706 Visited.insert(Ptr: BB);
8707
8708 while (true) {
8709 for (const auto &I : make_range(x: Begin, y: End)) {
8710 if (--ScanLimit == 0)
8711 return false;
8712 if (mustTriggerUB(I: &I, KnownPoison: YieldsPoison))
8713 return true;
8714 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8715 return false;
8716
8717 // If an operand is poison and propagates it, mark I as yielding poison.
8718 for (const Use &Op : I.operands()) {
8719 if (YieldsPoison.count(Ptr: Op) && propagatesPoison(PoisonOp: Op)) {
8720 YieldsPoison.insert(Ptr: &I);
8721 break;
8722 }
8723 }
8724
8725 // Special handling for select, which returns poison if its operand 0 is
8726 // poison (handled in the loop above) *or* if both its true/false operands
8727 // are poison (handled here).
8728 if (I.getOpcode() == Instruction::Select &&
8729 YieldsPoison.count(Ptr: I.getOperand(i: 1)) &&
8730 YieldsPoison.count(Ptr: I.getOperand(i: 2))) {
8731 YieldsPoison.insert(Ptr: &I);
8732 }
8733 }
8734
8735 BB = BB->getSingleSuccessor();
8736 if (!BB || !Visited.insert(Ptr: BB).second)
8737 break;
8738
8739 Begin = BB->getFirstNonPHIIt();
8740 End = BB->end();
8741 }
8742 return false;
8743}
8744
8745bool llvm::programUndefinedIfUndefOrPoison(const Instruction *Inst) {
8746 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: false);
8747}
8748
8749bool llvm::programUndefinedIfPoison(const Instruction *Inst) {
8750 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: true);
8751}
8752
8753static bool isKnownNonNaN(const Value *V, FastMathFlags FMF) {
8754 if (FMF.noNaNs())
8755 return true;
8756
8757 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8758 return !C->isNaN();
8759
8760 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8761 if (!C->getElementType()->isFloatingPointTy())
8762 return false;
8763 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8764 if (C->getElementAsAPFloat(i: I).isNaN())
8765 return false;
8766 }
8767 return true;
8768 }
8769
8770 if (isa<ConstantAggregateZero>(Val: V))
8771 return true;
8772
8773 return false;
8774}
8775
8776static bool isKnownNonZero(const Value *V) {
8777 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8778 return !C->isZero();
8779
8780 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8781 if (!C->getElementType()->isFloatingPointTy())
8782 return false;
8783 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8784 if (C->getElementAsAPFloat(i: I).isZero())
8785 return false;
8786 }
8787 return true;
8788 }
8789
8790 return false;
8791}
8792
8793/// Match clamp pattern for float types without care about NaNs or signed zeros.
8794/// Given non-min/max outer cmp/select from the clamp pattern this
8795/// function recognizes if it can be substitued by a "canonical" min/max
8796/// pattern.
8797static SelectPatternResult matchFastFloatClamp(CmpInst::Predicate Pred,
8798 Value *CmpLHS, Value *CmpRHS,
8799 Value *TrueVal, Value *FalseVal,
8800 Value *&LHS, Value *&RHS) {
8801 // Try to match
8802 // X < C1 ? C1 : Min(X, C2) --> Max(C1, Min(X, C2))
8803 // X > C1 ? C1 : Max(X, C2) --> Min(C1, Max(X, C2))
8804 // and return description of the outer Max/Min.
8805
8806 // First, check if select has inverse order:
8807 if (CmpRHS == FalseVal) {
8808 std::swap(a&: TrueVal, b&: FalseVal);
8809 Pred = CmpInst::getInversePredicate(pred: Pred);
8810 }
8811
8812 // Assume success now. If there's no match, callers should not use these anyway.
8813 LHS = TrueVal;
8814 RHS = FalseVal;
8815
8816 const APFloat *FC1;
8817 if (CmpRHS != TrueVal || !match(V: CmpRHS, P: m_APFloat(Res&: FC1)) || !FC1->isFinite())
8818 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8819
8820 const APFloat *FC2;
8821 switch (Pred) {
8822 case CmpInst::FCMP_OLT:
8823 case CmpInst::FCMP_OLE:
8824 case CmpInst::FCMP_ULT:
8825 case CmpInst::FCMP_ULE:
8826 if (match(V: FalseVal, P: m_OrdOrUnordFMin(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8827 *FC1 < *FC2)
8828 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8829 if (match(V: FalseVal, P: m_FMinNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
8830 *FC1 < *FC2)
8831 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8832 break;
8833 case CmpInst::FCMP_OGT:
8834 case CmpInst::FCMP_OGE:
8835 case CmpInst::FCMP_UGT:
8836 case CmpInst::FCMP_UGE:
8837 if (match(V: FalseVal, P: m_OrdOrUnordFMax(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8838 *FC1 > *FC2)
8839 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8840 if (match(V: FalseVal, P: m_FMaxNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
8841 *FC1 > *FC2)
8842 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8843 break;
8844 default:
8845 break;
8846 }
8847
8848 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8849}
8850
8851/// Recognize variations of:
8852/// CLAMP(v,l,h) ==> ((v) < (l) ? (l) : ((v) > (h) ? (h) : (v)))
8853static SelectPatternResult matchClamp(CmpInst::Predicate Pred,
8854 Value *CmpLHS, Value *CmpRHS,
8855 Value *TrueVal, Value *FalseVal) {
8856 // Swap the select operands and predicate to match the patterns below.
8857 if (CmpRHS != TrueVal) {
8858 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8859 std::swap(a&: TrueVal, b&: FalseVal);
8860 }
8861 const APInt *C1;
8862 if (CmpRHS == TrueVal && match(V: CmpRHS, P: m_APInt(Res&: C1))) {
8863 const APInt *C2;
8864 // (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
8865 if (match(V: FalseVal, P: m_SMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8866 C1->slt(RHS: *C2) && Pred == CmpInst::ICMP_SLT)
8867 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8868
8869 // (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
8870 if (match(V: FalseVal, P: m_SMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8871 C1->sgt(RHS: *C2) && Pred == CmpInst::ICMP_SGT)
8872 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8873
8874 // (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
8875 if (match(V: FalseVal, P: m_UMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8876 C1->ult(RHS: *C2) && Pred == CmpInst::ICMP_ULT)
8877 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8878
8879 // (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
8880 if (match(V: FalseVal, P: m_UMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8881 C1->ugt(RHS: *C2) && Pred == CmpInst::ICMP_UGT)
8882 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8883 }
8884 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8885}
8886
8887/// Recognize variations of:
8888/// a < c ? min(a,b) : min(b,c) ==> min(min(a,b),min(b,c))
8889static SelectPatternResult matchMinMaxOfMinMax(CmpInst::Predicate Pred,
8890 Value *CmpLHS, Value *CmpRHS,
8891 Value *TVal, Value *FVal,
8892 unsigned Depth) {
8893 // TODO: Allow FP min/max with nnan/nsz.
8894 assert(CmpInst::isIntPredicate(Pred) && "Expected integer comparison");
8895
8896 Value *A = nullptr, *B = nullptr;
8897 SelectPatternResult L = matchSelectPattern(V: TVal, LHS&: A, RHS&: B, CastOp: nullptr, Depth: Depth + 1);
8898 if (!SelectPatternResult::isMinOrMax(SPF: L.Flavor))
8899 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8900
8901 Value *C = nullptr, *D = nullptr;
8902 SelectPatternResult R = matchSelectPattern(V: FVal, LHS&: C, RHS&: D, CastOp: nullptr, Depth: Depth + 1);
8903 if (L.Flavor != R.Flavor)
8904 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8905
8906 // We have something like: x Pred y ? min(a, b) : min(c, d).
8907 // Try to match the compare to the min/max operations of the select operands.
8908 // First, make sure we have the right compare predicate.
8909 switch (L.Flavor) {
8910 case SPF_SMIN:
8911 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) {
8912 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8913 std::swap(a&: CmpLHS, b&: CmpRHS);
8914 }
8915 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
8916 break;
8917 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8918 case SPF_SMAX:
8919 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) {
8920 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8921 std::swap(a&: CmpLHS, b&: CmpRHS);
8922 }
8923 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)
8924 break;
8925 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8926 case SPF_UMIN:
8927 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
8928 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8929 std::swap(a&: CmpLHS, b&: CmpRHS);
8930 }
8931 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
8932 break;
8933 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8934 case SPF_UMAX:
8935 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
8936 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8937 std::swap(a&: CmpLHS, b&: CmpRHS);
8938 }
8939 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
8940 break;
8941 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8942 default:
8943 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8944 }
8945
8946 // If there is a common operand in the already matched min/max and the other
8947 // min/max operands match the compare operands (either directly or inverted),
8948 // then this is min/max of the same flavor.
8949
8950 // a pred c ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
8951 // ~c pred ~a ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
8952 if (D == B) {
8953 if ((CmpLHS == A && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8954 match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
8955 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8956 }
8957 // a pred d ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
8958 // ~d pred ~a ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
8959 if (C == B) {
8960 if ((CmpLHS == A && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8961 match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
8962 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8963 }
8964 // b pred c ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
8965 // ~c pred ~b ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
8966 if (D == A) {
8967 if ((CmpLHS == B && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8968 match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
8969 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8970 }
8971 // b pred d ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
8972 // ~d pred ~b ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
8973 if (C == A) {
8974 if ((CmpLHS == B && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
8975 match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
8976 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
8977 }
8978
8979 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8980}
8981
8982/// If the input value is the result of a 'not' op, constant integer, or vector
8983/// splat of a constant integer, return the bitwise-not source value.
8984/// TODO: This could be extended to handle non-splat vector integer constants.
8985static Value *getNotValue(Value *V) {
8986 Value *NotV;
8987 if (match(V, P: m_Not(V: m_Value(V&: NotV))))
8988 return NotV;
8989
8990 const APInt *C;
8991 if (match(V, P: m_APInt(Res&: C)))
8992 return ConstantInt::get(Ty: V->getType(), V: ~(*C));
8993
8994 return nullptr;
8995}
8996
8997/// Match non-obvious integer minimum and maximum sequences.
8998static SelectPatternResult matchMinMax(CmpInst::Predicate Pred,
8999 Value *CmpLHS, Value *CmpRHS,
9000 Value *TrueVal, Value *FalseVal,
9001 Value *&LHS, Value *&RHS,
9002 unsigned Depth) {
9003 // Assume success. If there's no match, callers should not use these anyway.
9004 LHS = TrueVal;
9005 RHS = FalseVal;
9006
9007 SelectPatternResult SPR = matchClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal);
9008 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
9009 return SPR;
9010
9011 SPR = matchMinMaxOfMinMax(Pred, CmpLHS, CmpRHS, TVal: TrueVal, FVal: FalseVal, Depth);
9012 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
9013 return SPR;
9014
9015 // Look through 'not' ops to find disguised min/max.
9016 // (X > Y) ? ~X : ~Y ==> (~X < ~Y) ? ~X : ~Y ==> MIN(~X, ~Y)
9017 // (X < Y) ? ~X : ~Y ==> (~X > ~Y) ? ~X : ~Y ==> MAX(~X, ~Y)
9018 if (CmpLHS == getNotValue(V: TrueVal) && CmpRHS == getNotValue(V: FalseVal)) {
9019 switch (Pred) {
9020 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9021 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9022 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9023 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9024 default: break;
9025 }
9026 }
9027
9028 // (X > Y) ? ~Y : ~X ==> (~X < ~Y) ? ~Y : ~X ==> MAX(~Y, ~X)
9029 // (X < Y) ? ~Y : ~X ==> (~X > ~Y) ? ~Y : ~X ==> MIN(~Y, ~X)
9030 if (CmpLHS == getNotValue(V: FalseVal) && CmpRHS == getNotValue(V: TrueVal)) {
9031 switch (Pred) {
9032 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9033 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9034 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9035 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9036 default: break;
9037 }
9038 }
9039
9040 if (Pred != CmpInst::ICMP_SGT && Pred != CmpInst::ICMP_SLT)
9041 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9042
9043 const APInt *C1;
9044 if (!match(V: CmpRHS, P: m_APInt(Res&: C1)))
9045 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9046
9047 // An unsigned min/max can be written with a signed compare.
9048 const APInt *C2;
9049 if ((CmpLHS == TrueVal && match(V: FalseVal, P: m_APInt(Res&: C2))) ||
9050 (CmpLHS == FalseVal && match(V: TrueVal, P: m_APInt(Res&: C2)))) {
9051 // Is the sign bit set?
9052 // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
9053 // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
9054 if (Pred == CmpInst::ICMP_SLT && C1->isZero() && C2->isMaxSignedValue())
9055 return {.Flavor: CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9056
9057 // Is the sign bit clear?
9058 // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
9059 // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
9060 if (Pred == CmpInst::ICMP_SGT && C1->isAllOnes() && C2->isMinSignedValue())
9061 return {.Flavor: CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9062 }
9063
9064 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9065}
9066
9067bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW,
9068 bool AllowPoison) {
9069 assert(X && Y && "Invalid operand");
9070
9071 auto IsNegationOf = [&](const Value *X, const Value *Y) {
9072 if (!match(V: X, P: m_Neg(V: m_Specific(V: Y))))
9073 return false;
9074
9075 auto *BO = cast<BinaryOperator>(Val: X);
9076 if (NeedNSW && !BO->hasNoSignedWrap())
9077 return false;
9078
9079 auto *Zero = cast<Constant>(Val: BO->getOperand(i_nocapture: 0));
9080 if (!AllowPoison && !Zero->isNullValue())
9081 return false;
9082
9083 return true;
9084 };
9085
9086 // X = -Y or Y = -X
9087 if (IsNegationOf(X, Y) || IsNegationOf(Y, X))
9088 return true;
9089
9090 // X = sub (A, B), Y = sub (B, A) || X = sub nsw (A, B), Y = sub nsw (B, A)
9091 Value *A, *B;
9092 return (!NeedNSW && (match(V: X, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9093 match(V: Y, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))))) ||
9094 (NeedNSW && (match(V: X, P: m_NSWSub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9095 match(V: Y, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A)))));
9096}
9097
9098bool llvm::isKnownInversion(const Value *X, const Value *Y) {
9099 // Handle X = icmp pred A, B, Y = icmp pred A, C.
9100 Value *A, *B, *C;
9101 CmpPredicate Pred1, Pred2;
9102 if (!match(V: X, P: m_ICmp(Pred&: Pred1, L: m_Value(V&: A), R: m_Value(V&: B))) ||
9103 !match(V: Y, P: m_c_ICmp(Pred&: Pred2, L: m_Specific(V: A), R: m_Value(V&: C))))
9104 return false;
9105
9106 // They must both have samesign flag or not.
9107 if (Pred1.hasSameSign() != Pred2.hasSameSign())
9108 return false;
9109
9110 if (B == C)
9111 return Pred1 == ICmpInst::getInversePredicate(pred: Pred2);
9112
9113 // Try to infer the relationship from constant ranges.
9114 const APInt *RHSC1, *RHSC2;
9115 if (!match(V: B, P: m_APInt(Res&: RHSC1)) || !match(V: C, P: m_APInt(Res&: RHSC2)))
9116 return false;
9117
9118 // Sign bits of two RHSCs should match.
9119 if (Pred1.hasSameSign() && RHSC1->isNonNegative() != RHSC2->isNonNegative())
9120 return false;
9121
9122 const auto CR1 = ConstantRange::makeExactICmpRegion(Pred: Pred1, Other: *RHSC1);
9123 const auto CR2 = ConstantRange::makeExactICmpRegion(Pred: Pred2, Other: *RHSC2);
9124
9125 return CR1.inverse() == CR2;
9126}
9127
9128SelectPatternResult llvm::getSelectPattern(CmpInst::Predicate Pred,
9129 SelectPatternNaNBehavior NaNBehavior,
9130 bool Ordered) {
9131 switch (Pred) {
9132 default:
9133 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false}; // Equality.
9134 case ICmpInst::ICMP_UGT:
9135 case ICmpInst::ICMP_UGE:
9136 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9137 case ICmpInst::ICMP_SGT:
9138 case ICmpInst::ICMP_SGE:
9139 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9140 case ICmpInst::ICMP_ULT:
9141 case ICmpInst::ICMP_ULE:
9142 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9143 case ICmpInst::ICMP_SLT:
9144 case ICmpInst::ICMP_SLE:
9145 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9146 case FCmpInst::FCMP_UGT:
9147 case FCmpInst::FCMP_UGE:
9148 case FCmpInst::FCMP_OGT:
9149 case FCmpInst::FCMP_OGE:
9150 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
9151 case FCmpInst::FCMP_ULT:
9152 case FCmpInst::FCMP_ULE:
9153 case FCmpInst::FCMP_OLT:
9154 case FCmpInst::FCMP_OLE:
9155 return {.Flavor: SPF_FMINNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
9156 }
9157}
9158
9159std::optional<std::pair<CmpPredicate, Constant *>>
9160llvm::getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred, Constant *C) {
9161 assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
9162 "Only for relational integer predicates.");
9163 if (isa<UndefValue>(Val: C))
9164 return std::nullopt;
9165
9166 Type *Type = C->getType();
9167 bool IsSigned = ICmpInst::isSigned(Pred);
9168
9169 CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred);
9170 bool WillIncrement =
9171 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
9172
9173 // Check if the constant operand can be safely incremented/decremented
9174 // without overflowing/underflowing.
9175 auto ConstantIsOk = [Pred, WillIncrement, IsSigned](ConstantInt *C) {
9176 if (WillIncrement ? C->isMaxValue(IsSigned) : C->isMinValue(IsSigned))
9177 return false;
9178
9179 if (!Pred.hasSameSign())
9180 return true;
9181
9182 // Crossing the corresponding boundary in the other ordering changes the
9183 // sign bit, and therefore changes the poison domain.
9184 return WillIncrement ? !C->isMaxValue(IsSigned: !IsSigned)
9185 : !C->isMinValue(IsSigned: !IsSigned);
9186 };
9187
9188 Constant *SafeReplacementConstant = nullptr;
9189 if (auto *CI = dyn_cast<ConstantInt>(Val: C)) {
9190 // Bail out if the constant can't be safely incremented/decremented.
9191 if (!ConstantIsOk(CI))
9192 return std::nullopt;
9193 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Type)) {
9194 unsigned NumElts = FVTy->getNumElements();
9195 for (unsigned i = 0; i != NumElts; ++i) {
9196 Constant *Elt = C->getAggregateElement(Elt: i);
9197 if (!Elt)
9198 return std::nullopt;
9199
9200 if (isa<UndefValue>(Val: Elt))
9201 continue;
9202
9203 // Bail out if we can't determine if this constant is min/max or if we
9204 // know that this constant is min/max.
9205 auto *CI = dyn_cast<ConstantInt>(Val: Elt);
9206 if (!CI || !ConstantIsOk(CI))
9207 return std::nullopt;
9208
9209 if (!SafeReplacementConstant)
9210 SafeReplacementConstant = CI;
9211 }
9212 } else if (isa<VectorType>(Val: C->getType())) {
9213 // Handle scalable splat
9214 Value *SplatC = C->getSplatValue();
9215 auto *CI = dyn_cast_or_null<ConstantInt>(Val: SplatC);
9216 // Bail out if the constant can't be safely incremented/decremented.
9217 if (!CI || !ConstantIsOk(CI))
9218 return std::nullopt;
9219 } else {
9220 // ConstantExpr?
9221 return std::nullopt;
9222 }
9223
9224 // It may not be safe to change a compare predicate in the presence of
9225 // undefined elements, so replace those elements with the first safe constant
9226 // that we found.
9227 // TODO: in case of poison, it is safe; let's replace undefs only.
9228 if (C->containsUndefOrPoisonElement()) {
9229 assert(SafeReplacementConstant && "Replacement constant not set");
9230 C = Constant::replaceUndefsWith(C, Replacement: SafeReplacementConstant);
9231 }
9232
9233 CmpPredicate NewPred(CmpInst::getFlippedStrictnessPredicate(pred: Pred),
9234 Pred.hasSameSign());
9235
9236 // Increment or decrement the constant.
9237 Constant *OneOrNegOne = ConstantInt::get(Ty: Type, V: WillIncrement ? 1 : -1, IsSigned: true);
9238 Constant *NewC = ConstantExpr::getAdd(C1: C, C2: OneOrNegOne);
9239
9240 return std::make_pair(x&: NewPred, y&: NewC);
9241}
9242
9243static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
9244 FastMathFlags FMF,
9245 Value *CmpLHS, Value *CmpRHS,
9246 Value *TrueVal, Value *FalseVal,
9247 Value *&LHS, Value *&RHS,
9248 unsigned Depth) {
9249 if (CmpInst::isFPPredicate(P: Pred)) {
9250 // IEEE-754 ignores the sign of 0.0 in comparisons. So if the select has one
9251 // 0.0 operand, set the compare's 0.0 operands to that same value for the
9252 // purpose of identifying min/max. Disregard vector constants with undefined
9253 // elements because those can not be back-propagated for analysis.
9254 Value *OutputZeroVal = nullptr;
9255 if (match(V: TrueVal, P: m_AnyZeroFP()) && !match(V: FalseVal, P: m_AnyZeroFP()) &&
9256 !cast<Constant>(Val: TrueVal)->containsUndefOrPoisonElement())
9257 OutputZeroVal = TrueVal;
9258 else if (match(V: FalseVal, P: m_AnyZeroFP()) && !match(V: TrueVal, P: m_AnyZeroFP()) &&
9259 !cast<Constant>(Val: FalseVal)->containsUndefOrPoisonElement())
9260 OutputZeroVal = FalseVal;
9261
9262 if (OutputZeroVal) {
9263 if (match(V: CmpLHS, P: m_AnyZeroFP()) && CmpLHS != OutputZeroVal)
9264 CmpLHS = OutputZeroVal;
9265 if (match(V: CmpRHS, P: m_AnyZeroFP()) && CmpRHS != OutputZeroVal)
9266 CmpRHS = OutputZeroVal;
9267 }
9268 }
9269
9270 LHS = CmpLHS;
9271 RHS = CmpRHS;
9272
9273 // Signed zero may return inconsistent results between implementations.
9274 // (0.0 <= -0.0) ? 0.0 : -0.0 // Returns 0.0
9275 // minNum(0.0, -0.0) // May return -0.0 or 0.0 (IEEE 754-2008 5.3.1)
9276 // Therefore, we behave conservatively and only proceed if at least one of the
9277 // operands is known to not be zero or if we don't care about signed zero.
9278 if (CmpInst::isFPPredicate(P: Pred)) {
9279 if (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9280 !isKnownNonZero(V: CmpRHS))
9281 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9282 }
9283
9284 SelectPatternNaNBehavior NaNBehavior = SPNB_NA;
9285 bool Ordered = false;
9286
9287 // When given one NaN and one non-NaN input:
9288 // - maxnum/minnum (C99 fmaxf()/fminf()) return the non-NaN input.
9289 // - A simple C99 (a < b ? a : b) construction will return 'b' (as the
9290 // ordered comparison fails), which could be NaN or non-NaN.
9291 // so here we discover exactly what NaN behavior is required/accepted.
9292 if (CmpInst::isFPPredicate(P: Pred)) {
9293 bool LHSSafe = isKnownNonNaN(V: CmpLHS, FMF);
9294 bool RHSSafe = isKnownNonNaN(V: CmpRHS, FMF);
9295
9296 if (LHSSafe && RHSSafe) {
9297 // Both operands are known non-NaN.
9298 NaNBehavior = SPNB_RETURNS_ANY;
9299 Ordered = CmpInst::isOrdered(predicate: Pred);
9300 } else if (CmpInst::isOrdered(predicate: Pred)) {
9301 // An ordered comparison will return false when given a NaN, so it
9302 // returns the RHS.
9303 Ordered = true;
9304 if (LHSSafe)
9305 // LHS is non-NaN, so if RHS is NaN then NaN will be returned.
9306 NaNBehavior = SPNB_RETURNS_NAN;
9307 else if (RHSSafe)
9308 NaNBehavior = SPNB_RETURNS_OTHER;
9309 else
9310 // Completely unsafe.
9311 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9312 } else {
9313 Ordered = false;
9314 // An unordered comparison will return true when given a NaN, so it
9315 // returns the LHS.
9316 if (LHSSafe)
9317 // LHS is non-NaN, so if RHS is NaN then non-NaN will be returned.
9318 NaNBehavior = SPNB_RETURNS_OTHER;
9319 else if (RHSSafe)
9320 NaNBehavior = SPNB_RETURNS_NAN;
9321 else
9322 // Completely unsafe.
9323 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9324 }
9325 }
9326
9327 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
9328 std::swap(a&: CmpLHS, b&: CmpRHS);
9329 Pred = CmpInst::getSwappedPredicate(pred: Pred);
9330 if (NaNBehavior == SPNB_RETURNS_NAN)
9331 NaNBehavior = SPNB_RETURNS_OTHER;
9332 else if (NaNBehavior == SPNB_RETURNS_OTHER)
9333 NaNBehavior = SPNB_RETURNS_NAN;
9334 Ordered = !Ordered;
9335 }
9336
9337 // ([if]cmp X, Y) ? X : Y
9338 if (TrueVal == CmpLHS && FalseVal == CmpRHS)
9339 return getSelectPattern(Pred, NaNBehavior, Ordered);
9340
9341 if (isKnownNegation(X: TrueVal, Y: FalseVal)) {
9342 // Sign-extending LHS does not change its sign, so TrueVal/FalseVal can
9343 // match against either LHS or sign-preserving operations on LHS, like
9344 // sext(LHS), or binary ops that do not wrap in signed sense.
9345 auto CmpLHSOrSExt =
9346 m_CombineOr(Ps: m_Specific(V: CmpLHS), Ps: m_SExt(Op: m_Specific(V: CmpLHS)));
9347 auto MaybeSExtOrMulCmpLHS =
9348 m_CombineOr(Ps: CmpLHSOrSExt, Ps: m_NSWMul(L: CmpLHSOrSExt, R: m_StrictlyPositive()),
9349 Ps: m_NSWShl(L: CmpLHSOrSExt, R: m_Value()));
9350 auto ZeroOrAllOnes = m_CombineOr(Ps: m_ZeroInt(), Ps: m_AllOnes());
9351 auto ZeroOrOne = m_CombineOr(Ps: m_ZeroInt(), Ps: m_One());
9352 if (match(V: TrueVal, P: MaybeSExtOrMulCmpLHS)) {
9353 // Set the return values. If the compare uses the negated value (-X >s 0),
9354 // swap the return values because the negated value is always 'RHS'.
9355 LHS = TrueVal;
9356 RHS = FalseVal;
9357 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: FalseVal))))
9358 std::swap(a&: LHS, b&: RHS);
9359
9360 // (X >s 0) ? X : -X or (X >s -1) ? X : -X --> ABS(X)
9361 // (-X >s 0) ? -X : X or (-X >s -1) ? -X : X --> ABS(X)
9362 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9363 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9364
9365 // (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X)
9366 if (Pred == ICmpInst::ICMP_SGE && match(V: CmpRHS, P: ZeroOrOne))
9367 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9368
9369 // (X <s 0) ? X : -X or (X <s 1) ? X : -X --> NABS(X)
9370 // (-X <s 0) ? -X : X or (-X <s 1) ? -X : X --> NABS(X)
9371 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9372 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9373 } else if (match(V: FalseVal, P: MaybeSExtOrMulCmpLHS)) {
9374 // Set the return values. If the compare uses the negated value (-X >s 0),
9375 // swap the return values because the negated value is always 'RHS'.
9376 LHS = FalseVal;
9377 RHS = TrueVal;
9378 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: TrueVal))))
9379 std::swap(a&: LHS, b&: RHS);
9380
9381 // (X >s 0) ? -X : X or (X >s -1) ? -X : X --> NABS(X)
9382 // (-X >s 0) ? X : -X or (-X >s -1) ? X : -X --> NABS(X)
9383 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9384 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9385
9386 // (X <s 0) ? -X : X or (X <s 1) ? -X : X --> ABS(X)
9387 // (-X <s 0) ? X : -X or (-X <s 1) ? X : -X --> ABS(X)
9388 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9389 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9390 }
9391 }
9392
9393 if (CmpInst::isIntPredicate(P: Pred))
9394 return matchMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS, Depth);
9395
9396 // According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar
9397 // may return either -0.0 or 0.0, so fcmp/select pair has stricter
9398 // semantics than minNum. Be conservative in such case.
9399 if (NaNBehavior != SPNB_RETURNS_ANY ||
9400 (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9401 !isKnownNonZero(V: CmpRHS)))
9402 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9403
9404 return matchFastFloatClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
9405}
9406
9407static Value *lookThroughCastConst(CmpInst *CmpI, Type *SrcTy, Constant *C,
9408 Instruction::CastOps *CastOp) {
9409 const DataLayout &DL = CmpI->getDataLayout();
9410
9411 Constant *CastedTo = nullptr;
9412 switch (*CastOp) {
9413 case Instruction::ZExt:
9414 if (CmpI->isUnsigned())
9415 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy);
9416 break;
9417 case Instruction::SExt:
9418 if (CmpI->isSigned())
9419 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy, OnlyIfReduced: true);
9420 break;
9421 case Instruction::Trunc:
9422 Constant *CmpConst;
9423 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_Constant(C&: CmpConst)) &&
9424 CmpConst->getType() == SrcTy) {
9425 // Here we have the following case:
9426 //
9427 // %cond = cmp iN %x, CmpConst
9428 // %tr = trunc iN %x to iK
9429 // %narrowsel = select i1 %cond, iK %t, iK C
9430 //
9431 // We can always move trunc after select operation:
9432 //
9433 // %cond = cmp iN %x, CmpConst
9434 // %widesel = select i1 %cond, iN %x, iN CmpConst
9435 // %tr = trunc iN %widesel to iK
9436 //
9437 // Note that C could be extended in any way because we don't care about
9438 // upper bits after truncation. It can't be abs pattern, because it would
9439 // look like:
9440 //
9441 // select i1 %cond, x, -x.
9442 //
9443 // So only min/max pattern could be matched. Such match requires widened C
9444 // == CmpConst. That is why set widened C = CmpConst, condition trunc
9445 // CmpConst == C is checked below.
9446 CastedTo = CmpConst;
9447 } else {
9448 unsigned ExtOp = CmpI->isSigned() ? Instruction::SExt : Instruction::ZExt;
9449 CastedTo = ConstantFoldCastOperand(Opcode: ExtOp, C, DestTy: SrcTy, DL);
9450 }
9451 break;
9452 case Instruction::FPTrunc:
9453 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPExt, C, DestTy: SrcTy, DL);
9454 break;
9455 case Instruction::FPExt:
9456 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPTrunc, C, DestTy: SrcTy, DL);
9457 break;
9458 case Instruction::FPToUI:
9459 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::UIToFP, C, DestTy: SrcTy, DL);
9460 break;
9461 case Instruction::FPToSI:
9462 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::SIToFP, C, DestTy: SrcTy, DL);
9463 break;
9464 case Instruction::UIToFP:
9465 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToUI, C, DestTy: SrcTy, DL);
9466 break;
9467 case Instruction::SIToFP:
9468 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToSI, C, DestTy: SrcTy, DL);
9469 break;
9470 default:
9471 break;
9472 }
9473
9474 if (!CastedTo)
9475 return nullptr;
9476
9477 // Make sure the cast doesn't lose any information.
9478 Constant *CastedBack =
9479 ConstantFoldCastOperand(Opcode: *CastOp, C: CastedTo, DestTy: C->getType(), DL);
9480 if (CastedBack && CastedBack != C)
9481 return nullptr;
9482
9483 return CastedTo;
9484}
9485
9486/// Helps to match a select pattern in case of a type mismatch.
9487///
9488/// The function processes the case when type of true and false values of a
9489/// select instruction differs from type of the cmp instruction operands because
9490/// of a cast instruction. The function checks if it is legal to move the cast
9491/// operation after "select". If yes, it returns the new second value of
9492/// "select" (with the assumption that cast is moved):
9493/// 1. As operand of cast instruction when both values of "select" are same cast
9494/// instructions.
9495/// 2. As restored constant (by applying reverse cast operation) when the first
9496/// value of the "select" is a cast operation and the second value is a
9497/// constant. It is implemented in lookThroughCastConst().
9498/// 3. As one operand is cast instruction and the other is not. The operands in
9499/// sel(cmp) are in different type integer.
9500/// NOTE: We return only the new second value because the first value could be
9501/// accessed as operand of cast instruction.
9502static Value *lookThroughCast(CmpInst *CmpI, Value *V1, Value *V2,
9503 Instruction::CastOps *CastOp) {
9504 auto *Cast1 = dyn_cast<CastInst>(Val: V1);
9505 if (!Cast1)
9506 return nullptr;
9507
9508 *CastOp = Cast1->getOpcode();
9509 Type *SrcTy = Cast1->getSrcTy();
9510 if (auto *Cast2 = dyn_cast<CastInst>(Val: V2)) {
9511 // If V1 and V2 are both the same cast from the same type, look through V1.
9512 if (*CastOp == Cast2->getOpcode() && SrcTy == Cast2->getSrcTy())
9513 return Cast2->getOperand(i_nocapture: 0);
9514 return nullptr;
9515 }
9516
9517 auto *C = dyn_cast<Constant>(Val: V2);
9518 if (C)
9519 return lookThroughCastConst(CmpI, SrcTy, C, CastOp);
9520
9521 Value *CastedTo = nullptr;
9522 if (*CastOp == Instruction::Trunc) {
9523 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_ZExtOrSExt(Op: m_Specific(V: V2)))) {
9524 // Here we have the following case:
9525 // %y_ext = sext iK %y to iN
9526 // %cond = cmp iN %x, %y_ext
9527 // %tr = trunc iN %x to iK
9528 // %narrowsel = select i1 %cond, iK %tr, iK %y
9529 //
9530 // We can always move trunc after select operation:
9531 // %y_ext = sext iK %y to iN
9532 // %cond = cmp iN %x, %y_ext
9533 // %widesel = select i1 %cond, iN %x, iN %y_ext
9534 // %tr = trunc iN %widesel to iK
9535 assert(V2->getType() == Cast1->getType() &&
9536 "V2 and Cast1 should be the same type.");
9537 CastedTo = CmpI->getOperand(i_nocapture: 1);
9538 }
9539 }
9540
9541 return CastedTo;
9542}
9543SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
9544 Instruction::CastOps *CastOp,
9545 unsigned Depth) {
9546 if (Depth >= MaxAnalysisRecursionDepth)
9547 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9548
9549 SelectInst *SI = dyn_cast<SelectInst>(Val: V);
9550 if (!SI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9551
9552 CmpInst *CmpI = dyn_cast<CmpInst>(Val: SI->getCondition());
9553 if (!CmpI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9554
9555 Value *TrueVal = SI->getTrueValue();
9556 Value *FalseVal = SI->getFalseValue();
9557
9558 return llvm::matchDecomposedSelectPattern(CmpI, TrueVal, FalseVal, LHS, RHS,
9559 FMF: SI->getFastMathFlagsOrNone(),
9560 CastOp, Depth);
9561}
9562
9563SelectPatternResult llvm::matchDecomposedSelectPattern(
9564 CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
9565 FastMathFlags FMF, Instruction::CastOps *CastOp, unsigned Depth) {
9566 CmpInst::Predicate Pred = CmpI->getPredicate();
9567 Value *CmpLHS = CmpI->getOperand(i_nocapture: 0);
9568 Value *CmpRHS = CmpI->getOperand(i_nocapture: 1);
9569 if (isa<FPMathOperator>(Val: CmpI) && CmpI->hasNoNaNs())
9570 FMF.setNoNaNs();
9571
9572 // Bail out early.
9573 if (CmpI->isEquality())
9574 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9575
9576 // Deal with type mismatches.
9577 if (CastOp && CmpLHS->getType() != TrueVal->getType()) {
9578 if (Value *C = lookThroughCast(CmpI, V1: TrueVal, V2: FalseVal, CastOp)) {
9579 // If this is a potential fmin/fmax with a cast to integer, then ignore
9580 // -0.0 because there is no corresponding integer value.
9581 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9582 FMF.setNoSignedZeros();
9583 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9584 TrueVal: cast<CastInst>(Val: TrueVal)->getOperand(i_nocapture: 0), FalseVal: C,
9585 LHS, RHS, Depth);
9586 }
9587 if (Value *C = lookThroughCast(CmpI, V1: FalseVal, V2: TrueVal, CastOp)) {
9588 // If this is a potential fmin/fmax with a cast to integer, then ignore
9589 // -0.0 because there is no corresponding integer value.
9590 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9591 FMF.setNoSignedZeros();
9592 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9593 TrueVal: C, FalseVal: cast<CastInst>(Val: FalseVal)->getOperand(i_nocapture: 0),
9594 LHS, RHS, Depth);
9595 }
9596 }
9597 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal, FalseVal,
9598 LHS, RHS, Depth);
9599}
9600
9601CmpInst::Predicate llvm::getMinMaxPred(SelectPatternFlavor SPF, bool Ordered) {
9602 if (SPF == SPF_SMIN) return ICmpInst::ICMP_SLT;
9603 if (SPF == SPF_UMIN) return ICmpInst::ICMP_ULT;
9604 if (SPF == SPF_SMAX) return ICmpInst::ICMP_SGT;
9605 if (SPF == SPF_UMAX) return ICmpInst::ICMP_UGT;
9606 if (SPF == SPF_FMINNUM)
9607 return Ordered ? FCmpInst::FCMP_OLT : FCmpInst::FCMP_ULT;
9608 if (SPF == SPF_FMAXNUM)
9609 return Ordered ? FCmpInst::FCMP_OGT : FCmpInst::FCMP_UGT;
9610 llvm_unreachable("unhandled!");
9611}
9612
9613Intrinsic::ID llvm::getMinMaxIntrinsic(SelectPatternFlavor SPF) {
9614 switch (SPF) {
9615 case SelectPatternFlavor::SPF_UMIN:
9616 return Intrinsic::umin;
9617 case SelectPatternFlavor::SPF_UMAX:
9618 return Intrinsic::umax;
9619 case SelectPatternFlavor::SPF_SMIN:
9620 return Intrinsic::smin;
9621 case SelectPatternFlavor::SPF_SMAX:
9622 return Intrinsic::smax;
9623 default:
9624 llvm_unreachable("Unexpected SPF");
9625 }
9626}
9627
9628SelectPatternFlavor llvm::getInverseMinMaxFlavor(SelectPatternFlavor SPF) {
9629 if (SPF == SPF_SMIN) return SPF_SMAX;
9630 if (SPF == SPF_UMIN) return SPF_UMAX;
9631 if (SPF == SPF_SMAX) return SPF_SMIN;
9632 if (SPF == SPF_UMAX) return SPF_UMIN;
9633 llvm_unreachable("unhandled!");
9634}
9635
9636Intrinsic::ID llvm::getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID) {
9637 switch (MinMaxID) {
9638 case Intrinsic::smax: return Intrinsic::smin;
9639 case Intrinsic::smin: return Intrinsic::smax;
9640 case Intrinsic::umax: return Intrinsic::umin;
9641 case Intrinsic::umin: return Intrinsic::umax;
9642 // Please note that next four intrinsics may produce the same result for
9643 // original and inverted case even if X != Y due to NaN is handled specially.
9644 case Intrinsic::maximum: return Intrinsic::minimum;
9645 case Intrinsic::minimum: return Intrinsic::maximum;
9646 case Intrinsic::maxnum: return Intrinsic::minnum;
9647 case Intrinsic::minnum: return Intrinsic::maxnum;
9648 case Intrinsic::maximumnum:
9649 return Intrinsic::minimumnum;
9650 case Intrinsic::minimumnum:
9651 return Intrinsic::maximumnum;
9652 default: llvm_unreachable("Unexpected intrinsic");
9653 }
9654}
9655
9656APInt llvm::getMinMaxLimit(SelectPatternFlavor SPF, unsigned BitWidth) {
9657 switch (SPF) {
9658 case SPF_SMAX: return APInt::getSignedMaxValue(numBits: BitWidth);
9659 case SPF_SMIN: return APInt::getSignedMinValue(numBits: BitWidth);
9660 case SPF_UMAX: return APInt::getMaxValue(numBits: BitWidth);
9661 case SPF_UMIN: return APInt::getMinValue(numBits: BitWidth);
9662 default: llvm_unreachable("Unexpected flavor");
9663 }
9664}
9665
9666std::pair<Intrinsic::ID, bool>
9667llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
9668 // Check if VL contains select instructions that can be folded into a min/max
9669 // vector intrinsic and return the intrinsic if it is possible.
9670 // TODO: Support floating point min/max.
9671 bool AllCmpSingleUse = true;
9672 SelectPatternResult SelectPattern;
9673 SelectPattern.Flavor = SPF_UNKNOWN;
9674 if (all_of(Range&: VL, P: [&SelectPattern, &AllCmpSingleUse](Value *I) {
9675 Value *LHS, *RHS;
9676 auto CurrentPattern = matchSelectPattern(V: I, LHS, RHS);
9677 if (!SelectPatternResult::isMinOrMax(SPF: CurrentPattern.Flavor))
9678 return false;
9679 if (SelectPattern.Flavor != SPF_UNKNOWN &&
9680 SelectPattern.Flavor != CurrentPattern.Flavor)
9681 return false;
9682 SelectPattern = CurrentPattern;
9683 AllCmpSingleUse &=
9684 match(V: I, P: m_Select(C: m_OneUse(SubPattern: m_Value()), L: m_Value(), R: m_Value()));
9685 return true;
9686 })) {
9687 switch (SelectPattern.Flavor) {
9688 case SPF_SMIN:
9689 return {Intrinsic::smin, AllCmpSingleUse};
9690 case SPF_UMIN:
9691 return {Intrinsic::umin, AllCmpSingleUse};
9692 case SPF_SMAX:
9693 return {Intrinsic::smax, AllCmpSingleUse};
9694 case SPF_UMAX:
9695 return {Intrinsic::umax, AllCmpSingleUse};
9696 case SPF_FMAXNUM:
9697 return {Intrinsic::maxnum, AllCmpSingleUse};
9698 case SPF_FMINNUM:
9699 return {Intrinsic::minnum, AllCmpSingleUse};
9700 default:
9701 llvm_unreachable("unexpected select pattern flavor");
9702 }
9703 }
9704 return {Intrinsic::not_intrinsic, false};
9705}
9706
9707template <typename InstTy>
9708static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
9709 Value *&Init, Value *&OtherOp) {
9710 // Handle the case of a simple two-predecessor recurrence PHI.
9711 // There's a lot more that could theoretically be done here, but
9712 // this is sufficient to catch some interesting cases.
9713 // TODO: Expand list -- gep, uadd.sat etc.
9714 if (PN->getNumIncomingValues() != 2)
9715 return false;
9716
9717 for (unsigned I = 0; I != 2; ++I) {
9718 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9719 Operation && Operation->getNumOperands() >= 2) {
9720 Value *LHS = Operation->getOperand(0);
9721 Value *RHS = Operation->getOperand(1);
9722 if (LHS != PN && RHS != PN)
9723 continue;
9724
9725 Inst = Operation;
9726 Init = PN->getIncomingValue(i: !I);
9727 OtherOp = (LHS == PN) ? RHS : LHS;
9728 return true;
9729 }
9730 }
9731 return false;
9732}
9733
9734template <typename InstTy>
9735static bool matchThreeInputRecurrence(const PHINode *PN, InstTy *&Inst,
9736 Value *&Init, Value *&OtherOp0,
9737 Value *&OtherOp1) {
9738 if (PN->getNumIncomingValues() != 2)
9739 return false;
9740
9741 for (unsigned I = 0; I != 2; ++I) {
9742 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9743 Operation && Operation->getNumOperands() >= 3) {
9744 Value *Op0 = Operation->getOperand(0);
9745 Value *Op1 = Operation->getOperand(1);
9746 Value *Op2 = Operation->getOperand(2);
9747
9748 if (Op0 != PN && Op1 != PN && Op2 != PN)
9749 continue;
9750
9751 Inst = Operation;
9752 Init = PN->getIncomingValue(i: !I);
9753 if (Op0 == PN) {
9754 OtherOp0 = Op1;
9755 OtherOp1 = Op2;
9756 } else if (Op1 == PN) {
9757 OtherOp0 = Op0;
9758 OtherOp1 = Op2;
9759 } else {
9760 OtherOp0 = Op0;
9761 OtherOp1 = Op1;
9762 }
9763 return true;
9764 }
9765 }
9766 return false;
9767}
9768bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
9769 Value *&Start, Value *&Step) {
9770 // We try to match a recurrence of the form:
9771 // %iv = [Start, %entry], [%iv.next, %backedge]
9772 // %iv.next = binop %iv, Step
9773 // Or:
9774 // %iv = [Start, %entry], [%iv.next, %backedge]
9775 // %iv.next = binop Step, %iv
9776 return matchTwoInputRecurrence(PN: P, Inst&: BO, Init&: Start, OtherOp&: Step);
9777}
9778
9779bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
9780 Value *&Start, Value *&Step) {
9781 BinaryOperator *BO = nullptr;
9782 return match(V: I, P: m_c_BinOp(L: m_Phi(PN&: P), R: m_Value())) &&
9783 matchSimpleRecurrence(P, BO, Start, Step) && BO == I;
9784}
9785
9786bool llvm::matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I,
9787 PHINode *&P, Value *&Init,
9788 Value *&OtherOp) {
9789 // Binary intrinsics only supported for now.
9790 if (I->arg_size() != 2 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9791 I->getType() != I->getArgOperand(i: 1)->getType())
9792 return false;
9793
9794 IntrinsicInst *II = nullptr;
9795 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9796 if (!P)
9797 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9798
9799 return P && matchTwoInputRecurrence(PN: P, Inst&: II, Init, OtherOp) && II == I;
9800}
9801
9802bool llvm::matchSimpleTernaryIntrinsicRecurrence(const IntrinsicInst *I,
9803 PHINode *&P, Value *&Init,
9804 Value *&OtherOp0,
9805 Value *&OtherOp1) {
9806 if (I->arg_size() != 3 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9807 I->getType() != I->getArgOperand(i: 1)->getType() ||
9808 I->getType() != I->getArgOperand(i: 2)->getType())
9809 return false;
9810 IntrinsicInst *II = nullptr;
9811 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9812 if (!P) {
9813 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9814 if (!P)
9815 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 2));
9816 }
9817 return P && matchThreeInputRecurrence(PN: P, Inst&: II, Init, OtherOp0, OtherOp1) &&
9818 II == I;
9819}
9820
9821/// Return true if "icmp Pred LHS RHS" is always true.
9822static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
9823 const Value *RHS) {
9824 if (ICmpInst::isTrueWhenEqual(predicate: Pred) && LHS == RHS)
9825 return true;
9826
9827 switch (Pred) {
9828 default:
9829 return false;
9830
9831 case CmpInst::ICMP_SLE: {
9832 const APInt *C;
9833
9834 // LHS s<= LHS +_{nsw} C if C >= 0
9835 // LHS s<= LHS | C if C >= 0
9836 if (match(V: RHS, P: m_NSWAdd(L: m_Specific(V: LHS), R: m_APInt(Res&: C))) ||
9837 match(V: RHS, P: m_Or(L: m_Specific(V: LHS), R: m_APInt(Res&: C))))
9838 return !C->isNegative();
9839
9840 // LHS s<= smax(LHS, V) for any V
9841 if (match(V: RHS, P: m_c_SMax(L: m_Specific(V: LHS), R: m_Value())))
9842 return true;
9843
9844 // smin(RHS, V) s<= RHS for any V
9845 if (match(V: LHS, P: m_c_SMin(L: m_Specific(V: RHS), R: m_Value())))
9846 return true;
9847
9848 // Match A to (X +_{nsw} CA) and B to (X +_{nsw} CB)
9849 const Value *X;
9850 const APInt *CLHS, *CRHS;
9851 if (match(V: LHS, P: m_NSWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9852 match(V: RHS, P: m_NSWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9853 return CLHS->sle(RHS: *CRHS);
9854
9855 return false;
9856 }
9857
9858 case CmpInst::ICMP_ULE: {
9859 // LHS u<= LHS +_{nuw} V for any V
9860 if (match(V: RHS, P: m_c_Add(L: m_Specific(V: LHS), R: m_Value())) &&
9861 cast<OverflowingBinaryOperator>(Val: RHS)->hasNoUnsignedWrap())
9862 return true;
9863
9864 // LHS u<= LHS | V for any V
9865 if (match(V: RHS, P: m_c_Or(L: m_Specific(V: LHS), R: m_Value())))
9866 return true;
9867
9868 // LHS u<= umax(LHS, V) for any V
9869 if (match(V: RHS, P: m_c_UMax(L: m_Specific(V: LHS), R: m_Value())))
9870 return true;
9871
9872 // RHS >> V u<= RHS for any V
9873 if (match(V: LHS, P: m_LShr(L: m_Specific(V: RHS), R: m_Value())))
9874 return true;
9875
9876 // RHS u/ C_ugt_1 u<= RHS
9877 const APInt *C;
9878 if (match(V: LHS, P: m_UDiv(L: m_Specific(V: RHS), R: m_APInt(Res&: C))) && C->ugt(RHS: 1))
9879 return true;
9880
9881 // RHS & V u<= RHS for any V
9882 if (match(V: LHS, P: m_c_And(L: m_Specific(V: RHS), R: m_Value())))
9883 return true;
9884
9885 // umin(RHS, V) u<= RHS for any V
9886 if (match(V: LHS, P: m_c_UMin(L: m_Specific(V: RHS), R: m_Value())))
9887 return true;
9888
9889 // Match A to (X +_{nuw} CA) and B to (X +_{nuw} CB)
9890 const Value *X;
9891 const APInt *CLHS, *CRHS;
9892 if (match(V: LHS, P: m_NUWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9893 match(V: RHS, P: m_NUWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9894 return CLHS->ule(RHS: *CRHS);
9895
9896 return false;
9897 }
9898 }
9899}
9900
9901/// Return true if "icmp Pred BLHS BRHS" is true whenever "icmp Pred
9902/// ALHS ARHS" is true. Otherwise, return std::nullopt.
9903static std::optional<bool>
9904isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS,
9905 const Value *ARHS, const Value *BLHS, const Value *BRHS) {
9906 switch (Pred) {
9907 default:
9908 return std::nullopt;
9909
9910 case CmpInst::ICMP_SLT:
9911 case CmpInst::ICMP_SLE:
9912 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BLHS, RHS: ALHS) &&
9913 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ARHS, RHS: BRHS))
9914 return true;
9915 return std::nullopt;
9916
9917 case CmpInst::ICMP_SGT:
9918 case CmpInst::ICMP_SGE:
9919 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ALHS, RHS: BLHS) &&
9920 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BRHS, RHS: ARHS))
9921 return true;
9922 return std::nullopt;
9923
9924 case CmpInst::ICMP_ULT:
9925 case CmpInst::ICMP_ULE:
9926 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BLHS, RHS: ALHS) &&
9927 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ARHS, RHS: BRHS))
9928 return true;
9929 return std::nullopt;
9930
9931 case CmpInst::ICMP_UGT:
9932 case CmpInst::ICMP_UGE:
9933 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ALHS, RHS: BLHS) &&
9934 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BRHS, RHS: ARHS))
9935 return true;
9936 return std::nullopt;
9937 }
9938}
9939
9940/// Return true if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is true.
9941/// Return false if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is false.
9942/// Otherwise, return std::nullopt if we can't infer anything.
9943static std::optional<bool>
9944isImpliedCondCommonOperandWithCR(CmpPredicate LPred, const ConstantRange &LCR,
9945 CmpPredicate RPred, const ConstantRange &RCR) {
9946 auto CRImpliesPred = [&](ConstantRange CR,
9947 CmpInst::Predicate Pred) -> std::optional<bool> {
9948 // If all true values for lhs and true for rhs, lhs implies rhs
9949 if (CR.icmp(Pred, Other: RCR))
9950 return true;
9951
9952 // If there is no overlap, lhs implies not rhs
9953 if (CR.icmp(Pred: CmpInst::getInversePredicate(pred: Pred), Other: RCR))
9954 return false;
9955
9956 return std::nullopt;
9957 };
9958 if (auto Res = CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
9959 RPred))
9960 return Res;
9961 if (LPred.hasSameSign() ^ RPred.hasSameSign()) {
9962 LPred = LPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: LPred)
9963 : LPred.dropSameSign();
9964 RPred = RPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: RPred)
9965 : RPred.dropSameSign();
9966 return CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
9967 RPred);
9968 }
9969 return std::nullopt;
9970}
9971
9972/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
9973/// is true. Return false if LHS implies RHS is false. Otherwise, return
9974/// std::nullopt if we can't infer anything.
9975static std::optional<bool>
9976isImpliedCondICmps(CmpPredicate LPred, const Value *L0, const Value *L1,
9977 CmpPredicate RPred, const Value *R0, const Value *R1,
9978 const DataLayout &DL, bool LHSIsTrue) {
9979 // The rest of the logic assumes the LHS condition is true. If that's not the
9980 // case, invert the predicate to make it so.
9981 if (!LHSIsTrue)
9982 LPred = ICmpInst::getInverseCmpPredicate(Pred: LPred);
9983
9984 // We can have non-canonical operands, so try to normalize any common operand
9985 // to L0/R0.
9986 if (L0 == R1) {
9987 std::swap(a&: R0, b&: R1);
9988 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
9989 }
9990 if (R0 == L1) {
9991 std::swap(a&: L0, b&: L1);
9992 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9993 }
9994 if (L1 == R1) {
9995 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
9996 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
9997 std::swap(a&: L0, b&: L1);
9998 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
9999 std::swap(a&: R0, b&: R1);
10000 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10001 }
10002 }
10003
10004 // See if we can infer anything if operand-0 matches and we have at least one
10005 // constant.
10006 const APInt *Unused;
10007 if (L0 == R0 && (match(V: L1, P: m_APInt(Res&: Unused)) || match(V: R1, P: m_APInt(Res&: Unused)))) {
10008 // Potential TODO: We could also further use the constant range of L0/R0 to
10009 // further constraint the constant ranges. At the moment this leads to
10010 // several regressions related to not transforming `multi_use(A + C0) eq/ne
10011 // C1` (see discussion: D58633).
10012 SimplifyQuery SQ(DL);
10013 ConstantRange LCR = computeConstantRange(V: L1, ForSigned: ICmpInst::isSigned(Pred: LPred), SQ,
10014 Depth: MaxAnalysisRecursionDepth - 1);
10015 ConstantRange RCR = computeConstantRange(V: R1, ForSigned: ICmpInst::isSigned(Pred: RPred), SQ,
10016 Depth: MaxAnalysisRecursionDepth - 1);
10017
10018 // Even if L1/R1 are not both constant, we can still sometimes deduce
10019 // relationship from a single constant. For example X u> Y implies X != 0.
10020 if (auto R = isImpliedCondCommonOperandWithCR(LPred, LCR, RPred, RCR))
10021 return R;
10022 // If both L1/R1 were exact constant ranges and we didn't get anything
10023 // here, we won't be able to deduce this.
10024 if (match(V: L1, P: m_APInt(Res&: Unused)) && match(V: R1, P: m_APInt(Res&: Unused)))
10025 return std::nullopt;
10026 }
10027
10028 // Can we infer anything when the two compares have matching operands?
10029 if (L0 == R0 && L1 == R1)
10030 return ICmpInst::isImpliedByMatchingCmp(Pred1: LPred, Pred2: RPred);
10031
10032 // It only really makes sense in the context of signed comparison for "X - Y
10033 // must be positive if X >= Y and no overflow".
10034 // Take SGT as an example: L0:x > L1:y and C >= 0
10035 // ==> R0:(x -nsw y) < R1:(-C) is false
10036 CmpInst::Predicate SignedLPred = LPred.getPreferredSignedPredicate();
10037 if ((SignedLPred == ICmpInst::ICMP_SGT ||
10038 SignedLPred == ICmpInst::ICMP_SGE) &&
10039 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
10040 if (match(V: R1, P: m_NonPositive()) &&
10041 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == false)
10042 return false;
10043 }
10044
10045 // Take SLT as an example: L0:x < L1:y and C <= 0
10046 // ==> R0:(x -nsw y) < R1:(-C) is true
10047 if ((SignedLPred == ICmpInst::ICMP_SLT ||
10048 SignedLPred == ICmpInst::ICMP_SLE) &&
10049 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
10050 if (match(V: R1, P: m_NonNegative()) &&
10051 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == true)
10052 return true;
10053 }
10054
10055 // a - b == NonZero -> a != b
10056 // ptrtoint(a) - ptrtoint(b) == NonZero -> a != b
10057 const APInt *L1C;
10058 Value *A, *B;
10059 if (LPred == ICmpInst::ICMP_EQ && ICmpInst::isEquality(P: RPred) &&
10060 match(V: L1, P: m_APInt(Res&: L1C)) && !L1C->isZero() &&
10061 match(V: L0, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
10062 ((A == R0 && B == R1) || (A == R1 && B == R0) ||
10063 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0))) &&
10064 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1)))) ||
10065 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1))) &&
10066 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0)))))) {
10067 return RPred.dropSameSign() == ICmpInst::ICMP_NE;
10068 }
10069
10070 // L0 = R0 = L1 + R1, L0 >=u L1 implies R0 >=u R1, L0 <u L1 implies R0 <u R1
10071 if (L0 == R0 &&
10072 (LPred == ICmpInst::ICMP_ULT || LPred == ICmpInst::ICMP_UGE) &&
10073 (RPred == ICmpInst::ICMP_ULT || RPred == ICmpInst::ICMP_UGE) &&
10074 match(V: L0, P: m_c_Add(L: m_Specific(V: L1), R: m_Specific(V: R1))))
10075 return CmpPredicate::getMatching(A: LPred, B: RPred).has_value();
10076
10077 if (auto P = CmpPredicate::getMatching(A: LPred, B: RPred))
10078 return isImpliedCondOperands(Pred: *P, ALHS: L0, ARHS: L1, BLHS: R0, BRHS: R1);
10079
10080 // L0 u< C sets limits to L0's bits which may imply (L0 & Mask) pred RC
10081 // Example: L0 u< 13 => (L0 & 16) == 0
10082 const APInt *LC, *RC, *MaskC;
10083 if (match(V: L1, P: m_APInt(Res&: LC)) && match(V: R1, P: m_APInt(Res&: RC)) &&
10084 match(V: R0, P: m_And(L: m_Specific(V: L0), R: m_APInt(Res&: MaskC)))) {
10085 ConstantRange LCRange = ConstantRange::makeExactICmpRegion(Pred: LPred, Other: *LC);
10086 ConstantRange MaskedCRange = LCRange.binaryAnd(Other: *MaskC);
10087 if (MaskedCRange.icmp(Pred: RPred, Other: ConstantRange(*RC)))
10088 return true;
10089 if (MaskedCRange.icmp(Pred: ICmpInst::getInversePredicate(pred: RPred),
10090 Other: ConstantRange(*RC)))
10091 return false;
10092 }
10093
10094 return std::nullopt;
10095}
10096
10097/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
10098/// is true. Return false if LHS implies RHS is false. Otherwise, return
10099/// std::nullopt if we can't infer anything.
10100static std::optional<bool>
10101isImpliedCondFCmps(FCmpInst::Predicate LPred, const Value *L0, const Value *L1,
10102 FCmpInst::Predicate RPred, const Value *R0, const Value *R1,
10103 const DataLayout &DL, bool LHSIsTrue) {
10104 // The rest of the logic assumes the LHS condition is true. If that's not the
10105 // case, invert the predicate to make it so.
10106 if (!LHSIsTrue)
10107 LPred = FCmpInst::getInversePredicate(pred: LPred);
10108
10109 // We can have non-canonical operands, so try to normalize any common operand
10110 // to L0/R0.
10111 if (L0 == R1) {
10112 std::swap(a&: R0, b&: R1);
10113 RPred = FCmpInst::getSwappedPredicate(pred: RPred);
10114 }
10115 if (R0 == L1) {
10116 std::swap(a&: L0, b&: L1);
10117 LPred = FCmpInst::getSwappedPredicate(pred: LPred);
10118 }
10119 if (L1 == R1) {
10120 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
10121 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
10122 std::swap(a&: L0, b&: L1);
10123 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
10124 std::swap(a&: R0, b&: R1);
10125 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10126 }
10127 }
10128
10129 // Can we infer anything when the two compares have matching operands?
10130 if (L0 == R0 && L1 == R1) {
10131 if ((LPred & RPred) == LPred)
10132 return true;
10133 if ((LPred & ~RPred) == LPred)
10134 return false;
10135 }
10136
10137 // See if we can infer anything if operand-0 matches and we have at least one
10138 // constant.
10139 const APFloat *L1C, *R1C;
10140 if (L0 == R0 && match(V: L1, P: m_APFloat(Res&: L1C)) && match(V: R1, P: m_APFloat(Res&: R1C))) {
10141 if (std::optional<ConstantFPRange> DomCR =
10142 ConstantFPRange::makeExactFCmpRegion(Pred: LPred, Other: *L1C)) {
10143 if (std::optional<ConstantFPRange> ImpliedCR =
10144 ConstantFPRange::makeExactFCmpRegion(Pred: RPred, Other: *R1C)) {
10145 if (ImpliedCR->contains(CR: *DomCR))
10146 return true;
10147 }
10148 if (std::optional<ConstantFPRange> ImpliedCR =
10149 ConstantFPRange::makeExactFCmpRegion(
10150 Pred: FCmpInst::getInversePredicate(pred: RPred), Other: *R1C)) {
10151 if (ImpliedCR->contains(CR: *DomCR))
10152 return false;
10153 }
10154 }
10155 }
10156
10157 return std::nullopt;
10158}
10159
10160/// Return true if LHS implies RHS is true. Return false if LHS implies RHS is
10161/// false. Otherwise, return std::nullopt if we can't infer anything. We
10162/// expect the RHS to be an icmp and the LHS to be an 'and', 'or', or a 'select'
10163/// instruction.
10164static std::optional<bool>
10165isImpliedCondAndOr(const Instruction *LHS, CmpPredicate RHSPred,
10166 const Value *RHSOp0, const Value *RHSOp1,
10167 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
10168 // The LHS must be an 'or', 'and', or a 'select' instruction.
10169 assert((LHS->getOpcode() == Instruction::And ||
10170 LHS->getOpcode() == Instruction::Or ||
10171 LHS->getOpcode() == Instruction::Select) &&
10172 "Expected LHS to be 'and', 'or', or 'select'.");
10173
10174 assert(Depth <= MaxAnalysisRecursionDepth && "Hit recursion limit");
10175
10176 // If the result of an 'or' is false, then we know both legs of the 'or' are
10177 // false. Similarly, if the result of an 'and' is true, then we know both
10178 // legs of the 'and' are true.
10179 const Value *ALHS, *ARHS;
10180 if ((!LHSIsTrue && match(V: LHS, P: m_LogicalOr(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS)))) ||
10181 (LHSIsTrue && match(V: LHS, P: m_LogicalAnd(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS))))) {
10182 // FIXME: Make this non-recursion.
10183 if (std::optional<bool> Implication = isImpliedCondition(
10184 LHS: ALHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
10185 return Implication;
10186 if (std::optional<bool> Implication = isImpliedCondition(
10187 LHS: ARHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
10188 return Implication;
10189 return std::nullopt;
10190 }
10191 return std::nullopt;
10192}
10193
10194std::optional<bool>
10195llvm::isImpliedCondition(const Value *LHS, CmpPredicate RHSPred,
10196 const Value *RHSOp0, const Value *RHSOp1,
10197 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
10198 // Bail out when we hit the limit.
10199 if (Depth == MaxAnalysisRecursionDepth)
10200 return std::nullopt;
10201
10202 // A mismatch occurs when we compare a scalar cmp to a vector cmp, for
10203 // example.
10204 if (RHSOp0->getType()->isVectorTy() != LHS->getType()->isVectorTy())
10205 return std::nullopt;
10206
10207 assert(LHS->getType()->isIntOrIntVectorTy(1) &&
10208 "Expected integer type only!");
10209
10210 // Match not
10211 if (match(V: LHS, P: m_Not(V: m_Value(V&: LHS))))
10212 LHSIsTrue = !LHSIsTrue;
10213
10214 // Both LHS and RHS are icmps.
10215 if (RHSOp0->getType()->getScalarType()->isIntOrPtrTy()) {
10216 CmpPredicate LHSPred;
10217 Value *LHSOp0, *LHSOp1;
10218 if (match(V: LHS, P: m_ICmpLike(Pred&: LHSPred, L: m_Value(V&: LHSOp0), R: m_Value(V&: LHSOp1))))
10219 return isImpliedCondICmps(LPred: LHSPred, L0: LHSOp0, L1: LHSOp1, RPred: RHSPred, R0: RHSOp0,
10220 R1: RHSOp1, DL, LHSIsTrue);
10221 } else {
10222 assert(RHSOp0->getType()->isFPOrFPVectorTy() &&
10223 "Expected floating point type only!");
10224 if (const auto *LHSCmp = dyn_cast<FCmpInst>(Val: LHS))
10225 return isImpliedCondFCmps(LPred: LHSCmp->getPredicate(), L0: LHSCmp->getOperand(i_nocapture: 0),
10226 L1: LHSCmp->getOperand(i_nocapture: 1), RPred: RHSPred, R0: RHSOp0, R1: RHSOp1,
10227 DL, LHSIsTrue);
10228 }
10229
10230 /// The LHS should be an 'or', 'and', or a 'select' instruction. We expect
10231 /// the RHS to be an icmp.
10232 /// FIXME: Add support for and/or/select on the RHS.
10233 if (const Instruction *LHSI = dyn_cast<Instruction>(Val: LHS)) {
10234 if ((LHSI->getOpcode() == Instruction::And ||
10235 LHSI->getOpcode() == Instruction::Or ||
10236 LHSI->getOpcode() == Instruction::Select))
10237 return isImpliedCondAndOr(LHS: LHSI, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue,
10238 Depth);
10239 }
10240 return std::nullopt;
10241}
10242
10243std::optional<bool> llvm::isImpliedCondition(const Value *LHS, const Value *RHS,
10244 const DataLayout &DL,
10245 bool LHSIsTrue, unsigned Depth) {
10246 // LHS ==> RHS by definition
10247 if (LHS == RHS)
10248 return LHSIsTrue;
10249
10250 // Match not
10251 bool InvertRHS = false;
10252 if (match(V: RHS, P: m_Not(V: m_Value(V&: RHS)))) {
10253 if (LHS == RHS)
10254 return !LHSIsTrue;
10255 InvertRHS = true;
10256 }
10257
10258 CmpPredicate RHSPred;
10259 Value *RHSOp0, *RHSOp1;
10260 if (match(V: RHS, P: m_ICmpLike(Pred&: RHSPred, L: m_Value(V&: RHSOp0), R: m_Value(V&: RHSOp1)))) {
10261 if (auto Implied = isImpliedCondition(LHS, RHSPred, RHSOp0, RHSOp1, DL,
10262 LHSIsTrue, Depth))
10263 return InvertRHS ? !*Implied : *Implied;
10264 return std::nullopt;
10265 }
10266 if (const FCmpInst *RHSCmp = dyn_cast<FCmpInst>(Val: RHS)) {
10267 if (auto Implied = isImpliedCondition(
10268 LHS, RHSPred: RHSCmp->getPredicate(), RHSOp0: RHSCmp->getOperand(i_nocapture: 0),
10269 RHSOp1: RHSCmp->getOperand(i_nocapture: 1), DL, LHSIsTrue, Depth))
10270 return InvertRHS ? !*Implied : *Implied;
10271 return std::nullopt;
10272 }
10273
10274 if (Depth == MaxAnalysisRecursionDepth)
10275 return std::nullopt;
10276
10277 // LHS ==> (RHS1 || RHS2) if LHS ==> RHS1 or LHS ==> RHS2
10278 // LHS ==> !(RHS1 && RHS2) if LHS ==> !RHS1 or LHS ==> !RHS2
10279 const Value *RHS1, *RHS2;
10280 if (match(V: RHS, P: m_LogicalOr(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
10281 if (std::optional<bool> Imp =
10282 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
10283 if (*Imp == true)
10284 return !InvertRHS;
10285 if (std::optional<bool> Imp =
10286 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
10287 if (*Imp == true)
10288 return !InvertRHS;
10289 }
10290 if (match(V: RHS, P: m_LogicalAnd(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
10291 if (std::optional<bool> Imp =
10292 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
10293 if (*Imp == false)
10294 return InvertRHS;
10295 if (std::optional<bool> Imp =
10296 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
10297 if (*Imp == false)
10298 return InvertRHS;
10299 }
10300
10301 return std::nullopt;
10302}
10303
10304// Returns a pair (Condition, ConditionIsTrue), where Condition is a branch
10305// condition dominating ContextI or nullptr, if no condition is found.
10306static std::pair<Value *, bool>
10307getDomPredecessorCondition(const Instruction *ContextI) {
10308 if (!ContextI || !ContextI->getParent())
10309 return {nullptr, false};
10310
10311 // TODO: This is a poor/cheap way to determine dominance. Should we use a
10312 // dominator tree (eg, from a SimplifyQuery) instead?
10313 const BasicBlock *ContextBB = ContextI->getParent();
10314 const BasicBlock *PredBB = ContextBB->getSinglePredecessor();
10315 if (!PredBB)
10316 return {nullptr, false};
10317
10318 // We need a conditional branch in the predecessor.
10319 Value *PredCond;
10320 BasicBlock *TrueBB, *FalseBB;
10321 if (!match(V: PredBB->getTerminator(), P: m_Br(C: m_Value(V&: PredCond), T&: TrueBB, F&: FalseBB)))
10322 return {nullptr, false};
10323
10324 // The branch should get simplified. Don't bother simplifying this condition.
10325 if (TrueBB == FalseBB)
10326 return {nullptr, false};
10327
10328 assert((TrueBB == ContextBB || FalseBB == ContextBB) &&
10329 "Predecessor block does not point to successor?");
10330
10331 // Is this condition implied by the predecessor condition?
10332 return {PredCond, TrueBB == ContextBB};
10333}
10334
10335std::optional<bool> llvm::isImpliedByDomCondition(const Value *Cond,
10336 const Instruction *ContextI,
10337 const DataLayout &DL) {
10338 assert(Cond->getType()->isIntOrIntVectorTy(1) && "Condition must be bool");
10339 auto PredCond = getDomPredecessorCondition(ContextI);
10340 if (PredCond.first)
10341 return isImpliedCondition(LHS: PredCond.first, RHS: Cond, DL, LHSIsTrue: PredCond.second);
10342 return std::nullopt;
10343}
10344
10345std::optional<bool> llvm::isImpliedByDomCondition(CmpPredicate Pred,
10346 const Value *LHS,
10347 const Value *RHS,
10348 const Instruction *ContextI,
10349 const DataLayout &DL) {
10350 auto PredCond = getDomPredecessorCondition(ContextI);
10351 if (PredCond.first)
10352 return isImpliedCondition(LHS: PredCond.first, RHSPred: Pred, RHSOp0: LHS, RHSOp1: RHS, DL,
10353 LHSIsTrue: PredCond.second);
10354 return std::nullopt;
10355}
10356
10357static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
10358 APInt &Upper, const InstrInfoQuery &IIQ,
10359 bool PreferSignedRange) {
10360 unsigned Width = Lower.getBitWidth();
10361 const APInt *C;
10362 switch (BO.getOpcode()) {
10363 case Instruction::Sub:
10364 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10365 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10366 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10367
10368 // If the caller expects a signed compare, then try to use a signed range.
10369 // Otherwise if both no-wraps are set, use the unsigned range because it
10370 // is never larger than the signed range. Example:
10371 // "sub nuw nsw i8 -2, x" is unsigned [0, 254] vs. signed [-128, 126].
10372 // "sub nuw nsw i8 2, x" is unsigned [0, 2] vs. signed [-125, 127].
10373 if (PreferSignedRange && HasNSW && HasNUW)
10374 HasNUW = false;
10375
10376 if (HasNUW) {
10377 // 'sub nuw c, x' produces [0, C].
10378 Upper = *C + 1;
10379 } else if (HasNSW) {
10380 if (C->isNegative()) {
10381 // 'sub nsw -C, x' produces [SINT_MIN, -C - SINT_MIN].
10382 Lower = APInt::getSignedMinValue(numBits: Width);
10383 Upper = *C - APInt::getSignedMaxValue(numBits: Width);
10384 } else {
10385 // Note that sub 0, INT_MIN is not NSW. It techically is a signed wrap
10386 // 'sub nsw C, x' produces [C - SINT_MAX, SINT_MAX].
10387 Lower = *C - APInt::getSignedMaxValue(numBits: Width);
10388 Upper = APInt::getSignedMinValue(numBits: Width);
10389 }
10390 }
10391 }
10392 break;
10393 case Instruction::Add:
10394 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10395 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10396 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10397
10398 // If the caller expects a signed compare, then try to use a signed
10399 // range. Otherwise if both no-wraps are set, use the unsigned range
10400 // because it is never larger than the signed range. Example: "add nuw
10401 // nsw i8 X, -2" is unsigned [254,255] vs. signed [-128, 125].
10402 if (PreferSignedRange && HasNSW && HasNUW)
10403 HasNUW = false;
10404
10405 if (HasNUW) {
10406 // 'add nuw x, C' produces [C, UINT_MAX].
10407 Lower = *C;
10408 } else if (HasNSW) {
10409 if (C->isNegative()) {
10410 // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C].
10411 Lower = APInt::getSignedMinValue(numBits: Width);
10412 Upper = APInt::getSignedMaxValue(numBits: Width) + *C + 1;
10413 } else {
10414 // 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX].
10415 Lower = APInt::getSignedMinValue(numBits: Width) + *C;
10416 Upper = APInt::getSignedMaxValue(numBits: Width) + 1;
10417 }
10418 }
10419 }
10420 break;
10421
10422 case Instruction::And:
10423 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10424 // 'and x, C' produces [0, C].
10425 Upper = *C + 1;
10426 // X & -X is a power of two or zero. So we can cap the value at max power of
10427 // two.
10428 if (match(V: BO.getOperand(i_nocapture: 0), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 1)))) ||
10429 match(V: BO.getOperand(i_nocapture: 1), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 0)))))
10430 Upper = APInt::getSignedMinValue(numBits: Width) + 1;
10431 break;
10432
10433 case Instruction::Or:
10434 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10435 // 'or x, C' produces [C, UINT_MAX].
10436 Lower = *C;
10437 break;
10438
10439 case Instruction::AShr:
10440 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10441 // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C].
10442 Lower = APInt::getSignedMinValue(numBits: Width).ashr(ShiftAmt: *C);
10443 Upper = APInt::getSignedMaxValue(numBits: Width).ashr(ShiftAmt: *C) + 1;
10444 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10445 unsigned ShiftAmount = Width - 1;
10446 if (!C->isZero() && IIQ.isExact(Op: &BO))
10447 ShiftAmount = C->countr_zero();
10448 if (C->isNegative()) {
10449 // 'ashr C, x' produces [C, C >> (Width-1)]
10450 Lower = *C;
10451 Upper = C->ashr(ShiftAmt: ShiftAmount) + 1;
10452 } else {
10453 // 'ashr C, x' produces [C >> (Width-1), C]
10454 Lower = C->ashr(ShiftAmt: ShiftAmount);
10455 Upper = *C + 1;
10456 }
10457 }
10458 break;
10459
10460 case Instruction::LShr:
10461 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10462 // 'lshr x, C' produces [0, UINT_MAX >> C].
10463 Upper = APInt::getAllOnes(numBits: Width).lshr(ShiftAmt: *C) + 1;
10464 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10465 // 'lshr C, x' produces [C >> (Width-1), C].
10466 unsigned ShiftAmount = Width - 1;
10467 if (!C->isZero() && IIQ.isExact(Op: &BO))
10468 ShiftAmount = C->countr_zero();
10469 Lower = C->lshr(shiftAmt: ShiftAmount);
10470 Upper = *C + 1;
10471 }
10472 break;
10473
10474 case Instruction::Shl:
10475 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10476 if (IIQ.hasNoUnsignedWrap(Op: &BO)) {
10477 // 'shl nuw C, x' produces [C, C << CLZ(C)]
10478 Lower = *C;
10479 Upper = Lower.shl(shiftAmt: Lower.countl_zero()) + 1;
10480 } else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw?
10481 if (C->isNegative()) {
10482 // 'shl nsw C, x' produces [C << CLO(C)-1, C]
10483 unsigned ShiftAmount = C->countl_one() - 1;
10484 Lower = C->shl(shiftAmt: ShiftAmount);
10485 Upper = *C + 1;
10486 } else {
10487 // 'shl nsw C, x' produces [C, C << CLZ(C)-1]
10488 unsigned ShiftAmount = C->countl_zero() - 1;
10489 Lower = *C;
10490 Upper = C->shl(shiftAmt: ShiftAmount) + 1;
10491 }
10492 } else {
10493 // If lowbit is set, value can never be zero.
10494 if ((*C)[0])
10495 Lower = APInt::getOneBitSet(numBits: Width, BitNo: 0);
10496 // If we are shifting a constant the largest it can be is if the longest
10497 // sequence of consecutive ones is shifted to the highbits (breaking
10498 // ties for which sequence is higher). At the moment we take a liberal
10499 // upper bound on this by just popcounting the constant.
10500 // TODO: There may be a bitwise trick for it longest/highest
10501 // consecutative sequence of ones (naive method is O(Width) loop).
10502 Upper = APInt::getHighBitsSet(numBits: Width, hiBitsSet: C->popcount()) + 1;
10503 }
10504 } else if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10505 Upper = APInt::getBitsSetFrom(numBits: Width, loBit: C->getZExtValue()) + 1;
10506 }
10507 break;
10508
10509 case Instruction::SDiv:
10510 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10511 APInt IntMin = APInt::getSignedMinValue(numBits: Width);
10512 APInt IntMax = APInt::getSignedMaxValue(numBits: Width);
10513 if (C->isAllOnes()) {
10514 // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX]
10515 // where C != -1 and C != 0 and C != 1
10516 Lower = IntMin + 1;
10517 Upper = IntMax + 1;
10518 } else if (C->countl_zero() < Width - 1) {
10519 // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C]
10520 // where C != -1 and C != 0 and C != 1
10521 Lower = IntMin.sdiv(RHS: *C);
10522 Upper = IntMax.sdiv(RHS: *C);
10523 if (Lower.sgt(RHS: Upper))
10524 std::swap(a&: Lower, b&: Upper);
10525 Upper = Upper + 1;
10526 assert(Upper != Lower && "Upper part of range has wrapped!");
10527 }
10528 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10529 if (C->isMinSignedValue()) {
10530 // 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2].
10531 Lower = *C;
10532 Upper = Lower.lshr(shiftAmt: 1) + 1;
10533 } else {
10534 // 'sdiv C, x' produces [-|C|, |C|].
10535 Upper = C->abs() + 1;
10536 Lower = (-Upper) + 1;
10537 }
10538 }
10539 break;
10540
10541 case Instruction::UDiv:
10542 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10543 // 'udiv x, C' produces [0, UINT_MAX / C].
10544 Upper = APInt::getMaxValue(numBits: Width).udiv(RHS: *C) + 1;
10545 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10546 // 'udiv C, x' produces [0, C].
10547 Upper = *C + 1;
10548 }
10549 break;
10550
10551 case Instruction::SRem:
10552 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10553 // 'srem x, C' produces (-|C|, |C|).
10554 Upper = C->abs();
10555 Lower = (-Upper) + 1;
10556 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10557 if (C->isNegative()) {
10558 // 'srem -|C|, x' produces [-|C|, 0].
10559 Upper = 1;
10560 Lower = *C;
10561 } else {
10562 // 'srem |C|, x' produces [0, |C|].
10563 Upper = *C + 1;
10564 }
10565 }
10566 break;
10567
10568 case Instruction::URem:
10569 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10570 // 'urem x, C' produces [0, C).
10571 Upper = *C;
10572 else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10573 // 'urem C, x' produces [0, C].
10574 Upper = *C + 1;
10575 break;
10576
10577 default:
10578 break;
10579 }
10580}
10581
10582static ConstantRange getRangeForIntrinsic(const IntrinsicInst &II,
10583 bool UseInstrInfo) {
10584 unsigned Width = II.getType()->getScalarSizeInBits();
10585 const APInt *C;
10586 switch (II.getIntrinsicID()) {
10587 case Intrinsic::ctlz:
10588 case Intrinsic::cttz: {
10589 APInt Upper(Width, Width);
10590 if (!UseInstrInfo || !match(V: II.getArgOperand(i: 1), P: m_One()))
10591 Upper += 1;
10592 // Maximum of set/clear bits is the bit width.
10593 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper);
10594 }
10595 case Intrinsic::ctpop:
10596 // Maximum of set/clear bits is the bit width.
10597 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10598 Upper: APInt(Width, Width) + 1);
10599 case Intrinsic::uadd_sat:
10600 // uadd.sat(x, C) produces [C, UINT_MAX].
10601 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10602 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10603 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10604 break;
10605 case Intrinsic::sadd_sat:
10606 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10607 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10608 if (C->isNegative())
10609 // sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)].
10610 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10611 Upper: APInt::getSignedMaxValue(numBits: Width) + *C +
10612 1);
10613
10614 // sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX].
10615 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) + *C,
10616 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10617 }
10618 break;
10619 case Intrinsic::usub_sat:
10620 // usub.sat(C, x) produces [0, C].
10621 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10622 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10623
10624 // usub.sat(x, C) produces [0, UINT_MAX - C].
10625 if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10626 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10627 Upper: APInt::getMaxValue(numBits: Width) - *C + 1);
10628 break;
10629 case Intrinsic::ssub_sat:
10630 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10631 if (C->isNegative())
10632 // ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)].
10633 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10634 Upper: *C - APInt::getSignedMinValue(numBits: Width) +
10635 1);
10636
10637 // ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX].
10638 return ConstantRange::getNonEmpty(Lower: *C - APInt::getSignedMaxValue(numBits: Width),
10639 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10640 } else if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10641 if (C->isNegative())
10642 // ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]:
10643 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) - *C,
10644 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10645
10646 // ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C].
10647 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10648 Upper: APInt::getSignedMaxValue(numBits: Width) - *C +
10649 1);
10650 }
10651 break;
10652 case Intrinsic::umin:
10653 case Intrinsic::umax:
10654 case Intrinsic::smin:
10655 case Intrinsic::smax:
10656 if (!match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) &&
10657 !match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10658 break;
10659
10660 switch (II.getIntrinsicID()) {
10661 case Intrinsic::umin:
10662 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10663 case Intrinsic::umax:
10664 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10665 case Intrinsic::smin:
10666 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10667 Upper: *C + 1);
10668 case Intrinsic::smax:
10669 return ConstantRange::getNonEmpty(Lower: *C,
10670 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10671 default:
10672 llvm_unreachable("Must be min/max intrinsic");
10673 }
10674 break;
10675 case Intrinsic::abs:
10676 // If abs of SIGNED_MIN is poison, then the result is [0..SIGNED_MAX],
10677 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10678 if (match(V: II.getOperand(i_nocapture: 1), P: m_One()))
10679 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10680 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10681
10682 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10683 Upper: APInt::getSignedMinValue(numBits: Width) + 1);
10684 case Intrinsic::vscale:
10685 if (!II.getParent() || !II.getFunction())
10686 break;
10687 return getVScaleRange(F: II.getFunction(), BitWidth: Width);
10688 case Intrinsic::read_register:
10689 case Intrinsic::read_volatile_register: {
10690 const Module *M = II.getModule();
10691 if (!M || !M->getTargetTriple().isRISCV())
10692 break;
10693 if (II.getFunction() && isReadVLENB(II))
10694 return getRISCVVLENBRange(II, Width);
10695 break;
10696 }
10697 default:
10698 break;
10699 }
10700
10701 return ConstantRange::getFull(BitWidth: Width);
10702}
10703
10704static ConstantRange getRangeForSelectPattern(const SelectInst &SI,
10705 const InstrInfoQuery &IIQ) {
10706 unsigned BitWidth = SI.getType()->getScalarSizeInBits();
10707 const Value *LHS = nullptr, *RHS = nullptr;
10708 SelectPatternResult R = matchSelectPattern(V: &SI, LHS, RHS);
10709 if (R.Flavor == SPF_UNKNOWN)
10710 return ConstantRange::getFull(BitWidth);
10711
10712 if (R.Flavor == SelectPatternFlavor::SPF_ABS) {
10713 // If the negation part of the abs (in RHS) has the NSW flag,
10714 // then the result of abs(X) is [0..SIGNED_MAX],
10715 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10716 if (match(V: RHS, P: m_Neg(V: m_Specific(V: LHS))) &&
10717 IIQ.hasNoSignedWrap(Op: cast<Instruction>(Val: RHS)))
10718 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10719 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10720
10721 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10722 Upper: APInt::getSignedMinValue(numBits: BitWidth) + 1);
10723 }
10724
10725 if (R.Flavor == SelectPatternFlavor::SPF_NABS) {
10726 // The result of -abs(X) is <= 0.
10727 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10728 Upper: APInt(BitWidth, 1));
10729 }
10730
10731 const APInt *C;
10732 if (!match(V: LHS, P: m_APInt(Res&: C)) && !match(V: RHS, P: m_APInt(Res&: C)))
10733 return ConstantRange::getFull(BitWidth);
10734
10735 switch (R.Flavor) {
10736 case SPF_UMIN:
10737 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth), Upper: *C + 1);
10738 case SPF_UMAX:
10739 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: BitWidth));
10740 case SPF_SMIN:
10741 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10742 Upper: *C + 1);
10743 case SPF_SMAX:
10744 return ConstantRange::getNonEmpty(Lower: *C,
10745 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10746 default:
10747 return ConstantRange::getFull(BitWidth);
10748 }
10749}
10750
10751static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) {
10752 // The maximum representable value of a half is 65504. For floats the maximum
10753 // value is 3.4e38 which requires roughly 129 bits.
10754 unsigned BitWidth = I->getType()->getScalarSizeInBits();
10755 if (!I->getOperand(i: 0)->getType()->getScalarType()->isHalfTy())
10756 return;
10757 if (isa<FPToSIInst>(Val: I) && BitWidth >= 17) {
10758 Lower = APInt(BitWidth, -65504, true);
10759 Upper = APInt(BitWidth, 65505);
10760 }
10761
10762 if (isa<FPToUIInst>(Val: I) && BitWidth >= 16) {
10763 // For a fptoui the lower limit is left as 0.
10764 Upper = APInt(BitWidth, 65505);
10765 }
10766}
10767
10768ConstantRange llvm::computeConstantRange(const Value *V, bool ForSigned,
10769 const SimplifyQuery &SQ,
10770 unsigned Depth) {
10771 assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction");
10772
10773 if (Depth == MaxAnalysisRecursionDepth)
10774 return ConstantRange::getFull(BitWidth: V->getType()->getScalarSizeInBits());
10775
10776 if (auto *C = dyn_cast<Constant>(Val: V))
10777 return C->toConstantRange();
10778
10779 unsigned BitWidth = V->getType()->getScalarSizeInBits();
10780 ConstantRange CR = ConstantRange::getFull(BitWidth);
10781 if (auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
10782 APInt Lower = APInt(BitWidth, 0);
10783 APInt Upper = APInt(BitWidth, 0);
10784 // TODO: Return ConstantRange.
10785 setLimitsForBinOp(BO: *BO, Lower, Upper, IIQ: SQ.IIQ, PreferSignedRange: ForSigned);
10786 CR = ConstantRange::getNonEmpty(Lower, Upper);
10787 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
10788 CR = getRangeForIntrinsic(II: *II, UseInstrInfo: SQ.IIQ.UseInstrInfo);
10789 else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
10790 ConstantRange CRTrue =
10791 computeConstantRange(V: SI->getTrueValue(), ForSigned, SQ, Depth: Depth + 1);
10792 ConstantRange CRFalse =
10793 computeConstantRange(V: SI->getFalseValue(), ForSigned, SQ, Depth: Depth + 1);
10794 CR = CRTrue.unionWith(CR: CRFalse);
10795 CR = CR.intersectWith(CR: getRangeForSelectPattern(SI: *SI, IIQ: SQ.IIQ));
10796 } else if (auto *TI = dyn_cast<TruncInst>(Val: V)) {
10797 ConstantRange SrcCR =
10798 computeConstantRange(V: TI->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
10799 CR = SrcCR.truncate(BitWidth);
10800 } else if (isa<FPToUIInst>(Val: V) || isa<FPToSIInst>(Val: V)) {
10801 APInt Lower = APInt(BitWidth, 0);
10802 APInt Upper = APInt(BitWidth, 0);
10803 // TODO: Return ConstantRange.
10804 setLimitForFPToI(I: cast<Instruction>(Val: V), Lower, Upper);
10805 CR = ConstantRange::getNonEmpty(Lower, Upper);
10806 } else if (const auto *A = dyn_cast<Argument>(Val: V))
10807 if (std::optional<ConstantRange> Range = A->getRange())
10808 CR = *Range;
10809
10810 if (auto *I = dyn_cast<Instruction>(Val: V)) {
10811 if (auto *Range = SQ.IIQ.getMetadata(I, KindID: LLVMContext::MD_range))
10812 CR = CR.intersectWith(CR: getConstantRangeFromMetadata(RangeMD: *Range));
10813
10814 Value *FrexpSrc;
10815 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
10816 if (std::optional<ConstantRange> Range = CB->getRange())
10817 CR = CR.intersectWith(CR: *Range);
10818 } else if (match(V: I, P: m_ExtractValue<1>(V: m_Intrinsic<Intrinsic::frexp>(
10819 Ops: m_Value(V&: FrexpSrc))))) {
10820 const fltSemantics &FltSem =
10821 FrexpSrc->getType()->getScalarType()->getFltSemantics();
10822 // It should be possible to implement this for any type, but this logic
10823 // only computes the range assuming standard subnormal handling.
10824 if (APFloat::isIEEELikeFP(FltSem)) {
10825 KnownFPClass KnownSrc = computeKnownFPClass(
10826 V: FrexpSrc, InterestedClasses: fcSubnormal | fcZero | fcNan | fcInf, SQ, Depth: Depth + 1);
10827
10828 // The exponent of frexp(NaN) and frexp(Inf) is unspecified. Only
10829 // constrain its range when the source can be neither.
10830 if (KnownSrc.isKnownNeverInfOrNaN()) {
10831 int MinExp = APFloat::semanticsMinExponent(FltSem) + 1;
10832
10833 // Offset to find the true minimum exponent value for a denormal.
10834 if (!KnownSrc.isKnownNeverSubnormal())
10835 MinExp -= (APFloat::semanticsPrecision(FltSem) - 1);
10836
10837 int MaxExp = APFloat::semanticsMaxExponent(FltSem) + 1;
10838
10839 auto [AdjustedMin, AdjustedMax, AdjustedMaxNonZero] =
10840 computeKnownExponentRangeFromContext(V: FrexpSrc, Q: SQ);
10841
10842 DenormalMode Mode = I->getFunction()->getDenormalMode(FPType: FltSem);
10843 bool NeverLogicalZero = KnownSrc.isKnownNeverLogicalZero(Mode);
10844
10845 MinExp = std::max(a: AdjustedMin, b: MinExp);
10846 MaxExp = std::min(a: NeverLogicalZero ? AdjustedMaxNonZero : AdjustedMax,
10847 b: MaxExp);
10848
10849 CR = ConstantRange::getNonEmpty(
10850 Lower: APInt(BitWidth, static_cast<int64_t>(MinExp), /*isSigned=*/true),
10851 Upper: APInt(BitWidth, static_cast<int64_t>(MaxExp) + 1,
10852 /*isSigned=*/true));
10853 }
10854 }
10855 }
10856 }
10857
10858 if (SQ.CxtI && SQ.AC) {
10859 // Try to restrict the range based on information from assumptions.
10860 for (auto &AssumeVH : SQ.AC->assumptionsFor(V)) {
10861 if (!AssumeVH)
10862 continue;
10863 CallInst *I = cast<CallInst>(Val&: AssumeVH);
10864 assert(I->getParent()->getParent() == SQ.CxtI->getParent()->getParent() &&
10865 "Got assumption for the wrong function!");
10866 assert(I->getIntrinsicID() == Intrinsic::assume &&
10867 "must be an assume intrinsic");
10868
10869 if (!isValidAssumeForContext(I, Q: SQ))
10870 continue;
10871 Value *Arg = I->getArgOperand(i: 0);
10872 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
10873 // Currently we just use information from comparisons.
10874 if (!Cmp || Cmp->getOperand(i_nocapture: 0) != V)
10875 continue;
10876 // TODO: Set "ForSigned" parameter via Cmp->isSigned()?
10877 ConstantRange RHS =
10878 computeConstantRange(V: Cmp->getOperand(i_nocapture: 1), /*ForSigned=*/false,
10879 SQ: SQ.getWithInstruction(I), Depth: Depth + 1);
10880 CR = CR.intersectWith(
10881 CR: ConstantRange::makeAllowedICmpRegion(Pred: Cmp->getCmpPredicate(), Other: RHS));
10882 }
10883 }
10884
10885 return CR;
10886}
10887
10888static void
10889addValueAffectedByCondition(Value *V,
10890 function_ref<void(Value *)> InsertAffected) {
10891 assert(V != nullptr);
10892 if (isa<Argument>(Val: V) || isa<GlobalValue>(Val: V)) {
10893 InsertAffected(V);
10894 } else if (auto *I = dyn_cast<Instruction>(Val: V)) {
10895 InsertAffected(V);
10896
10897 // Peek through unary operators to find the source of the condition.
10898 Value *Op;
10899 if (match(V: I, P: m_CombineOr(Ps: m_PtrToIntOrAddr(Op: m_Value(V&: Op)),
10900 Ps: m_Trunc(Op: m_Value(V&: Op))))) {
10901 if (isa<Instruction>(Val: Op) || isa<Argument>(Val: Op))
10902 InsertAffected(Op);
10903 }
10904 }
10905}
10906
10907void llvm::findValuesAffectedByCondition(
10908 Value *Cond, bool IsAssume, function_ref<void(Value *)> InsertAffected) {
10909 auto AddAffected = [&InsertAffected](Value *V) {
10910 addValueAffectedByCondition(V, InsertAffected);
10911 };
10912
10913 auto AddCmpOperands = [&AddAffected, IsAssume](Value *LHS, Value *RHS) {
10914 if (IsAssume) {
10915 AddAffected(LHS);
10916 AddAffected(RHS);
10917 } else if (match(V: RHS, P: m_Constant()))
10918 AddAffected(LHS);
10919 };
10920
10921 SmallVector<Value *, 8> Worklist;
10922 SmallPtrSet<Value *, 8> Visited;
10923 Worklist.push_back(Elt: Cond);
10924 while (!Worklist.empty()) {
10925 Value *V = Worklist.pop_back_val();
10926 if (!Visited.insert(Ptr: V).second)
10927 continue;
10928
10929 CmpPredicate Pred;
10930 Value *A, *B, *X;
10931
10932 if (IsAssume) {
10933 AddAffected(V);
10934 if (match(V, P: m_Not(V: m_Value(V&: X))))
10935 AddAffected(X);
10936 }
10937
10938 if (match(V, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
10939 // assume(A && B) is split to -> assume(A); assume(B);
10940 // assume(!(A || B)) is split to -> assume(!A); assume(!B);
10941 // Finally, assume(A || B) / assume(!(A && B)) generally don't provide
10942 // enough information to be worth handling (intersection of information as
10943 // opposed to union).
10944 if (!IsAssume) {
10945 Worklist.push_back(Elt: A);
10946 Worklist.push_back(Elt: B);
10947 }
10948 } else if (match(V, P: m_ICmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
10949 bool HasRHSC = match(V: B, P: m_ConstantInt());
10950 if (ICmpInst::isEquality(P: Pred)) {
10951 AddAffected(A);
10952 if (IsAssume)
10953 AddAffected(B);
10954 if (HasRHSC) {
10955 Value *Y;
10956 // (X << C) or (X >>_s C) or (X >>_u C).
10957 if (match(V: A, P: m_Shift(L: m_Value(V&: X), R: m_ConstantInt())))
10958 AddAffected(X);
10959 // (X & C) or (X | C).
10960 else if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10961 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10962 AddAffected(X);
10963 AddAffected(Y);
10964 }
10965 // X - Y
10966 else if (match(V: A, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10967 AddAffected(X);
10968 AddAffected(Y);
10969 }
10970 }
10971 } else {
10972 AddCmpOperands(A, B);
10973 if (HasRHSC) {
10974 // Handle (A + C1) u< C2, which is the canonical form of
10975 // A > C3 && A < C4.
10976 if (match(V: A, P: m_AddLike(L: m_Value(V&: X), R: m_ConstantInt())))
10977 AddAffected(X);
10978
10979 if (ICmpInst::isUnsigned(Pred)) {
10980 Value *Y;
10981 // X & Y u> C -> X >u C && Y >u C
10982 // X | Y u< C -> X u< C && Y u< C
10983 // X nuw+ Y u< C -> X u< C && Y u< C
10984 if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10985 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10986 match(V: A, P: m_NUWAdd(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10987 AddAffected(X);
10988 AddAffected(Y);
10989 }
10990 // X nuw- Y u> C -> X u> C
10991 if (match(V: A, P: m_NUWSub(L: m_Value(V&: X), R: m_Value())))
10992 AddAffected(X);
10993 }
10994 }
10995
10996 // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported
10997 // by computeKnownFPClass().
10998 if (match(V: A, P: m_ElementWiseBitCast(Op: m_Value(V&: X)))) {
10999 if (Pred == ICmpInst::ICMP_SLT && match(V: B, P: m_Zero()))
11000 InsertAffected(X);
11001 else if (Pred == ICmpInst::ICMP_SGT && match(V: B, P: m_AllOnes()))
11002 InsertAffected(X);
11003 }
11004 }
11005
11006 auto AddNuwSquareOperand = [&AddAffected](Value *Op) {
11007 Value *SquareOp = nullptr;
11008 if (match(V: Op, P: m_NUWMul(L: m_Value(V&: SquareOp), R: m_Deferred(V: SquareOp))))
11009 AddAffected(SquareOp);
11010 };
11011 AddNuwSquareOperand(A);
11012 AddNuwSquareOperand(B);
11013
11014 if (HasRHSC && match(V: A, P: m_Ctpop(Op0: m_Value(V&: X))))
11015 AddAffected(X);
11016 } else if (match(V, P: m_FCmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
11017 AddCmpOperands(A, B);
11018
11019 // fcmp fneg(x), y
11020 // fcmp fabs(x), y
11021 // fcmp fneg(fabs(x)), y
11022 if (match(V: A, P: m_FNeg(X: m_Value(V&: A))))
11023 AddAffected(A);
11024 if (match(V: A, P: m_FAbs(Op0: m_Value(V&: A))))
11025 AddAffected(A);
11026
11027 } else if (match(V, P: m_Intrinsic<Intrinsic::is_fpclass>(Ops: m_Value(V&: A),
11028 Ops: m_Value()))) {
11029 // Handle patterns that computeKnownFPClass() support.
11030 AddAffected(A);
11031 } else if (!IsAssume && match(V, P: m_Trunc(Op: m_Value(V&: X)))) {
11032 // Assume is checked here as X is already added above for assumes in
11033 // addValueAffectedByCondition
11034 AddAffected(X);
11035 } else if (!IsAssume && match(V, P: m_Not(V: m_Value(V&: X)))) {
11036 // Assume is checked here to avoid issues with ephemeral values
11037 Worklist.push_back(Elt: X);
11038 }
11039 }
11040}
11041
11042const Value *llvm::stripNullTest(const Value *V) {
11043 // (X >> C) or/add (X & mask(C) != 0)
11044 if (const auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
11045 if (BO->getOpcode() == Instruction::Add ||
11046 BO->getOpcode() == Instruction::Or) {
11047 const Value *X;
11048 const APInt *C1, *C2;
11049 if (match(V: BO, P: m_c_BinOp(L: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: C1)),
11050 R: m_ZExt(Op: m_SpecificICmp(
11051 MatchPred: ICmpInst::ICMP_NE,
11052 L: m_And(L: m_Deferred(V: X), R: m_LowBitMask(V&: C2)),
11053 R: m_Zero())))) &&
11054 C2->popcount() == C1->getZExtValue())
11055 return X;
11056 }
11057 }
11058 return nullptr;
11059}
11060
11061Value *llvm::stripNullTest(Value *V) {
11062 return const_cast<Value *>(stripNullTest(V: const_cast<const Value *>(V)));
11063}
11064
11065bool llvm::collectPossibleValues(const Value *V,
11066 SmallPtrSetImpl<const Constant *> &Constants,
11067 unsigned MaxCount, bool AllowUndefOrPoison) {
11068 SmallPtrSet<const Instruction *, 8> Visited;
11069 SmallVector<const Instruction *, 8> Worklist;
11070 auto Push = [&](const Value *V) -> bool {
11071 Constant *C;
11072 if (match(V: const_cast<Value *>(V), P: m_ImmConstant(C))) {
11073 if (!AllowUndefOrPoison && !isGuaranteedNotToBeUndefOrPoison(V: C))
11074 return false;
11075 // Check existence first to avoid unnecessary allocations.
11076 if (Constants.contains(Ptr: C))
11077 return true;
11078 if (Constants.size() == MaxCount)
11079 return false;
11080 Constants.insert(Ptr: C);
11081 return true;
11082 }
11083
11084 if (auto *Inst = dyn_cast<Instruction>(Val: V)) {
11085 if (Visited.insert(Ptr: Inst).second)
11086 Worklist.push_back(Elt: Inst);
11087 return true;
11088 }
11089 return false;
11090 };
11091 if (!Push(V))
11092 return false;
11093 while (!Worklist.empty()) {
11094 const Instruction *CurInst = Worklist.pop_back_val();
11095 switch (CurInst->getOpcode()) {
11096 case Instruction::Select:
11097 if (!Push(CurInst->getOperand(i: 1)))
11098 return false;
11099 if (!Push(CurInst->getOperand(i: 2)))
11100 return false;
11101 break;
11102 case Instruction::PHI:
11103 for (Value *IncomingValue : cast<PHINode>(Val: CurInst)->incoming_values()) {
11104 // Fast path for recurrence PHI.
11105 if (IncomingValue == CurInst)
11106 continue;
11107 if (!Push(IncomingValue))
11108 return false;
11109 }
11110 break;
11111 default:
11112 return false;
11113 }
11114 }
11115 return true;
11116}
11117