1//===- ValueTracking.cpp - Walk computations to compute properties --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains routines that help analyze properties that chains of
10// computations have.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ValueTracking.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/FloatingPointMode.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/ADT/SmallPtrSet.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/ADT/iterator_range.h"
24#include "llvm/Analysis/AliasAnalysis.h"
25#include "llvm/Analysis/AssumeBundleQueries.h"
26#include "llvm/Analysis/AssumptionCache.h"
27#include "llvm/Analysis/ConstantFolding.h"
28#include "llvm/Analysis/DomConditionCache.h"
29#include "llvm/Analysis/FloatingPointPredicateUtils.h"
30#include "llvm/Analysis/GuardUtils.h"
31#include "llvm/Analysis/InstructionSimplify.h"
32#include "llvm/Analysis/Loads.h"
33#include "llvm/Analysis/LoopInfo.h"
34#include "llvm/Analysis/TargetLibraryInfo.h"
35#include "llvm/Analysis/VectorUtils.h"
36#include "llvm/Analysis/WithCache.h"
37#include "llvm/IR/Argument.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/BasicBlock.h"
40#include "llvm/IR/BundleAttributes.h"
41#include "llvm/IR/Constant.h"
42#include "llvm/IR/ConstantFPRange.h"
43#include "llvm/IR/ConstantRange.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DerivedTypes.h"
46#include "llvm/IR/DiagnosticInfo.h"
47#include "llvm/IR/Dominators.h"
48#include "llvm/IR/EHPersonalities.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GetElementPtrTypeIterator.h"
51#include "llvm/IR/GlobalAlias.h"
52#include "llvm/IR/GlobalValue.h"
53#include "llvm/IR/GlobalVariable.h"
54#include "llvm/IR/InstrTypes.h"
55#include "llvm/IR/Instruction.h"
56#include "llvm/IR/Instructions.h"
57#include "llvm/IR/IntrinsicInst.h"
58#include "llvm/IR/Intrinsics.h"
59#include "llvm/IR/IntrinsicsAArch64.h"
60#include "llvm/IR/IntrinsicsAMDGPU.h"
61#include "llvm/IR/IntrinsicsRISCV.h"
62#include "llvm/IR/IntrinsicsX86.h"
63#include "llvm/IR/LLVMContext.h"
64#include "llvm/IR/Metadata.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/Operator.h"
67#include "llvm/IR/PatternMatch.h"
68#include "llvm/IR/Type.h"
69#include "llvm/IR/User.h"
70#include "llvm/IR/Value.h"
71#include "llvm/Support/Casting.h"
72#include "llvm/Support/CommandLine.h"
73#include "llvm/Support/Compiler.h"
74#include "llvm/Support/ErrorHandling.h"
75#include "llvm/Support/KnownBits.h"
76#include "llvm/Support/KnownFPClass.h"
77#include "llvm/Support/MathExtras.h"
78#include "llvm/Support/UndefPoison.h"
79#include "llvm/TargetParser/RISCVTargetParser.h"
80#include <algorithm>
81#include <cassert>
82#include <cstdint>
83#include <optional>
84#include <utility>
85
86using namespace llvm;
87using namespace llvm::PatternMatch;
88
89// Controls the number of uses of the value searched for possible
90// dominating comparisons.
91static cl::opt<unsigned> DomConditionsMaxUses("dom-conditions-max-uses",
92 cl::Hidden, cl::init(Val: 20));
93
94/// Maximum number of instructions to check between assume and context
95/// instruction.
96static constexpr unsigned MaxInstrsToCheckForFree = 32;
97
98template <typename InstTy>
99static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
100 Value *&Init, Value *&OtherOp);
101
102/// Returns the bitwidth of the given scalar or pointer type. For vector types,
103/// returns the element type's bitwidth.
104static unsigned getBitWidth(Type *Ty, const DataLayout &DL) {
105 if (unsigned BitWidth = Ty->getScalarSizeInBits())
106 return BitWidth;
107
108 return DL.getPointerTypeSizeInBits(Ty);
109}
110
111// Given the provided Value and, potentially, a context instruction, return
112// the preferred context instruction (if any).
113static const Instruction *safeCtxI(const Value *V, const Instruction *CtxI) {
114 // If we've been provided with a context instruction, then use that (provided
115 // it has been inserted).
116 if (CtxI && CtxI->getParent())
117 return CtxI;
118
119 // If the value is really an already-inserted instruction, then use that.
120 CtxI = dyn_cast<Instruction>(Val: V);
121 if (CtxI && CtxI->getParent())
122 return CtxI;
123
124 return nullptr;
125}
126
127static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
128 const APInt &DemandedElts,
129 APInt &DemandedLHS, APInt &DemandedRHS) {
130 if (isa<ScalableVectorType>(Val: Shuf->getType())) {
131 assert(DemandedElts == APInt(1,1));
132 DemandedLHS = DemandedRHS = DemandedElts;
133 return true;
134 }
135
136 int NumElts =
137 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
138 return llvm::getShuffleDemandedElts(SrcWidth: NumElts, Mask: Shuf->getShuffleMask(),
139 DemandedElts, DemandedLHS, DemandedRHS);
140}
141
142static void computeKnownBits(const Value *V, const APInt &DemandedElts,
143 KnownBits &Known, const SimplifyQuery &Q,
144 unsigned Depth);
145
146void llvm::computeKnownBits(const Value *V, KnownBits &Known,
147 const SimplifyQuery &Q, unsigned Depth) {
148 // Since the number of lanes in a scalable vector is unknown at compile time,
149 // we track one bit which is implicitly broadcast to all lanes. This means
150 // that all lanes in a scalable vector are considered demanded.
151 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
152 APInt DemandedElts =
153 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
154 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
155}
156
157void llvm::computeKnownBits(const Value *V, KnownBits &Known,
158 const DataLayout &DL, AssumptionCache *AC,
159 const Instruction *CtxI, const DominatorTree *DT,
160 bool UseInstrInfo, unsigned Depth) {
161 computeKnownBits(V, Known,
162 Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo),
163 Depth);
164}
165
166KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
167 AssumptionCache *AC, const Instruction *CtxI,
168 const DominatorTree *DT, bool UseInstrInfo,
169 unsigned Depth) {
170 return computeKnownBits(
171 V, Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo), Depth);
172}
173
174static NoCommonBitsSetResult
175haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
176 const SimplifyQuery &SQ) {
177 // Look for an inverted mask: (X & ~M) op (Y & M).
178 {
179 Value *M;
180 if (match(V: LHS, P: m_c_And(L: m_Not(V: m_Value(V&: M)), R: m_Value())) &&
181 match(V: RHS, P: m_c_And(L: m_Specific(V: M), R: m_Value())))
182 return isGuaranteedNotToBeUndef(V: M, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)
183 ? NoCommonBitsSetResult::Known
184 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
185 }
186
187 // X op (Y & ~X)
188 if (match(V: RHS, P: m_c_And(L: m_Not(V: m_Specific(V: LHS)), R: m_Value())))
189 return isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)
190 ? NoCommonBitsSetResult::Known
191 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
192
193 // X op ((X & Y) ^ Y) -- this is the canonical form of the previous pattern
194 // for constant Y.
195 Value *Y;
196 if (match(V: RHS,
197 P: m_c_Xor(L: m_c_And(L: m_Specific(V: LHS), R: m_Value(V&: Y)), R: m_Deferred(V: Y)))) {
198 bool IsNoUndef = isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT) &&
199 isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT);
200 return IsNoUndef ? NoCommonBitsSetResult::Known
201 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
202 }
203
204 // Peek through extends to find a 'not' of the other side:
205 // (ext Y) op ext(~Y)
206 if (match(V: LHS, P: m_ZExtOrSExt(Op: m_Value(V&: Y))) &&
207 match(V: RHS, P: m_ZExtOrSExt(Op: m_Not(V: m_Specific(V: Y)))))
208 return isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)
209 ? NoCommonBitsSetResult::Known
210 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
211
212 // Look for: (A & B) op ~(A | B)
213 {
214 Value *A, *B;
215 if (match(V: LHS, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
216 match(V: RHS, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))) {
217 bool IsNoUndef = isGuaranteedNotToBeUndef(V: A, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT) &&
218 isGuaranteedNotToBeUndef(V: B, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT);
219 return IsNoUndef ? NoCommonBitsSetResult::Known
220 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
221 }
222 }
223
224 // Look for: (X << V) op (Y >> (BitWidth - V))
225 // or (X >> V) op (Y << (BitWidth - V))
226 {
227 const Value *V;
228 const APInt *R;
229 if (((match(V: RHS, P: m_Shl(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
230 match(V: LHS, P: m_LShr(L: m_Value(), R: m_Specific(V)))) ||
231 (match(V: RHS, P: m_LShr(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
232 match(V: LHS, P: m_Shl(L: m_Value(), R: m_Specific(V))))) &&
233 R->uge(RHS: LHS->getType()->getScalarSizeInBits()))
234 return NoCommonBitsSetResult::Known;
235 }
236
237 return NoCommonBitsSetResult::Unknown;
238}
239
240NoCommonBitsSetResult
241llvm::getNoCommonBitsSetResult(const WithCache<const Value *> &LHSCache,
242 const WithCache<const Value *> &RHSCache,
243 const SimplifyQuery &SQ) {
244 const Value *LHS = LHSCache.getValue();
245 const Value *RHS = RHSCache.getValue();
246
247 assert(LHS->getType() == RHS->getType() &&
248 "LHS and RHS should have the same type");
249 assert(LHS->getType()->isIntOrIntVectorTy() &&
250 "LHS and RHS should be integers");
251
252 NoCommonBitsSetResult Result = haveNoCommonBitsSetSpecialCases(LHS, RHS, SQ);
253 if (Result == NoCommonBitsSetResult::Known)
254 return NoCommonBitsSetResult::Known;
255
256 NoCommonBitsSetResult CommuteResult =
257 haveNoCommonBitsSetSpecialCases(LHS: RHS, RHS: LHS, SQ);
258 if (CommuteResult == NoCommonBitsSetResult::Known)
259 return NoCommonBitsSetResult::Known;
260
261 if (KnownBits::haveNoCommonBitsSet(LHS: LHSCache.getKnownBits(Q: SQ),
262 RHS: RHSCache.getKnownBits(Q: SQ)))
263 return NoCommonBitsSetResult::Known;
264
265 if (Result == NoCommonBitsSetResult::OnlyIfUndefIgnored ||
266 CommuteResult == NoCommonBitsSetResult::OnlyIfUndefIgnored)
267 return NoCommonBitsSetResult::OnlyIfUndefIgnored;
268
269 return NoCommonBitsSetResult::Unknown;
270}
271
272bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
273 const WithCache<const Value *> &RHSCache,
274 const SimplifyQuery &SQ) {
275 NoCommonBitsSetResult Result =
276 getNoCommonBitsSetResult(LHSCache, RHSCache, SQ);
277 return Result == NoCommonBitsSetResult::Known;
278}
279
280bool llvm::isOnlyUsedInZeroComparison(const Instruction *I) {
281 return !I->user_empty() &&
282 all_of(Range: I->users(), P: match_fn(P: m_ICmp(L: m_Value(), R: m_Zero())));
283}
284
285bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) {
286 return !I->user_empty() && all_of(Range: I->users(), P: [](const User *U) {
287 CmpPredicate P;
288 return match(V: U, P: m_ICmp(Pred&: P, L: m_Value(), R: m_Zero())) && ICmpInst::isEquality(P);
289 });
290}
291
292bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
293 bool OrZero, AssumptionCache *AC,
294 const Instruction *CtxI,
295 const DominatorTree *DT, bool UseInstrInfo,
296 unsigned Depth) {
297 return ::isKnownToBeAPowerOfTwo(
298 V, OrZero, Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo),
299 Depth);
300}
301
302static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
303 const SimplifyQuery &Q, unsigned Depth);
304
305bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
306 unsigned Depth) {
307 return computeKnownBits(V, Q: SQ, Depth).isNonNegative();
308}
309
310bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
311 unsigned Depth) {
312 if (auto *CI = dyn_cast<ConstantInt>(Val: V))
313 return CI->getValue().isStrictlyPositive();
314
315 // If `isKnownNonNegative` ever becomes more sophisticated, make sure to keep
316 // this updated.
317 KnownBits Known = computeKnownBits(V, Q: SQ, Depth);
318 return Known.isNonNegative() &&
319 (Known.isNonZero() || isKnownNonZero(V, Q: SQ, Depth));
320}
321
322bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ,
323 unsigned Depth) {
324 return computeKnownBits(V, Q: SQ, Depth).isNegative();
325}
326
327static bool isKnownNonEqual(const Value *V1, const Value *V2,
328 const APInt &DemandedElts, const SimplifyQuery &Q,
329 unsigned Depth);
330
331static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
332 const Value *RHS);
333
334bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
335 const SimplifyQuery &Q, unsigned Depth) {
336 // We don't support looking through casts.
337 if (V1 == V2 || V1->getType() != V2->getType())
338 return false;
339 auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType());
340 APInt DemandedElts =
341 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
342 return ::isKnownNonEqual(V1, V2, DemandedElts, Q, Depth);
343}
344
345bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
346 const SimplifyQuery &SQ, unsigned Depth) {
347 KnownBits Known(Mask.getBitWidth());
348 computeKnownBits(V, Known, Q: SQ, Depth);
349 return Mask.isSubsetOf(RHS: Known.Zero);
350}
351
352static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
353 const SimplifyQuery &Q, unsigned Depth);
354
355static unsigned ComputeNumSignBits(const Value *V, const SimplifyQuery &Q,
356 unsigned Depth = 0) {
357 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
358 APInt DemandedElts =
359 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
360 return ComputeNumSignBits(V, DemandedElts, Q, Depth);
361}
362
363unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL,
364 AssumptionCache *AC, const Instruction *CtxI,
365 const DominatorTree *DT, bool UseInstrInfo,
366 unsigned Depth) {
367 return ::ComputeNumSignBits(
368 V, Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo), Depth);
369}
370
371unsigned llvm::ComputeMaxSignificantBits(const Value *V, const DataLayout &DL,
372 AssumptionCache *AC,
373 const Instruction *CtxI,
374 const DominatorTree *DT,
375 unsigned Depth) {
376 unsigned SignBits = ComputeNumSignBits(V, DL, AC, CtxI, DT, UseInstrInfo: Depth);
377 return V->getType()->getScalarSizeInBits() - SignBits + 1;
378}
379
380/// Try to detect the lerp pattern: a * (b - c) + c * d
381/// where a >= 0, b >= 0, c >= 0, d >= 0, and b >= c.
382///
383/// In that particular case, we can use the following chain of reasoning:
384///
385/// a * (b - c) + c * d <= a' * (b - c) + a' * c = a' * b where a' = max(a, d)
386///
387/// Since that is true for arbitrary a, b, c and d within our constraints, we
388/// can conclude that:
389///
390/// max(a * (b - c) + c * d) <= max(max(a), max(d)) * max(b) = U
391///
392/// Considering that any result of the lerp would be less or equal to U, it
393/// would have at least the number of leading 0s as in U.
394///
395/// While being quite a specific situation, it is fairly common in computer
396/// graphics in the shape of alpha blending.
397///
398/// Modifies given KnownOut in-place with the inferred information.
399static void computeKnownBitsFromLerpPattern(const Value *Op0, const Value *Op1,
400 const APInt &DemandedElts,
401 KnownBits &KnownOut,
402 const SimplifyQuery &Q,
403 unsigned Depth) {
404
405 Type *Ty = Op0->getType();
406 const unsigned BitWidth = Ty->getScalarSizeInBits();
407
408 // Only handle scalar types for now
409 if (Ty->isVectorTy())
410 return;
411
412 // Try to match: a * (b - c) + c * d.
413 // When a == 1 => A == nullptr, the same applies to d/D as well.
414 const Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
415 const Instruction *SubBC = nullptr;
416
417 const auto MatchSubBC = [&]() {
418 // (b - c) can have two forms that interest us:
419 //
420 // 1. sub nuw %b, %c
421 // 2. xor %c, %b
422 //
423 // For the first case, nuw flag guarantees our requirement b >= c.
424 //
425 // The second case might happen when the analysis can infer that b is a mask
426 // for c and we can transform sub operation into xor (that is usually true
427 // for constant b's). Even though xor is symmetrical, canonicalization
428 // ensures that the constant will be the RHS. We have additional checks
429 // later on to ensure that this xor operation is equivalent to subtraction.
430 return m_Instruction(I&: SubBC, P: m_CombineOr(Ps: m_NUWSub(L: m_Value(V&: B), R: m_Value(V&: C)),
431 Ps: m_Xor(L: m_Value(V&: C), R: m_Value(V&: B))));
432 };
433
434 const auto MatchASubBC = [&]() {
435 // Cases:
436 // - a * (b - c)
437 // - (b - c) * a
438 // - (b - c) <- a implicitly equals 1
439 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: A), R: MatchSubBC()), Ps: MatchSubBC());
440 };
441
442 const auto MatchCD = [&]() {
443 // Cases:
444 // - d * c
445 // - c * d
446 // - c <- d implicitly equals 1
447 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: D), R: m_Specific(V: C)), Ps: m_Specific(V: C));
448 };
449
450 const auto Match = [&](const Value *LHS, const Value *RHS) {
451 // We do use m_Specific(C) in MatchCD, so we have to make sure that
452 // it's bound to anything and match(LHS, MatchASubBC()) absolutely
453 // has to evaluate first and return true.
454 //
455 // If Match returns true, it is guaranteed that B != nullptr, C != nullptr.
456 return match(V: LHS, P: MatchASubBC()) && match(V: RHS, P: MatchCD());
457 };
458
459 if (!Match(Op0, Op1) && !Match(Op1, Op0))
460 return;
461
462 const auto ComputeKnownBitsOrOne = [&](const Value *V) {
463 // For some of the values we use the convention of leaving
464 // it nullptr to signify an implicit constant 1.
465 return V ? computeKnownBits(V, DemandedElts, Q, Depth: Depth + 1)
466 : KnownBits::makeConstant(C: APInt(BitWidth, 1));
467 };
468
469 // Check that all operands are non-negative
470 const KnownBits KnownA = ComputeKnownBitsOrOne(A);
471 if (!KnownA.isNonNegative())
472 return;
473
474 const KnownBits KnownD = ComputeKnownBitsOrOne(D);
475 if (!KnownD.isNonNegative())
476 return;
477
478 const KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
479 if (!KnownB.isNonNegative())
480 return;
481
482 const KnownBits KnownC = computeKnownBits(V: C, DemandedElts, Q, Depth: Depth + 1);
483 if (!KnownC.isNonNegative())
484 return;
485
486 // If we matched subtraction as xor, we need to actually check that xor
487 // is semantically equivalent to subtraction.
488 //
489 // For that to be true, b has to be a mask for c or that b's known
490 // ones cover all known and possible ones of c.
491 if (SubBC->getOpcode() == Instruction::Xor &&
492 !KnownC.getMaxValue().isSubsetOf(RHS: KnownB.getMinValue()))
493 return;
494
495 const APInt MaxA = KnownA.getMaxValue();
496 const APInt MaxD = KnownD.getMaxValue();
497 const APInt MaxAD = APIntOps::umax(A: MaxA, B: MaxD);
498 const APInt MaxB = KnownB.getMaxValue();
499
500 // We can't infer leading zeros info if the upper-bound estimate wraps.
501 bool Overflow;
502 const APInt UpperBound = MaxAD.umul_ov(RHS: MaxB, Overflow);
503
504 if (Overflow)
505 return;
506
507 // If we know that x <= y and both are positive than x has at least the same
508 // number of leading zeros as y.
509 const unsigned MinimumNumberOfLeadingZeros = UpperBound.countl_zero();
510 KnownOut.Zero.setHighBits(MinimumNumberOfLeadingZeros);
511}
512
513static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
514 bool NSW, bool NUW,
515 const APInt &DemandedElts,
516 KnownBits &KnownOut, KnownBits &Known2,
517 const SimplifyQuery &Q, unsigned Depth) {
518 computeKnownBits(V: Op1, DemandedElts, Known&: KnownOut, Q, Depth: Depth + 1);
519
520 // If one operand is unknown and we have no nowrap information,
521 // the result will be unknown independently of the second operand.
522 if (KnownOut.isUnknown() && !NSW && !NUW)
523 return;
524
525 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
526 KnownOut = KnownBits::computeForAddSub(Add, NSW, NUW, LHS: Known2, RHS: KnownOut);
527
528 if (!Add && NSW && !KnownOut.isNonNegative() &&
529 (isImpliedByDomCondition(Pred: ICmpInst::ICMP_SLE, LHS: Op1, RHS: Op0, ContextI: Q.CtxI, DL: Q.DL)
530 .value_or(u: false) ||
531 match(V: Op1, P: m_c_SMin(L: m_Specific(V: Op0), R: m_Value()))))
532 KnownOut.makeNonNegative();
533
534 if (Add)
535 // Try to match lerp pattern and combine results
536 computeKnownBitsFromLerpPattern(Op0, Op1, DemandedElts, KnownOut, Q, Depth);
537}
538
539static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
540 bool NUW, const APInt &DemandedElts,
541 KnownBits &Known, KnownBits &Known2,
542 const SimplifyQuery &Q, unsigned Depth) {
543 computeKnownBits(V: Op1, DemandedElts, Known, Q, Depth: Depth + 1);
544 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
545
546 bool isKnownNegative = false;
547 bool isKnownNonNegative = false;
548 // If the multiplication is known not to overflow, compute the sign bit.
549 if (NSW) {
550 if (Op0 == Op1) {
551 // The product of a number with itself is non-negative.
552 isKnownNonNegative = true;
553 } else {
554 bool isKnownNonNegativeOp1 = Known.isNonNegative();
555 bool isKnownNonNegativeOp0 = Known2.isNonNegative();
556 bool isKnownNegativeOp1 = Known.isNegative();
557 bool isKnownNegativeOp0 = Known2.isNegative();
558 // The product of two numbers with the same sign is non-negative.
559 isKnownNonNegative = (isKnownNegativeOp1 && isKnownNegativeOp0) ||
560 (isKnownNonNegativeOp1 && isKnownNonNegativeOp0);
561 if (!isKnownNonNegative && NUW) {
562 // mul nuw nsw with a factor > 1 is non-negative.
563 KnownBits One = KnownBits::makeConstant(C: APInt(Known.getBitWidth(), 1));
564 isKnownNonNegative = KnownBits::sgt(LHS: Known, RHS: One).value_or(u: false) ||
565 KnownBits::sgt(LHS: Known2, RHS: One).value_or(u: false);
566 }
567
568 // The product of a negative number and a non-negative number is either
569 // negative or zero.
570 if (!isKnownNonNegative)
571 isKnownNegative =
572 (isKnownNegativeOp1 && isKnownNonNegativeOp0 &&
573 Known2.isNonZero()) ||
574 (isKnownNegativeOp0 && isKnownNonNegativeOp1 && Known.isNonZero());
575 }
576 }
577
578 bool SelfMultiply = Op0 == Op1;
579 if (SelfMultiply)
580 SelfMultiply &=
581 isGuaranteedNotToBeUndef(V: Op0, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1);
582 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
583
584 if (SelfMultiply) {
585 unsigned SignBits = ComputeNumSignBits(V: Op0, DemandedElts, Q, Depth: Depth + 1);
586 unsigned TyBits = Op0->getType()->getScalarSizeInBits();
587 unsigned OutValidBits = 2 * (TyBits - SignBits + 1);
588
589 if (OutValidBits < TyBits) {
590 APInt KnownZeroMask =
591 APInt::getHighBitsSet(numBits: TyBits, hiBitsSet: TyBits - OutValidBits + 1);
592 Known.Zero |= KnownZeroMask;
593 }
594 }
595
596 // Only make use of no-wrap flags if we failed to compute the sign bit
597 // directly. This matters if the multiplication always overflows, in
598 // which case we prefer to follow the result of the direct computation,
599 // though as the program is invoking undefined behaviour we can choose
600 // whatever we like here.
601 if (isKnownNonNegative && !Known.isNegative())
602 Known.makeNonNegative();
603 else if (isKnownNegative && !Known.isNonNegative())
604 Known.makeNegative();
605}
606
607void llvm::computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
608 KnownBits &Known) {
609 unsigned BitWidth = Known.getBitWidth();
610 unsigned NumRanges = Ranges.getNumOperands() / 2;
611 assert(NumRanges >= 1);
612
613 Known.setAllConflict();
614
615 for (unsigned i = 0; i < NumRanges; ++i) {
616 ConstantInt *Lower =
617 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 0));
618 ConstantInt *Upper =
619 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 1));
620 ConstantRange Range(Lower->getValue(), Upper->getValue());
621 // BitWidth must equal the Ranges BitWidth for the correct number of high
622 // bits to be set.
623 assert(BitWidth == Range.getBitWidth() &&
624 "Known bit width must match range bit width!");
625
626 // The first CommonPrefixBits of all values in Range are equal.
627 unsigned CommonPrefixBits =
628 (Range.getUnsignedMax() ^ Range.getUnsignedMin()).countl_zero();
629 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: CommonPrefixBits);
630 APInt UnsignedMax = Range.getUnsignedMax().zextOrTrunc(width: BitWidth);
631 Known.One &= UnsignedMax & Mask;
632 Known.Zero &= ~UnsignedMax & Mask;
633 }
634}
635
636static bool isEphemeralValueOf(const Instruction *I, const Value *E) {
637 // The instruction defining an assumption's condition itself is always
638 // considered ephemeral to that assumption (even if it has other
639 // non-ephemeral users). See r246696's test case for an example.
640 if (is_contained(Range: I->operands(), Element: E))
641 return true;
642
643 const auto *EI = dyn_cast<Instruction>(Val: E);
644 if (!EI)
645 return false;
646
647 if (EI == I)
648 return true;
649
650 SmallPtrSet<const Instruction *, 16> Visited;
651 SmallVector<const Instruction *, 16> WorkList;
652 Visited.insert(Ptr: EI);
653 WorkList.push_back(Elt: EI);
654 bool ReachesI = false;
655 while (!WorkList.empty()) {
656 const Instruction *V = WorkList.pop_back_val();
657 for (const User *U : V->users()) {
658 const auto *UI = cast<Instruction>(Val: U);
659 if (UI == I) {
660 ReachesI = true;
661 continue;
662 }
663 if (UI->mayHaveSideEffects() || UI->isTerminator())
664 return false;
665 if (Visited.insert(Ptr: UI).second)
666 WorkList.push_back(Elt: UI);
667 }
668 }
669 return ReachesI;
670}
671
672// Is this an intrinsic that cannot be speculated but also cannot trap?
673bool llvm::isAssumeLikeIntrinsic(const Instruction *I) {
674 if (const IntrinsicInst *CI = dyn_cast<IntrinsicInst>(Val: I))
675 return CI->isAssumeLikeIntrinsic();
676
677 return false;
678}
679
680bool llvm::isValidAssumeForContext(const Instruction *Inv,
681 const Instruction *CtxI,
682 const DominatorTree *DT,
683 bool AllowEphemerals) {
684 // There are two restrictions on the use of an assume:
685 // 1. The assume must dominate the context (or the control flow must
686 // reach the assume whenever it reaches the context).
687 // 2. The context must not be in the assume's set of ephemeral values
688 // (otherwise we will use the assume to prove that the condition
689 // feeding the assume is trivially true, thus causing the removal of
690 // the assume).
691
692 if (Inv->getParent() == CtxI->getParent()) {
693 // If Inv and CtxI are in the same block, check if the assume (Inv) is first
694 // in the BB.
695 if (Inv->comesBefore(Other: CtxI))
696 return true;
697
698 // Don't let an assume affect itself - this would cause the problems
699 // `isEphemeralValueOf` is trying to prevent, and it would also make
700 // the loop below go out of bounds.
701 if (!AllowEphemerals && Inv == CtxI)
702 return false;
703
704 // The context comes first, but they're both in the same block.
705 // Make sure there is nothing in between that might interrupt
706 // the control flow, not even CtxI itself.
707 // We limit the scan distance between the assume and its context instruction
708 // to avoid a compile-time explosion. This limit is chosen arbitrarily, so
709 // it can be adjusted if needed (could be turned into a cl::opt).
710 auto Range = make_range(x: CtxI->getIterator(), y: Inv->getIterator());
711 if (!isGuaranteedToTransferExecutionToSuccessor(Range, ScanLimit: 15))
712 return false;
713
714 return AllowEphemerals || !isEphemeralValueOf(I: Inv, E: CtxI);
715 }
716
717 // Inv and CtxI are in different blocks.
718 if (DT) {
719 if (DT->dominates(Def: Inv, User: CtxI))
720 return true;
721 } else if (Inv->getParent() == CtxI->getParent()->getSinglePredecessor() ||
722 Inv->getParent()->isEntryBlock()) {
723 // We don't have a DT, but this trivially dominates.
724 return true;
725 }
726
727 return false;
728}
729
730static bool hasNoFreeInRange(BasicBlock::const_iterator Begin,
731 BasicBlock::const_iterator End,
732 unsigned &NumChecked) {
733 for (const Instruction &I : make_range(x: Begin, y: End)) {
734 if (NumChecked++ > MaxInstrsToCheckForFree)
735 return false;
736 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
737 if (!CB->hasFnAttr(Kind: Attribute::NoFree))
738 return false;
739 } else if (I.maySynchronize()) {
740 return false;
741 }
742 }
743 return true;
744}
745
746bool llvm::willNotFreeBetween(const Instruction *Assume,
747 const Instruction *CtxI,
748 const DominatorTree *DT) {
749 const BasicBlock *CtxBB = CtxI->getParent();
750 const BasicBlock *AssumeBB = Assume->getParent();
751 unsigned NumChecked = 0;
752 BasicBlock::const_iterator CtxIter = CtxI->getIterator();
753 if (CtxBB == AssumeBB) {
754 if (Assume != CtxI && !Assume->comesBefore(Other: CtxI))
755 return false;
756 return hasNoFreeInRange(Begin: Assume->getIterator(), End: CtxIter, NumChecked);
757 }
758 if (DT && !DT->dominates(Def: Assume, User: CtxI))
759 return false;
760 if (!hasNoFreeInRange(Begin: CtxBB->begin(), End: CtxIter, NumChecked))
761 return false;
762 if (pred_empty(BB: CtxBB))
763 return false;
764
765 // Note: CtxBB is NOT pre-inserted into Visited to ensure that loop
766 // backedges returning to CtxBB are enqueued and checked correctly.
767 SmallVector<const BasicBlock *, 16> Worklist(predecessors(BB: CtxBB));
768 SmallPtrSet<const BasicBlock *, 16> Visited;
769 while (!Worklist.empty()) {
770 const BasicBlock *CurBB = Worklist.pop_back_val();
771 if (!Visited.insert(Ptr: CurBB).second)
772 continue;
773
774 if (CurBB == AssumeBB) {
775 if (!hasNoFreeInRange(Begin: Assume->getIterator(), End: AssumeBB->end(), NumChecked))
776 return false;
777 continue;
778 }
779 assert((!DT || DT->dominates(AssumeBB, CurBB)) &&
780 "Blocks between Assume and CtxI must be dominated by AssumeBB");
781
782 if (pred_empty(BB: CurBB))
783 return false;
784
785 // If CurBB == CtxBB (due to a loop backedge targeting CtxBB), check
786 // instructions from CtxIter to the end of CtxBB (instructions before
787 // CtxIter were checked above). Otherwise, check the entire block.
788 auto StartIt = (CurBB == CtxBB) ? CtxIter : CurBB->begin();
789 if (!hasNoFreeInRange(Begin: StartIt, End: CurBB->end(), NumChecked))
790 return false;
791 append_range(C&: Worklist, R: predecessors(BB: CurBB));
792 }
793 return true;
794}
795
796// TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but
797// we still have enough information about `RHS` to conclude non-zero. For
798// example Pred=EQ, RHS=isKnownNonZero. cmpExcludesZero is called in loops
799// so the extra compile time may not be worth it, but possibly a second API
800// should be created for use outside of loops.
801static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) {
802 // v u> y implies v != 0.
803 if (Pred == ICmpInst::ICMP_UGT)
804 return true;
805
806 // Special-case v != 0 to also handle v != null.
807 if (Pred == ICmpInst::ICMP_NE)
808 return match(V: RHS, P: m_Zero());
809
810 // All other predicates - rely on generic ConstantRange handling.
811 const APInt *C;
812 auto Zero = APInt::getZero(numBits: RHS->getType()->getScalarSizeInBits());
813 if (match(V: RHS, P: m_APInt(Res&: C))) {
814 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
815 return !TrueValues.contains(Val: Zero);
816 }
817
818 auto *VC = dyn_cast<ConstantDataVector>(Val: RHS);
819 if (VC == nullptr)
820 return false;
821
822 for (unsigned ElemIdx = 0, NElem = VC->getNumElements(); ElemIdx < NElem;
823 ++ElemIdx) {
824 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(
825 Pred, Other: VC->getElementAsAPInt(i: ElemIdx));
826 if (TrueValues.contains(Val: Zero))
827 return false;
828 }
829 return true;
830}
831
832static void breakSelfRecursivePHI(const Use *U, const PHINode *PHI,
833 Value *&ValOut, Instruction *&CtxIOut,
834 const PHINode **PhiOut = nullptr) {
835 ValOut = U->get();
836 if (ValOut == PHI)
837 return;
838 CtxIOut = PHI->getIncomingBlock(U: *U)->getTerminator();
839 if (PhiOut)
840 *PhiOut = PHI;
841 Value *V;
842 // If the Use is a select of this phi, compute analysis on other arm to break
843 // recursion.
844 // TODO: Min/Max
845 if (match(V: ValOut, P: m_Select(C: m_Value(), L: m_Specific(V: PHI), R: m_Value(V))) ||
846 match(V: ValOut, P: m_Select(C: m_Value(), L: m_Value(V), R: m_Specific(V: PHI))))
847 ValOut = V;
848
849 // Same for select, if this phi is 2-operand phi, compute analysis on other
850 // incoming value to break recursion.
851 // TODO: We could handle any number of incoming edges as long as we only have
852 // two unique values.
853 if (auto *IncPhi = dyn_cast<PHINode>(Val: ValOut);
854 IncPhi && IncPhi->getNumIncomingValues() == 2) {
855 for (int Idx = 0; Idx < 2; ++Idx) {
856 if (IncPhi->getIncomingValue(i: Idx) == PHI) {
857 ValOut = IncPhi->getIncomingValue(i: 1 - Idx);
858 if (PhiOut)
859 *PhiOut = IncPhi;
860 CtxIOut = IncPhi->getIncomingBlock(i: 1 - Idx)->getTerminator();
861 break;
862 }
863 }
864 }
865}
866
867static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) {
868 // Use of assumptions is context-sensitive. If we don't have a context, we
869 // cannot use them!
870 if (!Q.AC || !Q.CtxI)
871 return false;
872
873 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
874 if (!Elem.Assume)
875 continue;
876
877 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
878 assert(I->getFunction() == Q.CtxI->getFunction() &&
879 "Got assumption for the wrong function!");
880
881 if (Elem.Index != AssumptionCache::ExprResultIdx) {
882 if (assumeBundleImpliesNonNull(Val: V, Context: Q.CtxI->getFunction(),
883 OBU: I->getOperandBundleAt(Index: Elem.Index)) &&
884 isValidAssumeForContext(I, Q))
885 return true;
886 continue;
887 }
888
889 // Warning: This loop can end up being somewhat performance sensitive.
890 // We're running this loop for once for each value queried resulting in a
891 // runtime of ~O(#assumes * #values).
892
893 Value *RHS;
894 CmpPredicate Pred;
895 auto m_V = m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToInt(Op: m_Specific(V)));
896 if (!match(V: I->getArgOperand(i: 0), P: m_c_ICmp(Pred, L: m_V, R: m_Value(V&: RHS))))
897 continue;
898
899 if (cmpExcludesZero(Pred, RHS) && isValidAssumeForContext(I, Q))
900 return true;
901 }
902
903 return false;
904}
905
906static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
907 Value *LHS, Value *RHS, KnownBits &Known,
908 const SimplifyQuery &Q) {
909 if (RHS->getType()->isPointerTy()) {
910 // Handle comparison of pointer to null explicitly, as it will not be
911 // covered by the m_APInt() logic below.
912 if (LHS == V && match(V: RHS, P: m_Zero())) {
913 switch (Pred) {
914 case ICmpInst::ICMP_EQ:
915 Known.setAllZero();
916 break;
917 case ICmpInst::ICMP_SGE:
918 case ICmpInst::ICMP_SGT:
919 Known.makeNonNegative();
920 break;
921 case ICmpInst::ICMP_SLT:
922 Known.makeNegative();
923 break;
924 default:
925 break;
926 }
927 }
928 return;
929 }
930
931 unsigned BitWidth = Known.getBitWidth();
932 auto m_V =
933 m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToIntSameSize(DL: Q.DL, Op: m_Specific(V)));
934
935 Value *Y;
936 const APInt *Mask, *C;
937 if (!match(V: RHS, P: m_APInt(Res&: C)))
938 return;
939
940 uint64_t ShAmt;
941 switch (Pred) {
942 case ICmpInst::ICMP_EQ:
943 // assume(V = C)
944 if (match(V: LHS, P: m_V)) {
945 Known = Known.unionWith(RHS: KnownBits::makeConstant(C: *C));
946 // assume(V & Mask = C)
947 } else if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value(V&: Y)))) {
948 // For one bits in Mask, we can propagate bits from C to V.
949 Known.One |= *C;
950 if (match(V: Y, P: m_APInt(Res&: Mask)))
951 Known.Zero |= ~*C & *Mask;
952 // assume(V | Mask = C)
953 } else if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value(V&: Y)))) {
954 // For zero bits in Mask, we can propagate bits from C to V.
955 Known.Zero |= ~*C;
956 if (match(V: Y, P: m_APInt(Res&: Mask)))
957 Known.One |= *C & ~*Mask;
958 // assume(V << ShAmt = C)
959 } else if (match(V: LHS, P: m_Shl(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
960 ShAmt < BitWidth) {
961 // For those bits in C that are known, we can propagate them to known
962 // bits in V shifted to the right by ShAmt.
963 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
964 RHSKnown >>= ShAmt;
965 Known = Known.unionWith(RHS: RHSKnown);
966 // assume(V >> ShAmt = C)
967 } else if (match(V: LHS, P: m_Shr(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
968 ShAmt < BitWidth) {
969 // For those bits in RHS that are known, we can propagate them to known
970 // bits in V shifted to the right by C.
971 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
972 RHSKnown <<= ShAmt;
973 Known = Known.unionWith(RHS: RHSKnown);
974 }
975 break;
976 case ICmpInst::ICMP_NE: {
977 // assume (V & B != 0) where B is a power of 2
978 const APInt *BPow2;
979 if (C->isZero() && match(V: LHS, P: m_And(L: m_V, R: m_Power2(V&: BPow2))))
980 Known.One |= *BPow2;
981 break;
982 }
983 default: {
984 const APInt *Offset = nullptr;
985 if (match(V: LHS, P: m_CombineOr(Ps: m_V, Ps: m_AddLike(L: m_V, R: m_APInt(Res&: Offset))))) {
986 ConstantRange LHSRange = ConstantRange::makeAllowedICmpRegion(Pred, Other: *C);
987 if (Offset)
988 LHSRange = LHSRange.sub(Other: *Offset);
989 Known = Known.unionWith(RHS: LHSRange.toKnownBits());
990 }
991 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
992 // X & Y u> C -> X u> C && Y u> C
993 // X nuw- Y u> C -> X u> C
994 if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value())) ||
995 match(V: LHS, P: m_NUWSub(L: m_V, R: m_Value())))
996 Known.One.setHighBits(
997 (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
998 }
999 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
1000 // X | Y u< C -> X u< C && Y u< C
1001 // X nuw+ Y u< C -> X u< C && Y u< C
1002 if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value())) ||
1003 match(V: LHS, P: m_c_NUWAdd(L: m_V, R: m_Value()))) {
1004 Known.Zero.setHighBits(
1005 (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
1006 }
1007 }
1008 } break;
1009 }
1010}
1011
1012static void computeKnownBitsFromICmpCond(const Value *V, ICmpInst *Cmp,
1013 KnownBits &Known,
1014 const SimplifyQuery &SQ, bool Invert) {
1015 ICmpInst::Predicate Pred =
1016 Invert ? Cmp->getInversePredicate() : Cmp->getPredicate();
1017 Value *LHS = Cmp->getOperand(i_nocapture: 0);
1018 Value *RHS = Cmp->getOperand(i_nocapture: 1);
1019
1020 // Handle icmp pred (trunc V), C
1021 if (match(V: LHS, P: m_Trunc(Op: m_Specific(V)))) {
1022 KnownBits DstKnown(LHS->getType()->getScalarSizeInBits());
1023 computeKnownBitsFromCmp(V: LHS, Pred, LHS, RHS, Known&: DstKnown, Q: SQ);
1024 if (cast<TruncInst>(Val: LHS)->hasNoUnsignedWrap())
1025 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1026 else
1027 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1028 return;
1029 }
1030
1031 computeKnownBitsFromCmp(V, Pred, LHS, RHS, Known, Q: SQ);
1032}
1033
1034static void computeKnownBitsFromCond(const Value *V, Value *Cond,
1035 KnownBits &Known, const SimplifyQuery &SQ,
1036 bool Invert, unsigned Depth) {
1037 Value *A, *B;
1038 if (Depth < MaxAnalysisRecursionDepth &&
1039 match(V: Cond, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
1040 KnownBits Known2(Known.getBitWidth());
1041 KnownBits Known3(Known.getBitWidth());
1042 computeKnownBitsFromCond(V, Cond: A, Known&: Known2, SQ, Invert, Depth: Depth + 1);
1043 computeKnownBitsFromCond(V, Cond: B, Known&: Known3, SQ, Invert, Depth: Depth + 1);
1044 if (Invert ? match(V: Cond, P: m_LogicalOr(L: m_Value(), R: m_Value()))
1045 : match(V: Cond, P: m_LogicalAnd(L: m_Value(), R: m_Value())))
1046 Known2 = Known2.unionWith(RHS: Known3);
1047 else
1048 Known2 = Known2.intersectWith(RHS: Known3);
1049 Known = Known.unionWith(RHS: Known2);
1050 return;
1051 }
1052
1053 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
1054 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ, Invert);
1055 return;
1056 }
1057
1058 if (match(V: Cond, P: m_Trunc(Op: m_Specific(V)))) {
1059 KnownBits DstKnown(1);
1060 if (Invert) {
1061 DstKnown.setAllZero();
1062 } else {
1063 DstKnown.setAllOnes();
1064 }
1065 if (cast<TruncInst>(Val: Cond)->hasNoUnsignedWrap()) {
1066 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1067 return;
1068 }
1069 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1070 return;
1071 }
1072
1073 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A))))
1074 computeKnownBitsFromCond(V, Cond: A, Known, SQ, Invert: !Invert, Depth: Depth + 1);
1075}
1076
1077void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
1078 const SimplifyQuery &Q, unsigned Depth) {
1079 // Handle injected condition.
1080 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
1081 computeKnownBitsFromCond(V, Cond: Q.CC->Cond, Known, SQ: Q, Invert: Q.CC->Invert, Depth);
1082
1083 if (!Q.CtxI)
1084 return;
1085
1086 if (Q.DC && Q.DT) {
1087 // Handle dominating conditions.
1088 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
1089 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
1090 if (Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()))
1091 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1092 /*Invert*/ false, Depth);
1093
1094 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
1095 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()))
1096 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1097 /*Invert*/ true, Depth);
1098 }
1099
1100 if (Known.hasConflict())
1101 Known.resetAll();
1102 }
1103
1104 if (!Q.AC)
1105 return;
1106
1107 unsigned BitWidth = Known.getBitWidth();
1108
1109 // Note that the patterns below need to be kept in sync with the code
1110 // in AssumptionCache::updateAffectedValues.
1111
1112 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
1113 if (!Elem.Assume)
1114 continue;
1115
1116 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
1117 assert(I->getParent()->getParent() == Q.CtxI->getParent()->getParent() &&
1118 "Got assumption for the wrong function!");
1119
1120 if (Elem.Index != AssumptionCache::ExprResultIdx) {
1121 if (auto OBU = I->getOperandBundleAt(Index: Elem.Index);
1122 getBundleAttrFromOBU(OBU) == BundleAttr::Align) {
1123 auto [Ptr, _, _2, Alignment, Offset] = getAssumeAlignInfo(OBU);
1124 if (Ptr == V && Alignment && Offset && isPowerOf2_64(Value: *Alignment) &&
1125 isValidAssumeForContext(I, Q)) {
1126 Known.Zero |= (*Alignment - 1) & ~*Offset;
1127 Known.One |= (*Alignment - 1) & *Offset;
1128 }
1129 }
1130 continue;
1131 }
1132
1133 // Warning: This loop can end up being somewhat performance sensitive.
1134 // We're running this loop for once for each value queried resulting in a
1135 // runtime of ~O(#assumes * #values).
1136
1137 Value *Arg = I->getArgOperand(i: 0);
1138
1139 if (Arg == V && isValidAssumeForContext(I, Q)) {
1140 assert(BitWidth == 1 && "assume operand is not i1?");
1141 (void)BitWidth;
1142 Known.setAllOnes();
1143 return;
1144 }
1145 if (match(V: Arg, P: m_Not(V: m_Specific(V))) &&
1146 isValidAssumeForContext(I, Q)) {
1147 assert(BitWidth == 1 && "assume operand is not i1?");
1148 (void)BitWidth;
1149 Known.setAllZero();
1150 return;
1151 }
1152 auto *Trunc = dyn_cast<TruncInst>(Val: Arg);
1153 if (Trunc && Trunc->getOperand(i_nocapture: 0) == V &&
1154 isValidAssumeForContext(I, Q)) {
1155 if (Trunc->hasNoUnsignedWrap()) {
1156 Known = KnownBits::makeConstant(C: APInt(BitWidth, 1));
1157 return;
1158 }
1159 Known.One.setBit(0);
1160 return;
1161 }
1162
1163 // The remaining tests are all recursive, so bail out if we hit the limit.
1164 if (Depth == MaxAnalysisRecursionDepth)
1165 continue;
1166
1167 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
1168 if (!Cmp)
1169 continue;
1170
1171 if (!isValidAssumeForContext(I, Q))
1172 continue;
1173
1174 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ: Q, /*Invert=*/false);
1175 }
1176
1177 // Conflicting assumption: Undefined behavior will occur on this execution
1178 // path.
1179 if (Known.hasConflict())
1180 Known.resetAll();
1181}
1182
1183/// Compute known bits from a shift operator, including those with a
1184/// non-constant shift amount. Known is the output of this function. Known2 is a
1185/// pre-allocated temporary with the same bit width as Known and on return
1186/// contains the known bit of the shift value source. KF is an
1187/// operator-specific function that, given the known-bits and a shift amount,
1188/// compute the implied known-bits of the shift operator's result respectively
1189/// for that shift amount. The results from calling KF are conservatively
1190/// combined for all permitted shift amounts.
1191static void computeKnownBitsFromShiftOperator(
1192 const Operator *I, const APInt &DemandedElts, KnownBits &Known,
1193 KnownBits &Known2, const SimplifyQuery &Q, unsigned Depth,
1194 function_ref<KnownBits(const KnownBits &, const KnownBits &, bool)> KF) {
1195 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1196 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1197 // To limit compile-time impact, only query isKnownNonZero() if we know at
1198 // least something about the shift amount.
1199 bool ShAmtNonZero =
1200 Known.isNonZero() ||
1201 (Known.getMaxValue().ult(RHS: Known.getBitWidth()) &&
1202 isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1));
1203 Known = KF(Known2, Known, ShAmtNonZero);
1204}
1205
1206static KnownBits
1207getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
1208 const KnownBits &KnownLHS, const KnownBits &KnownRHS,
1209 const SimplifyQuery &Q, unsigned Depth) {
1210 unsigned BitWidth = KnownLHS.getBitWidth();
1211 KnownBits KnownOut(BitWidth);
1212 bool IsAnd = false;
1213 bool HasKnownOne = !KnownLHS.One.isZero() || !KnownRHS.One.isZero();
1214 Value *X = nullptr, *Y = nullptr;
1215
1216 switch (I->getOpcode()) {
1217 case Instruction::And:
1218 KnownOut = KnownLHS & KnownRHS;
1219 IsAnd = true;
1220 // and(x, -x) is common idioms that will clear all but lowest set
1221 // bit. If we have a single known bit in x, we can clear all bits
1222 // above it.
1223 // TODO: instcombine often reassociates independent `and` which can hide
1224 // this pattern. Try to match and(x, and(-x, y)) / and(and(x, y), -x).
1225 if (HasKnownOne && match(V: I, P: m_c_And(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X))))) {
1226 // -(-x) == x so using whichever (LHS/RHS) gets us a better result.
1227 if (KnownLHS.countMaxTrailingZeros() <= KnownRHS.countMaxTrailingZeros())
1228 KnownOut = KnownLHS.blsi();
1229 else
1230 KnownOut = KnownRHS.blsi();
1231 }
1232 break;
1233 case Instruction::Or:
1234 KnownOut = KnownLHS | KnownRHS;
1235 break;
1236 case Instruction::Xor:
1237 KnownOut = KnownLHS ^ KnownRHS;
1238 // xor(x, x-1) is common idioms that will clear all but lowest set
1239 // bit. If we have a single known bit in x, we can clear all bits
1240 // above it.
1241 // TODO: xor(x, x-1) is often rewritting as xor(x, x-C) where C !=
1242 // -1 but for the purpose of demanded bits (xor(x, x-C) &
1243 // Demanded) == (xor(x, x-1) & Demanded). Extend the xor pattern
1244 // to use arbitrary C if xor(x, x-C) as the same as xor(x, x-1).
1245 if (HasKnownOne &&
1246 match(V: I, P: m_c_Xor(L: m_Value(V&: X), R: m_Add(L: m_Deferred(V: X), R: m_AllOnes())))) {
1247 const KnownBits &XBits = I->getOperand(i: 0) == X ? KnownLHS : KnownRHS;
1248 KnownOut = XBits.blsmsk();
1249 }
1250 break;
1251 default:
1252 llvm_unreachable("Invalid Op used in 'analyzeKnownBitsFromAndXorOr'");
1253 }
1254
1255 // and(x, add (x, -1)) is a common idiom that always clears the low bit;
1256 // xor/or(x, add (x, -1)) is an idiom that will always set the low bit.
1257 // here we handle the more general case of adding any odd number by
1258 // matching the form and/xor/or(x, add(x, y)) where y is odd.
1259 // TODO: This could be generalized to clearing any bit set in y where the
1260 // following bit is known to be unset in y.
1261 if (!KnownOut.Zero[0] && !KnownOut.One[0] &&
1262 (match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_c_Add(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1263 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1264 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Value(V&: Y), R: m_Deferred(V: X)))))) {
1265 KnownBits KnownY(BitWidth);
1266 computeKnownBits(V: Y, DemandedElts, Known&: KnownY, Q, Depth: Depth + 1);
1267 if (KnownY.countMinTrailingOnes() > 0) {
1268 if (IsAnd)
1269 KnownOut.Zero.setBit(0);
1270 else
1271 KnownOut.One.setBit(0);
1272 }
1273 }
1274 return KnownOut;
1275}
1276
1277static KnownBits computeKnownBitsForHorizontalOperation(
1278 const Operator *I, const APInt &DemandedElts, const SimplifyQuery &Q,
1279 unsigned Depth,
1280 const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
1281 KnownBitsFunc) {
1282 APInt DemandedEltsLHS, DemandedEltsRHS;
1283 getHorizDemandedEltsForFirstOperand(VectorBitWidth: Q.DL.getTypeSizeInBits(Ty: I->getType()),
1284 DemandedElts, DemandedLHS&: DemandedEltsLHS,
1285 DemandedRHS&: DemandedEltsRHS);
1286
1287 const auto ComputeForSingleOpFunc =
1288 [Depth, &Q, KnownBitsFunc](const Value *Op, APInt &DemandedEltsOp) {
1289 return KnownBitsFunc(
1290 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp, Q, Depth: Depth + 1),
1291 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp << 1, Q, Depth: Depth + 1));
1292 };
1293
1294 if (DemandedEltsRHS.isZero())
1295 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS);
1296 if (DemandedEltsLHS.isZero())
1297 return ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS);
1298
1299 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS)
1300 .intersectWith(RHS: ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS));
1301}
1302
1303// Public so this can be used in `SimplifyDemandedUseBits`.
1304KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
1305 const KnownBits &KnownLHS,
1306 const KnownBits &KnownRHS,
1307 const SimplifyQuery &SQ,
1308 unsigned Depth) {
1309 auto *FVTy = dyn_cast<FixedVectorType>(Val: I->getType());
1310 APInt DemandedElts =
1311 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
1312
1313 return getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS, KnownRHS, Q: SQ,
1314 Depth);
1315}
1316
1317ConstantRange llvm::getVScaleRange(const Function *F, unsigned BitWidth) {
1318 Attribute Attr = F->getFnAttribute(Kind: Attribute::VScaleRange);
1319 // Without vscale_range, we only know that vscale is non-zero.
1320 if (!Attr.isValid())
1321 return ConstantRange(APInt(BitWidth, 1), APInt::getZero(numBits: BitWidth));
1322
1323 unsigned AttrMin = Attr.getVScaleRangeMin();
1324 // Minimum is larger than vscale width, result is always poison.
1325 if ((unsigned)llvm::bit_width(Value: AttrMin) > BitWidth)
1326 return ConstantRange::getEmpty(BitWidth);
1327
1328 APInt Min(BitWidth, AttrMin);
1329 std::optional<unsigned> AttrMax = Attr.getVScaleRangeMax();
1330 if (!AttrMax || (unsigned)llvm::bit_width(Value: *AttrMax) > BitWidth)
1331 return ConstantRange(Min, APInt::getZero(numBits: BitWidth));
1332
1333 return ConstantRange(Min, APInt(BitWidth, *AttrMax) + 1);
1334}
1335
1336/// Return true if \p II reads a register named "vlenb". On RISC-V this is the
1337/// VLENB CSR, which holds VLEN/8: a non-zero power of two bounded by the
1338/// target's VLEN range. Callers must ensure the target is RISC-V.
1339static bool isReadVLENB(const IntrinsicInst &II) {
1340 auto *MAV = dyn_cast<MetadataAsValue>(Val: II.getArgOperand(i: 0));
1341 if (!MAV)
1342 return false;
1343 auto *MD = dyn_cast<MDNode>(Val: MAV->getMetadata());
1344 if (!MD || MD->getNumOperands() != 1)
1345 return false;
1346 auto *RegName = dyn_cast<MDString>(Val: MD->getOperand(I: 0));
1347 return RegName && RegName->getString() == "vlenb";
1348}
1349
1350/// Return the value range of a RISC-V vlenb CSR read. RVV requires VLEN to be a
1351/// power of two in [32, 65536] (Zvl32b is the smallest vector extension), so
1352/// VLENB = VLEN/8 is in [4, 8192]. This architectural bound is independent of
1353/// any function attribute and stays sound for Zvl32b, whose VLEN (32) is not
1354/// representable as an integer vscale (VLEN / RVVBitsPerBlock). A vscale_range
1355/// attribute, when present, pins the subtarget's VLEN in units of
1356/// RVVBitsPerBlock (64 bits) and so gives a tighter VLENB = vscale *
1357/// RVVBytesPerBlock.
1358static ConstantRange getRISCVVLENBRange(const IntrinsicInst &II,
1359 unsigned Width) {
1360 // Architectural bounds: VLEN in [32, 65536] => VLENB in [4, 8192].
1361 ConstantRange Range(APInt(Width, 32 / 8), APInt(Width, 65536 / 8) + 1);
1362
1363 const Function *F = II.getFunction();
1364 if (F->getFnAttribute(Kind: Attribute::VScaleRange).isValid()) {
1365 ConstantRange VScale = getVScaleRange(F, BitWidth: Width);
1366 Range = Range.intersectWith(
1367 CR: VScale.multiply(Other: ConstantRange(APInt(Width, RISCV::RVVBytesPerBlock))));
1368 }
1369 return Range;
1370}
1371
1372void llvm::adjustKnownBitsForSelectArm(KnownBits &Known, Value *Cond,
1373 Value *Arm, bool Invert,
1374 const SimplifyQuery &Q, unsigned Depth) {
1375 // If we have a constant arm, we are done.
1376 if (Known.isConstant())
1377 return;
1378
1379 // See what condition implies about the bits of the select arm.
1380 KnownBits CondRes(Known.getBitWidth());
1381 computeKnownBitsFromCond(V: Arm, Cond, Known&: CondRes, SQ: Q, Invert, Depth: Depth + 1);
1382 // If we don't get any information from the condition, no reason to
1383 // proceed.
1384 if (CondRes.isUnknown())
1385 return;
1386
1387 // We can have conflict if the condition is dead. I.e if we have
1388 // (x | 64) < 32 ? (x | 64) : y
1389 // we will have conflict at bit 6 from the condition/the `or`.
1390 // In that case just return. Its not particularly important
1391 // what we do, as this select is going to be simplified soon.
1392 CondRes = CondRes.unionWith(RHS: Known);
1393 if (CondRes.hasConflict())
1394 return;
1395
1396 // Finally make sure the information we found is valid. This is relatively
1397 // expensive so it's left for the very end.
1398 if (!isGuaranteedNotToBeUndef(V: Arm, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1))
1399 return;
1400
1401 // Finally, we know we get information from the condition and its valid,
1402 // so return it.
1403 Known = std::move(CondRes);
1404}
1405
1406// Match a signed min+max clamp pattern like smax(smin(In, CHigh), CLow).
1407// Returns the input and lower/upper bounds.
1408static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
1409 const APInt *&CLow, const APInt *&CHigh) {
1410 assert(isa<Operator>(Select) &&
1411 cast<Operator>(Select)->getOpcode() == Instruction::Select &&
1412 "Input should be a Select!");
1413
1414 const Value *LHS = nullptr, *RHS = nullptr;
1415 SelectPatternFlavor SPF = matchSelectPattern(V: Select, LHS, RHS).Flavor;
1416 if (SPF != SPF_SMAX && SPF != SPF_SMIN)
1417 return false;
1418
1419 if (!match(V: RHS, P: m_APInt(Res&: CLow)))
1420 return false;
1421
1422 const Value *LHS2 = nullptr, *RHS2 = nullptr;
1423 SelectPatternFlavor SPF2 = matchSelectPattern(V: LHS, LHS&: LHS2, RHS&: RHS2).Flavor;
1424 if (getInverseMinMaxFlavor(SPF) != SPF2)
1425 return false;
1426
1427 if (!match(V: RHS2, P: m_APInt(Res&: CHigh)))
1428 return false;
1429
1430 if (SPF == SPF_SMIN)
1431 std::swap(a&: CLow, b&: CHigh);
1432
1433 In = LHS2;
1434 return CLow->sle(RHS: *CHigh);
1435}
1436
1437static bool isSignedMinMaxIntrinsicClamp(const IntrinsicInst *II,
1438 const APInt *&CLow,
1439 const APInt *&CHigh) {
1440 assert((II->getIntrinsicID() == Intrinsic::smin ||
1441 II->getIntrinsicID() == Intrinsic::smax) &&
1442 "Must be smin/smax");
1443
1444 Intrinsic::ID InverseID = getInverseMinMaxIntrinsic(MinMaxID: II->getIntrinsicID());
1445 auto *InnerII = dyn_cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
1446 if (!InnerII || InnerII->getIntrinsicID() != InverseID ||
1447 !match(V: II->getArgOperand(i: 1), P: m_APInt(Res&: CLow)) ||
1448 !match(V: InnerII->getArgOperand(i: 1), P: m_APInt(Res&: CHigh)))
1449 return false;
1450
1451 if (II->getIntrinsicID() == Intrinsic::smin)
1452 std::swap(a&: CLow, b&: CHigh);
1453 return CLow->sle(RHS: *CHigh);
1454}
1455
1456static void unionWithMinMaxIntrinsicClamp(const IntrinsicInst *II,
1457 KnownBits &Known) {
1458 const APInt *CLow, *CHigh;
1459 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
1460 Known = Known.unionWith(
1461 RHS: ConstantRange::getNonEmpty(Lower: *CLow, Upper: *CHigh + 1).toKnownBits());
1462}
1463
1464static void computeKnownBitsForRecurrenceOperands(
1465 const PHINode *P, Value *Start, Value *Step, const APInt &DemandedElts,
1466 KnownBits &KnownStart, KnownBits &KnownStep, const SimplifyQuery &Q,
1467 unsigned Depth) {
1468 // Change the context instruction to the "edge" that flows into the phi. This
1469 // is important because that is where the value is actually "evaluated" even
1470 // though it is used later somewhere else. (see also D69571).
1471 SimplifyQuery RecQ = Q.getWithoutCondContext();
1472 unsigned OpNum = P->getOperand(i_nocapture: 0) == Start ? 0 : 1;
1473
1474 RecQ.CtxI = P->getIncomingBlock(i: OpNum)->getTerminator();
1475 computeKnownBits(V: Start, DemandedElts, Known&: KnownStart, Q: RecQ, Depth: Depth + 1);
1476
1477 RecQ.CtxI = P->getIncomingBlock(i: 1 - OpNum)->getTerminator();
1478 computeKnownBits(V: Step, DemandedElts, Known&: KnownStep, Q: RecQ, Depth: Depth + 1);
1479}
1480
1481static void computeKnownBitsFromOperator(const Operator *I,
1482 const APInt &DemandedElts,
1483 KnownBits &Known,
1484 const SimplifyQuery &Q,
1485 unsigned Depth) {
1486 unsigned BitWidth = Known.getBitWidth();
1487
1488 KnownBits Known2(BitWidth);
1489 switch (I->getOpcode()) {
1490 default: break;
1491 case Instruction::Load:
1492 if (MDNode *MD =
1493 Q.IIQ.getMetadata(I: cast<LoadInst>(Val: I), KindID: LLVMContext::MD_range))
1494 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
1495 break;
1496 case Instruction::And:
1497 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1498 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1499
1500 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1501 break;
1502 case Instruction::Or:
1503 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1504 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1505
1506 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1507 break;
1508 case Instruction::Xor:
1509 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1510 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1511
1512 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1513 break;
1514 case Instruction::Mul: {
1515 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1516 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1517 computeKnownBitsMul(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1518 DemandedElts, Known, Known2, Q, Depth);
1519 break;
1520 }
1521 case Instruction::UDiv: {
1522 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1523 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1524 Known =
1525 KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1526 break;
1527 }
1528 case Instruction::SDiv: {
1529 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1530 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1531 Known =
1532 KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1533 break;
1534 }
1535 case Instruction::Select: {
1536 auto ComputeForArm = [&](Value *Arm, bool Invert) {
1537 KnownBits Res(Known.getBitWidth());
1538 computeKnownBits(V: Arm, DemandedElts, Known&: Res, Q, Depth: Depth + 1);
1539 adjustKnownBitsForSelectArm(Known&: Res, Cond: I->getOperand(i: 0), Arm, Invert, Q, Depth);
1540 return Res;
1541 };
1542 // Only known if known in both the LHS and RHS.
1543 Known =
1544 ComputeForArm(I->getOperand(i: 1), /*Invert=*/false)
1545 .intersectWith(RHS: ComputeForArm(I->getOperand(i: 2), /*Invert=*/true));
1546 break;
1547 }
1548 case Instruction::FPToSI: {
1549 // fptosi is poison if the rounded value doesn't fit in the result type,
1550 // so we can assume the conversion is well-defined and rounds towards
1551 // zero. +-Inf can never fit in an integer type, so it is always poison,
1552 // like NaN. Negative subnormals and negative zero round to 0. That
1553 // leaves negative normals as the only class that can produce a defined
1554 // negative result.
1555 KnownFPClass SrcFPClass = computeKnownFPClass(
1556 V: I->getOperand(i: 0), DemandedElts, InterestedClasses: fcNegNormal, SQ: Q, Depth: Depth + 1);
1557 if (SrcFPClass.isKnownNever(Mask: fcNegNormal))
1558 Known.makeNonNegative();
1559 break;
1560 }
1561 case Instruction::FPTrunc:
1562 case Instruction::FPExt:
1563 case Instruction::FPToUI:
1564 case Instruction::SIToFP:
1565 case Instruction::UIToFP:
1566 break; // Can't work with floating point.
1567 case Instruction::PtrToInt:
1568 case Instruction::PtrToAddr:
1569 case Instruction::IntToPtr:
1570 // Fall through and handle them the same as zext/trunc.
1571 [[fallthrough]];
1572 case Instruction::ZExt:
1573 case Instruction::Trunc: {
1574 Type *SrcTy = I->getOperand(i: 0)->getType();
1575
1576 unsigned SrcBitWidth;
1577 // Note that we handle pointer operands here because of inttoptr/ptrtoint
1578 // which fall through here.
1579 Type *ScalarTy = SrcTy->getScalarType();
1580 SrcBitWidth = ScalarTy->isPointerTy() ?
1581 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
1582 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
1583
1584 assert(SrcBitWidth && "SrcBitWidth can't be zero");
1585 Known = Known.anyextOrTrunc(BitWidth: SrcBitWidth);
1586 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1587 if (auto *Inst = dyn_cast<PossiblyNonNegInst>(Val: I);
1588 Inst && Inst->hasNonNeg() && !Known.isNegative())
1589 Known.makeNonNegative();
1590 Known = Known.zextOrTrunc(BitWidth);
1591 break;
1592 }
1593 case Instruction::BitCast: {
1594 Type *SrcTy = I->getOperand(i: 0)->getType();
1595 if (SrcTy->isIntOrPtrTy() &&
1596 // TODO: For now, not handling conversions like:
1597 // (bitcast i64 %x to <2 x i32>)
1598 !I->getType()->isVectorTy()) {
1599 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1600 break;
1601 }
1602
1603 const Value *V;
1604 // Handle bitcast from floating point to integer.
1605 if (match(V: I, P: m_ElementWiseBitCast(Op: m_Value(V))) &&
1606 V->getType()->isFPOrFPVectorTy()) {
1607 Type *FPType = V->getType()->getScalarType();
1608 KnownFPClass Result =
1609 computeKnownFPClass(V, DemandedElts, InterestedClasses: fcAllFlags, SQ: Q, Depth: Depth + 1);
1610
1611 Known = Result.toKnownBits(FltSemantics: FPType->getFltSemantics());
1612
1613 break;
1614 }
1615
1616 // Handle cast from vector integer type to scalar or vector integer.
1617 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy);
1618 if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
1619 !I->getType()->isIntOrIntVectorTy() ||
1620 isa<ScalableVectorType>(Val: I->getType()))
1621 break;
1622
1623 unsigned NumElts = DemandedElts.getBitWidth();
1624 bool IsLE = Q.DL.isLittleEndian();
1625 // Look through a cast from narrow vector elements to wider type.
1626 // Examples: v4i32 -> v2i64, v3i8 -> v24
1627 unsigned SubBitWidth = SrcVecTy->getScalarSizeInBits();
1628 if (BitWidth % SubBitWidth == 0) {
1629 // Known bits are automatically intersected across demanded elements of a
1630 // vector. So for example, if a bit is computed as known zero, it must be
1631 // zero across all demanded elements of the vector.
1632 //
1633 // For this bitcast, each demanded element of the output is sub-divided
1634 // across a set of smaller vector elements in the source vector. To get
1635 // the known bits for an entire element of the output, compute the known
1636 // bits for each sub-element sequentially. This is done by shifting the
1637 // one-set-bit demanded elements parameter across the sub-elements for
1638 // consecutive calls to computeKnownBits. We are using the demanded
1639 // elements parameter as a mask operator.
1640 //
1641 // The known bits of each sub-element are then inserted into place
1642 // (dependent on endian) to form the full result of known bits.
1643 unsigned SubScale = BitWidth / SubBitWidth;
1644 APInt SubDemandedElts = APInt::getZero(numBits: NumElts * SubScale);
1645 for (unsigned i = 0; i != NumElts; ++i) {
1646 if (DemandedElts[i])
1647 SubDemandedElts.setBit(i * SubScale);
1648 }
1649
1650 KnownBits KnownSrc(SubBitWidth);
1651 for (unsigned i = 0; i != SubScale; ++i) {
1652 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts.shl(shiftAmt: i), Known&: KnownSrc, Q,
1653 Depth: Depth + 1);
1654 unsigned ShiftElt = IsLE ? i : SubScale - 1 - i;
1655 Known.insertBits(SubBits: KnownSrc, BitPosition: ShiftElt * SubBitWidth);
1656 }
1657 }
1658 // Look through a cast from wider vector elements to narrow type.
1659 // Examples: v2i64 -> v4i32
1660 if (SubBitWidth % BitWidth == 0) {
1661 unsigned SubScale = SubBitWidth / BitWidth;
1662 KnownBits KnownSrc(SubBitWidth);
1663 APInt SubDemandedElts =
1664 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
1665 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts, Known&: KnownSrc, Q,
1666 Depth: Depth + 1);
1667
1668 Known.setAllConflict();
1669 for (unsigned i = 0; i != NumElts; ++i) {
1670 if (DemandedElts[i]) {
1671 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
1672 unsigned Offset = (Shifts % SubScale) * BitWidth;
1673 Known = Known.intersectWith(RHS: KnownSrc.extractBits(NumBits: BitWidth, BitPosition: Offset));
1674 if (Known.isUnknown())
1675 break;
1676 }
1677 }
1678 }
1679 break;
1680 }
1681 case Instruction::SExt: {
1682 // Compute the bits in the result that are not present in the input.
1683 unsigned SrcBitWidth = I->getOperand(i: 0)->getType()->getScalarSizeInBits();
1684
1685 Known = Known.trunc(BitWidth: SrcBitWidth);
1686 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1687 // If the sign bit of the input is known set or clear, then we know the
1688 // top bits of the result.
1689 Known = Known.sext(BitWidth);
1690 break;
1691 }
1692 case Instruction::Shl: {
1693 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1694 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1695 auto KF = [NUW, NSW](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1696 bool ShAmtNonZero) {
1697 return KnownBits::shl(LHS: KnownVal, RHS: KnownAmt, NUW, NSW, ShAmtNonZero);
1698 };
1699 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1700 KF);
1701 // Trailing zeros of a right-shifted constant never decrease.
1702 const APInt *C;
1703 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1704 Known.Zero.setLowBits(C->countr_zero());
1705
1706 // shl X, sub(Y, xor(ctlz(X, true), BitWidth-1)) shifts X so that its MSB
1707 // lands at bit Y, when BitWidth is a power of 2.
1708 const APInt *YC;
1709 Value *X = I->getOperand(i: 0);
1710 if (isPowerOf2_32(Value: BitWidth) &&
1711 match(V: I->getOperand(i: 1),
1712 P: m_Sub(L: m_APInt(Res&: YC), R: m_Xor(L: m_Ctlz(Op0: m_Specific(V: X), Op1: m_One()),
1713 R: m_SpecificInt(V: BitWidth - 1)))) &&
1714 YC->ult(RHS: BitWidth - 1)) {
1715 unsigned Y = YC->getZExtValue();
1716 Known.One.setBit(Y);
1717 Known.Zero.setBitsFrom(Y + 1);
1718 }
1719 break;
1720 }
1721 case Instruction::LShr: {
1722 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1723 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1724 bool ShAmtNonZero) {
1725 return KnownBits::lshr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1726 };
1727 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1728 KF);
1729 // Leading zeros of a left-shifted constant never decrease.
1730 const APInt *C;
1731 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1732 Known.Zero.setHighBits(C->countl_zero());
1733 break;
1734 }
1735 case Instruction::AShr: {
1736 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1737 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1738 bool ShAmtNonZero) {
1739 return KnownBits::ashr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1740 };
1741 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1742 KF);
1743 break;
1744 }
1745 case Instruction::Sub: {
1746 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1747 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1748 computeKnownBitsAddSub(Add: false, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1749 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1750 break;
1751 }
1752 case Instruction::Add: {
1753 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1754 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1755 computeKnownBitsAddSub(Add: true, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1756 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1757 break;
1758 }
1759 case Instruction::SRem:
1760 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1761 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1762 Known = KnownBits::srem(LHS: Known, RHS: Known2);
1763 break;
1764
1765 case Instruction::URem:
1766 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1767 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1768 Known = KnownBits::urem(LHS: Known, RHS: Known2);
1769 break;
1770 case Instruction::Alloca:
1771 Known.Zero.setLowBits(Log2(A: cast<AllocaInst>(Val: I)->getAlign()));
1772 break;
1773 case Instruction::GetElementPtr: {
1774 // Analyze all of the subscripts of this getelementptr instruction
1775 // to determine if we can prove known low zero bits.
1776 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1777 // Accumulate the constant indices in a separate variable
1778 // to minimize the number of calls to computeForAddSub.
1779 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: I->getType());
1780 APInt AccConstIndices(IndexWidth, 0);
1781
1782 auto AddIndexToKnown = [&](KnownBits IndexBits) {
1783 if (IndexWidth == BitWidth) {
1784 // Note that inbounds does *not* guarantee nsw for the addition, as only
1785 // the offset is signed, while the base address is unsigned.
1786 Known = KnownBits::add(LHS: Known, RHS: IndexBits);
1787 } else {
1788 // If the index width is smaller than the pointer width, only add the
1789 // value to the low bits.
1790 assert(IndexWidth < BitWidth &&
1791 "Index width can't be larger than pointer width");
1792 Known.insertBits(SubBits: KnownBits::add(LHS: Known.trunc(BitWidth: IndexWidth), RHS: IndexBits), BitPosition: 0);
1793 }
1794 };
1795
1796 gep_type_iterator GTI = gep_type_begin(GEP: I);
1797 for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
1798 // TrailZ can only become smaller, short-circuit if we hit zero.
1799 if (Known.isUnknown())
1800 break;
1801
1802 Value *Index = I->getOperand(i);
1803
1804 // Handle case when index is zero.
1805 Constant *CIndex = dyn_cast<Constant>(Val: Index);
1806 if (CIndex && CIndex->isNullValue())
1807 continue;
1808
1809 if (StructType *STy = GTI.getStructTypeOrNull()) {
1810 // Handle struct member offset arithmetic.
1811
1812 assert(CIndex &&
1813 "Access to structure field must be known at compile time");
1814
1815 if (CIndex->getType()->isVectorTy())
1816 Index = CIndex->getSplatValue();
1817
1818 unsigned Idx = cast<ConstantInt>(Val: Index)->getZExtValue();
1819 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
1820 uint64_t Offset = SL->getElementOffset(Idx);
1821 AccConstIndices += Offset;
1822 continue;
1823 }
1824
1825 // Handle array index arithmetic.
1826 Type *IndexedTy = GTI.getIndexedType();
1827 if (!IndexedTy->isSized()) {
1828 Known.resetAll();
1829 break;
1830 }
1831
1832 TypeSize Stride = GTI.getSequentialElementStride(DL: Q.DL);
1833 uint64_t StrideInBytes = Stride.getKnownMinValue();
1834 if (!Stride.isScalable()) {
1835 // Fast path for constant offset.
1836 if (auto *CI = dyn_cast<ConstantInt>(Val: Index)) {
1837 AccConstIndices +=
1838 CI->getValue().sextOrTrunc(width: IndexWidth) * StrideInBytes;
1839 continue;
1840 }
1841 }
1842
1843 KnownBits IndexBits =
1844 computeKnownBits(V: Index, Q, Depth: Depth + 1).sextOrTrunc(BitWidth: IndexWidth);
1845 KnownBits ScalingFactor(IndexWidth);
1846 // Multiply by current sizeof type.
1847 // &A[i] == A + i * sizeof(*A[i]).
1848 if (Stride.isScalable()) {
1849 // For scalable types the only thing we know about sizeof is
1850 // that this is a multiple of the minimum size.
1851 ScalingFactor.Zero.setLowBits(llvm::countr_zero(Val: StrideInBytes));
1852 } else {
1853 ScalingFactor =
1854 KnownBits::makeConstant(C: APInt(IndexWidth, StrideInBytes));
1855 }
1856 AddIndexToKnown(KnownBits::mul(LHS: IndexBits, RHS: ScalingFactor));
1857 }
1858 if (!Known.isUnknown() && !AccConstIndices.isZero())
1859 AddIndexToKnown(KnownBits::makeConstant(C: AccConstIndices));
1860 break;
1861 }
1862 case Instruction::PHI: {
1863 const PHINode *P = cast<PHINode>(Val: I);
1864 BinaryOperator *BO = nullptr;
1865 Value *Start = nullptr, *Step = nullptr;
1866 KnownBits &KnownStart = Known2;
1867 if (matchSimpleRecurrence(P, BO, Start, Step)) {
1868 // Handle the case of a simple two-predecessor recurrence PHI.
1869 // There's a lot more that could theoretically be done here, but
1870 // this is sufficient to catch some interesting cases.
1871 unsigned Opcode = BO->getOpcode();
1872
1873 switch (Opcode) {
1874 // If this is a shift recurrence, we know the bits being shifted in. We
1875 // can combine that with information about the start value of the
1876 // recurrence to conclude facts about the result. If this is a udiv
1877 // recurrence, we know that the result can never exceed either the
1878 // numerator or the start value, whichever is greater.
1879 case Instruction::LShr:
1880 case Instruction::AShr:
1881 case Instruction::Shl:
1882 case Instruction::UDiv:
1883 if (BO->getOperand(i_nocapture: 0) != I)
1884 break;
1885 [[fallthrough]];
1886
1887 // For a urem recurrence, the result can never exceed the start value. The
1888 // phi could either be the numerator or the denominator.
1889 case Instruction::URem: {
1890 // We have matched a recurrence of the form:
1891 // %iv = [R, %entry], [%iv.next, %backedge]
1892 // %iv.next = shift_op %iv, L
1893
1894 // Recurse with the phi context to avoid concern about whether facts
1895 // inferred hold at original context instruction. TODO: It may be
1896 // correct to use the original context. IF warranted, explore and
1897 // add sufficient tests to cover.
1898 SimplifyQuery RecQ = Q.getWithoutCondContext();
1899 RecQ.CtxI = P;
1900 computeKnownBits(V: Start, DemandedElts, Known&: KnownStart, Q: RecQ, Depth: Depth + 1);
1901 switch (Opcode) {
1902 case Instruction::Shl:
1903 // A shl recurrence will only increase the tailing zeros
1904 Known.Zero.setLowBits(KnownStart.countMinTrailingZeros());
1905 break;
1906 case Instruction::LShr:
1907 case Instruction::UDiv:
1908 case Instruction::URem:
1909 // lshr, udiv, and urem recurrences will preserve the leading zeros of
1910 // the start value.
1911 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
1912 break;
1913 case Instruction::AShr:
1914 // An ashr recurrence will extend the initial sign bit
1915 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
1916 Known.One.setHighBits(KnownStart.countMinLeadingOnes());
1917 break;
1918 }
1919 break;
1920 }
1921
1922 case Instruction::And: {
1923 // Bits that are zero in the start value stay zero, and bits that are
1924 // one in both the start value and the step stay one.
1925 KnownBits KnownStep(BitWidth);
1926 computeKnownBitsForRecurrenceOperands(P, Start, Step, DemandedElts,
1927 KnownStart, KnownStep, Q, Depth);
1928 Known.Zero |= KnownStart.Zero;
1929 Known.One |= KnownStart.One & KnownStep.One;
1930 break;
1931 }
1932
1933 case Instruction::Or: {
1934 // Bits that are zero in both the start value and the step stay zero,
1935 // and bits that are one in the start value stay one.
1936 KnownBits KnownStep(BitWidth);
1937 computeKnownBitsForRecurrenceOperands(P, Start, Step, DemandedElts,
1938 KnownStart, KnownStep, Q, Depth);
1939 Known.Zero |= KnownStart.Zero & KnownStep.Zero;
1940 Known.One |= KnownStart.One;
1941 break;
1942 }
1943
1944 // Check for operations that have the property that if
1945 // both their operands have low zero bits, the result
1946 // will have low zero bits.
1947 case Instruction::Add:
1948 case Instruction::Sub:
1949 case Instruction::Mul: {
1950 // Ok, we have a recurrence of the form {Start,op,Step}. Check for low
1951 // zero bits.
1952 KnownBits KnownStep(BitWidth);
1953 computeKnownBitsForRecurrenceOperands(P, Start, Step, DemandedElts,
1954 KnownStart, KnownStep, Q, Depth);
1955
1956 Known.Zero.setLowBits(std::min(a: KnownStart.countMinTrailingZeros(),
1957 b: KnownStep.countMinTrailingZeros()));
1958
1959 auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(Val: BO);
1960 if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(Op: OverflowOp))
1961 break;
1962
1963 switch (Opcode) {
1964 // If initial value of recurrence is nonnegative, and we are adding
1965 // a nonnegative number with nsw, the result can only be nonnegative
1966 // or poison value regardless of the number of times we execute the
1967 // add in phi recurrence. If initial value is negative and we are
1968 // adding a negative number with nsw, the result can only be
1969 // negative or poison value. Similar arguments apply to sub and mul.
1970 //
1971 // (add non-negative, non-negative) --> non-negative
1972 // (add negative, negative) --> negative
1973 case Instruction::Add: {
1974 if (KnownStart.isNonNegative() && KnownStep.isNonNegative())
1975 Known.makeNonNegative();
1976 else if (KnownStart.isNegative() && KnownStep.isNegative())
1977 Known.makeNegative();
1978 break;
1979 }
1980
1981 // (sub nsw non-negative, negative) --> non-negative
1982 // (sub nsw negative, non-negative) --> negative
1983 case Instruction::Sub: {
1984 if (BO->getOperand(i_nocapture: 0) != I)
1985 break;
1986 if (KnownStart.isNonNegative() && KnownStep.isNegative())
1987 Known.makeNonNegative();
1988 else if (KnownStart.isNegative() && KnownStep.isNonNegative())
1989 Known.makeNegative();
1990 break;
1991 }
1992
1993 // (mul nsw non-negative, non-negative) --> non-negative
1994 case Instruction::Mul:
1995 if (KnownStart.isNonNegative() && KnownStep.isNonNegative())
1996 Known.makeNonNegative();
1997 break;
1998
1999 default:
2000 break;
2001 }
2002 break;
2003 }
2004
2005 default:
2006 break;
2007 }
2008 } else {
2009 IntrinsicInst *II = nullptr;
2010 if (matchTwoInputRecurrence<IntrinsicInst>(PN: P, Inst&: II, Init&: Start, OtherOp&: Step)) {
2011 // %iv = [<Start>, %entry], [%iv.next, %backedge]
2012 //
2013 // %iv.next = <II>(%iv, <Step>)
2014 // or
2015 // %iv.next = <II>(<Step>, %iv)
2016 Intrinsic::ID IntrinsicID = II->getIntrinsicID();
2017 if (IntrinsicID == Intrinsic::umin || IntrinsicID == Intrinsic::umax) {
2018 KnownBits KnownStep(BitWidth);
2019 computeKnownBitsForRecurrenceOperands(
2020 P, Start, Step, DemandedElts, KnownStart, KnownStep, Q, Depth);
2021
2022 if (IntrinsicID == Intrinsic::umin) {
2023 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
2024 Known.One.setHighBits(std::min(a: KnownStart.countMinLeadingOnes(),
2025 b: KnownStep.countMinLeadingOnes()));
2026 } else {
2027 // umax
2028 Known.Zero.setHighBits(std::min(a: KnownStart.countMinLeadingZeros(),
2029 b: KnownStep.countMinLeadingZeros()));
2030 Known.One.setHighBits(KnownStart.countMinLeadingOnes());
2031 }
2032 }
2033 }
2034 }
2035
2036 // Unreachable blocks may have zero-operand PHI nodes.
2037 if (P->getNumIncomingValues() == 0)
2038 break;
2039
2040 // Otherwise take the unions of the known bit sets of the operands,
2041 // taking conservative care to avoid excessive recursion.
2042 if (Depth < MaxAnalysisRecursionDepth - 1 && Known.isUnknown()) {
2043 // Skip if every incoming value references to ourself.
2044 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
2045 break;
2046
2047 Known.setAllConflict();
2048 for (const Use &U : P->operands()) {
2049 Value *IncValue;
2050 const PHINode *CtxPhi;
2051 Instruction *CtxI;
2052 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CtxI, PhiOut: &CtxPhi);
2053 // Skip direct self references.
2054 if (IncValue == P)
2055 continue;
2056
2057 // Change the context instruction to the "edge" that flows into the
2058 // phi. This is important because that is where the value is actually
2059 // "evaluated" even though it is used later somewhere else. (see also
2060 // D69571).
2061 SimplifyQuery RecQ = Q.getWithoutCondContext().getWithInstruction(I: CtxI);
2062
2063 Known2 = KnownBits(BitWidth);
2064
2065 // Recurse, but cap the recursion to one level, because we don't
2066 // want to waste time spinning around in loops.
2067 // TODO: See if we can base recursion limiter on number of incoming phi
2068 // edges so we don't overly clamp analysis.
2069 computeKnownBits(V: IncValue, DemandedElts, Known&: Known2, Q: RecQ,
2070 Depth: MaxAnalysisRecursionDepth - 1);
2071
2072 // See if we can further use a conditional branch into the phi
2073 // to help us determine the range of the value.
2074 if (!Known2.isConstant()) {
2075 CmpPredicate Pred;
2076 const APInt *RHSC;
2077 BasicBlock *TrueSucc, *FalseSucc;
2078 // TODO: Use RHS Value and compute range from its known bits.
2079 if (match(V: RecQ.CtxI,
2080 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: IncValue), R: m_APInt(Res&: RHSC)),
2081 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
2082 // Check for cases of duplicate successors.
2083 if ((TrueSucc == CtxPhi->getParent()) !=
2084 (FalseSucc == CtxPhi->getParent())) {
2085 // If we're using the false successor, invert the predicate.
2086 if (FalseSucc == CtxPhi->getParent())
2087 Pred = CmpInst::getInversePredicate(pred: Pred);
2088 // Get the knownbits implied by the incoming phi condition.
2089 auto CR = ConstantRange::makeExactICmpRegion(Pred, Other: *RHSC);
2090 KnownBits KnownUnion = Known2.unionWith(RHS: CR.toKnownBits());
2091 // We can have conflicts here if we are analyzing deadcode (its
2092 // impossible for us reach this BB based the icmp).
2093 if (KnownUnion.hasConflict()) {
2094 // No reason to continue analyzing in a known dead region, so
2095 // just resetAll and break. This will cause us to also exit the
2096 // outer loop.
2097 Known.resetAll();
2098 break;
2099 }
2100 Known2 = KnownUnion;
2101 }
2102 }
2103 }
2104
2105 Known = Known.intersectWith(RHS: Known2);
2106 // If all bits have been ruled out, there's no need to check
2107 // more operands.
2108 if (Known.isUnknown())
2109 break;
2110 }
2111 }
2112 break;
2113 }
2114 case Instruction::Call:
2115 case Instruction::Invoke: {
2116 // If range metadata is attached to this call, set known bits from that,
2117 // and then intersect with known bits based on other properties of the
2118 // function.
2119 if (MDNode *MD =
2120 Q.IIQ.getMetadata(I: cast<Instruction>(Val: I), KindID: LLVMContext::MD_range))
2121 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
2122
2123 const auto *CB = cast<CallBase>(Val: I);
2124
2125 if (std::optional<ConstantRange> Range = CB->getRange())
2126 Known = Known.unionWith(RHS: Range->toKnownBits());
2127
2128 if (const Value *RV = CB->getReturnedArgOperand()) {
2129 if (RV->getType() == I->getType()) {
2130 computeKnownBits(V: RV, Known&: Known2, Q, Depth: Depth + 1);
2131 Known = Known.unionWith(RHS: Known2);
2132 // If the function doesn't return properly for all input values
2133 // (e.g. unreachable exits) then there might be conflicts between the
2134 // argument value and the range metadata. Simply discard the known bits
2135 // in case of conflicts.
2136 if (Known.hasConflict())
2137 Known.resetAll();
2138 }
2139 }
2140 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
2141 switch (II->getIntrinsicID()) {
2142 default:
2143 break;
2144 case Intrinsic::abs: {
2145 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2146 bool IntMinIsPoison = match(V: II->getArgOperand(i: 1), P: m_One());
2147 Known = Known.unionWith(RHS: Known2.abs(IntMinIsPoison));
2148 break;
2149 }
2150 case Intrinsic::bitreverse:
2151 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2152 Known = Known.unionWith(RHS: Known2.reverseBits());
2153 break;
2154 case Intrinsic::bswap:
2155 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2156 Known = Known.unionWith(RHS: Known2.byteSwap());
2157 break;
2158 case Intrinsic::ctlz: {
2159 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2160 // If we have a known 1, its position is our upper bound.
2161 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2162 // If this call is poison for 0 input, the result will be less than 2^n.
2163 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2164 PossibleLZ = std::min(a: PossibleLZ, b: BitWidth - 1);
2165 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
2166 Known.Zero.setBitsFrom(LowBits);
2167 break;
2168 }
2169 case Intrinsic::cttz: {
2170 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2171 // If we have a known 1, its position is our upper bound.
2172 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2173 // If this call is poison for 0 input, the result will be less than 2^n.
2174 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2175 PossibleTZ = std::min(a: PossibleTZ, b: BitWidth - 1);
2176 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
2177 Known.Zero.setBitsFrom(LowBits);
2178 break;
2179 }
2180 case Intrinsic::ctpop: {
2181 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2182 // We can bound the space the count needs. Also, bits known to be zero
2183 // can't contribute to the population.
2184 unsigned BitsPossiblySet = Known2.countMaxPopulation();
2185 unsigned LowBits = llvm::bit_width(Value: BitsPossiblySet);
2186 Known.Zero.setBitsFrom(LowBits);
2187 // TODO: we could bound KnownOne using the lower bound on the number
2188 // of bits which might be set provided by popcnt KnownOne2.
2189 break;
2190 }
2191 case Intrinsic::fshr:
2192 case Intrinsic::fshl: {
2193 const APInt *SA;
2194 if (!match(V: I->getOperand(i: 2), P: m_APInt(Res&: SA)))
2195 break;
2196
2197 KnownBits Known3(BitWidth);
2198 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2199 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known3, Q, Depth: Depth + 1);
2200 Known = II->getIntrinsicID() == Intrinsic::fshl
2201 ? KnownBits::fshl(LHS: Known2, RHS: Known3, Amt: *SA)
2202 : KnownBits::fshr(LHS: Known2, RHS: Known3, Amt: *SA);
2203 break;
2204 }
2205 case Intrinsic::clmul:
2206 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2207 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2208 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
2209 break;
2210 case Intrinsic::pext:
2211 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2212 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2213 Known = KnownBits::pext(Val: Known, Mask: Known2);
2214 break;
2215 case Intrinsic::pdep:
2216 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2217 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2218 Known = KnownBits::pdep(Val: Known, Mask: Known2);
2219 break;
2220 case Intrinsic::smulh:
2221 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2222 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2223 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
2224 break;
2225 case Intrinsic::umulh:
2226 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2227 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2228 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
2229 break;
2230 case Intrinsic::uadd_sat:
2231 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2232 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2233 Known = KnownBits::uadd_sat(LHS: Known, RHS: Known2);
2234 break;
2235 case Intrinsic::usub_sat:
2236 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2237 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2238 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
2239 break;
2240 case Intrinsic::sadd_sat:
2241 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2242 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2243 Known = KnownBits::sadd_sat(LHS: Known, RHS: Known2);
2244 break;
2245 case Intrinsic::ssub_sat:
2246 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2247 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2248 Known = KnownBits::ssub_sat(LHS: Known, RHS: Known2);
2249 break;
2250 // Vec reverse preserves bits from input vec.
2251 case Intrinsic::vector_reverse:
2252 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: DemandedElts.reverseBits(), Known, Q,
2253 Depth: Depth + 1);
2254 break;
2255 // for min/max/and/or reduce, any bit common to each element in the
2256 // input vec is set in the output.
2257 case Intrinsic::vector_reduce_and:
2258 case Intrinsic::vector_reduce_or:
2259 case Intrinsic::vector_reduce_umax:
2260 case Intrinsic::vector_reduce_umin:
2261 case Intrinsic::vector_reduce_smax:
2262 case Intrinsic::vector_reduce_smin:
2263 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2264 break;
2265 case Intrinsic::vector_reduce_xor: {
2266 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2267 // The zeros common to all vecs are zero in the output.
2268 // If the number of elements is odd, then the common ones remain. If the
2269 // number of elements is even, then the common ones becomes zeros.
2270 auto *VecTy = cast<VectorType>(Val: I->getOperand(i: 0)->getType());
2271 // Even, so the ones become zeros.
2272 bool EvenCnt = VecTy->getElementCount().isKnownEven();
2273 if (EvenCnt)
2274 Known.Zero |= Known.One;
2275 // Maybe even element count so need to clear ones.
2276 if (VecTy->isScalableTy() || EvenCnt)
2277 Known.One.clearAllBits();
2278 break;
2279 }
2280 case Intrinsic::vector_reduce_add: {
2281 auto *VecTy = dyn_cast<FixedVectorType>(Val: I->getOperand(i: 0)->getType());
2282 if (!VecTy)
2283 break;
2284 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2285 Known = Known.reduceAdd(NumElts: VecTy->getNumElements());
2286 break;
2287 }
2288 case Intrinsic::umin:
2289 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2290 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2291 Known = KnownBits::umin(LHS: Known, RHS: Known2);
2292 break;
2293 case Intrinsic::umax:
2294 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2295 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2296 Known = KnownBits::umax(LHS: Known, RHS: Known2);
2297 break;
2298 case Intrinsic::smin:
2299 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2300 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2301 Known = KnownBits::smin(LHS: Known, RHS: Known2);
2302 unionWithMinMaxIntrinsicClamp(II, Known);
2303 break;
2304 case Intrinsic::smax:
2305 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2306 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2307 Known = KnownBits::smax(LHS: Known, RHS: Known2);
2308 unionWithMinMaxIntrinsicClamp(II, Known);
2309 break;
2310 case Intrinsic::ptrmask: {
2311 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2312
2313 const Value *Mask = I->getOperand(i: 1);
2314 Known2 = KnownBits(Mask->getType()->getScalarSizeInBits());
2315 computeKnownBits(V: Mask, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2316 // TODO: 1-extend would be more precise.
2317 Known &= Known2.anyextOrTrunc(BitWidth);
2318 break;
2319 }
2320 case Intrinsic::x86_sse42_crc32_64_64:
2321 Known.Zero.setBitsFrom(32);
2322 break;
2323 case Intrinsic::x86_ssse3_phadd_d_128:
2324 case Intrinsic::x86_ssse3_phadd_w_128:
2325 case Intrinsic::x86_avx2_phadd_d:
2326 case Intrinsic::x86_avx2_phadd_w: {
2327 Known = computeKnownBitsForHorizontalOperation(
2328 I, DemandedElts, Q, Depth,
2329 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2330 return KnownBits::add(LHS: KnownLHS, RHS: KnownRHS);
2331 });
2332 break;
2333 }
2334 case Intrinsic::x86_ssse3_phadd_sw_128:
2335 case Intrinsic::x86_avx2_phadd_sw: {
2336 Known = computeKnownBitsForHorizontalOperation(
2337 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::sadd_sat);
2338 break;
2339 }
2340 case Intrinsic::x86_ssse3_phsub_d_128:
2341 case Intrinsic::x86_ssse3_phsub_w_128:
2342 case Intrinsic::x86_avx2_phsub_d:
2343 case Intrinsic::x86_avx2_phsub_w: {
2344 Known = computeKnownBitsForHorizontalOperation(
2345 I, DemandedElts, Q, Depth,
2346 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2347 return KnownBits::sub(LHS: KnownLHS, RHS: KnownRHS);
2348 });
2349 break;
2350 }
2351 case Intrinsic::x86_ssse3_phsub_sw_128:
2352 case Intrinsic::x86_avx2_phsub_sw: {
2353 Known = computeKnownBitsForHorizontalOperation(
2354 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::ssub_sat);
2355 break;
2356 }
2357 case Intrinsic::riscv_vsetvli:
2358 case Intrinsic::riscv_vsetvlimax: {
2359 bool HasAVL = II->getIntrinsicID() == Intrinsic::riscv_vsetvli;
2360 const ConstantRange Range = getVScaleRange(F: II->getFunction(), BitWidth);
2361 uint64_t SEW = RISCVVType::decodeVSEW(
2362 VSEW: cast<ConstantInt>(Val: II->getArgOperand(i: HasAVL))->getZExtValue());
2363 RISCVVType::VLMUL VLMUL = static_cast<RISCVVType::VLMUL>(
2364 cast<ConstantInt>(Val: II->getArgOperand(i: 1 + HasAVL))->getZExtValue());
2365 uint64_t MaxVLEN =
2366 Range.getUnsignedMax().getZExtValue() * RISCV::RVVBitsPerBlock;
2367 uint64_t MaxVL = MaxVLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
2368
2369 // Result of vsetvli must be not larger than AVL.
2370 if (HasAVL)
2371 if (auto *CI = dyn_cast<ConstantInt>(Val: II->getArgOperand(i: 0)))
2372 MaxVL = std::min(a: MaxVL, b: CI->getZExtValue());
2373
2374 unsigned KnownZeroFirstBit = Log2_32(Value: MaxVL) + 1;
2375 if (BitWidth > KnownZeroFirstBit)
2376 Known.Zero.setBitsFrom(KnownZeroFirstBit);
2377 break;
2378 }
2379 case Intrinsic::amdgcn_mbcnt_hi:
2380 case Intrinsic::amdgcn_mbcnt_lo: {
2381 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
2382 // most 31 + src1.
2383 KnownBits MbcntKnown(BitWidth);
2384 MbcntKnown.Zero.setBitsFrom(
2385 II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo ? 6 : 5);
2386 computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Q, Depth: Depth + 1);
2387 Known = Known.unionWith(RHS: KnownBits::add(LHS: MbcntKnown, RHS: Known2));
2388 break;
2389 }
2390 case Intrinsic::vscale: {
2391 if (!II->getParent() || !II->getFunction())
2392 break;
2393
2394 Known = getVScaleRange(F: II->getFunction(), BitWidth).toKnownBits();
2395 break;
2396 }
2397 case Intrinsic::stepvector: {
2398 auto *VecTy = cast<VectorType>(Val: II->getType());
2399 unsigned MinNumElts = VecTy->getElementCount().getKnownMinValue();
2400 if (!isUIntN(N: BitWidth, x: MinNumElts))
2401 break;
2402
2403 bool Overflow = false;
2404 APInt MaxNumElts(BitWidth, MinNumElts);
2405 if (VecTy->isScalableTy()) {
2406 if (!II->getParent() || !II->getFunction())
2407 break;
2408 MaxNumElts = getVScaleRange(F: II->getFunction(), BitWidth)
2409 .getUnsignedMax()
2410 .umul_ov(RHS: MaxNumElts, Overflow);
2411 }
2412
2413 // Give up if the lane count could wrap. Stepvector truncates lane
2414 // indices that do not fit in the element type.
2415 if (Overflow)
2416 break;
2417
2418 Known.Zero.setHighBits((MaxNumElts - 1).countl_zero());
2419 break;
2420 }
2421 }
2422 }
2423 break;
2424 }
2425 case Instruction::ShuffleVector: {
2426 if (auto *Splat = getSplatValue(V: I)) {
2427 computeKnownBits(V: Splat, Known, Q, Depth: Depth + 1);
2428 break;
2429 }
2430
2431 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
2432 // FIXME: Do we need to handle ConstantExpr involving shufflevectors?
2433 if (!Shuf) {
2434 Known.resetAll();
2435 return;
2436 }
2437 // For undef elements, we don't know anything about the common state of
2438 // the shuffle result.
2439 APInt DemandedLHS, DemandedRHS;
2440 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS)) {
2441 Known.resetAll();
2442 return;
2443 }
2444 Known.setAllConflict();
2445 if (!!DemandedLHS) {
2446 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
2447 computeKnownBits(V: LHS, DemandedElts: DemandedLHS, Known, Q, Depth: Depth + 1);
2448 // If we don't know any bits, early out.
2449 if (Known.isUnknown())
2450 break;
2451 }
2452 if (!!DemandedRHS) {
2453 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
2454 computeKnownBits(V: RHS, DemandedElts: DemandedRHS, Known&: Known2, Q, Depth: Depth + 1);
2455 Known = Known.intersectWith(RHS: Known2);
2456 }
2457 break;
2458 }
2459 case Instruction::InsertElement: {
2460 if (isa<ScalableVectorType>(Val: I->getType())) {
2461 Known.resetAll();
2462 return;
2463 }
2464 const Value *Vec = I->getOperand(i: 0);
2465 const Value *Elt = I->getOperand(i: 1);
2466 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
2467 unsigned NumElts = DemandedElts.getBitWidth();
2468 APInt DemandedVecElts = DemandedElts;
2469 bool NeedsElt = true;
2470 // If we know the index we are inserting too, clear it from Vec check.
2471 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
2472 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
2473 NeedsElt = DemandedElts[CIdx->getZExtValue()];
2474 }
2475
2476 Known.setAllConflict();
2477 if (NeedsElt) {
2478 computeKnownBits(V: Elt, Known, Q, Depth: Depth + 1);
2479 // If we don't know any bits, early out.
2480 if (Known.isUnknown())
2481 break;
2482 }
2483
2484 if (!DemandedVecElts.isZero()) {
2485 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known&: Known2, Q, Depth: Depth + 1);
2486 Known = Known.intersectWith(RHS: Known2);
2487 }
2488 break;
2489 }
2490 case Instruction::ExtractElement: {
2491 // Look through extract element. If the index is non-constant or
2492 // out-of-range demand all elements, otherwise just the extracted element.
2493 const Value *Vec = I->getOperand(i: 0);
2494 const Value *Idx = I->getOperand(i: 1);
2495 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
2496 if (isa<ScalableVectorType>(Val: Vec->getType())) {
2497 // FIXME: there's probably *something* we can do with scalable vectors
2498 Known.resetAll();
2499 break;
2500 }
2501 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2502 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
2503 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
2504 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
2505 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known, Q, Depth: Depth + 1);
2506 break;
2507 }
2508 case Instruction::ExtractValue:
2509 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I->getOperand(i: 0))) {
2510 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: I);
2511 if (EVI->getNumIndices() != 1) break;
2512 if (EVI->getIndices()[0] == 0) {
2513 switch (II->getIntrinsicID()) {
2514 default: break;
2515 case Intrinsic::uadd_with_overflow:
2516 case Intrinsic::sadd_with_overflow:
2517 computeKnownBitsAddSub(
2518 Add: true, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2519 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2520 break;
2521 case Intrinsic::usub_with_overflow:
2522 case Intrinsic::ssub_with_overflow:
2523 computeKnownBitsAddSub(
2524 Add: false, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2525 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2526 break;
2527 case Intrinsic::umul_with_overflow:
2528 case Intrinsic::smul_with_overflow:
2529 computeKnownBitsMul(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), NSW: false,
2530 NUW: false, DemandedElts, Known, Known2, Q, Depth);
2531 break;
2532 }
2533 }
2534 }
2535 break;
2536 case Instruction::Freeze:
2537 if (isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT,
2538 Depth: Depth + 1))
2539 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2540 break;
2541 }
2542}
2543
2544/// Determine which bits of V are known to be either zero or one and return
2545/// them.
2546KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
2547 const SimplifyQuery &Q, unsigned Depth) {
2548 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2549 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
2550 return Known;
2551}
2552
2553/// Determine which bits of V are known to be either zero or one and return
2554/// them.
2555KnownBits llvm::computeKnownBits(const Value *V, const SimplifyQuery &Q,
2556 unsigned Depth) {
2557 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2558 computeKnownBits(V, Known, Q, Depth);
2559 return Known;
2560}
2561
2562/// Determine which bits of V are known to be either zero or one and return
2563/// them in the Known bit set.
2564///
2565/// NOTE: we cannot consider 'undef' to be "IsZero" here. The problem is that
2566/// we cannot optimize based on the assumption that it is zero without changing
2567/// it to be an explicit zero. If we don't change it to zero, other code could
2568/// optimized based on the contradictory assumption that it is non-zero.
2569/// Because instcombine aggressively folds operations with undef args anyway,
2570/// this won't lose us code quality.
2571///
2572/// This function is defined on values with integer type, values with pointer
2573/// type, and vectors of integers. In the case
2574/// where V is a vector, known zero, and known one values are the
2575/// same width as the vector element, and the bit is set only if it is true
2576/// for all of the demanded elements in the vector specified by DemandedElts.
2577void computeKnownBits(const Value *V, const APInt &DemandedElts,
2578 KnownBits &Known, const SimplifyQuery &Q,
2579 unsigned Depth) {
2580 if (!DemandedElts) {
2581 // No demanded elts, better to assume we don't know anything.
2582 Known.resetAll();
2583 return;
2584 }
2585
2586 assert(V && "No Value?");
2587 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2588
2589#ifndef NDEBUG
2590 Type *Ty = V->getType();
2591 unsigned BitWidth = Known.getBitWidth();
2592
2593 assert((Ty->isIntOrIntVectorTy(BitWidth) || Ty->isPtrOrPtrVectorTy()) &&
2594 "Not integer or pointer type!");
2595
2596 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
2597 assert(
2598 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
2599 "DemandedElt width should equal the fixed vector number of elements");
2600 } else {
2601 assert(DemandedElts == APInt(1, 1) &&
2602 "DemandedElt width should be 1 for scalars or scalable vectors");
2603 }
2604
2605 Type *ScalarTy = Ty->getScalarType();
2606 if (ScalarTy->isPointerTy()) {
2607 assert(BitWidth == Q.DL.getPointerTypeSizeInBits(ScalarTy) &&
2608 "V and Known should have same BitWidth");
2609 } else {
2610 assert(BitWidth == Q.DL.getTypeSizeInBits(ScalarTy) &&
2611 "V and Known should have same BitWidth");
2612 }
2613#endif
2614
2615 const APInt *C;
2616 if (match(V, P: m_APInt(Res&: C))) {
2617 // We know all of the bits for a scalar constant or a splat vector constant!
2618 Known = KnownBits::makeConstant(C: *C);
2619 return;
2620 }
2621 // Null and aggregate-zero are all-zeros.
2622 if (isa<ConstantPointerNull>(Val: V) || isa<ConstantAggregateZero>(Val: V)) {
2623 Known.setAllZero();
2624 return;
2625 }
2626 // Handle a constant vector by taking the intersection of the known bits of
2627 // each element.
2628 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: V)) {
2629 assert(!isa<ScalableVectorType>(V->getType()));
2630 // We know that CDV must be a vector of integers. Take the intersection of
2631 // each element.
2632 Known.setAllConflict();
2633 for (unsigned i = 0, e = CDV->getNumElements(); i != e; ++i) {
2634 if (!DemandedElts[i])
2635 continue;
2636 APInt Elt = CDV->getElementAsAPInt(i);
2637 Known.Zero &= ~Elt;
2638 Known.One &= Elt;
2639 }
2640 if (Known.hasConflict())
2641 Known.resetAll();
2642 return;
2643 }
2644
2645 if (const auto *CV = dyn_cast<ConstantVector>(Val: V)) {
2646 assert(!isa<ScalableVectorType>(V->getType()));
2647 // We know that CV must be a vector of integers. Take the intersection of
2648 // each element.
2649 Known.setAllConflict();
2650 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
2651 if (!DemandedElts[i])
2652 continue;
2653 Constant *Element = CV->getAggregateElement(Elt: i);
2654 if (isa<PoisonValue>(Val: Element))
2655 continue;
2656 auto *ElementCI = dyn_cast_or_null<ConstantInt>(Val: Element);
2657 if (!ElementCI) {
2658 Known.resetAll();
2659 return;
2660 }
2661 const APInt &Elt = ElementCI->getValue();
2662 Known.Zero &= ~Elt;
2663 Known.One &= Elt;
2664 }
2665 if (Known.hasConflict())
2666 Known.resetAll();
2667 return;
2668 }
2669
2670 // Start out not knowing anything.
2671 Known.resetAll();
2672
2673 // We can't imply anything about undefs.
2674 if (isa<UndefValue>(Val: V))
2675 return;
2676
2677 // There's no point in looking through other users of ConstantData for
2678 // assumptions. Confirm that we've handled them all.
2679 assert(!isa<ConstantData>(V) && "Unhandled constant data!");
2680
2681 if (const auto *A = dyn_cast<Argument>(Val: V))
2682 if (std::optional<ConstantRange> Range = A->getRange())
2683 Known = Range->toKnownBits();
2684
2685 // All recursive calls that increase depth must come after this.
2686 if (Depth == MaxAnalysisRecursionDepth)
2687 return;
2688
2689 // A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
2690 // the bits of its aliasee.
2691 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: V)) {
2692 if (!GA->isInterposable())
2693 computeKnownBits(V: GA->getAliasee(), Known, Q, Depth: Depth + 1);
2694 return;
2695 }
2696
2697 if (const Operator *I = dyn_cast<Operator>(Val: V))
2698 computeKnownBitsFromOperator(I, DemandedElts, Known, Q, Depth);
2699 else if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
2700 if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
2701 Known = CR->toKnownBits();
2702 }
2703
2704 // Aligned pointers have trailing zeros - refine Known.Zero set
2705 if (isa<PointerType>(Val: V->getType())) {
2706 Align Alignment = V->getPointerAlignment(DL: Q.DL);
2707 Known.Zero.setLowBits(Log2(A: Alignment));
2708 }
2709
2710 // computeKnownBitsFromContext strictly refines Known.
2711 // Therefore, we run them after computeKnownBitsFromOperator.
2712
2713 // Check whether we can determine known bits from context such as assumes.
2714 computeKnownBitsFromContext(V, Known, Q, Depth);
2715}
2716
2717/// Try to detect a recurrence that the value of the induction variable is
2718/// always a power of two (or zero).
2719static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
2720 SimplifyQuery &Q, unsigned Depth) {
2721 BinaryOperator *BO = nullptr;
2722 Value *Start = nullptr, *Step = nullptr;
2723 if (!matchSimpleRecurrence(P: PN, BO, Start, Step))
2724 return false;
2725
2726 // Initial value must be a power of two.
2727 for (const Use &U : PN->operands()) {
2728 if (U.get() == Start) {
2729 // Initial value comes from a different BB, need to adjust context
2730 // instruction for analysis.
2731 Q.CtxI = PN->getIncomingBlock(U)->getTerminator();
2732 if (!isKnownToBeAPowerOfTwo(V: Start, OrZero, Q, Depth))
2733 return false;
2734 }
2735 }
2736
2737 // Except for Mul, the induction variable must be on the left side of the
2738 // increment expression, otherwise its value can be arbitrary.
2739 if (BO->getOpcode() != Instruction::Mul && BO->getOperand(i_nocapture: 1) != Step)
2740 return false;
2741
2742 Q.CtxI = BO->getParent()->getTerminator();
2743 switch (BO->getOpcode()) {
2744 case Instruction::Mul:
2745 // Power of two is closed under multiplication.
2746 return (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) ||
2747 Q.IIQ.hasNoSignedWrap(Op: BO)) &&
2748 isKnownToBeAPowerOfTwo(V: Step, OrZero, Q, Depth);
2749 case Instruction::SDiv:
2750 // Start value must not be signmask for signed division, so simply being a
2751 // power of two is not sufficient, and it has to be a constant.
2752 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2753 return false;
2754 [[fallthrough]];
2755 case Instruction::UDiv:
2756 // Divisor must be a power of two.
2757 // If OrZero is false, cannot guarantee induction variable is non-zero after
2758 // division, same for Shr, unless it is exact division.
2759 return (OrZero || Q.IIQ.isExact(Op: BO)) &&
2760 isKnownToBeAPowerOfTwo(V: Step, OrZero: false, Q, Depth);
2761 case Instruction::Shl:
2762 return OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO);
2763 case Instruction::AShr:
2764 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2765 return false;
2766 [[fallthrough]];
2767 case Instruction::LShr:
2768 return OrZero || Q.IIQ.isExact(Op: BO);
2769 default:
2770 return false;
2771 }
2772}
2773
2774/// Return true if we can infer that \p V is known to be a power of 2 from
2775/// dominating condition \p Cond (e.g., ctpop(V) == 1).
2776static bool isImpliedToBeAPowerOfTwoFromCond(const Value *V, bool OrZero,
2777 const Value *Cond,
2778 bool CondIsTrue) {
2779 CmpPredicate Pred;
2780 const APInt *RHSC;
2781 if (!match(V: Cond, P: m_ICmp(Pred, L: m_Ctpop(Op0: m_Specific(V)), R: m_APInt(Res&: RHSC))))
2782 return false;
2783 if (!CondIsTrue)
2784 Pred = ICmpInst::getInversePredicate(pred: Pred);
2785 // ctpop(V) u< 2
2786 if (OrZero && Pred == ICmpInst::ICMP_ULT && *RHSC == 2)
2787 return true;
2788 // ctpop(V) == 1
2789 return Pred == ICmpInst::ICMP_EQ && *RHSC == 1;
2790}
2791
2792/// Return true if the given value is known to have exactly one
2793/// bit set when defined. For vectors return true if every element is known to
2794/// be a power of two when defined. Supports values with integer or pointer
2795/// types and vectors of integers.
2796bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero,
2797 const SimplifyQuery &Q, unsigned Depth) {
2798 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2799
2800 if (isa<Constant>(Val: V))
2801 return OrZero ? match(V, P: m_Power2OrZero()) : match(V, P: m_Power2());
2802
2803 // i1 is by definition a power of 2 or zero.
2804 if (OrZero && V->getType()->getScalarSizeInBits() == 1)
2805 return true;
2806
2807 // Try to infer from assumptions.
2808 if (Q.AC && Q.CtxI) {
2809 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
2810 if (!AssumeVH)
2811 continue;
2812 CallInst *I = cast<CallInst>(Val&: AssumeVH);
2813 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond: I->getArgOperand(i: 0),
2814 /*CondIsTrue=*/true) &&
2815 isValidAssumeForContext(I, Q))
2816 return true;
2817 }
2818 }
2819
2820 // Handle dominating conditions.
2821 if (Q.DC && Q.CtxI && Q.DT) {
2822 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
2823 Value *Cond = BI->getCondition();
2824
2825 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
2826 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2827 /*CondIsTrue=*/true) &&
2828 Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()))
2829 return true;
2830
2831 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
2832 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2833 /*CondIsTrue=*/false) &&
2834 Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()))
2835 return true;
2836 }
2837 }
2838
2839 auto *I = dyn_cast<Instruction>(Val: V);
2840 if (!I)
2841 return false;
2842
2843 if (Q.CtxI && match(V, P: m_VScale())) {
2844 const Function *F = Q.CtxI->getFunction();
2845 // The vscale_range indicates vscale is a power-of-two.
2846 return F->hasFnAttribute(Kind: Attribute::VScaleRange);
2847 }
2848
2849 // 1 << X is clearly a power of two if the one is not shifted off the end. If
2850 // it is shifted off the end then the result is undefined.
2851 if (match(V: I, P: m_Shl(L: m_One(), R: m_Value())))
2852 return true;
2853
2854 // (signmask) >>l X is clearly a power of two if the one is not shifted off
2855 // the bottom. If it is shifted off the bottom then the result is undefined.
2856 if (match(V: I, P: m_LShr(L: m_SignMask(), R: m_Value())))
2857 return true;
2858
2859 // The remaining tests are all recursive, so bail out if we hit the limit.
2860 if (Depth++ == MaxAnalysisRecursionDepth)
2861 return false;
2862
2863 switch (I->getOpcode()) {
2864 case Instruction::ZExt:
2865 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2866 case Instruction::Trunc:
2867 return OrZero && isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2868 case Instruction::Shl:
2869 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: I) || Q.IIQ.hasNoSignedWrap(Op: I))
2870 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2871 return false;
2872 case Instruction::LShr:
2873 if (OrZero || Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2874 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2875 return false;
2876 case Instruction::UDiv:
2877 if (Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2878 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2879 return false;
2880 case Instruction::Mul:
2881 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2882 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth) &&
2883 (OrZero || isKnownNonZero(V: I, Q, Depth));
2884 case Instruction::And:
2885 // A power of two and'd with anything is a power of two or zero.
2886 if (OrZero &&
2887 (isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), /*OrZero*/ true, Q, Depth) ||
2888 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), /*OrZero*/ true, Q, Depth)))
2889 return true;
2890 // X & (-X) is always a power of two or zero.
2891 if (match(V: I->getOperand(i: 0), P: m_Neg(V: m_Specific(V: I->getOperand(i: 1)))) ||
2892 match(V: I->getOperand(i: 1), P: m_Neg(V: m_Specific(V: I->getOperand(i: 0)))))
2893 return OrZero || isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
2894 return false;
2895 case Instruction::Add: {
2896 // Adding a power-of-two or zero to the same power-of-two or zero yields
2897 // either the original power-of-two, a larger power-of-two or zero.
2898 const OverflowingBinaryOperator *VOBO = cast<OverflowingBinaryOperator>(Val: V);
2899 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO) ||
2900 Q.IIQ.hasNoSignedWrap(Op: VOBO)) {
2901 if (match(V: I->getOperand(i: 0),
2902 P: m_c_And(L: m_Specific(V: I->getOperand(i: 1)), R: m_Value())) &&
2903 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth))
2904 return true;
2905 if (match(V: I->getOperand(i: 1),
2906 P: m_c_And(L: m_Specific(V: I->getOperand(i: 0)), R: m_Value())) &&
2907 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth))
2908 return true;
2909
2910 unsigned BitWidth = V->getType()->getScalarSizeInBits();
2911 KnownBits LHSBits(BitWidth);
2912 computeKnownBits(V: I->getOperand(i: 0), Known&: LHSBits, Q, Depth);
2913
2914 KnownBits RHSBits(BitWidth);
2915 computeKnownBits(V: I->getOperand(i: 1), Known&: RHSBits, Q, Depth);
2916 // If i8 V is a power of two or zero:
2917 // ZeroBits: 1 1 1 0 1 1 1 1
2918 // ~ZeroBits: 0 0 0 1 0 0 0 0
2919 if ((~(LHSBits.Zero & RHSBits.Zero)).isPowerOf2())
2920 // If OrZero isn't set, we cannot give back a zero result.
2921 // Make sure either the LHS or RHS has a bit set.
2922 if (OrZero || RHSBits.One.getBoolValue() || LHSBits.One.getBoolValue())
2923 return true;
2924 }
2925
2926 // LShr(UINT_MAX, Y) + 1 is a power of two (if add is nuw) or zero.
2927 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO))
2928 if (match(V: I, P: m_Add(L: m_LShr(L: m_AllOnes(), R: m_Value()), R: m_One())))
2929 return true;
2930 return false;
2931 }
2932 case Instruction::Select:
2933 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2934 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 2), OrZero, Q, Depth);
2935 case Instruction::PHI: {
2936 // A PHI node is power of two if all incoming values are power of two, or if
2937 // it is an induction variable where in each step its value is a power of
2938 // two.
2939 auto *PN = cast<PHINode>(Val: I);
2940 SimplifyQuery RecQ = Q.getWithoutCondContext();
2941
2942 // Check if it is an induction variable and always power of two.
2943 if (isPowerOfTwoRecurrence(PN, OrZero, Q&: RecQ, Depth))
2944 return true;
2945
2946 // Recursively check all incoming values. Limit recursion to 2 levels, so
2947 // that search complexity is limited to number of operands^2.
2948 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
2949 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
2950 // Value is power of 2 if it is coming from PHI node itself by induction.
2951 if (U.get() == PN)
2952 return true;
2953
2954 // Change the context instruction to the incoming block where it is
2955 // evaluated.
2956 RecQ.CtxI = PN->getIncomingBlock(U)->getTerminator();
2957 return isKnownToBeAPowerOfTwo(V: U.get(), OrZero, Q: RecQ, Depth: NewDepth);
2958 });
2959 }
2960 case Instruction::Invoke:
2961 case Instruction::Call: {
2962 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2963 switch (II->getIntrinsicID()) {
2964 case Intrinsic::umax:
2965 case Intrinsic::smax:
2966 case Intrinsic::umin:
2967 case Intrinsic::smin:
2968 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 1), OrZero, Q, Depth) &&
2969 isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2970 // bswap/bitreverse just move around bits, but don't change any 1s/0s
2971 // thus dont change pow2/non-pow2 status.
2972 case Intrinsic::bitreverse:
2973 case Intrinsic::bswap:
2974 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2975 case Intrinsic::fshr:
2976 case Intrinsic::fshl:
2977 // If Op0 == Op1, this is a rotate. is_pow2(rotate(x, y)) == is_pow2(x)
2978 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
2979 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2980 break;
2981 case Intrinsic::riscv_vsetvlimax:
2982 // VLMAX is VLEN * LMUL / SEW, which is always a non-zero power of two
2983 // for any valid vtype, so it is a power of two regardless of OrZero.
2984 return true;
2985 case Intrinsic::read_register:
2986 case Intrinsic::read_volatile_register: {
2987 // The RISC-V vlenb CSR holds VLEN/8, which is always a non-zero power
2988 // of two, so it is a power of two regardless of OrZero.
2989 const Module *M = II->getModule();
2990 if (!M || !M->getTargetTriple().isRISCV())
2991 break;
2992 return isReadVLENB(II: *II);
2993 }
2994 default:
2995 break;
2996 }
2997 }
2998 return false;
2999 }
3000 default:
3001 return false;
3002 }
3003}
3004
3005/// Test whether a GEP's result is known to be non-null.
3006///
3007/// Uses properties inherent in a GEP to try to determine whether it is known
3008/// to be non-null.
3009///
3010/// Currently this routine does not support vector GEPs.
3011static bool isGEPKnownNonNull(const GEPOperator *GEP, const SimplifyQuery &Q,
3012 unsigned Depth) {
3013 const Function *F = nullptr;
3014 if (const Instruction *I = dyn_cast<Instruction>(Val: GEP))
3015 F = I->getFunction();
3016
3017 // If the gep is nuw or inbounds with invalid null pointer, then the GEP
3018 // may be null iff the base pointer is null and the offset is zero.
3019 if (!GEP->hasNoUnsignedWrap() &&
3020 !(GEP->isInBounds() &&
3021 !NullPointerIsDefined(F, AS: GEP->getPointerAddressSpace())))
3022 return false;
3023
3024 // FIXME: Support vector-GEPs.
3025 assert(GEP->getType()->isPointerTy() && "We only support plain pointer GEP");
3026
3027 // If the base pointer is non-null, we cannot walk to a null address with an
3028 // inbounds GEP in address space zero.
3029 if (isKnownNonZero(V: GEP->getPointerOperand(), Q, Depth))
3030 return true;
3031
3032 // Walk the GEP operands and see if any operand introduces a non-zero offset.
3033 // If so, then the GEP cannot produce a null pointer, as doing so would
3034 // inherently violate the inbounds contract within address space zero.
3035 for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
3036 GTI != GTE; ++GTI) {
3037 // Struct types are easy -- they must always be indexed by a constant.
3038 if (StructType *STy = GTI.getStructTypeOrNull()) {
3039 ConstantInt *OpC = cast<ConstantInt>(Val: GTI.getOperand());
3040 unsigned ElementIdx = OpC->getZExtValue();
3041 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
3042 uint64_t ElementOffset = SL->getElementOffset(Idx: ElementIdx);
3043 if (ElementOffset > 0)
3044 return true;
3045 continue;
3046 }
3047
3048 // If we have a zero-sized type, the index doesn't matter. Keep looping.
3049 if (GTI.getSequentialElementStride(DL: Q.DL).isZero())
3050 continue;
3051
3052 // Fast path the constant operand case both for efficiency and so we don't
3053 // increment Depth when just zipping down an all-constant GEP.
3054 if (ConstantInt *OpC = dyn_cast<ConstantInt>(Val: GTI.getOperand())) {
3055 if (!OpC->isZero())
3056 return true;
3057 continue;
3058 }
3059
3060 // We post-increment Depth here because while isKnownNonZero increments it
3061 // as well, when we pop back up that increment won't persist. We don't want
3062 // to recurse 10k times just because we have 10k GEP operands. We don't
3063 // bail completely out because we want to handle constant GEPs regardless
3064 // of depth.
3065 if (Depth++ >= MaxAnalysisRecursionDepth)
3066 continue;
3067
3068 if (isKnownNonZero(V: GTI.getOperand(), Q, Depth))
3069 return true;
3070 }
3071
3072 return false;
3073}
3074
3075static bool isKnownNonNullFromDominatingCondition(const Value *V,
3076 const Instruction *CtxI,
3077 const DominatorTree *DT) {
3078 assert(!isa<Constant>(V) && "Called for constant?");
3079
3080 if (!CtxI || !DT)
3081 return false;
3082
3083 unsigned NumUsesExplored = 0;
3084 for (auto &U : V->uses()) {
3085 // Avoid massive lists
3086 if (NumUsesExplored >= DomConditionsMaxUses)
3087 break;
3088 NumUsesExplored++;
3089
3090 const Instruction *UI = cast<Instruction>(Val: U.getUser());
3091 // If the value is used as an argument to a call or invoke, then argument
3092 // attributes may provide an answer about null-ness.
3093 if (V->getType()->isPointerTy()) {
3094 if (const auto *CB = dyn_cast<CallBase>(Val: UI)) {
3095 if (CB->isArgOperand(U: &U) &&
3096 CB->paramHasNonNullAttr(ArgNo: CB->getArgOperandNo(U: &U),
3097 /*AllowUndefOrPoison=*/false) &&
3098 DT->dominates(Def: CB, User: CtxI))
3099 return true;
3100 }
3101 }
3102
3103 // If the value is used as a load/store, then the pointer must be non null.
3104 if (V == getLoadStorePointerOperand(V: UI)) {
3105 if (!NullPointerIsDefined(F: UI->getFunction(),
3106 AS: V->getType()->getPointerAddressSpace()) &&
3107 DT->dominates(Def: UI, User: CtxI))
3108 return true;
3109 }
3110
3111 if ((match(V: UI, P: m_IDiv(L: m_Value(), R: m_Specific(V))) ||
3112 match(V: UI, P: m_IRem(L: m_Value(), R: m_Specific(V)))) &&
3113 isValidAssumeForContext(Inv: UI, CtxI, DT))
3114 return true;
3115
3116 // Consider only compare instructions uniquely controlling a branch
3117 Value *RHS;
3118 CmpPredicate Pred;
3119 if (!match(V: UI, P: m_c_ICmp(Pred, L: m_Specific(V), R: m_Value(V&: RHS))))
3120 continue;
3121
3122 bool NonNullIfTrue;
3123 if (cmpExcludesZero(Pred, RHS))
3124 NonNullIfTrue = true;
3125 else if (cmpExcludesZero(Pred: CmpInst::getInversePredicate(pred: Pred), RHS))
3126 NonNullIfTrue = false;
3127 else
3128 continue;
3129
3130 SmallVector<const User *, 4> WorkList;
3131 SmallPtrSet<const User *, 4> Visited;
3132 for (const auto *CmpU : UI->users()) {
3133 assert(WorkList.empty() && "Should be!");
3134 if (Visited.insert(Ptr: CmpU).second)
3135 WorkList.push_back(Elt: CmpU);
3136
3137 while (!WorkList.empty()) {
3138 auto *Curr = WorkList.pop_back_val();
3139
3140 // If a user is an AND, add all its users to the work list. We only
3141 // propagate "pred != null" condition through AND because it is only
3142 // correct to assume that all conditions of AND are met in true branch.
3143 // TODO: Support similar logic of OR and EQ predicate?
3144 if (NonNullIfTrue)
3145 if (match(V: Curr, P: m_LogicalAnd(L: m_Value(), R: m_Value()))) {
3146 for (const auto *CurrU : Curr->users())
3147 if (Visited.insert(Ptr: CurrU).second)
3148 WorkList.push_back(Elt: CurrU);
3149 continue;
3150 }
3151
3152 if (const CondBrInst *BI = dyn_cast<CondBrInst>(Val: Curr)) {
3153 BasicBlock *NonNullSuccessor =
3154 BI->getSuccessor(i: NonNullIfTrue ? 0 : 1);
3155 BasicBlockEdge Edge(BI->getParent(), NonNullSuccessor);
3156 if (DT->dominates(BBE: Edge, BB: CtxI->getParent()))
3157 return true;
3158 } else if (NonNullIfTrue && isGuard(U: Curr) &&
3159 DT->dominates(Def: cast<Instruction>(Val: Curr), User: CtxI)) {
3160 return true;
3161 }
3162 }
3163 }
3164 }
3165
3166 return false;
3167}
3168
3169/// Does the 'Range' metadata (which must be a valid MD_range operand list)
3170/// ensure that the value it's attached to is never Value? 'RangeType' is
3171/// is the type of the value described by the range.
3172static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value) {
3173 const unsigned NumRanges = Ranges->getNumOperands() / 2;
3174 assert(NumRanges >= 1);
3175 for (unsigned i = 0; i < NumRanges; ++i) {
3176 ConstantInt *Lower =
3177 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 0));
3178 ConstantInt *Upper =
3179 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 1));
3180 ConstantRange Range(Lower->getValue(), Upper->getValue());
3181 if (Range.contains(Val: Value))
3182 return false;
3183 }
3184 return true;
3185}
3186
3187/// Try to detect a recurrence that monotonically increases/decreases from a
3188/// non-zero starting value. These are common as induction variables.
3189static bool isNonZeroRecurrence(const PHINode *PN) {
3190 BinaryOperator *BO = nullptr;
3191 Value *Start = nullptr, *Step = nullptr;
3192 const APInt *StartC, *StepC;
3193 if (!matchSimpleRecurrence(P: PN, BO, Start, Step) ||
3194 !match(V: Start, P: m_APInt(Res&: StartC)) || StartC->isZero())
3195 return false;
3196
3197 switch (BO->getOpcode()) {
3198 case Instruction::Add:
3199 // Starting from non-zero and stepping away from zero can never wrap back
3200 // to zero.
3201 return BO->hasNoUnsignedWrap() ||
3202 (BO->hasNoSignedWrap() && match(V: Step, P: m_APInt(Res&: StepC)) &&
3203 StartC->isNegative() == StepC->isNegative());
3204 case Instruction::Mul:
3205 return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) &&
3206 match(V: Step, P: m_APInt(Res&: StepC)) && !StepC->isZero();
3207 case Instruction::Shl:
3208 return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap();
3209 case Instruction::AShr:
3210 case Instruction::LShr:
3211 return BO->isExact();
3212 case Instruction::Or:
3213 return true;
3214 default:
3215 return false;
3216 }
3217}
3218
3219static bool matchOpWithOpEqZero(Value *Op0, Value *Op1) {
3220 return match(V: Op0, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3221 L: m_Specific(V: Op1), R: m_Zero()))) ||
3222 match(V: Op1, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3223 L: m_Specific(V: Op0), R: m_Zero())));
3224}
3225
3226static bool isNonZeroAdd(const APInt &DemandedElts, const SimplifyQuery &Q,
3227 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3228 bool NUW, unsigned Depth) {
3229 // (X + (X != 0)) is non zero
3230 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3231 return true;
3232
3233 if (NUW)
3234 return isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3235 isKnownNonZero(V: X, DemandedElts, Q, Depth);
3236
3237 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3238 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3239
3240 // If X and Y are both non-negative (as signed values) then their sum is not
3241 // zero unless both X and Y are zero.
3242 if (XKnown.isNonNegative() && YKnown.isNonNegative())
3243 if (isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3244 isKnownNonZero(V: X, DemandedElts, Q, Depth))
3245 return true;
3246
3247 // If X and Y are both negative (as signed values) then their sum is not
3248 // zero unless both X and Y equal INT_MIN.
3249 if (XKnown.isNegative() && YKnown.isNegative()) {
3250 APInt Mask = APInt::getSignedMaxValue(numBits: BitWidth);
3251 // The sign bit of X is set. If some other bit is set then X is not equal
3252 // to INT_MIN.
3253 if (XKnown.One.intersects(RHS: Mask))
3254 return true;
3255 // The sign bit of Y is set. If some other bit is set then Y is not equal
3256 // to INT_MIN.
3257 if (YKnown.One.intersects(RHS: Mask))
3258 return true;
3259 }
3260
3261 // The sum of a non-negative number and a power of two is not zero.
3262 if (XKnown.isNonNegative() &&
3263 isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ false, Q, Depth))
3264 return true;
3265 if (YKnown.isNonNegative() &&
3266 isKnownToBeAPowerOfTwo(V: X, /*OrZero*/ false, Q, Depth))
3267 return true;
3268
3269 return KnownBits::add(LHS: XKnown, RHS: YKnown, NSW, NUW).isNonZero();
3270}
3271
3272static bool isNonZeroSub(const APInt &DemandedElts, const SimplifyQuery &Q,
3273 unsigned BitWidth, Value *X, Value *Y,
3274 unsigned Depth) {
3275 // (X - (X != 0)) is non zero
3276 // ((X != 0) - X) is non zero
3277 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3278 return true;
3279
3280 // TODO: Move this case into isKnownNonEqual().
3281 if (auto *C = dyn_cast<Constant>(Val: X))
3282 if (C->isNullValue() && isKnownNonZero(V: Y, DemandedElts, Q, Depth))
3283 return true;
3284
3285 return ::isKnownNonEqual(V1: X, V2: Y, DemandedElts, Q, Depth);
3286}
3287
3288static bool isNonZeroMul(const APInt &DemandedElts, const SimplifyQuery &Q,
3289 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3290 bool NUW, unsigned Depth) {
3291 // If X and Y are non-zero then so is X * Y as long as the multiplication
3292 // does not overflow.
3293 if (NSW || NUW)
3294 return isKnownNonZero(V: X, DemandedElts, Q, Depth) &&
3295 isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3296
3297 // If either X or Y is odd, then if the other is non-zero the result can't
3298 // be zero.
3299 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3300 if (XKnown.One[0])
3301 return isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3302
3303 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3304 if (YKnown.One[0])
3305 return XKnown.isNonZero() || isKnownNonZero(V: X, DemandedElts, Q, Depth);
3306
3307 // If there exists any subset of X (sX) and subset of Y (sY) s.t sX * sY is
3308 // non-zero, then X * Y is non-zero. We can find sX and sY by just taking
3309 // the lowest known One of X and Y. If they are non-zero, the result
3310 // must be non-zero. We can check if LSB(X) * LSB(Y) != 0 by doing
3311 // X.CountLeadingZeros + Y.CountLeadingZeros < BitWidth.
3312 return (XKnown.countMaxTrailingZeros() + YKnown.countMaxTrailingZeros()) <
3313 BitWidth;
3314}
3315
3316static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
3317 const SimplifyQuery &Q, const KnownBits &KnownVal,
3318 unsigned Depth) {
3319 auto ShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3320 switch (I->getOpcode()) {
3321 case Instruction::Shl:
3322 return Lhs.shl(ShiftAmt: Rhs);
3323 case Instruction::LShr:
3324 return Lhs.lshr(ShiftAmt: Rhs);
3325 case Instruction::AShr:
3326 return Lhs.ashr(ShiftAmt: Rhs);
3327 default:
3328 llvm_unreachable("Unknown Shift Opcode");
3329 }
3330 };
3331
3332 auto InvShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3333 switch (I->getOpcode()) {
3334 case Instruction::Shl:
3335 return Lhs.lshr(ShiftAmt: Rhs);
3336 case Instruction::LShr:
3337 case Instruction::AShr:
3338 return Lhs.shl(ShiftAmt: Rhs);
3339 default:
3340 llvm_unreachable("Unknown Shift Opcode");
3341 }
3342 };
3343
3344 if (KnownVal.isUnknown())
3345 return false;
3346
3347 KnownBits KnownCnt =
3348 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3349 APInt MaxShift = KnownCnt.getMaxValue();
3350 unsigned NumBits = KnownVal.getBitWidth();
3351 if (MaxShift.uge(RHS: NumBits))
3352 return false;
3353
3354 if (!ShiftOp(KnownVal.One, MaxShift).isZero())
3355 return true;
3356
3357 // If all of the bits shifted out are known to be zero, and Val is known
3358 // non-zero then at least one non-zero bit must remain.
3359 if (InvShiftOp(KnownVal.Zero, NumBits - MaxShift)
3360 .eq(RHS: InvShiftOp(APInt::getAllOnes(numBits: NumBits), NumBits - MaxShift)) &&
3361 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth))
3362 return true;
3363
3364 return false;
3365}
3366
3367static bool isKnownNonZeroFromOperator(const Operator *I,
3368 const APInt &DemandedElts,
3369 const SimplifyQuery &Q, unsigned Depth) {
3370 unsigned BitWidth = getBitWidth(Ty: I->getType()->getScalarType(), DL: Q.DL);
3371 switch (I->getOpcode()) {
3372 case Instruction::Alloca:
3373 // Alloca never returns null, malloc might.
3374 return I->getType()->getPointerAddressSpace() == 0;
3375 case Instruction::GetElementPtr:
3376 if (I->getType()->isPointerTy())
3377 return isGEPKnownNonNull(GEP: cast<GEPOperator>(Val: I), Q, Depth);
3378 break;
3379 case Instruction::BitCast: {
3380 // We need to be a bit careful here. We can only peek through the bitcast
3381 // if the scalar size of elements in the operand are smaller than and a
3382 // multiple of the size they are casting too. Take three cases:
3383 //
3384 // 1) Unsafe:
3385 // bitcast <2 x i16> %NonZero to <4 x i8>
3386 //
3387 // %NonZero can have 2 non-zero i16 elements, but isKnownNonZero on a
3388 // <4 x i8> requires that all 4 i8 elements be non-zero which isn't
3389 // guranteed (imagine just sign bit set in the 2 i16 elements).
3390 //
3391 // 2) Unsafe:
3392 // bitcast <4 x i3> %NonZero to <3 x i4>
3393 //
3394 // Even though the scalar size of the src (`i3`) is smaller than the
3395 // scalar size of the dst `i4`, because `i3` is not a multiple of `i4`
3396 // its possible for the `3 x i4` elements to be zero because there are
3397 // some elements in the destination that don't contain any full src
3398 // element.
3399 //
3400 // 3) Safe:
3401 // bitcast <4 x i8> %NonZero to <2 x i16>
3402 //
3403 // This is always safe as non-zero in the 4 i8 elements implies
3404 // non-zero in the combination of any two adjacent ones. Since i8 is a
3405 // multiple of i16, each i16 is guranteed to have 2 full i8 elements.
3406 // This all implies the 2 i16 elements are non-zero.
3407 Type *FromTy = I->getOperand(i: 0)->getType();
3408 if ((FromTy->isIntOrIntVectorTy() || FromTy->isPtrOrPtrVectorTy()) &&
3409 (BitWidth % getBitWidth(Ty: FromTy->getScalarType(), DL: Q.DL)) == 0)
3410 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3411 } break;
3412 case Instruction::IntToPtr:
3413 // Note that we have to take special care to avoid looking through
3414 // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
3415 // as casts that can alter the value, e.g., AddrSpaceCasts.
3416 if (!isa<ScalableVectorType>(Val: I->getType()) &&
3417 Q.DL.getTypeSizeInBits(Ty: I->getOperand(i: 0)->getType()).getFixedValue() <=
3418 Q.DL.getTypeSizeInBits(Ty: I->getType()).getFixedValue())
3419 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3420 break;
3421 case Instruction::PtrToAddr:
3422 // isKnownNonZero() for pointers refers to the address bits being non-zero,
3423 // so we can directly forward.
3424 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3425 case Instruction::PtrToInt:
3426 // For inttoptr, make sure the result size is >= the address size. If the
3427 // address is non-zero, any larger value is also non-zero.
3428 if (Q.DL.getAddressSizeInBits(Ty: I->getOperand(i: 0)->getType()) <=
3429 I->getType()->getScalarSizeInBits())
3430 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3431 break;
3432 case Instruction::Trunc:
3433 // nuw/nsw trunc preserves zero/non-zero status of input.
3434 if (auto *TI = dyn_cast<TruncInst>(Val: I))
3435 if (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap())
3436 return isKnownNonZero(V: TI->getOperand(i_nocapture: 0), DemandedElts, Q, Depth);
3437 break;
3438
3439 // Iff x - y != 0, then x ^ y != 0
3440 // Therefore we can do the same exact checks
3441 case Instruction::Xor:
3442 case Instruction::Sub:
3443 return isNonZeroSub(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3444 Y: I->getOperand(i: 1), Depth);
3445 case Instruction::Or:
3446 // (X | (X != 0)) is non zero
3447 if (matchOpWithOpEqZero(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1)))
3448 return true;
3449 // X | Y != 0 if X != Y.
3450 if (isKnownNonEqual(V1: I->getOperand(i: 0), V2: I->getOperand(i: 1), DemandedElts, Q,
3451 Depth))
3452 return true;
3453 // X | Y != 0 if X != 0 or Y != 0.
3454 return isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth) ||
3455 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3456 case Instruction::SExt:
3457 case Instruction::ZExt:
3458 // ext X != 0 if X != 0.
3459 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3460
3461 case Instruction::Shl: {
3462 // shl nsw/nuw can't remove any non-zero bits.
3463 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3464 if (Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO))
3465 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3466
3467 // shl X, Y != 0 if X is odd. Note that the value of the shift is undefined
3468 // if the lowest bit is shifted off the end.
3469 KnownBits Known(BitWidth);
3470 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth);
3471 if (Known.One[0])
3472 return true;
3473
3474 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3475 }
3476 case Instruction::LShr:
3477 case Instruction::AShr: {
3478 // shr exact can only shift out zero bits.
3479 const PossiblyExactOperator *BO = cast<PossiblyExactOperator>(Val: I);
3480 if (BO->isExact())
3481 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3482
3483 // shr X, Y != 0 if X is negative. Note that the value of the shift is not
3484 // defined if the sign bit is shifted off the end.
3485 KnownBits Known =
3486 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3487 if (Known.isNegative())
3488 return true;
3489
3490 // shr (add nuw A, B), C is non-zero if A or B has a known-one bit at
3491 // position >= C, because the sum >= max(A, B).
3492 Value *A, *B;
3493 const APInt *C;
3494 if (Depth + 1 < MaxAnalysisRecursionDepth &&
3495 match(V: I->getOperand(i: 0), P: m_NUWAdd(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3496 match(V: I->getOperand(i: 1), P: m_APInt(Res&: C)) && C->ult(RHS: BitWidth)) {
3497 KnownBits KnownA = computeKnownBits(V: A, DemandedElts, Q, Depth: Depth + 1);
3498 if (!KnownA.One.lshr(ShiftAmt: *C).isZero())
3499 return true;
3500 KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
3501 if (!KnownB.One.lshr(ShiftAmt: *C).isZero())
3502 return true;
3503 }
3504
3505 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3506 }
3507 case Instruction::UDiv:
3508 case Instruction::SDiv: {
3509 // X / Y
3510 // div exact can only produce a zero if the dividend is zero.
3511 if (cast<PossiblyExactOperator>(Val: I)->isExact())
3512 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3513
3514 KnownBits XKnown =
3515 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3516 // If X is fully unknown we won't be able to figure anything out so don't
3517 // both computing knownbits for Y.
3518 if (XKnown.isUnknown())
3519 return false;
3520
3521 KnownBits YKnown =
3522 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3523 if (I->getOpcode() == Instruction::SDiv) {
3524 // For signed division need to compare abs value of the operands.
3525 XKnown = XKnown.abs(/*IntMinIsPoison*/ false);
3526 YKnown = YKnown.abs(/*IntMinIsPoison*/ false);
3527 }
3528 // If X u>= Y then div is non zero (0/0 is UB).
3529 std::optional<bool> XUgeY = KnownBits::uge(LHS: XKnown, RHS: YKnown);
3530 // If X is total unknown or X u< Y we won't be able to prove non-zero
3531 // with compute known bits so just return early.
3532 return XUgeY && *XUgeY;
3533 }
3534 case Instruction::Add: {
3535 // X + Y.
3536
3537 // If Add has nuw wrap flag, then if either X or Y is non-zero the result is
3538 // non-zero.
3539 auto *BO = cast<OverflowingBinaryOperator>(Val: I);
3540 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3541 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3542 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3543 }
3544 case Instruction::Mul: {
3545 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3546 return isNonZeroMul(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3547 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3548 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3549 }
3550 case Instruction::Select: {
3551 // (C ? X : Y) != 0 if X != 0 and Y != 0.
3552
3553 // First check if the arm is non-zero using `isKnownNonZero`. If that fails,
3554 // then see if the select condition implies the arm is non-zero. For example
3555 // (X != 0 ? X : Y), we know the true arm is non-zero as the `X` "return" is
3556 // dominated by `X != 0`.
3557 auto SelectArmIsNonZero = [&](bool IsTrueArm) {
3558 Value *Op;
3559 Op = IsTrueArm ? I->getOperand(i: 1) : I->getOperand(i: 2);
3560 // Op is trivially non-zero.
3561 if (isKnownNonZero(V: Op, DemandedElts, Q, Depth))
3562 return true;
3563
3564 // The condition of the select dominates the true/false arm. Check if the
3565 // condition implies that a given arm is non-zero.
3566 Value *X;
3567 CmpPredicate Pred;
3568 if (!match(V: I->getOperand(i: 0), P: m_c_ICmp(Pred, L: m_Specific(V: Op), R: m_Value(V&: X))))
3569 return false;
3570
3571 if (!IsTrueArm)
3572 Pred = ICmpInst::getInversePredicate(pred: Pred);
3573
3574 return cmpExcludesZero(Pred, RHS: X);
3575 };
3576
3577 if (SelectArmIsNonZero(/* IsTrueArm */ true) &&
3578 SelectArmIsNonZero(/* IsTrueArm */ false))
3579 return true;
3580 break;
3581 }
3582 case Instruction::PHI: {
3583 auto *PN = cast<PHINode>(Val: I);
3584 if (Q.IIQ.UseInstrInfo && isNonZeroRecurrence(PN))
3585 return true;
3586
3587 // Check if all incoming values are non-zero using recursion.
3588 SimplifyQuery RecQ = Q.getWithoutCondContext();
3589 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
3590 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
3591 if (U.get() == PN)
3592 return true;
3593 RecQ.CtxI = PN->getIncomingBlock(U)->getTerminator();
3594 // Check if the branch on the phi excludes zero.
3595 CmpPredicate Pred;
3596 Value *X;
3597 BasicBlock *TrueSucc, *FalseSucc;
3598 if (match(V: RecQ.CtxI,
3599 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: U.get()), R: m_Value(V&: X)),
3600 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
3601 // Check for cases of duplicate successors.
3602 if ((TrueSucc == PN->getParent()) != (FalseSucc == PN->getParent())) {
3603 // If we're using the false successor, invert the predicate.
3604 if (FalseSucc == PN->getParent())
3605 Pred = CmpInst::getInversePredicate(pred: Pred);
3606 if (cmpExcludesZero(Pred, RHS: X))
3607 return true;
3608 }
3609 }
3610 // Finally recurse on the edge and check it directly.
3611 return isKnownNonZero(V: U.get(), DemandedElts, Q: RecQ, Depth: NewDepth);
3612 });
3613 }
3614 case Instruction::InsertElement: {
3615 if (isa<ScalableVectorType>(Val: I->getType()))
3616 break;
3617
3618 const Value *Vec = I->getOperand(i: 0);
3619 const Value *Elt = I->getOperand(i: 1);
3620 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
3621
3622 unsigned NumElts = DemandedElts.getBitWidth();
3623 APInt DemandedVecElts = DemandedElts;
3624 bool SkipElt = false;
3625 // If we know the index we are inserting too, clear it from Vec check.
3626 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
3627 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
3628 SkipElt = !DemandedElts[CIdx->getZExtValue()];
3629 }
3630
3631 // Result is zero if Elt is non-zero and rest of the demanded elts in Vec
3632 // are non-zero.
3633 return (SkipElt || isKnownNonZero(V: Elt, Q, Depth)) &&
3634 (DemandedVecElts.isZero() ||
3635 isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth));
3636 }
3637 case Instruction::ExtractElement:
3638 if (const auto *EEI = dyn_cast<ExtractElementInst>(Val: I)) {
3639 const Value *Vec = EEI->getVectorOperand();
3640 const Value *Idx = EEI->getIndexOperand();
3641 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
3642 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
3643 unsigned NumElts = VecTy->getNumElements();
3644 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
3645 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
3646 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
3647 return isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth);
3648 }
3649 }
3650 break;
3651 case Instruction::ShuffleVector: {
3652 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
3653 if (!Shuf)
3654 break;
3655 APInt DemandedLHS, DemandedRHS;
3656 // For undef elements, we don't know anything about the common state of
3657 // the shuffle result.
3658 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
3659 break;
3660 // If demanded elements for both vecs are non-zero, the shuffle is non-zero.
3661 return (DemandedRHS.isZero() ||
3662 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 1), DemandedElts: DemandedRHS, Q, Depth)) &&
3663 (DemandedLHS.isZero() ||
3664 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 0), DemandedElts: DemandedLHS, Q, Depth));
3665 }
3666 case Instruction::Freeze:
3667 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth) &&
3668 isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT,
3669 Depth);
3670 case Instruction::Load: {
3671 auto *LI = cast<LoadInst>(Val: I);
3672 // A Load tagged with nonnull or dereferenceable with null pointer undefined
3673 // is never null.
3674 if (auto *PtrT = dyn_cast<PointerType>(Val: I->getType())) {
3675 if (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_nonnull) ||
3676 (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_dereferenceable) &&
3677 !NullPointerIsDefined(F: LI->getFunction(), AS: PtrT->getAddressSpace())))
3678 return true;
3679 } else if (MDNode *Ranges = Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_range)) {
3680 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3681 }
3682
3683 // No need to fall through to computeKnownBits as range metadata is already
3684 // handled in isKnownNonZero.
3685 return false;
3686 }
3687 case Instruction::ExtractValue: {
3688 const WithOverflowInst *WO;
3689 if (match(V: I, P: m_ExtractValue<0>(V: m_WithOverflowInst(I&: WO)))) {
3690 switch (WO->getBinaryOp()) {
3691 default:
3692 break;
3693 case Instruction::Add:
3694 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3695 Y: WO->getArgOperand(i: 1),
3696 /*NSW=*/false,
3697 /*NUW=*/false, Depth);
3698 case Instruction::Sub:
3699 return isNonZeroSub(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3700 Y: WO->getArgOperand(i: 1), Depth);
3701 case Instruction::Mul:
3702 return isNonZeroMul(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3703 Y: WO->getArgOperand(i: 1),
3704 /*NSW=*/false, /*NUW=*/false, Depth);
3705 break;
3706 }
3707 }
3708 break;
3709 }
3710 case Instruction::Call:
3711 case Instruction::Invoke: {
3712 const auto *Call = cast<CallBase>(Val: I);
3713 if (I->getType()->isPointerTy()) {
3714 if (Call->isReturnNonNull())
3715 return true;
3716 if (const auto *RP = getArgumentAliasingToReturnedPointer(
3717 Call, /*MustPreserveOffset=*/true))
3718 return isKnownNonZero(V: RP, Q, Depth);
3719 } else {
3720 if (MDNode *Ranges = Q.IIQ.getMetadata(I: Call, KindID: LLVMContext::MD_range))
3721 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3722 if (std::optional<ConstantRange> Range = Call->getRange()) {
3723 const APInt ZeroValue(Range->getBitWidth(), 0);
3724 if (!Range->contains(Val: ZeroValue))
3725 return true;
3726 }
3727 if (const Value *RV = Call->getReturnedArgOperand())
3728 if (RV->getType() == I->getType() && isKnownNonZero(V: RV, Q, Depth))
3729 return true;
3730 }
3731
3732 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3733 switch (II->getIntrinsicID()) {
3734 case Intrinsic::sshl_sat:
3735 case Intrinsic::ushl_sat:
3736 case Intrinsic::abs:
3737 case Intrinsic::bitreverse:
3738 case Intrinsic::bswap:
3739 case Intrinsic::ctpop:
3740 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3741 // NB: We don't do usub_sat here as in any case we can prove its
3742 // non-zero, we will fold it to `sub nuw` in InstCombine.
3743 case Intrinsic::ssub_sat:
3744 // For most types, if x != y then ssub.sat x, y != 0. But
3745 // ssub.sat.i1 0, -1 = 0, because 1 saturates to 0. This means
3746 // isNonZeroSub will do the wrong thing for ssub.sat.i1.
3747 if (BitWidth == 1)
3748 return false;
3749 return isNonZeroSub(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3750 Y: II->getArgOperand(i: 1), Depth);
3751 case Intrinsic::sadd_sat:
3752 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3753 Y: II->getArgOperand(i: 1),
3754 /*NSW=*/true, /* NUW=*/false, Depth);
3755 // Vec reverse preserves zero/non-zero status from input vec.
3756 case Intrinsic::vector_reverse:
3757 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
3758 Q, Depth);
3759 // umin/smin/smax/smin/or of all non-zero elements is always non-zero.
3760 case Intrinsic::vector_reduce_or:
3761 case Intrinsic::vector_reduce_umax:
3762 case Intrinsic::vector_reduce_umin:
3763 case Intrinsic::vector_reduce_smax:
3764 case Intrinsic::vector_reduce_smin:
3765 return isKnownNonZero(V: II->getArgOperand(i: 0), Q, Depth);
3766 case Intrinsic::umax:
3767 case Intrinsic::uadd_sat:
3768 // umax(X, (X != 0)) is non zero
3769 // X +usat (X != 0) is non zero
3770 if (matchOpWithOpEqZero(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1)))
3771 return true;
3772
3773 return isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth) ||
3774 isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3775 case Intrinsic::smax: {
3776 // If either arg is strictly positive the result is non-zero. Otherwise
3777 // the result is non-zero if both ops are non-zero.
3778 auto IsNonZero = [&](Value *Op, std::optional<bool> &OpNonZero,
3779 const KnownBits &OpKnown) {
3780 if (!OpNonZero.has_value())
3781 OpNonZero = OpKnown.isNonZero() ||
3782 isKnownNonZero(V: Op, DemandedElts, Q, Depth);
3783 return *OpNonZero;
3784 };
3785 // Avoid re-computing isKnownNonZero.
3786 std::optional<bool> Op0NonZero, Op1NonZero;
3787 KnownBits Op1Known =
3788 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3789 if (Op1Known.isNonNegative() &&
3790 IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known))
3791 return true;
3792 KnownBits Op0Known =
3793 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3794 if (Op0Known.isNonNegative() &&
3795 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known))
3796 return true;
3797 return IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known) &&
3798 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known);
3799 }
3800 case Intrinsic::smin: {
3801 // If either arg is negative the result is non-zero. Otherwise
3802 // the result is non-zero if both ops are non-zero.
3803 KnownBits Op1Known =
3804 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3805 if (Op1Known.isNegative())
3806 return true;
3807 KnownBits Op0Known =
3808 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3809 if (Op0Known.isNegative())
3810 return true;
3811
3812 if (Op1Known.isNonZero() && Op0Known.isNonZero())
3813 return true;
3814 }
3815 [[fallthrough]];
3816 case Intrinsic::umin:
3817 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth) &&
3818 isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3819 case Intrinsic::cttz:
3820 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3821 .Zero[0];
3822 case Intrinsic::ctlz:
3823 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3824 .isNonNegative();
3825 case Intrinsic::fshr:
3826 case Intrinsic::fshl:
3827 // If Op0 == Op1, this is a rotate. rotate(x, y) != 0 iff x != 0.
3828 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
3829 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3830 break;
3831 case Intrinsic::vscale:
3832 return true;
3833 case Intrinsic::experimental_get_vector_length:
3834 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3835 default:
3836 break;
3837 }
3838 break;
3839 }
3840
3841 return false;
3842 }
3843 }
3844
3845 KnownBits Known(BitWidth);
3846 computeKnownBits(V: I, DemandedElts, Known, Q, Depth);
3847 return Known.One != 0;
3848}
3849
3850/// Return true if the given value is known to be non-zero when defined. For
3851/// vectors, return true if every demanded element is known to be non-zero when
3852/// defined. For pointers, if the context instruction and dominator tree are
3853/// specified, perform context-sensitive analysis and return true if the
3854/// pointer couldn't possibly be null at the specified instruction.
3855/// Supports values with integer or pointer type and vectors of integers.
3856bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
3857 const SimplifyQuery &Q, unsigned Depth) {
3858 Type *Ty = V->getType();
3859
3860#ifndef NDEBUG
3861 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
3862
3863 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
3864 assert(
3865 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
3866 "DemandedElt width should equal the fixed vector number of elements");
3867 } else {
3868 assert(DemandedElts == APInt(1, 1) &&
3869 "DemandedElt width should be 1 for scalars");
3870 }
3871#endif
3872
3873 if (auto *C = dyn_cast<Constant>(Val: V)) {
3874 if (C->isNullValue())
3875 return false;
3876 if (isa<ConstantInt>(Val: C))
3877 // Must be non-zero due to null test above.
3878 return true;
3879
3880 // For constant vectors, check that all elements are poison or known
3881 // non-zero to determine that the whole vector is known non-zero.
3882 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Ty)) {
3883 for (unsigned i = 0, e = VecTy->getNumElements(); i != e; ++i) {
3884 if (!DemandedElts[i])
3885 continue;
3886 Constant *Elt = C->getAggregateElement(Elt: i);
3887 if (!Elt || Elt->isNullValue())
3888 return false;
3889 if (!isa<PoisonValue>(Val: Elt) && !isa<ConstantInt>(Val: Elt))
3890 return false;
3891 }
3892 return true;
3893 }
3894
3895 // Constant ptrauth can be null, iff the base pointer can be.
3896 if (auto *CPA = dyn_cast<ConstantPtrAuth>(Val: V))
3897 return isKnownNonZero(V: CPA->getPointer(), DemandedElts, Q, Depth);
3898
3899 // A global variable in address space 0 is non null unless extern weak
3900 // or an absolute symbol reference. Other address spaces may have null as a
3901 // valid address for a global, so we can't assume anything.
3902 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
3903 if (!GV->isAbsoluteSymbolRef() && !GV->hasExternalWeakLinkage() &&
3904 GV->getType()->getAddressSpace() == 0)
3905 return true;
3906 }
3907
3908 // For constant expressions, fall through to the Operator code below.
3909 if (!isa<ConstantExpr>(Val: V))
3910 return false;
3911 }
3912
3913 if (const auto *A = dyn_cast<Argument>(Val: V))
3914 if (std::optional<ConstantRange> Range = A->getRange()) {
3915 const APInt ZeroValue(Range->getBitWidth(), 0);
3916 if (!Range->contains(Val: ZeroValue))
3917 return true;
3918 }
3919
3920 if (!isa<Constant>(Val: V) && isKnownNonZeroFromAssume(V, Q))
3921 return true;
3922
3923 // Some of the tests below are recursive, so bail out if we hit the limit.
3924 if (Depth++ >= MaxAnalysisRecursionDepth)
3925 return false;
3926
3927 // Check for pointer simplifications.
3928
3929 if (PointerType *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
3930 // A byval, inalloca may not be null in a non-default addres space. A
3931 // nonnull argument is assumed never 0.
3932 if (const Argument *A = dyn_cast<Argument>(Val: V)) {
3933 if (((A->hasPassPointeeByValueCopyAttr() &&
3934 !NullPointerIsDefined(F: A->getParent(), AS: PtrTy->getAddressSpace())) ||
3935 A->hasNonNullAttr()))
3936 return true;
3937 }
3938 }
3939
3940 if (const auto *I = dyn_cast<Operator>(Val: V))
3941 if (isKnownNonZeroFromOperator(I, DemandedElts, Q, Depth))
3942 return true;
3943
3944 if (!isa<Constant>(Val: V) &&
3945 isKnownNonNullFromDominatingCondition(V, CtxI: Q.CtxI, DT: Q.DT))
3946 return true;
3947
3948 if (const Value *Stripped = stripNullTest(V))
3949 return isKnownNonZero(V: Stripped, DemandedElts, Q, Depth);
3950
3951 return false;
3952}
3953
3954bool llvm::isKnownNonZero(const Value *V, const SimplifyQuery &Q,
3955 unsigned Depth) {
3956 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
3957 APInt DemandedElts =
3958 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
3959 return ::isKnownNonZero(V, DemandedElts, Q, Depth);
3960}
3961
3962/// If the pair of operators are the same invertible function, return the
3963/// the operands of the function corresponding to each input. Otherwise,
3964/// return std::nullopt. An invertible function is one that is 1-to-1 and maps
3965/// every input value to exactly one output value. This is equivalent to
3966/// saying that Op1 and Op2 are equal exactly when the specified pair of
3967/// operands are equal, (except that Op1 and Op2 may be poison more often.)
3968static std::optional<std::pair<Value*, Value*>>
3969getInvertibleOperands(const Operator *Op1,
3970 const Operator *Op2) {
3971 if (Op1->getOpcode() != Op2->getOpcode())
3972 return std::nullopt;
3973
3974 auto getOperands = [&](unsigned OpNum) -> auto {
3975 return std::make_pair(x: Op1->getOperand(i: OpNum), y: Op2->getOperand(i: OpNum));
3976 };
3977
3978 switch (Op1->getOpcode()) {
3979 default:
3980 break;
3981 case Instruction::Or:
3982 if (!cast<PossiblyDisjointInst>(Val: Op1)->isDisjoint() ||
3983 !cast<PossiblyDisjointInst>(Val: Op2)->isDisjoint())
3984 break;
3985 [[fallthrough]];
3986 case Instruction::Xor:
3987 case Instruction::Add: {
3988 Value *Other;
3989 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 0)), R: m_Value(V&: Other))))
3990 return std::make_pair(x: Op1->getOperand(i: 1), y&: Other);
3991 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 1)), R: m_Value(V&: Other))))
3992 return std::make_pair(x: Op1->getOperand(i: 0), y&: Other);
3993 break;
3994 }
3995 case Instruction::Sub:
3996 if (Op1->getOperand(i: 0) == Op2->getOperand(i: 0))
3997 return getOperands(1);
3998 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
3999 return getOperands(0);
4000 break;
4001 case Instruction::Mul: {
4002 // invertible if A * B == (A * B) mod 2^N where A, and B are integers
4003 // and N is the bitwdith. The nsw case is non-obvious, but proven by
4004 // alive2: https://alive2.llvm.org/ce/z/Z6D5qK
4005 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
4006 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
4007 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
4008 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
4009 break;
4010
4011 // Assume operand order has been canonicalized
4012 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1) &&
4013 isa<ConstantInt>(Val: Op1->getOperand(i: 1)) &&
4014 !cast<ConstantInt>(Val: Op1->getOperand(i: 1))->isZero())
4015 return getOperands(0);
4016 break;
4017 }
4018 case Instruction::Shl: {
4019 // Same as multiplies, with the difference that we don't need to check
4020 // for a non-zero multiply. Shifts always multiply by non-zero.
4021 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
4022 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
4023 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
4024 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
4025 break;
4026
4027 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
4028 return getOperands(0);
4029 break;
4030 }
4031 case Instruction::AShr:
4032 case Instruction::LShr: {
4033 auto *PEO1 = cast<PossiblyExactOperator>(Val: Op1);
4034 auto *PEO2 = cast<PossiblyExactOperator>(Val: Op2);
4035 if (!PEO1->isExact() || !PEO2->isExact())
4036 break;
4037
4038 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
4039 return getOperands(0);
4040 break;
4041 }
4042 case Instruction::SExt:
4043 case Instruction::ZExt:
4044 if (Op1->getOperand(i: 0)->getType() == Op2->getOperand(i: 0)->getType())
4045 return getOperands(0);
4046 break;
4047 case Instruction::PHI: {
4048 const PHINode *PN1 = cast<PHINode>(Val: Op1);
4049 const PHINode *PN2 = cast<PHINode>(Val: Op2);
4050
4051 // If PN1 and PN2 are both recurrences, can we prove the entire recurrences
4052 // are a single invertible function of the start values? Note that repeated
4053 // application of an invertible function is also invertible
4054 BinaryOperator *BO1 = nullptr;
4055 Value *Start1 = nullptr, *Step1 = nullptr;
4056 BinaryOperator *BO2 = nullptr;
4057 Value *Start2 = nullptr, *Step2 = nullptr;
4058 if (PN1->getParent() != PN2->getParent() ||
4059 !matchSimpleRecurrence(P: PN1, BO&: BO1, Start&: Start1, Step&: Step1) ||
4060 !matchSimpleRecurrence(P: PN2, BO&: BO2, Start&: Start2, Step&: Step2))
4061 break;
4062
4063 auto Values = getInvertibleOperands(Op1: cast<Operator>(Val: BO1),
4064 Op2: cast<Operator>(Val: BO2));
4065 if (!Values)
4066 break;
4067
4068 // We have to be careful of mutually defined recurrences here. Ex:
4069 // * X_i = X_(i-1) OP Y_(i-1), and Y_i = X_(i-1) OP V
4070 // * X_i = Y_i = X_(i-1) OP Y_(i-1)
4071 // The invertibility of these is complicated, and not worth reasoning
4072 // about (yet?).
4073 if (Values->first != PN1 || Values->second != PN2)
4074 break;
4075
4076 return std::make_pair(x&: Start1, y&: Start2);
4077 }
4078 }
4079 return std::nullopt;
4080}
4081
4082/// Return true if V1 == (binop V2, X), where X is known non-zero.
4083/// Only handle a small subset of binops where (binop V2, X) with non-zero X
4084/// implies V2 != V1.
4085static bool isModifyingBinopOfNonZero(const Value *V1, const Value *V2,
4086 const APInt &DemandedElts,
4087 const SimplifyQuery &Q, unsigned Depth) {
4088 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V1);
4089 if (!BO)
4090 return false;
4091 switch (BO->getOpcode()) {
4092 default:
4093 break;
4094 case Instruction::Or:
4095 if (!cast<PossiblyDisjointInst>(Val: V1)->isDisjoint())
4096 break;
4097 [[fallthrough]];
4098 case Instruction::Xor:
4099 case Instruction::Add:
4100 Value *Op = nullptr;
4101 if (V2 == BO->getOperand(i_nocapture: 0))
4102 Op = BO->getOperand(i_nocapture: 1);
4103 else if (V2 == BO->getOperand(i_nocapture: 1))
4104 Op = BO->getOperand(i_nocapture: 0);
4105 else
4106 return false;
4107 return isKnownNonZero(V: Op, DemandedElts, Q, Depth: Depth + 1);
4108 }
4109 return false;
4110}
4111
4112/// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
4113/// the multiplication is nuw or nsw.
4114static bool isNonEqualMul(const Value *V1, const Value *V2,
4115 const APInt &DemandedElts, const SimplifyQuery &Q,
4116 unsigned Depth) {
4117 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
4118 const APInt *C;
4119 return match(V: OBO, P: m_Mul(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
4120 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
4121 !C->isZero() && !C->isOne() &&
4122 isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
4123 }
4124 return false;
4125}
4126
4127/// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
4128/// the shift is nuw or nsw.
4129static bool isNonEqualShl(const Value *V1, const Value *V2,
4130 const APInt &DemandedElts, const SimplifyQuery &Q,
4131 unsigned Depth) {
4132 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
4133 const APInt *C;
4134 return match(V: OBO, P: m_Shl(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
4135 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
4136 !C->isZero() && isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
4137 }
4138 return false;
4139}
4140
4141static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
4142 const APInt &DemandedElts, const SimplifyQuery &Q,
4143 unsigned Depth) {
4144 // Check two PHIs are in same block.
4145 if (PN1->getParent() != PN2->getParent())
4146 return false;
4147
4148 SmallPtrSet<const BasicBlock *, 8> VisitedBBs;
4149 bool UsedFullRecursion = false;
4150 for (const BasicBlock *IncomBB : PN1->blocks()) {
4151 if (!VisitedBBs.insert(Ptr: IncomBB).second)
4152 continue; // Don't reprocess blocks that we have dealt with already.
4153 const Value *IV1 = PN1->getIncomingValueForBlock(BB: IncomBB);
4154 const Value *IV2 = PN2->getIncomingValueForBlock(BB: IncomBB);
4155 const APInt *C1, *C2;
4156 if (match(V: IV1, P: m_APInt(Res&: C1)) && match(V: IV2, P: m_APInt(Res&: C2)) && *C1 != *C2)
4157 continue;
4158
4159 // Only one pair of phi operands is allowed for full recursion.
4160 if (UsedFullRecursion)
4161 return false;
4162
4163 SimplifyQuery RecQ = Q.getWithoutCondContext();
4164 RecQ.CtxI = IncomBB->getTerminator();
4165 if (!isKnownNonEqual(V1: IV1, V2: IV2, DemandedElts, Q: RecQ, Depth: Depth + 1))
4166 return false;
4167 UsedFullRecursion = true;
4168 }
4169 return true;
4170}
4171
4172static bool isNonEqualSelect(const Value *V1, const Value *V2,
4173 const APInt &DemandedElts, const SimplifyQuery &Q,
4174 unsigned Depth) {
4175 const SelectInst *SI1 = dyn_cast<SelectInst>(Val: V1);
4176 if (!SI1)
4177 return false;
4178
4179 if (const SelectInst *SI2 = dyn_cast<SelectInst>(Val: V2)) {
4180 const Value *Cond1 = SI1->getCondition();
4181 const Value *Cond2 = SI2->getCondition();
4182 if (Cond1 == Cond2)
4183 return isKnownNonEqual(V1: SI1->getTrueValue(), V2: SI2->getTrueValue(),
4184 DemandedElts, Q, Depth: Depth + 1) &&
4185 isKnownNonEqual(V1: SI1->getFalseValue(), V2: SI2->getFalseValue(),
4186 DemandedElts, Q, Depth: Depth + 1);
4187 }
4188 return isKnownNonEqual(V1: SI1->getTrueValue(), V2, DemandedElts, Q, Depth: Depth + 1) &&
4189 isKnownNonEqual(V1: SI1->getFalseValue(), V2, DemandedElts, Q, Depth: Depth + 1);
4190}
4191
4192// Check to see if A is both a GEP and is the incoming value for a PHI in the
4193// loop, and B is either a ptr or another GEP. If the PHI has 2 incoming values,
4194// one of them being the recursive GEP A and the other a ptr at same base and at
4195// the same/higher offset than B we are only incrementing the pointer further in
4196// loop if offset of recursive GEP is greater than 0.
4197static bool isNonEqualPointersWithRecursiveGEP(const Value *A, const Value *B,
4198 const SimplifyQuery &Q) {
4199 if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
4200 return false;
4201
4202 auto *GEPA = dyn_cast<GEPOperator>(Val: A);
4203 if (!GEPA || GEPA->getNumIndices() != 1 || !isa<Constant>(Val: GEPA->idx_begin()))
4204 return false;
4205
4206 // Handle 2 incoming PHI values with one being a recursive GEP.
4207 auto *PN = dyn_cast<PHINode>(Val: GEPA->getPointerOperand());
4208 if (!PN || PN->getNumIncomingValues() != 2)
4209 return false;
4210
4211 // Search for the recursive GEP as an incoming operand, and record that as
4212 // Step.
4213 Value *Start = nullptr;
4214 Value *Step = const_cast<Value *>(A);
4215 if (PN->getIncomingValue(i: 0) == Step)
4216 Start = PN->getIncomingValue(i: 1);
4217 else if (PN->getIncomingValue(i: 1) == Step)
4218 Start = PN->getIncomingValue(i: 0);
4219 else
4220 return false;
4221
4222 // Other incoming node base should match the B base.
4223 // StartOffset >= OffsetB && StepOffset > 0?
4224 // StartOffset <= OffsetB && StepOffset < 0?
4225 // Is non-equal if above are true.
4226 // We use stripAndAccumulateInBoundsConstantOffsets to restrict the
4227 // optimisation to inbounds GEPs only.
4228 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: Start->getType());
4229 APInt StartOffset(IndexWidth, 0);
4230 Start = Start->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StartOffset);
4231 APInt StepOffset(IndexWidth, 0);
4232 Step = Step->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StepOffset);
4233
4234 // Check if Base Pointer of Step matches the PHI.
4235 if (Step != PN)
4236 return false;
4237 APInt OffsetB(IndexWidth, 0);
4238 B = B->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: OffsetB);
4239 return Start == B &&
4240 ((StartOffset.sge(RHS: OffsetB) && StepOffset.isStrictlyPositive()) ||
4241 (StartOffset.sle(RHS: OffsetB) && StepOffset.isNegative()));
4242}
4243
4244static bool isKnownNonEqualFromContext(const Value *V1, const Value *V2,
4245 const SimplifyQuery &Q, unsigned Depth) {
4246 if (!Q.CtxI)
4247 return false;
4248
4249 // Try to infer NonEqual based on information from dominating conditions.
4250 if (Q.DC && Q.DT) {
4251 auto IsKnownNonEqualFromDominatingCondition = [&](const Value *V) {
4252 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4253 Value *Cond = BI->getCondition();
4254 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4255 if (Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()) &&
4256 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4257 /*LHSIsTrue=*/true, Depth)
4258 .value_or(u: false))
4259 return true;
4260
4261 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4262 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()) &&
4263 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4264 /*LHSIsTrue=*/false, Depth)
4265 .value_or(u: false))
4266 return true;
4267 }
4268
4269 return false;
4270 };
4271
4272 if (IsKnownNonEqualFromDominatingCondition(V1) ||
4273 IsKnownNonEqualFromDominatingCondition(V2))
4274 return true;
4275 }
4276
4277 if (!Q.AC)
4278 return false;
4279
4280 // Try to infer NonEqual based on information from assumptions.
4281 for (auto &AssumeVH : Q.AC->assumptionsFor(V: V1)) {
4282 if (!AssumeVH)
4283 continue;
4284 CallInst *I = cast<CallInst>(Val&: AssumeVH);
4285
4286 assert(I->getFunction() == Q.CtxI->getFunction() &&
4287 "Got assumption for the wrong function!");
4288 assert(I->getIntrinsicID() == Intrinsic::assume &&
4289 "must be an assume intrinsic");
4290
4291 if (isImpliedCondition(LHS: I->getArgOperand(i: 0), RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4292 /*LHSIsTrue=*/true, Depth)
4293 .value_or(u: false) &&
4294 isValidAssumeForContext(I, Q))
4295 return true;
4296 }
4297
4298 return false;
4299}
4300
4301static bool isNonEqualURem(const Value *X, const Value *Rem,
4302 const SimplifyQuery &Q) {
4303 const Value *Y;
4304 if (!match(V: Rem, P: m_URem(L: m_Specific(V: X), R: m_Value(V&: Y))))
4305 return false;
4306
4307 // For a defined urem, X != X urem Y exactly when X u>= Y.
4308 // isTruePredicate does not handle UGE, so use the equivalent Y u<= X.
4309 if (isTruePredicate(Pred: ICmpInst::ICMP_ULE, LHS: Y, RHS: X))
4310 return true;
4311
4312 std::optional<bool> Implied =
4313 isImpliedByDomCondition(Pred: ICmpInst::ICMP_UGE, LHS: X, RHS: Y, ContextI: Q.CtxI, DL: Q.DL);
4314 return Implied && *Implied;
4315}
4316
4317/// Return true if it is known that V1 != V2.
4318static bool isKnownNonEqual(const Value *V1, const Value *V2,
4319 const APInt &DemandedElts, const SimplifyQuery &Q,
4320 unsigned Depth) {
4321 if (V1 == V2)
4322 return false;
4323 if (V1->getType() != V2->getType())
4324 // We can't look through casts yet.
4325 return false;
4326
4327 if (Depth >= MaxAnalysisRecursionDepth)
4328 return false;
4329
4330 // See if we can recurse through (exactly one of) our operands. This
4331 // requires our operation be 1-to-1 and map every input value to exactly
4332 // one output value. Such an operation is invertible.
4333 auto *O1 = dyn_cast<Operator>(Val: V1);
4334 auto *O2 = dyn_cast<Operator>(Val: V2);
4335 if (O1 && O2 && O1->getOpcode() == O2->getOpcode()) {
4336 if (auto Values = getInvertibleOperands(Op1: O1, Op2: O2))
4337 return isKnownNonEqual(V1: Values->first, V2: Values->second, DemandedElts, Q,
4338 Depth: Depth + 1);
4339
4340 if (const PHINode *PN1 = dyn_cast<PHINode>(Val: V1)) {
4341 const PHINode *PN2 = cast<PHINode>(Val: V2);
4342 // FIXME: This is missing a generalization to handle the case where one is
4343 // a PHI and another one isn't.
4344 if (isNonEqualPHIs(PN1, PN2, DemandedElts, Q, Depth))
4345 return true;
4346 };
4347 }
4348
4349 if (isModifyingBinopOfNonZero(V1, V2, DemandedElts, Q, Depth) ||
4350 isModifyingBinopOfNonZero(V1: V2, V2: V1, DemandedElts, Q, Depth))
4351 return true;
4352
4353 if (isNonEqualMul(V1, V2, DemandedElts, Q, Depth) ||
4354 isNonEqualMul(V1: V2, V2: V1, DemandedElts, Q, Depth))
4355 return true;
4356
4357 if (isNonEqualShl(V1, V2, DemandedElts, Q, Depth) ||
4358 isNonEqualShl(V1: V2, V2: V1, DemandedElts, Q, Depth))
4359 return true;
4360
4361 if (V1->getType()->isIntOrIntVectorTy()) {
4362 // Are any known bits in V1 contradictory to known bits in V2? If V1
4363 // has a known zero where V2 has a known one, they must not be equal.
4364 KnownBits Known1 = computeKnownBits(V: V1, DemandedElts, Q, Depth);
4365 if (!Known1.isUnknown()) {
4366 KnownBits Known2 = computeKnownBits(V: V2, DemandedElts, Q, Depth);
4367 if (Known1.Zero.intersects(RHS: Known2.One) ||
4368 Known2.Zero.intersects(RHS: Known1.One))
4369 return true;
4370 }
4371 }
4372
4373 if (isNonEqualSelect(V1, V2, DemandedElts, Q, Depth) ||
4374 isNonEqualSelect(V1: V2, V2: V1, DemandedElts, Q, Depth))
4375 return true;
4376
4377 if (isNonEqualPointersWithRecursiveGEP(A: V1, B: V2, Q) ||
4378 isNonEqualPointersWithRecursiveGEP(A: V2, B: V1, Q))
4379 return true;
4380
4381 Value *A, *B;
4382 // PtrToInts are NonEqual if their Ptrs are NonEqual.
4383 // Check PtrToInt type matches the pointer size.
4384 if (match(V: V1, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: A))) &&
4385 match(V: V2, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: B))))
4386 return isKnownNonEqual(V1: A, V2: B, DemandedElts, Q, Depth: Depth + 1);
4387
4388 if (isNonEqualURem(X: V1, Rem: V2, Q) || isNonEqualURem(X: V2, Rem: V1, Q))
4389 return true;
4390
4391 if (isKnownNonEqualFromContext(V1, V2, Q, Depth))
4392 return true;
4393
4394 return false;
4395}
4396
4397/// For vector constants, loop over the elements and find the constant with the
4398/// minimum number of sign bits. Return 0 if the value is not a vector constant
4399/// or if any element was not analyzed; otherwise, return the count for the
4400/// element with the minimum number of sign bits.
4401static unsigned computeNumSignBitsVectorConstant(const Value *V,
4402 const APInt &DemandedElts,
4403 unsigned TyBits) {
4404 const auto *CV = dyn_cast<Constant>(Val: V);
4405 if (!CV || !isa<FixedVectorType>(Val: CV->getType()))
4406 return 0;
4407
4408 unsigned MinSignBits = TyBits;
4409 unsigned NumElts = cast<FixedVectorType>(Val: CV->getType())->getNumElements();
4410 for (unsigned i = 0; i != NumElts; ++i) {
4411 if (!DemandedElts[i])
4412 continue;
4413 // If we find a non-ConstantInt, bail out.
4414 auto *Elt = dyn_cast_or_null<ConstantInt>(Val: CV->getAggregateElement(Elt: i));
4415 if (!Elt)
4416 return 0;
4417
4418 MinSignBits = std::min(a: MinSignBits, b: Elt->getValue().getNumSignBits());
4419 }
4420
4421 return MinSignBits;
4422}
4423
4424static unsigned ComputeNumSignBitsImpl(const Value *V,
4425 const APInt &DemandedElts,
4426 const SimplifyQuery &Q, unsigned Depth);
4427
4428static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
4429 const SimplifyQuery &Q, unsigned Depth) {
4430 unsigned Result = ComputeNumSignBitsImpl(V, DemandedElts, Q, Depth);
4431 assert(Result > 0 && "At least one sign bit needs to be present!");
4432 return Result;
4433}
4434
4435/// Return the number of times the sign bit of the register is replicated into
4436/// the other bits. We know that at least 1 bit is always equal to the sign bit
4437/// (itself), but other cases can give us information. For example, immediately
4438/// after an "ashr X, 2", we know that the top 3 bits are all equal to each
4439/// other, so we return 3. For vectors, return the number of sign bits for the
4440/// vector element with the minimum number of known sign bits of the demanded
4441/// elements in the vector specified by DemandedElts.
4442static unsigned ComputeNumSignBitsImpl(const Value *V,
4443 const APInt &DemandedElts,
4444 const SimplifyQuery &Q, unsigned Depth) {
4445 Type *Ty = V->getType();
4446#ifndef NDEBUG
4447 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
4448
4449 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
4450 assert(
4451 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
4452 "DemandedElt width should equal the fixed vector number of elements");
4453 } else {
4454 assert(DemandedElts == APInt(1, 1) &&
4455 "DemandedElt width should be 1 for scalars");
4456 }
4457#endif
4458
4459 // We return the minimum number of sign bits that are guaranteed to be present
4460 // in V, so for undef we have to conservatively return 1. We don't have the
4461 // same behavior for poison though -- that's a FIXME today.
4462
4463 Type *ScalarTy = Ty->getScalarType();
4464 unsigned TyBits = ScalarTy->isPointerTy() ?
4465 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
4466 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
4467
4468 unsigned Tmp, Tmp2;
4469 unsigned FirstAnswer = 1;
4470
4471 // Note that ConstantInt is handled by the general computeKnownBits case
4472 // below.
4473
4474 if (Depth == MaxAnalysisRecursionDepth)
4475 return 1;
4476
4477 if (auto *U = dyn_cast<Operator>(Val: V)) {
4478 switch (Operator::getOpcode(V)) {
4479 default: break;
4480 case Instruction::BitCast: {
4481 Value *Src = U->getOperand(i: 0);
4482 Type *SrcTy = Src->getType();
4483
4484 // Skip if the source type is not an integer or integer vector type
4485 // This ensures we only process integer-like types
4486 if (!SrcTy->isIntOrIntVectorTy())
4487 break;
4488
4489 unsigned SrcBits = SrcTy->getScalarSizeInBits();
4490
4491 // Bitcast 'large element' scalar/vector to 'small element' vector.
4492 if ((SrcBits % TyBits) != 0)
4493 break;
4494
4495 // Only proceed if the destination type is a fixed-size vector
4496 if (isa<FixedVectorType>(Val: Ty)) {
4497 // Fast case - sign splat can be simply split across the small elements.
4498 // This works for both vector and scalar sources
4499 Tmp = ComputeNumSignBits(V: Src, Q, Depth: Depth + 1);
4500 if (Tmp == SrcBits)
4501 return TyBits;
4502 }
4503 break;
4504 }
4505 case Instruction::SExt:
4506 Tmp = TyBits - U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4507 return ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1) +
4508 Tmp;
4509
4510 case Instruction::SDiv: {
4511 const APInt *Denominator;
4512 // sdiv X, C -> adds log(C) sign bits.
4513 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4514
4515 // Ignore non-positive denominator.
4516 if (!Denominator->isStrictlyPositive())
4517 break;
4518
4519 // Calculate the incoming numerator bits.
4520 unsigned NumBits =
4521 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4522
4523 // Add floor(log(C)) bits to the numerator bits.
4524 return std::min(a: TyBits, b: NumBits + Denominator->logBase2());
4525 }
4526 break;
4527 }
4528
4529 case Instruction::SRem: {
4530 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4531
4532 const APInt *Denominator;
4533 // srem X, C -> we know that the result is within [-C+1,C) when C is a
4534 // positive constant. This let us put a lower bound on the number of sign
4535 // bits.
4536 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4537
4538 // Ignore non-positive denominator.
4539 if (Denominator->isStrictlyPositive()) {
4540 // Calculate the leading sign bit constraints by examining the
4541 // denominator. Given that the denominator is positive, there are two
4542 // cases:
4543 //
4544 // 1. The numerator is positive. The result range is [0,C) and
4545 // [0,C) u< (1 << ceilLogBase2(C)).
4546 //
4547 // 2. The numerator is negative. Then the result range is (-C,0] and
4548 // integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
4549 //
4550 // Thus a lower bound on the number of sign bits is `TyBits -
4551 // ceilLogBase2(C)`.
4552
4553 unsigned ResBits = TyBits - Denominator->ceilLogBase2();
4554 Tmp = std::max(a: Tmp, b: ResBits);
4555 }
4556 }
4557 return Tmp;
4558 }
4559
4560 case Instruction::AShr: {
4561 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4562 // ashr X, C -> adds C sign bits. Vectors too.
4563 const APInt *ShAmt;
4564 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4565 if (ShAmt->uge(RHS: TyBits))
4566 break; // Bad shift.
4567 unsigned ShAmtLimited = ShAmt->getZExtValue();
4568 Tmp += ShAmtLimited;
4569 if (Tmp > TyBits) Tmp = TyBits;
4570 }
4571 return Tmp;
4572 }
4573 case Instruction::Shl: {
4574 const APInt *ShAmt;
4575 Value *X = nullptr;
4576 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4577 // shl destroys sign bits.
4578 if (ShAmt->uge(RHS: TyBits))
4579 break; // Bad shift.
4580 // We can look through a zext (more or less treating it as a sext) if
4581 // all extended bits are shifted out.
4582 if (match(V: U->getOperand(i: 0), P: m_ZExt(Op: m_Value(V&: X))) &&
4583 ShAmt->uge(RHS: TyBits - X->getType()->getScalarSizeInBits())) {
4584 Tmp = ComputeNumSignBits(V: X, DemandedElts, Q, Depth: Depth + 1);
4585 Tmp += TyBits - X->getType()->getScalarSizeInBits();
4586 } else
4587 Tmp =
4588 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4589 if (ShAmt->uge(RHS: Tmp))
4590 break; // Shifted all sign bits out.
4591 Tmp2 = ShAmt->getZExtValue();
4592 return Tmp - Tmp2;
4593 }
4594 break;
4595 }
4596 case Instruction::And:
4597 case Instruction::Or:
4598 case Instruction::Xor: // NOT is handled here.
4599 // Logical binary ops preserve the number of sign bits at the worst.
4600 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4601 if (Tmp != 1) {
4602 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4603 FirstAnswer = std::min(a: Tmp, b: Tmp2);
4604 // We computed what we know about the sign bits as our first
4605 // answer. Now proceed to the generic code that uses
4606 // computeKnownBits, and pick whichever answer is better.
4607 }
4608 break;
4609
4610 case Instruction::Select: {
4611 // If we have a clamp pattern, we know that the number of sign bits will
4612 // be the minimum of the clamp min/max range.
4613 const Value *X;
4614 const APInt *CLow, *CHigh;
4615 if (isSignedMinMaxClamp(Select: U, In&: X, CLow, CHigh))
4616 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4617
4618 Tmp = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4619 if (Tmp == 1)
4620 break;
4621 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 2), DemandedElts, Q, Depth: Depth + 1);
4622 return std::min(a: Tmp, b: Tmp2);
4623 }
4624
4625 case Instruction::Add:
4626 // Add can have at most one carry bit. Thus we know that the output
4627 // is, at worst, one more bit than the inputs.
4628 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4629 if (Tmp == 1) break;
4630
4631 // Special case decrementing a value (ADD X, -1):
4632 if (const auto *CRHS = dyn_cast<Constant>(Val: U->getOperand(i: 1)))
4633 if (CRHS->isAllOnesValue()) {
4634 KnownBits Known(TyBits);
4635 computeKnownBits(V: U->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
4636
4637 // If the input is known to be 0 or 1, the output is 0/-1, which is
4638 // all sign bits set.
4639 if ((Known.Zero | 1).isAllOnes())
4640 return TyBits;
4641
4642 // If we are subtracting one from a positive number, there is no carry
4643 // out of the result.
4644 if (Known.isNonNegative())
4645 return Tmp;
4646 }
4647
4648 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4649 if (Tmp2 == 1)
4650 break;
4651 return std::min(a: Tmp, b: Tmp2) - 1;
4652
4653 case Instruction::Sub:
4654 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4655 if (Tmp2 == 1)
4656 break;
4657
4658 // Handle NEG.
4659 if (const auto *CLHS = dyn_cast<Constant>(Val: U->getOperand(i: 0)))
4660 if (CLHS->isNullValue()) {
4661 KnownBits Known(TyBits);
4662 computeKnownBits(V: U->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
4663 // If the input is known to be 0 or 1, the output is 0/-1, which is
4664 // all sign bits set.
4665 if ((Known.Zero | 1).isAllOnes())
4666 return TyBits;
4667
4668 // If the input is known to be positive (the sign bit is known clear),
4669 // the output of the NEG has the same number of sign bits as the
4670 // input.
4671 if (Known.isNonNegative())
4672 return Tmp2;
4673
4674 // Otherwise, we treat this like a SUB.
4675 }
4676
4677 // Sub can have at most one carry bit. Thus we know that the output
4678 // is, at worst, one more bit than the inputs.
4679 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4680 if (Tmp == 1)
4681 break;
4682 return std::min(a: Tmp, b: Tmp2) - 1;
4683
4684 case Instruction::Mul: {
4685 // The output of the Mul can be at most twice the valid bits in the
4686 // inputs.
4687 unsigned SignBitsOp0 =
4688 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4689 if (SignBitsOp0 == 1)
4690 break;
4691 unsigned SignBitsOp1 =
4692 ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4693 if (SignBitsOp1 == 1)
4694 break;
4695 unsigned OutValidBits =
4696 (TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
4697 return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
4698 }
4699
4700 case Instruction::PHI: {
4701 const PHINode *PN = cast<PHINode>(Val: U);
4702 unsigned NumIncomingValues = PN->getNumIncomingValues();
4703 // Don't analyze large in-degree PHIs.
4704 if (NumIncomingValues > 4) break;
4705 // Unreachable blocks may have zero-operand PHI nodes.
4706 if (NumIncomingValues == 0) break;
4707
4708 // Take the minimum of all incoming values. This can't infinitely loop
4709 // because of our depth threshold.
4710 SimplifyQuery RecQ = Q.getWithoutCondContext();
4711 Tmp = TyBits;
4712 for (unsigned i = 0, e = NumIncomingValues; i != e; ++i) {
4713 if (Tmp == 1) return Tmp;
4714 RecQ.CtxI = PN->getIncomingBlock(i)->getTerminator();
4715 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(V: PN->getIncomingValue(i),
4716 DemandedElts, Q: RecQ, Depth: Depth + 1));
4717 }
4718 return Tmp;
4719 }
4720
4721 case Instruction::Trunc: {
4722 // If the input contained enough sign bits that some remain after the
4723 // truncation, then we can make use of that. Otherwise we don't know
4724 // anything.
4725 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4726 unsigned OperandTyBits = U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4727 if (Tmp > (OperandTyBits - TyBits))
4728 return Tmp - (OperandTyBits - TyBits);
4729
4730 return 1;
4731 }
4732
4733 case Instruction::ExtractElement:
4734 // Look through extract element. At the moment we keep this simple and
4735 // skip tracking the specific element. But at least we might find
4736 // information valid for all elements of the vector (for example if vector
4737 // is sign extended, shifted, etc).
4738 return ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4739
4740 case Instruction::ShuffleVector: {
4741 // Collect the minimum number of sign bits that are shared by every vector
4742 // element referenced by the shuffle.
4743 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: U);
4744 if (!Shuf) {
4745 // FIXME: Add support for shufflevector constant expressions.
4746 return 1;
4747 }
4748 APInt DemandedLHS, DemandedRHS;
4749 // For undef elements, we don't know anything about the common state of
4750 // the shuffle result.
4751 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
4752 return 1;
4753 Tmp = std::numeric_limits<unsigned>::max();
4754 if (!!DemandedLHS) {
4755 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
4756 Tmp = ComputeNumSignBits(V: LHS, DemandedElts: DemandedLHS, Q, Depth: Depth + 1);
4757 }
4758 // If we don't know anything, early out and try computeKnownBits
4759 // fall-back.
4760 if (Tmp == 1)
4761 break;
4762 if (!!DemandedRHS) {
4763 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
4764 Tmp2 = ComputeNumSignBits(V: RHS, DemandedElts: DemandedRHS, Q, Depth: Depth + 1);
4765 Tmp = std::min(a: Tmp, b: Tmp2);
4766 }
4767 // If we don't know anything, early out and try computeKnownBits
4768 // fall-back.
4769 if (Tmp == 1)
4770 break;
4771 assert(Tmp <= TyBits && "Failed to determine minimum sign bits");
4772 return Tmp;
4773 }
4774 case Instruction::Call: {
4775 if (const auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
4776 switch (II->getIntrinsicID()) {
4777 default:
4778 break;
4779 case Intrinsic::abs:
4780 Tmp =
4781 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4782 if (Tmp == 1)
4783 break;
4784
4785 // Absolute value reduces number of sign bits by at most 1.
4786 return Tmp - 1;
4787 case Intrinsic::smin:
4788 case Intrinsic::smax: {
4789 const APInt *CLow, *CHigh;
4790 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
4791 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4792 }
4793 }
4794 }
4795 }
4796 }
4797 }
4798
4799 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4800 // use this information.
4801
4802 // If we can examine all elements of a vector constant successfully, we're
4803 // done (we can't do any better than that). If not, keep trying.
4804 if (unsigned VecSignBits =
4805 computeNumSignBitsVectorConstant(V, DemandedElts, TyBits))
4806 return VecSignBits;
4807
4808 KnownBits Known(TyBits);
4809 computeKnownBits(V, DemandedElts, Known, Q, Depth);
4810
4811 // If we know that the sign bit is either zero or one, determine the number of
4812 // identical bits in the top of the input value.
4813 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
4814}
4815
4816Intrinsic::ID llvm::getIntrinsicForCallSite(const CallBase &CB,
4817 const TargetLibraryInfo *TLI) {
4818 const Function *F = CB.getCalledFunction();
4819 if (!F)
4820 return Intrinsic::not_intrinsic;
4821
4822 if (F->isIntrinsic())
4823 return F->getIntrinsicID();
4824
4825 // We are going to infer semantics of a library function based on mapping it
4826 // to an LLVM intrinsic. Check that the library function is available from
4827 // this callbase and in this environment.
4828 if (F->hasLocalLinkage() || !TLI || !CB.onlyReadsMemory())
4829 return Intrinsic::not_intrinsic;
4830
4831 LibFunc Func = TLI->getLibFunc(CB);
4832 if (Func == NotLibFunc)
4833 return Intrinsic::not_intrinsic;
4834
4835 switch (Func) {
4836 default:
4837 break;
4838 case LibFunc_sin:
4839 case LibFunc_sinf:
4840 case LibFunc_sinl:
4841 return Intrinsic::sin;
4842 case LibFunc_cos:
4843 case LibFunc_cosf:
4844 case LibFunc_cosl:
4845 return Intrinsic::cos;
4846 case LibFunc_tan:
4847 case LibFunc_tanf:
4848 case LibFunc_tanl:
4849 return Intrinsic::tan;
4850 case LibFunc_asin:
4851 case LibFunc_asinf:
4852 case LibFunc_asinl:
4853 return Intrinsic::asin;
4854 case LibFunc_acos:
4855 case LibFunc_acosf:
4856 case LibFunc_acosl:
4857 return Intrinsic::acos;
4858 case LibFunc_atan:
4859 case LibFunc_atanf:
4860 case LibFunc_atanl:
4861 return Intrinsic::atan;
4862 case LibFunc_atan2:
4863 case LibFunc_atan2f:
4864 case LibFunc_atan2l:
4865 return Intrinsic::atan2;
4866 case LibFunc_sinh:
4867 case LibFunc_sinhf:
4868 case LibFunc_sinhl:
4869 return Intrinsic::sinh;
4870 case LibFunc_cosh:
4871 case LibFunc_coshf:
4872 case LibFunc_coshl:
4873 return Intrinsic::cosh;
4874 case LibFunc_tanh:
4875 case LibFunc_tanhf:
4876 case LibFunc_tanhl:
4877 return Intrinsic::tanh;
4878 case LibFunc_exp:
4879 case LibFunc_expf:
4880 case LibFunc_expl:
4881 return Intrinsic::exp;
4882 case LibFunc_exp2:
4883 case LibFunc_exp2f:
4884 case LibFunc_exp2l:
4885 return Intrinsic::exp2;
4886 case LibFunc_exp10:
4887 case LibFunc_exp10f:
4888 case LibFunc_exp10l:
4889 return Intrinsic::exp10;
4890 case LibFunc_log:
4891 case LibFunc_logf:
4892 case LibFunc_logl:
4893 return Intrinsic::log;
4894 case LibFunc_log10:
4895 case LibFunc_log10f:
4896 case LibFunc_log10l:
4897 return Intrinsic::log10;
4898 case LibFunc_log2:
4899 case LibFunc_log2f:
4900 case LibFunc_log2l:
4901 return Intrinsic::log2;
4902 case LibFunc_fabs:
4903 case LibFunc_fabsf:
4904 case LibFunc_fabsl:
4905 return Intrinsic::fabs;
4906 case LibFunc_fmin:
4907 case LibFunc_fminf:
4908 case LibFunc_fminl:
4909 return Intrinsic::minnum;
4910 case LibFunc_fmax:
4911 case LibFunc_fmaxf:
4912 case LibFunc_fmaxl:
4913 return Intrinsic::maxnum;
4914 case LibFunc_copysign:
4915 case LibFunc_copysignf:
4916 case LibFunc_copysignl:
4917 return Intrinsic::copysign;
4918 case LibFunc_floor:
4919 case LibFunc_floorf:
4920 case LibFunc_floorl:
4921 return Intrinsic::floor;
4922 case LibFunc_ceil:
4923 case LibFunc_ceilf:
4924 case LibFunc_ceill:
4925 return Intrinsic::ceil;
4926 case LibFunc_trunc:
4927 case LibFunc_truncf:
4928 case LibFunc_truncl:
4929 return Intrinsic::trunc;
4930 case LibFunc_rint:
4931 case LibFunc_rintf:
4932 case LibFunc_rintl:
4933 return Intrinsic::rint;
4934 case LibFunc_nearbyint:
4935 case LibFunc_nearbyintf:
4936 case LibFunc_nearbyintl:
4937 return Intrinsic::nearbyint;
4938 case LibFunc_round:
4939 case LibFunc_roundf:
4940 case LibFunc_roundl:
4941 return Intrinsic::round;
4942 case LibFunc_roundeven:
4943 case LibFunc_roundevenf:
4944 case LibFunc_roundevenl:
4945 return Intrinsic::roundeven;
4946 case LibFunc_pow:
4947 case LibFunc_powf:
4948 case LibFunc_powl:
4949 return Intrinsic::pow;
4950 case LibFunc_sqrt:
4951 case LibFunc_sqrtf:
4952 case LibFunc_sqrtl:
4953 return Intrinsic::sqrt;
4954 }
4955
4956 return Intrinsic::not_intrinsic;
4957}
4958
4959/// Given an exploded icmp instruction, return true if the comparison only
4960/// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if
4961/// the result of the comparison is true when the input value is signed.
4962bool llvm::isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
4963 bool &TrueIfSigned) {
4964 switch (Pred) {
4965 case ICmpInst::ICMP_SLT: // True if LHS s< 0
4966 TrueIfSigned = true;
4967 return RHS.isZero();
4968 case ICmpInst::ICMP_SLE: // True if LHS s<= -1
4969 TrueIfSigned = true;
4970 return RHS.isAllOnes();
4971 case ICmpInst::ICMP_SGT: // True if LHS s> -1
4972 TrueIfSigned = false;
4973 return RHS.isAllOnes();
4974 case ICmpInst::ICMP_SGE: // True if LHS s>= 0
4975 TrueIfSigned = false;
4976 return RHS.isZero();
4977 case ICmpInst::ICMP_UGT:
4978 // True if LHS u> RHS and RHS == sign-bit-mask - 1
4979 TrueIfSigned = true;
4980 return RHS.isMaxSignedValue();
4981 case ICmpInst::ICMP_UGE:
4982 // True if LHS u>= RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
4983 TrueIfSigned = true;
4984 return RHS.isMinSignedValue();
4985 case ICmpInst::ICMP_ULT:
4986 // True if LHS u< RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
4987 TrueIfSigned = false;
4988 return RHS.isMinSignedValue();
4989 case ICmpInst::ICMP_ULE:
4990 // True if LHS u<= RHS and RHS == sign-bit-mask - 1
4991 TrueIfSigned = false;
4992 return RHS.isMaxSignedValue();
4993 default:
4994 return false;
4995 }
4996}
4997
4998static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
4999 bool CondIsTrue,
5000 const Instruction *CtxI,
5001 KnownFPClass &KnownFromContext,
5002 unsigned Depth = 0) {
5003 Value *A, *B;
5004 if (Depth < MaxAnalysisRecursionDepth &&
5005 (CondIsTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B)))
5006 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))))) {
5007 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue, CtxI, KnownFromContext,
5008 Depth: Depth + 1);
5009 computeKnownFPClassFromCond(V, Cond: B, CondIsTrue, CtxI, KnownFromContext,
5010 Depth: Depth + 1);
5011 return;
5012 }
5013 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A)))) {
5014 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue: !CondIsTrue, CtxI, KnownFromContext,
5015 Depth: Depth + 1);
5016 return;
5017 }
5018 CmpPredicate Pred;
5019 Value *LHS;
5020 uint64_t ClassVal = 0;
5021 const APFloat *CRHS;
5022 const APInt *RHS;
5023 if (match(V: Cond, P: m_FCmp(Pred, L: m_Value(V&: LHS), R: m_APFloat(Res&: CRHS)))) {
5024 auto [CmpVal, MaskIfTrue, MaskIfFalse] = fcmpImpliesClass(
5025 Pred, F: *cast<Instruction>(Val: Cond)->getParent()->getParent(), LHS, ConstRHS: *CRHS,
5026 LookThroughSrc: LHS != V);
5027 if (CmpVal == V)
5028 KnownFromContext.knownNot(RuleOut: ~(CondIsTrue ? MaskIfTrue : MaskIfFalse));
5029 } else if (match(V: Cond, P: m_Intrinsic<Intrinsic::is_fpclass>(
5030 Ops: m_Specific(V), Ops: m_ConstantInt(V&: ClassVal)))) {
5031 FPClassTest Mask = static_cast<FPClassTest>(ClassVal);
5032 KnownFromContext.knownNot(RuleOut: CondIsTrue ? ~Mask : Mask);
5033 } else if (match(V: Cond, P: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Specific(V)),
5034 R: m_APInt(Res&: RHS)))) {
5035 bool TrueIfSigned;
5036 if (!isSignBitCheck(Pred, RHS: *RHS, TrueIfSigned))
5037 return;
5038 if (TrueIfSigned == CondIsTrue)
5039 KnownFromContext.signBitMustBeOne();
5040 else
5041 KnownFromContext.signBitMustBeZero();
5042 }
5043}
5044
5045/// Compute the minimum and maximum values (inclusive) for the exponent of \p V,
5046/// assuming it is not nan. Returns {min, max, max-assuming-nonzero}. A value
5047/// frexp(0) = 0, so the tighter max-assuming-nonzero bound is only usable when
5048/// \p V is known not to be a logical zero (e.g., for fabs(x) < 0.25, the non-0
5049/// exponent range is [-149, -2], but the 0 edge case is above this range).
5050static std::tuple<int, int, int>
5051computeKnownExponentRangeFromContext(const Value *V, const SimplifyQuery &Q) {
5052 if (!Q.CtxI || !Q.DC || !Q.DT)
5053 return {APFloat::IEK_NaN, APFloat::IEK_Inf, APFloat::IEK_Inf};
5054
5055 // Intersect the bounds implied by every dominating condition, keeping the
5056 // tightest maximum. A value may participate in multiple compares
5057 // (e.g. fabs(x) < 2.0 and fabs(x) < 1.0), and the tighter one wins.
5058 int MaxExp = APFloat::IEK_Inf;
5059 int MaxExpNonZero = APFloat::IEK_Inf;
5060
5061 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
5062 CmpPredicate Pred;
5063 const APFloat *LimitC;
5064 if (!match(V: BI->getCondition(),
5065 P: m_FCmp(Pred, L: m_FAbs(Op0: m_Specific(V)), R: m_Finite(V&: LimitC))))
5066 continue;
5067
5068 if (Pred == FCmpInst::FCMP_ORD || Pred == FCmpInst::FCMP_UNO ||
5069 Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE)
5070 continue;
5071
5072 // If fabs(x) <= K, implies the exponent min exp range.
5073 // if fabs(x) >= K, swap the successor
5074 bool IsLessEqual =
5075 Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE ||
5076 Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE ||
5077 Pred == FCmpInst::FCMP_OEQ || Pred == FCmpInst::FCMP_UEQ;
5078
5079 bool KnownStrictlyLess =
5080 Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_ULT ||
5081 Pred == FCmpInst::FCMP_OGE || Pred == FCmpInst::FCMP_UGE;
5082
5083 BasicBlockEdge Edge1(BI->getParent(),
5084 BI->getSuccessor(i: IsLessEqual ? 0 : 1));
5085 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent())) {
5086 // frexp returns an exponent one greater than ilogb.
5087 int Exp = ilogb(Arg: *LimitC) + 1;
5088
5089 // A strict bound fabs(V) < 2^n forces ilogb(V) <= n - 1, so the max frexp
5090 // exponent drops by one when K is exact power of two.
5091 if (KnownStrictlyLess && LimitC->getExactLog2Abs() != INT_MIN)
5092 --Exp;
5093
5094 // frexp(0) = 0, which the bound above (assuming a normal nonzero value)
5095 // may exclude.
5096
5097 // TODO: Figure out lower bound to detect no-underflow.
5098 MaxExpNonZero = std::min(a: MaxExpNonZero, b: Exp);
5099 MaxExp = std::min(a: MaxExp, b: std::max(a: Exp, b: 0));
5100 }
5101 }
5102
5103 return {APFloat::IEK_NaN, MaxExp, MaxExpNonZero};
5104}
5105
5106static KnownFPClass computeKnownFPClassFromContext(const Value *V,
5107 const SimplifyQuery &Q) {
5108 KnownFPClass KnownFromContext;
5109
5110 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
5111 computeKnownFPClassFromCond(V, Cond: Q.CC->Cond, CondIsTrue: !Q.CC->Invert, CtxI: Q.CtxI,
5112 KnownFromContext);
5113
5114 if (!Q.CtxI)
5115 return KnownFromContext;
5116
5117 if (Q.DC && Q.DT) {
5118 // Handle dominating conditions.
5119 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
5120 Value *Cond = BI->getCondition();
5121
5122 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
5123 if (Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()))
5124 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, CtxI: Q.CtxI,
5125 KnownFromContext);
5126
5127 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
5128 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()))
5129 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, CtxI: Q.CtxI,
5130 KnownFromContext);
5131 }
5132 }
5133
5134 if (!Q.AC)
5135 return KnownFromContext;
5136
5137 // Try to restrict the floating-point classes based on information from
5138 // assumptions.
5139 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
5140 if (!AssumeVH)
5141 continue;
5142 CallInst *I = cast<CallInst>(Val&: AssumeVH);
5143
5144 assert(I->getFunction() == Q.CtxI->getParent()->getParent() &&
5145 "Got assumption for the wrong function!");
5146 assert(I->getIntrinsicID() == Intrinsic::assume &&
5147 "must be an assume intrinsic");
5148
5149 if (!isValidAssumeForContext(I, Q))
5150 continue;
5151
5152 computeKnownFPClassFromCond(V, Cond: I->getArgOperand(i: 0),
5153 /*CondIsTrue=*/true, CtxI: Q.CtxI, KnownFromContext);
5154 }
5155
5156 return KnownFromContext;
5157}
5158
5159void llvm::adjustKnownFPClassForSelectArm(KnownFPClass &Known, Value *Cond,
5160 Value *Arm, bool Invert,
5161 const SimplifyQuery &SQ,
5162 unsigned Depth) {
5163
5164 KnownFPClass KnownSrc;
5165 computeKnownFPClassFromCond(V: Arm, Cond,
5166 /*CondIsTrue=*/!Invert, CtxI: SQ.CtxI, KnownFromContext&: KnownSrc,
5167 Depth: Depth + 1);
5168 KnownSrc = KnownSrc.unionWith(RHS: Known);
5169 if (KnownSrc.isUnknown())
5170 return;
5171
5172 if (isGuaranteedNotToBeUndef(V: Arm, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT, Depth: Depth + 1))
5173 Known = KnownSrc;
5174}
5175
5176void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5177 FPClassTest InterestedClasses, KnownFPClass &Known,
5178 const SimplifyQuery &Q, unsigned Depth);
5179
5180static void computeKnownFPClass(const Value *V, KnownFPClass &Known,
5181 FPClassTest InterestedClasses,
5182 const SimplifyQuery &Q, unsigned Depth) {
5183 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5184 APInt DemandedElts =
5185 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
5186 computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Q, Depth);
5187}
5188
5189static void computeKnownFPClassForFPTrunc(const Operator *Op,
5190 const APInt &DemandedElts,
5191 FPClassTest InterestedClasses,
5192 KnownFPClass &Known,
5193 const SimplifyQuery &Q,
5194 unsigned Depth) {
5195 if ((InterestedClasses &
5196 (KnownFPClass::OrderedLessThanZeroMask | fcNan)) == fcNone)
5197 return;
5198
5199 KnownFPClass KnownSrc;
5200 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5201 Known&: KnownSrc, Q, Depth: Depth + 1);
5202 Known = KnownFPClass::fptrunc(KnownSrc);
5203}
5204
5205static constexpr KnownFPClass::MinMaxKind getMinMaxKind(Intrinsic::ID IID) {
5206 switch (IID) {
5207 case Intrinsic::minimum:
5208 return KnownFPClass::MinMaxKind::minimum;
5209 case Intrinsic::maximum:
5210 return KnownFPClass::MinMaxKind::maximum;
5211 case Intrinsic::minimumnum:
5212 return KnownFPClass::MinMaxKind::minimumnum;
5213 case Intrinsic::maximumnum:
5214 return KnownFPClass::MinMaxKind::maximumnum;
5215 case Intrinsic::minnum:
5216 return KnownFPClass::MinMaxKind::minnum;
5217 case Intrinsic::maxnum:
5218 return KnownFPClass::MinMaxKind::maxnum;
5219 default:
5220 llvm_unreachable("not a floating-point min-max intrinsic");
5221 }
5222}
5223
5224/// \return true if this is a floating point value that is known to have a
5225/// magnitude smaller than 1. i.e., fabs(X) <= 1.0 or is nan.
5226static bool isAbsoluteValueULEOne(const Value *V) {
5227 // TODO: Handle frexp
5228 // TODO: Other rounding intrinsics?
5229 // TODO: Try computeKnownExponentRangeFromContext
5230
5231 // fabs(x - floor(x)) <= 1
5232 const Value *SubFloorX;
5233 if (match(V, P: m_FSub(L: m_Value(V&: SubFloorX),
5234 R: m_Intrinsic<Intrinsic::floor>(Ops: m_Deferred(V: SubFloorX)))))
5235 return true;
5236
5237 return match(V, P: m_Intrinsic<Intrinsic::amdgcn_trig_preop>(Ops: m_Value())) ||
5238 match(V, P: m_Intrinsic<Intrinsic::amdgcn_fract>(Ops: m_Value()));
5239}
5240
5241void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5242 FPClassTest InterestedClasses, KnownFPClass &Known,
5243 const SimplifyQuery &Q, unsigned Depth) {
5244 assert(Known.isUnknown() && "should not be called with known information");
5245
5246 if (!DemandedElts) {
5247 // No demanded elts, better to assume we don't know anything.
5248 Known.resetAll();
5249 return;
5250 }
5251
5252 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
5253
5254 if (auto *CFP = dyn_cast<ConstantFP>(Val: V)) {
5255 Known = KnownFPClass(CFP->getValueAPF());
5256 return;
5257 }
5258
5259 if (isa<ConstantAggregateZero>(Val: V)) {
5260 Known.setKnownFPClasses(fcPosZero);
5261 Known.setSignBit(false);
5262 return;
5263 }
5264
5265 if (isa<PoisonValue>(Val: V)) {
5266 Known.setKnownFPClasses(fcNone);
5267 Known.setSignBit(false);
5268 return;
5269 }
5270
5271 // Try to handle fixed width vector constants
5272 auto *VFVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5273 const Constant *CV = dyn_cast<Constant>(Val: V);
5274 if (VFVTy && CV) {
5275 Known.setKnownFPClasses(fcNone);
5276 bool SignBitAllZero = true;
5277 bool SignBitAllOne = true;
5278
5279 // For vectors, verify that each element is not NaN.
5280 unsigned NumElts = VFVTy->getNumElements();
5281 for (unsigned i = 0; i != NumElts; ++i) {
5282 if (!DemandedElts[i])
5283 continue;
5284
5285 Constant *Elt = CV->getAggregateElement(Elt: i);
5286 if (!Elt) {
5287 Known = KnownFPClass();
5288 return;
5289 }
5290 if (isa<PoisonValue>(Val: Elt))
5291 continue;
5292 auto *CElt = dyn_cast<ConstantFP>(Val: Elt);
5293 if (!CElt) {
5294 Known = KnownFPClass();
5295 return;
5296 }
5297
5298 const APFloat &C = CElt->getValueAPF();
5299 Known.setKnownFPClasses(Known.getKnownFPClasses() | C.classify());
5300 if (C.isNegative())
5301 SignBitAllZero = false;
5302 else
5303 SignBitAllOne = false;
5304 }
5305 if (SignBitAllOne != SignBitAllZero)
5306 Known.setSignBit(SignBitAllOne);
5307 return;
5308 }
5309
5310 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) {
5311 Known.setKnownFPClasses(fcNone);
5312 for (size_t I = 0, E = CDS->getNumElements(); I != E; ++I)
5313 Known |= CDS->getElementAsAPFloat(i: I).classify();
5314 return;
5315 }
5316
5317 if (const auto *CA = dyn_cast<ConstantAggregate>(Val: V)) {
5318 // TODO: Handle complex aggregates
5319 Known.setKnownFPClasses(fcNone);
5320 for (const Use &Op : CA->operands()) {
5321 auto *CFP = dyn_cast<ConstantFP>(Val: Op.get());
5322 if (!CFP) {
5323 Known = KnownFPClass();
5324 return;
5325 }
5326
5327 Known |= CFP->getValueAPF().classify();
5328 }
5329
5330 return;
5331 }
5332
5333 FPClassTest KnownNotFromFlags = fcNone;
5334 if (const auto *CB = dyn_cast<CallBase>(Val: V))
5335 KnownNotFromFlags |= CB->getRetNoFPClass();
5336 else if (const auto *Arg = dyn_cast<Argument>(Val: V))
5337 KnownNotFromFlags |= Arg->getNoFPClass();
5338
5339 const Operator *Op = dyn_cast<Operator>(Val: V);
5340 if (const FPMathOperator *FPOp = dyn_cast_or_null<FPMathOperator>(Val: Op)) {
5341 if (FPOp->hasNoNaNs())
5342 KnownNotFromFlags |= fcNan;
5343 if (FPOp->hasNoInfs())
5344 KnownNotFromFlags |= fcInf;
5345 }
5346
5347 KnownFPClass AssumedClasses = computeKnownFPClassFromContext(V, Q);
5348 KnownNotFromFlags |= ~AssumedClasses.getKnownFPClasses();
5349
5350 // We no longer need to find out about these bits from inputs if we can
5351 // assume this from flags/attributes.
5352 InterestedClasses &= ~KnownNotFromFlags;
5353
5354 llvm::scope_exit ClearClassesFromFlags([=, &Known] {
5355 Known.knownNot(RuleOut: KnownNotFromFlags);
5356 if (!Known.getSignBit() && AssumedClasses.getSignBit()) {
5357 if (*AssumedClasses.getSignBit())
5358 Known.signBitMustBeOne();
5359 else
5360 Known.signBitMustBeZero();
5361 }
5362 });
5363
5364 if (!Op)
5365 return;
5366
5367 // All recursive calls that increase depth must come after this.
5368 if (Depth == MaxAnalysisRecursionDepth)
5369 return;
5370
5371 const unsigned Opc = Op->getOpcode();
5372 switch (Opc) {
5373 case Instruction::FNeg: {
5374 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5375 Known, Q, Depth: Depth + 1);
5376 Known.fneg();
5377 break;
5378 }
5379 case Instruction::Select: {
5380 auto ComputeForArm = [&](Value *Arm, bool Invert) {
5381 KnownFPClass Res;
5382 computeKnownFPClass(V: Arm, DemandedElts, InterestedClasses, Known&: Res, Q,
5383 Depth: Depth + 1);
5384 adjustKnownFPClassForSelectArm(Known&: Res, Cond: Op->getOperand(i: 0), Arm, Invert, SQ: Q,
5385 Depth);
5386 return Res;
5387 };
5388 // Only known if known in both the LHS and RHS.
5389 Known =
5390 ComputeForArm(Op->getOperand(i: 1), /*Invert=*/false)
5391 .intersectWith(RHS: ComputeForArm(Op->getOperand(i: 2), /*Invert=*/true));
5392 break;
5393 }
5394 case Instruction::Load: {
5395 const MDNode *NoFPClass =
5396 cast<LoadInst>(Val: Op)->getMetadata(KindID: LLVMContext::MD_nofpclass);
5397 if (!NoFPClass)
5398 break;
5399
5400 ConstantInt *MaskVal =
5401 mdconst::extract<ConstantInt>(MD: NoFPClass->getOperand(I: 0));
5402 Known.knownNot(RuleOut: static_cast<FPClassTest>(MaskVal->getZExtValue()));
5403 break;
5404 }
5405 case Instruction::Call: {
5406 const CallInst *II = cast<CallInst>(Val: Op);
5407 const Intrinsic::ID IID = II->getIntrinsicID();
5408 switch (IID) {
5409 case Intrinsic::fabs: {
5410 if ((InterestedClasses & (fcNan | fcPositive)) != fcNone) {
5411 // If we only care about the sign bit we don't need to inspect the
5412 // operand.
5413 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5414 InterestedClasses, Known, Q, Depth: Depth + 1);
5415 }
5416
5417 Known.fabs();
5418 break;
5419 }
5420 case Intrinsic::copysign: {
5421 KnownFPClass KnownSign;
5422
5423 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5424 Known, Q, Depth: Depth + 1);
5425 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5426 Known&: KnownSign, Q, Depth: Depth + 1);
5427 Known.copysign(Sign: KnownSign);
5428 break;
5429 }
5430 case Intrinsic::fma:
5431 case Intrinsic::fmuladd: {
5432 if ((InterestedClasses & fcNegative) == fcNone)
5433 break;
5434
5435 // FIXME: This should check isGuaranteedNotToBeUndef
5436 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) {
5437 KnownFPClass KnownSrc, KnownAddend;
5438 computeKnownFPClass(V: II->getArgOperand(i: 2), DemandedElts,
5439 InterestedClasses, Known&: KnownAddend, Q, Depth: Depth + 1);
5440 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5441 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5442
5443 const Function *F = II->getFunction();
5444 const fltSemantics &FltSem =
5445 II->getType()->getScalarType()->getFltSemantics();
5446 DenormalMode Mode =
5447 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5448
5449 if (KnownNotFromFlags & fcNan) {
5450 KnownSrc.knownNot(RuleOut: fcNan);
5451 KnownAddend.knownNot(RuleOut: fcNan);
5452 }
5453
5454 if (KnownNotFromFlags & fcInf) {
5455 KnownSrc.knownNot(RuleOut: fcInf);
5456 KnownAddend.knownNot(RuleOut: fcInf);
5457 }
5458
5459 Known = KnownFPClass::fma_square(Squared: KnownSrc, Addend: KnownAddend, Mode);
5460 break;
5461 }
5462
5463 KnownFPClass KnownSrc[3];
5464 for (int I = 0; I != 3; ++I) {
5465 computeKnownFPClass(V: II->getArgOperand(i: I), DemandedElts,
5466 InterestedClasses, Known&: KnownSrc[I], Q, Depth: Depth + 1);
5467 if (KnownSrc[I].isUnknown())
5468 return;
5469
5470 if (KnownNotFromFlags & fcNan)
5471 KnownSrc[I].knownNot(RuleOut: fcNan);
5472 if (KnownNotFromFlags & fcInf)
5473 KnownSrc[I].knownNot(RuleOut: fcInf);
5474 }
5475
5476 const Function *F = II->getFunction();
5477 const fltSemantics &FltSem =
5478 II->getType()->getScalarType()->getFltSemantics();
5479 DenormalMode Mode =
5480 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5481 Known = KnownFPClass::fma(LHS: KnownSrc[0], RHS: KnownSrc[1], Addend: KnownSrc[2], Mode);
5482 break;
5483 }
5484 case Intrinsic::sqrt:
5485 case Intrinsic::experimental_constrained_sqrt: {
5486 KnownFPClass KnownSrc;
5487 FPClassTest InterestedSrcs = InterestedClasses;
5488 if (InterestedClasses & fcNan)
5489 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5490
5491 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5492 Known&: KnownSrc, Q, Depth: Depth + 1);
5493
5494 DenormalMode Mode = DenormalMode::getDynamic();
5495
5496 bool HasNSZ = Q.IIQ.hasNoSignedZeros(Op: II);
5497 if (!HasNSZ) {
5498 const Function *F = II->getFunction();
5499 const fltSemantics &FltSem =
5500 II->getType()->getScalarType()->getFltSemantics();
5501 Mode = F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5502 }
5503
5504 Known = KnownFPClass::sqrt(Src: KnownSrc, Mode);
5505 if (HasNSZ)
5506 Known.knownNot(RuleOut: fcNegZero);
5507
5508 break;
5509 }
5510 case Intrinsic::sin: {
5511 KnownFPClass KnownSrc;
5512 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5513 Known&: KnownSrc, Q, Depth: Depth + 1);
5514 Known = KnownFPClass::sin(Src: KnownSrc);
5515 break;
5516 }
5517 case Intrinsic::cos: {
5518 KnownFPClass KnownSrc;
5519 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5520 Known&: KnownSrc, Q, Depth: Depth + 1);
5521 Known = KnownFPClass::cos(Src: KnownSrc);
5522 break;
5523 }
5524 case Intrinsic::tan: {
5525 KnownFPClass KnownSrc;
5526 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5527 Known&: KnownSrc, Q, Depth: Depth + 1);
5528 Known = KnownFPClass::tan(Src: KnownSrc);
5529 break;
5530 }
5531 case Intrinsic::sinh: {
5532 KnownFPClass KnownSrc;
5533 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5534 Known&: KnownSrc, Q, Depth: Depth + 1);
5535 Known = KnownFPClass::sinh(Src: KnownSrc);
5536 break;
5537 }
5538 case Intrinsic::cosh: {
5539 KnownFPClass KnownSrc;
5540 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5541 Known&: KnownSrc, Q, Depth: Depth + 1);
5542 Known = KnownFPClass::cosh(Src: KnownSrc);
5543 break;
5544 }
5545 case Intrinsic::tanh: {
5546 KnownFPClass KnownSrc;
5547 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5548 Known&: KnownSrc, Q, Depth: Depth + 1);
5549 Known = KnownFPClass::tanh(Src: KnownSrc);
5550 break;
5551 }
5552 case Intrinsic::asin: {
5553 KnownFPClass KnownSrc;
5554 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5555 Known&: KnownSrc, Q, Depth: Depth + 1);
5556 Known = KnownFPClass::asin(Src: KnownSrc);
5557 break;
5558 }
5559 case Intrinsic::acos: {
5560 KnownFPClass KnownSrc;
5561 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5562 Known&: KnownSrc, Q, Depth: Depth + 1);
5563 Known = KnownFPClass::acos(Src: KnownSrc);
5564 break;
5565 }
5566 case Intrinsic::atan: {
5567 KnownFPClass KnownSrc;
5568 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5569 Known&: KnownSrc, Q, Depth: Depth + 1);
5570 Known = KnownFPClass::atan(Src: KnownSrc);
5571 break;
5572 }
5573 case Intrinsic::atan2: {
5574 FPClassTest InterestedY = InterestedClasses;
5575 FPClassTest InterestedX = InterestedClasses;
5576
5577 // We can rule out negative values if y cannot have a negative value.
5578 if ((InterestedClasses & fcNegFinite) != fcNone)
5579 InterestedY |= fcNegative;
5580
5581 // We can rule out positive values if y cannot have a positive value.
5582 if ((InterestedClasses & fcPosFinite) != fcNone)
5583 InterestedY |= fcPositive | fcNegSubnormal;
5584
5585 // We can rule out zero and subnormal if x cannot have a positive value.
5586 if ((InterestedClasses & (fcZero | fcSubnormal)) != fcNone)
5587 InterestedX |= fcPositive | fcNegSubnormal;
5588
5589 KnownFPClass KnownY, KnownX;
5590 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedY,
5591 Known&: KnownY, Q, Depth: Depth + 1);
5592 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses: InterestedX,
5593 Known&: KnownX, Q, Depth: Depth + 1);
5594
5595 const Function *F = II->getFunction();
5596 DenormalMode Mode =
5597 F ? F->getDenormalMode(
5598 FPType: II->getType()->getScalarType()->getFltSemantics())
5599 : DenormalMode::getDynamic();
5600 Known = KnownFPClass::atan2(LHS: KnownY, RHS: KnownX, Mode);
5601 break;
5602 }
5603 case Intrinsic::maxnum:
5604 case Intrinsic::minnum:
5605 case Intrinsic::minimum:
5606 case Intrinsic::maximum:
5607 case Intrinsic::minimumnum:
5608 case Intrinsic::maximumnum: {
5609 KnownFPClass KnownLHS, KnownRHS;
5610 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5611 Known&: KnownLHS, Q, Depth: Depth + 1);
5612 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5613 Known&: KnownRHS, Q, Depth: Depth + 1);
5614
5615 const Function *F = II->getFunction();
5616
5617 DenormalMode Mode =
5618 F ? F->getDenormalMode(
5619 FPType: II->getType()->getScalarType()->getFltSemantics())
5620 : DenormalMode::getDynamic();
5621
5622 Known = KnownFPClass::minMaxLike(LHS: KnownLHS, RHS: KnownRHS, Kind: getMinMaxKind(IID),
5623 DenormMode: Mode);
5624 break;
5625 }
5626 case Intrinsic::canonicalize: {
5627 KnownFPClass KnownSrc;
5628 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5629 Known&: KnownSrc, Q, Depth: Depth + 1);
5630
5631 const Function *F = II->getFunction();
5632 DenormalMode DenormMode =
5633 F ? F->getDenormalMode(
5634 FPType: II->getType()->getScalarType()->getFltSemantics())
5635 : DenormalMode::getDynamic();
5636 Known = KnownFPClass::canonicalize(Src: KnownSrc, DenormMode);
5637 break;
5638 }
5639 case Intrinsic::vector_reduce_fmax:
5640 case Intrinsic::vector_reduce_fmin:
5641 case Intrinsic::vector_reduce_fmaximum:
5642 case Intrinsic::vector_reduce_fminimum:
5643 case Intrinsic::vector_reduce_fmaximumnum:
5644 case Intrinsic::vector_reduce_fminimumnum: {
5645 // reduce min/max will choose an element from one of the vector elements,
5646 // so we can infer and class information that is common to all elements.
5647 Known = computeKnownFPClass(V: II->getArgOperand(i: 0), FMF: II->getFastMathFlags(),
5648 InterestedClasses, SQ: Q, Depth: Depth + 1);
5649 // Can only propagate sign if output is never NaN.
5650 if (!Known.isKnownNeverNaN())
5651 Known.setSignBit(std::nullopt);
5652 break;
5653 }
5654 // reverse preserves all characteristics of the input vec's element.
5655 case Intrinsic::vector_reverse:
5656 Known = computeKnownFPClass(
5657 V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
5658 FMF: II->getFastMathFlags(), InterestedClasses, SQ: Q, Depth: Depth + 1);
5659 break;
5660 case Intrinsic::trunc:
5661 case Intrinsic::floor:
5662 case Intrinsic::ceil:
5663 case Intrinsic::rint:
5664 case Intrinsic::nearbyint:
5665 case Intrinsic::round:
5666 case Intrinsic::roundeven: {
5667 KnownFPClass KnownSrc;
5668 FPClassTest InterestedSrcs = InterestedClasses;
5669
5670 // Negative round ups towards zero produce negative zero.
5671 if (InterestedSrcs & fcNegFinite)
5672 InterestedSrcs |= fcNegFinite;
5673
5674 // Negative subnormals may flush to positive zero.
5675 if (InterestedSrcs & fcPosFinite)
5676 InterestedSrcs |= fcPosFinite | fcNegSubnormal;
5677
5678 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5679 Known&: KnownSrc, Q, Depth: Depth + 1);
5680
5681 const Function *F = II->getFunction();
5682 DenormalMode Mode =
5683 F ? F->getDenormalMode(
5684 FPType: II->getType()->getScalarType()->getFltSemantics())
5685 : DenormalMode::getDynamic();
5686 const bool IsMultiUnitFPType =
5687 V->getType()->getScalarType()->isMultiUnitFPType();
5688
5689 const bool IsTrunc = IID == Intrinsic::trunc;
5690 Known = KnownFPClass::roundToIntegral(Src: KnownSrc, IsTrunc,
5691 IsMultiUnitFPType, Mode);
5692 break;
5693 }
5694 case Intrinsic::exp:
5695 case Intrinsic::exp2:
5696 case Intrinsic::exp10:
5697 case Intrinsic::amdgcn_exp2: {
5698 KnownFPClass KnownSrc;
5699 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5700 Known&: KnownSrc, Q, Depth: Depth + 1);
5701
5702 Known = KnownFPClass::exp(Src: KnownSrc);
5703
5704 Type *EltTy = II->getType()->getScalarType();
5705 if (IID == Intrinsic::amdgcn_exp2 && EltTy->isFloatTy())
5706 Known.knownNot(RuleOut: fcSubnormal);
5707
5708 break;
5709 }
5710 case Intrinsic::fptrunc_round: {
5711 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
5712 Q, Depth);
5713 break;
5714 }
5715 case Intrinsic::log:
5716 case Intrinsic::log10:
5717 case Intrinsic::log2:
5718 case Intrinsic::experimental_constrained_log:
5719 case Intrinsic::experimental_constrained_log10:
5720 case Intrinsic::experimental_constrained_log2:
5721 case Intrinsic::amdgcn_log: {
5722 FPClassTest InterestedSrcs = fcNone;
5723
5724 // log(negative) produces NaN.
5725 if ((InterestedClasses & fcNan) != fcNone)
5726 InterestedSrcs |= fcNan | fcNegative;
5727
5728 // log(logical-zero) produces negative infinity.
5729 if ((InterestedClasses & fcNegInf) != fcNone)
5730 InterestedSrcs |= fcZero | fcSubnormal;
5731
5732 // log(x) < -0.0 if x < +1.0
5733 if ((InterestedClasses & fcNegNormal) != fcNone)
5734 InterestedSrcs |= fcPosSubnormal | fcPosNormal;
5735
5736 // log(x) >= +0.0 if x >= +1.0
5737 if ((InterestedClasses & (fcPosZero | fcPosNormal)) != fcNone)
5738 InterestedSrcs |= fcPosNormal;
5739
5740 // log(x) is positive infinity iff x is positive infinity.
5741 if ((InterestedClasses & fcPosInf) != fcNone)
5742 InterestedSrcs |= fcPosInf;
5743
5744 KnownFPClass KnownSrc;
5745 if (InterestedSrcs != fcNone)
5746 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5747 Known&: KnownSrc, Q, Depth: Depth + 1);
5748 const Function *F = II->getFunction();
5749 DenormalMode Mode =
5750 F ? F->getDenormalMode(
5751 FPType: II->getType()->getScalarType()->getFltSemantics())
5752 : DenormalMode::getDynamic();
5753 Known = KnownFPClass::log(Src: KnownSrc, Mode);
5754 break;
5755 }
5756 case Intrinsic::pow: {
5757 const bool WantNaN = (InterestedClasses & fcNan) != fcNone;
5758 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
5759 if (!WantNaN && !WantNegative)
5760 break;
5761
5762 FPClassTest InterestedLHS = fcNone;
5763 FPClassTest InterestedRHS = fcNone;
5764 if (WantNaN) {
5765 // pow may return NaN if one of the arguments is NaN. NaN may also be
5766 // produced from a negative, non-zero finite base and a non-integer
5767 // exponent.
5768 InterestedLHS |= fcNan | fcNegNormal | fcNegSubnormal;
5769 InterestedRHS |= fcNan;
5770 }
5771 if (WantNegative) {
5772 // A negative value is returned when a negative base is raised to an odd
5773 // integer power. Only normal values can be odd integers.
5774 InterestedLHS |= fcNegative;
5775 InterestedRHS |= fcNormal;
5776 }
5777
5778 KnownFPClass KnownLHS;
5779 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedLHS,
5780 Known&: KnownLHS, Q, Depth: Depth + 1);
5781
5782 // If the LHS is unknown, then querying the RHS is only useful for rare
5783 // edge cases.
5784 if (KnownLHS.isUnknown())
5785 break;
5786
5787 KnownFPClass KnownRHS;
5788 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses: InterestedRHS,
5789 Known&: KnownRHS, Q, Depth: Depth + 1);
5790 Known = KnownFPClass::pow(LHS: KnownLHS, RHS: KnownRHS);
5791 break;
5792 }
5793 case Intrinsic::powi: {
5794 if ((InterestedClasses & (fcNan | fcInf | fcNegative)) == fcNone)
5795 break;
5796
5797 // The exponent is always a scalar, even when raising a vector to a power.
5798 const Value *Exp = II->getArgOperand(i: 1);
5799 unsigned BitWidth = Exp->getType()->getIntegerBitWidth();
5800 KnownBits ExponentKnownBits(BitWidth);
5801 computeKnownBits(V: Exp, DemandedElts: APInt(1, 1), Known&: ExponentKnownBits, Q, Depth: Depth + 1);
5802
5803 FPClassTest InterestedSrcs = fcNone;
5804 if (InterestedClasses & fcNan)
5805 InterestedSrcs |= fcNan;
5806 if (!ExponentKnownBits.isZero()) {
5807 if (InterestedClasses & fcInf)
5808 InterestedSrcs |= fcFinite | fcInf;
5809 if ((InterestedClasses & fcNegative) && !ExponentKnownBits.isEven())
5810 InterestedSrcs |= fcNegative;
5811 }
5812
5813 KnownFPClass KnownSrc;
5814 if (InterestedSrcs != fcNone)
5815 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5816 Known&: KnownSrc, Q, Depth: Depth + 1);
5817
5818 Known = KnownFPClass::powi(Src: KnownSrc, N: ExponentKnownBits);
5819 break;
5820 }
5821 case Intrinsic::ldexp: {
5822 KnownFPClass KnownSrc;
5823 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5824 Known&: KnownSrc, Q, Depth: Depth + 1);
5825 // Can refine inf/zero handling based on the exponent operand.
5826 const FPClassTest ExpInfoMask = fcZero | fcSubnormal | fcInf;
5827
5828 const Value *ExpArg = II->getArgOperand(i: 1);
5829 ConstantRange ExpKnownRange =
5830 ((KnownSrc.getKnownFPClasses() & ExpInfoMask) != fcNone)
5831 ? computeConstantRange(V: ExpArg, /*ForSigned=*/true, SQ: Q, Depth: Depth + 1)
5832 : ConstantRange::getFull(
5833 BitWidth: ExpArg->getType()->getScalarSizeInBits());
5834
5835 const fltSemantics &Flt =
5836 II->getType()->getScalarType()->getFltSemantics();
5837
5838 const Function *F = II->getFunction();
5839 DenormalMode Mode =
5840 F ? F->getDenormalMode(FPType: Flt) : DenormalMode::getDynamic();
5841
5842 Known = KnownFPClass::ldexp(Src: KnownSrc, ConstantRangeMin: ExpKnownRange.getSignedMin(),
5843 ConstantRangeMax: ExpKnownRange.getSignedMax(), Flt, Mode);
5844 break;
5845 }
5846 case Intrinsic::arithmetic_fence: {
5847 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5848 Known, Q, Depth: Depth + 1);
5849 break;
5850 }
5851 case Intrinsic::experimental_constrained_sitofp:
5852 case Intrinsic::experimental_constrained_uitofp:
5853 // Cannot produce nan
5854 Known.knownNot(RuleOut: fcNan);
5855
5856 // sitofp and uitofp turn into +0.0 for zero.
5857 Known.knownNot(RuleOut: fcNegZero);
5858
5859 // Integers cannot be subnormal
5860 Known.knownNot(RuleOut: fcSubnormal);
5861
5862 if (IID == Intrinsic::experimental_constrained_uitofp)
5863 Known.signBitMustBeZero();
5864
5865 // TODO: Copy inf handling from instructions
5866 break;
5867
5868 case Intrinsic::amdgcn_fract: {
5869 Known.knownNot(RuleOut: fcInf);
5870
5871 if (InterestedClasses & fcNan) {
5872 KnownFPClass KnownSrc;
5873 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5874 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5875
5876 if (KnownSrc.isKnownNeverInfOrNaN())
5877 Known.knownNot(RuleOut: fcNan);
5878 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5879 Known.knownNot(RuleOut: fcSNan);
5880 }
5881
5882 break;
5883 }
5884 case Intrinsic::amdgcn_rcp: {
5885 KnownFPClass KnownSrc;
5886 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5887 Known&: KnownSrc, Q, Depth: Depth + 1);
5888
5889 Known.propagateNonNaN(Src: KnownSrc);
5890
5891 Type *EltTy = II->getType()->getScalarType();
5892
5893 // f32 denormal always flushed.
5894 if (EltTy->isFloatTy()) {
5895 Known.knownNot(RuleOut: fcSubnormal);
5896 KnownSrc.knownNot(RuleOut: fcSubnormal);
5897 }
5898
5899 if (KnownSrc.isKnownNever(Mask: fcNegative))
5900 Known.knownNot(RuleOut: fcNegative);
5901 if (KnownSrc.isKnownNever(Mask: fcPositive))
5902 Known.knownNot(RuleOut: fcPositive);
5903
5904 if (const Function *F = II->getFunction()) {
5905 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5906 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5907 Known.knownNot(RuleOut: fcPosInf);
5908 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5909 Known.knownNot(RuleOut: fcNegInf);
5910 }
5911
5912 break;
5913 }
5914 case Intrinsic::amdgcn_rsq: {
5915 KnownFPClass KnownSrc;
5916 // The only negative value that can be returned is -inf for -0 inputs.
5917 Known.knownNot(RuleOut: fcNegZero | fcNegSubnormal | fcNegNormal);
5918
5919 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5920 Known&: KnownSrc, Q, Depth: Depth + 1);
5921
5922 // Negative -> nan
5923 if (KnownSrc.isKnownNeverNaN() && KnownSrc.cannotBeOrderedLessThanZero())
5924 Known.knownNot(RuleOut: fcNan);
5925 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5926 Known.knownNot(RuleOut: fcSNan);
5927
5928 // +inf -> +0
5929 if (KnownSrc.isKnownNeverPosInfinity())
5930 Known.knownNot(RuleOut: fcPosZero);
5931
5932 Type *EltTy = II->getType()->getScalarType();
5933
5934 // f32 denormal always flushed.
5935 if (EltTy->isFloatTy())
5936 Known.knownNot(RuleOut: fcPosSubnormal);
5937
5938 if (const Function *F = II->getFunction()) {
5939 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5940
5941 // -0 -> -inf
5942 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5943 Known.knownNot(RuleOut: fcNegInf);
5944
5945 // +0 -> +inf
5946 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5947 Known.knownNot(RuleOut: fcPosInf);
5948 }
5949
5950 break;
5951 }
5952 case Intrinsic::amdgcn_trig_preop: {
5953 // Always returns a value [0, 1)
5954 Known.knownNot(RuleOut: fcNan | fcInf | fcNegative);
5955 break;
5956 }
5957 case Intrinsic::convert_from_arbitrary_fp: {
5958 auto *MD = cast<MetadataAsValue>(Val: II->getArgOperand(i: 1))->getMetadata();
5959 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
5960
5961 const fltSemantics *SrcSemantics =
5962 APFloat::getArbitraryFPSemantics(Format: FormatStr);
5963 if (!SrcSemantics)
5964 break;
5965
5966 const fltSemantics DstSemantics =
5967 II->getType()->getScalarType()->getFltSemantics();
5968
5969 if (!APFloat::semanticsHasNaN(*SrcSemantics))
5970 Known.knownNot(RuleOut: fcNan);
5971
5972 // fcInf can only be cleared if the source format has no Inf encoding
5973 // and the dst max exp can accommodate src max exp.
5974 if (!APFloat::semanticsHasInf(*SrcSemantics) &&
5975 APFloat::semanticsMaxExponent(*SrcSemantics) <=
5976 APFloat::semanticsMaxExponent(DstSemantics))
5977 Known.knownNot(RuleOut: fcInf);
5978
5979 // Check and clear all neg flags for formats that do not have signed
5980 // representation.
5981 if (!APFloat::semanticsHasSignedRepr(*SrcSemantics))
5982 Known.knownNot(RuleOut: fcNegative);
5983
5984 // Check if format has no zero at all (Float8E8M0FNU), or no negative
5985 // zero.
5986 if (!APFloat::semanticsHasZero(*SrcSemantics))
5987 Known.knownNot(RuleOut: fcZero);
5988 else if (SrcSemantics->nanEncoding == fltNanEncoding::NegativeZero)
5989 Known.knownNot(RuleOut: fcNegZero);
5990
5991 // If src lands normally in dest, the result can never be subnormal.
5992 if (APFloat::isRepresentableAsNormalIn(Src: *SrcSemantics, Dst: DstSemantics))
5993 Known.knownNot(RuleOut: fcSubnormal);
5994 break;
5995 }
5996 default:
5997 break;
5998 }
5999
6000 break;
6001 }
6002 case Instruction::FAdd:
6003 case Instruction::FSub: {
6004 KnownFPClass KnownLHS, KnownRHS;
6005 bool WantNegative =
6006 Op->getOpcode() == Instruction::FAdd &&
6007 (InterestedClasses & KnownFPClass::OrderedLessThanZeroMask) != fcNone;
6008 bool WantNaN = (InterestedClasses & fcNan) != fcNone;
6009 bool WantNegZero = (InterestedClasses & fcNegZero) != fcNone;
6010
6011 if (!WantNaN && !WantNegative && !WantNegZero)
6012 break;
6013
6014 FPClassTest InterestedSrcs = InterestedClasses;
6015 if (WantNegative)
6016 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
6017 if (InterestedClasses & fcNan)
6018 InterestedSrcs |= fcInf;
6019 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: InterestedSrcs,
6020 Known&: KnownRHS, Q, Depth: Depth + 1);
6021
6022 // Special case fadd x, x, which is the canonical form of fmul x, 2.
6023 bool Self = Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
6024 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT,
6025 Depth: Depth + 1);
6026 if (Self)
6027 KnownLHS = KnownRHS;
6028
6029 if ((WantNaN && KnownRHS.isKnownNeverNaN()) ||
6030 (WantNegative && KnownRHS.cannotBeOrderedLessThanZero()) ||
6031 WantNegZero || Opc == Instruction::FSub) {
6032
6033 // FIXME: Context function should always be passed in separately
6034 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6035 const fltSemantics &FltSem =
6036 Op->getType()->getScalarType()->getFltSemantics();
6037 DenormalMode Mode =
6038 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6039
6040 if (Self && Opc == Instruction::FAdd) {
6041 Known = KnownFPClass::fadd_self(Src: KnownLHS, Mode);
6042 } else {
6043 // RHS is canonically cheaper to compute. Skip inspecting the LHS if
6044 // there's no point.
6045
6046 if (!Self) {
6047 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
6048 Known&: KnownLHS, Q, Depth: Depth + 1);
6049 }
6050
6051 Known = Opc == Instruction::FAdd
6052 ? KnownFPClass::fadd(LHS: KnownLHS, RHS: KnownRHS, Mode)
6053 : KnownFPClass::fsub(LHS: KnownLHS, RHS: KnownRHS, Mode);
6054 }
6055 }
6056
6057 break;
6058 }
6059 case Instruction::FMul: {
6060 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6061 DenormalMode Mode =
6062 F ? F->getDenormalMode(
6063 FPType: Op->getType()->getScalarType()->getFltSemantics())
6064 : DenormalMode::getDynamic();
6065
6066 Value *LHS = Op->getOperand(i: 0);
6067 Value *RHS = Op->getOperand(i: 1);
6068 // X * X is always non-negative or a NaN.
6069 // FIXME: Should check isGuaranteedNotToBeUndef
6070 if (LHS == RHS) {
6071 KnownFPClass KnownSrc;
6072 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownSrc, Q,
6073 Depth: Depth + 1);
6074 Known = KnownFPClass::square(Src: KnownSrc, Mode);
6075 break;
6076 }
6077
6078 KnownFPClass KnownLHS, KnownRHS;
6079
6080 const APFloat *CRHS;
6081 if (match(V: RHS, P: m_APFloat(Res&: CRHS))) {
6082 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
6083 Depth: Depth + 1);
6084 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: *CRHS, Mode);
6085 } else {
6086 computeKnownFPClass(V: RHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS, Q,
6087 Depth: Depth + 1);
6088 // TODO: Improve accuracy in unfused FMA pattern. We can prove an
6089 // additional not-nan if the addend is known-not negative infinity if the
6090 // multiply is known-not infinity.
6091
6092 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
6093 Depth: Depth + 1);
6094 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: KnownRHS, Mode);
6095 }
6096
6097 /// Propgate no-infs if the other source is known smaller than one, such
6098 /// that this cannot introduce overflow.
6099 if (KnownLHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: RHS))
6100 Known.knownNot(RuleOut: fcInf);
6101 else if (KnownRHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: LHS))
6102 Known.knownNot(RuleOut: fcInf);
6103
6104 break;
6105 }
6106 case Instruction::FDiv: {
6107 const bool WantNan = (InterestedClasses & fcNan) != fcNone;
6108
6109 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6110 const fltSemantics &FltSem =
6111 Op->getType()->getScalarType()->getFltSemantics();
6112 DenormalMode Mode =
6113 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6114
6115 if (Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
6116 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT)) {
6117 // X / X is always exactly 1.0 or a NaN.
6118 Known.setKnownFPClasses(fcNan | fcPosNormal);
6119
6120 if (!WantNan)
6121 break;
6122
6123 KnownFPClass KnownSrc;
6124 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts,
6125 InterestedClasses: fcNan | fcInf | fcZero | fcSubnormal, Known&: KnownSrc, Q,
6126 Depth: Depth + 1);
6127
6128 Known = KnownFPClass::fdiv_self(Src: KnownSrc, Mode);
6129 break;
6130 }
6131
6132 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
6133 const bool WantPositive = (InterestedClasses & fcPositive) != fcNone;
6134 if (!WantNan && !WantNegative && !WantPositive)
6135 break;
6136
6137 KnownFPClass KnownLHS, KnownRHS;
6138 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS,
6139 Q, Depth: Depth + 1);
6140
6141 bool KnowSomethingUseful =
6142 KnownRHS.isKnownNeverNaN() ||
6143 KnownRHS.isKnownNever(Mask: fcNegNormal | fcNegSubnormal) ||
6144 KnownRHS.isKnownNever(Mask: fcPosNormal | fcPosSubnormal);
6145
6146 if (KnowSomethingUseful)
6147 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS,
6148 Q, Depth: Depth + 1);
6149
6150 Known = KnownFPClass::fdiv(LHS: KnownLHS, RHS: KnownRHS, Mode);
6151 break;
6152 }
6153 case Instruction::FRem: {
6154 FPClassTest InterestedLHS = fcNone;
6155 FPClassTest InterestedRHS = fcNone;
6156
6157 // NaN is also generated for frem(Inf, x) and frem(x, 0.0).
6158 if (InterestedClasses & fcNan) {
6159 InterestedLHS |= fcNan | fcInf;
6160 InterestedRHS |= fcNan | fcZero | fcSubnormal;
6161 }
6162
6163 // The sign for frem is the same as the first operand.
6164 if (InterestedClasses & (fcPosNormal | fcPosSubnormal))
6165 InterestedLHS |= fcPosNormal | fcPosSubnormal;
6166 if (InterestedClasses & (fcNegNormal | fcNegSubnormal))
6167 InterestedLHS |= fcNegNormal | fcNegSubnormal;
6168
6169 // A negative zero result requires a negative finite first operand.
6170 if (InterestedClasses & fcNegZero)
6171 InterestedLHS |= fcNegFinite;
6172
6173 // A positive zero result can additionally come from a negative finite
6174 // result being flushed to positive zero.
6175 if (InterestedClasses & fcPosZero)
6176 InterestedLHS |= fcPosFinite | fcNegNormal | fcNegSubnormal;
6177
6178 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6179 DenormalMode Mode =
6180 F ? F->getDenormalMode(
6181 FPType: Op->getType()->getScalarType()->getFltSemantics())
6182 : DenormalMode::getDynamic();
6183
6184 if (Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
6185 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT)) {
6186 // X % X is always exactly [+-]0.0 or a NaN.
6187 FPClassTest InterestedSrcs = InterestedLHS | InterestedRHS;
6188 KnownFPClass KnownSrc;
6189 if (InterestedSrcs != fcNone)
6190 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
6191 Known&: KnownSrc, Q, Depth: Depth + 1);
6192 Known = KnownFPClass::frem_self(Src: KnownSrc, Mode);
6193 break;
6194 }
6195
6196 KnownFPClass KnownLHS;
6197 if (InterestedLHS != fcNone)
6198 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedLHS,
6199 Known&: KnownLHS, Q, Depth: Depth + 1);
6200
6201 KnownFPClass KnownRHS;
6202 // RHS is only useful for refining NaN classes.
6203 if (InterestedRHS != fcNone && KnownLHS.isKnownNever(Mask: fcSNan))
6204 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: InterestedRHS,
6205 Known&: KnownRHS, Q, Depth: Depth + 1);
6206
6207 Known = KnownFPClass::frem(LHS: KnownLHS, RHS: KnownRHS, Mode);
6208
6209 break;
6210 }
6211 case Instruction::FPExt: {
6212 KnownFPClass KnownSrc;
6213 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
6214 Known&: KnownSrc, Q, Depth: Depth + 1);
6215
6216 const fltSemantics &DstTy =
6217 Op->getType()->getScalarType()->getFltSemantics();
6218 const fltSemantics &SrcTy =
6219 Op->getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
6220
6221 Known = KnownFPClass::fpext(KnownSrc, DstTy, SrcTy);
6222 break;
6223 }
6224 case Instruction::FPTrunc: {
6225 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known, Q,
6226 Depth);
6227 break;
6228 }
6229 case Instruction::SIToFP:
6230 case Instruction::UIToFP: {
6231 // Cannot produce nan
6232 Known.knownNot(RuleOut: fcNan);
6233
6234 // Integers cannot be subnormal
6235 Known.knownNot(RuleOut: fcSubnormal);
6236
6237 // sitofp and uitofp turn into +0.0 for zero.
6238 Known.knownNot(RuleOut: fcNegZero);
6239
6240 // UIToFP is always non-negative regardless of known bits.
6241 if (Op->getOpcode() == Instruction::UIToFP)
6242 Known.signBitMustBeZero();
6243
6244 // Only compute known bits if we can learn something useful from them.
6245 if (!(InterestedClasses & (fcPosZero | fcNormal | fcInf)))
6246 break;
6247
6248 KnownBits IntKnown =
6249 computeKnownBits(V: Op->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
6250
6251 // If the integer is non-zero, the result cannot be +0.0
6252 if (IntKnown.isNonZero())
6253 Known.knownNot(RuleOut: fcPosZero);
6254
6255 if (Op->getOpcode() == Instruction::SIToFP) {
6256 // If the signed integer is known non-negative, the result is
6257 // non-negative. If the signed integer is known negative, the result is
6258 // negative.
6259 if (IntKnown.isNonNegative()) {
6260 Known.signBitMustBeZero();
6261 } else if (IntKnown.isNegative()) {
6262 Known.signBitMustBeOne();
6263 }
6264 }
6265
6266 // Guard kept for ilogb()
6267 if (InterestedClasses & fcInf) {
6268 // Get width of largest magnitude integer known.
6269 // This still works for a signed minimum value because the largest FP
6270 // value is scaled by some fraction close to 2.0 (1.0 + 0.xxxx).
6271 int IntSize = IntKnown.getBitWidth();
6272 if (Op->getOpcode() == Instruction::UIToFP)
6273 IntSize -= IntKnown.countMinLeadingZeros();
6274 else if (Op->getOpcode() == Instruction::SIToFP)
6275 IntSize -= IntKnown.countMinSignBits();
6276
6277 // If the exponent of the largest finite FP value can hold the largest
6278 // integer, the result of the cast must be finite.
6279 Type *FPTy = Op->getType()->getScalarType();
6280 if (ilogb(Arg: APFloat::getLargest(Sem: FPTy->getFltSemantics())) >= IntSize)
6281 Known.knownNot(RuleOut: fcInf);
6282 }
6283
6284 break;
6285 }
6286 case Instruction::ExtractElement: {
6287 // Look through extract element. If the index is non-constant or
6288 // out-of-range demand all elements, otherwise just the extracted element.
6289 const Value *Vec = Op->getOperand(i: 0);
6290
6291 APInt DemandedVecElts;
6292 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
6293 unsigned NumElts = VecTy->getNumElements();
6294 DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
6295 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1));
6296 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
6297 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
6298 } else {
6299 DemandedVecElts = APInt(1, 1);
6300 }
6301
6302 return computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known,
6303 Q, Depth: Depth + 1);
6304 }
6305 case Instruction::InsertElement: {
6306 if (isa<ScalableVectorType>(Val: Op->getType()))
6307 return;
6308
6309 const Value *Vec = Op->getOperand(i: 0);
6310 const Value *Elt = Op->getOperand(i: 1);
6311 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 2));
6312 unsigned NumElts = DemandedElts.getBitWidth();
6313 APInt DemandedVecElts = DemandedElts;
6314 bool NeedsElt = true;
6315 // If we know the index we are inserting to, clear it from Vec check.
6316 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
6317 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
6318 NeedsElt = DemandedElts[CIdx->getZExtValue()];
6319 }
6320
6321 // Do we demand the inserted element?
6322 if (NeedsElt) {
6323 computeKnownFPClass(V: Elt, Known, InterestedClasses, Q, Depth: Depth + 1);
6324 // If we don't know any bits, early out.
6325 if (Known.isUnknown())
6326 break;
6327 } else {
6328 Known.setKnownFPClasses(fcNone);
6329 }
6330
6331 // Do we need anymore elements from Vec?
6332 if (!DemandedVecElts.isZero()) {
6333 KnownFPClass Known2;
6334 computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known&: Known2, Q,
6335 Depth: Depth + 1);
6336 Known |= Known2;
6337 }
6338
6339 break;
6340 }
6341 case Instruction::ShuffleVector: {
6342 // Handle vector splat idiom
6343 if (Value *Splat = getSplatValue(V)) {
6344 computeKnownFPClass(V: Splat, Known, InterestedClasses, Q, Depth: Depth + 1);
6345 break;
6346 }
6347
6348 // For undef elements, we don't know anything about the common state of
6349 // the shuffle result.
6350 APInt DemandedLHS, DemandedRHS;
6351 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: Op);
6352 if (!Shuf || !getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
6353 return;
6354
6355 if (!!DemandedLHS) {
6356 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
6357 computeKnownFPClass(V: LHS, DemandedElts: DemandedLHS, InterestedClasses, Known, Q,
6358 Depth: Depth + 1);
6359
6360 // If we don't know any bits, early out.
6361 if (Known.isUnknown())
6362 break;
6363 } else {
6364 Known.setKnownFPClasses(fcNone);
6365 }
6366
6367 if (!!DemandedRHS) {
6368 KnownFPClass Known2;
6369 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
6370 computeKnownFPClass(V: RHS, DemandedElts: DemandedRHS, InterestedClasses, Known&: Known2, Q,
6371 Depth: Depth + 1);
6372 Known |= Known2;
6373 }
6374
6375 break;
6376 }
6377 case Instruction::ExtractValue: {
6378 const ExtractValueInst *Extract = cast<ExtractValueInst>(Val: Op);
6379 ArrayRef<unsigned> Indices = Extract->getIndices();
6380 const Value *Src = Extract->getAggregateOperand();
6381 if (isa<StructType>(Val: Src->getType()) && Indices.size() == 1 &&
6382 Indices[0] == 0) {
6383 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Src)) {
6384 switch (II->getIntrinsicID()) {
6385 case Intrinsic::frexp: {
6386 FPClassTest InterestedSrcs = InterestedClasses;
6387
6388 // Positive subnormals and negative subnormals could become positive
6389 // zero.
6390 if (InterestedClasses & fcPosZero)
6391 InterestedSrcs |= fcSubnormal;
6392
6393 // Negative subnormals could become negative zero.
6394 if (InterestedClasses & fcNegZero)
6395 InterestedSrcs |= fcNegSubnormal;
6396
6397 if (InterestedClasses & fcPosNormal)
6398 InterestedSrcs |= fcPosSubnormal;
6399
6400 if (InterestedClasses & fcNegNormal)
6401 InterestedSrcs |= fcNegSubnormal;
6402
6403 KnownFPClass KnownSrc;
6404 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
6405 InterestedClasses: InterestedSrcs, Known&: KnownSrc, Q, Depth: Depth + 1);
6406
6407 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6408 const fltSemantics &FltSem =
6409 Op->getType()->getScalarType()->getFltSemantics();
6410
6411 DenormalMode Mode =
6412 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6413 Known = KnownFPClass::frexp_mant(Src: KnownSrc, Mode);
6414 return;
6415 }
6416 default:
6417 break;
6418 }
6419 }
6420 }
6421
6422 computeKnownFPClass(V: Src, DemandedElts, InterestedClasses, Known, Q,
6423 Depth: Depth + 1);
6424 break;
6425 }
6426 case Instruction::PHI: {
6427 const PHINode *P = cast<PHINode>(Val: Op);
6428 // Unreachable blocks may have zero-operand PHI nodes.
6429 if (P->getNumIncomingValues() == 0)
6430 break;
6431
6432 // Otherwise take the unions of the known bit sets of the operands,
6433 // taking conservative care to avoid excessive recursion.
6434 const unsigned PhiRecursionLimit = MaxAnalysisRecursionDepth - 2;
6435
6436 if (Depth < PhiRecursionLimit) {
6437 // Skip if every incoming value references to ourself.
6438 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
6439 break;
6440
6441 bool First = true;
6442
6443 for (const Use &U : P->operands()) {
6444 Value *IncValue;
6445 Instruction *CtxI;
6446 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CtxI);
6447 // Skip direct self references.
6448 if (IncValue == P)
6449 continue;
6450
6451 KnownFPClass KnownSrc;
6452 // Recurse, but cap the recursion to two levels, because we don't want
6453 // to waste time spinning around in loops. We need at least depth 2 to
6454 // detect known sign bits.
6455 computeKnownFPClass(V: IncValue, DemandedElts, InterestedClasses, Known&: KnownSrc,
6456 Q: Q.getWithoutCondContext().getWithInstruction(I: CtxI),
6457 Depth: PhiRecursionLimit);
6458
6459 if (First) {
6460 Known = KnownSrc;
6461 First = false;
6462 } else {
6463 Known |= KnownSrc;
6464 }
6465
6466 if (Known.getKnownFPClasses() == fcAllFlags)
6467 break;
6468 }
6469 }
6470
6471 // Look for the case of a for loop which has a positive
6472 // initial value and is incremented by a squared value.
6473 // This will propagate sign information out of such loops.
6474 if (P->getNumIncomingValues() != 2 || Known.cannotBeOrderedLessThanZero())
6475 break;
6476 for (unsigned I = 0; I < 2; I++) {
6477 Value *RecurValue = P->getIncomingValue(i: 1 - I);
6478 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: RecurValue);
6479 if (!II)
6480 continue;
6481 Value *R, *L, *Init;
6482 PHINode *PN;
6483 if (matchSimpleTernaryIntrinsicRecurrence(I: II, P&: PN, Init, OtherOp0&: L, OtherOp1&: R) &&
6484 PN == P) {
6485 switch (II->getIntrinsicID()) {
6486 case Intrinsic::fma:
6487 case Intrinsic::fmuladd: {
6488 KnownFPClass KnownStart;
6489 computeKnownFPClass(V: Init, DemandedElts, InterestedClasses, Known&: KnownStart,
6490 Q, Depth: Depth + 1);
6491 if (KnownStart.cannotBeOrderedLessThanZero() && L == R &&
6492 isGuaranteedNotToBeUndef(V: L, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1))
6493 Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
6494 break;
6495 }
6496 }
6497 }
6498 }
6499 break;
6500 }
6501 case Instruction::BitCast: {
6502 const Value *Src;
6503 if (!match(V: Op, P: m_ElementWiseBitCast(Op: m_Value(V&: Src))) ||
6504 !Src->getType()->isIntOrIntVectorTy())
6505 break;
6506
6507 const Type *Ty = Op->getType();
6508
6509 Value *CastLHS, *CastRHS;
6510
6511 // Match bitcast(umax(bitcast(a), bitcast(b)))
6512 if (match(V: Src, P: m_c_MaxOrMin(L: m_BitCast(Op: m_Value(V&: CastLHS)),
6513 R: m_BitCast(Op: m_Value(V&: CastRHS)))) &&
6514 CastLHS->getType() == Ty && CastRHS->getType() == Ty) {
6515 KnownFPClass KnownLHS, KnownRHS;
6516 computeKnownFPClass(V: CastRHS, DemandedElts, InterestedClasses, Known&: KnownRHS, Q,
6517 Depth: Depth + 1);
6518 if (!KnownRHS.isUnknown()) {
6519 computeKnownFPClass(V: CastLHS, DemandedElts, InterestedClasses, Known&: KnownLHS,
6520 Q, Depth: Depth + 1);
6521 Known = KnownLHS | KnownRHS;
6522 }
6523
6524 return;
6525 }
6526
6527 const Type *EltTy = Ty->getScalarType();
6528 KnownBits Bits(EltTy->getPrimitiveSizeInBits());
6529 computeKnownBits(V: Src, DemandedElts, Known&: Bits, Q, Depth: Depth + 1);
6530
6531 Known = KnownFPClass::bitcast(FltSemantics: EltTy->getFltSemantics(), Bits);
6532 break;
6533 }
6534 default:
6535 break;
6536 }
6537}
6538
6539KnownFPClass llvm::computeKnownFPClass(const Value *V,
6540 const APInt &DemandedElts,
6541 FPClassTest InterestedClasses,
6542 const SimplifyQuery &SQ,
6543 unsigned Depth) {
6544 KnownFPClass KnownClasses;
6545 ::computeKnownFPClass(V, DemandedElts, InterestedClasses, Known&: KnownClasses, Q: SQ,
6546 Depth);
6547 return KnownClasses;
6548}
6549
6550KnownFPClass llvm::computeKnownFPClass(const Value *V,
6551 FPClassTest InterestedClasses,
6552 const SimplifyQuery &SQ,
6553 unsigned Depth) {
6554 KnownFPClass Known;
6555 ::computeKnownFPClass(V, Known, InterestedClasses, Q: SQ, Depth);
6556 return Known;
6557}
6558
6559KnownFPClass llvm::computeKnownFPClass(
6560 const Value *V, const DataLayout &DL, FPClassTest InterestedClasses,
6561 const TargetLibraryInfo *TLI, AssumptionCache *AC, const Instruction *CtxI,
6562 const DominatorTree *DT, bool UseInstrInfo, unsigned Depth) {
6563 return computeKnownFPClass(V, InterestedClasses,
6564 SQ: SimplifyQuery(DL, TLI, DT, AC, CtxI, UseInstrInfo),
6565 Depth);
6566}
6567
6568KnownFPClass
6569llvm::computeKnownFPClass(const Value *V, const APInt &DemandedElts,
6570 FastMathFlags FMF, FPClassTest InterestedClasses,
6571 const SimplifyQuery &SQ, unsigned Depth) {
6572 if (FMF.noNaNs())
6573 InterestedClasses &= ~fcNan;
6574 if (FMF.noInfs())
6575 InterestedClasses &= ~fcInf;
6576
6577 KnownFPClass Result =
6578 computeKnownFPClass(V, DemandedElts, InterestedClasses, SQ, Depth);
6579
6580 if (FMF.noNaNs())
6581 Result.setKnownFPClasses(Result.getKnownFPClasses() & ~fcNan);
6582 if (FMF.noInfs())
6583 Result.setKnownFPClasses(Result.getKnownFPClasses() & ~fcInf);
6584 return Result;
6585}
6586
6587KnownFPClass llvm::computeKnownFPClass(const Value *V, FastMathFlags FMF,
6588 FPClassTest InterestedClasses,
6589 const SimplifyQuery &SQ,
6590 unsigned Depth) {
6591 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
6592 APInt DemandedElts =
6593 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
6594 return computeKnownFPClass(V, DemandedElts, FMF, InterestedClasses, SQ,
6595 Depth);
6596}
6597
6598bool llvm::cannotBeNegativeZero(const Value *V, const SimplifyQuery &SQ,
6599 unsigned Depth) {
6600 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNegZero, SQ, Depth);
6601 return Known.isKnownNeverNegZero();
6602}
6603
6604bool llvm::cannotBeOrderedLessThanZero(const Value *V, const SimplifyQuery &SQ,
6605 unsigned Depth) {
6606 KnownFPClass Known =
6607 computeKnownFPClass(V, InterestedClasses: KnownFPClass::OrderedLessThanZeroMask, SQ, Depth);
6608 return Known.cannotBeOrderedLessThanZero();
6609}
6610
6611bool llvm::isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ,
6612 unsigned Depth) {
6613 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf, SQ, Depth);
6614 return Known.isKnownNeverInfinity();
6615}
6616
6617/// Return true if the floating-point value can never contain a NaN or infinity.
6618bool llvm::isKnownNeverInfOrNaN(const Value *V, const SimplifyQuery &SQ,
6619 unsigned Depth) {
6620 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf | fcNan, SQ, Depth);
6621 return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity();
6622}
6623
6624/// Return true if the floating-point scalar value is not a NaN or if the
6625/// floating-point vector value has no NaN elements. Return false if a value
6626/// could ever be NaN.
6627bool llvm::isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ,
6628 unsigned Depth) {
6629 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNan, SQ, Depth);
6630 return Known.isKnownNeverNaN();
6631}
6632
6633/// Return false if we can prove that the specified FP value's sign bit is 0.
6634/// Return true if we can prove that the specified FP value's sign bit is 1.
6635/// Otherwise return std::nullopt.
6636std::optional<bool> llvm::computeKnownFPSignBit(const Value *V,
6637 const SimplifyQuery &SQ,
6638 unsigned Depth) {
6639 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcAllFlags, SQ, Depth);
6640 return Known.getSignBit();
6641}
6642
6643bool llvm::canIgnoreSignBitOfZero(const Use &U) {
6644 auto *User = cast<Instruction>(Val: U.getUser());
6645 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6646 if (FPOp->hasNoSignedZeros())
6647 return true;
6648 }
6649
6650 switch (User->getOpcode()) {
6651 case Instruction::FPToSI:
6652 case Instruction::FPToUI:
6653 return true;
6654 case Instruction::FCmp:
6655 // fcmp treats both positive and negative zero as equal.
6656 return true;
6657 case Instruction::Call:
6658 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6659 switch (II->getIntrinsicID()) {
6660 case Intrinsic::fabs:
6661 return true;
6662 case Intrinsic::copysign:
6663 return U.getOperandNo() == 0;
6664 case Intrinsic::is_fpclass: {
6665 auto Test =
6666 static_cast<FPClassTest>(
6667 cast<ConstantInt>(Val: II->getArgOperand(i: 1))->getZExtValue()) &
6668 FPClassTest::fcZero;
6669 return Test == FPClassTest::fcZero || Test == FPClassTest::fcNone;
6670 }
6671 default:
6672 return false;
6673 }
6674 }
6675 return false;
6676 default:
6677 return false;
6678 }
6679}
6680
6681bool llvm::canIgnoreSignBitOfNaN(const Use &U) {
6682 auto *User = cast<Instruction>(Val: U.getUser());
6683 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6684 if (FPOp->hasNoNaNs())
6685 return true;
6686 }
6687
6688 switch (User->getOpcode()) {
6689 case Instruction::FPToSI:
6690 case Instruction::FPToUI:
6691 return true;
6692 // Proper FP math operations ignore the sign bit of NaN.
6693 case Instruction::FAdd:
6694 case Instruction::FSub:
6695 case Instruction::FMul:
6696 case Instruction::FDiv:
6697 case Instruction::FRem:
6698 case Instruction::FPTrunc:
6699 case Instruction::FPExt:
6700 case Instruction::FCmp:
6701 return true;
6702 // Bitwise FP operations should preserve the sign bit of NaN.
6703 case Instruction::FNeg:
6704 case Instruction::Select:
6705 case Instruction::PHI:
6706 return false;
6707 case Instruction::Ret:
6708 return User->getFunction()->getAttributes().getRetNoFPClass() &
6709 FPClassTest::fcNan;
6710 case Instruction::Call:
6711 case Instruction::Invoke: {
6712 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6713 switch (II->getIntrinsicID()) {
6714 case Intrinsic::fabs:
6715 return true;
6716 case Intrinsic::copysign:
6717 return U.getOperandNo() == 0;
6718 // Other proper FP math intrinsics ignore the sign bit of NaN.
6719 case Intrinsic::maxnum:
6720 case Intrinsic::minnum:
6721 case Intrinsic::maximum:
6722 case Intrinsic::minimum:
6723 case Intrinsic::maximumnum:
6724 case Intrinsic::minimumnum:
6725 case Intrinsic::canonicalize:
6726 case Intrinsic::fma:
6727 case Intrinsic::fmuladd:
6728 case Intrinsic::sqrt:
6729 case Intrinsic::pow:
6730 case Intrinsic::powi:
6731 case Intrinsic::fptoui_sat:
6732 case Intrinsic::fptosi_sat:
6733 case Intrinsic::is_fpclass:
6734 return true;
6735 default:
6736 return false;
6737 }
6738 }
6739
6740 FPClassTest NoFPClass =
6741 cast<CallBase>(Val: User)->getParamNoFPClass(i: U.getOperandNo());
6742 return NoFPClass & FPClassTest::fcNan;
6743 }
6744 default:
6745 return false;
6746 }
6747}
6748
6749bool llvm::isKnownIntegral(const Value *V, const SimplifyQuery &SQ,
6750 FastMathFlags FMF) {
6751 if (isa<PoisonValue>(Val: V))
6752 return true;
6753 if (isa<UndefValue>(Val: V))
6754 return false;
6755
6756 if (match(V, P: m_CheckedFp(CheckFn: [](const APFloat &Val) { return Val.isInteger(); })))
6757 return true;
6758
6759 const Instruction *I = dyn_cast<Instruction>(Val: V);
6760 if (!I)
6761 return false;
6762
6763 switch (I->getOpcode()) {
6764 case Instruction::SIToFP:
6765 case Instruction::UIToFP:
6766 // TODO: Could check nofpclass(inf) on incoming argument
6767 if (FMF.noInfs())
6768 return true;
6769
6770 // Need to check int size cannot produce infinity, which computeKnownFPClass
6771 // knows how to do already.
6772 return isKnownNeverInfinity(V: I, SQ);
6773 case Instruction::Call: {
6774 const CallInst *CI = cast<CallInst>(Val: I);
6775 switch (CI->getIntrinsicID()) {
6776 case Intrinsic::trunc:
6777 case Intrinsic::floor:
6778 case Intrinsic::ceil:
6779 case Intrinsic::rint:
6780 case Intrinsic::nearbyint:
6781 case Intrinsic::round:
6782 case Intrinsic::roundeven:
6783 return (FMF.noInfs() && FMF.noNaNs()) || isKnownNeverInfOrNaN(V: I, SQ);
6784 default:
6785 break;
6786 }
6787
6788 break;
6789 }
6790 default:
6791 break;
6792 }
6793
6794 return false;
6795}
6796
6797Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
6798
6799 // All byte-wide stores are splatable, even of arbitrary variables.
6800 if (V->getType()->isIntegerTy(BitWidth: 8))
6801 return V;
6802
6803 LLVMContext &Ctx = V->getContext();
6804
6805 // Undef don't care.
6806 auto *UndefInt8 = UndefValue::get(T: Type::getInt8Ty(C&: Ctx));
6807 if (isa<UndefValue>(Val: V))
6808 return UndefInt8;
6809
6810 // Return poison for zero-sized type.
6811 if (DL.getTypeStoreSize(Ty: V->getType()).isZero())
6812 return PoisonValue::get(T: Type::getInt8Ty(C&: Ctx));
6813
6814 Constant *C = dyn_cast<Constant>(Val: V);
6815 if (!C) {
6816 // Conceptually, we could handle things like:
6817 // %a = zext i8 %X to i16
6818 // %b = shl i16 %a, 8
6819 // %c = or i16 %a, %b
6820 // but until there is an example that actually needs this, it doesn't seem
6821 // worth worrying about.
6822 return nullptr;
6823 }
6824
6825 // Handle 'null' ConstantArrayZero etc.
6826 if (C->isNullValue())
6827 return Constant::getNullValue(Ty: Type::getInt8Ty(C&: Ctx));
6828
6829 // Constant floating-point values can be handled as integer values if the
6830 // corresponding integer value is "byteable". An important case is 0.0.
6831 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
6832 Type *ScalarTy = CFP->getType()->getScalarType();
6833 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
6834 return isBytewiseValue(
6835 V: ConstantInt::get(Context&: Ctx, V: CFP->getValue().bitcastToAPInt()), DL);
6836
6837 // Don't handle long double formats, which have strange constraints.
6838 return nullptr;
6839 }
6840
6841 // We can handle constant integers that are multiple of 8 bits.
6842 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
6843 if (CI->getBitWidth() % 8 == 0) {
6844 if (!CI->getValue().isSplat(SplatSizeInBits: 8))
6845 return nullptr;
6846 return ConstantInt::get(Context&: Ctx, V: CI->getValue().trunc(width: 8));
6847 }
6848 }
6849
6850 if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
6851 if (CE->getOpcode() == Instruction::IntToPtr) {
6852 if (auto *PtrTy = dyn_cast<PointerType>(Val: CE->getType())) {
6853 unsigned BitWidth = DL.getPointerSizeInBits(AS: PtrTy->getAddressSpace());
6854 if (Constant *Op = ConstantFoldIntegerCast(
6855 C: CE->getOperand(i_nocapture: 0), DestTy: Type::getIntNTy(C&: Ctx, N: BitWidth), IsSigned: false, DL))
6856 return isBytewiseValue(V: Op, DL);
6857 }
6858 }
6859 }
6860
6861 auto Merge = [&](Value *LHS, Value *RHS) -> Value * {
6862 if (LHS == RHS)
6863 return LHS;
6864 if (!LHS || !RHS)
6865 return nullptr;
6866 if (LHS == UndefInt8)
6867 return RHS;
6868 if (RHS == UndefInt8)
6869 return LHS;
6870 return nullptr;
6871 };
6872
6873 if (ConstantDataSequential *CA = dyn_cast<ConstantDataSequential>(Val: C)) {
6874 Value *Val = UndefInt8;
6875 for (uint64_t I = 0, E = CA->getNumElements(); I != E; ++I)
6876 if (!(Val = Merge(Val, isBytewiseValue(V: CA->getElementAsConstant(i: I), DL))))
6877 return nullptr;
6878 return Val;
6879 }
6880
6881 if (isa<ConstantAggregate>(Val: C)) {
6882 Value *Val = UndefInt8;
6883 for (Value *Op : C->operands())
6884 if (!(Val = Merge(Val, isBytewiseValue(V: Op, DL))))
6885 return nullptr;
6886 return Val;
6887 }
6888
6889 // Don't try to handle the handful of other constants.
6890 return nullptr;
6891}
6892
6893// This is the recursive version of BuildSubAggregate. It takes a few different
6894// arguments. Idxs is the index within the nested struct From that we are
6895// looking at now (which is of type IndexedType). IdxSkip is the number of
6896// indices from Idxs that should be left out when inserting into the resulting
6897// struct. To is the result struct built so far, new insertvalue instructions
6898// build on that.
6899static Value *BuildSubAggregate(Value *From, Value *To, Type *IndexedType,
6900 SmallVectorImpl<unsigned> &Idxs,
6901 unsigned IdxSkip,
6902 BasicBlock::iterator InsertBefore) {
6903 StructType *STy = dyn_cast<StructType>(Val: IndexedType);
6904 if (STy) {
6905 // Save the original To argument so we can modify it
6906 Value *OrigTo = To;
6907 // General case, the type indexed by Idxs is a struct
6908 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6909 // Process each struct element recursively
6910 Idxs.push_back(Elt: i);
6911 Value *PrevTo = To;
6912 To = BuildSubAggregate(From, To, IndexedType: STy->getElementType(N: i), Idxs, IdxSkip,
6913 InsertBefore);
6914 Idxs.pop_back();
6915 if (!To) {
6916 // Couldn't find any inserted value for this index? Cleanup
6917 while (PrevTo != OrigTo) {
6918 InsertValueInst* Del = cast<InsertValueInst>(Val: PrevTo);
6919 PrevTo = Del->getAggregateOperand();
6920 Del->eraseFromParent();
6921 }
6922 // Stop processing elements
6923 break;
6924 }
6925 }
6926 // If we successfully found a value for each of our subaggregates
6927 if (To)
6928 return To;
6929 }
6930 // Base case, the type indexed by SourceIdxs is not a struct, or not all of
6931 // the struct's elements had a value that was inserted directly. In the latter
6932 // case, perhaps we can't determine each of the subelements individually, but
6933 // we might be able to find the complete struct somewhere.
6934
6935 // Find the value that is at that particular spot
6936 Value *V = FindInsertedValue(V: From, idx_range: Idxs);
6937
6938 if (!V)
6939 return nullptr;
6940
6941 // Insert the value in the new (sub) aggregate
6942 return InsertValueInst::Create(Agg: To, Val: V, Idxs: ArrayRef(Idxs).slice(N: IdxSkip), NameStr: "tmp",
6943 InsertBefore);
6944}
6945
6946// This helper takes a nested struct and extracts a part of it (which is again a
6947// struct) into a new value. For example, given the struct:
6948// { a, { b, { c, d }, e } }
6949// and the indices "1, 1" this returns
6950// { c, d }.
6951//
6952// It does this by inserting an insertvalue for each element in the resulting
6953// struct, as opposed to just inserting a single struct. This will only work if
6954// each of the elements of the substruct are known (ie, inserted into From by an
6955// insertvalue instruction somewhere).
6956//
6957// All inserted insertvalue instructions are inserted before InsertBefore
6958static Value *BuildSubAggregate(Value *From, ArrayRef<unsigned> idx_range,
6959 BasicBlock::iterator InsertBefore) {
6960 Type *IndexedType = ExtractValueInst::getIndexedType(Agg: From->getType(),
6961 Idxs: idx_range);
6962 Value *To = PoisonValue::get(T: IndexedType);
6963 SmallVector<unsigned, 10> Idxs(idx_range);
6964 unsigned IdxSkip = Idxs.size();
6965
6966 return BuildSubAggregate(From, To, IndexedType, Idxs, IdxSkip, InsertBefore);
6967}
6968
6969/// Given an aggregate and a sequence of indices, see if the scalar value
6970/// indexed is already around as a register, for example if it was inserted
6971/// directly into the aggregate.
6972///
6973/// If InsertBefore is not null, this function will duplicate (modified)
6974/// insertvalues when a part of a nested struct is extracted.
6975Value *
6976llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
6977 std::optional<BasicBlock::iterator> InsertBefore) {
6978 // Nothing to index? Just return V then (this is useful at the end of our
6979 // recursion).
6980 if (idx_range.empty())
6981 return V;
6982 // We have indices, so V should have an indexable type.
6983 assert((V->getType()->isStructTy() || V->getType()->isArrayTy()) &&
6984 "Not looking at a struct or array?");
6985 assert(ExtractValueInst::getIndexedType(V->getType(), idx_range) &&
6986 "Invalid indices for type?");
6987
6988 if (Constant *C = dyn_cast<Constant>(Val: V)) {
6989 C = C->getAggregateElement(Elt: idx_range[0]);
6990 if (!C) return nullptr;
6991 return FindInsertedValue(V: C, idx_range: idx_range.slice(N: 1), InsertBefore);
6992 }
6993
6994 if (InsertValueInst *I = dyn_cast<InsertValueInst>(Val: V)) {
6995 // Loop the indices for the insertvalue instruction in parallel with the
6996 // requested indices
6997 const unsigned *req_idx = idx_range.begin();
6998 for (const unsigned *i = I->idx_begin(), *e = I->idx_end();
6999 i != e; ++i, ++req_idx) {
7000 if (req_idx == idx_range.end()) {
7001 // We can't handle this without inserting insertvalues
7002 if (!InsertBefore)
7003 return nullptr;
7004
7005 // The requested index identifies a part of a nested aggregate. Handle
7006 // this specially. For example,
7007 // %A = insertvalue { i32, {i32, i32 } } undef, i32 10, 1, 0
7008 // %B = insertvalue { i32, {i32, i32 } } %A, i32 11, 1, 1
7009 // %C = extractvalue {i32, { i32, i32 } } %B, 1
7010 // This can be changed into
7011 // %A = insertvalue {i32, i32 } undef, i32 10, 0
7012 // %C = insertvalue {i32, i32 } %A, i32 11, 1
7013 // which allows the unused 0,0 element from the nested struct to be
7014 // removed.
7015 return BuildSubAggregate(From: V, idx_range: ArrayRef(idx_range.begin(), req_idx),
7016 InsertBefore: *InsertBefore);
7017 }
7018
7019 // This insert value inserts something else than what we are looking for.
7020 // See if the (aggregate) value inserted into has the value we are
7021 // looking for, then.
7022 if (*req_idx != *i)
7023 return FindInsertedValue(V: I->getAggregateOperand(), idx_range,
7024 InsertBefore);
7025 }
7026 // If we end up here, the indices of the insertvalue match with those
7027 // requested (though possibly only partially). Now we recursively look at
7028 // the inserted value, passing any remaining indices.
7029 return FindInsertedValue(V: I->getInsertedValueOperand(),
7030 idx_range: ArrayRef(req_idx, idx_range.end()), InsertBefore);
7031 }
7032
7033 if (ExtractValueInst *I = dyn_cast<ExtractValueInst>(Val: V)) {
7034 // If we're extracting a value from an aggregate that was extracted from
7035 // something else, we can extract from that something else directly instead.
7036 // However, we will need to chain I's indices with the requested indices.
7037
7038 // Calculate the number of indices required
7039 unsigned size = I->getNumIndices() + idx_range.size();
7040 // Allocate some space to put the new indices in
7041 SmallVector<unsigned, 5> Idxs;
7042 Idxs.reserve(N: size);
7043 // Add indices from the extract value instruction
7044 Idxs.append(in_start: I->idx_begin(), in_end: I->idx_end());
7045
7046 // Add requested indices
7047 Idxs.append(in_start: idx_range.begin(), in_end: idx_range.end());
7048
7049 assert(Idxs.size() == size
7050 && "Number of indices added not correct?");
7051
7052 return FindInsertedValue(V: I->getAggregateOperand(), idx_range: Idxs, InsertBefore);
7053 }
7054 // Otherwise, we don't know (such as, extracting from a function return value
7055 // or load instruction)
7056 return nullptr;
7057}
7058
7059// If V refers to an initialized global constant, set Slice either to
7060// its initializer if the size of its elements equals ElementSize, or,
7061// for ElementSize == 8, to its representation as an array of unsiged
7062// char. Return true on success.
7063// Offset is in the unit "nr of ElementSize sized elements".
7064bool llvm::getConstantDataArrayInfo(const Value *V,
7065 ConstantDataArraySlice &Slice,
7066 unsigned ElementSize, uint64_t Offset) {
7067 assert(V && "V should not be null.");
7068 assert((ElementSize % 8) == 0 &&
7069 "ElementSize expected to be a multiple of the size of a byte.");
7070 unsigned ElementSizeInBytes = ElementSize / 8;
7071
7072 // Drill down into the pointer expression V, ignoring any intervening
7073 // casts, and determine the identity of the object it references along
7074 // with the cumulative byte offset into it.
7075 const GlobalVariable *GV =
7076 dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V));
7077 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
7078 // Fail if V is not based on constant global object.
7079 return false;
7080
7081 const DataLayout &DL = GV->getDataLayout();
7082 APInt Off(DL.getIndexTypeSizeInBits(Ty: V->getType()), 0);
7083
7084 if (GV != V->stripAndAccumulateConstantOffsets(DL, Offset&: Off,
7085 /*AllowNonInbounds*/ true))
7086 // Fail if a constant offset could not be determined.
7087 return false;
7088
7089 uint64_t StartIdx = Off.getLimitedValue();
7090 if (StartIdx == UINT64_MAX)
7091 // Fail if the constant offset is excessive.
7092 return false;
7093
7094 // Off/StartIdx is in the unit of bytes. So we need to convert to number of
7095 // elements. Simply bail out if that isn't possible.
7096 if ((StartIdx % ElementSizeInBytes) != 0)
7097 return false;
7098
7099 Offset += StartIdx / ElementSizeInBytes;
7100 ConstantDataArray *Array = nullptr;
7101 ArrayType *ArrayTy = nullptr;
7102
7103 if (GV->getInitializer()->isNullValue()) {
7104 Type *GVTy = GV->getValueType();
7105 uint64_t SizeInBytes = DL.getTypeStoreSize(Ty: GVTy).getFixedValue();
7106 uint64_t Length = SizeInBytes / ElementSizeInBytes;
7107
7108 Slice.Array = nullptr;
7109 Slice.Offset = 0;
7110 // Return an empty Slice for undersized constants to let callers
7111 // transform even undefined library calls into simpler, well-defined
7112 // expressions. This is preferable to making the calls although it
7113 // prevents sanitizers from detecting such calls.
7114 Slice.Length = Length < Offset ? 0 : Length - Offset;
7115 return true;
7116 }
7117
7118 auto *Init = const_cast<Constant *>(GV->getInitializer());
7119 if (auto *ArrayInit = dyn_cast<ConstantDataArray>(Val: Init)) {
7120 Type *InitElTy = ArrayInit->getElementType();
7121 if (InitElTy->isIntegerTy(BitWidth: ElementSize)) {
7122 // If Init is an initializer for an array of the expected type
7123 // and size, use it as is.
7124 Array = ArrayInit;
7125 ArrayTy = ArrayInit->getType();
7126 }
7127 }
7128
7129 if (!Array) {
7130 if (ElementSize != 8)
7131 // TODO: Handle conversions to larger integral types.
7132 return false;
7133
7134 // Otherwise extract the portion of the initializer starting
7135 // at Offset as an array of bytes, and reset Offset.
7136 Init = ReadByteArrayFromGlobal(GV, Offset);
7137 if (!Init)
7138 return false;
7139
7140 Offset = 0;
7141 Array = dyn_cast<ConstantDataArray>(Val: Init);
7142 ArrayTy = dyn_cast<ArrayType>(Val: Init->getType());
7143 }
7144
7145 uint64_t NumElts = ArrayTy->getArrayNumElements();
7146 if (Offset > NumElts)
7147 return false;
7148
7149 Slice.Array = Array;
7150 Slice.Offset = Offset;
7151 Slice.Length = NumElts - Offset;
7152 return true;
7153}
7154
7155/// Extract bytes from the initializer of the constant array V, which need
7156/// not be a nul-terminated string. On success, store the bytes in Str and
7157/// return true. When TrimAtNul is set, Str will contain only the bytes up
7158/// to but not including the first nul. Return false on failure.
7159bool llvm::getConstantStringInfo(const Value *V, StringRef &Str,
7160 bool TrimAtNul) {
7161 ConstantDataArraySlice Slice;
7162 if (!getConstantDataArrayInfo(V, Slice, ElementSize: 8))
7163 return false;
7164
7165 if (Slice.Array == nullptr) {
7166 if (TrimAtNul) {
7167 // Return a nul-terminated string even for an empty Slice. This is
7168 // safe because all existing SimplifyLibcalls callers require string
7169 // arguments and the behavior of the functions they fold is undefined
7170 // otherwise. Folding the calls this way is preferable to making
7171 // the undefined library calls, even though it prevents sanitizers
7172 // from reporting such calls.
7173 Str = StringRef();
7174 return true;
7175 }
7176 if (Slice.Length == 1) {
7177 Str = StringRef("", 1);
7178 return true;
7179 }
7180 // We cannot instantiate a StringRef as we do not have an appropriate string
7181 // of 0s at hand.
7182 return false;
7183 }
7184
7185 // Start out with the entire array in the StringRef.
7186 Str = Slice.Array->getAsString();
7187 // Skip over 'offset' bytes.
7188 Str = Str.substr(Start: Slice.Offset);
7189
7190 if (TrimAtNul) {
7191 // Trim off the \0 and anything after it. If the array is not nul
7192 // terminated, we just return the whole end of string. The client may know
7193 // some other way that the string is length-bound.
7194 Str = Str.substr(Start: 0, N: Str.find(C: '\0'));
7195 }
7196 return true;
7197}
7198
7199// These next two are very similar to the above, but also look through PHI
7200// nodes.
7201// TODO: See if we can integrate these two together.
7202
7203/// If we can compute the length of the string pointed to by
7204/// the specified pointer, return 'len+1'. If we can't, return 0.
7205static uint64_t GetStringLengthH(const Value *V,
7206 SmallPtrSetImpl<const PHINode*> &PHIs,
7207 unsigned CharSize) {
7208 // Look through noop bitcast instructions.
7209 V = V->stripPointerCasts();
7210
7211 // If this is a PHI node, there are two cases: either we have already seen it
7212 // or we haven't.
7213 if (const PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7214 if (!PHIs.insert(Ptr: PN).second)
7215 return ~0ULL; // already in the set.
7216
7217 // If it was new, see if all the input strings are the same length.
7218 uint64_t LenSoFar = ~0ULL;
7219 for (Value *IncValue : PN->incoming_values()) {
7220 uint64_t Len = GetStringLengthH(V: IncValue, PHIs, CharSize);
7221 if (Len == 0) return 0; // Unknown length -> unknown.
7222
7223 if (Len == ~0ULL) continue;
7224
7225 if (Len != LenSoFar && LenSoFar != ~0ULL)
7226 return 0; // Disagree -> unknown.
7227 LenSoFar = Len;
7228 }
7229
7230 // Success, all agree.
7231 return LenSoFar;
7232 }
7233
7234 // strlen(select(c,x,y)) -> strlen(x) ^ strlen(y)
7235 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: V)) {
7236 uint64_t Len1 = GetStringLengthH(V: SI->getTrueValue(), PHIs, CharSize);
7237 if (Len1 == 0) return 0;
7238 uint64_t Len2 = GetStringLengthH(V: SI->getFalseValue(), PHIs, CharSize);
7239 if (Len2 == 0) return 0;
7240 if (Len1 == ~0ULL) return Len2;
7241 if (Len2 == ~0ULL) return Len1;
7242 if (Len1 != Len2) return 0;
7243 return Len1;
7244 }
7245
7246 // Otherwise, see if we can read the string.
7247 ConstantDataArraySlice Slice;
7248 if (!getConstantDataArrayInfo(V, Slice, ElementSize: CharSize))
7249 return 0;
7250
7251 if (Slice.Array == nullptr)
7252 // Zeroinitializer (including an empty one).
7253 return 1;
7254
7255 // Search for the first nul character. Return a conservative result even
7256 // when there is no nul. This is safe since otherwise the string function
7257 // being folded such as strlen is undefined, and can be preferable to
7258 // making the undefined library call.
7259 unsigned NullIndex = 0;
7260 for (unsigned E = Slice.Length; NullIndex < E; ++NullIndex) {
7261 if (Slice.Array->getElementAsInteger(i: Slice.Offset + NullIndex) == 0)
7262 break;
7263 }
7264
7265 return NullIndex + 1;
7266}
7267
7268/// If we can compute the length of the string pointed to by
7269/// the specified pointer, return 'len+1'. If we can't, return 0.
7270uint64_t llvm::GetStringLength(const Value *V, unsigned CharSize) {
7271 if (!V->getType()->isPointerTy())
7272 return 0;
7273
7274 SmallPtrSet<const PHINode*, 32> PHIs;
7275 uint64_t Len = GetStringLengthH(V, PHIs, CharSize);
7276 // If Len is ~0ULL, we had an infinite phi cycle: this is dead code, so return
7277 // an empty string as a length.
7278 return Len == ~0ULL ? 1 : Len;
7279}
7280
7281const Value *
7282llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call,
7283 bool MustPreserveOffset,
7284 bool MustPreserveProvenance) {
7285 assert(Call &&
7286 "getArgumentAliasingToReturnedPointer only works on nonnull calls");
7287 if (const Value *RV = Call->getReturnedArgOperand())
7288 return RV;
7289 // This can be used only as a aliasing property.
7290 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
7291 Call, MustPreserveOffset, MustPreserveProvenance))
7292 return Call->getArgOperand(i: 0);
7293 return nullptr;
7294}
7295
7296bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
7297 const CallBase *Call, bool MustPreserveOffset,
7298 bool MustPreserveProvenance) {
7299 switch (Call->getIntrinsicID()) {
7300 case Intrinsic::launder_invariant_group:
7301 case Intrinsic::aarch64_irg:
7302 case Intrinsic::aarch64_tagp:
7303 // The amdgcn_make_buffer_rsrc function does not alter the address of the
7304 // input pointer (and thus preserves the byte offset, which is the property
7305 // the MustPreserveOffset flag selects). However, it will not necessarily
7306 // map ptr addrspace(N) null to ptr addrspace(8) null, aka the "null
7307 // descriptor", which has "all loads return 0, all stores are dropped"
7308 // semantics. Given the context of this intrinsic list, no one should be
7309 // relying on such a strict bit-exact null mapping (and, at time of
7310 // writing, they are not), but we document this fact out of an abundance
7311 // of caution.
7312 case Intrinsic::amdgcn_make_buffer_rsrc:
7313 return !MustPreserveProvenance;
7314 case Intrinsic::ptrmask:
7315 return !MustPreserveOffset;
7316 case Intrinsic::threadlocal_address:
7317 // The underlying variable changes with thread ID. The Thread ID may change
7318 // at coroutine suspend points.
7319 return !Call->getParent()->getParent()->isPresplitCoroutine();
7320 default:
7321 return false;
7322 }
7323}
7324
7325/// \p PN defines a loop-variant pointer to an object. Check if the
7326/// previous iteration of the loop was referring to the same object as \p PN.
7327static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
7328 const LoopInfo *LI) {
7329 // Find the loop-defined value.
7330 Loop *L = LI->getLoopFor(BB: PN->getParent());
7331 if (PN->getNumIncomingValues() != 2)
7332 return true;
7333
7334 // Find the value from previous iteration.
7335 auto *PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 0));
7336 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
7337 PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 1));
7338 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
7339 return true;
7340
7341 // If a new pointer is loaded in the loop, the pointer references a different
7342 // object in every iteration. E.g.:
7343 // for (i)
7344 // int *p = a[i];
7345 // ...
7346 if (auto *Load = dyn_cast<LoadInst>(Val: PrevValue))
7347 if (!L->isLoopInvariant(V: Load->getPointerOperand()))
7348 return false;
7349 return true;
7350}
7351
7352const Value *llvm::getUnderlyingObject(const Value *V, unsigned MaxLookup,
7353 bool MustPreserveProvenance) {
7354 for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
7355 if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
7356 const Value *PtrOp = GEP->getPointerOperand();
7357 if (!PtrOp->getType()->isPointerTy()) // Only handle scalar pointer base.
7358 return V;
7359 V = PtrOp;
7360 } else if (Operator::getOpcode(V) == Instruction::BitCast ||
7361 Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
7362 Value *NewV = cast<Operator>(Val: V)->getOperand(i: 0);
7363 if (!NewV->getType()->isPointerTy())
7364 return V;
7365 V = NewV;
7366 } else if (auto *GA = dyn_cast<GlobalAlias>(Val: V)) {
7367 if (GA->isInterposable())
7368 return V;
7369 V = GA->getAliasee();
7370 } else {
7371 if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
7372 // Look through single-arg phi nodes created by LCSSA.
7373 if (PHI->getNumIncomingValues() == 1) {
7374 V = PHI->getIncomingValue(i: 0);
7375 continue;
7376 }
7377 } else if (auto *Call = dyn_cast<CallBase>(Val: V)) {
7378 // CaptureTracking can know about special capturing properties of some
7379 // intrinsics like launder.invariant.group, that can't be expressed with
7380 // the attributes, but have properties like returning aliasing pointer.
7381 // Because some analysis may assume that nocaptured pointer is not
7382 // returned from some special intrinsic (because function would have to
7383 // be marked with returns attribute), it is crucial to use this function
7384 // because it should be in sync with CaptureTracking. Not using it may
7385 // cause weird miscompilations where 2 aliasing pointers are assumed to
7386 // noalias.
7387 if (auto *RP = getArgumentAliasingToReturnedPointer(
7388 Call, /*MustPreserveOffset=*/false, MustPreserveProvenance)) {
7389 V = RP;
7390 continue;
7391 }
7392 }
7393
7394 return V;
7395 }
7396 assert(V->getType()->isPointerTy() && "Unexpected operand type!");
7397 }
7398 return V;
7399}
7400
7401void llvm::getUnderlyingObjects(const Value *V,
7402 SmallVectorImpl<const Value *> &Objects,
7403 const LoopInfo *LI, unsigned MaxLookup) {
7404 SmallPtrSet<const Value *, 4> Visited;
7405 SmallVector<const Value *, 4> Worklist;
7406 Worklist.push_back(Elt: V);
7407 do {
7408 const Value *P = Worklist.pop_back_val();
7409 P = getUnderlyingObject(V: P, MaxLookup);
7410
7411 if (!Visited.insert(Ptr: P).second)
7412 continue;
7413
7414 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7415 Worklist.push_back(Elt: SI->getTrueValue());
7416 Worklist.push_back(Elt: SI->getFalseValue());
7417 continue;
7418 }
7419
7420 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7421 // If this PHI changes the underlying object in every iteration of the
7422 // loop, don't look through it. Consider:
7423 // int **A;
7424 // for (i) {
7425 // Prev = Curr; // Prev = PHI (Prev_0, Curr)
7426 // Curr = A[i];
7427 // *Prev, *Curr;
7428 //
7429 // Prev is tracking Curr one iteration behind so they refer to different
7430 // underlying objects.
7431 if (!LI || !LI->isLoopHeader(BB: PN->getParent()) ||
7432 isSameUnderlyingObjectInLoop(PN, LI))
7433 append_range(C&: Worklist, R: PN->incoming_values());
7434 else
7435 Objects.push_back(Elt: P);
7436 continue;
7437 }
7438
7439 Objects.push_back(Elt: P);
7440 } while (!Worklist.empty());
7441}
7442
7443const Value *llvm::getUnderlyingObjectAggressive(const Value *V,
7444 bool MustPreserveProvenance) {
7445 const unsigned MaxVisited = 8;
7446
7447 SmallPtrSet<const Value *, 8> Visited;
7448 SmallVector<const Value *, 8> Worklist;
7449 Worklist.push_back(Elt: V);
7450 const Value *Object = nullptr;
7451 // Used as fallback if we can't find a common underlying object through
7452 // recursion.
7453 bool First = true;
7454 const Value *FirstObject =
7455 getUnderlyingObject(V, MaxLookup: MaxLookupSearchDepth, MustPreserveProvenance);
7456 do {
7457 const Value *P = Worklist.pop_back_val();
7458 P = First ? FirstObject
7459 : getUnderlyingObject(V: P, MaxLookup: MaxLookupSearchDepth,
7460 MustPreserveProvenance);
7461 First = false;
7462
7463 if (!Visited.insert(Ptr: P).second)
7464 continue;
7465
7466 if (Visited.size() == MaxVisited)
7467 return FirstObject;
7468
7469 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7470 Worklist.push_back(Elt: SI->getTrueValue());
7471 Worklist.push_back(Elt: SI->getFalseValue());
7472 continue;
7473 }
7474
7475 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7476 append_range(C&: Worklist, R: PN->incoming_values());
7477 continue;
7478 }
7479
7480 if (!Object)
7481 Object = P;
7482 else if (Object != P)
7483 return FirstObject;
7484 } while (!Worklist.empty());
7485
7486 return Object ? Object : FirstObject;
7487}
7488
7489/// This is the function that does the work of looking through basic
7490/// ptrtoint+arithmetic+inttoptr sequences.
7491static const Value *getUnderlyingObjectFromInt(const Value *V) {
7492 do {
7493 if (const Operator *U = dyn_cast<Operator>(Val: V)) {
7494 // If we find a ptrtoint, we can transfer control back to the
7495 // regular getUnderlyingObjectFromInt.
7496 if (U->getOpcode() == Instruction::PtrToInt)
7497 return U->getOperand(i: 0);
7498 // If we find an add of a constant, a multiplied value, or a phi, it's
7499 // likely that the other operand will lead us to the base
7500 // object. We don't have to worry about the case where the
7501 // object address is somehow being computed by the multiply,
7502 // because our callers only care when the result is an
7503 // identifiable object.
7504 if (U->getOpcode() != Instruction::Add ||
7505 (!isa<ConstantInt>(Val: U->getOperand(i: 1)) &&
7506 Operator::getOpcode(V: U->getOperand(i: 1)) != Instruction::Mul &&
7507 !isa<PHINode>(Val: U->getOperand(i: 1))))
7508 return V;
7509 V = U->getOperand(i: 0);
7510 } else {
7511 return V;
7512 }
7513 assert(V->getType()->isIntegerTy() && "Unexpected operand type!");
7514 } while (true);
7515}
7516
7517/// This is a wrapper around getUnderlyingObjects and adds support for basic
7518/// ptrtoint+arithmetic+inttoptr sequences.
7519/// It returns false if unidentified object is found in getUnderlyingObjects.
7520bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
7521 SmallVectorImpl<Value *> &Objects) {
7522 SmallPtrSet<const Value *, 16> Visited;
7523 SmallVector<const Value *, 4> Working(1, V);
7524 bool AllObjectsIdentified = true;
7525 do {
7526 V = Working.pop_back_val();
7527
7528 SmallVector<const Value *, 4> Objs;
7529 getUnderlyingObjects(V, Objects&: Objs);
7530
7531 for (const Value *V : Objs) {
7532 if (!Visited.insert(Ptr: V).second)
7533 continue;
7534 if (Operator::getOpcode(V) == Instruction::IntToPtr) {
7535 const Value *O =
7536 getUnderlyingObjectFromInt(V: cast<User>(Val: V)->getOperand(i: 0));
7537 if (O->getType()->isPointerTy()) {
7538 Working.push_back(Elt: O);
7539 continue;
7540 }
7541 }
7542 AllObjectsIdentified &= isIdentifiedObject(V);
7543 Objects.push_back(Elt: const_cast<Value *>(V));
7544 }
7545 } while (!Working.empty());
7546 return AllObjectsIdentified;
7547}
7548
7549AllocaInst *llvm::findAllocaForValue(Value *V, bool OffsetZero) {
7550 AllocaInst *Result = nullptr;
7551 SmallPtrSet<Value *, 4> Visited;
7552 SmallVector<Value *, 4> Worklist;
7553
7554 auto AddWork = [&](Value *V) {
7555 if (Visited.insert(Ptr: V).second)
7556 Worklist.push_back(Elt: V);
7557 };
7558
7559 AddWork(V);
7560 do {
7561 V = Worklist.pop_back_val();
7562 assert(Visited.count(V));
7563
7564 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
7565 if (Result && Result != AI)
7566 return nullptr;
7567 Result = AI;
7568 } else if (CastInst *CI = dyn_cast<CastInst>(Val: V)) {
7569 AddWork(CI->getOperand(i_nocapture: 0));
7570 } else if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7571 for (Value *IncValue : PN->incoming_values())
7572 AddWork(IncValue);
7573 } else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
7574 AddWork(SI->getTrueValue());
7575 AddWork(SI->getFalseValue());
7576 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: V)) {
7577 if (OffsetZero && !GEP->hasAllZeroIndices())
7578 return nullptr;
7579 AddWork(GEP->getPointerOperand());
7580 } else if (CallBase *CB = dyn_cast<CallBase>(Val: V)) {
7581 Value *Returned = CB->getReturnedArgOperand();
7582 if (Returned)
7583 AddWork(Returned);
7584 else
7585 return nullptr;
7586 } else {
7587 return nullptr;
7588 }
7589 } while (!Worklist.empty());
7590
7591 return Result;
7592}
7593
7594static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7595 const Value *V, bool AllowLifetime, bool AllowDroppable) {
7596 for (const User *U : V->users()) {
7597 const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
7598 if (!II)
7599 return false;
7600
7601 if (AllowLifetime && II->isLifetimeStartOrEnd())
7602 continue;
7603
7604 if (AllowDroppable && II->isDroppable())
7605 continue;
7606
7607 return false;
7608 }
7609 return true;
7610}
7611
7612bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
7613 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7614 V, /* AllowLifetime */ true, /* AllowDroppable */ false);
7615}
7616bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) {
7617 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7618 V, /* AllowLifetime */ true, /* AllowDroppable */ true);
7619}
7620
7621bool llvm::isNotCrossLaneOperation(const Instruction *I) {
7622 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
7623 return isTriviallyVectorizable(ID: II->getIntrinsicID());
7624 auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: I);
7625 return (!Shuffle || Shuffle->isSelect()) &&
7626 !isa<CallBase, BitCastInst, ExtractElementInst>(Val: I);
7627}
7628
7629bool llvm::isSafeToSpeculativelyExecute(
7630 const Instruction *Inst, const Instruction *CtxI, AssumptionCache *AC,
7631 const DominatorTree *DT, const TargetLibraryInfo *TLI, bool UseVariableInfo,
7632 bool IgnoreUBImplyingAttrs) {
7633 return isSafeToSpeculativelyExecuteWithOpcode(Opcode: Inst->getOpcode(), Inst, CtxI,
7634 AC, DT, TLI, UseVariableInfo,
7635 IgnoreUBImplyingAttrs);
7636}
7637
7638bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
7639 unsigned Opcode, const Instruction *Inst, const Instruction *CtxI,
7640 AssumptionCache *AC, const DominatorTree *DT, const TargetLibraryInfo *TLI,
7641 bool UseVariableInfo, bool IgnoreUBImplyingAttrs) {
7642#ifndef NDEBUG
7643 if (Inst->getOpcode() != Opcode) {
7644 // Check that the operands are actually compatible with the Opcode override.
7645 auto hasEqualReturnAndLeadingOperandTypes =
7646 [](const Instruction *Inst, unsigned NumLeadingOperands) {
7647 if (Inst->getNumOperands() < NumLeadingOperands)
7648 return false;
7649 const Type *ExpectedType = Inst->getType();
7650 for (unsigned ItOp = 0; ItOp < NumLeadingOperands; ++ItOp)
7651 if (Inst->getOperand(ItOp)->getType() != ExpectedType)
7652 return false;
7653 return true;
7654 };
7655 assert(!Instruction::isBinaryOp(Opcode) ||
7656 hasEqualReturnAndLeadingOperandTypes(Inst, 2));
7657 assert(!Instruction::isUnaryOp(Opcode) ||
7658 hasEqualReturnAndLeadingOperandTypes(Inst, 1));
7659 }
7660#endif
7661
7662 switch (Opcode) {
7663 default:
7664 return true;
7665 case Instruction::UDiv:
7666 case Instruction::URem: {
7667 // x / y is undefined if y == 0.
7668 const APInt *V;
7669 if (match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: V)))
7670 return *V != 0;
7671 return false;
7672 }
7673 case Instruction::SDiv:
7674 case Instruction::SRem: {
7675 // x / y is undefined if y == 0 or x == INT_MIN and y == -1
7676 const APInt *Numerator, *Denominator;
7677 if (!match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: Denominator)))
7678 return false;
7679 // We cannot hoist this division if the denominator is 0.
7680 if (*Denominator == 0)
7681 return false;
7682 // It's safe to hoist if the denominator is not 0 or -1.
7683 if (!Denominator->isAllOnes())
7684 return true;
7685 // At this point we know that the denominator is -1. It is safe to hoist as
7686 // long we know that the numerator is not INT_MIN.
7687 if (match(V: Inst->getOperand(i: 0), P: m_APInt(Res&: Numerator)))
7688 return !Numerator->isMinSignedValue();
7689 // The numerator *might* be MinSignedValue.
7690 return false;
7691 }
7692 case Instruction::Load: {
7693 if (!UseVariableInfo)
7694 return false;
7695
7696 const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst);
7697 if (!LI)
7698 return false;
7699 if (mustSuppressSpeculation(LI: *LI))
7700 return false;
7701 const DataLayout &DL = LI->getDataLayout();
7702 return isDereferenceableAndAlignedPointer(
7703 V: LI->getPointerOperand(), Ty: LI->getType(), Alignment: LI->getAlign(),
7704 Q: SimplifyQuery(DL, TLI, DT, AC, CtxI));
7705 }
7706 case Instruction::Call: {
7707 auto *CI = dyn_cast<const CallInst>(Val: Inst);
7708 if (!CI)
7709 return false;
7710 const Function *Callee = CI->getCalledFunction();
7711
7712 // The called function could have undefined behavior or side-effects, even
7713 // if marked readnone nounwind.
7714 if (!Callee || !Callee->isSpeculatable())
7715 return false;
7716 // Since the operands may be changed after hoisting, undefined behavior may
7717 // be triggered by some UB-implying attributes.
7718 return IgnoreUBImplyingAttrs || !CI->hasUBImplyingAttrs();
7719 }
7720 case Instruction::VAArg:
7721 case Instruction::Alloca:
7722 case Instruction::Invoke:
7723 case Instruction::CallBr:
7724 case Instruction::PHI:
7725 case Instruction::Store:
7726 case Instruction::Ret:
7727 case Instruction::UncondBr:
7728 case Instruction::CondBr:
7729 case Instruction::IndirectBr:
7730 case Instruction::Switch:
7731 case Instruction::Unreachable:
7732 case Instruction::Fence:
7733 case Instruction::AtomicRMW:
7734 case Instruction::AtomicCmpXchg:
7735 case Instruction::LandingPad:
7736 case Instruction::Resume:
7737 case Instruction::CatchSwitch:
7738 case Instruction::CatchPad:
7739 case Instruction::CatchRet:
7740 case Instruction::CleanupPad:
7741 case Instruction::CleanupRet:
7742 return false; // Misc instructions which have effects
7743 }
7744}
7745
7746bool llvm::mayHaveNonDefUseDependency(const Instruction &I) {
7747 if (I.mayReadOrWriteMemory())
7748 // Memory dependency possible
7749 return true;
7750 if (!isSafeToSpeculativelyExecute(Inst: &I))
7751 // Can't move above a maythrow call or infinite loop. Or if an
7752 // inalloca alloca, above a stacksave call.
7753 return true;
7754 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7755 // 1) Can't reorder two inf-loop calls, even if readonly
7756 // 2) Also can't reorder an inf-loop call below a instruction which isn't
7757 // safe to speculative execute. (Inverse of above)
7758 return true;
7759 return false;
7760}
7761
7762/// Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
7763static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) {
7764 switch (OR) {
7765 case ConstantRange::OverflowResult::MayOverflow:
7766 return OverflowResult::MayOverflow;
7767 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
7768 return OverflowResult::AlwaysOverflowsLow;
7769 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
7770 return OverflowResult::AlwaysOverflowsHigh;
7771 case ConstantRange::OverflowResult::NeverOverflows:
7772 return OverflowResult::NeverOverflows;
7773 }
7774 llvm_unreachable("Unknown OverflowResult");
7775}
7776
7777/// Combine constant ranges from computeConstantRange() and computeKnownBits().
7778ConstantRange
7779llvm::computeConstantRangeIncludingKnownBits(const WithCache<const Value *> &V,
7780 bool ForSigned,
7781 const SimplifyQuery &SQ) {
7782 ConstantRange CR1 =
7783 ConstantRange::fromKnownBits(Known: V.getKnownBits(Q: SQ), IsSigned: ForSigned);
7784 ConstantRange CR2 = computeConstantRange(V, ForSigned, SQ);
7785 ConstantRange::PreferredRangeType RangeType =
7786 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
7787 return CR1.intersectWith(CR: CR2, Type: RangeType);
7788}
7789
7790OverflowResult llvm::computeOverflowForUnsignedMul(const Value *LHS,
7791 const Value *RHS,
7792 const SimplifyQuery &SQ,
7793 bool IsNSW) {
7794 ConstantRange LHSRange =
7795 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7796 ConstantRange RHSRange =
7797 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7798
7799 // mul nsw of two non-negative numbers is also nuw.
7800 if (IsNSW && LHSRange.isAllNonNegative() && RHSRange.isAllNonNegative())
7801 return OverflowResult::NeverOverflows;
7802
7803 return mapOverflowResult(OR: LHSRange.unsignedMulMayOverflow(Other: RHSRange));
7804}
7805
7806OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
7807 const Value *RHS,
7808 const SimplifyQuery &SQ) {
7809 // Multiplying n * m significant bits yields a result of n + m significant
7810 // bits. If the total number of significant bits does not exceed the
7811 // result bit width (minus 1), there is no overflow.
7812 // This means if we have enough leading sign bits in the operands
7813 // we can guarantee that the result does not overflow.
7814 // Ref: "Hacker's Delight" by Henry Warren
7815 unsigned BitWidth = LHS->getType()->getScalarSizeInBits();
7816
7817 // Note that underestimating the number of sign bits gives a more
7818 // conservative answer.
7819 unsigned SignBits =
7820 ::ComputeNumSignBits(V: LHS, Q: SQ) + ::ComputeNumSignBits(V: RHS, Q: SQ);
7821
7822 // First handle the easy case: if we have enough sign bits there's
7823 // definitely no overflow.
7824 if (SignBits > BitWidth + 1)
7825 return OverflowResult::NeverOverflows;
7826
7827 // There are two ambiguous cases where there can be no overflow:
7828 // SignBits == BitWidth + 1 and
7829 // SignBits == BitWidth
7830 // The second case is difficult to check, therefore we only handle the
7831 // first case.
7832 if (SignBits == BitWidth + 1) {
7833 // It overflows only when both arguments are negative and the true
7834 // product is exactly the minimum negative number.
7835 // E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
7836 // For simplicity we just check if at least one side is not negative.
7837 KnownBits LHSKnown = computeKnownBits(V: LHS, Q: SQ);
7838 KnownBits RHSKnown = computeKnownBits(V: RHS, Q: SQ);
7839 if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
7840 return OverflowResult::NeverOverflows;
7841 }
7842 return OverflowResult::MayOverflow;
7843}
7844
7845OverflowResult
7846llvm::computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
7847 const WithCache<const Value *> &RHS,
7848 const SimplifyQuery &SQ) {
7849 ConstantRange LHSRange =
7850 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7851 ConstantRange RHSRange =
7852 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7853 return mapOverflowResult(OR: LHSRange.unsignedAddMayOverflow(Other: RHSRange));
7854}
7855
7856static OverflowResult
7857computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
7858 const WithCache<const Value *> &RHS,
7859 const AddOperator *Add, const SimplifyQuery &SQ) {
7860 if (Add && Add->hasNoSignedWrap()) {
7861 return OverflowResult::NeverOverflows;
7862 }
7863
7864 // If LHS and RHS each have at least two sign bits, the addition will look
7865 // like
7866 //
7867 // XX..... +
7868 // YY.....
7869 //
7870 // If the carry into the most significant position is 0, X and Y can't both
7871 // be 1 and therefore the carry out of the addition is also 0.
7872 //
7873 // If the carry into the most significant position is 1, X and Y can't both
7874 // be 0 and therefore the carry out of the addition is also 1.
7875 //
7876 // Since the carry into the most significant position is always equal to
7877 // the carry out of the addition, there is no signed overflow.
7878 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7879 return OverflowResult::NeverOverflows;
7880
7881 ConstantRange LHSRange =
7882 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7883 ConstantRange RHSRange =
7884 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7885 OverflowResult OR =
7886 mapOverflowResult(OR: LHSRange.signedAddMayOverflow(Other: RHSRange));
7887 if (OR != OverflowResult::MayOverflow)
7888 return OR;
7889
7890 // The remaining code needs Add to be available. Early returns if not so.
7891 if (!Add)
7892 return OverflowResult::MayOverflow;
7893
7894 // If the sign of Add is the same as at least one of the operands, this add
7895 // CANNOT overflow. If this can be determined from the known bits of the
7896 // operands the above signedAddMayOverflow() check will have already done so.
7897 // The only other way to improve on the known bits is from an assumption, so
7898 // call computeKnownBitsFromContext() directly.
7899 bool LHSOrRHSKnownNonNegative =
7900 (LHSRange.isAllNonNegative() || RHSRange.isAllNonNegative());
7901 bool LHSOrRHSKnownNegative =
7902 (LHSRange.isAllNegative() || RHSRange.isAllNegative());
7903 if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) {
7904 KnownBits AddKnown(LHSRange.getBitWidth());
7905 computeKnownBitsFromContext(V: Add, Known&: AddKnown, Q: SQ);
7906 if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) ||
7907 (AddKnown.isNegative() && LHSOrRHSKnownNegative))
7908 return OverflowResult::NeverOverflows;
7909 }
7910
7911 return OverflowResult::MayOverflow;
7912}
7913
7914OverflowResult llvm::computeOverflowForUnsignedSub(const Value *LHS,
7915 const Value *RHS,
7916 const SimplifyQuery &SQ) {
7917 // X - (X % ?)
7918 // The remainder of a value can't have greater magnitude than itself,
7919 // so the subtraction can't overflow.
7920
7921 // X - (X -nuw ?)
7922 // In the minimal case, this would simplify to "?", so there's no subtract
7923 // at all. But if this analysis is used to peek through casts, for example,
7924 // then determining no-overflow may allow other transforms.
7925
7926 // TODO: There are other patterns like this.
7927 // See simplifyICmpWithBinOpOnLHS() for candidates.
7928 if (match(V: RHS, P: m_URem(L: m_Specific(V: LHS), R: m_Value())) ||
7929 match(V: RHS, P: m_NUWSub(L: m_Specific(V: LHS), R: m_Value())))
7930 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT))
7931 return OverflowResult::NeverOverflows;
7932
7933 if (auto C = isImpliedByDomCondition(Pred: CmpInst::ICMP_UGE, LHS, RHS, ContextI: SQ.CtxI,
7934 DL: SQ.DL)) {
7935 if (*C)
7936 return OverflowResult::NeverOverflows;
7937 return OverflowResult::AlwaysOverflowsLow;
7938 }
7939
7940 ConstantRange LHSRange =
7941 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7942 ConstantRange RHSRange =
7943 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7944 return mapOverflowResult(OR: LHSRange.unsignedSubMayOverflow(Other: RHSRange));
7945}
7946
7947OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS,
7948 const Value *RHS,
7949 const SimplifyQuery &SQ) {
7950 // X - (X % ?)
7951 // The remainder of a value can't have greater magnitude than itself,
7952 // so the subtraction can't overflow.
7953
7954 // X - (X -nsw ?)
7955 // In the minimal case, this would simplify to "?", so there's no subtract
7956 // at all. But if this analysis is used to peek through casts, for example,
7957 // then determining no-overflow may allow other transforms.
7958 if (match(V: RHS, P: m_SRem(L: m_Specific(V: LHS), R: m_Value())) ||
7959 match(V: RHS, P: m_NSWSub(L: m_Specific(V: LHS), R: m_Value())))
7960 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT))
7961 return OverflowResult::NeverOverflows;
7962
7963 // If LHS and RHS each have at least two sign bits, the subtraction
7964 // cannot overflow.
7965 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7966 return OverflowResult::NeverOverflows;
7967
7968 ConstantRange LHSRange =
7969 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7970 ConstantRange RHSRange =
7971 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7972 return mapOverflowResult(OR: LHSRange.signedSubMayOverflow(Other: RHSRange));
7973}
7974
7975bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO,
7976 const DominatorTree &DT) {
7977 SmallVector<const CondBrInst *, 2> GuardingBranches;
7978 SmallVector<const ExtractValueInst *, 2> Results;
7979
7980 for (const User *U : WO->users()) {
7981 if (const auto *EVI = dyn_cast<ExtractValueInst>(Val: U)) {
7982 assert(EVI->getNumIndices() == 1 && "Obvious from CI's type");
7983
7984 if (EVI->getIndices()[0] == 0)
7985 Results.push_back(Elt: EVI);
7986 else {
7987 assert(EVI->getIndices()[0] == 1 && "Obvious from CI's type");
7988
7989 for (const auto *U : EVI->users())
7990 if (const auto *B = dyn_cast<CondBrInst>(Val: U))
7991 GuardingBranches.push_back(Elt: B);
7992 }
7993 } else {
7994 // We are using the aggregate directly in a way we don't want to analyze
7995 // here (storing it to a global, say).
7996 return false;
7997 }
7998 }
7999
8000 auto AllUsesGuardedByBranch = [&](const CondBrInst *BI) {
8001 BasicBlockEdge NoWrapEdge(BI->getParent(), BI->getSuccessor(i: 1));
8002
8003 // Check if all users of the add are provably no-wrap.
8004 for (const auto *Result : Results) {
8005 // If the extractvalue itself is not executed on overflow, the we don't
8006 // need to check each use separately, since domination is transitive.
8007 if (DT.dominates(BBE: NoWrapEdge, BB: Result->getParent()))
8008 continue;
8009
8010 for (const auto &RU : Result->uses())
8011 if (!DT.dominates(BBE: NoWrapEdge, U: RU))
8012 return false;
8013 }
8014
8015 return true;
8016 };
8017
8018 return llvm::any_of(Range&: GuardingBranches, P: AllUsesGuardedByBranch);
8019}
8020
8021/// Shifts return poison if shiftwidth is larger than the bitwidth.
8022static bool shiftAmountKnownInRange(const Value *ShiftAmount) {
8023 auto *C = dyn_cast<Constant>(Val: ShiftAmount);
8024 if (!C)
8025 return false;
8026
8027 // Shifts return poison if shiftwidth is larger than the bitwidth.
8028 SmallVector<const Constant *, 4> ShiftAmounts;
8029 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: C->getType())) {
8030 unsigned NumElts = FVTy->getNumElements();
8031 for (unsigned i = 0; i < NumElts; ++i)
8032 ShiftAmounts.push_back(Elt: C->getAggregateElement(Elt: i));
8033 } else if (isa<ScalableVectorType>(Val: C->getType()))
8034 return false; // Can't tell, just return false to be safe
8035 else
8036 ShiftAmounts.push_back(Elt: C);
8037
8038 bool Safe = llvm::all_of(Range&: ShiftAmounts, P: [](const Constant *C) {
8039 auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
8040 return CI && CI->getValue().ult(RHS: C->getType()->getIntegerBitWidth());
8041 });
8042
8043 return Safe;
8044}
8045
8046static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
8047 bool ConsiderFlagsAndMetadata) {
8048
8049 if (ConsiderFlagsAndMetadata && includesPoison(Kind) &&
8050 Op->hasPoisonGeneratingAnnotations())
8051 return true;
8052
8053 unsigned Opcode = Op->getOpcode();
8054
8055 // Check whether opcode is a poison/undef-generating operation
8056 switch (Opcode) {
8057 case Instruction::Shl:
8058 case Instruction::AShr:
8059 case Instruction::LShr:
8060 return includesPoison(Kind) && !shiftAmountKnownInRange(ShiftAmount: Op->getOperand(i: 1));
8061 case Instruction::FPToSI:
8062 case Instruction::FPToUI:
8063 // fptosi/ui yields poison if the resulting value does not fit in the
8064 // destination type.
8065 return true;
8066 case Instruction::Call:
8067 if (auto *II = dyn_cast<IntrinsicInst>(Val: Op)) {
8068 switch (II->getIntrinsicID()) {
8069 // NOTE: Use IntrNoCreateUndefOrPoison when possible.
8070 case Intrinsic::ctlz:
8071 case Intrinsic::cttz:
8072 case Intrinsic::abs:
8073 // We're not considering flags so it is safe to just return false.
8074 return false;
8075 case Intrinsic::sshl_sat:
8076 case Intrinsic::ushl_sat:
8077 if (!includesPoison(Kind) ||
8078 shiftAmountKnownInRange(ShiftAmount: II->getArgOperand(i: 1)))
8079 return false;
8080 break;
8081 }
8082 }
8083 [[fallthrough]];
8084 case Instruction::CallBr:
8085 case Instruction::Invoke: {
8086 const auto *CB = cast<CallBase>(Val: Op);
8087 return !CB->hasRetAttr(Kind: Attribute::NoUndef) &&
8088 !CB->hasFnAttr(Kind: Attribute::NoCreateUndefOrPoison);
8089 }
8090 case Instruction::InsertElement:
8091 case Instruction::ExtractElement: {
8092 // If index exceeds the length of the vector, it returns poison
8093 auto *VTy = cast<VectorType>(Val: Op->getOperand(i: 0)->getType());
8094 unsigned IdxOp = Op->getOpcode() == Instruction::InsertElement ? 2 : 1;
8095 auto *Idx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: IdxOp));
8096 if (includesPoison(Kind))
8097 return !Idx ||
8098 Idx->getValue().uge(RHS: VTy->getElementCount().getKnownMinValue());
8099 return false;
8100 }
8101 case Instruction::ShuffleVector: {
8102 ArrayRef<int> Mask = isa<ConstantExpr>(Val: Op)
8103 ? cast<ConstantExpr>(Val: Op)->getShuffleMask()
8104 : cast<ShuffleVectorInst>(Val: Op)->getShuffleMask();
8105 return includesPoison(Kind) && is_contained(Range&: Mask, Element: PoisonMaskElem);
8106 }
8107 case Instruction::FNeg:
8108 case Instruction::PHI:
8109 case Instruction::Select:
8110 case Instruction::ExtractValue:
8111 case Instruction::InsertValue:
8112 case Instruction::Freeze:
8113 case Instruction::ICmp:
8114 case Instruction::FCmp:
8115 case Instruction::GetElementPtr:
8116 return false;
8117 case Instruction::AddrSpaceCast:
8118 return true;
8119 default: {
8120 const auto *CE = dyn_cast<ConstantExpr>(Val: Op);
8121 if (isa<CastInst>(Val: Op) || (CE && CE->isCast()))
8122 return false;
8123 else if (Instruction::isBinaryOp(Opcode))
8124 return false;
8125 // Be conservative and return true.
8126 return true;
8127 }
8128 }
8129}
8130
8131bool llvm::canCreateUndefOrPoison(const Operator *Op,
8132 bool ConsiderFlagsAndMetadata) {
8133 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::UndefOrPoison,
8134 ConsiderFlagsAndMetadata);
8135}
8136
8137bool llvm::canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata) {
8138 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::PoisonOnly,
8139 ConsiderFlagsAndMetadata);
8140}
8141
8142static bool directlyImpliesPoison(const Value *ValAssumedPoison, const Value *V,
8143 unsigned Depth) {
8144 if (ValAssumedPoison == V)
8145 return true;
8146
8147 const unsigned MaxDepth = 2;
8148 if (Depth >= MaxDepth)
8149 return false;
8150
8151 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
8152 if (any_of(Range: I->operands(), P: [=](const Use &Op) {
8153 return propagatesPoison(PoisonOp: Op) &&
8154 directlyImpliesPoison(ValAssumedPoison, V: Op, Depth: Depth + 1);
8155 }))
8156 return true;
8157
8158 // V = extractvalue V0, idx
8159 // V2 = extractvalue V0, idx2
8160 // V0's elements are all poison or not. (e.g., add_with_overflow)
8161 const WithOverflowInst *II;
8162 if (match(V: I, P: m_ExtractValue(V: m_WithOverflowInst(I&: II))) &&
8163 (match(V: ValAssumedPoison, P: m_ExtractValue(V: m_Specific(V: II))) ||
8164 llvm::is_contained(Range: II->args(), Element: ValAssumedPoison)))
8165 return true;
8166 }
8167 return false;
8168}
8169
8170static bool impliesPoison(const Value *ValAssumedPoison, const Value *V,
8171 unsigned Depth) {
8172 if (isGuaranteedNotToBePoison(V: ValAssumedPoison))
8173 return true;
8174
8175 if (directlyImpliesPoison(ValAssumedPoison, V, /* Depth */ 0))
8176 return true;
8177
8178 const unsigned MaxDepth = 2;
8179 if (Depth >= MaxDepth)
8180 return false;
8181
8182 const auto *I = dyn_cast<Instruction>(Val: ValAssumedPoison);
8183 if (I && !canCreatePoison(Op: cast<Operator>(Val: I))) {
8184 return all_of(Range: I->operands(), P: [=](const Value *Op) {
8185 return impliesPoison(ValAssumedPoison: Op, V, Depth: Depth + 1);
8186 });
8187 }
8188 return false;
8189}
8190
8191bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
8192 return ::impliesPoison(ValAssumedPoison, V, /* Depth */ 0);
8193}
8194
8195static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
8196
8197static bool isGuaranteedNotToBeUndefOrPoison(
8198 const Value *V, AssumptionCache *AC, const Instruction *CtxI,
8199 const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
8200 if (Depth >= MaxAnalysisRecursionDepth)
8201 return false;
8202
8203 if (isa<MetadataAsValue>(Val: V))
8204 return false;
8205
8206 if (const auto *A = dyn_cast<Argument>(Val: V)) {
8207 if (A->hasAttribute(Kind: Attribute::NoUndef) ||
8208 A->hasAttribute(Kind: Attribute::Dereferenceable) ||
8209 A->hasAttribute(Kind: Attribute::DereferenceableOrNull))
8210 return true;
8211 }
8212
8213 if (auto *C = dyn_cast<Constant>(Val: V)) {
8214 if (isa<PoisonValue>(Val: C))
8215 return !includesPoison(Kind);
8216
8217 if (isa<UndefValue>(Val: C))
8218 return !includesUndef(Kind);
8219
8220 if (isa<ConstantInt>(Val: C) || isa<GlobalVariable>(Val: C) || isa<ConstantFP>(Val: C) ||
8221 isa<ConstantPointerNull>(Val: C) || isa<Function>(Val: C))
8222 return true;
8223
8224 if (C->getType()->isVectorTy() || C->getType()->isAggregateType()) {
8225 if (isa<ConstantExpr>(Val: C)) {
8226 // Scalable vectors can use a ConstantExpr to build a splat.
8227 if (Constant *SplatC = C->getSplatValue())
8228 if (isa<ConstantInt>(Val: SplatC) || isa<ConstantFP>(Val: SplatC))
8229 return true;
8230 } else {
8231 if (includesUndef(Kind) && C->containsUndefElement())
8232 return false;
8233 if (includesPoison(Kind) && C->containsPoisonElement())
8234 return false;
8235 return !C->containsConstantExpression();
8236 }
8237 }
8238 }
8239
8240 // Strip cast operations from a pointer value.
8241 // Note that stripPointerCastsSameRepresentation can strip off getelementptr
8242 // inbounds with zero offset. To guarantee that the result isn't poison, the
8243 // stripped pointer is checked as it has to be pointing into an allocated
8244 // object or be null `null` to ensure `inbounds` getelement pointers with a
8245 // zero offset could not produce poison.
8246 // It can strip off addrspacecast that do not change bit representation as
8247 // well. We believe that such addrspacecast is equivalent to no-op.
8248 auto *StrippedV = V->stripPointerCastsSameRepresentation();
8249 if (isa<AllocaInst>(Val: StrippedV) || isa<GlobalVariable>(Val: StrippedV) ||
8250 isa<Function>(Val: StrippedV) || isa<ConstantPointerNull>(Val: StrippedV))
8251 return true;
8252
8253 auto OpCheck = [&](const Value *V) {
8254 return isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth: Depth + 1, Kind);
8255 };
8256
8257 if (auto *Opr = dyn_cast<Operator>(Val: V)) {
8258 // If the value is a freeze instruction, then it can never
8259 // be undef or poison.
8260 if (isa<FreezeInst>(Val: V))
8261 return true;
8262
8263 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
8264 if (CB->hasRetAttr(Kind: Attribute::NoUndef) ||
8265 CB->hasRetAttr(Kind: Attribute::Dereferenceable) ||
8266 CB->hasRetAttr(Kind: Attribute::DereferenceableOrNull))
8267 return true;
8268 }
8269
8270 if (!::canCreateUndefOrPoison(Op: Opr, Kind,
8271 /*ConsiderFlagsAndMetadata=*/true)) {
8272 if (const auto *PN = dyn_cast<PHINode>(Val: V)) {
8273 unsigned Num = PN->getNumIncomingValues();
8274 bool IsWellDefined = true;
8275 for (unsigned i = 0; i < Num; ++i) {
8276 if (PN == PN->getIncomingValue(i))
8277 continue;
8278 auto *TI = PN->getIncomingBlock(i)->getTerminator();
8279 if (!isGuaranteedNotToBeUndefOrPoison(V: PN->getIncomingValue(i), AC, CtxI: TI,
8280 DT, Depth: Depth + 1, Kind)) {
8281 IsWellDefined = false;
8282 break;
8283 }
8284 }
8285 if (IsWellDefined)
8286 return true;
8287 } else if (auto *Splat = isa<ShuffleVectorInst>(Val: Opr) ? getSplatValue(V: Opr)
8288 : nullptr) {
8289 // For splats we only need to check the value being splatted.
8290 if (OpCheck(Splat))
8291 return true;
8292 } else if (all_of(Range: Opr->operands(), P: OpCheck))
8293 return true;
8294 }
8295 }
8296
8297 if (auto *I = dyn_cast<LoadInst>(Val: V))
8298 if (I->hasMetadata(KindID: LLVMContext::MD_noundef) ||
8299 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable) ||
8300 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable_or_null))
8301 return true;
8302
8303 if (programUndefinedIfUndefOrPoison(V, PoisonOnly: !includesUndef(Kind)))
8304 return true;
8305
8306 // CtxI may be null or a cloned instruction.
8307 if (!CtxI || !CtxI->getParent() || !DT)
8308 return false;
8309
8310 auto *DNode = DT->getNode(BB: CtxI->getParent());
8311 if (!DNode)
8312 // Unreachable block
8313 return false;
8314
8315 // If V is used as a branch condition before reaching CtxI, V cannot be
8316 // undef or poison.
8317 // br V, BB1, BB2
8318 // BB1:
8319 // CtxI ; V cannot be undef or poison here
8320 auto *Dominator = DNode->getIDom();
8321 // This check is purely for compile time reasons: we can skip the IDom walk
8322 // if what we are checking for includes undef and the value is not an integer.
8323 if (!includesUndef(Kind) || V->getType()->isIntegerTy())
8324 while (Dominator) {
8325 auto *TI = Dominator->getBlock()->getTerminatorOrNull();
8326
8327 Value *Cond = nullptr;
8328 if (auto BI = dyn_cast_or_null<CondBrInst>(Val: TI)) {
8329 Cond = BI->getCondition();
8330 } else if (auto SI = dyn_cast_or_null<SwitchInst>(Val: TI)) {
8331 Cond = SI->getCondition();
8332 }
8333
8334 if (Cond) {
8335 if (Cond == V)
8336 return true;
8337 else if (!includesUndef(Kind) && isa<Operator>(Val: Cond)) {
8338 // For poison, we can analyze further
8339 auto *Opr = cast<Operator>(Val: Cond);
8340 if (any_of(Range: Opr->operands(), P: [V](const Use &U) {
8341 return V == U && propagatesPoison(PoisonOp: U);
8342 }))
8343 return true;
8344 }
8345 }
8346
8347 Dominator = Dominator->getIDom();
8348 }
8349
8350 if (AC && getKnowledgeValidInContext(V, AttrKinds: {Attribute::NoUndef}, AC&: *AC, CtxI, DT))
8351 return true;
8352
8353 return false;
8354}
8355
8356bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC,
8357 const Instruction *CtxI,
8358 const DominatorTree *DT,
8359 unsigned Depth) {
8360 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8361 Kind: UndefPoisonKind::UndefOrPoison);
8362}
8363
8364bool llvm::isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC,
8365 const Instruction *CtxI,
8366 const DominatorTree *DT, unsigned Depth) {
8367 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8368 Kind: UndefPoisonKind::PoisonOnly);
8369}
8370
8371bool llvm::isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC,
8372 const Instruction *CtxI,
8373 const DominatorTree *DT, unsigned Depth) {
8374 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8375 Kind: UndefPoisonKind::UndefOnly);
8376}
8377
8378/// Return true if undefined behavior would provably be executed on the path to
8379/// OnPathTo if Root produced a posion result. Note that this doesn't say
8380/// anything about whether OnPathTo is actually executed or whether Root is
8381/// actually poison. This can be used to assess whether a new use of Root can
8382/// be added at a location which is control equivalent with OnPathTo (such as
8383/// immediately before it) without introducing UB which didn't previously
8384/// exist. Note that a false result conveys no information.
8385bool llvm::mustExecuteUBIfPoisonOnPathTo(Instruction *Root,
8386 Instruction *OnPathTo,
8387 DominatorTree *DT) {
8388 // Basic approach is to assume Root is poison, propagate poison forward
8389 // through all users we can easily track, and then check whether any of those
8390 // users are provable UB and must execute before out exiting block might
8391 // exit.
8392
8393 // The set of all recursive users we've visited (which are assumed to all be
8394 // poison because of said visit)
8395 SmallPtrSet<const Value *, 16> KnownPoison;
8396 SmallVector<const Instruction*, 16> Worklist;
8397 Worklist.push_back(Elt: Root);
8398 while (!Worklist.empty()) {
8399 const Instruction *I = Worklist.pop_back_val();
8400
8401 // If we know this must trigger UB on a path leading our target.
8402 if (mustTriggerUB(I, KnownPoison) && DT->dominates(Def: I, User: OnPathTo))
8403 return true;
8404
8405 // If we can't analyze propagation through this instruction, just skip it
8406 // and transitive users. Safe as false is a conservative result.
8407 if (I != Root && !any_of(Range: I->operands(), P: [&KnownPoison](const Use &U) {
8408 return KnownPoison.contains(Ptr: U) && propagatesPoison(PoisonOp: U);
8409 }))
8410 continue;
8411
8412 if (KnownPoison.insert(Ptr: I).second)
8413 for (const User *User : I->users())
8414 Worklist.push_back(Elt: cast<Instruction>(Val: User));
8415 }
8416
8417 // Might be non-UB, or might have a path we couldn't prove must execute on
8418 // way to exiting bb.
8419 return false;
8420}
8421
8422OverflowResult llvm::computeOverflowForSignedAdd(const AddOperator *Add,
8423 const SimplifyQuery &SQ) {
8424 return ::computeOverflowForSignedAdd(LHS: Add->getOperand(i_nocapture: 0), RHS: Add->getOperand(i_nocapture: 1),
8425 Add, SQ);
8426}
8427
8428OverflowResult
8429llvm::computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
8430 const WithCache<const Value *> &RHS,
8431 const SimplifyQuery &SQ) {
8432 return ::computeOverflowForSignedAdd(LHS, RHS, Add: nullptr, SQ);
8433}
8434
8435bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
8436 // Note: An atomic operation isn't guaranteed to return in a reasonable amount
8437 // of time because it's possible for another thread to interfere with it for an
8438 // arbitrary length of time, but programs aren't allowed to rely on that.
8439
8440 // If there is no successor, then execution can't transfer to it.
8441 if (isa<ReturnInst>(Val: I))
8442 return false;
8443 if (isa<UnreachableInst>(Val: I))
8444 return false;
8445
8446 // Note: Do not add new checks here; instead, change Instruction::mayThrow or
8447 // Instruction::willReturn.
8448 //
8449 // FIXME: Move this check into Instruction::willReturn.
8450 if (isa<CatchPadInst>(Val: I)) {
8451 switch (classifyEHPersonality(Pers: I->getFunction()->getPersonalityFn())) {
8452 default:
8453 // A catchpad may invoke exception object constructors and such, which
8454 // in some languages can be arbitrary code, so be conservative by default.
8455 return false;
8456 case EHPersonality::CoreCLR:
8457 // For CoreCLR, it just involves a type test.
8458 return true;
8459 }
8460 }
8461
8462 // An instruction that returns without throwing must transfer control flow
8463 // to a successor.
8464 return !I->mayThrow() && I->willReturn();
8465}
8466
8467bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) {
8468 // TODO: This is slightly conservative for invoke instruction since exiting
8469 // via an exception *is* normal control for them.
8470 for (const Instruction &I : *BB)
8471 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8472 return false;
8473 return true;
8474}
8475
8476bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8477 BasicBlock::const_iterator Begin, BasicBlock::const_iterator End,
8478 unsigned ScanLimit) {
8479 return isGuaranteedToTransferExecutionToSuccessor(Range: make_range(x: Begin, y: End),
8480 ScanLimit);
8481}
8482
8483bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8484 iterator_range<BasicBlock::const_iterator> Range, unsigned ScanLimit) {
8485 assert(ScanLimit && "scan limit must be non-zero");
8486 for (const Instruction &I : Range) {
8487 if (--ScanLimit == 0)
8488 return false;
8489 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8490 return false;
8491 }
8492 return true;
8493}
8494
8495bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
8496 const Loop *L) {
8497 // The loop header is guaranteed to be executed for every iteration.
8498 //
8499 // FIXME: Relax this constraint to cover all basic blocks that are
8500 // guaranteed to be executed at every iteration.
8501 if (I->getParent() != L->getHeader()) return false;
8502
8503 for (const Instruction &LI : *L->getHeader()) {
8504 if (&LI == I) return true;
8505 if (!isGuaranteedToTransferExecutionToSuccessor(I: &LI)) return false;
8506 }
8507 llvm_unreachable("Instruction not contained in its own parent basic block.");
8508}
8509
8510bool llvm::intrinsicPropagatesPoison(Intrinsic::ID IID) {
8511 switch (IID) {
8512 // TODO: Add more intrinsics.
8513 case Intrinsic::sadd_with_overflow:
8514 case Intrinsic::ssub_with_overflow:
8515 case Intrinsic::smul_with_overflow:
8516 case Intrinsic::uadd_with_overflow:
8517 case Intrinsic::usub_with_overflow:
8518 case Intrinsic::umul_with_overflow:
8519 // If an input is a vector containing a poison element, the
8520 // two output vectors (calculated results, overflow bits)'
8521 // corresponding lanes are poison.
8522 return true;
8523 case Intrinsic::ctpop:
8524 case Intrinsic::ctlz:
8525 case Intrinsic::cttz:
8526 case Intrinsic::abs:
8527 case Intrinsic::smax:
8528 case Intrinsic::smin:
8529 case Intrinsic::umax:
8530 case Intrinsic::umin:
8531 case Intrinsic::scmp:
8532 case Intrinsic::smulh:
8533 case Intrinsic::umulh:
8534 case Intrinsic::is_fpclass:
8535 case Intrinsic::ptrmask:
8536 case Intrinsic::ucmp:
8537 case Intrinsic::bitreverse:
8538 case Intrinsic::bswap:
8539 case Intrinsic::sadd_sat:
8540 case Intrinsic::ssub_sat:
8541 case Intrinsic::sshl_sat:
8542 case Intrinsic::uadd_sat:
8543 case Intrinsic::usub_sat:
8544 case Intrinsic::ushl_sat:
8545 case Intrinsic::smul_fix:
8546 case Intrinsic::smul_fix_sat:
8547 case Intrinsic::umul_fix:
8548 case Intrinsic::umul_fix_sat:
8549 case Intrinsic::pow:
8550 case Intrinsic::powi:
8551 case Intrinsic::sin:
8552 case Intrinsic::sinh:
8553 case Intrinsic::cos:
8554 case Intrinsic::cosh:
8555 case Intrinsic::sincos:
8556 case Intrinsic::sincospi:
8557 case Intrinsic::tan:
8558 case Intrinsic::tanh:
8559 case Intrinsic::asin:
8560 case Intrinsic::acos:
8561 case Intrinsic::atan:
8562 case Intrinsic::atan2:
8563 case Intrinsic::canonicalize:
8564 case Intrinsic::sqrt:
8565 case Intrinsic::fma:
8566 case Intrinsic::fmuladd:
8567 case Intrinsic::exp:
8568 case Intrinsic::exp2:
8569 case Intrinsic::exp10:
8570 case Intrinsic::log:
8571 case Intrinsic::log2:
8572 case Intrinsic::log10:
8573 case Intrinsic::modf:
8574 case Intrinsic::floor:
8575 case Intrinsic::ceil:
8576 case Intrinsic::trunc:
8577 case Intrinsic::rint:
8578 case Intrinsic::nearbyint:
8579 case Intrinsic::round:
8580 case Intrinsic::roundeven:
8581 case Intrinsic::lrint:
8582 case Intrinsic::llrint:
8583 case Intrinsic::fshl:
8584 case Intrinsic::fshr:
8585 case Intrinsic::frexp:
8586 case Intrinsic::get_active_lane_mask:
8587 return true;
8588 default:
8589 return false;
8590 }
8591}
8592
8593bool llvm::propagatesPoison(const Use &PoisonOp) {
8594 const Operator *I = cast<Operator>(Val: PoisonOp.getUser());
8595 switch (I->getOpcode()) {
8596 case Instruction::Freeze:
8597 case Instruction::PHI:
8598 case Instruction::Invoke:
8599 return false;
8600 case Instruction::Select:
8601 return PoisonOp.getOperandNo() == 0;
8602 case Instruction::Call:
8603 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
8604 return intrinsicPropagatesPoison(IID: II->getIntrinsicID());
8605 return false;
8606 case Instruction::ICmp:
8607 case Instruction::FCmp:
8608 case Instruction::GetElementPtr:
8609 return true;
8610 default:
8611 if (isa<BinaryOperator>(Val: I) || isa<UnaryOperator>(Val: I) || isa<CastInst>(Val: I))
8612 return true;
8613
8614 // Be conservative and return false.
8615 return false;
8616 }
8617}
8618
8619/// Enumerates all operands of \p I that are guaranteed to not be undef or
8620/// poison. If the callback \p Handle returns true, stop processing and return
8621/// true. Otherwise, return false.
8622template <typename CallableT>
8623static bool handleGuaranteedWellDefinedOps(const Instruction *I,
8624 const CallableT &Handle) {
8625 switch (I->getOpcode()) {
8626 case Instruction::Store:
8627 if (Handle(cast<StoreInst>(Val: I)->getPointerOperand()))
8628 return true;
8629 break;
8630
8631 case Instruction::Load:
8632 if (Handle(cast<LoadInst>(Val: I)->getPointerOperand()))
8633 return true;
8634 break;
8635
8636 // Since dereferenceable attribute imply noundef, atomic operations
8637 // also implicitly have noundef pointers too
8638 case Instruction::AtomicCmpXchg:
8639 if (Handle(cast<AtomicCmpXchgInst>(Val: I)->getPointerOperand()))
8640 return true;
8641 break;
8642
8643 case Instruction::AtomicRMW:
8644 if (Handle(cast<AtomicRMWInst>(Val: I)->getPointerOperand()))
8645 return true;
8646 break;
8647
8648 case Instruction::Call:
8649 case Instruction::Invoke: {
8650 const CallBase *CB = cast<CallBase>(Val: I);
8651 if (CB->isIndirectCall() && Handle(CB->getCalledOperand()))
8652 return true;
8653 for (unsigned i = 0; i < CB->arg_size(); ++i)
8654 if ((CB->paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef) ||
8655 CB->paramHasAttr(ArgNo: i, Kind: Attribute::Dereferenceable) ||
8656 CB->paramHasAttr(ArgNo: i, Kind: Attribute::DereferenceableOrNull)) &&
8657 Handle(CB->getArgOperand(i)))
8658 return true;
8659 break;
8660 }
8661 case Instruction::Ret:
8662 if (I->getFunction()->hasRetAttribute(Kind: Attribute::NoUndef) &&
8663 Handle(I->getOperand(i: 0)))
8664 return true;
8665 break;
8666 case Instruction::Switch:
8667 if (Handle(cast<SwitchInst>(Val: I)->getCondition()))
8668 return true;
8669 break;
8670 case Instruction::CondBr:
8671 if (Handle(cast<CondBrInst>(Val: I)->getCondition()))
8672 return true;
8673 break;
8674 default:
8675 break;
8676 }
8677
8678 return false;
8679}
8680
8681/// Enumerates all operands of \p I that are guaranteed to not be poison.
8682template <typename CallableT>
8683static bool handleGuaranteedNonPoisonOps(const Instruction *I,
8684 const CallableT &Handle) {
8685 if (handleGuaranteedWellDefinedOps(I, Handle))
8686 return true;
8687 switch (I->getOpcode()) {
8688 // Divisors of these operations are allowed to be partially undef.
8689 case Instruction::UDiv:
8690 case Instruction::SDiv:
8691 case Instruction::URem:
8692 case Instruction::SRem:
8693 return Handle(I->getOperand(i: 1));
8694 default:
8695 return false;
8696 }
8697}
8698
8699bool llvm::mustTriggerUB(const Instruction *I,
8700 const SmallPtrSetImpl<const Value *> &KnownPoison) {
8701 return handleGuaranteedNonPoisonOps(
8702 I, Handle: [&](const Value *V) { return KnownPoison.count(Ptr: V); });
8703}
8704
8705static bool programUndefinedIfUndefOrPoison(const Value *V,
8706 bool PoisonOnly) {
8707 // We currently only look for uses of values within the same basic
8708 // block, as that makes it easier to guarantee that the uses will be
8709 // executed given that Inst is executed.
8710 //
8711 // FIXME: Expand this to consider uses beyond the same basic block. To do
8712 // this, look out for the distinction between post-dominance and strong
8713 // post-dominance.
8714 const BasicBlock *BB = nullptr;
8715 BasicBlock::const_iterator Begin;
8716 if (const auto *Inst = dyn_cast<Instruction>(Val: V)) {
8717 BB = Inst->getParent();
8718 Begin = Inst->getIterator();
8719 Begin++;
8720 } else if (const auto *Arg = dyn_cast<Argument>(Val: V)) {
8721 if (Arg->getParent()->isDeclaration())
8722 return false;
8723 BB = &Arg->getParent()->getEntryBlock();
8724 Begin = BB->begin();
8725 } else {
8726 return false;
8727 }
8728
8729 // Limit number of instructions we look at, to avoid scanning through large
8730 // blocks. The current limit is chosen arbitrarily.
8731 unsigned ScanLimit = 32;
8732 BasicBlock::const_iterator End = BB->end();
8733
8734 if (!PoisonOnly) {
8735 // Since undef does not propagate eagerly, be conservative & just check
8736 // whether a value is directly passed to an instruction that must take
8737 // well-defined operands.
8738
8739 for (const auto &I : make_range(x: Begin, y: End)) {
8740 if (--ScanLimit == 0)
8741 break;
8742
8743 if (handleGuaranteedWellDefinedOps(I: &I, Handle: [V](const Value *WellDefinedOp) {
8744 return WellDefinedOp == V;
8745 }))
8746 return true;
8747
8748 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8749 break;
8750 }
8751 return false;
8752 }
8753
8754 // Set of instructions that we have proved will yield poison if Inst
8755 // does.
8756 SmallPtrSet<const Value *, 16> YieldsPoison;
8757 SmallPtrSet<const BasicBlock *, 4> Visited;
8758
8759 YieldsPoison.insert(Ptr: V);
8760 Visited.insert(Ptr: BB);
8761
8762 while (true) {
8763 for (const auto &I : make_range(x: Begin, y: End)) {
8764 if (--ScanLimit == 0)
8765 return false;
8766 if (mustTriggerUB(I: &I, KnownPoison: YieldsPoison))
8767 return true;
8768 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8769 return false;
8770
8771 // If an operand is poison and propagates it, mark I as yielding poison.
8772 for (const Use &Op : I.operands()) {
8773 if (YieldsPoison.count(Ptr: Op) && propagatesPoison(PoisonOp: Op)) {
8774 YieldsPoison.insert(Ptr: &I);
8775 break;
8776 }
8777 }
8778
8779 // Special handling for select, which returns poison if its operand 0 is
8780 // poison (handled in the loop above) *or* if both its true/false operands
8781 // are poison (handled here).
8782 if (I.getOpcode() == Instruction::Select &&
8783 YieldsPoison.count(Ptr: I.getOperand(i: 1)) &&
8784 YieldsPoison.count(Ptr: I.getOperand(i: 2))) {
8785 YieldsPoison.insert(Ptr: &I);
8786 }
8787 }
8788
8789 BB = BB->getSingleSuccessor();
8790 if (!BB || !Visited.insert(Ptr: BB).second)
8791 break;
8792
8793 Begin = BB->getFirstNonPHIIt();
8794 End = BB->end();
8795 }
8796 return false;
8797}
8798
8799bool llvm::programUndefinedIfUndefOrPoison(const Instruction *Inst) {
8800 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: false);
8801}
8802
8803bool llvm::programUndefinedIfPoison(const Instruction *Inst) {
8804 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: true);
8805}
8806
8807static bool isKnownNonNaN(const Value *V, FastMathFlags FMF) {
8808 if (FMF.noNaNs())
8809 return true;
8810
8811 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8812 return !C->isNaN();
8813
8814 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8815 if (!C->getElementType()->isFloatingPointTy())
8816 return false;
8817 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8818 if (C->getElementAsAPFloat(i: I).isNaN())
8819 return false;
8820 }
8821 return true;
8822 }
8823
8824 if (isa<ConstantAggregateZero>(Val: V))
8825 return true;
8826
8827 return false;
8828}
8829
8830static bool isKnownNonZero(const Value *V) {
8831 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8832 return !C->isZero();
8833
8834 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8835 if (!C->getElementType()->isFloatingPointTy())
8836 return false;
8837 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8838 if (C->getElementAsAPFloat(i: I).isZero())
8839 return false;
8840 }
8841 return true;
8842 }
8843
8844 return false;
8845}
8846
8847/// Match clamp pattern for float types without care about NaNs or signed zeros.
8848/// Given non-min/max outer cmp/select from the clamp pattern this
8849/// function recognizes if it can be substitued by a "canonical" min/max
8850/// pattern.
8851static SelectPatternResult matchFastFloatClamp(CmpInst::Predicate Pred,
8852 Value *CmpLHS, Value *CmpRHS,
8853 Value *TrueVal, Value *FalseVal,
8854 Value *&LHS, Value *&RHS) {
8855 // Try to match
8856 // X < C1 ? C1 : Min(X, C2) --> Max(C1, Min(X, C2))
8857 // X > C1 ? C1 : Max(X, C2) --> Min(C1, Max(X, C2))
8858 // and return description of the outer Max/Min.
8859
8860 // First, check if select has inverse order:
8861 if (CmpRHS == FalseVal) {
8862 std::swap(a&: TrueVal, b&: FalseVal);
8863 Pred = CmpInst::getInversePredicate(pred: Pred);
8864 }
8865
8866 // Assume success now. If there's no match, callers should not use these anyway.
8867 LHS = TrueVal;
8868 RHS = FalseVal;
8869
8870 const APFloat *FC1;
8871 if (CmpRHS != TrueVal || !match(V: CmpRHS, P: m_APFloat(Res&: FC1)) || !FC1->isFinite())
8872 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8873
8874 const APFloat *FC2;
8875 switch (Pred) {
8876 case CmpInst::FCMP_OLT:
8877 case CmpInst::FCMP_OLE:
8878 case CmpInst::FCMP_ULT:
8879 case CmpInst::FCMP_ULE:
8880 if (match(V: FalseVal, P: m_OrdOrUnordFMin(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8881 *FC1 < *FC2)
8882 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8883 if (match(V: FalseVal, P: m_FMinNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
8884 *FC1 < *FC2)
8885 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8886 break;
8887 case CmpInst::FCMP_OGT:
8888 case CmpInst::FCMP_OGE:
8889 case CmpInst::FCMP_UGT:
8890 case CmpInst::FCMP_UGE:
8891 if (match(V: FalseVal, P: m_OrdOrUnordFMax(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8892 *FC1 > *FC2)
8893 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8894 if (match(V: FalseVal, P: m_FMaxNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
8895 *FC1 > *FC2)
8896 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8897 break;
8898 default:
8899 break;
8900 }
8901
8902 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8903}
8904
8905/// Recognize variations of:
8906/// CLAMP(v,l,h) ==> ((v) < (l) ? (l) : ((v) > (h) ? (h) : (v)))
8907static SelectPatternResult matchClamp(CmpInst::Predicate Pred,
8908 Value *CmpLHS, Value *CmpRHS,
8909 Value *TrueVal, Value *FalseVal) {
8910 // Swap the select operands and predicate to match the patterns below.
8911 if (CmpRHS != TrueVal) {
8912 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8913 std::swap(a&: TrueVal, b&: FalseVal);
8914 }
8915 const APInt *C1;
8916 if (CmpRHS == TrueVal && match(V: CmpRHS, P: m_APInt(Res&: C1))) {
8917 const APInt *C2;
8918 // (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
8919 if (match(V: FalseVal, P: m_SMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8920 C1->slt(RHS: *C2) && Pred == CmpInst::ICMP_SLT)
8921 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8922
8923 // (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
8924 if (match(V: FalseVal, P: m_SMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8925 C1->sgt(RHS: *C2) && Pred == CmpInst::ICMP_SGT)
8926 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8927
8928 // (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
8929 if (match(V: FalseVal, P: m_UMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8930 C1->ult(RHS: *C2) && Pred == CmpInst::ICMP_ULT)
8931 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8932
8933 // (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
8934 if (match(V: FalseVal, P: m_UMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8935 C1->ugt(RHS: *C2) && Pred == CmpInst::ICMP_UGT)
8936 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8937 }
8938 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8939}
8940
8941/// Recognize variations of:
8942/// a < c ? min(a,b) : min(b,c) ==> min(min(a,b),min(b,c))
8943static SelectPatternResult matchMinMaxOfMinMax(CmpInst::Predicate Pred,
8944 Value *CmpLHS, Value *CmpRHS,
8945 Value *TVal, Value *FVal,
8946 unsigned Depth) {
8947 // TODO: Allow FP min/max with nnan/nsz.
8948 assert(CmpInst::isIntPredicate(Pred) && "Expected integer comparison");
8949
8950 Value *A = nullptr, *B = nullptr;
8951 SelectPatternResult L = matchSelectPattern(V: TVal, LHS&: A, RHS&: B, CastOp: nullptr, Depth: Depth + 1);
8952 if (!SelectPatternResult::isMinOrMax(SPF: L.Flavor))
8953 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8954
8955 Value *C = nullptr, *D = nullptr;
8956 SelectPatternResult R = matchSelectPattern(V: FVal, LHS&: C, RHS&: D, CastOp: nullptr, Depth: Depth + 1);
8957 if (L.Flavor != R.Flavor)
8958 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8959
8960 // We have something like: x Pred y ? min(a, b) : min(c, d).
8961 // Try to match the compare to the min/max operations of the select operands.
8962 // First, make sure we have the right compare predicate.
8963 switch (L.Flavor) {
8964 case SPF_SMIN:
8965 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) {
8966 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8967 std::swap(a&: CmpLHS, b&: CmpRHS);
8968 }
8969 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
8970 break;
8971 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8972 case SPF_SMAX:
8973 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE) {
8974 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8975 std::swap(a&: CmpLHS, b&: CmpRHS);
8976 }
8977 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE)
8978 break;
8979 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8980 case SPF_UMIN:
8981 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
8982 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8983 std::swap(a&: CmpLHS, b&: CmpRHS);
8984 }
8985 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
8986 break;
8987 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8988 case SPF_UMAX:
8989 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
8990 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8991 std::swap(a&: CmpLHS, b&: CmpRHS);
8992 }
8993 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE)
8994 break;
8995 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8996 default:
8997 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8998 }
8999
9000 // If there is a common operand in the already matched min/max and the other
9001 // min/max operands match the compare operands (either directly or inverted),
9002 // then this is min/max of the same flavor.
9003
9004 // a pred c ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
9005 // ~c pred ~a ? m(a, b) : m(c, b) --> m(m(a, b), m(c, b))
9006 if (D == B) {
9007 if ((CmpLHS == A && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
9008 match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
9009 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
9010 }
9011 // a pred d ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
9012 // ~d pred ~a ? m(a, b) : m(b, d) --> m(m(a, b), m(b, d))
9013 if (C == B) {
9014 if ((CmpLHS == A && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
9015 match(V: A, P: m_Not(V: m_Specific(V: CmpRHS)))))
9016 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
9017 }
9018 // b pred c ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
9019 // ~c pred ~b ? m(a, b) : m(c, a) --> m(m(a, b), m(c, a))
9020 if (D == A) {
9021 if ((CmpLHS == B && CmpRHS == C) || (match(V: C, P: m_Not(V: m_Specific(V: CmpLHS))) &&
9022 match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
9023 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
9024 }
9025 // b pred d ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
9026 // ~d pred ~b ? m(a, b) : m(a, d) --> m(m(a, b), m(a, d))
9027 if (C == A) {
9028 if ((CmpLHS == B && CmpRHS == D) || (match(V: D, P: m_Not(V: m_Specific(V: CmpLHS))) &&
9029 match(V: B, P: m_Not(V: m_Specific(V: CmpRHS)))))
9030 return {.Flavor: L.Flavor, .NaNBehavior: SPNB_NA, .Ordered: false};
9031 }
9032
9033 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9034}
9035
9036/// If the input value is the result of a 'not' op, constant integer, or vector
9037/// splat of a constant integer, return the bitwise-not source value.
9038/// TODO: This could be extended to handle non-splat vector integer constants.
9039static Value *getNotValue(Value *V) {
9040 Value *NotV;
9041 if (match(V, P: m_Not(V: m_Value(V&: NotV))))
9042 return NotV;
9043
9044 const APInt *C;
9045 if (match(V, P: m_APInt(Res&: C)))
9046 return ConstantInt::get(Ty: V->getType(), V: ~(*C));
9047
9048 return nullptr;
9049}
9050
9051/// Match non-obvious integer minimum and maximum sequences.
9052static SelectPatternResult matchMinMax(CmpInst::Predicate Pred,
9053 Value *CmpLHS, Value *CmpRHS,
9054 Value *TrueVal, Value *FalseVal,
9055 Value *&LHS, Value *&RHS,
9056 unsigned Depth) {
9057 // Assume success. If there's no match, callers should not use these anyway.
9058 LHS = TrueVal;
9059 RHS = FalseVal;
9060
9061 SelectPatternResult SPR = matchClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal);
9062 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
9063 return SPR;
9064
9065 SPR = matchMinMaxOfMinMax(Pred, CmpLHS, CmpRHS, TVal: TrueVal, FVal: FalseVal, Depth);
9066 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
9067 return SPR;
9068
9069 // Look through 'not' ops to find disguised min/max.
9070 // (X > Y) ? ~X : ~Y ==> (~X < ~Y) ? ~X : ~Y ==> MIN(~X, ~Y)
9071 // (X < Y) ? ~X : ~Y ==> (~X > ~Y) ? ~X : ~Y ==> MAX(~X, ~Y)
9072 if (CmpLHS == getNotValue(V: TrueVal) && CmpRHS == getNotValue(V: FalseVal)) {
9073 switch (Pred) {
9074 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9075 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9076 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9077 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9078 default: break;
9079 }
9080 }
9081
9082 // (X > Y) ? ~Y : ~X ==> (~X < ~Y) ? ~Y : ~X ==> MAX(~Y, ~X)
9083 // (X < Y) ? ~Y : ~X ==> (~X > ~Y) ? ~Y : ~X ==> MIN(~Y, ~X)
9084 if (CmpLHS == getNotValue(V: FalseVal) && CmpRHS == getNotValue(V: TrueVal)) {
9085 switch (Pred) {
9086 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9087 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9088 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9089 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9090 default: break;
9091 }
9092 }
9093
9094 if (Pred != CmpInst::ICMP_SGT && Pred != CmpInst::ICMP_SLT)
9095 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9096
9097 const APInt *C1;
9098 if (!match(V: CmpRHS, P: m_APInt(Res&: C1)))
9099 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9100
9101 // An unsigned min/max can be written with a signed compare.
9102 const APInt *C2;
9103 if ((CmpLHS == TrueVal && match(V: FalseVal, P: m_APInt(Res&: C2))) ||
9104 (CmpLHS == FalseVal && match(V: TrueVal, P: m_APInt(Res&: C2)))) {
9105 // Is the sign bit set?
9106 // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
9107 // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
9108 if (Pred == CmpInst::ICMP_SLT && C1->isZero() && C2->isMaxSignedValue())
9109 return {.Flavor: CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9110
9111 // Is the sign bit clear?
9112 // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
9113 // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
9114 if (Pred == CmpInst::ICMP_SGT && C1->isAllOnes() && C2->isMinSignedValue())
9115 return {.Flavor: CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9116 }
9117
9118 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9119}
9120
9121bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW,
9122 bool AllowPoison) {
9123 assert(X && Y && "Invalid operand");
9124
9125 auto IsNegationOf = [&](const Value *X, const Value *Y) {
9126 if (!match(V: X, P: m_Neg(V: m_Specific(V: Y))))
9127 return false;
9128
9129 auto *BO = cast<BinaryOperator>(Val: X);
9130 if (NeedNSW && !BO->hasNoSignedWrap())
9131 return false;
9132
9133 auto *Zero = cast<Constant>(Val: BO->getOperand(i_nocapture: 0));
9134 if (!AllowPoison && !Zero->isNullValue())
9135 return false;
9136
9137 return true;
9138 };
9139
9140 // X = -Y or Y = -X
9141 if (IsNegationOf(X, Y) || IsNegationOf(Y, X))
9142 return true;
9143
9144 // X = sub (A, B), Y = sub (B, A) || X = sub nsw (A, B), Y = sub nsw (B, A)
9145 Value *A, *B;
9146 return (!NeedNSW && (match(V: X, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9147 match(V: Y, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))))) ||
9148 (NeedNSW && (match(V: X, P: m_NSWSub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9149 match(V: Y, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A)))));
9150}
9151
9152bool llvm::isKnownInversion(const Value *X, const Value *Y) {
9153 // Handle X = icmp pred A, B, Y = icmp pred A, C.
9154 Value *A, *B, *C;
9155 CmpPredicate Pred1, Pred2;
9156 if (!match(V: X, P: m_ICmp(Pred&: Pred1, L: m_Value(V&: A), R: m_Value(V&: B))) ||
9157 !match(V: Y, P: m_c_ICmp(Pred&: Pred2, L: m_Specific(V: A), R: m_Value(V&: C))))
9158 return false;
9159
9160 // They must both have samesign flag or not.
9161 if (Pred1.hasSameSign() != Pred2.hasSameSign())
9162 return false;
9163
9164 if (B == C)
9165 return Pred1 == ICmpInst::getInversePredicate(pred: Pred2);
9166
9167 // Try to infer the relationship from constant ranges.
9168 const APInt *RHSC1, *RHSC2;
9169 if (!match(V: B, P: m_APInt(Res&: RHSC1)) || !match(V: C, P: m_APInt(Res&: RHSC2)))
9170 return false;
9171
9172 // Sign bits of two RHSCs should match.
9173 if (Pred1.hasSameSign() && RHSC1->isNonNegative() != RHSC2->isNonNegative())
9174 return false;
9175
9176 const auto CR1 = ConstantRange::makeExactICmpRegion(Pred: Pred1, Other: *RHSC1);
9177 const auto CR2 = ConstantRange::makeExactICmpRegion(Pred: Pred2, Other: *RHSC2);
9178
9179 return CR1.inverse() == CR2;
9180}
9181
9182SelectPatternResult llvm::getSelectPattern(CmpInst::Predicate Pred,
9183 SelectPatternNaNBehavior NaNBehavior,
9184 bool Ordered) {
9185 switch (Pred) {
9186 default:
9187 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false}; // Equality.
9188 case ICmpInst::ICMP_UGT:
9189 case ICmpInst::ICMP_UGE:
9190 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9191 case ICmpInst::ICMP_SGT:
9192 case ICmpInst::ICMP_SGE:
9193 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9194 case ICmpInst::ICMP_ULT:
9195 case ICmpInst::ICMP_ULE:
9196 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9197 case ICmpInst::ICMP_SLT:
9198 case ICmpInst::ICMP_SLE:
9199 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9200 case FCmpInst::FCMP_UGT:
9201 case FCmpInst::FCMP_UGE:
9202 case FCmpInst::FCMP_OGT:
9203 case FCmpInst::FCMP_OGE:
9204 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
9205 case FCmpInst::FCMP_ULT:
9206 case FCmpInst::FCMP_ULE:
9207 case FCmpInst::FCMP_OLT:
9208 case FCmpInst::FCMP_OLE:
9209 return {.Flavor: SPF_FMINNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
9210 }
9211}
9212
9213std::optional<std::pair<CmpPredicate, Constant *>>
9214llvm::getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred, Constant *C) {
9215 assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
9216 "Only for relational integer predicates.");
9217 if (isa<UndefValue>(Val: C))
9218 return std::nullopt;
9219
9220 Type *Type = C->getType();
9221 bool IsSigned = ICmpInst::isSigned(Pred);
9222
9223 CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred);
9224 bool WillIncrement =
9225 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
9226
9227 // Check if the constant operand can be safely incremented/decremented
9228 // without overflowing/underflowing.
9229 auto ConstantIsOk = [Pred, WillIncrement, IsSigned](ConstantInt *C) {
9230 if (WillIncrement ? C->isMaxValue(IsSigned) : C->isMinValue(IsSigned))
9231 return false;
9232
9233 if (!Pred.hasSameSign())
9234 return true;
9235
9236 // Crossing the corresponding boundary in the other ordering changes the
9237 // sign bit, and therefore changes the poison domain.
9238 return WillIncrement ? !C->isMaxValue(IsSigned: !IsSigned)
9239 : !C->isMinValue(IsSigned: !IsSigned);
9240 };
9241
9242 Constant *SafeReplacementConstant = nullptr;
9243 if (auto *CI = dyn_cast<ConstantInt>(Val: C)) {
9244 // Bail out if the constant can't be safely incremented/decremented.
9245 if (!ConstantIsOk(CI))
9246 return std::nullopt;
9247 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Type)) {
9248 unsigned NumElts = FVTy->getNumElements();
9249 for (unsigned i = 0; i != NumElts; ++i) {
9250 Constant *Elt = C->getAggregateElement(Elt: i);
9251 if (!Elt)
9252 return std::nullopt;
9253
9254 if (isa<UndefValue>(Val: Elt))
9255 continue;
9256
9257 // Bail out if we can't determine if this constant is min/max or if we
9258 // know that this constant is min/max.
9259 auto *CI = dyn_cast<ConstantInt>(Val: Elt);
9260 if (!CI || !ConstantIsOk(CI))
9261 return std::nullopt;
9262
9263 if (!SafeReplacementConstant)
9264 SafeReplacementConstant = CI;
9265 }
9266 } else if (isa<VectorType>(Val: C->getType())) {
9267 // Handle scalable splat
9268 Value *SplatC = C->getSplatValue();
9269 auto *CI = dyn_cast_or_null<ConstantInt>(Val: SplatC);
9270 // Bail out if the constant can't be safely incremented/decremented.
9271 if (!CI || !ConstantIsOk(CI))
9272 return std::nullopt;
9273 } else {
9274 // ConstantExpr?
9275 return std::nullopt;
9276 }
9277
9278 // It may not be safe to change a compare predicate in the presence of
9279 // undefined elements, so replace those elements with the first safe constant
9280 // that we found.
9281 // TODO: in case of poison, it is safe; let's replace undefs only.
9282 if (C->containsUndefOrPoisonElement()) {
9283 assert(SafeReplacementConstant && "Replacement constant not set");
9284 C = Constant::replaceUndefsWith(C, Replacement: SafeReplacementConstant);
9285 }
9286
9287 CmpPredicate NewPred(CmpInst::getFlippedStrictnessPredicate(pred: Pred),
9288 Pred.hasSameSign());
9289
9290 // Increment or decrement the constant.
9291 Constant *OneOrNegOne = ConstantInt::get(Ty: Type, V: WillIncrement ? 1 : -1, IsSigned: true);
9292 Constant *NewC = ConstantExpr::getAdd(C1: C, C2: OneOrNegOne);
9293
9294 return std::make_pair(x&: NewPred, y&: NewC);
9295}
9296
9297static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
9298 FastMathFlags FMF,
9299 Value *CmpLHS, Value *CmpRHS,
9300 Value *TrueVal, Value *FalseVal,
9301 Value *&LHS, Value *&RHS,
9302 unsigned Depth) {
9303 if (CmpInst::isFPPredicate(P: Pred)) {
9304 // IEEE-754 ignores the sign of 0.0 in comparisons. So if the select has one
9305 // 0.0 operand, set the compare's 0.0 operands to that same value for the
9306 // purpose of identifying min/max. Disregard vector constants with undefined
9307 // elements because those can not be back-propagated for analysis.
9308 Value *OutputZeroVal = nullptr;
9309 if (match(V: TrueVal, P: m_AnyZeroFP()) && !match(V: FalseVal, P: m_AnyZeroFP()) &&
9310 !cast<Constant>(Val: TrueVal)->containsUndefOrPoisonElement())
9311 OutputZeroVal = TrueVal;
9312 else if (match(V: FalseVal, P: m_AnyZeroFP()) && !match(V: TrueVal, P: m_AnyZeroFP()) &&
9313 !cast<Constant>(Val: FalseVal)->containsUndefOrPoisonElement())
9314 OutputZeroVal = FalseVal;
9315
9316 if (OutputZeroVal) {
9317 if (match(V: CmpLHS, P: m_AnyZeroFP()) && CmpLHS != OutputZeroVal)
9318 CmpLHS = OutputZeroVal;
9319 if (match(V: CmpRHS, P: m_AnyZeroFP()) && CmpRHS != OutputZeroVal)
9320 CmpRHS = OutputZeroVal;
9321 }
9322 }
9323
9324 LHS = CmpLHS;
9325 RHS = CmpRHS;
9326
9327 // Signed zero may return inconsistent results between implementations.
9328 // (0.0 <= -0.0) ? 0.0 : -0.0 // Returns 0.0
9329 // minNum(0.0, -0.0) // May return -0.0 or 0.0 (IEEE 754-2008 5.3.1)
9330 // Therefore, we behave conservatively and only proceed if at least one of the
9331 // operands is known to not be zero or if we don't care about signed zero.
9332 if (CmpInst::isFPPredicate(P: Pred)) {
9333 if (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9334 !isKnownNonZero(V: CmpRHS))
9335 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9336 }
9337
9338 SelectPatternNaNBehavior NaNBehavior = SPNB_NA;
9339 bool Ordered = false;
9340
9341 // When given one NaN and one non-NaN input:
9342 // - maxnum/minnum (C99 fmaxf()/fminf()) return the non-NaN input.
9343 // - A simple C99 (a < b ? a : b) construction will return 'b' (as the
9344 // ordered comparison fails), which could be NaN or non-NaN.
9345 // so here we discover exactly what NaN behavior is required/accepted.
9346 if (CmpInst::isFPPredicate(P: Pred)) {
9347 bool LHSSafe = isKnownNonNaN(V: CmpLHS, FMF);
9348 bool RHSSafe = isKnownNonNaN(V: CmpRHS, FMF);
9349
9350 if (LHSSafe && RHSSafe) {
9351 // Both operands are known non-NaN.
9352 NaNBehavior = SPNB_RETURNS_ANY;
9353 Ordered = CmpInst::isOrdered(predicate: Pred);
9354 } else if (CmpInst::isOrdered(predicate: Pred)) {
9355 // An ordered comparison will return false when given a NaN, so it
9356 // returns the RHS.
9357 Ordered = true;
9358 if (LHSSafe)
9359 // LHS is non-NaN, so if RHS is NaN then NaN will be returned.
9360 NaNBehavior = SPNB_RETURNS_NAN;
9361 else if (RHSSafe)
9362 NaNBehavior = SPNB_RETURNS_OTHER;
9363 else
9364 // Completely unsafe.
9365 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9366 } else {
9367 Ordered = false;
9368 // An unordered comparison will return true when given a NaN, so it
9369 // returns the LHS.
9370 if (LHSSafe)
9371 // LHS is non-NaN, so if RHS is NaN then non-NaN will be returned.
9372 NaNBehavior = SPNB_RETURNS_OTHER;
9373 else if (RHSSafe)
9374 NaNBehavior = SPNB_RETURNS_NAN;
9375 else
9376 // Completely unsafe.
9377 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9378 }
9379 }
9380
9381 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
9382 std::swap(a&: CmpLHS, b&: CmpRHS);
9383 Pred = CmpInst::getSwappedPredicate(pred: Pred);
9384 if (NaNBehavior == SPNB_RETURNS_NAN)
9385 NaNBehavior = SPNB_RETURNS_OTHER;
9386 else if (NaNBehavior == SPNB_RETURNS_OTHER)
9387 NaNBehavior = SPNB_RETURNS_NAN;
9388 Ordered = !Ordered;
9389 }
9390
9391 // ([if]cmp X, Y) ? X : Y
9392 if (TrueVal == CmpLHS && FalseVal == CmpRHS)
9393 return getSelectPattern(Pred, NaNBehavior, Ordered);
9394
9395 if (isKnownNegation(X: TrueVal, Y: FalseVal)) {
9396 // Sign-extending LHS does not change its sign, so TrueVal/FalseVal can
9397 // match against either LHS or sign-preserving operations on LHS, like
9398 // sext(LHS), or binary ops that do not wrap in signed sense.
9399 auto CmpLHSOrSExt =
9400 m_CombineOr(Ps: m_Specific(V: CmpLHS), Ps: m_SExt(Op: m_Specific(V: CmpLHS)));
9401 auto MaybeSExtOrMulCmpLHS =
9402 m_CombineOr(Ps: CmpLHSOrSExt, Ps: m_NSWMul(L: CmpLHSOrSExt, R: m_StrictlyPositive()),
9403 Ps: m_NSWShl(L: CmpLHSOrSExt, R: m_Value()));
9404 auto ZeroOrAllOnes = m_CombineOr(Ps: m_ZeroInt(), Ps: m_AllOnes());
9405 auto ZeroOrOne = m_CombineOr(Ps: m_ZeroInt(), Ps: m_One());
9406 if (match(V: TrueVal, P: MaybeSExtOrMulCmpLHS)) {
9407 // Set the return values. If the compare uses the negated value (-X >s 0),
9408 // swap the return values because the negated value is always 'RHS'.
9409 LHS = TrueVal;
9410 RHS = FalseVal;
9411 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: FalseVal))))
9412 std::swap(a&: LHS, b&: RHS);
9413
9414 // (X >s 0) ? X : -X or (X >s -1) ? X : -X --> ABS(X)
9415 // (-X >s 0) ? -X : X or (-X >s -1) ? -X : X --> ABS(X)
9416 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9417 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9418
9419 // (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X)
9420 if (Pred == ICmpInst::ICMP_SGE && match(V: CmpRHS, P: ZeroOrOne))
9421 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9422
9423 // (X <s 0) ? X : -X or (X <s 1) ? X : -X --> NABS(X)
9424 // (-X <s 0) ? -X : X or (-X <s 1) ? -X : X --> NABS(X)
9425 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9426 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9427 } else if (match(V: FalseVal, P: MaybeSExtOrMulCmpLHS)) {
9428 // Set the return values. If the compare uses the negated value (-X >s 0),
9429 // swap the return values because the negated value is always 'RHS'.
9430 LHS = FalseVal;
9431 RHS = TrueVal;
9432 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: TrueVal))))
9433 std::swap(a&: LHS, b&: RHS);
9434
9435 // (X >s 0) ? -X : X or (X >s -1) ? -X : X --> NABS(X)
9436 // (-X >s 0) ? X : -X or (-X >s -1) ? X : -X --> NABS(X)
9437 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9438 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9439
9440 // (X <s 0) ? -X : X or (X <s 1) ? -X : X --> ABS(X)
9441 // (-X <s 0) ? X : -X or (-X <s 1) ? X : -X --> ABS(X)
9442 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9443 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9444 }
9445 }
9446
9447 if (CmpInst::isIntPredicate(P: Pred))
9448 return matchMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS, Depth);
9449
9450 // According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar
9451 // may return either -0.0 or 0.0, so fcmp/select pair has stricter
9452 // semantics than minNum. Be conservative in such case.
9453 if (NaNBehavior != SPNB_RETURNS_ANY ||
9454 (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9455 !isKnownNonZero(V: CmpRHS)))
9456 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9457
9458 return matchFastFloatClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
9459}
9460
9461static Value *lookThroughCastConst(CmpInst *CmpI, Type *SrcTy, Constant *C,
9462 Instruction::CastOps *CastOp) {
9463 const DataLayout &DL = CmpI->getDataLayout();
9464
9465 Constant *CastedTo = nullptr;
9466 switch (*CastOp) {
9467 case Instruction::ZExt:
9468 if (CmpI->isUnsigned())
9469 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy);
9470 break;
9471 case Instruction::SExt:
9472 if (CmpI->isSigned())
9473 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy, OnlyIfReduced: true);
9474 break;
9475 case Instruction::Trunc:
9476 Constant *CmpConst;
9477 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_Constant(C&: CmpConst)) &&
9478 CmpConst->getType() == SrcTy) {
9479 // Here we have the following case:
9480 //
9481 // %cond = cmp iN %x, CmpConst
9482 // %tr = trunc iN %x to iK
9483 // %narrowsel = select i1 %cond, iK %t, iK C
9484 //
9485 // We can always move trunc after select operation:
9486 //
9487 // %cond = cmp iN %x, CmpConst
9488 // %widesel = select i1 %cond, iN %x, iN CmpConst
9489 // %tr = trunc iN %widesel to iK
9490 //
9491 // Note that C could be extended in any way because we don't care about
9492 // upper bits after truncation. It can't be abs pattern, because it would
9493 // look like:
9494 //
9495 // select i1 %cond, x, -x.
9496 //
9497 // So only min/max pattern could be matched. Such match requires widened C
9498 // == CmpConst. That is why set widened C = CmpConst, condition trunc
9499 // CmpConst == C is checked below.
9500 CastedTo = CmpConst;
9501 } else {
9502 unsigned ExtOp = CmpI->isSigned() ? Instruction::SExt : Instruction::ZExt;
9503 CastedTo = ConstantFoldCastOperand(Opcode: ExtOp, C, DestTy: SrcTy, DL);
9504 }
9505 break;
9506 case Instruction::FPTrunc:
9507 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPExt, C, DestTy: SrcTy, DL);
9508 break;
9509 case Instruction::FPExt:
9510 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPTrunc, C, DestTy: SrcTy, DL);
9511 break;
9512 case Instruction::FPToUI:
9513 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::UIToFP, C, DestTy: SrcTy, DL);
9514 break;
9515 case Instruction::FPToSI:
9516 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::SIToFP, C, DestTy: SrcTy, DL);
9517 break;
9518 case Instruction::UIToFP:
9519 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToUI, C, DestTy: SrcTy, DL);
9520 break;
9521 case Instruction::SIToFP:
9522 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToSI, C, DestTy: SrcTy, DL);
9523 break;
9524 default:
9525 break;
9526 }
9527
9528 if (!CastedTo)
9529 return nullptr;
9530
9531 // Make sure the cast doesn't lose any information.
9532 Constant *CastedBack =
9533 ConstantFoldCastOperand(Opcode: *CastOp, C: CastedTo, DestTy: C->getType(), DL);
9534 if (CastedBack && CastedBack != C)
9535 return nullptr;
9536
9537 return CastedTo;
9538}
9539
9540/// Helps to match a select pattern in case of a type mismatch.
9541///
9542/// The function processes the case when type of true and false values of a
9543/// select instruction differs from type of the cmp instruction operands because
9544/// of a cast instruction. The function checks if it is legal to move the cast
9545/// operation after "select". If yes, it returns the new second value of
9546/// "select" (with the assumption that cast is moved):
9547/// 1. As operand of cast instruction when both values of "select" are same cast
9548/// instructions.
9549/// 2. As restored constant (by applying reverse cast operation) when the first
9550/// value of the "select" is a cast operation and the second value is a
9551/// constant. It is implemented in lookThroughCastConst().
9552/// 3. As one operand is cast instruction and the other is not. The operands in
9553/// sel(cmp) are in different type integer.
9554/// NOTE: We return only the new second value because the first value could be
9555/// accessed as operand of cast instruction.
9556static Value *lookThroughCast(CmpInst *CmpI, Value *V1, Value *V2,
9557 Instruction::CastOps *CastOp) {
9558 auto *Cast1 = dyn_cast<CastInst>(Val: V1);
9559 if (!Cast1)
9560 return nullptr;
9561
9562 *CastOp = Cast1->getOpcode();
9563 Type *SrcTy = Cast1->getSrcTy();
9564 if (auto *Cast2 = dyn_cast<CastInst>(Val: V2)) {
9565 // If V1 and V2 are both the same cast from the same type, look through V1.
9566 if (*CastOp == Cast2->getOpcode() && SrcTy == Cast2->getSrcTy())
9567 return Cast2->getOperand(i_nocapture: 0);
9568 return nullptr;
9569 }
9570
9571 auto *C = dyn_cast<Constant>(Val: V2);
9572 if (C)
9573 return lookThroughCastConst(CmpI, SrcTy, C, CastOp);
9574
9575 Value *CastedTo = nullptr;
9576 if (*CastOp == Instruction::Trunc) {
9577 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_ZExtOrSExt(Op: m_Specific(V: V2)))) {
9578 // Here we have the following case:
9579 // %y_ext = sext iK %y to iN
9580 // %cond = cmp iN %x, %y_ext
9581 // %tr = trunc iN %x to iK
9582 // %narrowsel = select i1 %cond, iK %tr, iK %y
9583 //
9584 // We can always move trunc after select operation:
9585 // %y_ext = sext iK %y to iN
9586 // %cond = cmp iN %x, %y_ext
9587 // %widesel = select i1 %cond, iN %x, iN %y_ext
9588 // %tr = trunc iN %widesel to iK
9589 assert(V2->getType() == Cast1->getType() &&
9590 "V2 and Cast1 should be the same type.");
9591 CastedTo = CmpI->getOperand(i_nocapture: 1);
9592 }
9593 }
9594
9595 return CastedTo;
9596}
9597SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
9598 Instruction::CastOps *CastOp,
9599 unsigned Depth) {
9600 if (Depth >= MaxAnalysisRecursionDepth)
9601 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9602
9603 SelectInst *SI = dyn_cast<SelectInst>(Val: V);
9604 if (!SI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9605
9606 CmpInst *CmpI = dyn_cast<CmpInst>(Val: SI->getCondition());
9607 if (!CmpI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9608
9609 Value *TrueVal = SI->getTrueValue();
9610 Value *FalseVal = SI->getFalseValue();
9611
9612 return llvm::matchDecomposedSelectPattern(CmpI, TrueVal, FalseVal, LHS, RHS,
9613 FMF: SI->getFastMathFlagsOrNone(),
9614 CastOp, Depth);
9615}
9616
9617SelectPatternResult llvm::matchDecomposedSelectPattern(
9618 CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
9619 FastMathFlags FMF, Instruction::CastOps *CastOp, unsigned Depth) {
9620 CmpInst::Predicate Pred = CmpI->getPredicate();
9621 Value *CmpLHS = CmpI->getOperand(i_nocapture: 0);
9622 Value *CmpRHS = CmpI->getOperand(i_nocapture: 1);
9623 if (isa<FPMathOperator>(Val: CmpI) && CmpI->hasNoNaNs())
9624 FMF.setNoNaNs();
9625
9626 // Bail out early.
9627 if (CmpI->isEquality())
9628 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9629
9630 // Deal with type mismatches.
9631 if (CastOp && CmpLHS->getType() != TrueVal->getType()) {
9632 if (Value *C = lookThroughCast(CmpI, V1: TrueVal, V2: FalseVal, CastOp)) {
9633 // If this is a potential fmin/fmax with a cast to integer, then ignore
9634 // -0.0 because there is no corresponding integer value.
9635 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9636 FMF.setNoSignedZeros();
9637 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9638 TrueVal: cast<CastInst>(Val: TrueVal)->getOperand(i_nocapture: 0), FalseVal: C,
9639 LHS, RHS, Depth);
9640 }
9641 if (Value *C = lookThroughCast(CmpI, V1: FalseVal, V2: TrueVal, CastOp)) {
9642 // If this is a potential fmin/fmax with a cast to integer, then ignore
9643 // -0.0 because there is no corresponding integer value.
9644 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9645 FMF.setNoSignedZeros();
9646 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9647 TrueVal: C, FalseVal: cast<CastInst>(Val: FalseVal)->getOperand(i_nocapture: 0),
9648 LHS, RHS, Depth);
9649 }
9650 }
9651 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal, FalseVal,
9652 LHS, RHS, Depth);
9653}
9654
9655CmpInst::Predicate llvm::getMinMaxPred(SelectPatternFlavor SPF, bool Ordered) {
9656 if (SPF == SPF_SMIN) return ICmpInst::ICMP_SLT;
9657 if (SPF == SPF_UMIN) return ICmpInst::ICMP_ULT;
9658 if (SPF == SPF_SMAX) return ICmpInst::ICMP_SGT;
9659 if (SPF == SPF_UMAX) return ICmpInst::ICMP_UGT;
9660 if (SPF == SPF_FMINNUM)
9661 return Ordered ? FCmpInst::FCMP_OLT : FCmpInst::FCMP_ULT;
9662 if (SPF == SPF_FMAXNUM)
9663 return Ordered ? FCmpInst::FCMP_OGT : FCmpInst::FCMP_UGT;
9664 llvm_unreachable("unhandled!");
9665}
9666
9667Intrinsic::ID llvm::getMinMaxIntrinsic(SelectPatternFlavor SPF) {
9668 switch (SPF) {
9669 case SelectPatternFlavor::SPF_UMIN:
9670 return Intrinsic::umin;
9671 case SelectPatternFlavor::SPF_UMAX:
9672 return Intrinsic::umax;
9673 case SelectPatternFlavor::SPF_SMIN:
9674 return Intrinsic::smin;
9675 case SelectPatternFlavor::SPF_SMAX:
9676 return Intrinsic::smax;
9677 default:
9678 llvm_unreachable("Unexpected SPF");
9679 }
9680}
9681
9682SelectPatternFlavor llvm::getInverseMinMaxFlavor(SelectPatternFlavor SPF) {
9683 if (SPF == SPF_SMIN) return SPF_SMAX;
9684 if (SPF == SPF_UMIN) return SPF_UMAX;
9685 if (SPF == SPF_SMAX) return SPF_SMIN;
9686 if (SPF == SPF_UMAX) return SPF_UMIN;
9687 llvm_unreachable("unhandled!");
9688}
9689
9690Intrinsic::ID llvm::getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID) {
9691 switch (MinMaxID) {
9692 case Intrinsic::smax: return Intrinsic::smin;
9693 case Intrinsic::smin: return Intrinsic::smax;
9694 case Intrinsic::umax: return Intrinsic::umin;
9695 case Intrinsic::umin: return Intrinsic::umax;
9696 // Please note that next four intrinsics may produce the same result for
9697 // original and inverted case even if X != Y due to NaN is handled specially.
9698 case Intrinsic::maximum: return Intrinsic::minimum;
9699 case Intrinsic::minimum: return Intrinsic::maximum;
9700 case Intrinsic::maxnum: return Intrinsic::minnum;
9701 case Intrinsic::minnum: return Intrinsic::maxnum;
9702 case Intrinsic::maximumnum:
9703 return Intrinsic::minimumnum;
9704 case Intrinsic::minimumnum:
9705 return Intrinsic::maximumnum;
9706 default: llvm_unreachable("Unexpected intrinsic");
9707 }
9708}
9709
9710APInt llvm::getMinMaxLimit(SelectPatternFlavor SPF, unsigned BitWidth) {
9711 switch (SPF) {
9712 case SPF_SMAX: return APInt::getSignedMaxValue(numBits: BitWidth);
9713 case SPF_SMIN: return APInt::getSignedMinValue(numBits: BitWidth);
9714 case SPF_UMAX: return APInt::getMaxValue(numBits: BitWidth);
9715 case SPF_UMIN: return APInt::getMinValue(numBits: BitWidth);
9716 default: llvm_unreachable("Unexpected flavor");
9717 }
9718}
9719
9720std::pair<Intrinsic::ID, bool>
9721llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
9722 // Check if VL contains select instructions that can be folded into a min/max
9723 // vector intrinsic and return the intrinsic if it is possible.
9724 // TODO: Support floating point min/max.
9725 bool AllCmpSingleUse = true;
9726 SelectPatternResult SelectPattern;
9727 SelectPattern.Flavor = SPF_UNKNOWN;
9728 if (all_of(Range&: VL, P: [&SelectPattern, &AllCmpSingleUse](Value *I) {
9729 Value *LHS, *RHS;
9730 auto CurrentPattern = matchSelectPattern(V: I, LHS, RHS);
9731 if (!SelectPatternResult::isMinOrMax(SPF: CurrentPattern.Flavor))
9732 return false;
9733 if (SelectPattern.Flavor != SPF_UNKNOWN &&
9734 SelectPattern.Flavor != CurrentPattern.Flavor)
9735 return false;
9736 SelectPattern = CurrentPattern;
9737 AllCmpSingleUse &=
9738 match(V: I, P: m_Select(C: m_OneUse(SubPattern: m_Value()), L: m_Value(), R: m_Value()));
9739 return true;
9740 })) {
9741 switch (SelectPattern.Flavor) {
9742 case SPF_SMIN:
9743 return {Intrinsic::smin, AllCmpSingleUse};
9744 case SPF_UMIN:
9745 return {Intrinsic::umin, AllCmpSingleUse};
9746 case SPF_SMAX:
9747 return {Intrinsic::smax, AllCmpSingleUse};
9748 case SPF_UMAX:
9749 return {Intrinsic::umax, AllCmpSingleUse};
9750 case SPF_FMAXNUM:
9751 return {Intrinsic::maxnum, AllCmpSingleUse};
9752 case SPF_FMINNUM:
9753 return {Intrinsic::minnum, AllCmpSingleUse};
9754 default:
9755 llvm_unreachable("unexpected select pattern flavor");
9756 }
9757 }
9758 return {Intrinsic::not_intrinsic, false};
9759}
9760
9761template <typename InstTy>
9762static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
9763 Value *&Init, Value *&OtherOp) {
9764 // Handle the case of a simple two-predecessor recurrence PHI.
9765 // There's a lot more that could theoretically be done here, but
9766 // this is sufficient to catch some interesting cases.
9767 // TODO: Expand list -- gep, uadd.sat etc.
9768 if (PN->getNumIncomingValues() != 2)
9769 return false;
9770
9771 for (unsigned I = 0; I != 2; ++I) {
9772 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9773 Operation && Operation->getNumOperands() >= 2) {
9774 Value *LHS = Operation->getOperand(0);
9775 Value *RHS = Operation->getOperand(1);
9776 if (LHS != PN && RHS != PN)
9777 continue;
9778
9779 Inst = Operation;
9780 Init = PN->getIncomingValue(i: !I);
9781 OtherOp = (LHS == PN) ? RHS : LHS;
9782 return true;
9783 }
9784 }
9785 return false;
9786}
9787
9788template <typename InstTy>
9789static bool matchThreeInputRecurrence(const PHINode *PN, InstTy *&Inst,
9790 Value *&Init, Value *&OtherOp0,
9791 Value *&OtherOp1) {
9792 if (PN->getNumIncomingValues() != 2)
9793 return false;
9794
9795 for (unsigned I = 0; I != 2; ++I) {
9796 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9797 Operation && Operation->getNumOperands() >= 3) {
9798 Value *Op0 = Operation->getOperand(0);
9799 Value *Op1 = Operation->getOperand(1);
9800 Value *Op2 = Operation->getOperand(2);
9801
9802 if (Op0 != PN && Op1 != PN && Op2 != PN)
9803 continue;
9804
9805 Inst = Operation;
9806 Init = PN->getIncomingValue(i: !I);
9807 if (Op0 == PN) {
9808 OtherOp0 = Op1;
9809 OtherOp1 = Op2;
9810 } else if (Op1 == PN) {
9811 OtherOp0 = Op0;
9812 OtherOp1 = Op2;
9813 } else {
9814 OtherOp0 = Op0;
9815 OtherOp1 = Op1;
9816 }
9817 return true;
9818 }
9819 }
9820 return false;
9821}
9822bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
9823 Value *&Start, Value *&Step) {
9824 // We try to match a recurrence of the form:
9825 // %iv = [Start, %entry], [%iv.next, %backedge]
9826 // %iv.next = binop %iv, Step
9827 // Or:
9828 // %iv = [Start, %entry], [%iv.next, %backedge]
9829 // %iv.next = binop Step, %iv
9830 return matchTwoInputRecurrence(PN: P, Inst&: BO, Init&: Start, OtherOp&: Step);
9831}
9832
9833bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
9834 Value *&Start, Value *&Step) {
9835 BinaryOperator *BO = nullptr;
9836 return match(V: I, P: m_c_BinOp(L: m_Phi(PN&: P), R: m_Value())) &&
9837 matchSimpleRecurrence(P, BO, Start, Step) && BO == I;
9838}
9839
9840bool llvm::matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I,
9841 PHINode *&P, Value *&Init,
9842 Value *&OtherOp) {
9843 // Binary intrinsics only supported for now.
9844 if (I->arg_size() != 2 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9845 I->getType() != I->getArgOperand(i: 1)->getType())
9846 return false;
9847
9848 IntrinsicInst *II = nullptr;
9849 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9850 if (!P)
9851 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9852
9853 return P && matchTwoInputRecurrence(PN: P, Inst&: II, Init, OtherOp) && II == I;
9854}
9855
9856bool llvm::matchSimpleTernaryIntrinsicRecurrence(const IntrinsicInst *I,
9857 PHINode *&P, Value *&Init,
9858 Value *&OtherOp0,
9859 Value *&OtherOp1) {
9860 if (I->arg_size() != 3 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9861 I->getType() != I->getArgOperand(i: 1)->getType() ||
9862 I->getType() != I->getArgOperand(i: 2)->getType())
9863 return false;
9864 IntrinsicInst *II = nullptr;
9865 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9866 if (!P) {
9867 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9868 if (!P)
9869 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 2));
9870 }
9871 return P && matchThreeInputRecurrence(PN: P, Inst&: II, Init, OtherOp0, OtherOp1) &&
9872 II == I;
9873}
9874
9875/// Return true if "icmp Pred LHS RHS" is always true.
9876static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
9877 const Value *RHS) {
9878 if (ICmpInst::isTrueWhenEqual(predicate: Pred) && LHS == RHS)
9879 return true;
9880
9881 switch (Pred) {
9882 default:
9883 return false;
9884
9885 case CmpInst::ICMP_SLE: {
9886 const APInt *C;
9887
9888 // LHS s<= LHS +_{nsw} C if C >= 0
9889 // LHS s<= LHS | C if C >= 0
9890 if (match(V: RHS, P: m_NSWAdd(L: m_Specific(V: LHS), R: m_APInt(Res&: C))) ||
9891 match(V: RHS, P: m_Or(L: m_Specific(V: LHS), R: m_APInt(Res&: C))))
9892 return !C->isNegative();
9893
9894 // LHS s<= smax(LHS, V) for any V
9895 if (match(V: RHS, P: m_c_SMax(L: m_Specific(V: LHS), R: m_Value())))
9896 return true;
9897
9898 // smin(RHS, V) s<= RHS for any V
9899 if (match(V: LHS, P: m_c_SMin(L: m_Specific(V: RHS), R: m_Value())))
9900 return true;
9901
9902 // Match A to (X +_{nsw} CA) and B to (X +_{nsw} CB)
9903 const Value *X;
9904 const APInt *CLHS, *CRHS;
9905 if (match(V: LHS, P: m_NSWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9906 match(V: RHS, P: m_NSWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9907 return CLHS->sle(RHS: *CRHS);
9908
9909 return false;
9910 }
9911
9912 case CmpInst::ICMP_ULE: {
9913 // LHS u<= LHS +_{nuw} V for any V
9914 if (match(V: RHS, P: m_c_Add(L: m_Specific(V: LHS), R: m_Value())) &&
9915 cast<OverflowingBinaryOperator>(Val: RHS)->hasNoUnsignedWrap())
9916 return true;
9917
9918 // LHS u<= LHS | V for any V
9919 if (match(V: RHS, P: m_c_Or(L: m_Specific(V: LHS), R: m_Value())))
9920 return true;
9921
9922 // LHS u<= umax(LHS, V) for any V
9923 if (match(V: RHS, P: m_c_UMax(L: m_Specific(V: LHS), R: m_Value())))
9924 return true;
9925
9926 // RHS >> V u<= RHS for any V
9927 if (match(V: LHS, P: m_LShr(L: m_Specific(V: RHS), R: m_Value())))
9928 return true;
9929
9930 // RHS u/ C_ugt_1 u<= RHS
9931 const APInt *C;
9932 if (match(V: LHS, P: m_UDiv(L: m_Specific(V: RHS), R: m_APInt(Res&: C))) && C->ugt(RHS: 1))
9933 return true;
9934
9935 // RHS & V u<= RHS for any V
9936 if (match(V: LHS, P: m_c_And(L: m_Specific(V: RHS), R: m_Value())))
9937 return true;
9938
9939 // umin(RHS, V) u<= RHS for any V
9940 if (match(V: LHS, P: m_c_UMin(L: m_Specific(V: RHS), R: m_Value())))
9941 return true;
9942
9943 // Match A to (X +_{nuw} CA) and B to (X +_{nuw} CB)
9944 const Value *X;
9945 const APInt *CLHS, *CRHS;
9946 if (match(V: LHS, P: m_NUWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9947 match(V: RHS, P: m_NUWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9948 return CLHS->ule(RHS: *CRHS);
9949
9950 return false;
9951 }
9952 }
9953}
9954
9955/// Return true if "icmp Pred BLHS BRHS" is true whenever "icmp Pred
9956/// ALHS ARHS" is true. Otherwise, return std::nullopt.
9957static std::optional<bool>
9958isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS,
9959 const Value *ARHS, const Value *BLHS, const Value *BRHS) {
9960 switch (Pred) {
9961 default:
9962 return std::nullopt;
9963
9964 case CmpInst::ICMP_SLT:
9965 case CmpInst::ICMP_SLE:
9966 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BLHS, RHS: ALHS) &&
9967 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ARHS, RHS: BRHS))
9968 return true;
9969 return std::nullopt;
9970
9971 case CmpInst::ICMP_SGT:
9972 case CmpInst::ICMP_SGE:
9973 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ALHS, RHS: BLHS) &&
9974 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BRHS, RHS: ARHS))
9975 return true;
9976 return std::nullopt;
9977
9978 case CmpInst::ICMP_ULT:
9979 case CmpInst::ICMP_ULE:
9980 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BLHS, RHS: ALHS) &&
9981 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ARHS, RHS: BRHS))
9982 return true;
9983 return std::nullopt;
9984
9985 case CmpInst::ICMP_UGT:
9986 case CmpInst::ICMP_UGE:
9987 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ALHS, RHS: BLHS) &&
9988 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BRHS, RHS: ARHS))
9989 return true;
9990 return std::nullopt;
9991 }
9992}
9993
9994/// Return true if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is true.
9995/// Return false if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is false.
9996/// Otherwise, return std::nullopt if we can't infer anything.
9997static std::optional<bool>
9998isImpliedCondCommonOperandWithCR(CmpPredicate LPred, const ConstantRange &LCR,
9999 CmpPredicate RPred, const ConstantRange &RCR) {
10000 auto CRImpliesPred = [&](ConstantRange CR,
10001 CmpInst::Predicate Pred) -> std::optional<bool> {
10002 // If all true values for lhs and true for rhs, lhs implies rhs
10003 if (CR.icmp(Pred, Other: RCR))
10004 return true;
10005
10006 // If there is no overlap, lhs implies not rhs
10007 if (CR.icmp(Pred: CmpInst::getInversePredicate(pred: Pred), Other: RCR))
10008 return false;
10009
10010 return std::nullopt;
10011 };
10012 if (auto Res = CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
10013 RPred))
10014 return Res;
10015 if (LPred.hasSameSign() ^ RPred.hasSameSign()) {
10016 LPred = LPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: LPred)
10017 : LPred.dropSameSign();
10018 RPred = RPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: RPred)
10019 : RPred.dropSameSign();
10020 return CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
10021 RPred);
10022 }
10023 return std::nullopt;
10024}
10025
10026/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
10027/// is true. Return false if LHS implies RHS is false. Otherwise, return
10028/// std::nullopt if we can't infer anything.
10029static std::optional<bool>
10030isImpliedCondICmps(CmpPredicate LPred, const Value *L0, const Value *L1,
10031 CmpPredicate RPred, const Value *R0, const Value *R1,
10032 const DataLayout &DL, bool LHSIsTrue) {
10033 // The rest of the logic assumes the LHS condition is true. If that's not the
10034 // case, invert the predicate to make it so.
10035 if (!LHSIsTrue)
10036 LPred = ICmpInst::getInverseCmpPredicate(Pred: LPred);
10037
10038 // We can have non-canonical operands, so try to normalize any common operand
10039 // to L0/R0.
10040 if (L0 == R1) {
10041 std::swap(a&: R0, b&: R1);
10042 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10043 }
10044 if (R0 == L1) {
10045 std::swap(a&: L0, b&: L1);
10046 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
10047 }
10048 if (L1 == R1) {
10049 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
10050 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
10051 std::swap(a&: L0, b&: L1);
10052 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
10053 std::swap(a&: R0, b&: R1);
10054 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10055 }
10056 }
10057
10058 // See if we can infer anything if operand-0 matches and we have at least one
10059 // constant.
10060 const APInt *Unused;
10061 if (L0 == R0 && (match(V: L1, P: m_APInt(Res&: Unused)) || match(V: R1, P: m_APInt(Res&: Unused)))) {
10062 // Potential TODO: We could also further use the constant range of L0/R0 to
10063 // further constraint the constant ranges. At the moment this leads to
10064 // several regressions related to not transforming `multi_use(A + C0) eq/ne
10065 // C1` (see discussion: D58633).
10066 SimplifyQuery SQ(DL);
10067 ConstantRange LCR = computeConstantRange(V: L1, ForSigned: ICmpInst::isSigned(Pred: LPred), SQ,
10068 Depth: MaxAnalysisRecursionDepth - 1);
10069 ConstantRange RCR = computeConstantRange(V: R1, ForSigned: ICmpInst::isSigned(Pred: RPred), SQ,
10070 Depth: MaxAnalysisRecursionDepth - 1);
10071
10072 // Even if L1/R1 are not both constant, we can still sometimes deduce
10073 // relationship from a single constant. For example X u> Y implies X != 0.
10074 if (auto R = isImpliedCondCommonOperandWithCR(LPred, LCR, RPred, RCR))
10075 return R;
10076 // If both L1/R1 were exact constant ranges and we didn't get anything
10077 // here, we won't be able to deduce this.
10078 if (match(V: L1, P: m_APInt(Res&: Unused)) && match(V: R1, P: m_APInt(Res&: Unused)))
10079 return std::nullopt;
10080 }
10081
10082 // Can we infer anything when the two compares have matching operands?
10083 if (L0 == R0 && L1 == R1)
10084 return ICmpInst::isImpliedByMatchingCmp(Pred1: LPred, Pred2: RPred);
10085
10086 // It only really makes sense in the context of signed comparison for "X - Y
10087 // must be positive if X >= Y and no overflow".
10088 // Take SGT as an example: L0:x > L1:y and C >= 0
10089 // ==> R0:(x -nsw y) < R1:(-C) is false
10090 CmpInst::Predicate SignedLPred = LPred.getPreferredSignedPredicate();
10091 if ((SignedLPred == ICmpInst::ICMP_SGT ||
10092 SignedLPred == ICmpInst::ICMP_SGE) &&
10093 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
10094 if (match(V: R1, P: m_NonPositive()) &&
10095 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == false)
10096 return false;
10097 }
10098
10099 // Take SLT as an example: L0:x < L1:y and C <= 0
10100 // ==> R0:(x -nsw y) < R1:(-C) is true
10101 if ((SignedLPred == ICmpInst::ICMP_SLT ||
10102 SignedLPred == ICmpInst::ICMP_SLE) &&
10103 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
10104 if (match(V: R1, P: m_NonNegative()) &&
10105 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == true)
10106 return true;
10107 }
10108
10109 // a - b == NonZero -> a != b
10110 // ptrtoint(a) - ptrtoint(b) == NonZero -> a != b
10111 const APInt *L1C;
10112 Value *A, *B;
10113 if (LPred == ICmpInst::ICMP_EQ && ICmpInst::isEquality(P: RPred) &&
10114 match(V: L1, P: m_APInt(Res&: L1C)) && !L1C->isZero() &&
10115 match(V: L0, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
10116 ((A == R0 && B == R1) || (A == R1 && B == R0) ||
10117 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0))) &&
10118 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1)))) ||
10119 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1))) &&
10120 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0)))))) {
10121 return RPred.dropSameSign() == ICmpInst::ICMP_NE;
10122 }
10123
10124 // L0 = R0 = L1 + R1, L0 >=u L1 implies R0 >=u R1, L0 <u L1 implies R0 <u R1
10125 if (L0 == R0 &&
10126 (LPred == ICmpInst::ICMP_ULT || LPred == ICmpInst::ICMP_UGE) &&
10127 (RPred == ICmpInst::ICMP_ULT || RPred == ICmpInst::ICMP_UGE) &&
10128 match(V: L0, P: m_c_Add(L: m_Specific(V: L1), R: m_Specific(V: R1))))
10129 return CmpPredicate::getMatching(A: LPred, B: RPred).has_value();
10130
10131 if (auto P = CmpPredicate::getMatching(A: LPred, B: RPred))
10132 return isImpliedCondOperands(Pred: *P, ALHS: L0, ARHS: L1, BLHS: R0, BRHS: R1);
10133
10134 // L0 u< C sets limits to L0's bits which may imply (L0 & Mask) pred RC
10135 // Example: L0 u< 13 => (L0 & 16) == 0
10136 const APInt *LC, *RC, *MaskC;
10137 if (match(V: L1, P: m_APInt(Res&: LC)) && match(V: R1, P: m_APInt(Res&: RC)) &&
10138 match(V: R0, P: m_And(L: m_Specific(V: L0), R: m_APInt(Res&: MaskC)))) {
10139 ConstantRange LCRange = ConstantRange::makeExactICmpRegion(Pred: LPred, Other: *LC);
10140 ConstantRange MaskedCRange = LCRange.binaryAnd(Other: *MaskC);
10141 if (MaskedCRange.icmp(Pred: RPred, Other: ConstantRange(*RC)))
10142 return true;
10143 if (MaskedCRange.icmp(Pred: ICmpInst::getInversePredicate(pred: RPred),
10144 Other: ConstantRange(*RC)))
10145 return false;
10146 }
10147
10148 return std::nullopt;
10149}
10150
10151/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
10152/// is true. Return false if LHS implies RHS is false. Otherwise, return
10153/// std::nullopt if we can't infer anything.
10154static std::optional<bool>
10155isImpliedCondFCmps(FCmpInst::Predicate LPred, const Value *L0, const Value *L1,
10156 FCmpInst::Predicate RPred, const Value *R0, const Value *R1,
10157 const DataLayout &DL, bool LHSIsTrue) {
10158 // The rest of the logic assumes the LHS condition is true. If that's not the
10159 // case, invert the predicate to make it so.
10160 if (!LHSIsTrue)
10161 LPred = FCmpInst::getInversePredicate(pred: LPred);
10162
10163 // We can have non-canonical operands, so try to normalize any common operand
10164 // to L0/R0.
10165 if (L0 == R1) {
10166 std::swap(a&: R0, b&: R1);
10167 RPred = FCmpInst::getSwappedPredicate(pred: RPred);
10168 }
10169 if (R0 == L1) {
10170 std::swap(a&: L0, b&: L1);
10171 LPred = FCmpInst::getSwappedPredicate(pred: LPred);
10172 }
10173 if (L1 == R1) {
10174 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
10175 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
10176 std::swap(a&: L0, b&: L1);
10177 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
10178 std::swap(a&: R0, b&: R1);
10179 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10180 }
10181 }
10182
10183 // Can we infer anything when the two compares have matching operands?
10184 if (L0 == R0 && L1 == R1) {
10185 if ((LPred & RPred) == LPred)
10186 return true;
10187 if ((LPred & ~RPred) == LPred)
10188 return false;
10189 }
10190
10191 // See if we can infer anything if operand-0 matches and we have at least one
10192 // constant.
10193 const APFloat *L1C, *R1C;
10194 if (L0 == R0 && match(V: L1, P: m_APFloat(Res&: L1C)) && match(V: R1, P: m_APFloat(Res&: R1C))) {
10195 if (std::optional<ConstantFPRange> DomCR =
10196 ConstantFPRange::makeExactFCmpRegion(Pred: LPred, Other: *L1C)) {
10197 if (std::optional<ConstantFPRange> ImpliedCR =
10198 ConstantFPRange::makeExactFCmpRegion(Pred: RPred, Other: *R1C)) {
10199 if (ImpliedCR->contains(CR: *DomCR))
10200 return true;
10201 }
10202 if (std::optional<ConstantFPRange> ImpliedCR =
10203 ConstantFPRange::makeExactFCmpRegion(
10204 Pred: FCmpInst::getInversePredicate(pred: RPred), Other: *R1C)) {
10205 if (ImpliedCR->contains(CR: *DomCR))
10206 return false;
10207 }
10208 }
10209 }
10210
10211 return std::nullopt;
10212}
10213
10214/// Return true if LHS implies RHS is true. Return false if LHS implies RHS is
10215/// false. Otherwise, return std::nullopt if we can't infer anything. We
10216/// expect the RHS to be an icmp and the LHS to be an 'and', 'or', or a 'select'
10217/// instruction.
10218static std::optional<bool>
10219isImpliedCondAndOr(const Instruction *LHS, CmpPredicate RHSPred,
10220 const Value *RHSOp0, const Value *RHSOp1,
10221 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
10222 // The LHS must be an 'or', 'and', or a 'select' instruction.
10223 assert((LHS->getOpcode() == Instruction::And ||
10224 LHS->getOpcode() == Instruction::Or ||
10225 LHS->getOpcode() == Instruction::Select) &&
10226 "Expected LHS to be 'and', 'or', or 'select'.");
10227
10228 assert(Depth <= MaxAnalysisRecursionDepth && "Hit recursion limit");
10229
10230 // If the result of an 'or' is false, then we know both legs of the 'or' are
10231 // false. Similarly, if the result of an 'and' is true, then we know both
10232 // legs of the 'and' are true.
10233 const Value *ALHS, *ARHS;
10234 if ((!LHSIsTrue && match(V: LHS, P: m_LogicalOr(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS)))) ||
10235 (LHSIsTrue && match(V: LHS, P: m_LogicalAnd(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS))))) {
10236 // FIXME: Make this non-recursion.
10237 if (std::optional<bool> Implication = isImpliedCondition(
10238 LHS: ALHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
10239 return Implication;
10240 if (std::optional<bool> Implication = isImpliedCondition(
10241 LHS: ARHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
10242 return Implication;
10243 return std::nullopt;
10244 }
10245 return std::nullopt;
10246}
10247
10248std::optional<bool>
10249llvm::isImpliedCondition(const Value *LHS, CmpPredicate RHSPred,
10250 const Value *RHSOp0, const Value *RHSOp1,
10251 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
10252 // Bail out when we hit the limit.
10253 if (Depth == MaxAnalysisRecursionDepth)
10254 return std::nullopt;
10255
10256 // A mismatch occurs when we compare a scalar cmp to a vector cmp, for
10257 // example.
10258 if (RHSOp0->getType()->isVectorTy() != LHS->getType()->isVectorTy())
10259 return std::nullopt;
10260
10261 assert(LHS->getType()->isIntOrIntVectorTy(1) &&
10262 "Expected integer type only!");
10263
10264 // Match not
10265 if (match(V: LHS, P: m_Not(V: m_Value(V&: LHS))))
10266 LHSIsTrue = !LHSIsTrue;
10267
10268 // Both LHS and RHS are icmps.
10269 if (RHSOp0->getType()->getScalarType()->isIntOrPtrTy()) {
10270 CmpPredicate LHSPred;
10271 Value *LHSOp0, *LHSOp1;
10272 if (match(V: LHS, P: m_ICmpLike(Pred&: LHSPred, L: m_Value(V&: LHSOp0), R: m_Value(V&: LHSOp1))))
10273 return isImpliedCondICmps(LPred: LHSPred, L0: LHSOp0, L1: LHSOp1, RPred: RHSPred, R0: RHSOp0,
10274 R1: RHSOp1, DL, LHSIsTrue);
10275 } else {
10276 assert(RHSOp0->getType()->isFPOrFPVectorTy() &&
10277 "Expected floating point type only!");
10278 if (const auto *LHSCmp = dyn_cast<FCmpInst>(Val: LHS))
10279 return isImpliedCondFCmps(LPred: LHSCmp->getPredicate(), L0: LHSCmp->getOperand(i_nocapture: 0),
10280 L1: LHSCmp->getOperand(i_nocapture: 1), RPred: RHSPred, R0: RHSOp0, R1: RHSOp1,
10281 DL, LHSIsTrue);
10282 }
10283
10284 /// The LHS should be an 'or', 'and', or a 'select' instruction. We expect
10285 /// the RHS to be an icmp.
10286 /// FIXME: Add support for and/or/select on the RHS.
10287 if (const Instruction *LHSI = dyn_cast<Instruction>(Val: LHS)) {
10288 if ((LHSI->getOpcode() == Instruction::And ||
10289 LHSI->getOpcode() == Instruction::Or ||
10290 LHSI->getOpcode() == Instruction::Select))
10291 return isImpliedCondAndOr(LHS: LHSI, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue,
10292 Depth);
10293 }
10294 return std::nullopt;
10295}
10296
10297std::optional<bool> llvm::isImpliedCondition(const Value *LHS, const Value *RHS,
10298 const DataLayout &DL,
10299 bool LHSIsTrue, unsigned Depth) {
10300 // LHS ==> RHS by definition
10301 if (LHS == RHS)
10302 return LHSIsTrue;
10303
10304 // Match not
10305 bool InvertRHS = false;
10306 if (match(V: RHS, P: m_Not(V: m_Value(V&: RHS)))) {
10307 if (LHS == RHS)
10308 return !LHSIsTrue;
10309 InvertRHS = true;
10310 }
10311
10312 CmpPredicate RHSPred;
10313 Value *RHSOp0, *RHSOp1;
10314 if (match(V: RHS, P: m_ICmpLike(Pred&: RHSPred, L: m_Value(V&: RHSOp0), R: m_Value(V&: RHSOp1)))) {
10315 if (auto Implied = isImpliedCondition(LHS, RHSPred, RHSOp0, RHSOp1, DL,
10316 LHSIsTrue, Depth))
10317 return InvertRHS ? !*Implied : *Implied;
10318 return std::nullopt;
10319 }
10320 if (const FCmpInst *RHSCmp = dyn_cast<FCmpInst>(Val: RHS)) {
10321 if (auto Implied = isImpliedCondition(
10322 LHS, RHSPred: RHSCmp->getPredicate(), RHSOp0: RHSCmp->getOperand(i_nocapture: 0),
10323 RHSOp1: RHSCmp->getOperand(i_nocapture: 1), DL, LHSIsTrue, Depth))
10324 return InvertRHS ? !*Implied : *Implied;
10325 return std::nullopt;
10326 }
10327
10328 if (Depth == MaxAnalysisRecursionDepth)
10329 return std::nullopt;
10330
10331 // LHS ==> (RHS1 || RHS2) if LHS ==> RHS1 or LHS ==> RHS2
10332 // LHS ==> !(RHS1 && RHS2) if LHS ==> !RHS1 or LHS ==> !RHS2
10333 const Value *RHS1, *RHS2;
10334 if (match(V: RHS, P: m_LogicalOr(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
10335 if (std::optional<bool> Imp =
10336 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
10337 if (*Imp == true)
10338 return !InvertRHS;
10339 if (std::optional<bool> Imp =
10340 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
10341 if (*Imp == true)
10342 return !InvertRHS;
10343 }
10344 if (match(V: RHS, P: m_LogicalAnd(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
10345 if (std::optional<bool> Imp =
10346 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
10347 if (*Imp == false)
10348 return InvertRHS;
10349 if (std::optional<bool> Imp =
10350 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
10351 if (*Imp == false)
10352 return InvertRHS;
10353 }
10354
10355 return std::nullopt;
10356}
10357
10358// Returns a pair (Condition, ConditionIsTrue), where Condition is a branch
10359// condition dominating ContextI or nullptr, if no condition is found.
10360static std::pair<Value *, bool>
10361getDomPredecessorCondition(const Instruction *ContextI) {
10362 if (!ContextI || !ContextI->getParent())
10363 return {nullptr, false};
10364
10365 // TODO: This is a poor/cheap way to determine dominance. Should we use a
10366 // dominator tree (eg, from a SimplifyQuery) instead?
10367 const BasicBlock *ContextBB = ContextI->getParent();
10368 const BasicBlock *PredBB = ContextBB->getSinglePredecessor();
10369 if (!PredBB)
10370 return {nullptr, false};
10371
10372 // We need a conditional branch in the predecessor.
10373 Value *PredCond;
10374 BasicBlock *TrueBB, *FalseBB;
10375 if (!match(V: PredBB->getTerminator(), P: m_Br(C: m_Value(V&: PredCond), T&: TrueBB, F&: FalseBB)))
10376 return {nullptr, false};
10377
10378 // The branch should get simplified. Don't bother simplifying this condition.
10379 if (TrueBB == FalseBB)
10380 return {nullptr, false};
10381
10382 assert((TrueBB == ContextBB || FalseBB == ContextBB) &&
10383 "Predecessor block does not point to successor?");
10384
10385 // Is this condition implied by the predecessor condition?
10386 return {PredCond, TrueBB == ContextBB};
10387}
10388
10389std::optional<bool> llvm::isImpliedByDomCondition(const Value *Cond,
10390 const Instruction *ContextI,
10391 const DataLayout &DL) {
10392 assert(Cond->getType()->isIntOrIntVectorTy(1) && "Condition must be bool");
10393 auto PredCond = getDomPredecessorCondition(ContextI);
10394 if (PredCond.first)
10395 return isImpliedCondition(LHS: PredCond.first, RHS: Cond, DL, LHSIsTrue: PredCond.second);
10396 return std::nullopt;
10397}
10398
10399std::optional<bool> llvm::isImpliedByDomCondition(CmpPredicate Pred,
10400 const Value *LHS,
10401 const Value *RHS,
10402 const Instruction *ContextI,
10403 const DataLayout &DL) {
10404 auto PredCond = getDomPredecessorCondition(ContextI);
10405 if (PredCond.first)
10406 return isImpliedCondition(LHS: PredCond.first, RHSPred: Pred, RHSOp0: LHS, RHSOp1: RHS, DL,
10407 LHSIsTrue: PredCond.second);
10408 return std::nullopt;
10409}
10410
10411static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
10412 APInt &Upper, const InstrInfoQuery &IIQ,
10413 bool PreferSignedRange) {
10414 unsigned Width = Lower.getBitWidth();
10415 const APInt *C;
10416 switch (BO.getOpcode()) {
10417 case Instruction::Sub:
10418 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10419 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10420 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10421
10422 // If the caller expects a signed compare, then try to use a signed range.
10423 // Otherwise if both no-wraps are set, use the unsigned range because it
10424 // is never larger than the signed range. Example:
10425 // "sub nuw nsw i8 -2, x" is unsigned [0, 254] vs. signed [-128, 126].
10426 // "sub nuw nsw i8 2, x" is unsigned [0, 2] vs. signed [-125, 127].
10427 if (PreferSignedRange && HasNSW && HasNUW)
10428 HasNUW = false;
10429
10430 if (HasNUW) {
10431 // 'sub nuw c, x' produces [0, C].
10432 Upper = *C + 1;
10433 } else if (HasNSW) {
10434 if (C->isNegative()) {
10435 // 'sub nsw -C, x' produces [SINT_MIN, -C - SINT_MIN].
10436 Lower = APInt::getSignedMinValue(numBits: Width);
10437 Upper = *C - APInt::getSignedMaxValue(numBits: Width);
10438 } else {
10439 // Note that sub 0, INT_MIN is not NSW. It techically is a signed wrap
10440 // 'sub nsw C, x' produces [C - SINT_MAX, SINT_MAX].
10441 Lower = *C - APInt::getSignedMaxValue(numBits: Width);
10442 Upper = APInt::getSignedMinValue(numBits: Width);
10443 }
10444 }
10445 }
10446 break;
10447 case Instruction::Add:
10448 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10449 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10450 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10451
10452 // If the caller expects a signed compare, then try to use a signed
10453 // range. Otherwise if both no-wraps are set, use the unsigned range
10454 // because it is never larger than the signed range. Example: "add nuw
10455 // nsw i8 X, -2" is unsigned [254,255] vs. signed [-128, 125].
10456 if (PreferSignedRange && HasNSW && HasNUW)
10457 HasNUW = false;
10458
10459 if (HasNUW) {
10460 // 'add nuw x, C' produces [C, UINT_MAX].
10461 Lower = *C;
10462 } else if (HasNSW) {
10463 if (C->isNegative()) {
10464 // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C].
10465 Lower = APInt::getSignedMinValue(numBits: Width);
10466 Upper = APInt::getSignedMaxValue(numBits: Width) + *C + 1;
10467 } else {
10468 // 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX].
10469 Lower = APInt::getSignedMinValue(numBits: Width) + *C;
10470 Upper = APInt::getSignedMaxValue(numBits: Width) + 1;
10471 }
10472 }
10473 }
10474 break;
10475
10476 case Instruction::And:
10477 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10478 // 'and x, C' produces [0, C].
10479 Upper = *C + 1;
10480 // X & -X is a power of two or zero. So we can cap the value at max power of
10481 // two.
10482 if (match(V: BO.getOperand(i_nocapture: 0), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 1)))) ||
10483 match(V: BO.getOperand(i_nocapture: 1), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 0)))))
10484 Upper = APInt::getSignedMinValue(numBits: Width) + 1;
10485 break;
10486
10487 case Instruction::Or:
10488 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10489 // 'or x, C' produces [C, UINT_MAX].
10490 Lower = *C;
10491 break;
10492
10493 case Instruction::AShr:
10494 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10495 // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C].
10496 Lower = APInt::getSignedMinValue(numBits: Width).ashr(ShiftAmt: *C);
10497 Upper = APInt::getSignedMaxValue(numBits: Width).ashr(ShiftAmt: *C) + 1;
10498 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10499 unsigned ShiftAmount = Width - 1;
10500 if (!C->isZero() && IIQ.isExact(Op: &BO))
10501 ShiftAmount = C->countr_zero();
10502 if (C->isNegative()) {
10503 // 'ashr C, x' produces [C, C >> (Width-1)]
10504 Lower = *C;
10505 Upper = C->ashr(ShiftAmt: ShiftAmount) + 1;
10506 } else {
10507 // 'ashr C, x' produces [C >> (Width-1), C]
10508 Lower = C->ashr(ShiftAmt: ShiftAmount);
10509 Upper = *C + 1;
10510 }
10511 }
10512 break;
10513
10514 case Instruction::LShr:
10515 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10516 // 'lshr x, C' produces [0, UINT_MAX >> C].
10517 Upper = APInt::getAllOnes(numBits: Width).lshr(ShiftAmt: *C) + 1;
10518 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10519 // 'lshr C, x' produces [C >> (Width-1), C].
10520 unsigned ShiftAmount = Width - 1;
10521 if (!C->isZero() && IIQ.isExact(Op: &BO))
10522 ShiftAmount = C->countr_zero();
10523 Lower = C->lshr(shiftAmt: ShiftAmount);
10524 Upper = *C + 1;
10525 }
10526 break;
10527
10528 case Instruction::Shl:
10529 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10530 if (IIQ.hasNoUnsignedWrap(Op: &BO)) {
10531 // 'shl nuw C, x' produces [C, C << CLZ(C)]
10532 Lower = *C;
10533 Upper = Lower.shl(shiftAmt: Lower.countl_zero()) + 1;
10534 } else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw?
10535 if (C->isNegative()) {
10536 // 'shl nsw C, x' produces [C << CLO(C)-1, C]
10537 unsigned ShiftAmount = C->countl_one() - 1;
10538 Lower = C->shl(shiftAmt: ShiftAmount);
10539 Upper = *C + 1;
10540 } else {
10541 // 'shl nsw C, x' produces [C, C << CLZ(C)-1]
10542 unsigned ShiftAmount = C->countl_zero() - 1;
10543 Lower = *C;
10544 Upper = C->shl(shiftAmt: ShiftAmount) + 1;
10545 }
10546 } else {
10547 // If lowbit is set, value can never be zero.
10548 if ((*C)[0])
10549 Lower = APInt::getOneBitSet(numBits: Width, BitNo: 0);
10550 // If we are shifting a constant the largest it can be is if the longest
10551 // sequence of consecutive ones is shifted to the highbits (breaking
10552 // ties for which sequence is higher). At the moment we take a liberal
10553 // upper bound on this by just popcounting the constant.
10554 // TODO: There may be a bitwise trick for it longest/highest
10555 // consecutative sequence of ones (naive method is O(Width) loop).
10556 Upper = APInt::getHighBitsSet(numBits: Width, hiBitsSet: C->popcount()) + 1;
10557 }
10558 } else if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10559 Upper = APInt::getBitsSetFrom(numBits: Width, loBit: C->getZExtValue()) + 1;
10560 }
10561 break;
10562
10563 case Instruction::SDiv:
10564 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10565 APInt IntMin = APInt::getSignedMinValue(numBits: Width);
10566 APInt IntMax = APInt::getSignedMaxValue(numBits: Width);
10567 if (C->isAllOnes()) {
10568 // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX]
10569 // where C != -1 and C != 0 and C != 1
10570 Lower = IntMin + 1;
10571 Upper = IntMax + 1;
10572 } else if (C->countl_zero() < Width - 1) {
10573 // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C]
10574 // where C != -1 and C != 0 and C != 1
10575 Lower = IntMin.sdiv(RHS: *C);
10576 Upper = IntMax.sdiv(RHS: *C);
10577 if (Lower.sgt(RHS: Upper))
10578 std::swap(a&: Lower, b&: Upper);
10579 Upper = Upper + 1;
10580 assert(Upper != Lower && "Upper part of range has wrapped!");
10581 }
10582 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10583 if (C->isMinSignedValue()) {
10584 // 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2].
10585 Lower = *C;
10586 Upper = Lower.lshr(shiftAmt: 1) + 1;
10587 } else {
10588 // 'sdiv C, x' produces [-|C|, |C|].
10589 Upper = C->abs() + 1;
10590 Lower = (-Upper) + 1;
10591 }
10592 }
10593 break;
10594
10595 case Instruction::UDiv:
10596 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10597 // 'udiv x, C' produces [0, UINT_MAX / C].
10598 Upper = APInt::getMaxValue(numBits: Width).udiv(RHS: *C) + 1;
10599 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10600 // 'udiv C, x' produces [0, C].
10601 Upper = *C + 1;
10602 }
10603 break;
10604
10605 case Instruction::SRem:
10606 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10607 // 'srem x, C' produces (-|C|, |C|).
10608 Upper = C->abs();
10609 Lower = (-Upper) + 1;
10610 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10611 if (C->isNegative()) {
10612 // 'srem -|C|, x' produces [-|C|, 0].
10613 Upper = 1;
10614 Lower = *C;
10615 } else {
10616 // 'srem |C|, x' produces [0, |C|].
10617 Upper = *C + 1;
10618 }
10619 }
10620 break;
10621
10622 case Instruction::URem:
10623 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10624 // 'urem x, C' produces [0, C).
10625 Upper = *C;
10626 else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10627 // 'urem C, x' produces [0, C].
10628 Upper = *C + 1;
10629 break;
10630
10631 default:
10632 break;
10633 }
10634}
10635
10636static ConstantRange getRangeForIntrinsic(const IntrinsicInst &II,
10637 bool UseInstrInfo) {
10638 unsigned Width = II.getType()->getScalarSizeInBits();
10639 const APInt *C;
10640 switch (II.getIntrinsicID()) {
10641 case Intrinsic::ctlz:
10642 case Intrinsic::cttz: {
10643 APInt Upper(Width, Width);
10644 if (!UseInstrInfo || !match(V: II.getArgOperand(i: 1), P: m_One()))
10645 Upper += 1;
10646 // Maximum of set/clear bits is the bit width.
10647 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper);
10648 }
10649 case Intrinsic::ctpop:
10650 // Maximum of set/clear bits is the bit width.
10651 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10652 Upper: APInt(Width, Width) + 1);
10653 case Intrinsic::uadd_sat:
10654 // uadd.sat(x, C) produces [C, UINT_MAX].
10655 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10656 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10657 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10658 break;
10659 case Intrinsic::sadd_sat:
10660 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10661 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10662 if (C->isNegative())
10663 // sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)].
10664 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10665 Upper: APInt::getSignedMaxValue(numBits: Width) + *C +
10666 1);
10667
10668 // sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX].
10669 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) + *C,
10670 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10671 }
10672 break;
10673 case Intrinsic::usub_sat:
10674 // usub.sat(C, x) produces [0, C].
10675 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10676 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10677
10678 // usub.sat(x, C) produces [0, UINT_MAX - C].
10679 if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10680 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10681 Upper: APInt::getMaxValue(numBits: Width) - *C + 1);
10682 break;
10683 case Intrinsic::ssub_sat:
10684 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10685 if (C->isNegative())
10686 // ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)].
10687 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10688 Upper: *C - APInt::getSignedMinValue(numBits: Width) +
10689 1);
10690
10691 // ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX].
10692 return ConstantRange::getNonEmpty(Lower: *C - APInt::getSignedMaxValue(numBits: Width),
10693 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10694 } else if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10695 if (C->isNegative())
10696 // ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]:
10697 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) - *C,
10698 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10699
10700 // ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C].
10701 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10702 Upper: APInt::getSignedMaxValue(numBits: Width) - *C +
10703 1);
10704 }
10705 break;
10706 case Intrinsic::umin:
10707 case Intrinsic::umax:
10708 case Intrinsic::smin:
10709 case Intrinsic::smax:
10710 if (!match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) &&
10711 !match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10712 break;
10713
10714 switch (II.getIntrinsicID()) {
10715 case Intrinsic::umin:
10716 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10717 case Intrinsic::umax:
10718 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10719 case Intrinsic::smin:
10720 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10721 Upper: *C + 1);
10722 case Intrinsic::smax:
10723 return ConstantRange::getNonEmpty(Lower: *C,
10724 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10725 default:
10726 llvm_unreachable("Must be min/max intrinsic");
10727 }
10728 break;
10729 case Intrinsic::abs:
10730 // If abs of SIGNED_MIN is poison, then the result is [0..SIGNED_MAX],
10731 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10732 if (match(V: II.getOperand(i_nocapture: 1), P: m_One()))
10733 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10734 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10735
10736 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10737 Upper: APInt::getSignedMinValue(numBits: Width) + 1);
10738 case Intrinsic::vscale:
10739 if (!II.getParent() || !II.getFunction())
10740 break;
10741 return getVScaleRange(F: II.getFunction(), BitWidth: Width);
10742 case Intrinsic::read_register:
10743 case Intrinsic::read_volatile_register: {
10744 const Module *M = II.getModule();
10745 if (!M || !M->getTargetTriple().isRISCV())
10746 break;
10747 if (II.getFunction() && isReadVLENB(II))
10748 return getRISCVVLENBRange(II, Width);
10749 break;
10750 }
10751 default:
10752 break;
10753 }
10754
10755 return ConstantRange::getFull(BitWidth: Width);
10756}
10757
10758static ConstantRange getRangeForSelectPattern(const SelectInst &SI,
10759 const InstrInfoQuery &IIQ) {
10760 unsigned BitWidth = SI.getType()->getScalarSizeInBits();
10761 const Value *LHS = nullptr, *RHS = nullptr;
10762 SelectPatternResult R = matchSelectPattern(V: &SI, LHS, RHS);
10763 if (R.Flavor == SPF_UNKNOWN)
10764 return ConstantRange::getFull(BitWidth);
10765
10766 if (R.Flavor == SelectPatternFlavor::SPF_ABS) {
10767 // If the negation part of the abs (in RHS) has the NSW flag,
10768 // then the result of abs(X) is [0..SIGNED_MAX],
10769 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10770 if (match(V: RHS, P: m_Neg(V: m_Specific(V: LHS))) &&
10771 IIQ.hasNoSignedWrap(Op: cast<Instruction>(Val: RHS)))
10772 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10773 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10774
10775 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10776 Upper: APInt::getSignedMinValue(numBits: BitWidth) + 1);
10777 }
10778
10779 if (R.Flavor == SelectPatternFlavor::SPF_NABS) {
10780 // The result of -abs(X) is <= 0.
10781 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10782 Upper: APInt(BitWidth, 1));
10783 }
10784
10785 const APInt *C;
10786 if (!match(V: LHS, P: m_APInt(Res&: C)) && !match(V: RHS, P: m_APInt(Res&: C)))
10787 return ConstantRange::getFull(BitWidth);
10788
10789 switch (R.Flavor) {
10790 case SPF_UMIN:
10791 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth), Upper: *C + 1);
10792 case SPF_UMAX:
10793 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: BitWidth));
10794 case SPF_SMIN:
10795 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10796 Upper: *C + 1);
10797 case SPF_SMAX:
10798 return ConstantRange::getNonEmpty(Lower: *C,
10799 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10800 default:
10801 return ConstantRange::getFull(BitWidth);
10802 }
10803}
10804
10805static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) {
10806 // The maximum representable value of a half is 65504. For floats the maximum
10807 // value is 3.4e38 which requires roughly 129 bits.
10808 unsigned BitWidth = I->getType()->getScalarSizeInBits();
10809 if (!I->getOperand(i: 0)->getType()->getScalarType()->isHalfTy())
10810 return;
10811 if (isa<FPToSIInst>(Val: I) && BitWidth >= 17) {
10812 Lower = APInt(BitWidth, -65504, true);
10813 Upper = APInt(BitWidth, 65505);
10814 }
10815
10816 if (isa<FPToUIInst>(Val: I) && BitWidth >= 16) {
10817 // For a fptoui the lower limit is left as 0.
10818 Upper = APInt(BitWidth, 65505);
10819 }
10820}
10821
10822ConstantRange llvm::computeConstantRange(const Value *V, bool ForSigned,
10823 const SimplifyQuery &SQ,
10824 unsigned Depth) {
10825 assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction");
10826
10827 if (Depth == MaxAnalysisRecursionDepth)
10828 return ConstantRange::getFull(BitWidth: V->getType()->getScalarSizeInBits());
10829
10830 if (auto *C = dyn_cast<Constant>(Val: V))
10831 return C->toConstantRange();
10832
10833 unsigned BitWidth = V->getType()->getScalarSizeInBits();
10834 ConstantRange CR = ConstantRange::getFull(BitWidth);
10835 if (auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
10836 APInt Lower = APInt(BitWidth, 0);
10837 APInt Upper = APInt(BitWidth, 0);
10838 // TODO: Return ConstantRange.
10839 setLimitsForBinOp(BO: *BO, Lower, Upper, IIQ: SQ.IIQ, PreferSignedRange: ForSigned);
10840 CR = ConstantRange::getNonEmpty(Lower, Upper);
10841 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
10842 CR = getRangeForIntrinsic(II: *II, UseInstrInfo: SQ.IIQ.UseInstrInfo);
10843 else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
10844 ConstantRange CRTrue =
10845 computeConstantRange(V: SI->getTrueValue(), ForSigned, SQ, Depth: Depth + 1);
10846 ConstantRange CRFalse =
10847 computeConstantRange(V: SI->getFalseValue(), ForSigned, SQ, Depth: Depth + 1);
10848 CR = CRTrue.unionWith(CR: CRFalse);
10849 CR = CR.intersectWith(CR: getRangeForSelectPattern(SI: *SI, IIQ: SQ.IIQ));
10850 } else if (auto *TI = dyn_cast<TruncInst>(Val: V)) {
10851 ConstantRange SrcCR =
10852 computeConstantRange(V: TI->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
10853 CR = SrcCR.truncate(BitWidth);
10854 } else if (auto *ZExt = dyn_cast<ZExtInst>(Val: V)) {
10855 ConstantRange SrcCR =
10856 computeConstantRange(V: ZExt->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
10857 CR = SrcCR.zeroExtend(BitWidth);
10858 } else if (auto *SExt = dyn_cast<SExtInst>(Val: V)) {
10859 ConstantRange SrcCR =
10860 computeConstantRange(V: SExt->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
10861 CR = SrcCR.signExtend(BitWidth);
10862 } else if (isa<FPToUIInst>(Val: V) || isa<FPToSIInst>(Val: V)) {
10863 APInt Lower = APInt(BitWidth, 0);
10864 APInt Upper = APInt(BitWidth, 0);
10865 // TODO: Return ConstantRange.
10866 setLimitForFPToI(I: cast<Instruction>(Val: V), Lower, Upper);
10867 CR = ConstantRange::getNonEmpty(Lower, Upper);
10868 } else if (const auto *A = dyn_cast<Argument>(Val: V))
10869 if (std::optional<ConstantRange> Range = A->getRange())
10870 CR = *Range;
10871
10872 if (auto *I = dyn_cast<Instruction>(Val: V)) {
10873 if (auto *Range = SQ.IIQ.getMetadata(I, KindID: LLVMContext::MD_range))
10874 CR = CR.intersectWith(CR: getConstantRangeFromMetadata(RangeMD: *Range));
10875
10876 Value *FrexpSrc;
10877 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
10878 if (std::optional<ConstantRange> Range = CB->getRange())
10879 CR = CR.intersectWith(CR: *Range);
10880 } else if (match(V: I, P: m_ExtractValue<1>(V: m_Intrinsic<Intrinsic::frexp>(
10881 Ops: m_Value(V&: FrexpSrc))))) {
10882 const fltSemantics &FltSem =
10883 FrexpSrc->getType()->getScalarType()->getFltSemantics();
10884 // It should be possible to implement this for any type, but this logic
10885 // only computes the range assuming standard subnormal handling.
10886 if (APFloat::isIEEELikeFP(FltSem)) {
10887 KnownFPClass KnownSrc = computeKnownFPClass(
10888 V: FrexpSrc, InterestedClasses: fcSubnormal | fcZero | fcNan | fcInf, SQ, Depth: Depth + 1);
10889
10890 // The exponent of frexp(NaN) and frexp(Inf) is unspecified. Only
10891 // constrain its range when the source can be neither.
10892 if (KnownSrc.isKnownNeverInfOrNaN()) {
10893 int MinExp = APFloat::semanticsMinExponent(FltSem) + 1;
10894
10895 // Offset to find the true minimum exponent value for a denormal.
10896 if (!KnownSrc.isKnownNeverSubnormal())
10897 MinExp -= (APFloat::semanticsPrecision(FltSem) - 1);
10898
10899 int MaxExp = APFloat::semanticsMaxExponent(FltSem) + 1;
10900
10901 auto [AdjustedMin, AdjustedMax, AdjustedMaxNonZero] =
10902 computeKnownExponentRangeFromContext(V: FrexpSrc, Q: SQ);
10903
10904 DenormalMode Mode = I->getFunction()->getDenormalMode(FPType: FltSem);
10905 bool NeverLogicalZero = KnownSrc.isKnownNeverLogicalZero(Mode);
10906
10907 MinExp = std::max(a: AdjustedMin, b: MinExp);
10908 MaxExp = std::min(a: NeverLogicalZero ? AdjustedMaxNonZero : AdjustedMax,
10909 b: MaxExp);
10910
10911 CR = ConstantRange::getNonEmpty(
10912 Lower: APInt(BitWidth, static_cast<int64_t>(MinExp), /*isSigned=*/true),
10913 Upper: APInt(BitWidth, static_cast<int64_t>(MaxExp) + 1,
10914 /*isSigned=*/true));
10915 }
10916 }
10917 }
10918 }
10919
10920 if (SQ.CtxI && SQ.AC) {
10921 // Try to restrict the range based on information from assumptions.
10922 for (auto &AssumeVH : SQ.AC->assumptionsFor(V)) {
10923 if (!AssumeVH)
10924 continue;
10925 CallInst *I = cast<CallInst>(Val&: AssumeVH);
10926 assert(I->getParent()->getParent() == SQ.CtxI->getParent()->getParent() &&
10927 "Got assumption for the wrong function!");
10928 assert(I->getIntrinsicID() == Intrinsic::assume &&
10929 "must be an assume intrinsic");
10930
10931 if (!isValidAssumeForContext(I, Q: SQ))
10932 continue;
10933 Value *Arg = I->getArgOperand(i: 0);
10934 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
10935 // Currently we just use information from comparisons.
10936 if (!Cmp || Cmp->getOperand(i_nocapture: 0) != V)
10937 continue;
10938 // TODO: Set "ForSigned" parameter via Cmp->isSigned()?
10939 ConstantRange RHS =
10940 computeConstantRange(V: Cmp->getOperand(i_nocapture: 1), /*ForSigned=*/false,
10941 SQ: SQ.getWithInstruction(I), Depth: Depth + 1);
10942 CR = CR.intersectWith(
10943 CR: ConstantRange::makeAllowedICmpRegion(Pred: Cmp->getCmpPredicate(), Other: RHS));
10944 }
10945 }
10946
10947 return CR;
10948}
10949
10950static void
10951addValueAffectedByCondition(Value *V,
10952 function_ref<void(Value *)> InsertAffected) {
10953 assert(V != nullptr);
10954 if (isa<Argument>(Val: V) || isa<GlobalValue>(Val: V)) {
10955 InsertAffected(V);
10956 } else if (auto *I = dyn_cast<Instruction>(Val: V)) {
10957 InsertAffected(V);
10958
10959 // Peek through unary operators to find the source of the condition.
10960 Value *Op;
10961 if (match(V: I, P: m_CombineOr(Ps: m_PtrToIntOrAddr(Op: m_Value(V&: Op)),
10962 Ps: m_Trunc(Op: m_Value(V&: Op))))) {
10963 if (isa<Instruction>(Val: Op) || isa<Argument>(Val: Op))
10964 InsertAffected(Op);
10965 }
10966 }
10967}
10968
10969void llvm::findValuesAffectedByCondition(
10970 Value *Cond, bool IsAssume, function_ref<void(Value *)> InsertAffected) {
10971 auto AddAffected = [&InsertAffected](Value *V) {
10972 addValueAffectedByCondition(V, InsertAffected);
10973 };
10974
10975 auto AddCmpOperands = [&AddAffected, IsAssume](Value *LHS, Value *RHS) {
10976 if (IsAssume) {
10977 AddAffected(LHS);
10978 AddAffected(RHS);
10979 } else if (match(V: RHS, P: m_Constant()))
10980 AddAffected(LHS);
10981 };
10982
10983 SmallVector<Value *, 8> Worklist;
10984 SmallPtrSet<Value *, 8> Visited;
10985 Worklist.push_back(Elt: Cond);
10986 while (!Worklist.empty()) {
10987 Value *V = Worklist.pop_back_val();
10988 if (!Visited.insert(Ptr: V).second)
10989 continue;
10990
10991 CmpPredicate Pred;
10992 Value *A, *B, *X;
10993
10994 if (IsAssume) {
10995 AddAffected(V);
10996 if (match(V, P: m_Not(V: m_Value(V&: X))))
10997 AddAffected(X);
10998 }
10999
11000 if (match(V, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
11001 // assume(A && B) is split to -> assume(A); assume(B);
11002 // assume(!(A || B)) is split to -> assume(!A); assume(!B);
11003 // Finally, assume(A || B) / assume(!(A && B)) generally don't provide
11004 // enough information to be worth handling (intersection of information as
11005 // opposed to union).
11006 if (!IsAssume) {
11007 Worklist.push_back(Elt: A);
11008 Worklist.push_back(Elt: B);
11009 }
11010 } else if (match(V, P: m_ICmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
11011 bool HasRHSC = match(V: B, P: m_ConstantInt());
11012 if (ICmpInst::isEquality(P: Pred)) {
11013 AddAffected(A);
11014 if (IsAssume)
11015 AddAffected(B);
11016 if (HasRHSC) {
11017 Value *Y;
11018 // (X << C) or (X >>_s C) or (X >>_u C).
11019 if (match(V: A, P: m_Shift(L: m_Value(V&: X), R: m_ConstantInt())))
11020 AddAffected(X);
11021 // (X & C) or (X | C).
11022 else if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
11023 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
11024 AddAffected(X);
11025 AddAffected(Y);
11026 }
11027 // X - Y
11028 else if (match(V: A, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
11029 AddAffected(X);
11030 AddAffected(Y);
11031 }
11032 }
11033 } else {
11034 AddCmpOperands(A, B);
11035 if (HasRHSC) {
11036 // Handle (A + C1) u< C2, which is the canonical form of
11037 // A > C3 && A < C4.
11038 if (match(V: A, P: m_AddLike(L: m_Value(V&: X), R: m_ConstantInt())))
11039 AddAffected(X);
11040
11041 if (ICmpInst::isUnsigned(Pred)) {
11042 Value *Y;
11043 // X & Y u> C -> X >u C && Y >u C
11044 // X | Y u< C -> X u< C && Y u< C
11045 // X nuw+ Y u< C -> X u< C && Y u< C
11046 if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
11047 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
11048 match(V: A, P: m_NUWAdd(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
11049 AddAffected(X);
11050 AddAffected(Y);
11051 }
11052 // X nuw- Y u> C -> X u> C
11053 if (match(V: A, P: m_NUWSub(L: m_Value(V&: X), R: m_Value())))
11054 AddAffected(X);
11055 }
11056 }
11057
11058 // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported
11059 // by computeKnownFPClass().
11060 if (match(V: A, P: m_ElementWiseBitCast(Op: m_Value(V&: X)))) {
11061 if (Pred == ICmpInst::ICMP_SLT && match(V: B, P: m_Zero()))
11062 InsertAffected(X);
11063 else if (Pred == ICmpInst::ICMP_SGT && match(V: B, P: m_AllOnes()))
11064 InsertAffected(X);
11065 }
11066 }
11067
11068 auto AddNuwSquareOperand = [&AddAffected](Value *Op) {
11069 Value *SquareOp = nullptr;
11070 if (match(V: Op, P: m_NUWMul(L: m_Value(V&: SquareOp), R: m_Deferred(V: SquareOp))))
11071 AddAffected(SquareOp);
11072 };
11073 AddNuwSquareOperand(A);
11074 AddNuwSquareOperand(B);
11075
11076 if (HasRHSC && match(V: A, P: m_Ctpop(Op0: m_Value(V&: X))))
11077 AddAffected(X);
11078 } else if (match(V, P: m_FCmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
11079 AddCmpOperands(A, B);
11080
11081 // fcmp fneg(x), y
11082 // fcmp fabs(x), y
11083 // fcmp fneg(fabs(x)), y
11084 if (match(V: A, P: m_FNeg(X: m_Value(V&: A))))
11085 AddAffected(A);
11086 if (match(V: A, P: m_FAbs(Op0: m_Value(V&: A))))
11087 AddAffected(A);
11088
11089 } else if (match(V, P: m_Intrinsic<Intrinsic::is_fpclass>(Ops: m_Value(V&: A),
11090 Ops: m_Value()))) {
11091 // Handle patterns that computeKnownFPClass() support.
11092 AddAffected(A);
11093 } else if (!IsAssume && match(V, P: m_Trunc(Op: m_Value(V&: X)))) {
11094 // Assume is checked here as X is already added above for assumes in
11095 // addValueAffectedByCondition
11096 AddAffected(X);
11097 } else if (!IsAssume && match(V, P: m_Not(V: m_Value(V&: X)))) {
11098 // Assume is checked here to avoid issues with ephemeral values
11099 Worklist.push_back(Elt: X);
11100 }
11101 }
11102}
11103
11104const Value *llvm::stripNullTest(const Value *V) {
11105 // (X >> C) or/add (X & mask(C) != 0)
11106 if (const auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
11107 if (BO->getOpcode() == Instruction::Add ||
11108 BO->getOpcode() == Instruction::Or) {
11109 const Value *X;
11110 const APInt *C1, *C2;
11111 if (match(V: BO, P: m_c_BinOp(L: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: C1)),
11112 R: m_ZExt(Op: m_SpecificICmp(
11113 MatchPred: ICmpInst::ICMP_NE,
11114 L: m_And(L: m_Deferred(V: X), R: m_LowBitMask(V&: C2)),
11115 R: m_Zero())))) &&
11116 C2->popcount() == C1->getZExtValue())
11117 return X;
11118 }
11119 }
11120 return nullptr;
11121}
11122
11123Value *llvm::stripNullTest(Value *V) {
11124 return const_cast<Value *>(stripNullTest(V: const_cast<const Value *>(V)));
11125}
11126
11127bool llvm::collectPossibleValues(const Value *V,
11128 SmallPtrSetImpl<const Constant *> &Constants,
11129 unsigned MaxCount, bool AllowUndefOrPoison) {
11130 SmallPtrSet<const Instruction *, 8> Visited;
11131 SmallVector<const Instruction *, 8> Worklist;
11132 auto Push = [&](const Value *V) -> bool {
11133 Constant *C;
11134 if (match(V: const_cast<Value *>(V), P: m_ImmConstant(C))) {
11135 if (!AllowUndefOrPoison && !isGuaranteedNotToBeUndefOrPoison(V: C))
11136 return false;
11137 // Check existence first to avoid unnecessary allocations.
11138 if (Constants.contains(Ptr: C))
11139 return true;
11140 if (Constants.size() == MaxCount)
11141 return false;
11142 Constants.insert(Ptr: C);
11143 return true;
11144 }
11145
11146 if (auto *Inst = dyn_cast<Instruction>(Val: V)) {
11147 if (Visited.insert(Ptr: Inst).second)
11148 Worklist.push_back(Elt: Inst);
11149 return true;
11150 }
11151 return false;
11152 };
11153 if (!Push(V))
11154 return false;
11155 while (!Worklist.empty()) {
11156 const Instruction *CurInst = Worklist.pop_back_val();
11157 switch (CurInst->getOpcode()) {
11158 case Instruction::Select:
11159 if (!Push(CurInst->getOperand(i: 1)))
11160 return false;
11161 if (!Push(CurInst->getOperand(i: 2)))
11162 return false;
11163 break;
11164 case Instruction::PHI:
11165 for (Value *IncomingValue : cast<PHINode>(Val: CurInst)->incoming_values()) {
11166 // Fast path for recurrence PHI.
11167 if (IncomingValue == CurInst)
11168 continue;
11169 if (!Push(IncomingValue))
11170 return false;
11171 }
11172 break;
11173 default:
11174 return false;
11175 }
11176 }
11177 return true;
11178}
11179