1//===- ValueTracking.cpp - Walk computations to compute properties --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains routines that help analyze properties that chains of
10// computations have.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/ValueTracking.h"
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/FloatingPointMode.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/ADT/SmallPtrSet.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/ADT/iterator_range.h"
24#include "llvm/Analysis/AliasAnalysis.h"
25#include "llvm/Analysis/AssumeBundleQueries.h"
26#include "llvm/Analysis/AssumptionCache.h"
27#include "llvm/Analysis/ConstantFolding.h"
28#include "llvm/Analysis/DomConditionCache.h"
29#include "llvm/Analysis/FloatingPointPredicateUtils.h"
30#include "llvm/Analysis/GuardUtils.h"
31#include "llvm/Analysis/InstructionSimplify.h"
32#include "llvm/Analysis/Loads.h"
33#include "llvm/Analysis/LoopInfo.h"
34#include "llvm/Analysis/TargetLibraryInfo.h"
35#include "llvm/Analysis/VectorUtils.h"
36#include "llvm/Analysis/WithCache.h"
37#include "llvm/IR/Argument.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/BasicBlock.h"
40#include "llvm/IR/BundleAttributes.h"
41#include "llvm/IR/Constant.h"
42#include "llvm/IR/ConstantFPRange.h"
43#include "llvm/IR/ConstantRange.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DerivedTypes.h"
46#include "llvm/IR/DiagnosticInfo.h"
47#include "llvm/IR/Dominators.h"
48#include "llvm/IR/EHPersonalities.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GetElementPtrTypeIterator.h"
51#include "llvm/IR/GlobalAlias.h"
52#include "llvm/IR/GlobalValue.h"
53#include "llvm/IR/GlobalVariable.h"
54#include "llvm/IR/InstrTypes.h"
55#include "llvm/IR/Instruction.h"
56#include "llvm/IR/Instructions.h"
57#include "llvm/IR/IntrinsicInst.h"
58#include "llvm/IR/Intrinsics.h"
59#include "llvm/IR/IntrinsicsAArch64.h"
60#include "llvm/IR/IntrinsicsAMDGPU.h"
61#include "llvm/IR/IntrinsicsRISCV.h"
62#include "llvm/IR/IntrinsicsX86.h"
63#include "llvm/IR/LLVMContext.h"
64#include "llvm/IR/Metadata.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/Operator.h"
67#include "llvm/IR/PatternMatch.h"
68#include "llvm/IR/Type.h"
69#include "llvm/IR/User.h"
70#include "llvm/IR/Value.h"
71#include "llvm/Support/Casting.h"
72#include "llvm/Support/CommandLine.h"
73#include "llvm/Support/Compiler.h"
74#include "llvm/Support/ErrorHandling.h"
75#include "llvm/Support/KnownBits.h"
76#include "llvm/Support/KnownFPClass.h"
77#include "llvm/Support/MathExtras.h"
78#include "llvm/Support/UndefPoison.h"
79#include "llvm/TargetParser/RISCVTargetParser.h"
80#include <algorithm>
81#include <cassert>
82#include <cstdint>
83#include <optional>
84#include <utility>
85
86using namespace llvm;
87using namespace llvm::PatternMatch;
88
89// Controls the number of uses of the value searched for possible
90// dominating comparisons.
91static cl::opt<unsigned> DomConditionsMaxUses("dom-conditions-max-uses",
92 cl::Hidden, cl::init(Val: 20));
93
94/// Maximum number of instructions to check between assume and context
95/// instruction.
96static constexpr unsigned MaxInstrsToCheckForFree = 32;
97
98template <typename InstTy>
99static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
100 Value *&Init, Value *&OtherOp);
101
102/// Returns the bitwidth of the given scalar or pointer type. For vector types,
103/// returns the element type's bitwidth.
104static unsigned getBitWidth(Type *Ty, const DataLayout &DL) {
105 if (unsigned BitWidth = Ty->getScalarSizeInBits())
106 return BitWidth;
107
108 return DL.getPointerTypeSizeInBits(Ty);
109}
110
111// Given the provided Value and, potentially, a context instruction, return
112// the preferred context instruction (if any).
113static const Instruction *safeCtxI(const Value *V, const Instruction *CtxI) {
114 // If we've been provided with a context instruction, then use that (provided
115 // it has been inserted).
116 if (CtxI && CtxI->getParent())
117 return CtxI;
118
119 // If the value is really an already-inserted instruction, then use that.
120 CtxI = dyn_cast<Instruction>(Val: V);
121 if (CtxI && CtxI->getParent())
122 return CtxI;
123
124 return nullptr;
125}
126
127static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
128 const APInt &DemandedElts,
129 APInt &DemandedLHS, APInt &DemandedRHS) {
130 if (isa<ScalableVectorType>(Val: Shuf->getType())) {
131 assert(DemandedElts == APInt(1,1));
132 DemandedLHS = DemandedRHS = DemandedElts;
133 return true;
134 }
135
136 int NumElts =
137 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
138 return llvm::getShuffleDemandedElts(SrcWidth: NumElts, Mask: Shuf->getShuffleMask(),
139 DemandedElts, DemandedLHS, DemandedRHS);
140}
141
142static void computeKnownBits(const Value *V, const APInt &DemandedElts,
143 KnownBits &Known, const SimplifyQuery &Q,
144 unsigned Depth);
145
146void llvm::computeKnownBits(const Value *V, KnownBits &Known,
147 const SimplifyQuery &Q, unsigned Depth) {
148 // Since the number of lanes in a scalable vector is unknown at compile time,
149 // we track one bit which is implicitly broadcast to all lanes. This means
150 // that all lanes in a scalable vector are considered demanded.
151 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
152 APInt DemandedElts =
153 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
154 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
155}
156
157void llvm::computeKnownBits(const Value *V, KnownBits &Known,
158 const DataLayout &DL, AssumptionCache *AC,
159 const Instruction *CtxI, const DominatorTree *DT,
160 bool UseInstrInfo, unsigned Depth) {
161 computeKnownBits(V, Known,
162 Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo),
163 Depth);
164}
165
166KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
167 AssumptionCache *AC, const Instruction *CtxI,
168 const DominatorTree *DT, bool UseInstrInfo,
169 unsigned Depth) {
170 return computeKnownBits(
171 V, Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo), Depth);
172}
173
174static NoCommonBitsSetResult
175haveNoCommonBitsSetSpecialCases(const Value *LHS, const Value *RHS,
176 const SimplifyQuery &SQ) {
177 // Look for an inverted mask: (X & ~M) op (Y & M).
178 {
179 Value *M;
180 if (match(V: LHS, P: m_c_And(L: m_Not(V: m_Value(V&: M)), R: m_Value())) &&
181 match(V: RHS, P: m_c_And(L: m_Specific(V: M), R: m_Value())))
182 return isGuaranteedNotToBeUndef(V: M, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)
183 ? NoCommonBitsSetResult::Known
184 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
185 }
186
187 // X op (Y & ~X)
188 if (match(V: RHS, P: m_c_And(L: m_Not(V: m_Specific(V: LHS)), R: m_Value())))
189 return isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)
190 ? NoCommonBitsSetResult::Known
191 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
192
193 // X op ((X & Y) ^ Y) -- this is the canonical form of the previous pattern
194 // for constant Y.
195 Value *Y;
196 if (match(V: RHS,
197 P: m_c_Xor(L: m_c_And(L: m_Specific(V: LHS), R: m_Value(V&: Y)), R: m_Deferred(V: Y)))) {
198 bool IsNoUndef = isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT) &&
199 isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT);
200 return IsNoUndef ? NoCommonBitsSetResult::Known
201 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
202 }
203
204 // Peek through extends to find a 'not' of the other side:
205 // (ext Y) op ext(~Y)
206 if (match(V: LHS, P: m_ZExtOrSExt(Op: m_Value(V&: Y))) &&
207 match(V: RHS, P: m_ZExtOrSExt(Op: m_Not(V: m_Specific(V: Y)))))
208 return isGuaranteedNotToBeUndef(V: Y, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT)
209 ? NoCommonBitsSetResult::Known
210 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
211
212 // Look for: (A & B) op ~(A | B)
213 {
214 Value *A, *B;
215 if (match(V: LHS, P: m_And(L: m_Value(V&: A), R: m_Value(V&: B))) &&
216 match(V: RHS, P: m_Not(V: m_c_Or(L: m_Specific(V: A), R: m_Specific(V: B))))) {
217 bool IsNoUndef = isGuaranteedNotToBeUndef(V: A, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT) &&
218 isGuaranteedNotToBeUndef(V: B, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT);
219 return IsNoUndef ? NoCommonBitsSetResult::Known
220 : NoCommonBitsSetResult::OnlyIfUndefIgnored;
221 }
222 }
223
224 // Look for: (X << V) op (Y >> (BitWidth - V))
225 // or (X >> V) op (Y << (BitWidth - V))
226 {
227 const Value *V;
228 const APInt *R;
229 if (((match(V: RHS, P: m_Shl(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
230 match(V: LHS, P: m_LShr(L: m_Value(), R: m_Specific(V)))) ||
231 (match(V: RHS, P: m_LShr(L: m_Value(), R: m_Sub(L: m_APInt(Res&: R), R: m_Value(V)))) &&
232 match(V: LHS, P: m_Shl(L: m_Value(), R: m_Specific(V))))) &&
233 R->uge(RHS: LHS->getType()->getScalarSizeInBits()))
234 return NoCommonBitsSetResult::Known;
235 }
236
237 return NoCommonBitsSetResult::Unknown;
238}
239
240NoCommonBitsSetResult
241llvm::getNoCommonBitsSetResult(const WithCache<const Value *> &LHSCache,
242 const WithCache<const Value *> &RHSCache,
243 const SimplifyQuery &SQ) {
244 const Value *LHS = LHSCache.getValue();
245 const Value *RHS = RHSCache.getValue();
246
247 assert(LHS->getType() == RHS->getType() &&
248 "LHS and RHS should have the same type");
249 assert(LHS->getType()->isIntOrIntVectorTy() &&
250 "LHS and RHS should be integers");
251
252 NoCommonBitsSetResult Result = haveNoCommonBitsSetSpecialCases(LHS, RHS, SQ);
253 if (Result == NoCommonBitsSetResult::Known)
254 return NoCommonBitsSetResult::Known;
255
256 NoCommonBitsSetResult CommuteResult =
257 haveNoCommonBitsSetSpecialCases(LHS: RHS, RHS: LHS, SQ);
258 if (CommuteResult == NoCommonBitsSetResult::Known)
259 return NoCommonBitsSetResult::Known;
260
261 if (KnownBits::haveNoCommonBitsSet(LHS: LHSCache.getKnownBits(Q: SQ),
262 RHS: RHSCache.getKnownBits(Q: SQ)))
263 return NoCommonBitsSetResult::Known;
264
265 if (Result == NoCommonBitsSetResult::OnlyIfUndefIgnored ||
266 CommuteResult == NoCommonBitsSetResult::OnlyIfUndefIgnored)
267 return NoCommonBitsSetResult::OnlyIfUndefIgnored;
268
269 return NoCommonBitsSetResult::Unknown;
270}
271
272bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
273 const WithCache<const Value *> &RHSCache,
274 const SimplifyQuery &SQ) {
275 NoCommonBitsSetResult Result =
276 getNoCommonBitsSetResult(LHSCache, RHSCache, SQ);
277 return Result == NoCommonBitsSetResult::Known;
278}
279
280bool llvm::isOnlyUsedInZeroComparison(const Instruction *I) {
281 return !I->user_empty() &&
282 all_of(Range: I->users(), P: match_fn(P: m_ICmp(L: m_Value(), R: m_Zero())));
283}
284
285bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) {
286 return !I->user_empty() && all_of(Range: I->users(), P: [](const User *U) {
287 CmpPredicate P;
288 return match(V: U, P: m_ICmp(Pred&: P, L: m_Value(), R: m_Zero())) && ICmpInst::isEquality(P);
289 });
290}
291
292bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
293 bool OrZero, AssumptionCache *AC,
294 const Instruction *CtxI,
295 const DominatorTree *DT, bool UseInstrInfo,
296 unsigned Depth) {
297 return ::isKnownToBeAPowerOfTwo(
298 V, OrZero, Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo),
299 Depth);
300}
301
302static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
303 const SimplifyQuery &Q, unsigned Depth);
304
305bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
306 unsigned Depth) {
307 return computeKnownBits(V, Q: SQ, Depth).isNonNegative();
308}
309
310bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
311 unsigned Depth) {
312 if (auto *CI = dyn_cast<ConstantInt>(Val: V))
313 return CI->getValue().isStrictlyPositive();
314
315 // If `isKnownNonNegative` ever becomes more sophisticated, make sure to keep
316 // this updated.
317 KnownBits Known = computeKnownBits(V, Q: SQ, Depth);
318 return Known.isNonNegative() &&
319 (Known.isNonZero() || isKnownNonZero(V, Q: SQ, Depth));
320}
321
322bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ,
323 unsigned Depth) {
324 return computeKnownBits(V, Q: SQ, Depth).isNegative();
325}
326
327static bool isKnownNonEqual(const Value *V1, const Value *V2,
328 const APInt &DemandedElts, const SimplifyQuery &Q,
329 unsigned Depth);
330
331static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
332 const Value *RHS);
333
334bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
335 const SimplifyQuery &Q, unsigned Depth) {
336 // We don't support looking through casts.
337 if (V1 == V2 || V1->getType() != V2->getType())
338 return false;
339 auto *FVTy = dyn_cast<FixedVectorType>(Val: V1->getType());
340 APInt DemandedElts =
341 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
342 return ::isKnownNonEqual(V1, V2, DemandedElts, Q, Depth);
343}
344
345bool llvm::MaskedValueIsZero(const Value *V, const APInt &Mask,
346 const SimplifyQuery &SQ, unsigned Depth) {
347 KnownBits Known(Mask.getBitWidth());
348 computeKnownBits(V, Known, Q: SQ, Depth);
349 return Mask.isSubsetOf(RHS: Known.Zero);
350}
351
352static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
353 const SimplifyQuery &Q, unsigned Depth);
354
355static unsigned ComputeNumSignBits(const Value *V, const SimplifyQuery &Q,
356 unsigned Depth = 0) {
357 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
358 APInt DemandedElts =
359 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
360 return ComputeNumSignBits(V, DemandedElts, Q, Depth);
361}
362
363unsigned llvm::ComputeNumSignBits(const Value *V, const DataLayout &DL,
364 AssumptionCache *AC, const Instruction *CtxI,
365 const DominatorTree *DT, bool UseInstrInfo,
366 unsigned Depth) {
367 return ::ComputeNumSignBits(
368 V, Q: SimplifyQuery(DL, DT, AC, safeCtxI(V, CtxI), UseInstrInfo), Depth);
369}
370
371unsigned llvm::ComputeMaxSignificantBits(const Value *V, const DataLayout &DL,
372 AssumptionCache *AC,
373 const Instruction *CtxI,
374 const DominatorTree *DT,
375 unsigned Depth) {
376 unsigned SignBits = ComputeNumSignBits(V, DL, AC, CtxI, DT, UseInstrInfo: Depth);
377 return V->getType()->getScalarSizeInBits() - SignBits + 1;
378}
379
380/// Try to detect the lerp pattern: a * (b - c) + c * d
381/// where a >= 0, b >= 0, c >= 0, d >= 0, and b >= c.
382///
383/// In that particular case, we can use the following chain of reasoning:
384///
385/// a * (b - c) + c * d <= a' * (b - c) + a' * c = a' * b where a' = max(a, d)
386///
387/// Since that is true for arbitrary a, b, c and d within our constraints, we
388/// can conclude that:
389///
390/// max(a * (b - c) + c * d) <= max(max(a), max(d)) * max(b) = U
391///
392/// Considering that any result of the lerp would be less or equal to U, it
393/// would have at least the number of leading 0s as in U.
394///
395/// While being quite a specific situation, it is fairly common in computer
396/// graphics in the shape of alpha blending.
397///
398/// Modifies given KnownOut in-place with the inferred information.
399static void computeKnownBitsFromLerpPattern(const Value *Op0, const Value *Op1,
400 const APInt &DemandedElts,
401 KnownBits &KnownOut,
402 const SimplifyQuery &Q,
403 unsigned Depth) {
404
405 Type *Ty = Op0->getType();
406 const unsigned BitWidth = Ty->getScalarSizeInBits();
407
408 // Only handle scalar types for now
409 if (Ty->isVectorTy())
410 return;
411
412 // Try to match: a * (b - c) + c * d.
413 // When a == 1 => A == nullptr, the same applies to d/D as well.
414 const Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr;
415 const Instruction *SubBC = nullptr;
416
417 const auto MatchSubBC = [&]() {
418 // (b - c) can have two forms that interest us:
419 //
420 // 1. sub nuw %b, %c
421 // 2. xor %c, %b
422 //
423 // For the first case, nuw flag guarantees our requirement b >= c.
424 //
425 // The second case might happen when the analysis can infer that b is a mask
426 // for c and we can transform sub operation into xor (that is usually true
427 // for constant b's). Even though xor is symmetrical, canonicalization
428 // ensures that the constant will be the RHS. We have additional checks
429 // later on to ensure that this xor operation is equivalent to subtraction.
430 return m_Instruction(I&: SubBC, P: m_CombineOr(Ps: m_NUWSub(L: m_Value(V&: B), R: m_Value(V&: C)),
431 Ps: m_Xor(L: m_Value(V&: C), R: m_Value(V&: B))));
432 };
433
434 const auto MatchASubBC = [&]() {
435 // Cases:
436 // - a * (b - c)
437 // - (b - c) * a
438 // - (b - c) <- a implicitly equals 1
439 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: A), R: MatchSubBC()), Ps: MatchSubBC());
440 };
441
442 const auto MatchCD = [&]() {
443 // Cases:
444 // - d * c
445 // - c * d
446 // - c <- d implicitly equals 1
447 return m_CombineOr(Ps: m_c_Mul(L: m_Value(V&: D), R: m_Specific(V: C)), Ps: m_Specific(V: C));
448 };
449
450 const auto Match = [&](const Value *LHS, const Value *RHS) {
451 // We do use m_Specific(C) in MatchCD, so we have to make sure that
452 // it's bound to anything and match(LHS, MatchASubBC()) absolutely
453 // has to evaluate first and return true.
454 //
455 // If Match returns true, it is guaranteed that B != nullptr, C != nullptr.
456 return match(V: LHS, P: MatchASubBC()) && match(V: RHS, P: MatchCD());
457 };
458
459 if (!Match(Op0, Op1) && !Match(Op1, Op0))
460 return;
461
462 const auto ComputeKnownBitsOrOne = [&](const Value *V) {
463 // For some of the values we use the convention of leaving
464 // it nullptr to signify an implicit constant 1.
465 return V ? computeKnownBits(V, DemandedElts, Q, Depth: Depth + 1)
466 : KnownBits::makeConstant(C: APInt(BitWidth, 1));
467 };
468
469 // Check that all operands are non-negative
470 const KnownBits KnownA = ComputeKnownBitsOrOne(A);
471 if (!KnownA.isNonNegative())
472 return;
473
474 const KnownBits KnownD = ComputeKnownBitsOrOne(D);
475 if (!KnownD.isNonNegative())
476 return;
477
478 const KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
479 if (!KnownB.isNonNegative())
480 return;
481
482 const KnownBits KnownC = computeKnownBits(V: C, DemandedElts, Q, Depth: Depth + 1);
483 if (!KnownC.isNonNegative())
484 return;
485
486 // If we matched subtraction as xor, we need to actually check that xor
487 // is semantically equivalent to subtraction.
488 //
489 // For that to be true, b has to be a mask for c or that b's known
490 // ones cover all known and possible ones of c.
491 if (SubBC->getOpcode() == Instruction::Xor &&
492 !KnownC.getMaxValue().isSubsetOf(RHS: KnownB.getMinValue()))
493 return;
494
495 const APInt MaxA = KnownA.getMaxValue();
496 const APInt MaxD = KnownD.getMaxValue();
497 const APInt MaxAD = APIntOps::umax(A: MaxA, B: MaxD);
498 const APInt MaxB = KnownB.getMaxValue();
499
500 // We can't infer leading zeros info if the upper-bound estimate wraps.
501 bool Overflow;
502 const APInt UpperBound = MaxAD.umul_ov(RHS: MaxB, Overflow);
503
504 if (Overflow)
505 return;
506
507 // If we know that x <= y and both are positive than x has at least the same
508 // number of leading zeros as y.
509 const unsigned MinimumNumberOfLeadingZeros = UpperBound.countl_zero();
510 KnownOut.Zero.setHighBits(MinimumNumberOfLeadingZeros);
511}
512
513static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
514 bool NSW, bool NUW,
515 const APInt &DemandedElts,
516 KnownBits &KnownOut, KnownBits &Known2,
517 const SimplifyQuery &Q, unsigned Depth) {
518 computeKnownBits(V: Op1, DemandedElts, Known&: KnownOut, Q, Depth: Depth + 1);
519
520 // If one operand is unknown and we have no nowrap information,
521 // the result will be unknown independently of the second operand.
522 if (KnownOut.isUnknown() && !NSW && !NUW)
523 return;
524
525 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
526 KnownOut = KnownBits::computeForAddSub(Add, NSW, NUW, LHS: Known2, RHS: KnownOut);
527
528 if (!Add && NSW && !KnownOut.isNonNegative() &&
529 (isImpliedByDomCondition(Pred: ICmpInst::ICMP_SLE, LHS: Op1, RHS: Op0, ContextI: Q.CtxI, DL: Q.DL)
530 .value_or(u: false) ||
531 match(V: Op1, P: m_c_SMin(L: m_Specific(V: Op0), R: m_Value()))))
532 KnownOut.makeNonNegative();
533
534 if (Add)
535 // Try to match lerp pattern and combine results
536 computeKnownBitsFromLerpPattern(Op0, Op1, DemandedElts, KnownOut, Q, Depth);
537}
538
539static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
540 bool NUW, const APInt &DemandedElts,
541 KnownBits &Known, KnownBits &Known2,
542 const SimplifyQuery &Q, unsigned Depth) {
543 computeKnownBits(V: Op1, DemandedElts, Known, Q, Depth: Depth + 1);
544 computeKnownBits(V: Op0, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
545
546 bool isKnownNegative = false;
547 bool isKnownNonNegative = false;
548 // If the multiplication is known not to overflow, compute the sign bit.
549 if (NSW) {
550 if (Op0 == Op1) {
551 // The product of a number with itself is non-negative.
552 isKnownNonNegative = true;
553 } else {
554 bool isKnownNonNegativeOp1 = Known.isNonNegative();
555 bool isKnownNonNegativeOp0 = Known2.isNonNegative();
556 bool isKnownNegativeOp1 = Known.isNegative();
557 bool isKnownNegativeOp0 = Known2.isNegative();
558 // The product of two numbers with the same sign is non-negative.
559 isKnownNonNegative = (isKnownNegativeOp1 && isKnownNegativeOp0) ||
560 (isKnownNonNegativeOp1 && isKnownNonNegativeOp0);
561 if (!isKnownNonNegative && NUW) {
562 // mul nuw nsw with a factor > 1 is non-negative.
563 KnownBits One = KnownBits::makeConstant(C: APInt(Known.getBitWidth(), 1));
564 isKnownNonNegative = KnownBits::sgt(LHS: Known, RHS: One).value_or(u: false) ||
565 KnownBits::sgt(LHS: Known2, RHS: One).value_or(u: false);
566 }
567
568 // The product of a negative number and a non-negative number is either
569 // negative or zero.
570 if (!isKnownNonNegative)
571 isKnownNegative =
572 (isKnownNegativeOp1 && isKnownNonNegativeOp0 &&
573 Known2.isNonZero()) ||
574 (isKnownNegativeOp0 && isKnownNonNegativeOp1 && Known.isNonZero());
575 }
576 }
577
578 bool SelfMultiply = Op0 == Op1;
579 if (SelfMultiply)
580 SelfMultiply &=
581 isGuaranteedNotToBeUndef(V: Op0, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1);
582 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
583
584 if (SelfMultiply) {
585 unsigned SignBits = ComputeNumSignBits(V: Op0, DemandedElts, Q, Depth: Depth + 1);
586 unsigned TyBits = Op0->getType()->getScalarSizeInBits();
587 unsigned OutValidBits = 2 * (TyBits - SignBits + 1);
588
589 if (OutValidBits < TyBits) {
590 APInt KnownZeroMask =
591 APInt::getHighBitsSet(numBits: TyBits, hiBitsSet: TyBits - OutValidBits + 1);
592 Known.Zero |= KnownZeroMask;
593 }
594 }
595
596 // Only make use of no-wrap flags if we failed to compute the sign bit
597 // directly. This matters if the multiplication always overflows, in
598 // which case we prefer to follow the result of the direct computation,
599 // though as the program is invoking undefined behaviour we can choose
600 // whatever we like here.
601 if (isKnownNonNegative && !Known.isNegative())
602 Known.makeNonNegative();
603 else if (isKnownNegative && !Known.isNonNegative())
604 Known.makeNegative();
605}
606
607void llvm::computeKnownBitsFromRangeMetadata(const MDNode &Ranges,
608 KnownBits &Known) {
609 unsigned BitWidth = Known.getBitWidth();
610 unsigned NumRanges = Ranges.getNumOperands() / 2;
611 assert(NumRanges >= 1);
612
613 Known.setAllConflict();
614
615 for (unsigned i = 0; i < NumRanges; ++i) {
616 ConstantInt *Lower =
617 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 0));
618 ConstantInt *Upper =
619 mdconst::extract<ConstantInt>(MD: Ranges.getOperand(I: 2 * i + 1));
620 ConstantRange Range(Lower->getValue(), Upper->getValue());
621 // BitWidth must equal the Ranges BitWidth for the correct number of high
622 // bits to be set.
623 assert(BitWidth == Range.getBitWidth() &&
624 "Known bit width must match range bit width!");
625
626 // The first CommonPrefixBits of all values in Range are equal.
627 unsigned CommonPrefixBits =
628 (Range.getUnsignedMax() ^ Range.getUnsignedMin()).countl_zero();
629 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: CommonPrefixBits);
630 APInt UnsignedMax = Range.getUnsignedMax().zextOrTrunc(width: BitWidth);
631 Known.One &= UnsignedMax & Mask;
632 Known.Zero &= ~UnsignedMax & Mask;
633 }
634}
635
636static bool isEphemeralValueOf(const Instruction *I, const Value *E) {
637 // The instruction defining an assumption's condition itself is always
638 // considered ephemeral to that assumption (even if it has other
639 // non-ephemeral users). See r246696's test case for an example.
640 if (is_contained(Range: I->operands(), Element: E))
641 return true;
642
643 const auto *EI = dyn_cast<Instruction>(Val: E);
644 if (!EI)
645 return false;
646
647 if (EI == I)
648 return true;
649
650 SmallPtrSet<const Instruction *, 16> Visited;
651 SmallVector<const Instruction *, 16> WorkList;
652 Visited.insert(Ptr: EI);
653 WorkList.push_back(Elt: EI);
654 bool ReachesI = false;
655 while (!WorkList.empty()) {
656 const Instruction *V = WorkList.pop_back_val();
657 for (const User *U : V->users()) {
658 const auto *UI = cast<Instruction>(Val: U);
659 if (UI == I) {
660 ReachesI = true;
661 continue;
662 }
663 if (UI->mayHaveSideEffects() || UI->isTerminator())
664 return false;
665 if (Visited.insert(Ptr: UI).second)
666 WorkList.push_back(Elt: UI);
667 }
668 }
669 return ReachesI;
670}
671
672// Is this an intrinsic that cannot be speculated but also cannot trap?
673bool llvm::isAssumeLikeIntrinsic(const Instruction *I) {
674 if (const IntrinsicInst *CI = dyn_cast<IntrinsicInst>(Val: I))
675 return CI->isAssumeLikeIntrinsic();
676
677 return false;
678}
679
680bool llvm::isValidAssumeForContext(const Instruction *Inv,
681 const Instruction *CtxI,
682 const DominatorTree *DT,
683 bool AllowEphemerals) {
684 // There are two restrictions on the use of an assume:
685 // 1. The assume must dominate the context (or the control flow must
686 // reach the assume whenever it reaches the context).
687 // 2. The context must not be in the assume's set of ephemeral values
688 // (otherwise we will use the assume to prove that the condition
689 // feeding the assume is trivially true, thus causing the removal of
690 // the assume).
691
692 if (Inv->getParent() == CtxI->getParent()) {
693 // If Inv and CtxI are in the same block, check if the assume (Inv) is first
694 // in the BB.
695 if (Inv->comesBefore(Other: CtxI))
696 return true;
697
698 // Don't let an assume affect itself - this would cause the problems
699 // `isEphemeralValueOf` is trying to prevent, and it would also make
700 // the loop below go out of bounds.
701 if (!AllowEphemerals && Inv == CtxI)
702 return false;
703
704 // The context comes first, but they're both in the same block.
705 // Make sure there is nothing in between that might interrupt
706 // the control flow, not even CtxI itself.
707 // We limit the scan distance between the assume and its context instruction
708 // to avoid a compile-time explosion. This limit is chosen arbitrarily, so
709 // it can be adjusted if needed (could be turned into a cl::opt).
710 auto Range = make_range(x: CtxI->getIterator(), y: Inv->getIterator());
711 if (!isGuaranteedToTransferExecutionToSuccessor(Range, ScanLimit: 15))
712 return false;
713
714 return AllowEphemerals || !isEphemeralValueOf(I: Inv, E: CtxI);
715 }
716
717 // Inv and CtxI are in different blocks.
718 if (DT) {
719 if (DT->dominates(Def: Inv, User: CtxI))
720 return true;
721 } else if (Inv->getParent() == CtxI->getParent()->getSinglePredecessor() ||
722 Inv->getParent()->isEntryBlock()) {
723 // We don't have a DT, but this trivially dominates.
724 return true;
725 }
726
727 return false;
728}
729
730static bool hasNoFreeInRange(BasicBlock::const_iterator Begin,
731 BasicBlock::const_iterator End,
732 unsigned &NumChecked) {
733 for (const Instruction &I : make_range(x: Begin, y: End)) {
734 if (NumChecked++ > MaxInstrsToCheckForFree)
735 return false;
736 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
737 if (!CB->hasFnAttr(Kind: Attribute::NoFree))
738 return false;
739 } else if (I.maySynchronize()) {
740 return false;
741 }
742 }
743 return true;
744}
745
746bool llvm::willNotFreeBetween(const Instruction *Assume,
747 const Instruction *CtxI,
748 const DominatorTree *DT) {
749 const BasicBlock *CtxBB = CtxI->getParent();
750 const BasicBlock *AssumeBB = Assume->getParent();
751 unsigned NumChecked = 0;
752 BasicBlock::const_iterator CtxIter = CtxI->getIterator();
753 if (CtxBB == AssumeBB) {
754 if (Assume != CtxI && !Assume->comesBefore(Other: CtxI))
755 return false;
756 return hasNoFreeInRange(Begin: Assume->getIterator(), End: CtxIter, NumChecked);
757 }
758 if (DT && !DT->dominates(Def: Assume, User: CtxI))
759 return false;
760 if (!hasNoFreeInRange(Begin: CtxBB->begin(), End: CtxIter, NumChecked))
761 return false;
762 if (pred_empty(BB: CtxBB))
763 return false;
764
765 // Note: CtxBB is NOT pre-inserted into Visited to ensure that loop
766 // backedges returning to CtxBB are enqueued and checked correctly.
767 SmallVector<const BasicBlock *, 16> Worklist(predecessors(BB: CtxBB));
768 SmallPtrSet<const BasicBlock *, 16> Visited;
769 while (!Worklist.empty()) {
770 const BasicBlock *CurBB = Worklist.pop_back_val();
771 if (!Visited.insert(Ptr: CurBB).second)
772 continue;
773
774 if (CurBB == AssumeBB) {
775 if (!hasNoFreeInRange(Begin: Assume->getIterator(), End: AssumeBB->end(), NumChecked))
776 return false;
777 continue;
778 }
779 assert((!DT || DT->dominates(AssumeBB, CurBB)) &&
780 "Blocks between Assume and CtxI must be dominated by AssumeBB");
781
782 if (pred_empty(BB: CurBB))
783 return false;
784
785 // If CurBB == CtxBB (due to a loop backedge targeting CtxBB), check
786 // instructions from CtxIter to the end of CtxBB (instructions before
787 // CtxIter were checked above). Otherwise, check the entire block.
788 auto StartIt = (CurBB == CtxBB) ? CtxIter : CurBB->begin();
789 if (!hasNoFreeInRange(Begin: StartIt, End: CurBB->end(), NumChecked))
790 return false;
791 append_range(C&: Worklist, R: predecessors(BB: CurBB));
792 }
793 return true;
794}
795
796// TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but
797// we still have enough information about `RHS` to conclude non-zero. For
798// example Pred=EQ, RHS=isKnownNonZero. cmpExcludesZero is called in loops
799// so the extra compile time may not be worth it, but possibly a second API
800// should be created for use outside of loops.
801static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) {
802 // v u> y implies v != 0.
803 if (Pred == ICmpInst::ICMP_UGT)
804 return true;
805
806 // Special-case v != 0 to also handle v != null.
807 if (Pred == ICmpInst::ICMP_NE)
808 return match(V: RHS, P: m_Zero());
809
810 // All other predicates - rely on generic ConstantRange handling.
811 const APInt *C;
812 auto Zero = APInt::getZero(numBits: RHS->getType()->getScalarSizeInBits());
813 if (match(V: RHS, P: m_APInt(Res&: C))) {
814 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, Other: *C);
815 return !TrueValues.contains(Val: Zero);
816 }
817
818 auto *VC = dyn_cast<ConstantDataVector>(Val: RHS);
819 if (VC == nullptr)
820 return false;
821
822 for (unsigned ElemIdx = 0, NElem = VC->getNumElements(); ElemIdx < NElem;
823 ++ElemIdx) {
824 ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(
825 Pred, Other: VC->getElementAsAPInt(i: ElemIdx));
826 if (TrueValues.contains(Val: Zero))
827 return false;
828 }
829 return true;
830}
831
832static void breakSelfRecursivePHI(const Use *U, const PHINode *PHI,
833 Value *&ValOut, Instruction *&CtxIOut,
834 const PHINode **PhiOut = nullptr) {
835 ValOut = U->get();
836 if (ValOut == PHI)
837 return;
838 CtxIOut = PHI->getIncomingBlock(U: *U)->getTerminator();
839 if (PhiOut)
840 *PhiOut = PHI;
841 Value *V;
842 // If the Use is a select of this phi, compute analysis on other arm to break
843 // recursion.
844 // TODO: Min/Max
845 if (match(V: ValOut, P: m_Select(C: m_Value(), L: m_Specific(V: PHI), R: m_Value(V))) ||
846 match(V: ValOut, P: m_Select(C: m_Value(), L: m_Value(V), R: m_Specific(V: PHI))))
847 ValOut = V;
848
849 // Same for select, if this phi is 2-operand phi, compute analysis on other
850 // incoming value to break recursion.
851 // TODO: We could handle any number of incoming edges as long as we only have
852 // two unique values.
853 if (auto *IncPhi = dyn_cast<PHINode>(Val: ValOut);
854 IncPhi && IncPhi->getNumIncomingValues() == 2) {
855 for (int Idx = 0; Idx < 2; ++Idx) {
856 if (IncPhi->getIncomingValue(i: Idx) == PHI) {
857 ValOut = IncPhi->getIncomingValue(i: 1 - Idx);
858 if (PhiOut)
859 *PhiOut = IncPhi;
860 CtxIOut = IncPhi->getIncomingBlock(i: 1 - Idx)->getTerminator();
861 break;
862 }
863 }
864 }
865}
866
867static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) {
868 // Use of assumptions is context-sensitive. If we don't have a context, we
869 // cannot use them!
870 if (!Q.AC || !Q.CtxI)
871 return false;
872
873 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
874 if (!Elem.Assume)
875 continue;
876
877 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
878 assert(I->getFunction() == Q.CtxI->getFunction() &&
879 "Got assumption for the wrong function!");
880
881 if (Elem.Index != AssumptionCache::ExprResultIdx) {
882 if (assumeBundleImpliesNonNull(Val: V, Context: Q.CtxI->getFunction(),
883 OBU: I->getOperandBundleAt(Index: Elem.Index)) &&
884 isValidAssumeForContext(I, Q))
885 return true;
886 continue;
887 }
888
889 // Warning: This loop can end up being somewhat performance sensitive.
890 // We're running this loop for once for each value queried resulting in a
891 // runtime of ~O(#assumes * #values).
892
893 if (match(V: I->getArgOperand(i: 0), P: m_Trunc(Op: m_Specific(V))) &&
894 isValidAssumeForContext(I, Q))
895 return true;
896
897 Value *RHS;
898 CmpPredicate Pred;
899 auto m_V = m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToInt(Op: m_Specific(V)));
900 if (!match(V: I->getArgOperand(i: 0), P: m_c_ICmp(Pred, L: m_V, R: m_Value(V&: RHS))))
901 continue;
902
903 if (cmpExcludesZero(Pred, RHS) && isValidAssumeForContext(I, Q))
904 return true;
905 }
906
907 return false;
908}
909
910static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred,
911 Value *LHS, Value *RHS, KnownBits &Known,
912 const SimplifyQuery &Q) {
913 if (RHS->getType()->isPointerTy()) {
914 // Handle comparison of pointer to null explicitly, as it will not be
915 // covered by the m_APInt() logic below.
916 if (LHS == V && match(V: RHS, P: m_Zero())) {
917 switch (Pred) {
918 case ICmpInst::ICMP_EQ:
919 Known.setAllZero();
920 break;
921 case ICmpInst::ICMP_SGE:
922 case ICmpInst::ICMP_SGT:
923 Known.makeNonNegative();
924 break;
925 case ICmpInst::ICMP_SLT:
926 Known.makeNegative();
927 break;
928 default:
929 break;
930 }
931 }
932 return;
933 }
934
935 unsigned BitWidth = Known.getBitWidth();
936 auto m_V =
937 m_CombineOr(Ps: m_Specific(V), Ps: m_PtrToIntSameSize(DL: Q.DL, Op: m_Specific(V)));
938
939 Value *Y;
940 const APInt *Mask, *C;
941 if (!match(V: RHS, P: m_APInt(Res&: C)))
942 return;
943
944 uint64_t ShAmt;
945 switch (Pred) {
946 case ICmpInst::ICMP_EQ:
947 // assume(V = C)
948 if (match(V: LHS, P: m_V)) {
949 Known = Known.unionWith(RHS: KnownBits::makeConstant(C: *C));
950 // assume(V & Mask = C)
951 } else if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value(V&: Y)))) {
952 // For one bits in Mask, we can propagate bits from C to V.
953 Known.One |= *C;
954 if (match(V: Y, P: m_APInt(Res&: Mask)))
955 Known.Zero |= ~*C & *Mask;
956 // assume(V | Mask = C)
957 } else if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value(V&: Y)))) {
958 // For zero bits in Mask, we can propagate bits from C to V.
959 Known.Zero |= ~*C;
960 if (match(V: Y, P: m_APInt(Res&: Mask)))
961 Known.One |= *C & ~*Mask;
962 // assume(V << ShAmt = C)
963 } else if (match(V: LHS, P: m_Shl(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
964 ShAmt < BitWidth) {
965 // For those bits in C that are known, we can propagate them to known
966 // bits in V shifted to the right by ShAmt.
967 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
968 RHSKnown >>= ShAmt;
969 Known = Known.unionWith(RHS: RHSKnown);
970 // assume(V >> ShAmt = C)
971 } else if (match(V: LHS, P: m_Shr(L: m_V, R: m_ConstantInt(V&: ShAmt))) &&
972 ShAmt < BitWidth) {
973 // For those bits in RHS that are known, we can propagate them to known
974 // bits in V shifted to the right by C.
975 KnownBits RHSKnown = KnownBits::makeConstant(C: *C);
976 RHSKnown <<= ShAmt;
977 Known = Known.unionWith(RHS: RHSKnown);
978 }
979 break;
980 case ICmpInst::ICMP_NE: {
981 // assume (V & B != 0) where B is a power of 2
982 const APInt *BPow2;
983 if (C->isZero() && match(V: LHS, P: m_And(L: m_V, R: m_Power2(V&: BPow2))))
984 Known.One |= *BPow2;
985 break;
986 }
987 default: {
988 const APInt *Offset = nullptr;
989 if (match(V: LHS, P: m_CombineOr(Ps: m_V, Ps: m_AddLike(L: m_V, R: m_APInt(Res&: Offset))))) {
990 ConstantRange LHSRange = ConstantRange::makeAllowedICmpRegion(Pred, Other: *C);
991 if (Offset)
992 LHSRange = LHSRange.sub(Other: *Offset);
993 Known = Known.unionWith(RHS: LHSRange.toKnownBits());
994 }
995 if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
996 // X & Y u> C -> X u> C && Y u> C
997 // X nuw- Y u> C -> X u> C
998 if (match(V: LHS, P: m_c_And(L: m_V, R: m_Value())) ||
999 match(V: LHS, P: m_NUWSub(L: m_V, R: m_Value())))
1000 Known.One.setHighBits(
1001 (*C + (Pred == ICmpInst::ICMP_UGT)).countLeadingOnes());
1002 }
1003 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE) {
1004 // X | Y u< C -> X u< C && Y u< C
1005 // X nuw+ Y u< C -> X u< C && Y u< C
1006 if (match(V: LHS, P: m_c_Or(L: m_V, R: m_Value())) ||
1007 match(V: LHS, P: m_c_NUWAdd(L: m_V, R: m_Value()))) {
1008 Known.Zero.setHighBits(
1009 (*C - (Pred == ICmpInst::ICMP_ULT)).countLeadingZeros());
1010 }
1011 }
1012 } break;
1013 }
1014}
1015
1016static void computeKnownBitsFromICmpCond(const Value *V, ICmpInst *Cmp,
1017 KnownBits &Known,
1018 const SimplifyQuery &SQ, bool Invert) {
1019 ICmpInst::Predicate Pred =
1020 Invert ? Cmp->getInversePredicate() : Cmp->getPredicate();
1021 Value *LHS = Cmp->getOperand(i_nocapture: 0);
1022 Value *RHS = Cmp->getOperand(i_nocapture: 1);
1023
1024 // Handle icmp pred (trunc V), C
1025 if (match(V: LHS, P: m_Trunc(Op: m_Specific(V)))) {
1026 KnownBits DstKnown(LHS->getType()->getScalarSizeInBits());
1027 computeKnownBitsFromCmp(V: LHS, Pred, LHS, RHS, Known&: DstKnown, Q: SQ);
1028 if (cast<TruncInst>(Val: LHS)->hasNoUnsignedWrap())
1029 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1030 else
1031 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1032 return;
1033 }
1034
1035 computeKnownBitsFromCmp(V, Pred, LHS, RHS, Known, Q: SQ);
1036}
1037
1038static void computeKnownBitsFromCond(const Value *V, Value *Cond,
1039 KnownBits &Known, const SimplifyQuery &SQ,
1040 bool Invert, unsigned Depth) {
1041 Value *A, *B;
1042 if (Depth < MaxAnalysisRecursionDepth &&
1043 match(V: Cond, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
1044 KnownBits Known2(Known.getBitWidth());
1045 KnownBits Known3(Known.getBitWidth());
1046 computeKnownBitsFromCond(V, Cond: A, Known&: Known2, SQ, Invert, Depth: Depth + 1);
1047 computeKnownBitsFromCond(V, Cond: B, Known&: Known3, SQ, Invert, Depth: Depth + 1);
1048 if (Invert ? match(V: Cond, P: m_LogicalOr(L: m_Value(), R: m_Value()))
1049 : match(V: Cond, P: m_LogicalAnd(L: m_Value(), R: m_Value())))
1050 Known2 = Known2.unionWith(RHS: Known3);
1051 else
1052 Known2 = Known2.intersectWith(RHS: Known3);
1053 Known = Known.unionWith(RHS: Known2);
1054 return;
1055 }
1056
1057 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
1058 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ, Invert);
1059 return;
1060 }
1061
1062 if (match(V: Cond, P: m_Trunc(Op: m_Specific(V)))) {
1063 KnownBits DstKnown(1);
1064 if (Invert) {
1065 DstKnown.setAllZero();
1066 } else {
1067 DstKnown.setAllOnes();
1068 }
1069 if (cast<TruncInst>(Val: Cond)->hasNoUnsignedWrap()) {
1070 Known = Known.unionWith(RHS: DstKnown.zext(BitWidth: Known.getBitWidth()));
1071 return;
1072 }
1073 Known = Known.unionWith(RHS: DstKnown.anyext(BitWidth: Known.getBitWidth()));
1074 return;
1075 }
1076
1077 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A))))
1078 computeKnownBitsFromCond(V, Cond: A, Known, SQ, Invert: !Invert, Depth: Depth + 1);
1079}
1080
1081void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known,
1082 const SimplifyQuery &Q, unsigned Depth) {
1083 // Handle injected condition.
1084 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
1085 computeKnownBitsFromCond(V, Cond: Q.CC->Cond, Known, SQ: Q, Invert: Q.CC->Invert, Depth);
1086
1087 if (!Q.CtxI)
1088 return;
1089
1090 if (Q.DC && Q.DT) {
1091 // Handle dominating conditions.
1092 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
1093 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
1094 if (Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()))
1095 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1096 /*Invert*/ false, Depth);
1097
1098 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
1099 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()))
1100 computeKnownBitsFromCond(V, Cond: BI->getCondition(), Known, SQ: Q,
1101 /*Invert*/ true, Depth);
1102 }
1103
1104 if (Known.hasConflict())
1105 Known.resetAll();
1106 }
1107
1108 if (!Q.AC)
1109 return;
1110
1111 unsigned BitWidth = Known.getBitWidth();
1112
1113 // Note that the patterns below need to be kept in sync with the code
1114 // in AssumptionCache::updateAffectedValues.
1115
1116 for (AssumptionCache::ResultElem &Elem : Q.AC->assumptionsFor(V)) {
1117 if (!Elem.Assume)
1118 continue;
1119
1120 AssumeInst *I = cast<AssumeInst>(Val&: Elem.Assume);
1121 assert(I->getParent()->getParent() == Q.CtxI->getParent()->getParent() &&
1122 "Got assumption for the wrong function!");
1123
1124 if (Elem.Index != AssumptionCache::ExprResultIdx) {
1125 if (auto OBU = I->getOperandBundleAt(Index: Elem.Index);
1126 getBundleAttrFromOBU(OBU) == BundleAttr::Align) {
1127 auto [Ptr, _, _2, Alignment, Offset] = getAssumeAlignInfo(OBU);
1128 if (Ptr == V && Alignment && Offset && isPowerOf2_64(Value: *Alignment) &&
1129 isValidAssumeForContext(I, Q)) {
1130 Known.Zero |= (*Alignment - 1) & ~*Offset;
1131 Known.One |= (*Alignment - 1) & *Offset;
1132 }
1133 }
1134 continue;
1135 }
1136
1137 // Warning: This loop can end up being somewhat performance sensitive.
1138 // We're running this loop for once for each value queried resulting in a
1139 // runtime of ~O(#assumes * #values).
1140
1141 Value *Arg = I->getArgOperand(i: 0);
1142
1143 if (Arg == V && isValidAssumeForContext(I, Q)) {
1144 assert(BitWidth == 1 && "assume operand is not i1?");
1145 (void)BitWidth;
1146 Known.setAllOnes();
1147 return;
1148 }
1149 if (match(V: Arg, P: m_Not(V: m_Specific(V))) &&
1150 isValidAssumeForContext(I, Q)) {
1151 assert(BitWidth == 1 && "assume operand is not i1?");
1152 (void)BitWidth;
1153 Known.setAllZero();
1154 return;
1155 }
1156 auto *Trunc = dyn_cast<TruncInst>(Val: Arg);
1157 if (Trunc && Trunc->getOperand(i_nocapture: 0) == V &&
1158 isValidAssumeForContext(I, Q)) {
1159 if (Trunc->hasNoUnsignedWrap()) {
1160 Known = KnownBits::makeConstant(C: APInt(BitWidth, 1));
1161 return;
1162 }
1163 Known.One.setBit(0);
1164 return;
1165 }
1166
1167 // The remaining tests are all recursive, so bail out if we hit the limit.
1168 if (Depth == MaxAnalysisRecursionDepth)
1169 continue;
1170
1171 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
1172 if (!Cmp)
1173 continue;
1174
1175 if (!isValidAssumeForContext(I, Q))
1176 continue;
1177
1178 computeKnownBitsFromICmpCond(V, Cmp, Known, SQ: Q, /*Invert=*/false);
1179 }
1180
1181 // Conflicting assumption: Undefined behavior will occur on this execution
1182 // path.
1183 if (Known.hasConflict())
1184 Known.resetAll();
1185}
1186
1187/// Compute known bits from a shift operator, including those with a
1188/// non-constant shift amount. Known is the output of this function. Known2 is a
1189/// pre-allocated temporary with the same bit width as Known and on return
1190/// contains the known bit of the shift value source. KF is an
1191/// operator-specific function that, given the known-bits and a shift amount,
1192/// compute the implied known-bits of the shift operator's result respectively
1193/// for that shift amount. The results from calling KF are conservatively
1194/// combined for all permitted shift amounts.
1195static void computeKnownBitsFromShiftOperator(
1196 const Operator *I, const APInt &DemandedElts, KnownBits &Known,
1197 KnownBits &Known2, const SimplifyQuery &Q, unsigned Depth,
1198 function_ref<KnownBits(const KnownBits &, const KnownBits &, bool)> KF) {
1199 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1200 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1201 // To limit compile-time impact, only query isKnownNonZero() if we know at
1202 // least something about the shift amount.
1203 bool ShAmtNonZero =
1204 Known.isNonZero() ||
1205 (Known.getMaxValue().ult(RHS: Known.getBitWidth()) &&
1206 isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1));
1207 Known = KF(Known2, Known, ShAmtNonZero);
1208}
1209
1210static KnownBits
1211getKnownBitsFromAndXorOr(const Operator *I, const APInt &DemandedElts,
1212 const KnownBits &KnownLHS, const KnownBits &KnownRHS,
1213 const SimplifyQuery &Q, unsigned Depth) {
1214 unsigned BitWidth = KnownLHS.getBitWidth();
1215 KnownBits KnownOut(BitWidth);
1216 bool IsAnd = false;
1217 bool HasKnownOne = !KnownLHS.One.isZero() || !KnownRHS.One.isZero();
1218 Value *X = nullptr, *Y = nullptr;
1219
1220 switch (I->getOpcode()) {
1221 case Instruction::And:
1222 KnownOut = KnownLHS & KnownRHS;
1223 IsAnd = true;
1224 // and(x, -x) is common idioms that will clear all but lowest set
1225 // bit. If we have a single known bit in x, we can clear all bits
1226 // above it.
1227 // TODO: instcombine often reassociates independent `and` which can hide
1228 // this pattern. Try to match and(x, and(-x, y)) / and(and(x, y), -x).
1229 if (HasKnownOne && match(V: I, P: m_c_And(L: m_Value(V&: X), R: m_Neg(V: m_Deferred(V: X))))) {
1230 // -(-x) == x so using whichever (LHS/RHS) gets us a better result.
1231 if (KnownLHS.countMaxTrailingZeros() <= KnownRHS.countMaxTrailingZeros())
1232 KnownOut = KnownLHS.blsi();
1233 else
1234 KnownOut = KnownRHS.blsi();
1235 }
1236 break;
1237 case Instruction::Or:
1238 KnownOut = KnownLHS | KnownRHS;
1239 break;
1240 case Instruction::Xor:
1241 KnownOut = KnownLHS ^ KnownRHS;
1242 // xor(x, x-1) is common idioms that will clear all but lowest set
1243 // bit. If we have a single known bit in x, we can clear all bits
1244 // above it.
1245 // TODO: xor(x, x-1) is often rewritting as xor(x, x-C) where C !=
1246 // -1 but for the purpose of demanded bits (xor(x, x-C) &
1247 // Demanded) == (xor(x, x-1) & Demanded). Extend the xor pattern
1248 // to use arbitrary C if xor(x, x-C) as the same as xor(x, x-1).
1249 if (HasKnownOne &&
1250 match(V: I, P: m_c_Xor(L: m_Value(V&: X), R: m_Add(L: m_Deferred(V: X), R: m_AllOnes())))) {
1251 const KnownBits &XBits = I->getOperand(i: 0) == X ? KnownLHS : KnownRHS;
1252 KnownOut = XBits.blsmsk();
1253 }
1254 break;
1255 default:
1256 llvm_unreachable("Invalid Op used in 'analyzeKnownBitsFromAndXorOr'");
1257 }
1258
1259 // and(x, add (x, -1)) is a common idiom that always clears the low bit;
1260 // xor/or(x, add (x, -1)) is an idiom that will always set the low bit.
1261 // here we handle the more general case of adding any odd number by
1262 // matching the form and/xor/or(x, add(x, y)) where y is odd.
1263 // TODO: This could be generalized to clearing any bit set in y where the
1264 // following bit is known to be unset in y.
1265 if (!KnownOut.Zero[0] && !KnownOut.One[0] &&
1266 (match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_c_Add(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1267 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Deferred(V: X), R: m_Value(V&: Y)))) ||
1268 match(V: I, P: m_c_BinOp(L: m_Value(V&: X), R: m_Sub(L: m_Value(V&: Y), R: m_Deferred(V: X)))))) {
1269 KnownBits KnownY(BitWidth);
1270 computeKnownBits(V: Y, DemandedElts, Known&: KnownY, Q, Depth: Depth + 1);
1271 if (KnownY.countMinTrailingOnes() > 0) {
1272 if (IsAnd)
1273 KnownOut.Zero.setBit(0);
1274 else
1275 KnownOut.One.setBit(0);
1276 }
1277 }
1278 return KnownOut;
1279}
1280
1281static KnownBits computeKnownBitsForHorizontalOperation(
1282 const Operator *I, const APInt &DemandedElts, const SimplifyQuery &Q,
1283 unsigned Depth,
1284 const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
1285 KnownBitsFunc) {
1286 APInt DemandedEltsLHS, DemandedEltsRHS;
1287 getHorizDemandedEltsForFirstOperand(VectorBitWidth: Q.DL.getTypeSizeInBits(Ty: I->getType()),
1288 DemandedElts, DemandedLHS&: DemandedEltsLHS,
1289 DemandedRHS&: DemandedEltsRHS);
1290
1291 const auto ComputeForSingleOpFunc =
1292 [Depth, &Q, KnownBitsFunc](const Value *Op, APInt &DemandedEltsOp) {
1293 return KnownBitsFunc(
1294 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp, Q, Depth: Depth + 1),
1295 computeKnownBits(V: Op, DemandedElts: DemandedEltsOp << 1, Q, Depth: Depth + 1));
1296 };
1297
1298 if (DemandedEltsRHS.isZero())
1299 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS);
1300 if (DemandedEltsLHS.isZero())
1301 return ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS);
1302
1303 return ComputeForSingleOpFunc(I->getOperand(i: 0), DemandedEltsLHS)
1304 .intersectWith(RHS: ComputeForSingleOpFunc(I->getOperand(i: 1), DemandedEltsRHS));
1305}
1306
1307// Public so this can be used in `SimplifyDemandedUseBits`.
1308KnownBits llvm::analyzeKnownBitsFromAndXorOr(const Operator *I,
1309 const KnownBits &KnownLHS,
1310 const KnownBits &KnownRHS,
1311 const SimplifyQuery &SQ,
1312 unsigned Depth) {
1313 auto *FVTy = dyn_cast<FixedVectorType>(Val: I->getType());
1314 APInt DemandedElts =
1315 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
1316
1317 return getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS, KnownRHS, Q: SQ,
1318 Depth);
1319}
1320
1321ConstantRange llvm::getVScaleRange(const Function *F, unsigned BitWidth) {
1322 Attribute Attr = F->getFnAttribute(Kind: Attribute::VScaleRange);
1323 // Without vscale_range, we only know that vscale is non-zero.
1324 if (!Attr.isValid())
1325 return ConstantRange(APInt(BitWidth, 1), APInt::getZero(numBits: BitWidth));
1326
1327 unsigned AttrMin = Attr.getVScaleRangeMin();
1328 // Minimum is larger than vscale width, result is always poison.
1329 if ((unsigned)llvm::bit_width(Value: AttrMin) > BitWidth)
1330 return ConstantRange::getEmpty(BitWidth);
1331
1332 APInt Min(BitWidth, AttrMin);
1333 std::optional<unsigned> AttrMax = Attr.getVScaleRangeMax();
1334 if (!AttrMax || (unsigned)llvm::bit_width(Value: *AttrMax) > BitWidth)
1335 return ConstantRange(Min, APInt::getZero(numBits: BitWidth));
1336
1337 return ConstantRange(Min, APInt(BitWidth, *AttrMax) + 1);
1338}
1339
1340/// Return true if \p II reads a register named "vlenb". On RISC-V this is the
1341/// VLENB CSR, which holds VLEN/8: a non-zero power of two bounded by the
1342/// target's VLEN range. Callers must ensure the target is RISC-V.
1343static bool isReadVLENB(const IntrinsicInst &II) {
1344 auto *MAV = dyn_cast<MetadataAsValue>(Val: II.getArgOperand(i: 0));
1345 if (!MAV)
1346 return false;
1347 auto *MD = dyn_cast<MDNode>(Val: MAV->getMetadata());
1348 if (!MD || MD->getNumOperands() != 1)
1349 return false;
1350 auto *RegName = dyn_cast<MDString>(Val: MD->getOperand(I: 0));
1351 return RegName && RegName->getString() == "vlenb";
1352}
1353
1354/// Return the value range of a RISC-V vlenb CSR read. RVV requires VLEN to be a
1355/// power of two in [32, 65536] (Zvl32b is the smallest vector extension), so
1356/// VLENB = VLEN/8 is in [4, 8192]. This architectural bound is independent of
1357/// any function attribute and stays sound for Zvl32b, whose VLEN (32) is not
1358/// representable as an integer vscale (VLEN / RVVBitsPerBlock). A vscale_range
1359/// attribute, when present, pins the subtarget's VLEN in units of
1360/// RVVBitsPerBlock (64 bits) and so gives a tighter VLENB = vscale *
1361/// RVVBytesPerBlock.
1362static ConstantRange getRISCVVLENBRange(const IntrinsicInst &II,
1363 unsigned Width) {
1364 // Architectural bounds: VLEN in [32, 65536] => VLENB in [4, 8192].
1365 ConstantRange Range(APInt(Width, 32 / 8), APInt(Width, 65536 / 8) + 1);
1366
1367 const Function *F = II.getFunction();
1368 if (F->getFnAttribute(Kind: Attribute::VScaleRange).isValid()) {
1369 ConstantRange VScale = getVScaleRange(F, BitWidth: Width);
1370 Range = Range.intersectWith(
1371 CR: VScale.multiply(Other: ConstantRange(APInt(Width, RISCV::RVVBytesPerBlock))));
1372 }
1373 return Range;
1374}
1375
1376void llvm::adjustKnownBitsForSelectArm(KnownBits &Known, Value *Cond,
1377 Value *Arm, bool Invert,
1378 const SimplifyQuery &Q, unsigned Depth) {
1379 // If we have a constant arm, we are done.
1380 if (Known.isConstant())
1381 return;
1382
1383 // See what condition implies about the bits of the select arm.
1384 KnownBits CondRes(Known.getBitWidth());
1385 computeKnownBitsFromCond(V: Arm, Cond, Known&: CondRes, SQ: Q, Invert, Depth: Depth + 1);
1386 // If we don't get any information from the condition, no reason to
1387 // proceed.
1388 if (CondRes.isUnknown())
1389 return;
1390
1391 // We can have conflict if the condition is dead. I.e if we have
1392 // (x | 64) < 32 ? (x | 64) : y
1393 // we will have conflict at bit 6 from the condition/the `or`.
1394 // In that case just return. Its not particularly important
1395 // what we do, as this select is going to be simplified soon.
1396 CondRes = CondRes.unionWith(RHS: Known);
1397 if (CondRes.hasConflict())
1398 return;
1399
1400 // Finally make sure the information we found is valid. This is relatively
1401 // expensive so it's left for the very end.
1402 if (!isGuaranteedNotToBeUndef(V: Arm, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1))
1403 return;
1404
1405 // Finally, we know we get information from the condition and its valid,
1406 // so return it.
1407 Known = std::move(CondRes);
1408}
1409
1410// Match a signed min+max clamp pattern like smax(smin(In, CHigh), CLow).
1411// Returns the input and lower/upper bounds.
1412static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
1413 const APInt *&CLow, const APInt *&CHigh) {
1414 assert(isa<Operator>(Select) &&
1415 cast<Operator>(Select)->getOpcode() == Instruction::Select &&
1416 "Input should be a Select!");
1417
1418 const Value *LHS = nullptr, *RHS = nullptr;
1419 SelectPatternFlavor SPF = matchSelectPattern(V: Select, LHS, RHS).Flavor;
1420 if (SPF != SPF_SMAX && SPF != SPF_SMIN)
1421 return false;
1422
1423 if (!match(V: RHS, P: m_APInt(Res&: CLow)))
1424 return false;
1425
1426 const Value *LHS2 = nullptr, *RHS2 = nullptr;
1427 SelectPatternFlavor SPF2 = matchSelectPattern(V: LHS, LHS&: LHS2, RHS&: RHS2).Flavor;
1428 if (getInverseMinMaxFlavor(SPF) != SPF2)
1429 return false;
1430
1431 if (!match(V: RHS2, P: m_APInt(Res&: CHigh)))
1432 return false;
1433
1434 if (SPF == SPF_SMIN)
1435 std::swap(a&: CLow, b&: CHigh);
1436
1437 In = LHS2;
1438 return CLow->sle(RHS: *CHigh);
1439}
1440
1441static bool isSignedMinMaxIntrinsicClamp(const IntrinsicInst *II,
1442 const APInt *&CLow,
1443 const APInt *&CHigh) {
1444 assert((II->getIntrinsicID() == Intrinsic::smin ||
1445 II->getIntrinsicID() == Intrinsic::smax) &&
1446 "Must be smin/smax");
1447
1448 Intrinsic::ID InverseID = getInverseMinMaxIntrinsic(MinMaxID: II->getIntrinsicID());
1449 auto *InnerII = dyn_cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
1450 if (!InnerII || InnerII->getIntrinsicID() != InverseID ||
1451 !match(V: II->getArgOperand(i: 1), P: m_APInt(Res&: CLow)) ||
1452 !match(V: InnerII->getArgOperand(i: 1), P: m_APInt(Res&: CHigh)))
1453 return false;
1454
1455 if (II->getIntrinsicID() == Intrinsic::smin)
1456 std::swap(a&: CLow, b&: CHigh);
1457 return CLow->sle(RHS: *CHigh);
1458}
1459
1460static void unionWithMinMaxIntrinsicClamp(const IntrinsicInst *II,
1461 KnownBits &Known) {
1462 const APInt *CLow, *CHigh;
1463 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
1464 Known = Known.unionWith(
1465 RHS: ConstantRange::getNonEmpty(Lower: *CLow, Upper: *CHigh + 1).toKnownBits());
1466}
1467
1468static void computeKnownBitsForRecurrenceOperands(
1469 const PHINode *P, Value *Start, Value *Step, const APInt &DemandedElts,
1470 KnownBits &KnownStart, KnownBits &KnownStep, const SimplifyQuery &Q,
1471 unsigned Depth) {
1472 // Change the context instruction to the "edge" that flows into the phi. This
1473 // is important because that is where the value is actually "evaluated" even
1474 // though it is used later somewhere else. (see also D69571).
1475 SimplifyQuery RecQ = Q.getWithoutCondContext();
1476 unsigned OpNum = P->getOperand(i_nocapture: 0) == Start ? 0 : 1;
1477
1478 RecQ.CtxI = P->getIncomingBlock(i: OpNum)->getTerminator();
1479 computeKnownBits(V: Start, DemandedElts, Known&: KnownStart, Q: RecQ, Depth: Depth + 1);
1480
1481 RecQ.CtxI = P->getIncomingBlock(i: 1 - OpNum)->getTerminator();
1482 computeKnownBits(V: Step, DemandedElts, Known&: KnownStep, Q: RecQ, Depth: Depth + 1);
1483}
1484
1485static void computeKnownBitsFromOperator(const Operator *I,
1486 const APInt &DemandedElts,
1487 KnownBits &Known,
1488 const SimplifyQuery &Q,
1489 unsigned Depth) {
1490 unsigned BitWidth = Known.getBitWidth();
1491
1492 KnownBits Known2(BitWidth);
1493 switch (I->getOpcode()) {
1494 default: break;
1495 case Instruction::Load:
1496 if (MDNode *MD =
1497 Q.IIQ.getMetadata(I: cast<LoadInst>(Val: I), KindID: LLVMContext::MD_range))
1498 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
1499 break;
1500 case Instruction::And:
1501 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1502 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1503
1504 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1505 break;
1506 case Instruction::Or:
1507 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1508 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1509
1510 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1511 break;
1512 case Instruction::Xor:
1513 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
1514 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1515
1516 Known = getKnownBitsFromAndXorOr(I, DemandedElts, KnownLHS: Known2, KnownRHS: Known, Q, Depth);
1517 break;
1518 case Instruction::Mul: {
1519 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1520 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1521 computeKnownBitsMul(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1522 DemandedElts, Known, Known2, Q, Depth);
1523 break;
1524 }
1525 case Instruction::UDiv: {
1526 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1527 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1528 Known =
1529 KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1530 break;
1531 }
1532 case Instruction::SDiv: {
1533 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1534 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1535 Known =
1536 KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)));
1537 break;
1538 }
1539 case Instruction::Select: {
1540 auto ComputeForArm = [&](Value *Arm, bool Invert) {
1541 KnownBits Res(Known.getBitWidth());
1542 computeKnownBits(V: Arm, DemandedElts, Known&: Res, Q, Depth: Depth + 1);
1543 adjustKnownBitsForSelectArm(Known&: Res, Cond: I->getOperand(i: 0), Arm, Invert, Q, Depth);
1544 return Res;
1545 };
1546 // Only known if known in both the LHS and RHS.
1547 Known =
1548 ComputeForArm(I->getOperand(i: 1), /*Invert=*/false)
1549 .intersectWith(RHS: ComputeForArm(I->getOperand(i: 2), /*Invert=*/true));
1550 break;
1551 }
1552 case Instruction::FPToSI: {
1553 // fptosi is poison if the rounded value doesn't fit in the result type,
1554 // so we can assume the conversion is well-defined and rounds towards
1555 // zero. +-Inf can never fit in an integer type, so it is always poison,
1556 // like NaN. Negative subnormals and negative zero round to 0. That
1557 // leaves negative normals as the only class that can produce a defined
1558 // negative result.
1559 KnownFPClass SrcFPClass = computeKnownFPClass(
1560 V: I->getOperand(i: 0), DemandedElts, InterestedClasses: fcNegNormal, SQ: Q, Depth: Depth + 1);
1561 if (SrcFPClass.isKnownNever(Mask: fcNegNormal))
1562 Known.makeNonNegative();
1563 break;
1564 }
1565 case Instruction::FPTrunc:
1566 case Instruction::FPExt:
1567 case Instruction::FPToUI:
1568 case Instruction::SIToFP:
1569 case Instruction::UIToFP:
1570 break; // Can't work with floating point.
1571 case Instruction::PtrToInt:
1572 case Instruction::PtrToAddr:
1573 case Instruction::IntToPtr:
1574 // Fall through and handle them the same as zext/trunc.
1575 [[fallthrough]];
1576 case Instruction::ZExt:
1577 case Instruction::Trunc: {
1578 Type *SrcTy = I->getOperand(i: 0)->getType();
1579
1580 unsigned SrcBitWidth;
1581 // Note that we handle pointer operands here because of inttoptr/ptrtoint
1582 // which fall through here.
1583 Type *ScalarTy = SrcTy->getScalarType();
1584 SrcBitWidth = ScalarTy->isPointerTy() ?
1585 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
1586 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
1587
1588 assert(SrcBitWidth && "SrcBitWidth can't be zero");
1589 Known = Known.anyextOrTrunc(BitWidth: SrcBitWidth);
1590 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1591 if (auto *Inst = dyn_cast<PossiblyNonNegInst>(Val: I);
1592 Inst && Inst->hasNonNeg() && !Known.isNegative())
1593 Known.makeNonNegative();
1594 Known = Known.zextOrTrunc(BitWidth);
1595 break;
1596 }
1597 case Instruction::BitCast: {
1598 Type *SrcTy = I->getOperand(i: 0)->getType();
1599 if (SrcTy->isIntOrPtrTy() &&
1600 // TODO: For now, not handling conversions like:
1601 // (bitcast i64 %x to <2 x i32>)
1602 !I->getType()->isVectorTy()) {
1603 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1604 break;
1605 }
1606
1607 const Value *V;
1608 // Handle bitcast from floating point to integer.
1609 if (match(V: I, P: m_ElementWiseBitCast(Op: m_Value(V))) &&
1610 V->getType()->isFPOrFPVectorTy()) {
1611 Type *FPType = V->getType()->getScalarType();
1612 KnownFPClass Result =
1613 computeKnownFPClass(V, DemandedElts, InterestedClasses: fcAllFlags, SQ: Q, Depth: Depth + 1);
1614
1615 Known = Result.toKnownBits(FltSemantics: FPType->getFltSemantics());
1616
1617 break;
1618 }
1619
1620 // Handle cast from vector integer type to scalar or vector integer.
1621 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy);
1622 if (!SrcVecTy || !SrcVecTy->getElementType()->isIntegerTy() ||
1623 !I->getType()->isIntOrIntVectorTy() ||
1624 isa<ScalableVectorType>(Val: I->getType()))
1625 break;
1626
1627 unsigned NumElts = DemandedElts.getBitWidth();
1628 bool IsLE = Q.DL.isLittleEndian();
1629 // Look through a cast from narrow vector elements to wider type.
1630 // Examples: v4i32 -> v2i64, v3i8 -> v24
1631 unsigned SubBitWidth = SrcVecTy->getScalarSizeInBits();
1632 if (BitWidth % SubBitWidth == 0) {
1633 // Known bits are automatically intersected across demanded elements of a
1634 // vector. So for example, if a bit is computed as known zero, it must be
1635 // zero across all demanded elements of the vector.
1636 //
1637 // For this bitcast, each demanded element of the output is sub-divided
1638 // across a set of smaller vector elements in the source vector. To get
1639 // the known bits for an entire element of the output, compute the known
1640 // bits for each sub-element sequentially. This is done by shifting the
1641 // one-set-bit demanded elements parameter across the sub-elements for
1642 // consecutive calls to computeKnownBits. We are using the demanded
1643 // elements parameter as a mask operator.
1644 //
1645 // The known bits of each sub-element are then inserted into place
1646 // (dependent on endian) to form the full result of known bits.
1647 unsigned SubScale = BitWidth / SubBitWidth;
1648 APInt SubDemandedElts = APInt::getZero(numBits: NumElts * SubScale);
1649 for (unsigned i = 0; i != NumElts; ++i) {
1650 if (DemandedElts[i])
1651 SubDemandedElts.setBit(i * SubScale);
1652 }
1653
1654 KnownBits KnownSrc(SubBitWidth);
1655 for (unsigned i = 0; i != SubScale; ++i) {
1656 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts.shl(shiftAmt: i), Known&: KnownSrc, Q,
1657 Depth: Depth + 1);
1658 unsigned ShiftElt = IsLE ? i : SubScale - 1 - i;
1659 Known.insertBits(SubBits: KnownSrc, BitPosition: ShiftElt * SubBitWidth);
1660 }
1661 }
1662 // Look through a cast from wider vector elements to narrow type.
1663 // Examples: v2i64 -> v4i32
1664 if (SubBitWidth % BitWidth == 0) {
1665 unsigned SubScale = SubBitWidth / BitWidth;
1666 KnownBits KnownSrc(SubBitWidth);
1667 APInt SubDemandedElts =
1668 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
1669 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: SubDemandedElts, Known&: KnownSrc, Q,
1670 Depth: Depth + 1);
1671
1672 Known.setAllConflict();
1673 for (unsigned i = 0; i != NumElts; ++i) {
1674 if (DemandedElts[i]) {
1675 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
1676 unsigned Offset = (Shifts % SubScale) * BitWidth;
1677 Known = Known.intersectWith(RHS: KnownSrc.extractBits(NumBits: BitWidth, BitPosition: Offset));
1678 if (Known.isUnknown())
1679 break;
1680 }
1681 }
1682 }
1683 break;
1684 }
1685 case Instruction::SExt: {
1686 // Compute the bits in the result that are not present in the input.
1687 unsigned SrcBitWidth = I->getOperand(i: 0)->getType()->getScalarSizeInBits();
1688
1689 Known = Known.trunc(BitWidth: SrcBitWidth);
1690 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1691 // If the sign bit of the input is known set or clear, then we know the
1692 // top bits of the result.
1693 Known = Known.sext(BitWidth);
1694 break;
1695 }
1696 case Instruction::Shl: {
1697 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1698 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1699 auto KF = [NUW, NSW](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1700 bool ShAmtNonZero) {
1701 return KnownBits::shl(LHS: KnownVal, RHS: KnownAmt, NUW, NSW, ShAmtNonZero);
1702 };
1703 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1704 KF);
1705 // Trailing zeros of a right-shifted constant never decrease.
1706 const APInt *C;
1707 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1708 Known.Zero.setLowBits(C->countr_zero());
1709
1710 // shl X, sub(Y, xor(ctlz(X, true), BitWidth-1)) shifts X so that its MSB
1711 // lands at bit Y, when BitWidth is a power of 2.
1712 const APInt *YC;
1713 Value *X = I->getOperand(i: 0);
1714 if (isPowerOf2_32(Value: BitWidth) &&
1715 match(V: I->getOperand(i: 1),
1716 P: m_Sub(L: m_APInt(Res&: YC), R: m_Xor(L: m_Ctlz(Op0: m_Specific(V: X), Op1: m_One()),
1717 R: m_SpecificInt(V: BitWidth - 1)))) &&
1718 YC->ult(RHS: BitWidth - 1)) {
1719 unsigned Y = YC->getZExtValue();
1720 Known.One.setBit(Y);
1721 Known.Zero.setBitsFrom(Y + 1);
1722 }
1723 break;
1724 }
1725 case Instruction::LShr: {
1726 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1727 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1728 bool ShAmtNonZero) {
1729 return KnownBits::lshr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1730 };
1731 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1732 KF);
1733 // Leading zeros of a left-shifted constant never decrease.
1734 const APInt *C;
1735 if (match(V: I->getOperand(i: 0), P: m_APInt(Res&: C)))
1736 Known.Zero.setHighBits(C->countl_zero());
1737 break;
1738 }
1739 case Instruction::AShr: {
1740 bool Exact = Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I));
1741 auto KF = [Exact](const KnownBits &KnownVal, const KnownBits &KnownAmt,
1742 bool ShAmtNonZero) {
1743 return KnownBits::ashr(LHS: KnownVal, RHS: KnownAmt, ShAmtNonZero, Exact);
1744 };
1745 computeKnownBitsFromShiftOperator(I, DemandedElts, Known, Known2, Q, Depth,
1746 KF);
1747 break;
1748 }
1749 case Instruction::Sub: {
1750 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1751 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1752 computeKnownBitsAddSub(Add: false, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1753 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1754 break;
1755 }
1756 case Instruction::Add: {
1757 bool NSW = Q.IIQ.hasNoSignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1758 bool NUW = Q.IIQ.hasNoUnsignedWrap(Op: cast<OverflowingBinaryOperator>(Val: I));
1759 computeKnownBitsAddSub(Add: true, Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1), NSW, NUW,
1760 DemandedElts, KnownOut&: Known, Known2, Q, Depth);
1761 break;
1762 }
1763 case Instruction::SRem:
1764 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1765 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1766 Known = KnownBits::srem(LHS: Known, RHS: Known2);
1767 break;
1768
1769 case Instruction::URem:
1770 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
1771 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
1772 Known = KnownBits::urem(LHS: Known, RHS: Known2);
1773 break;
1774 case Instruction::Alloca:
1775 Known.Zero.setLowBits(Log2(A: cast<AllocaInst>(Val: I)->getAlign()));
1776 break;
1777 case Instruction::GetElementPtr: {
1778 // Analyze all of the subscripts of this getelementptr instruction
1779 // to determine if we can prove known low zero bits.
1780 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
1781 // Accumulate the constant indices in a separate variable
1782 // to minimize the number of calls to computeForAddSub.
1783 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: I->getType());
1784 APInt AccConstIndices(IndexWidth, 0);
1785
1786 auto AddIndexToKnown = [&](KnownBits IndexBits) {
1787 if (IndexWidth == BitWidth) {
1788 // Note that inbounds does *not* guarantee nsw for the addition, as only
1789 // the offset is signed, while the base address is unsigned.
1790 Known = KnownBits::add(LHS: Known, RHS: IndexBits);
1791 } else {
1792 // If the index width is smaller than the pointer width, only add the
1793 // value to the low bits.
1794 assert(IndexWidth < BitWidth &&
1795 "Index width can't be larger than pointer width");
1796 Known.insertBits(SubBits: KnownBits::add(LHS: Known.trunc(BitWidth: IndexWidth), RHS: IndexBits), BitPosition: 0);
1797 }
1798 };
1799
1800 gep_type_iterator GTI = gep_type_begin(GEP: I);
1801 for (unsigned i = 1, e = I->getNumOperands(); i != e; ++i, ++GTI) {
1802 // TrailZ can only become smaller, short-circuit if we hit zero.
1803 if (Known.isUnknown())
1804 break;
1805
1806 Value *Index = I->getOperand(i);
1807
1808 // Handle case when index is zero.
1809 Constant *CIndex = dyn_cast<Constant>(Val: Index);
1810 if (CIndex && CIndex->isNullValue())
1811 continue;
1812
1813 if (StructType *STy = GTI.getStructTypeOrNull()) {
1814 // Handle struct member offset arithmetic.
1815
1816 assert(CIndex &&
1817 "Access to structure field must be known at compile time");
1818
1819 if (CIndex->getType()->isVectorTy())
1820 Index = CIndex->getSplatValue();
1821
1822 unsigned Idx = cast<ConstantInt>(Val: Index)->getZExtValue();
1823 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
1824 uint64_t Offset = SL->getElementOffset(Idx);
1825 AccConstIndices += Offset;
1826 continue;
1827 }
1828
1829 // Handle array index arithmetic.
1830 Type *IndexedTy = GTI.getIndexedType();
1831 if (!IndexedTy->isSized()) {
1832 Known.resetAll();
1833 break;
1834 }
1835
1836 TypeSize Stride = GTI.getSequentialElementStride(DL: Q.DL);
1837 uint64_t StrideInBytes = Stride.getKnownMinValue();
1838 if (!Stride.isScalable()) {
1839 // Fast path for constant offset.
1840 if (auto *CI = dyn_cast<ConstantInt>(Val: Index)) {
1841 AccConstIndices +=
1842 CI->getValue().sextOrTrunc(width: IndexWidth) * StrideInBytes;
1843 continue;
1844 }
1845 }
1846
1847 KnownBits IndexBits =
1848 computeKnownBits(V: Index, Q, Depth: Depth + 1).sextOrTrunc(BitWidth: IndexWidth);
1849 KnownBits ScalingFactor(IndexWidth);
1850 // Multiply by current sizeof type.
1851 // &A[i] == A + i * sizeof(*A[i]).
1852 if (Stride.isScalable()) {
1853 // For scalable types the only thing we know about sizeof is
1854 // that this is a multiple of the minimum size.
1855 ScalingFactor.Zero.setLowBits(llvm::countr_zero(Val: StrideInBytes));
1856 } else {
1857 ScalingFactor =
1858 KnownBits::makeConstant(C: APInt(IndexWidth, StrideInBytes));
1859 }
1860 AddIndexToKnown(KnownBits::mul(LHS: IndexBits, RHS: ScalingFactor));
1861 }
1862 if (!Known.isUnknown() && !AccConstIndices.isZero())
1863 AddIndexToKnown(KnownBits::makeConstant(C: AccConstIndices));
1864 break;
1865 }
1866 case Instruction::PHI: {
1867 const PHINode *P = cast<PHINode>(Val: I);
1868 BinaryOperator *BO = nullptr;
1869 Value *Start = nullptr, *Step = nullptr;
1870 KnownBits &KnownStart = Known2;
1871 if (matchSimpleRecurrence(P, BO, Start, Step)) {
1872 // Handle the case of a simple two-predecessor recurrence PHI.
1873 // There's a lot more that could theoretically be done here, but
1874 // this is sufficient to catch some interesting cases.
1875 unsigned Opcode = BO->getOpcode();
1876
1877 switch (Opcode) {
1878 // If this is a shift recurrence, we know the bits being shifted in. We
1879 // can combine that with information about the start value of the
1880 // recurrence to conclude facts about the result. If this is a udiv
1881 // recurrence, we know that the result can never exceed either the
1882 // numerator or the start value, whichever is greater.
1883 case Instruction::LShr:
1884 case Instruction::AShr:
1885 case Instruction::Shl:
1886 case Instruction::UDiv:
1887 if (BO->getOperand(i_nocapture: 0) != I)
1888 break;
1889 [[fallthrough]];
1890
1891 // For a urem recurrence, the result can never exceed the start value. The
1892 // phi could either be the numerator or the denominator.
1893 case Instruction::URem: {
1894 // We have matched a recurrence of the form:
1895 // %iv = [R, %entry], [%iv.next, %backedge]
1896 // %iv.next = shift_op %iv, L
1897
1898 // Recurse with the phi context to avoid concern about whether facts
1899 // inferred hold at original context instruction. TODO: It may be
1900 // correct to use the original context. IF warranted, explore and
1901 // add sufficient tests to cover.
1902 SimplifyQuery RecQ = Q.getWithoutCondContext();
1903 RecQ.CtxI = P;
1904 computeKnownBits(V: Start, DemandedElts, Known&: KnownStart, Q: RecQ, Depth: Depth + 1);
1905 switch (Opcode) {
1906 case Instruction::Shl:
1907 // A shl recurrence will only increase the tailing zeros
1908 Known.Zero.setLowBits(KnownStart.countMinTrailingZeros());
1909 break;
1910 case Instruction::LShr:
1911 case Instruction::UDiv:
1912 case Instruction::URem:
1913 // lshr, udiv, and urem recurrences will preserve the leading zeros of
1914 // the start value.
1915 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
1916 break;
1917 case Instruction::AShr:
1918 // An ashr recurrence will extend the initial sign bit
1919 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
1920 Known.One.setHighBits(KnownStart.countMinLeadingOnes());
1921 break;
1922 }
1923 break;
1924 }
1925
1926 case Instruction::And: {
1927 // Bits that are zero in the start value stay zero, and bits that are
1928 // one in both the start value and the step stay one.
1929 KnownBits KnownStep(BitWidth);
1930 computeKnownBitsForRecurrenceOperands(P, Start, Step, DemandedElts,
1931 KnownStart, KnownStep, Q, Depth);
1932 Known.Zero |= KnownStart.Zero;
1933 Known.One |= KnownStart.One & KnownStep.One;
1934 break;
1935 }
1936
1937 case Instruction::Or: {
1938 // Bits that are zero in both the start value and the step stay zero,
1939 // and bits that are one in the start value stay one.
1940 KnownBits KnownStep(BitWidth);
1941 computeKnownBitsForRecurrenceOperands(P, Start, Step, DemandedElts,
1942 KnownStart, KnownStep, Q, Depth);
1943 Known.Zero |= KnownStart.Zero & KnownStep.Zero;
1944 Known.One |= KnownStart.One;
1945 break;
1946 }
1947
1948 // Check for operations that have the property that if
1949 // both their operands have low zero bits, the result
1950 // will have low zero bits.
1951 case Instruction::Add:
1952 case Instruction::Sub:
1953 case Instruction::Mul: {
1954 // Ok, we have a recurrence of the form {Start,op,Step}. Check for low
1955 // zero bits.
1956 KnownBits KnownStep(BitWidth);
1957 computeKnownBitsForRecurrenceOperands(P, Start, Step, DemandedElts,
1958 KnownStart, KnownStep, Q, Depth);
1959
1960 Known.Zero.setLowBits(std::min(a: KnownStart.countMinTrailingZeros(),
1961 b: KnownStep.countMinTrailingZeros()));
1962
1963 auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(Val: BO);
1964 if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(Op: OverflowOp))
1965 break;
1966
1967 switch (Opcode) {
1968 // If initial value of recurrence is nonnegative, and we are adding
1969 // a nonnegative number with nsw, the result can only be nonnegative
1970 // or poison value regardless of the number of times we execute the
1971 // add in phi recurrence. If initial value is negative and we are
1972 // adding a negative number with nsw, the result can only be
1973 // negative or poison value. Similar arguments apply to sub and mul.
1974 //
1975 // (add non-negative, non-negative) --> non-negative
1976 // (add negative, negative) --> negative
1977 case Instruction::Add: {
1978 if (KnownStart.isNonNegative() && KnownStep.isNonNegative())
1979 Known.makeNonNegative();
1980 else if (KnownStart.isNegative() && KnownStep.isNegative())
1981 Known.makeNegative();
1982 break;
1983 }
1984
1985 // (sub nsw non-negative, negative) --> non-negative
1986 // (sub nsw negative, non-negative) --> negative
1987 case Instruction::Sub: {
1988 if (BO->getOperand(i_nocapture: 0) != I)
1989 break;
1990 if (KnownStart.isNonNegative() && KnownStep.isNegative())
1991 Known.makeNonNegative();
1992 else if (KnownStart.isNegative() && KnownStep.isNonNegative())
1993 Known.makeNegative();
1994 break;
1995 }
1996
1997 // (mul nsw non-negative, non-negative) --> non-negative
1998 case Instruction::Mul:
1999 if (KnownStart.isNonNegative() && KnownStep.isNonNegative())
2000 Known.makeNonNegative();
2001 break;
2002
2003 default:
2004 break;
2005 }
2006 break;
2007 }
2008
2009 default:
2010 break;
2011 }
2012 } else {
2013 IntrinsicInst *II = nullptr;
2014 if (matchTwoInputRecurrence<IntrinsicInst>(PN: P, Inst&: II, Init&: Start, OtherOp&: Step)) {
2015 // %iv = [<Start>, %entry], [%iv.next, %backedge]
2016 //
2017 // %iv.next = <II>(%iv, <Step>)
2018 // or
2019 // %iv.next = <II>(<Step>, %iv)
2020 Intrinsic::ID IntrinsicID = II->getIntrinsicID();
2021 if (IntrinsicID == Intrinsic::umin || IntrinsicID == Intrinsic::umax) {
2022 KnownBits KnownStep(BitWidth);
2023 computeKnownBitsForRecurrenceOperands(
2024 P, Start, Step, DemandedElts, KnownStart, KnownStep, Q, Depth);
2025
2026 if (IntrinsicID == Intrinsic::umin) {
2027 Known.Zero.setHighBits(KnownStart.countMinLeadingZeros());
2028 Known.One.setHighBits(std::min(a: KnownStart.countMinLeadingOnes(),
2029 b: KnownStep.countMinLeadingOnes()));
2030 } else {
2031 // umax
2032 Known.Zero.setHighBits(std::min(a: KnownStart.countMinLeadingZeros(),
2033 b: KnownStep.countMinLeadingZeros()));
2034 Known.One.setHighBits(KnownStart.countMinLeadingOnes());
2035 }
2036 }
2037 }
2038 }
2039
2040 // Unreachable blocks may have zero-operand PHI nodes.
2041 if (P->getNumIncomingValues() == 0)
2042 break;
2043
2044 // Otherwise take the unions of the known bit sets of the operands,
2045 // taking conservative care to avoid excessive recursion.
2046 if (Depth < MaxAnalysisRecursionDepth - 1 && Known.isUnknown()) {
2047 // Skip if every incoming value references to ourself.
2048 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
2049 break;
2050
2051 Known.setAllConflict();
2052 for (const Use &U : P->operands()) {
2053 Value *IncValue;
2054 const PHINode *CtxPhi;
2055 Instruction *CtxI;
2056 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CtxI, PhiOut: &CtxPhi);
2057 // Skip direct self references.
2058 if (IncValue == P)
2059 continue;
2060
2061 // Change the context instruction to the "edge" that flows into the
2062 // phi. This is important because that is where the value is actually
2063 // "evaluated" even though it is used later somewhere else. (see also
2064 // D69571).
2065 SimplifyQuery RecQ = Q.getWithoutCondContext().getWithInstruction(I: CtxI);
2066
2067 Known2 = KnownBits(BitWidth);
2068
2069 // Recurse, but cap the recursion to one level, because we don't
2070 // want to waste time spinning around in loops.
2071 // TODO: See if we can base recursion limiter on number of incoming phi
2072 // edges so we don't overly clamp analysis.
2073 computeKnownBits(V: IncValue, DemandedElts, Known&: Known2, Q: RecQ,
2074 Depth: MaxAnalysisRecursionDepth - 1);
2075
2076 // See if we can further use a conditional branch into the phi
2077 // to help us determine the range of the value.
2078 if (!Known2.isConstant()) {
2079 CmpPredicate Pred;
2080 const APInt *RHSC;
2081 BasicBlock *TrueSucc, *FalseSucc;
2082 // TODO: Use RHS Value and compute range from its known bits.
2083 if (match(V: RecQ.CtxI,
2084 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: IncValue), R: m_APInt(Res&: RHSC)),
2085 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
2086 // Check for cases of duplicate successors.
2087 if ((TrueSucc == CtxPhi->getParent()) !=
2088 (FalseSucc == CtxPhi->getParent())) {
2089 // If we're using the false successor, invert the predicate.
2090 if (FalseSucc == CtxPhi->getParent())
2091 Pred = CmpInst::getInversePredicate(pred: Pred);
2092 // Get the knownbits implied by the incoming phi condition.
2093 auto CR = ConstantRange::makeExactICmpRegion(Pred, Other: *RHSC);
2094 KnownBits KnownUnion = Known2.unionWith(RHS: CR.toKnownBits());
2095 // We can have conflicts here if we are analyzing deadcode (its
2096 // impossible for us reach this BB based the icmp).
2097 if (KnownUnion.hasConflict()) {
2098 // No reason to continue analyzing in a known dead region, so
2099 // just resetAll and break. This will cause us to also exit the
2100 // outer loop.
2101 Known.resetAll();
2102 break;
2103 }
2104 Known2 = KnownUnion;
2105 }
2106 }
2107 }
2108
2109 Known = Known.intersectWith(RHS: Known2);
2110 // If all bits have been ruled out, there's no need to check
2111 // more operands.
2112 if (Known.isUnknown())
2113 break;
2114 }
2115 }
2116 break;
2117 }
2118 case Instruction::Call:
2119 case Instruction::Invoke: {
2120 // If range metadata is attached to this call, set known bits from that,
2121 // and then intersect with known bits based on other properties of the
2122 // function.
2123 if (MDNode *MD =
2124 Q.IIQ.getMetadata(I: cast<Instruction>(Val: I), KindID: LLVMContext::MD_range))
2125 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known);
2126
2127 const auto *CB = cast<CallBase>(Val: I);
2128
2129 if (std::optional<ConstantRange> Range = CB->getRange())
2130 Known = Known.unionWith(RHS: Range->toKnownBits());
2131
2132 if (const Value *RV = CB->getReturnedArgOperand()) {
2133 if (RV->getType() == I->getType()) {
2134 computeKnownBits(V: RV, Known&: Known2, Q, Depth: Depth + 1);
2135 Known = Known.unionWith(RHS: Known2);
2136 // If the function doesn't return properly for all input values
2137 // (e.g. unreachable exits) then there might be conflicts between the
2138 // argument value and the range metadata. Simply discard the known bits
2139 // in case of conflicts.
2140 if (Known.hasConflict())
2141 Known.resetAll();
2142 }
2143 }
2144 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
2145 switch (II->getIntrinsicID()) {
2146 default:
2147 break;
2148 case Intrinsic::abs: {
2149 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2150 bool IntMinIsPoison = match(V: II->getArgOperand(i: 1), P: m_One());
2151 Known = Known.unionWith(RHS: Known2.abs(IntMinIsPoison));
2152 break;
2153 }
2154 case Intrinsic::bitreverse:
2155 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2156 Known = Known.unionWith(RHS: Known2.reverseBits());
2157 break;
2158 case Intrinsic::bswap:
2159 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2160 Known = Known.unionWith(RHS: Known2.byteSwap());
2161 break;
2162 case Intrinsic::ctlz: {
2163 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2164 // If we have a known 1, its position is our upper bound.
2165 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2166 // If this call is poison for 0 input, the result will be less than 2^n.
2167 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2168 PossibleLZ = std::min(a: PossibleLZ, b: BitWidth - 1);
2169 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
2170 Known.Zero.setBitsFrom(LowBits);
2171 break;
2172 }
2173 case Intrinsic::cttz: {
2174 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2175 // If we have a known 1, its position is our upper bound.
2176 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2177 // If this call is poison for 0 input, the result will be less than 2^n.
2178 if (II->getArgOperand(i: 1) == ConstantInt::getTrue(Context&: II->getContext()))
2179 PossibleTZ = std::min(a: PossibleTZ, b: BitWidth - 1);
2180 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
2181 Known.Zero.setBitsFrom(LowBits);
2182 break;
2183 }
2184 case Intrinsic::ctpop: {
2185 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2186 // We can bound the space the count needs. Also, bits known to be zero
2187 // can't contribute to the population.
2188 unsigned BitsPossiblySet = Known2.countMaxPopulation();
2189 unsigned LowBits = llvm::bit_width(Value: BitsPossiblySet);
2190 Known.Zero.setBitsFrom(LowBits);
2191 // TODO: we could bound KnownOne using the lower bound on the number
2192 // of bits which might be set provided by popcnt KnownOne2.
2193 break;
2194 }
2195 case Intrinsic::fshr:
2196 case Intrinsic::fshl: {
2197 const APInt *SA;
2198 if (!match(V: I->getOperand(i: 2), P: m_APInt(Res&: SA)))
2199 break;
2200
2201 KnownBits Known3(BitWidth);
2202 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2203 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known3, Q, Depth: Depth + 1);
2204 Known = II->getIntrinsicID() == Intrinsic::fshl
2205 ? KnownBits::fshl(LHS: Known2, RHS: Known3, Amt: *SA)
2206 : KnownBits::fshr(LHS: Known2, RHS: Known3, Amt: *SA);
2207 break;
2208 }
2209 case Intrinsic::clmul:
2210 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2211 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2212 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
2213 break;
2214 case Intrinsic::pext:
2215 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2216 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2217 Known = KnownBits::pext(Val: Known, Mask: Known2);
2218 break;
2219 case Intrinsic::pdep:
2220 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2221 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2222 Known = KnownBits::pdep(Val: Known, Mask: Known2);
2223 break;
2224 case Intrinsic::smulh:
2225 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2226 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2227 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
2228 break;
2229 case Intrinsic::umulh:
2230 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2231 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2232 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
2233 break;
2234 case Intrinsic::uadd_sat:
2235 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2236 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2237 Known = KnownBits::uadd_sat(LHS: Known, RHS: Known2);
2238 break;
2239 case Intrinsic::usub_sat:
2240 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2241 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2242 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
2243 break;
2244 case Intrinsic::sadd_sat:
2245 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2246 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2247 Known = KnownBits::sadd_sat(LHS: Known, RHS: Known2);
2248 break;
2249 case Intrinsic::ssub_sat:
2250 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2251 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2252 Known = KnownBits::ssub_sat(LHS: Known, RHS: Known2);
2253 break;
2254 // Vec reverse preserves bits from input vec.
2255 case Intrinsic::vector_reverse:
2256 computeKnownBits(V: I->getOperand(i: 0), DemandedElts: DemandedElts.reverseBits(), Known, Q,
2257 Depth: Depth + 1);
2258 break;
2259 // for min/max/and/or reduce, any bit common to each element in the
2260 // input vec is set in the output.
2261 case Intrinsic::vector_reduce_and:
2262 case Intrinsic::vector_reduce_or:
2263 case Intrinsic::vector_reduce_umax:
2264 case Intrinsic::vector_reduce_umin:
2265 case Intrinsic::vector_reduce_smax:
2266 case Intrinsic::vector_reduce_smin:
2267 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2268 break;
2269 case Intrinsic::vector_reduce_xor: {
2270 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2271 // The zeros common to all vecs are zero in the output.
2272 // If the number of elements is odd, then the common ones remain. If the
2273 // number of elements is even, then the common ones becomes zeros.
2274 auto *VecTy = cast<VectorType>(Val: I->getOperand(i: 0)->getType());
2275 // Even, so the ones become zeros.
2276 bool EvenCnt = VecTy->getElementCount().isKnownEven();
2277 if (EvenCnt)
2278 Known.Zero |= Known.One;
2279 // Maybe even element count so need to clear ones.
2280 if (VecTy->isScalableTy() || EvenCnt)
2281 Known.One.clearAllBits();
2282 break;
2283 }
2284 case Intrinsic::vector_reduce_add: {
2285 auto *VecTy = dyn_cast<FixedVectorType>(Val: I->getOperand(i: 0)->getType());
2286 if (!VecTy)
2287 break;
2288 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2289 Known = Known.reduceAdd(NumElts: VecTy->getNumElements());
2290 break;
2291 }
2292 case Intrinsic::umin:
2293 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2294 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2295 Known = KnownBits::umin(LHS: Known, RHS: Known2);
2296 break;
2297 case Intrinsic::umax:
2298 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2299 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2300 Known = KnownBits::umax(LHS: Known, RHS: Known2);
2301 break;
2302 case Intrinsic::smin:
2303 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2304 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2305 Known = KnownBits::smin(LHS: Known, RHS: Known2);
2306 unionWithMinMaxIntrinsicClamp(II, Known);
2307 break;
2308 case Intrinsic::smax:
2309 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2310 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2311 Known = KnownBits::smax(LHS: Known, RHS: Known2);
2312 unionWithMinMaxIntrinsicClamp(II, Known);
2313 break;
2314 case Intrinsic::ptrmask: {
2315 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
2316
2317 const Value *Mask = I->getOperand(i: 1);
2318 Known2 = KnownBits(Mask->getType()->getScalarSizeInBits());
2319 computeKnownBits(V: Mask, DemandedElts, Known&: Known2, Q, Depth: Depth + 1);
2320 // TODO: 1-extend would be more precise.
2321 Known &= Known2.anyextOrTrunc(BitWidth);
2322 break;
2323 }
2324 case Intrinsic::x86_sse42_crc32_64_64:
2325 Known.Zero.setBitsFrom(32);
2326 break;
2327 case Intrinsic::x86_ssse3_phadd_d_128:
2328 case Intrinsic::x86_ssse3_phadd_w_128:
2329 case Intrinsic::x86_avx2_phadd_d:
2330 case Intrinsic::x86_avx2_phadd_w: {
2331 Known = computeKnownBitsForHorizontalOperation(
2332 I, DemandedElts, Q, Depth,
2333 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2334 return KnownBits::add(LHS: KnownLHS, RHS: KnownRHS);
2335 });
2336 break;
2337 }
2338 case Intrinsic::x86_ssse3_phadd_sw_128:
2339 case Intrinsic::x86_avx2_phadd_sw: {
2340 Known = computeKnownBitsForHorizontalOperation(
2341 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::sadd_sat);
2342 break;
2343 }
2344 case Intrinsic::x86_ssse3_phsub_d_128:
2345 case Intrinsic::x86_ssse3_phsub_w_128:
2346 case Intrinsic::x86_avx2_phsub_d:
2347 case Intrinsic::x86_avx2_phsub_w: {
2348 Known = computeKnownBitsForHorizontalOperation(
2349 I, DemandedElts, Q, Depth,
2350 KnownBitsFunc: [](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
2351 return KnownBits::sub(LHS: KnownLHS, RHS: KnownRHS);
2352 });
2353 break;
2354 }
2355 case Intrinsic::x86_ssse3_phsub_sw_128:
2356 case Intrinsic::x86_avx2_phsub_sw: {
2357 Known = computeKnownBitsForHorizontalOperation(
2358 I, DemandedElts, Q, Depth, KnownBitsFunc: KnownBits::ssub_sat);
2359 break;
2360 }
2361 case Intrinsic::riscv_vsetvli:
2362 case Intrinsic::riscv_vsetvlimax: {
2363 bool HasAVL = II->getIntrinsicID() == Intrinsic::riscv_vsetvli;
2364 const ConstantRange Range = getVScaleRange(F: II->getFunction(), BitWidth);
2365 uint64_t SEW = RISCVVType::decodeVSEW(
2366 VSEW: cast<ConstantInt>(Val: II->getArgOperand(i: HasAVL))->getZExtValue());
2367 RISCVVType::VLMUL VLMUL = static_cast<RISCVVType::VLMUL>(
2368 cast<ConstantInt>(Val: II->getArgOperand(i: 1 + HasAVL))->getZExtValue());
2369 uint64_t MaxVLEN =
2370 Range.getUnsignedMax().getZExtValue() * RISCV::RVVBitsPerBlock;
2371 uint64_t MaxVL = MaxVLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
2372
2373 // Result of vsetvli must be not larger than AVL.
2374 if (HasAVL)
2375 if (auto *CI = dyn_cast<ConstantInt>(Val: II->getArgOperand(i: 0)))
2376 MaxVL = std::min(a: MaxVL, b: CI->getZExtValue());
2377
2378 unsigned KnownZeroFirstBit = Log2_32(Value: MaxVL) + 1;
2379 if (BitWidth > KnownZeroFirstBit)
2380 Known.Zero.setBitsFrom(KnownZeroFirstBit);
2381 break;
2382 }
2383 case Intrinsic::amdgcn_mbcnt_hi:
2384 case Intrinsic::amdgcn_mbcnt_lo: {
2385 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
2386 // most 31 + src1.
2387 KnownBits MbcntKnown(BitWidth);
2388 MbcntKnown.Zero.setBitsFrom(
2389 II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo ? 6 : 5);
2390 computeKnownBits(V: I->getOperand(i: 1), Known&: Known2, Q, Depth: Depth + 1);
2391 Known = Known.unionWith(RHS: KnownBits::add(LHS: MbcntKnown, RHS: Known2));
2392 break;
2393 }
2394 case Intrinsic::vscale: {
2395 if (!II->getParent() || !II->getFunction())
2396 break;
2397
2398 Known = getVScaleRange(F: II->getFunction(), BitWidth).toKnownBits();
2399 break;
2400 }
2401 case Intrinsic::stepvector: {
2402 auto *VecTy = cast<VectorType>(Val: II->getType());
2403 unsigned MinNumElts = VecTy->getElementCount().getKnownMinValue();
2404 if (!isUIntN(N: BitWidth, x: MinNumElts))
2405 break;
2406
2407 bool Overflow = false;
2408 APInt MaxNumElts(BitWidth, MinNumElts);
2409 if (VecTy->isScalableTy()) {
2410 if (!II->getParent() || !II->getFunction())
2411 break;
2412 MaxNumElts = getVScaleRange(F: II->getFunction(), BitWidth)
2413 .getUnsignedMax()
2414 .umul_ov(RHS: MaxNumElts, Overflow);
2415 }
2416
2417 // Give up if the lane count could wrap. Stepvector truncates lane
2418 // indices that do not fit in the element type.
2419 if (Overflow)
2420 break;
2421
2422 Known.Zero.setHighBits((MaxNumElts - 1).countl_zero());
2423 break;
2424 }
2425 }
2426 }
2427 break;
2428 }
2429 case Instruction::ShuffleVector: {
2430 if (auto *Splat = getSplatValue(V: I)) {
2431 computeKnownBits(V: Splat, Known, Q, Depth: Depth + 1);
2432 break;
2433 }
2434
2435 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
2436 // FIXME: Do we need to handle ConstantExpr involving shufflevectors?
2437 if (!Shuf) {
2438 Known.resetAll();
2439 return;
2440 }
2441 // For undef elements, we don't know anything about the common state of
2442 // the shuffle result.
2443 APInt DemandedLHS, DemandedRHS;
2444 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS)) {
2445 Known.resetAll();
2446 return;
2447 }
2448 Known.setAllConflict();
2449 if (!!DemandedLHS) {
2450 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
2451 computeKnownBits(V: LHS, DemandedElts: DemandedLHS, Known, Q, Depth: Depth + 1);
2452 // If we don't know any bits, early out.
2453 if (Known.isUnknown())
2454 break;
2455 }
2456 if (!!DemandedRHS) {
2457 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
2458 computeKnownBits(V: RHS, DemandedElts: DemandedRHS, Known&: Known2, Q, Depth: Depth + 1);
2459 Known = Known.intersectWith(RHS: Known2);
2460 }
2461 break;
2462 }
2463 case Instruction::InsertElement: {
2464 if (isa<ScalableVectorType>(Val: I->getType())) {
2465 Known.resetAll();
2466 return;
2467 }
2468 const Value *Vec = I->getOperand(i: 0);
2469 const Value *Elt = I->getOperand(i: 1);
2470 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
2471 unsigned NumElts = DemandedElts.getBitWidth();
2472 APInt DemandedVecElts = DemandedElts;
2473 bool NeedsElt = true;
2474 // If we know the index we are inserting too, clear it from Vec check.
2475 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
2476 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
2477 NeedsElt = DemandedElts[CIdx->getZExtValue()];
2478 }
2479
2480 Known.setAllConflict();
2481 if (NeedsElt) {
2482 computeKnownBits(V: Elt, Known, Q, Depth: Depth + 1);
2483 // If we don't know any bits, early out.
2484 if (Known.isUnknown())
2485 break;
2486 }
2487
2488 if (!DemandedVecElts.isZero()) {
2489 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known&: Known2, Q, Depth: Depth + 1);
2490 Known = Known.intersectWith(RHS: Known2);
2491 }
2492 break;
2493 }
2494 case Instruction::ExtractElement: {
2495 // Look through extract element. If the index is non-constant or
2496 // out-of-range demand all elements, otherwise just the extracted element.
2497 const Value *Vec = I->getOperand(i: 0);
2498 const Value *Idx = I->getOperand(i: 1);
2499 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
2500 if (isa<ScalableVectorType>(Val: Vec->getType())) {
2501 // FIXME: there's probably *something* we can do with scalable vectors
2502 Known.resetAll();
2503 break;
2504 }
2505 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
2506 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
2507 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
2508 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
2509 computeKnownBits(V: Vec, DemandedElts: DemandedVecElts, Known, Q, Depth: Depth + 1);
2510 break;
2511 }
2512 case Instruction::ExtractValue:
2513 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I->getOperand(i: 0))) {
2514 const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: I);
2515 if (EVI->getNumIndices() != 1) break;
2516 if (EVI->getIndices()[0] == 0) {
2517 switch (II->getIntrinsicID()) {
2518 default: break;
2519 case Intrinsic::uadd_with_overflow:
2520 case Intrinsic::sadd_with_overflow:
2521 computeKnownBitsAddSub(
2522 Add: true, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2523 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2524 break;
2525 case Intrinsic::usub_with_overflow:
2526 case Intrinsic::ssub_with_overflow:
2527 computeKnownBitsAddSub(
2528 Add: false, Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), /*NSW=*/false,
2529 /* NUW=*/false, DemandedElts, KnownOut&: Known, Known2, Q, Depth);
2530 break;
2531 case Intrinsic::umul_with_overflow:
2532 case Intrinsic::smul_with_overflow:
2533 computeKnownBitsMul(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1), NSW: false,
2534 NUW: false, DemandedElts, Known, Known2, Q, Depth);
2535 break;
2536 }
2537 }
2538 }
2539 break;
2540 case Instruction::Freeze:
2541 if (isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT,
2542 Depth: Depth + 1))
2543 computeKnownBits(V: I->getOperand(i: 0), Known, Q, Depth: Depth + 1);
2544 break;
2545 }
2546}
2547
2548/// Determine which bits of V are known to be either zero or one and return
2549/// them.
2550KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
2551 const SimplifyQuery &Q, unsigned Depth) {
2552 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2553 ::computeKnownBits(V, DemandedElts, Known, Q, Depth);
2554 return Known;
2555}
2556
2557/// Determine which bits of V are known to be either zero or one and return
2558/// them.
2559KnownBits llvm::computeKnownBits(const Value *V, const SimplifyQuery &Q,
2560 unsigned Depth) {
2561 KnownBits Known(getBitWidth(Ty: V->getType(), DL: Q.DL));
2562 computeKnownBits(V, Known, Q, Depth);
2563 return Known;
2564}
2565
2566/// Determine which bits of V are known to be either zero or one and return
2567/// them in the Known bit set.
2568///
2569/// NOTE: we cannot consider 'undef' to be "IsZero" here. The problem is that
2570/// we cannot optimize based on the assumption that it is zero without changing
2571/// it to be an explicit zero. If we don't change it to zero, other code could
2572/// optimized based on the contradictory assumption that it is non-zero.
2573/// Because instcombine aggressively folds operations with undef args anyway,
2574/// this won't lose us code quality.
2575///
2576/// This function is defined on values with integer type, values with pointer
2577/// type, and vectors of integers. In the case
2578/// where V is a vector, known zero, and known one values are the
2579/// same width as the vector element, and the bit is set only if it is true
2580/// for all of the demanded elements in the vector specified by DemandedElts.
2581void computeKnownBits(const Value *V, const APInt &DemandedElts,
2582 KnownBits &Known, const SimplifyQuery &Q,
2583 unsigned Depth) {
2584 if (!DemandedElts) {
2585 // No demanded elts, better to assume we don't know anything.
2586 Known.resetAll();
2587 return;
2588 }
2589
2590 assert(V && "No Value?");
2591 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2592
2593#ifndef NDEBUG
2594 Type *Ty = V->getType();
2595 unsigned BitWidth = Known.getBitWidth();
2596
2597 assert((Ty->isIntOrIntVectorTy(BitWidth) || Ty->isPtrOrPtrVectorTy()) &&
2598 "Not integer or pointer type!");
2599
2600 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
2601 assert(
2602 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
2603 "DemandedElt width should equal the fixed vector number of elements");
2604 } else {
2605 assert(DemandedElts == APInt(1, 1) &&
2606 "DemandedElt width should be 1 for scalars or scalable vectors");
2607 }
2608
2609 Type *ScalarTy = Ty->getScalarType();
2610 if (ScalarTy->isPointerTy()) {
2611 assert(BitWidth == Q.DL.getPointerTypeSizeInBits(ScalarTy) &&
2612 "V and Known should have same BitWidth");
2613 } else {
2614 assert(BitWidth == Q.DL.getTypeSizeInBits(ScalarTy) &&
2615 "V and Known should have same BitWidth");
2616 }
2617#endif
2618
2619 const APInt *C;
2620 if (match(V, P: m_APInt(Res&: C))) {
2621 // We know all of the bits for a scalar constant or a splat vector constant!
2622 Known = KnownBits::makeConstant(C: *C);
2623 return;
2624 }
2625 // Null and aggregate-zero are all-zeros.
2626 if (isa<ConstantPointerNull>(Val: V) || isa<ConstantAggregateZero>(Val: V)) {
2627 Known.setAllZero();
2628 return;
2629 }
2630 // Handle a constant vector by taking the intersection of the known bits of
2631 // each element.
2632 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(Val: V)) {
2633 assert(!isa<ScalableVectorType>(V->getType()));
2634 // We know that CDV must be a vector of integers. Take the intersection of
2635 // each element.
2636 Known.setAllConflict();
2637 for (unsigned i = 0, e = CDV->getNumElements(); i != e; ++i) {
2638 if (!DemandedElts[i])
2639 continue;
2640 APInt Elt = CDV->getElementAsAPInt(i);
2641 Known.Zero &= ~Elt;
2642 Known.One &= Elt;
2643 }
2644 if (Known.hasConflict())
2645 Known.resetAll();
2646 return;
2647 }
2648
2649 if (const auto *CV = dyn_cast<ConstantVector>(Val: V)) {
2650 assert(!isa<ScalableVectorType>(V->getType()));
2651 // We know that CV must be a vector of integers. Take the intersection of
2652 // each element.
2653 Known.setAllConflict();
2654 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
2655 if (!DemandedElts[i])
2656 continue;
2657 Constant *Element = CV->getAggregateElement(Elt: i);
2658 if (isa<PoisonValue>(Val: Element))
2659 continue;
2660 auto *ElementCI = dyn_cast_or_null<ConstantInt>(Val: Element);
2661 if (!ElementCI) {
2662 Known.resetAll();
2663 return;
2664 }
2665 const APInt &Elt = ElementCI->getValue();
2666 Known.Zero &= ~Elt;
2667 Known.One &= Elt;
2668 }
2669 if (Known.hasConflict())
2670 Known.resetAll();
2671 return;
2672 }
2673
2674 // Start out not knowing anything.
2675 Known.resetAll();
2676
2677 // We can't imply anything about undefs.
2678 if (isa<UndefValue>(Val: V))
2679 return;
2680
2681 // There's no point in looking through other users of ConstantData for
2682 // assumptions. Confirm that we've handled them all.
2683 assert(!isa<ConstantData>(V) && "Unhandled constant data!");
2684
2685 if (const auto *A = dyn_cast<Argument>(Val: V))
2686 if (std::optional<ConstantRange> Range = A->getRange())
2687 Known = Range->toKnownBits();
2688
2689 // All recursive calls that increase depth must come after this.
2690 if (Depth == MaxAnalysisRecursionDepth)
2691 return;
2692
2693 // A weak GlobalAlias is totally unknown. A non-weak GlobalAlias has
2694 // the bits of its aliasee.
2695 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: V)) {
2696 if (!GA->isInterposable())
2697 computeKnownBits(V: GA->getAliasee(), Known, Q, Depth: Depth + 1);
2698 return;
2699 }
2700
2701 if (const Operator *I = dyn_cast<Operator>(Val: V))
2702 computeKnownBitsFromOperator(I, DemandedElts, Known, Q, Depth);
2703 else if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
2704 if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
2705 Known = CR->toKnownBits();
2706 }
2707
2708 // Aligned pointers have trailing zeros - refine Known.Zero set
2709 if (isa<PointerType>(Val: V->getType())) {
2710 Align Alignment = V->getPointerAlignment(DL: Q.DL);
2711 Known.Zero.setLowBits(Log2(A: Alignment));
2712 }
2713
2714 // computeKnownBitsFromContext strictly refines Known.
2715 // Therefore, we run them after computeKnownBitsFromOperator.
2716
2717 // Check whether we can determine known bits from context such as assumes.
2718 computeKnownBitsFromContext(V, Known, Q, Depth);
2719}
2720
2721/// Try to detect a recurrence that the value of the induction variable is
2722/// always a power of two (or zero).
2723static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
2724 SimplifyQuery &Q, unsigned Depth) {
2725 BinaryOperator *BO = nullptr;
2726 Value *Start = nullptr, *Step = nullptr;
2727 if (!matchSimpleRecurrence(P: PN, BO, Start, Step))
2728 return false;
2729
2730 // Initial value must be a power of two.
2731 for (const Use &U : PN->operands()) {
2732 if (U.get() == Start) {
2733 // Initial value comes from a different BB, need to adjust context
2734 // instruction for analysis.
2735 Q.CtxI = PN->getIncomingBlock(U)->getTerminator();
2736 if (!isKnownToBeAPowerOfTwo(V: Start, OrZero, Q, Depth))
2737 return false;
2738 }
2739 }
2740
2741 // Except for Mul, the induction variable must be on the left side of the
2742 // increment expression, otherwise its value can be arbitrary.
2743 if (BO->getOpcode() != Instruction::Mul && BO->getOperand(i_nocapture: 1) != Step)
2744 return false;
2745
2746 Q.CtxI = BO->getParent()->getTerminator();
2747 switch (BO->getOpcode()) {
2748 case Instruction::Mul:
2749 // Power of two is closed under multiplication.
2750 return (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) ||
2751 Q.IIQ.hasNoSignedWrap(Op: BO)) &&
2752 isKnownToBeAPowerOfTwo(V: Step, OrZero, Q, Depth);
2753 case Instruction::SDiv:
2754 // Start value must not be signmask for signed division, so simply being a
2755 // power of two is not sufficient, and it has to be a constant.
2756 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2757 return false;
2758 [[fallthrough]];
2759 case Instruction::UDiv:
2760 // Divisor must be a power of two.
2761 // If OrZero is false, cannot guarantee induction variable is non-zero after
2762 // division, same for Shr, unless it is exact division.
2763 return (OrZero || Q.IIQ.isExact(Op: BO)) &&
2764 isKnownToBeAPowerOfTwo(V: Step, OrZero: false, Q, Depth);
2765 case Instruction::Shl:
2766 return OrZero || Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO);
2767 case Instruction::AShr:
2768 if (!match(V: Start, P: m_Power2()) || match(V: Start, P: m_SignMask()))
2769 return false;
2770 [[fallthrough]];
2771 case Instruction::LShr:
2772 return OrZero || Q.IIQ.isExact(Op: BO);
2773 default:
2774 return false;
2775 }
2776}
2777
2778/// Return true if we can infer that \p V is known to be a power of 2 from
2779/// dominating condition \p Cond (e.g., ctpop(V) == 1).
2780static bool isImpliedToBeAPowerOfTwoFromCond(const Value *V, bool OrZero,
2781 const Value *Cond,
2782 bool CondIsTrue) {
2783 CmpPredicate Pred;
2784 const APInt *RHSC;
2785 if (!match(V: Cond, P: m_ICmp(Pred, L: m_Ctpop(Op0: m_Specific(V)), R: m_APInt(Res&: RHSC))))
2786 return false;
2787 if (!CondIsTrue)
2788 Pred = ICmpInst::getInversePredicate(pred: Pred);
2789 // ctpop(V) u< 2
2790 if (OrZero && Pred == ICmpInst::ICMP_ULT && *RHSC == 2)
2791 return true;
2792 // ctpop(V) == 1
2793 return Pred == ICmpInst::ICMP_EQ && *RHSC == 1;
2794}
2795
2796/// Return true if the given value is known to have exactly one
2797/// bit set when defined. For vectors return true if every element is known to
2798/// be a power of two when defined. Supports values with integer or pointer
2799/// types and vectors of integers.
2800bool llvm::isKnownToBeAPowerOfTwo(const Value *V, bool OrZero,
2801 const SimplifyQuery &Q, unsigned Depth) {
2802 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2803
2804 if (isa<Constant>(Val: V))
2805 return OrZero ? match(V, P: m_Power2OrZero()) : match(V, P: m_Power2());
2806
2807 // i1 is by definition a power of 2 or zero.
2808 if (OrZero && V->getType()->getScalarSizeInBits() == 1)
2809 return true;
2810
2811 // Try to infer from assumptions.
2812 if (Q.AC && Q.CtxI) {
2813 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
2814 if (!AssumeVH)
2815 continue;
2816 CallInst *I = cast<CallInst>(Val&: AssumeVH);
2817 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond: I->getArgOperand(i: 0),
2818 /*CondIsTrue=*/true) &&
2819 isValidAssumeForContext(I, Q))
2820 return true;
2821 }
2822 }
2823
2824 // Handle dominating conditions.
2825 if (Q.DC && Q.CtxI && Q.DT) {
2826 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
2827 Value *Cond = BI->getCondition();
2828
2829 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
2830 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2831 /*CondIsTrue=*/true) &&
2832 Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()))
2833 return true;
2834
2835 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
2836 if (isImpliedToBeAPowerOfTwoFromCond(V, OrZero, Cond,
2837 /*CondIsTrue=*/false) &&
2838 Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()))
2839 return true;
2840 }
2841 }
2842
2843 auto *I = dyn_cast<Instruction>(Val: V);
2844 if (!I)
2845 return false;
2846
2847 if (Q.CtxI && match(V, P: m_VScale())) {
2848 const Function *F = Q.CtxI->getFunction();
2849 // The vscale_range indicates vscale is a power-of-two.
2850 return F->hasFnAttribute(Kind: Attribute::VScaleRange);
2851 }
2852
2853 // 1 << X is clearly a power of two if the one is not shifted off the end. If
2854 // it is shifted off the end then the result is undefined.
2855 if (match(V: I, P: m_Shl(L: m_One(), R: m_Value())))
2856 return true;
2857
2858 // (signmask) >>l X is clearly a power of two if the one is not shifted off
2859 // the bottom. If it is shifted off the bottom then the result is undefined.
2860 if (match(V: I, P: m_LShr(L: m_SignMask(), R: m_Value())))
2861 return true;
2862
2863 // The remaining tests are all recursive, so bail out if we hit the limit.
2864 if (Depth++ == MaxAnalysisRecursionDepth)
2865 return false;
2866
2867 switch (I->getOpcode()) {
2868 case Instruction::ZExt:
2869 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2870 case Instruction::Trunc:
2871 return OrZero && isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2872 case Instruction::Shl:
2873 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: I) || Q.IIQ.hasNoSignedWrap(Op: I))
2874 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2875 return false;
2876 case Instruction::LShr:
2877 if (OrZero || Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2878 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2879 return false;
2880 case Instruction::UDiv:
2881 if (Q.IIQ.isExact(Op: cast<BinaryOperator>(Val: I)))
2882 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth);
2883 return false;
2884 case Instruction::Mul:
2885 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2886 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth) &&
2887 (OrZero || isKnownNonZero(V: I, Q, Depth));
2888 case Instruction::And:
2889 // A power of two and'd with anything is a power of two or zero.
2890 if (OrZero &&
2891 (isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), /*OrZero*/ true, Q, Depth) ||
2892 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), /*OrZero*/ true, Q, Depth)))
2893 return true;
2894 // X & (-X) is always a power of two or zero.
2895 if (match(V: I->getOperand(i: 0), P: m_Neg(V: m_Specific(V: I->getOperand(i: 1)))) ||
2896 match(V: I->getOperand(i: 1), P: m_Neg(V: m_Specific(V: I->getOperand(i: 0)))))
2897 return OrZero || isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
2898 return false;
2899 case Instruction::Add: {
2900 // Adding a power-of-two or zero to the same power-of-two or zero yields
2901 // either the original power-of-two, a larger power-of-two or zero.
2902 const OverflowingBinaryOperator *VOBO = cast<OverflowingBinaryOperator>(Val: V);
2903 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO) ||
2904 Q.IIQ.hasNoSignedWrap(Op: VOBO)) {
2905 if (match(V: I->getOperand(i: 0),
2906 P: m_c_And(L: m_Specific(V: I->getOperand(i: 1)), R: m_Value())) &&
2907 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth))
2908 return true;
2909 if (match(V: I->getOperand(i: 1),
2910 P: m_c_And(L: m_Specific(V: I->getOperand(i: 0)), R: m_Value())) &&
2911 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 0), OrZero, Q, Depth))
2912 return true;
2913
2914 unsigned BitWidth = V->getType()->getScalarSizeInBits();
2915 KnownBits LHSBits(BitWidth);
2916 computeKnownBits(V: I->getOperand(i: 0), Known&: LHSBits, Q, Depth);
2917
2918 KnownBits RHSBits(BitWidth);
2919 computeKnownBits(V: I->getOperand(i: 1), Known&: RHSBits, Q, Depth);
2920 // If i8 V is a power of two or zero:
2921 // ZeroBits: 1 1 1 0 1 1 1 1
2922 // ~ZeroBits: 0 0 0 1 0 0 0 0
2923 if ((~(LHSBits.Zero & RHSBits.Zero)).isPowerOf2())
2924 // If OrZero isn't set, we cannot give back a zero result.
2925 // Make sure either the LHS or RHS has a bit set.
2926 if (OrZero || RHSBits.One.getBoolValue() || LHSBits.One.getBoolValue())
2927 return true;
2928 }
2929
2930 // LShr(UINT_MAX, Y) + 1 is a power of two (if add is nuw) or zero.
2931 if (OrZero || Q.IIQ.hasNoUnsignedWrap(Op: VOBO))
2932 if (match(V: I, P: m_Add(L: m_LShr(L: m_AllOnes(), R: m_Value()), R: m_One())))
2933 return true;
2934 return false;
2935 }
2936 case Instruction::Select:
2937 return isKnownToBeAPowerOfTwo(V: I->getOperand(i: 1), OrZero, Q, Depth) &&
2938 isKnownToBeAPowerOfTwo(V: I->getOperand(i: 2), OrZero, Q, Depth);
2939 case Instruction::PHI: {
2940 // A PHI node is power of two if all incoming values are power of two, or if
2941 // it is an induction variable where in each step its value is a power of
2942 // two.
2943 auto *PN = cast<PHINode>(Val: I);
2944 SimplifyQuery RecQ = Q.getWithoutCondContext();
2945
2946 // Check if it is an induction variable and always power of two.
2947 if (isPowerOfTwoRecurrence(PN, OrZero, Q&: RecQ, Depth))
2948 return true;
2949
2950 // Recursively check all incoming values. Limit recursion to 2 levels, so
2951 // that search complexity is limited to number of operands^2.
2952 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
2953 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
2954 // Value is power of 2 if it is coming from PHI node itself by induction.
2955 if (U.get() == PN)
2956 return true;
2957
2958 // Change the context instruction to the incoming block where it is
2959 // evaluated.
2960 RecQ.CtxI = PN->getIncomingBlock(U)->getTerminator();
2961 return isKnownToBeAPowerOfTwo(V: U.get(), OrZero, Q: RecQ, Depth: NewDepth);
2962 });
2963 }
2964 case Instruction::Invoke:
2965 case Instruction::Call: {
2966 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2967 switch (II->getIntrinsicID()) {
2968 case Intrinsic::umax:
2969 case Intrinsic::smax:
2970 case Intrinsic::umin:
2971 case Intrinsic::smin:
2972 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 1), OrZero, Q, Depth) &&
2973 isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2974 // bswap/bitreverse just move around bits, but don't change any 1s/0s
2975 // thus dont change pow2/non-pow2 status.
2976 case Intrinsic::bitreverse:
2977 case Intrinsic::bswap:
2978 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2979 case Intrinsic::fshr:
2980 case Intrinsic::fshl:
2981 // If Op0 == Op1, this is a rotate. is_pow2(rotate(x, y)) == is_pow2(x)
2982 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
2983 return isKnownToBeAPowerOfTwo(V: II->getArgOperand(i: 0), OrZero, Q, Depth);
2984 break;
2985 case Intrinsic::riscv_vsetvlimax:
2986 // VLMAX is VLEN * LMUL / SEW, which is always a non-zero power of two
2987 // for any valid vtype, so it is a power of two regardless of OrZero.
2988 return true;
2989 case Intrinsic::read_register:
2990 case Intrinsic::read_volatile_register: {
2991 // The RISC-V vlenb CSR holds VLEN/8, which is always a non-zero power
2992 // of two, so it is a power of two regardless of OrZero.
2993 const Module *M = II->getModule();
2994 if (!M || !M->getTargetTriple().isRISCV())
2995 break;
2996 return isReadVLENB(II: *II);
2997 }
2998 default:
2999 break;
3000 }
3001 }
3002 return false;
3003 }
3004 default:
3005 return false;
3006 }
3007}
3008
3009/// Test whether a GEP's result is known to be non-null.
3010///
3011/// Uses properties inherent in a GEP to try to determine whether it is known
3012/// to be non-null.
3013///
3014/// Currently this routine does not support vector GEPs.
3015static bool isGEPKnownNonNull(const GEPOperator *GEP, const SimplifyQuery &Q,
3016 unsigned Depth) {
3017 const Function *F = nullptr;
3018 if (const Instruction *I = dyn_cast<Instruction>(Val: GEP))
3019 F = I->getFunction();
3020
3021 // If the gep is nuw or inbounds with invalid null pointer, then the GEP
3022 // may be null iff the base pointer is null and the offset is zero.
3023 if (!GEP->hasNoUnsignedWrap() &&
3024 !(GEP->isInBounds() &&
3025 !NullPointerIsDefined(F, AS: GEP->getPointerAddressSpace())))
3026 return false;
3027
3028 // FIXME: Support vector-GEPs.
3029 assert(GEP->getType()->isPointerTy() && "We only support plain pointer GEP");
3030
3031 // If the base pointer is non-null, we cannot walk to a null address with an
3032 // inbounds GEP in address space zero.
3033 if (isKnownNonZero(V: GEP->getPointerOperand(), Q, Depth))
3034 return true;
3035
3036 // Walk the GEP operands and see if any operand introduces a non-zero offset.
3037 // If so, then the GEP cannot produce a null pointer, as doing so would
3038 // inherently violate the inbounds contract within address space zero.
3039 for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
3040 GTI != GTE; ++GTI) {
3041 // Struct types are easy -- they must always be indexed by a constant.
3042 if (StructType *STy = GTI.getStructTypeOrNull()) {
3043 ConstantInt *OpC = cast<ConstantInt>(Val: GTI.getOperand());
3044 unsigned ElementIdx = OpC->getZExtValue();
3045 const StructLayout *SL = Q.DL.getStructLayout(Ty: STy);
3046 uint64_t ElementOffset = SL->getElementOffset(Idx: ElementIdx);
3047 if (ElementOffset > 0)
3048 return true;
3049 continue;
3050 }
3051
3052 // If we have a zero-sized type, the index doesn't matter. Keep looping.
3053 if (GTI.getSequentialElementStride(DL: Q.DL).isZero())
3054 continue;
3055
3056 // Fast path the constant operand case both for efficiency and so we don't
3057 // increment Depth when just zipping down an all-constant GEP.
3058 if (ConstantInt *OpC = dyn_cast<ConstantInt>(Val: GTI.getOperand())) {
3059 if (!OpC->isZero())
3060 return true;
3061 continue;
3062 }
3063
3064 // We post-increment Depth here because while isKnownNonZero increments it
3065 // as well, when we pop back up that increment won't persist. We don't want
3066 // to recurse 10k times just because we have 10k GEP operands. We don't
3067 // bail completely out because we want to handle constant GEPs regardless
3068 // of depth.
3069 if (Depth++ >= MaxAnalysisRecursionDepth)
3070 continue;
3071
3072 if (isKnownNonZero(V: GTI.getOperand(), Q, Depth))
3073 return true;
3074 }
3075
3076 return false;
3077}
3078
3079static bool isKnownNonNullFromDominatingCondition(const Value *V,
3080 const Instruction *CtxI,
3081 const DominatorTree *DT) {
3082 assert(!isa<Constant>(V) && "Called for constant?");
3083
3084 if (!CtxI || !DT)
3085 return false;
3086
3087 unsigned NumUsesExplored = 0;
3088 for (auto &U : V->uses()) {
3089 // Avoid massive lists
3090 if (NumUsesExplored >= DomConditionsMaxUses)
3091 break;
3092 NumUsesExplored++;
3093
3094 const Instruction *UI = cast<Instruction>(Val: U.getUser());
3095 // If the value is used as an argument to a call or invoke, then argument
3096 // attributes may provide an answer about null-ness.
3097 if (V->getType()->isPointerTy()) {
3098 if (const auto *CB = dyn_cast<CallBase>(Val: UI)) {
3099 if (CB->isArgOperand(U: &U) &&
3100 CB->paramHasNonNullAttr(ArgNo: CB->getArgOperandNo(U: &U),
3101 /*AllowUndefOrPoison=*/false) &&
3102 DT->dominates(Def: CB, User: CtxI))
3103 return true;
3104 }
3105 }
3106
3107 // If the value is used as a load/store, then the pointer must be non null.
3108 if (V == getLoadStorePointerOperand(V: UI)) {
3109 if (!NullPointerIsDefined(F: UI->getFunction(),
3110 AS: V->getType()->getPointerAddressSpace()) &&
3111 DT->dominates(Def: UI, User: CtxI))
3112 return true;
3113 }
3114
3115 if ((match(V: UI, P: m_IDiv(L: m_Value(), R: m_Specific(V))) ||
3116 match(V: UI, P: m_IRem(L: m_Value(), R: m_Specific(V)))) &&
3117 isValidAssumeForContext(Inv: UI, CtxI, DT))
3118 return true;
3119
3120 // Consider only compare instructions uniquely controlling a branch
3121 Value *RHS;
3122 CmpPredicate Pred;
3123 bool NonNullIfTrue;
3124 if (match(V: UI, P: m_c_ICmp(Pred, L: m_Specific(V), R: m_Value(V&: RHS)))) {
3125 if (cmpExcludesZero(Pred, RHS))
3126 NonNullIfTrue = true;
3127 else if (cmpExcludesZero(Pred: CmpInst::getInversePredicate(pred: Pred), RHS))
3128 NonNullIfTrue = false;
3129 else
3130 continue;
3131 } else if (UI->getType()->isIntegerTy(BitWidth: 1) &&
3132 match(V: UI, P: m_Trunc(Op: m_Specific(V)))) {
3133 NonNullIfTrue = true;
3134 } else
3135 continue;
3136
3137 SmallVector<const User *, 4> WorkList;
3138 SmallPtrSet<const User *, 4> Visited;
3139 for (const auto *CmpU : UI->users()) {
3140 assert(WorkList.empty() && "Should be!");
3141 if (Visited.insert(Ptr: CmpU).second)
3142 WorkList.push_back(Elt: CmpU);
3143
3144 while (!WorkList.empty()) {
3145 auto *Curr = WorkList.pop_back_val();
3146
3147 // If a user is an AND, add all its users to the work list. We only
3148 // propagate "pred != null" condition through AND because it is only
3149 // correct to assume that all conditions of AND are met in true branch.
3150 // TODO: Support similar logic of OR and EQ predicate?
3151 if (NonNullIfTrue)
3152 if (match(V: Curr, P: m_LogicalAnd(L: m_Value(), R: m_Value()))) {
3153 for (const auto *CurrU : Curr->users())
3154 if (Visited.insert(Ptr: CurrU).second)
3155 WorkList.push_back(Elt: CurrU);
3156 continue;
3157 }
3158
3159 if (const CondBrInst *BI = dyn_cast<CondBrInst>(Val: Curr)) {
3160 BasicBlock *NonNullSuccessor =
3161 BI->getSuccessor(i: NonNullIfTrue ? 0 : 1);
3162 BasicBlockEdge Edge(BI->getParent(), NonNullSuccessor);
3163 if (DT->dominates(BBE: Edge, BB: CtxI->getParent()))
3164 return true;
3165 } else if (NonNullIfTrue && isGuard(U: Curr) &&
3166 DT->dominates(Def: cast<Instruction>(Val: Curr), User: CtxI)) {
3167 return true;
3168 }
3169 }
3170 }
3171 }
3172
3173 return false;
3174}
3175
3176/// Does the 'Range' metadata (which must be a valid MD_range operand list)
3177/// ensure that the value it's attached to is never Value? 'RangeType' is
3178/// is the type of the value described by the range.
3179static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value) {
3180 const unsigned NumRanges = Ranges->getNumOperands() / 2;
3181 assert(NumRanges >= 1);
3182 for (unsigned i = 0; i < NumRanges; ++i) {
3183 ConstantInt *Lower =
3184 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 0));
3185 ConstantInt *Upper =
3186 mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: 2 * i + 1));
3187 ConstantRange Range(Lower->getValue(), Upper->getValue());
3188 if (Range.contains(Val: Value))
3189 return false;
3190 }
3191 return true;
3192}
3193
3194/// Try to detect a recurrence that monotonically increases/decreases from a
3195/// non-zero starting value. These are common as induction variables.
3196static bool isNonZeroRecurrence(const PHINode *PN) {
3197 BinaryOperator *BO = nullptr;
3198 Value *Start = nullptr, *Step = nullptr;
3199 const APInt *StartC, *StepC;
3200 if (!matchSimpleRecurrence(P: PN, BO, Start, Step) ||
3201 !match(V: Start, P: m_APInt(Res&: StartC)) || StartC->isZero())
3202 return false;
3203
3204 switch (BO->getOpcode()) {
3205 case Instruction::Add:
3206 // Starting from non-zero and stepping away from zero can never wrap back
3207 // to zero.
3208 return BO->hasNoUnsignedWrap() ||
3209 (BO->hasNoSignedWrap() && match(V: Step, P: m_APInt(Res&: StepC)) &&
3210 StartC->isNegative() == StepC->isNegative());
3211 case Instruction::Mul:
3212 return (BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap()) &&
3213 match(V: Step, P: m_APInt(Res&: StepC)) && !StepC->isZero();
3214 case Instruction::Shl:
3215 return BO->hasNoUnsignedWrap() || BO->hasNoSignedWrap();
3216 case Instruction::AShr:
3217 case Instruction::LShr:
3218 return BO->isExact();
3219 case Instruction::Or:
3220 return true;
3221 default:
3222 return false;
3223 }
3224}
3225
3226static bool matchOpWithOpEqZero(Value *Op0, Value *Op1) {
3227 return match(V: Op0, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3228 L: m_Specific(V: Op1), R: m_Zero()))) ||
3229 match(V: Op1, P: m_ZExtOrSExt(Op: m_SpecificICmp(MatchPred: ICmpInst::ICMP_EQ,
3230 L: m_Specific(V: Op0), R: m_Zero())));
3231}
3232
3233/// Return true if "addr(P) + Off" cannot be zero. The object P points to
3234/// cannot cross the unsigned address space boundary, including the address one
3235/// past its end, so an offset that stays within the object cannot wrap around
3236/// to zero. This holds even if the object starts at the zero address, which is
3237/// all that null_pointer_is_valid permits, so a non-zero offset is enough.
3238static bool isNonZeroAddrPlusOffset(const SimplifyQuery &Q, Value *X,
3239 Value *Y) {
3240 const APInt *Off;
3241 if (!match(V: Y, P: m_APInt(Res&: Off)) || Off->isZero())
3242 return false;
3243
3244 // ptrtoaddr always produces the address. ptrtoint only does so when its
3245 // result is at least as wide as the address: a narrower result is truncated
3246 // and may wrap to zero, while bits above the address can only make the
3247 // result more non-zero.
3248 Value *P;
3249 if (!match(V: X, P: m_PtrToAddr(Op: m_Value(V&: P))) &&
3250 !(match(V: X, P: m_PtrToInt(Op: m_Value(V&: P))) &&
3251 X->getType()->getScalarSizeInBits() >=
3252 Q.DL.getAddressSizeInBits(Ty: P->getType())))
3253 return false;
3254
3255 // A vector of pointers has no single object whose size can be taken, and
3256 // getPointerDereferenceableBytes() requires a plain pointer.
3257 if (!P->getType()->isPointerTy())
3258 return false;
3259
3260 bool CanBeNull;
3261 uint64_t DerefBytes = P->getPointerDereferenceableBytes(
3262 DL: Q.DL, CanBeNull, /*CanBeFreed=*/nullptr);
3263 if (CanBeNull || DerefBytes == 0 || !Off->ule(RHS: DerefBytes))
3264 return false;
3265
3266 return true;
3267}
3268
3269static bool isNonZeroAdd(const APInt &DemandedElts, const SimplifyQuery &Q,
3270 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3271 bool NUW, unsigned Depth) {
3272 if (isNonZeroAddrPlusOffset(Q, X, Y) || isNonZeroAddrPlusOffset(Q, X: Y, Y: X))
3273 return true;
3274
3275 // (X + (X != 0)) is non zero
3276 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3277 return true;
3278
3279 if (NUW)
3280 return isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3281 isKnownNonZero(V: X, DemandedElts, Q, Depth);
3282
3283 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3284 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3285
3286 // If X and Y are both non-negative (as signed values) then their sum is not
3287 // zero unless both X and Y are zero.
3288 if (XKnown.isNonNegative() && YKnown.isNonNegative())
3289 if (isKnownNonZero(V: Y, DemandedElts, Q, Depth) ||
3290 isKnownNonZero(V: X, DemandedElts, Q, Depth))
3291 return true;
3292
3293 // If X and Y are both negative (as signed values) then their sum is not
3294 // zero unless both X and Y equal INT_MIN.
3295 if (XKnown.isNegative() && YKnown.isNegative()) {
3296 APInt Mask = APInt::getSignedMaxValue(numBits: BitWidth);
3297 // The sign bit of X is set. If some other bit is set then X is not equal
3298 // to INT_MIN.
3299 if (XKnown.One.intersects(RHS: Mask))
3300 return true;
3301 // The sign bit of Y is set. If some other bit is set then Y is not equal
3302 // to INT_MIN.
3303 if (YKnown.One.intersects(RHS: Mask))
3304 return true;
3305 }
3306
3307 // The sum of a non-negative number and a power of two is not zero.
3308 if (XKnown.isNonNegative() &&
3309 isKnownToBeAPowerOfTwo(V: Y, /*OrZero*/ false, Q, Depth))
3310 return true;
3311 if (YKnown.isNonNegative() &&
3312 isKnownToBeAPowerOfTwo(V: X, /*OrZero*/ false, Q, Depth))
3313 return true;
3314
3315 return KnownBits::add(LHS: XKnown, RHS: YKnown, NSW, NUW).isNonZero();
3316}
3317
3318static bool isNonZeroSub(const APInt &DemandedElts, const SimplifyQuery &Q,
3319 unsigned BitWidth, Value *X, Value *Y,
3320 unsigned Depth) {
3321 // (X - (X != 0)) is non zero
3322 // ((X != 0) - X) is non zero
3323 if (matchOpWithOpEqZero(Op0: X, Op1: Y))
3324 return true;
3325
3326 // TODO: Move this case into isKnownNonEqual().
3327 if (auto *C = dyn_cast<Constant>(Val: X))
3328 if (C->isNullValue() && isKnownNonZero(V: Y, DemandedElts, Q, Depth))
3329 return true;
3330
3331 return ::isKnownNonEqual(V1: X, V2: Y, DemandedElts, Q, Depth);
3332}
3333
3334static bool isNonZeroMul(const APInt &DemandedElts, const SimplifyQuery &Q,
3335 unsigned BitWidth, Value *X, Value *Y, bool NSW,
3336 bool NUW, unsigned Depth) {
3337 // If X and Y are non-zero then so is X * Y as long as the multiplication
3338 // does not overflow.
3339 if (NSW || NUW)
3340 return isKnownNonZero(V: X, DemandedElts, Q, Depth) &&
3341 isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3342
3343 // If either X or Y is odd, then if the other is non-zero the result can't
3344 // be zero.
3345 KnownBits XKnown = computeKnownBits(V: X, DemandedElts, Q, Depth);
3346 if (XKnown.One[0])
3347 return isKnownNonZero(V: Y, DemandedElts, Q, Depth);
3348
3349 KnownBits YKnown = computeKnownBits(V: Y, DemandedElts, Q, Depth);
3350 if (YKnown.One[0])
3351 return XKnown.isNonZero() || isKnownNonZero(V: X, DemandedElts, Q, Depth);
3352
3353 // If there exists any subset of X (sX) and subset of Y (sY) s.t sX * sY is
3354 // non-zero, then X * Y is non-zero. We can find sX and sY by just taking
3355 // the lowest known One of X and Y. If they are non-zero, the result
3356 // must be non-zero. We can check if LSB(X) * LSB(Y) != 0 by doing
3357 // X.CountLeadingZeros + Y.CountLeadingZeros < BitWidth.
3358 return (XKnown.countMaxTrailingZeros() + YKnown.countMaxTrailingZeros()) <
3359 BitWidth;
3360}
3361
3362static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
3363 const SimplifyQuery &Q, const KnownBits &KnownVal,
3364 unsigned Depth) {
3365 auto ShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3366 switch (I->getOpcode()) {
3367 case Instruction::Shl:
3368 return Lhs.shl(ShiftAmt: Rhs);
3369 case Instruction::LShr:
3370 return Lhs.lshr(ShiftAmt: Rhs);
3371 case Instruction::AShr:
3372 return Lhs.ashr(ShiftAmt: Rhs);
3373 default:
3374 llvm_unreachable("Unknown Shift Opcode");
3375 }
3376 };
3377
3378 auto InvShiftOp = [&](const APInt &Lhs, const APInt &Rhs) {
3379 switch (I->getOpcode()) {
3380 case Instruction::Shl:
3381 return Lhs.lshr(ShiftAmt: Rhs);
3382 case Instruction::LShr:
3383 case Instruction::AShr:
3384 return Lhs.shl(ShiftAmt: Rhs);
3385 default:
3386 llvm_unreachable("Unknown Shift Opcode");
3387 }
3388 };
3389
3390 if (KnownVal.isUnknown())
3391 return false;
3392
3393 KnownBits KnownCnt =
3394 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3395 APInt MaxShift = KnownCnt.getMaxValue();
3396 unsigned NumBits = KnownVal.getBitWidth();
3397 if (MaxShift.uge(RHS: NumBits))
3398 return false;
3399
3400 if (!ShiftOp(KnownVal.One, MaxShift).isZero())
3401 return true;
3402
3403 // If all of the bits shifted out are known to be zero, and Val is known
3404 // non-zero then at least one non-zero bit must remain.
3405 if (InvShiftOp(KnownVal.Zero, NumBits - MaxShift)
3406 .eq(RHS: InvShiftOp(APInt::getAllOnes(numBits: NumBits), NumBits - MaxShift)) &&
3407 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth))
3408 return true;
3409
3410 return false;
3411}
3412
3413static bool isKnownNonZeroFromOperator(const Operator *I,
3414 const APInt &DemandedElts,
3415 const SimplifyQuery &Q, unsigned Depth) {
3416 unsigned BitWidth = getBitWidth(Ty: I->getType()->getScalarType(), DL: Q.DL);
3417 switch (I->getOpcode()) {
3418 case Instruction::Alloca:
3419 // Alloca never returns null, malloc might.
3420 return I->getType()->getPointerAddressSpace() == 0;
3421 case Instruction::GetElementPtr:
3422 if (I->getType()->isPointerTy())
3423 return isGEPKnownNonNull(GEP: cast<GEPOperator>(Val: I), Q, Depth);
3424 break;
3425 case Instruction::BitCast: {
3426 // We need to be a bit careful here. We can only peek through the bitcast
3427 // if the scalar size of elements in the operand are smaller than and a
3428 // multiple of the size they are casting too. Take three cases:
3429 //
3430 // 1) Unsafe:
3431 // bitcast <2 x i16> %NonZero to <4 x i8>
3432 //
3433 // %NonZero can have 2 non-zero i16 elements, but isKnownNonZero on a
3434 // <4 x i8> requires that all 4 i8 elements be non-zero which isn't
3435 // guranteed (imagine just sign bit set in the 2 i16 elements).
3436 //
3437 // 2) Unsafe:
3438 // bitcast <4 x i3> %NonZero to <3 x i4>
3439 //
3440 // Even though the scalar size of the src (`i3`) is smaller than the
3441 // scalar size of the dst `i4`, because `i3` is not a multiple of `i4`
3442 // its possible for the `3 x i4` elements to be zero because there are
3443 // some elements in the destination that don't contain any full src
3444 // element.
3445 //
3446 // 3) Safe:
3447 // bitcast <4 x i8> %NonZero to <2 x i16>
3448 //
3449 // This is always safe as non-zero in the 4 i8 elements implies
3450 // non-zero in the combination of any two adjacent ones. Since i8 is a
3451 // multiple of i16, each i16 is guranteed to have 2 full i8 elements.
3452 // This all implies the 2 i16 elements are non-zero.
3453 Type *FromTy = I->getOperand(i: 0)->getType();
3454 if ((FromTy->isIntOrIntVectorTy() || FromTy->isPtrOrPtrVectorTy()) &&
3455 (BitWidth % getBitWidth(Ty: FromTy->getScalarType(), DL: Q.DL)) == 0)
3456 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3457 } break;
3458 case Instruction::IntToPtr:
3459 // Note that we have to take special care to avoid looking through
3460 // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
3461 // as casts that can alter the value, e.g., AddrSpaceCasts.
3462 if (!isa<ScalableVectorType>(Val: I->getType()) &&
3463 Q.DL.getTypeSizeInBits(Ty: I->getOperand(i: 0)->getType()).getFixedValue() <=
3464 Q.DL.getTypeSizeInBits(Ty: I->getType()).getFixedValue())
3465 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3466 break;
3467 case Instruction::PtrToAddr:
3468 // isKnownNonZero() for pointers refers to the address bits being non-zero,
3469 // so we can directly forward.
3470 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3471 case Instruction::PtrToInt:
3472 // For inttoptr, make sure the result size is >= the address size. If the
3473 // address is non-zero, any larger value is also non-zero.
3474 if (Q.DL.getAddressSizeInBits(Ty: I->getOperand(i: 0)->getType()) <=
3475 I->getType()->getScalarSizeInBits())
3476 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3477 break;
3478 case Instruction::Trunc:
3479 // nuw/nsw trunc preserves zero/non-zero status of input.
3480 if (auto *TI = dyn_cast<TruncInst>(Val: I))
3481 if (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap())
3482 return isKnownNonZero(V: TI->getOperand(i_nocapture: 0), DemandedElts, Q, Depth);
3483 break;
3484
3485 // Iff x - y != 0, then x ^ y != 0
3486 // Therefore we can do the same exact checks
3487 case Instruction::Xor:
3488 case Instruction::Sub:
3489 return isNonZeroSub(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3490 Y: I->getOperand(i: 1), Depth);
3491 case Instruction::Or:
3492 // (X | (X != 0)) is non zero
3493 if (matchOpWithOpEqZero(Op0: I->getOperand(i: 0), Op1: I->getOperand(i: 1)))
3494 return true;
3495 // X | Y != 0 if X != Y.
3496 if (isKnownNonEqual(V1: I->getOperand(i: 0), V2: I->getOperand(i: 1), DemandedElts, Q,
3497 Depth))
3498 return true;
3499 // X | Y != 0 if X != 0 or Y != 0.
3500 return isKnownNonZero(V: I->getOperand(i: 1), DemandedElts, Q, Depth) ||
3501 isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3502 case Instruction::SExt:
3503 case Instruction::ZExt:
3504 // ext X != 0 if X != 0.
3505 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3506
3507 case Instruction::Shl: {
3508 // shl nsw/nuw can't remove any non-zero bits.
3509 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3510 if (Q.IIQ.hasNoUnsignedWrap(Op: BO) || Q.IIQ.hasNoSignedWrap(Op: BO))
3511 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3512
3513 // shl X, Y != 0 if X is odd. Note that the value of the shift is undefined
3514 // if the lowest bit is shifted off the end.
3515 KnownBits Known(BitWidth);
3516 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Known, Q, Depth);
3517 if (Known.One[0])
3518 return true;
3519
3520 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3521 }
3522 case Instruction::LShr:
3523 case Instruction::AShr: {
3524 // shr exact can only shift out zero bits.
3525 const PossiblyExactOperator *BO = cast<PossiblyExactOperator>(Val: I);
3526 if (BO->isExact())
3527 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3528
3529 // shr X, Y != 0 if X is negative. Note that the value of the shift is not
3530 // defined if the sign bit is shifted off the end.
3531 KnownBits Known =
3532 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3533 if (Known.isNegative())
3534 return true;
3535
3536 // shr (add nuw A, B), C is non-zero if A or B has a known-one bit at
3537 // position >= C, because the sum >= max(A, B).
3538 Value *A, *B;
3539 const APInt *C;
3540 if (Depth + 1 < MaxAnalysisRecursionDepth &&
3541 match(V: I->getOperand(i: 0), P: m_NUWAdd(L: m_Value(V&: A), R: m_Value(V&: B))) &&
3542 match(V: I->getOperand(i: 1), P: m_APInt(Res&: C)) && C->ult(RHS: BitWidth)) {
3543 KnownBits KnownA = computeKnownBits(V: A, DemandedElts, Q, Depth: Depth + 1);
3544 if (!KnownA.One.lshr(ShiftAmt: *C).isZero())
3545 return true;
3546 KnownBits KnownB = computeKnownBits(V: B, DemandedElts, Q, Depth: Depth + 1);
3547 if (!KnownB.One.lshr(ShiftAmt: *C).isZero())
3548 return true;
3549 }
3550
3551 return isNonZeroShift(I, DemandedElts, Q, KnownVal: Known, Depth);
3552 }
3553 case Instruction::UDiv:
3554 case Instruction::SDiv: {
3555 // X / Y
3556 // div exact can only produce a zero if the dividend is zero.
3557 if (cast<PossiblyExactOperator>(Val: I)->isExact())
3558 return isKnownNonZero(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3559
3560 KnownBits XKnown =
3561 computeKnownBits(V: I->getOperand(i: 0), DemandedElts, Q, Depth);
3562 // If X is fully unknown we won't be able to figure anything out so don't
3563 // both computing knownbits for Y.
3564 if (XKnown.isUnknown())
3565 return false;
3566
3567 KnownBits YKnown =
3568 computeKnownBits(V: I->getOperand(i: 1), DemandedElts, Q, Depth);
3569 if (I->getOpcode() == Instruction::SDiv) {
3570 // For signed division need to compare abs value of the operands.
3571 XKnown = XKnown.abs(/*IntMinIsPoison*/ false);
3572 YKnown = YKnown.abs(/*IntMinIsPoison*/ false);
3573 }
3574 // If X u>= Y then div is non zero (0/0 is UB).
3575 std::optional<bool> XUgeY = KnownBits::uge(LHS: XKnown, RHS: YKnown);
3576 // If X is total unknown or X u< Y we won't be able to prove non-zero
3577 // with compute known bits so just return early.
3578 return XUgeY && *XUgeY;
3579 }
3580 case Instruction::Add: {
3581 // X + Y.
3582
3583 // If Add has nuw wrap flag, then if either X or Y is non-zero the result is
3584 // non-zero.
3585 auto *BO = cast<OverflowingBinaryOperator>(Val: I);
3586 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3587 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3588 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3589 }
3590 case Instruction::Mul: {
3591 const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(Val: I);
3592 return isNonZeroMul(DemandedElts, Q, BitWidth, X: I->getOperand(i: 0),
3593 Y: I->getOperand(i: 1), NSW: Q.IIQ.hasNoSignedWrap(Op: BO),
3594 NUW: Q.IIQ.hasNoUnsignedWrap(Op: BO), Depth);
3595 }
3596 case Instruction::Select: {
3597 // (C ? X : Y) != 0 if X != 0 and Y != 0.
3598
3599 // First check if the arm is non-zero using `isKnownNonZero`. If that fails,
3600 // then see if the select condition implies the arm is non-zero. For example
3601 // (X != 0 ? X : Y), we know the true arm is non-zero as the `X` "return" is
3602 // dominated by `X != 0`.
3603 auto SelectArmIsNonZero = [&](bool IsTrueArm) {
3604 Value *Op;
3605 Op = IsTrueArm ? I->getOperand(i: 1) : I->getOperand(i: 2);
3606 // Op is trivially non-zero.
3607 if (isKnownNonZero(V: Op, DemandedElts, Q, Depth))
3608 return true;
3609
3610 // The condition of the select dominates the true/false arm. Check if the
3611 // condition implies that a given arm is non-zero.
3612 Value *X;
3613 CmpPredicate Pred;
3614 if (!match(V: I->getOperand(i: 0), P: m_c_ICmp(Pred, L: m_Specific(V: Op), R: m_Value(V&: X))))
3615 return false;
3616
3617 if (!IsTrueArm)
3618 Pred = ICmpInst::getInversePredicate(pred: Pred);
3619
3620 return cmpExcludesZero(Pred, RHS: X);
3621 };
3622
3623 if (SelectArmIsNonZero(/* IsTrueArm */ true) &&
3624 SelectArmIsNonZero(/* IsTrueArm */ false))
3625 return true;
3626 break;
3627 }
3628 case Instruction::PHI: {
3629 auto *PN = cast<PHINode>(Val: I);
3630 if (Q.IIQ.UseInstrInfo && isNonZeroRecurrence(PN))
3631 return true;
3632
3633 // Check if all incoming values are non-zero using recursion.
3634 SimplifyQuery RecQ = Q.getWithoutCondContext();
3635 unsigned NewDepth = std::max(a: Depth, b: MaxAnalysisRecursionDepth - 1);
3636 return llvm::all_of(Range: PN->operands(), P: [&](const Use &U) {
3637 if (U.get() == PN)
3638 return true;
3639 RecQ.CtxI = PN->getIncomingBlock(U)->getTerminator();
3640 // Check if the branch on the phi excludes zero.
3641 CmpPredicate Pred;
3642 Value *X;
3643 BasicBlock *TrueSucc, *FalseSucc;
3644 if (match(V: RecQ.CtxI,
3645 P: m_Br(C: m_c_ICmp(Pred, L: m_Specific(V: U.get()), R: m_Value(V&: X)),
3646 T: m_BasicBlock(V&: TrueSucc), F: m_BasicBlock(V&: FalseSucc)))) {
3647 // Check for cases of duplicate successors.
3648 if ((TrueSucc == PN->getParent()) != (FalseSucc == PN->getParent())) {
3649 // If we're using the false successor, invert the predicate.
3650 if (FalseSucc == PN->getParent())
3651 Pred = CmpInst::getInversePredicate(pred: Pred);
3652 if (cmpExcludesZero(Pred, RHS: X))
3653 return true;
3654 }
3655 }
3656 // Finally recurse on the edge and check it directly.
3657 return isKnownNonZero(V: U.get(), DemandedElts, Q: RecQ, Depth: NewDepth);
3658 });
3659 }
3660 case Instruction::InsertElement: {
3661 if (isa<ScalableVectorType>(Val: I->getType()))
3662 break;
3663
3664 const Value *Vec = I->getOperand(i: 0);
3665 const Value *Elt = I->getOperand(i: 1);
3666 auto *CIdx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
3667
3668 unsigned NumElts = DemandedElts.getBitWidth();
3669 APInt DemandedVecElts = DemandedElts;
3670 bool SkipElt = false;
3671 // If we know the index we are inserting too, clear it from Vec check.
3672 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
3673 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
3674 SkipElt = !DemandedElts[CIdx->getZExtValue()];
3675 }
3676
3677 // Result is zero if Elt is non-zero and rest of the demanded elts in Vec
3678 // are non-zero.
3679 return (SkipElt || isKnownNonZero(V: Elt, Q, Depth)) &&
3680 (DemandedVecElts.isZero() ||
3681 isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth));
3682 }
3683 case Instruction::ExtractElement:
3684 if (const auto *EEI = dyn_cast<ExtractElementInst>(Val: I)) {
3685 const Value *Vec = EEI->getVectorOperand();
3686 const Value *Idx = EEI->getIndexOperand();
3687 auto *CIdx = dyn_cast<ConstantInt>(Val: Idx);
3688 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
3689 unsigned NumElts = VecTy->getNumElements();
3690 APInt DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
3691 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
3692 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
3693 return isKnownNonZero(V: Vec, DemandedElts: DemandedVecElts, Q, Depth);
3694 }
3695 }
3696 break;
3697 case Instruction::ShuffleVector: {
3698 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I);
3699 if (!Shuf)
3700 break;
3701 APInt DemandedLHS, DemandedRHS;
3702 // For undef elements, we don't know anything about the common state of
3703 // the shuffle result.
3704 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
3705 break;
3706 // If demanded elements for both vecs are non-zero, the shuffle is non-zero.
3707 return (DemandedRHS.isZero() ||
3708 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 1), DemandedElts: DemandedRHS, Q, Depth)) &&
3709 (DemandedLHS.isZero() ||
3710 isKnownNonZero(V: Shuf->getOperand(i_nocapture: 0), DemandedElts: DemandedLHS, Q, Depth));
3711 }
3712 case Instruction::Freeze:
3713 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth) &&
3714 isGuaranteedNotToBePoison(V: I->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT,
3715 Depth);
3716 case Instruction::Load: {
3717 auto *LI = cast<LoadInst>(Val: I);
3718 // A Load tagged with nonnull or dereferenceable with null pointer undefined
3719 // is never null.
3720 if (auto *PtrT = dyn_cast<PointerType>(Val: I->getType())) {
3721 if (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_nonnull) ||
3722 (Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_dereferenceable) &&
3723 !NullPointerIsDefined(F: LI->getFunction(), AS: PtrT->getAddressSpace())))
3724 return true;
3725 } else if (MDNode *Ranges = Q.IIQ.getMetadata(I: LI, KindID: LLVMContext::MD_range)) {
3726 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3727 }
3728
3729 // No need to fall through to computeKnownBits as range metadata is already
3730 // handled in isKnownNonZero.
3731 return false;
3732 }
3733 case Instruction::ExtractValue: {
3734 const WithOverflowInst *WO;
3735 if (match(V: I, P: m_ExtractValue<0>(V: m_WithOverflowInst(I&: WO)))) {
3736 switch (WO->getBinaryOp()) {
3737 default:
3738 break;
3739 case Instruction::Add:
3740 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3741 Y: WO->getArgOperand(i: 1),
3742 /*NSW=*/false,
3743 /*NUW=*/false, Depth);
3744 case Instruction::Sub:
3745 return isNonZeroSub(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3746 Y: WO->getArgOperand(i: 1), Depth);
3747 case Instruction::Mul:
3748 return isNonZeroMul(DemandedElts, Q, BitWidth, X: WO->getArgOperand(i: 0),
3749 Y: WO->getArgOperand(i: 1),
3750 /*NSW=*/false, /*NUW=*/false, Depth);
3751 break;
3752 }
3753 }
3754 break;
3755 }
3756 case Instruction::Call:
3757 case Instruction::Invoke: {
3758 const auto *Call = cast<CallBase>(Val: I);
3759 if (I->getType()->isPointerTy()) {
3760 if (Call->isReturnNonNull())
3761 return true;
3762 if (const auto *RP = getArgumentAliasingToReturnedPointer(
3763 Call, /*MustPreserveOffset=*/true))
3764 return isKnownNonZero(V: RP, Q, Depth);
3765 } else {
3766 if (MDNode *Ranges = Q.IIQ.getMetadata(I: Call, KindID: LLVMContext::MD_range))
3767 return rangeMetadataExcludesValue(Ranges, Value: APInt::getZero(numBits: BitWidth));
3768 if (std::optional<ConstantRange> Range = Call->getRange()) {
3769 const APInt ZeroValue(Range->getBitWidth(), 0);
3770 if (!Range->contains(Val: ZeroValue))
3771 return true;
3772 }
3773 if (const Value *RV = Call->getReturnedArgOperand())
3774 if (RV->getType() == I->getType() && isKnownNonZero(V: RV, Q, Depth))
3775 return true;
3776 }
3777
3778 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3779 switch (II->getIntrinsicID()) {
3780 case Intrinsic::sshl_sat:
3781 case Intrinsic::ushl_sat:
3782 case Intrinsic::abs:
3783 case Intrinsic::bitreverse:
3784 case Intrinsic::bswap:
3785 case Intrinsic::ctpop:
3786 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3787 // NB: We don't do usub_sat here as in any case we can prove its
3788 // non-zero, we will fold it to `sub nuw` in InstCombine.
3789 case Intrinsic::ssub_sat:
3790 // For most types, if x != y then ssub.sat x, y != 0. But
3791 // ssub.sat.i1 0, -1 = 0, because 1 saturates to 0. This means
3792 // isNonZeroSub will do the wrong thing for ssub.sat.i1.
3793 if (BitWidth == 1)
3794 return false;
3795 return isNonZeroSub(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3796 Y: II->getArgOperand(i: 1), Depth);
3797 case Intrinsic::sadd_sat:
3798 return isNonZeroAdd(DemandedElts, Q, BitWidth, X: II->getArgOperand(i: 0),
3799 Y: II->getArgOperand(i: 1),
3800 /*NSW=*/true, /* NUW=*/false, Depth);
3801 // Vec reverse preserves zero/non-zero status from input vec.
3802 case Intrinsic::vector_reverse:
3803 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
3804 Q, Depth);
3805 // umin/smin/smax/smin/or of all non-zero elements is always non-zero.
3806 case Intrinsic::vector_reduce_or:
3807 case Intrinsic::vector_reduce_umax:
3808 case Intrinsic::vector_reduce_umin:
3809 case Intrinsic::vector_reduce_smax:
3810 case Intrinsic::vector_reduce_smin:
3811 return isKnownNonZero(V: II->getArgOperand(i: 0), Q, Depth);
3812 case Intrinsic::umax:
3813 case Intrinsic::uadd_sat:
3814 // umax(X, (X != 0)) is non zero
3815 // X +usat (X != 0) is non zero
3816 if (matchOpWithOpEqZero(Op0: II->getArgOperand(i: 0), Op1: II->getArgOperand(i: 1)))
3817 return true;
3818
3819 return isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth) ||
3820 isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3821 case Intrinsic::smax: {
3822 // If either arg is strictly positive the result is non-zero. Otherwise
3823 // the result is non-zero if both ops are non-zero.
3824 auto IsNonZero = [&](Value *Op, std::optional<bool> &OpNonZero,
3825 const KnownBits &OpKnown) {
3826 if (!OpNonZero.has_value())
3827 OpNonZero = OpKnown.isNonZero() ||
3828 isKnownNonZero(V: Op, DemandedElts, Q, Depth);
3829 return *OpNonZero;
3830 };
3831 // Avoid re-computing isKnownNonZero.
3832 std::optional<bool> Op0NonZero, Op1NonZero;
3833 KnownBits Op1Known =
3834 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3835 if (Op1Known.isNonNegative() &&
3836 IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known))
3837 return true;
3838 KnownBits Op0Known =
3839 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3840 if (Op0Known.isNonNegative() &&
3841 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known))
3842 return true;
3843 return IsNonZero(II->getArgOperand(i: 1), Op1NonZero, Op1Known) &&
3844 IsNonZero(II->getArgOperand(i: 0), Op0NonZero, Op0Known);
3845 }
3846 case Intrinsic::smin: {
3847 // If either arg is negative the result is non-zero. Otherwise
3848 // the result is non-zero if both ops are non-zero.
3849 KnownBits Op1Known =
3850 computeKnownBits(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3851 if (Op1Known.isNegative())
3852 return true;
3853 KnownBits Op0Known =
3854 computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3855 if (Op0Known.isNegative())
3856 return true;
3857
3858 if (Op1Known.isNonZero() && Op0Known.isNonZero())
3859 return true;
3860 }
3861 [[fallthrough]];
3862 case Intrinsic::umin:
3863 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth) &&
3864 isKnownNonZero(V: II->getArgOperand(i: 1), DemandedElts, Q, Depth);
3865 case Intrinsic::cttz:
3866 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3867 .Zero[0];
3868 case Intrinsic::ctlz:
3869 return computeKnownBits(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth)
3870 .isNonNegative();
3871 case Intrinsic::fshr:
3872 case Intrinsic::fshl:
3873 // If Op0 == Op1, this is a rotate. rotate(x, y) != 0 iff x != 0.
3874 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1))
3875 return isKnownNonZero(V: II->getArgOperand(i: 0), DemandedElts, Q, Depth);
3876 break;
3877 case Intrinsic::vscale:
3878 return true;
3879 case Intrinsic::experimental_get_vector_length:
3880 return isKnownNonZero(V: I->getOperand(i: 0), Q, Depth);
3881 default:
3882 break;
3883 }
3884 break;
3885 }
3886
3887 return false;
3888 }
3889 }
3890
3891 KnownBits Known(BitWidth);
3892 computeKnownBits(V: I, DemandedElts, Known, Q, Depth);
3893 return Known.One != 0;
3894}
3895
3896/// Return true if the given value is known to be non-zero when defined. For
3897/// vectors, return true if every demanded element is known to be non-zero when
3898/// defined. For pointers, if the context instruction and dominator tree are
3899/// specified, perform context-sensitive analysis and return true if the
3900/// pointer couldn't possibly be null at the specified instruction.
3901/// Supports values with integer or pointer type and vectors of integers.
3902bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
3903 const SimplifyQuery &Q, unsigned Depth) {
3904 Type *Ty = V->getType();
3905
3906#ifndef NDEBUG
3907 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
3908
3909 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
3910 assert(
3911 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
3912 "DemandedElt width should equal the fixed vector number of elements");
3913 } else {
3914 assert(DemandedElts == APInt(1, 1) &&
3915 "DemandedElt width should be 1 for scalars");
3916 }
3917#endif
3918
3919 if (auto *C = dyn_cast<Constant>(Val: V)) {
3920 if (C->isNullValue())
3921 return false;
3922 if (isa<ConstantInt>(Val: C))
3923 // Must be non-zero due to null test above.
3924 return true;
3925
3926 // For constant vectors, check that all elements are poison or known
3927 // non-zero to determine that the whole vector is known non-zero.
3928 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Ty)) {
3929 for (unsigned i = 0, e = VecTy->getNumElements(); i != e; ++i) {
3930 if (!DemandedElts[i])
3931 continue;
3932 Constant *Elt = C->getAggregateElement(Elt: i);
3933 if (!Elt || Elt->isNullValue())
3934 return false;
3935 if (!isa<PoisonValue>(Val: Elt) && !isa<ConstantInt>(Val: Elt))
3936 return false;
3937 }
3938 return true;
3939 }
3940
3941 // Constant ptrauth can be null, iff the base pointer can be.
3942 if (auto *CPA = dyn_cast<ConstantPtrAuth>(Val: V))
3943 return isKnownNonZero(V: CPA->getPointer(), DemandedElts, Q, Depth);
3944
3945 // A global variable is non null unless extern weak or an absolute symbol
3946 // reference. An alias is also excluded, as it may resolve to a weak
3947 // undefined symbol and thus to the null address. Where null is a valid
3948 // address an allocation may start there, so nothing can be assumed; that
3949 // covers both null_pointer_is_valid and address spaces other than zero.
3950 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
3951 if (!NullPointerIsDefined(F: Q.getFunction(),
3952 AS: GV->getType()->getAddressSpace()) &&
3953 !GV->isAbsoluteSymbolRef() && !GV->hasExternalWeakLinkage() &&
3954 !isa<GlobalAlias>(Val: GV))
3955 return true;
3956 }
3957
3958 // For constant expressions, fall through to the Operator code below.
3959 if (!isa<ConstantExpr>(Val: V))
3960 return false;
3961 }
3962
3963 if (const auto *A = dyn_cast<Argument>(Val: V))
3964 if (std::optional<ConstantRange> Range = A->getRange()) {
3965 const APInt ZeroValue(Range->getBitWidth(), 0);
3966 if (!Range->contains(Val: ZeroValue))
3967 return true;
3968 }
3969
3970 if (!isa<Constant>(Val: V) && isKnownNonZeroFromAssume(V, Q))
3971 return true;
3972
3973 // Some of the tests below are recursive, so bail out if we hit the limit.
3974 if (Depth++ >= MaxAnalysisRecursionDepth)
3975 return false;
3976
3977 // Check for pointer simplifications.
3978
3979 if (PointerType *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
3980 // A byval, inalloca may not be null in a non-default addres space. A
3981 // nonnull argument is assumed never 0.
3982 if (const Argument *A = dyn_cast<Argument>(Val: V)) {
3983 if (((A->hasPassPointeeByValueCopyAttr() &&
3984 !NullPointerIsDefined(F: A->getParent(), AS: PtrTy->getAddressSpace())) ||
3985 A->hasNonNullAttr()))
3986 return true;
3987 }
3988 }
3989
3990 if (const auto *I = dyn_cast<Operator>(Val: V))
3991 if (isKnownNonZeroFromOperator(I, DemandedElts, Q, Depth))
3992 return true;
3993
3994 if (!isa<Constant>(Val: V) &&
3995 isKnownNonNullFromDominatingCondition(V, CtxI: Q.CtxI, DT: Q.DT))
3996 return true;
3997
3998 if (const Value *Stripped = stripNullTest(V))
3999 return isKnownNonZero(V: Stripped, DemandedElts, Q, Depth);
4000
4001 return false;
4002}
4003
4004bool llvm::isKnownNonZero(const Value *V, const SimplifyQuery &Q,
4005 unsigned Depth) {
4006 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
4007 APInt DemandedElts =
4008 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
4009 return ::isKnownNonZero(V, DemandedElts, Q, Depth);
4010}
4011
4012/// If the pair of operators are the same invertible function, return the
4013/// the operands of the function corresponding to each input. Otherwise,
4014/// return std::nullopt. An invertible function is one that is 1-to-1 and maps
4015/// every input value to exactly one output value. This is equivalent to
4016/// saying that Op1 and Op2 are equal exactly when the specified pair of
4017/// operands are equal, (except that Op1 and Op2 may be poison more often.)
4018static std::optional<std::pair<Value*, Value*>>
4019getInvertibleOperands(const Operator *Op1,
4020 const Operator *Op2) {
4021 if (Op1->getOpcode() != Op2->getOpcode())
4022 return std::nullopt;
4023
4024 auto getOperands = [&](unsigned OpNum) -> auto {
4025 return std::make_pair(x: Op1->getOperand(i: OpNum), y: Op2->getOperand(i: OpNum));
4026 };
4027
4028 switch (Op1->getOpcode()) {
4029 default:
4030 break;
4031 case Instruction::Or:
4032 if (!cast<PossiblyDisjointInst>(Val: Op1)->isDisjoint() ||
4033 !cast<PossiblyDisjointInst>(Val: Op2)->isDisjoint())
4034 break;
4035 [[fallthrough]];
4036 case Instruction::Xor:
4037 case Instruction::Add: {
4038 Value *Other;
4039 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 0)), R: m_Value(V&: Other))))
4040 return std::make_pair(x: Op1->getOperand(i: 1), y&: Other);
4041 if (match(V: Op2, P: m_c_BinOp(L: m_Specific(V: Op1->getOperand(i: 1)), R: m_Value(V&: Other))))
4042 return std::make_pair(x: Op1->getOperand(i: 0), y&: Other);
4043 break;
4044 }
4045 case Instruction::Sub:
4046 if (Op1->getOperand(i: 0) == Op2->getOperand(i: 0))
4047 return getOperands(1);
4048 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
4049 return getOperands(0);
4050 break;
4051 case Instruction::Mul: {
4052 // invertible if A * B == (A * B) mod 2^N where A, and B are integers
4053 // and N is the bitwdith. The nsw case is non-obvious, but proven by
4054 // alive2: https://alive2.llvm.org/ce/z/Z6D5qK
4055 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
4056 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
4057 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
4058 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
4059 break;
4060
4061 // Assume operand order has been canonicalized
4062 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1) &&
4063 isa<ConstantInt>(Val: Op1->getOperand(i: 1)) &&
4064 !cast<ConstantInt>(Val: Op1->getOperand(i: 1))->isZero())
4065 return getOperands(0);
4066 break;
4067 }
4068 case Instruction::Shl: {
4069 // Same as multiplies, with the difference that we don't need to check
4070 // for a non-zero multiply. Shifts always multiply by non-zero.
4071 auto *OBO1 = cast<OverflowingBinaryOperator>(Val: Op1);
4072 auto *OBO2 = cast<OverflowingBinaryOperator>(Val: Op2);
4073 if ((!OBO1->hasNoUnsignedWrap() || !OBO2->hasNoUnsignedWrap()) &&
4074 (!OBO1->hasNoSignedWrap() || !OBO2->hasNoSignedWrap()))
4075 break;
4076
4077 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
4078 return getOperands(0);
4079 break;
4080 }
4081 case Instruction::AShr:
4082 case Instruction::LShr: {
4083 auto *PEO1 = cast<PossiblyExactOperator>(Val: Op1);
4084 auto *PEO2 = cast<PossiblyExactOperator>(Val: Op2);
4085 if (!PEO1->isExact() || !PEO2->isExact())
4086 break;
4087
4088 if (Op1->getOperand(i: 1) == Op2->getOperand(i: 1))
4089 return getOperands(0);
4090 break;
4091 }
4092 case Instruction::SExt:
4093 case Instruction::ZExt:
4094 if (Op1->getOperand(i: 0)->getType() == Op2->getOperand(i: 0)->getType())
4095 return getOperands(0);
4096 break;
4097 case Instruction::PHI: {
4098 const PHINode *PN1 = cast<PHINode>(Val: Op1);
4099 const PHINode *PN2 = cast<PHINode>(Val: Op2);
4100
4101 // If PN1 and PN2 are both recurrences, can we prove the entire recurrences
4102 // are a single invertible function of the start values? Note that repeated
4103 // application of an invertible function is also invertible
4104 BinaryOperator *BO1 = nullptr;
4105 Value *Start1 = nullptr, *Step1 = nullptr;
4106 BinaryOperator *BO2 = nullptr;
4107 Value *Start2 = nullptr, *Step2 = nullptr;
4108 if (PN1->getParent() != PN2->getParent() ||
4109 !matchSimpleRecurrence(P: PN1, BO&: BO1, Start&: Start1, Step&: Step1) ||
4110 !matchSimpleRecurrence(P: PN2, BO&: BO2, Start&: Start2, Step&: Step2))
4111 break;
4112
4113 auto Values = getInvertibleOperands(Op1: cast<Operator>(Val: BO1),
4114 Op2: cast<Operator>(Val: BO2));
4115 if (!Values)
4116 break;
4117
4118 // We have to be careful of mutually defined recurrences here. Ex:
4119 // * X_i = X_(i-1) OP Y_(i-1), and Y_i = X_(i-1) OP V
4120 // * X_i = Y_i = X_(i-1) OP Y_(i-1)
4121 // The invertibility of these is complicated, and not worth reasoning
4122 // about (yet?).
4123 if (Values->first != PN1 || Values->second != PN2)
4124 break;
4125
4126 return std::make_pair(x&: Start1, y&: Start2);
4127 }
4128 }
4129 return std::nullopt;
4130}
4131
4132/// Return true if V1 == (binop V2, X), where X is known non-zero.
4133/// Only handle a small subset of binops where (binop V2, X) with non-zero X
4134/// implies V2 != V1.
4135static bool isModifyingBinopOfNonZero(const Value *V1, const Value *V2,
4136 const APInt &DemandedElts,
4137 const SimplifyQuery &Q, unsigned Depth) {
4138 const BinaryOperator *BO = dyn_cast<BinaryOperator>(Val: V1);
4139 if (!BO)
4140 return false;
4141 switch (BO->getOpcode()) {
4142 default:
4143 break;
4144 case Instruction::Or:
4145 if (!cast<PossiblyDisjointInst>(Val: V1)->isDisjoint())
4146 break;
4147 [[fallthrough]];
4148 case Instruction::Xor:
4149 case Instruction::Add:
4150 Value *Op = nullptr;
4151 if (V2 == BO->getOperand(i_nocapture: 0))
4152 Op = BO->getOperand(i_nocapture: 1);
4153 else if (V2 == BO->getOperand(i_nocapture: 1))
4154 Op = BO->getOperand(i_nocapture: 0);
4155 else
4156 return false;
4157 return isKnownNonZero(V: Op, DemandedElts, Q, Depth: Depth + 1);
4158 }
4159 return false;
4160}
4161
4162/// Return true if V2 == V1 * C, where V1 is known non-zero, C is not 0/1 and
4163/// the multiplication is nuw or nsw.
4164static bool isNonEqualMul(const Value *V1, const Value *V2,
4165 const APInt &DemandedElts, const SimplifyQuery &Q,
4166 unsigned Depth) {
4167 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
4168 const APInt *C;
4169 return match(V: OBO, P: m_Mul(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
4170 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
4171 !C->isZero() && !C->isOne() &&
4172 isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
4173 }
4174 return false;
4175}
4176
4177/// Return true if V2 == V1 << C, where V1 is known non-zero, C is not 0 and
4178/// the shift is nuw or nsw.
4179static bool isNonEqualShl(const Value *V1, const Value *V2,
4180 const APInt &DemandedElts, const SimplifyQuery &Q,
4181 unsigned Depth) {
4182 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V2)) {
4183 const APInt *C;
4184 return match(V: OBO, P: m_Shl(L: m_Specific(V: V1), R: m_APInt(Res&: C))) &&
4185 (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
4186 !C->isZero() && isKnownNonZero(V: V1, DemandedElts, Q, Depth: Depth + 1);
4187 }
4188 return false;
4189}
4190
4191static bool isNonEqualPHIs(const PHINode *PN1, const PHINode *PN2,
4192 const APInt &DemandedElts, const SimplifyQuery &Q,
4193 unsigned Depth) {
4194 // Check two PHIs are in same block.
4195 if (PN1->getParent() != PN2->getParent())
4196 return false;
4197
4198 SmallPtrSet<const BasicBlock *, 8> VisitedBBs;
4199 bool UsedFullRecursion = false;
4200 for (const BasicBlock *IncomBB : PN1->blocks()) {
4201 if (!VisitedBBs.insert(Ptr: IncomBB).second)
4202 continue; // Don't reprocess blocks that we have dealt with already.
4203 const Value *IV1 = PN1->getIncomingValueForBlock(BB: IncomBB);
4204 const Value *IV2 = PN2->getIncomingValueForBlock(BB: IncomBB);
4205 const APInt *C1, *C2;
4206 if (match(V: IV1, P: m_APInt(Res&: C1)) && match(V: IV2, P: m_APInt(Res&: C2)) && *C1 != *C2)
4207 continue;
4208
4209 // Only one pair of phi operands is allowed for full recursion.
4210 if (UsedFullRecursion)
4211 return false;
4212
4213 SimplifyQuery RecQ = Q.getWithoutCondContext();
4214 RecQ.CtxI = IncomBB->getTerminator();
4215 if (!isKnownNonEqual(V1: IV1, V2: IV2, DemandedElts, Q: RecQ, Depth: Depth + 1))
4216 return false;
4217 UsedFullRecursion = true;
4218 }
4219 return true;
4220}
4221
4222static bool isNonEqualSelect(const Value *V1, const Value *V2,
4223 const APInt &DemandedElts, const SimplifyQuery &Q,
4224 unsigned Depth) {
4225 const SelectInst *SI1 = dyn_cast<SelectInst>(Val: V1);
4226 if (!SI1)
4227 return false;
4228
4229 if (const SelectInst *SI2 = dyn_cast<SelectInst>(Val: V2)) {
4230 const Value *Cond1 = SI1->getCondition();
4231 const Value *Cond2 = SI2->getCondition();
4232 if (Cond1 == Cond2)
4233 return isKnownNonEqual(V1: SI1->getTrueValue(), V2: SI2->getTrueValue(),
4234 DemandedElts, Q, Depth: Depth + 1) &&
4235 isKnownNonEqual(V1: SI1->getFalseValue(), V2: SI2->getFalseValue(),
4236 DemandedElts, Q, Depth: Depth + 1);
4237 }
4238 return isKnownNonEqual(V1: SI1->getTrueValue(), V2, DemandedElts, Q, Depth: Depth + 1) &&
4239 isKnownNonEqual(V1: SI1->getFalseValue(), V2, DemandedElts, Q, Depth: Depth + 1);
4240}
4241
4242// Check to see if A is both a GEP and is the incoming value for a PHI in the
4243// loop, and B is either a ptr or another GEP. If the PHI has 2 incoming values,
4244// one of them being the recursive GEP A and the other a ptr at same base and at
4245// the same/higher offset than B we are only incrementing the pointer further in
4246// loop if offset of recursive GEP is greater than 0.
4247static bool isNonEqualPointersWithRecursiveGEP(const Value *A, const Value *B,
4248 const SimplifyQuery &Q) {
4249 if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
4250 return false;
4251
4252 auto *GEPA = dyn_cast<GEPOperator>(Val: A);
4253 if (!GEPA || GEPA->getNumIndices() != 1 || !isa<Constant>(Val: GEPA->idx_begin()))
4254 return false;
4255
4256 // Handle 2 incoming PHI values with one being a recursive GEP.
4257 auto *PN = dyn_cast<PHINode>(Val: GEPA->getPointerOperand());
4258 if (!PN || PN->getNumIncomingValues() != 2)
4259 return false;
4260
4261 // Search for the recursive GEP as an incoming operand, and record that as
4262 // Step.
4263 Value *Start = nullptr;
4264 Value *Step = const_cast<Value *>(A);
4265 if (PN->getIncomingValue(i: 0) == Step)
4266 Start = PN->getIncomingValue(i: 1);
4267 else if (PN->getIncomingValue(i: 1) == Step)
4268 Start = PN->getIncomingValue(i: 0);
4269 else
4270 return false;
4271
4272 // Other incoming node base should match the B base.
4273 // StartOffset >= OffsetB && StepOffset > 0?
4274 // StartOffset <= OffsetB && StepOffset < 0?
4275 // Is non-equal if above are true.
4276 // We use stripAndAccumulateInBoundsConstantOffsets to restrict the
4277 // optimisation to inbounds GEPs only.
4278 unsigned IndexWidth = Q.DL.getIndexTypeSizeInBits(Ty: Start->getType());
4279 APInt StartOffset(IndexWidth, 0);
4280 Start = Start->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StartOffset);
4281 APInt StepOffset(IndexWidth, 0);
4282 Step = Step->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: StepOffset);
4283
4284 // Check if Base Pointer of Step matches the PHI.
4285 if (Step != PN)
4286 return false;
4287 APInt OffsetB(IndexWidth, 0);
4288 B = B->stripAndAccumulateInBoundsConstantOffsets(DL: Q.DL, Offset&: OffsetB);
4289 return Start == B &&
4290 ((StartOffset.sge(RHS: OffsetB) && StepOffset.isStrictlyPositive()) ||
4291 (StartOffset.sle(RHS: OffsetB) && StepOffset.isNegative()));
4292}
4293
4294static bool isKnownNonEqualFromContext(const Value *V1, const Value *V2,
4295 const SimplifyQuery &Q, unsigned Depth) {
4296 if (!Q.CtxI)
4297 return false;
4298
4299 // Try to infer NonEqual based on information from dominating conditions.
4300 if (Q.DC && Q.DT) {
4301 auto IsKnownNonEqualFromDominatingCondition = [&](const Value *V) {
4302 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
4303 Value *Cond = BI->getCondition();
4304 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
4305 if (Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()) &&
4306 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4307 /*LHSIsTrue=*/true, Depth)
4308 .value_or(u: false))
4309 return true;
4310
4311 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
4312 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()) &&
4313 isImpliedCondition(LHS: Cond, RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4314 /*LHSIsTrue=*/false, Depth)
4315 .value_or(u: false))
4316 return true;
4317 }
4318
4319 return false;
4320 };
4321
4322 if (IsKnownNonEqualFromDominatingCondition(V1) ||
4323 IsKnownNonEqualFromDominatingCondition(V2))
4324 return true;
4325 }
4326
4327 if (!Q.AC)
4328 return false;
4329
4330 // Try to infer NonEqual based on information from assumptions.
4331 for (auto &AssumeVH : Q.AC->assumptionsFor(V: V1)) {
4332 if (!AssumeVH)
4333 continue;
4334 CallInst *I = cast<CallInst>(Val&: AssumeVH);
4335
4336 assert(I->getFunction() == Q.CtxI->getFunction() &&
4337 "Got assumption for the wrong function!");
4338 assert(I->getIntrinsicID() == Intrinsic::assume &&
4339 "must be an assume intrinsic");
4340
4341 if (isImpliedCondition(LHS: I->getArgOperand(i: 0), RHSPred: ICmpInst::ICMP_NE, RHSOp0: V1, RHSOp1: V2, DL: Q.DL,
4342 /*LHSIsTrue=*/true, Depth)
4343 .value_or(u: false) &&
4344 isValidAssumeForContext(I, Q))
4345 return true;
4346 }
4347
4348 return false;
4349}
4350
4351static bool isNonEqualURem(const Value *X, const Value *Rem,
4352 const SimplifyQuery &Q) {
4353 const Value *Y;
4354 if (!match(V: Rem, P: m_URem(L: m_Specific(V: X), R: m_Value(V&: Y))))
4355 return false;
4356
4357 // For a defined urem, X != X urem Y exactly when X u>= Y.
4358 // isTruePredicate does not handle UGE, so use the equivalent Y u<= X.
4359 if (isTruePredicate(Pred: ICmpInst::ICMP_ULE, LHS: Y, RHS: X))
4360 return true;
4361
4362 std::optional<bool> Implied =
4363 isImpliedByDomCondition(Pred: ICmpInst::ICMP_UGE, LHS: X, RHS: Y, ContextI: Q.CtxI, DL: Q.DL);
4364 return Implied && *Implied;
4365}
4366
4367/// Return true if it is known that V1 != V2.
4368static bool isKnownNonEqual(const Value *V1, const Value *V2,
4369 const APInt &DemandedElts, const SimplifyQuery &Q,
4370 unsigned Depth) {
4371 if (V1 == V2)
4372 return false;
4373 if (V1->getType() != V2->getType())
4374 // We can't look through casts yet.
4375 return false;
4376
4377 if (Depth >= MaxAnalysisRecursionDepth)
4378 return false;
4379
4380 // See if we can recurse through (exactly one of) our operands. This
4381 // requires our operation be 1-to-1 and map every input value to exactly
4382 // one output value. Such an operation is invertible.
4383 auto *O1 = dyn_cast<Operator>(Val: V1);
4384 auto *O2 = dyn_cast<Operator>(Val: V2);
4385 if (O1 && O2 && O1->getOpcode() == O2->getOpcode()) {
4386 if (auto Values = getInvertibleOperands(Op1: O1, Op2: O2))
4387 return isKnownNonEqual(V1: Values->first, V2: Values->second, DemandedElts, Q,
4388 Depth: Depth + 1);
4389
4390 if (const PHINode *PN1 = dyn_cast<PHINode>(Val: V1)) {
4391 const PHINode *PN2 = cast<PHINode>(Val: V2);
4392 // FIXME: This is missing a generalization to handle the case where one is
4393 // a PHI and another one isn't.
4394 if (isNonEqualPHIs(PN1, PN2, DemandedElts, Q, Depth))
4395 return true;
4396 };
4397 }
4398
4399 if (isModifyingBinopOfNonZero(V1, V2, DemandedElts, Q, Depth) ||
4400 isModifyingBinopOfNonZero(V1: V2, V2: V1, DemandedElts, Q, Depth))
4401 return true;
4402
4403 if (isNonEqualMul(V1, V2, DemandedElts, Q, Depth) ||
4404 isNonEqualMul(V1: V2, V2: V1, DemandedElts, Q, Depth))
4405 return true;
4406
4407 if (isNonEqualShl(V1, V2, DemandedElts, Q, Depth) ||
4408 isNonEqualShl(V1: V2, V2: V1, DemandedElts, Q, Depth))
4409 return true;
4410
4411 if (V1->getType()->isIntOrIntVectorTy()) {
4412 // Are any known bits in V1 contradictory to known bits in V2? If V1
4413 // has a known zero where V2 has a known one, they must not be equal.
4414 KnownBits Known1 = computeKnownBits(V: V1, DemandedElts, Q, Depth);
4415 if (!Known1.isUnknown()) {
4416 KnownBits Known2 = computeKnownBits(V: V2, DemandedElts, Q, Depth);
4417 if (Known1.Zero.intersects(RHS: Known2.One) ||
4418 Known2.Zero.intersects(RHS: Known1.One))
4419 return true;
4420 }
4421 }
4422
4423 if (isNonEqualSelect(V1, V2, DemandedElts, Q, Depth) ||
4424 isNonEqualSelect(V1: V2, V2: V1, DemandedElts, Q, Depth))
4425 return true;
4426
4427 if (isNonEqualPointersWithRecursiveGEP(A: V1, B: V2, Q) ||
4428 isNonEqualPointersWithRecursiveGEP(A: V2, B: V1, Q))
4429 return true;
4430
4431 Value *A, *B;
4432 // PtrToInts are NonEqual if their Ptrs are NonEqual.
4433 // Check PtrToInt type matches the pointer size.
4434 if (match(V: V1, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: A))) &&
4435 match(V: V2, P: m_PtrToIntSameSize(DL: Q.DL, Op: m_Value(V&: B))))
4436 return isKnownNonEqual(V1: A, V2: B, DemandedElts, Q, Depth: Depth + 1);
4437
4438 if (isNonEqualURem(X: V1, Rem: V2, Q) || isNonEqualURem(X: V2, Rem: V1, Q))
4439 return true;
4440
4441 if (isKnownNonEqualFromContext(V1, V2, Q, Depth))
4442 return true;
4443
4444 return false;
4445}
4446
4447/// For vector constants, loop over the elements and find the constant with the
4448/// minimum number of sign bits. Return 0 if the value is not a vector constant
4449/// or if any element was not analyzed; otherwise, return the count for the
4450/// element with the minimum number of sign bits.
4451static unsigned computeNumSignBitsVectorConstant(const Value *V,
4452 const APInt &DemandedElts,
4453 unsigned TyBits) {
4454 const auto *CV = dyn_cast<Constant>(Val: V);
4455 if (!CV || !isa<FixedVectorType>(Val: CV->getType()))
4456 return 0;
4457
4458 unsigned MinSignBits = TyBits;
4459 unsigned NumElts = cast<FixedVectorType>(Val: CV->getType())->getNumElements();
4460 for (unsigned i = 0; i != NumElts; ++i) {
4461 if (!DemandedElts[i])
4462 continue;
4463 // If we find a non-ConstantInt, bail out.
4464 auto *Elt = dyn_cast_or_null<ConstantInt>(Val: CV->getAggregateElement(Elt: i));
4465 if (!Elt)
4466 return 0;
4467
4468 MinSignBits = std::min(a: MinSignBits, b: Elt->getValue().getNumSignBits());
4469 }
4470
4471 return MinSignBits;
4472}
4473
4474static unsigned ComputeNumSignBitsImpl(const Value *V,
4475 const APInt &DemandedElts,
4476 const SimplifyQuery &Q, unsigned Depth);
4477
4478static unsigned ComputeNumSignBits(const Value *V, const APInt &DemandedElts,
4479 const SimplifyQuery &Q, unsigned Depth) {
4480 unsigned Result = ComputeNumSignBitsImpl(V, DemandedElts, Q, Depth);
4481 assert(Result > 0 && "At least one sign bit needs to be present!");
4482 return Result;
4483}
4484
4485/// Return the number of times the sign bit of the register is replicated into
4486/// the other bits. We know that at least 1 bit is always equal to the sign bit
4487/// (itself), but other cases can give us information. For example, immediately
4488/// after an "ashr X, 2", we know that the top 3 bits are all equal to each
4489/// other, so we return 3. For vectors, return the number of sign bits for the
4490/// vector element with the minimum number of known sign bits of the demanded
4491/// elements in the vector specified by DemandedElts.
4492static unsigned ComputeNumSignBitsImpl(const Value *V,
4493 const APInt &DemandedElts,
4494 const SimplifyQuery &Q, unsigned Depth) {
4495 Type *Ty = V->getType();
4496#ifndef NDEBUG
4497 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
4498
4499 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
4500 assert(
4501 FVTy->getNumElements() == DemandedElts.getBitWidth() &&
4502 "DemandedElt width should equal the fixed vector number of elements");
4503 } else {
4504 assert(DemandedElts == APInt(1, 1) &&
4505 "DemandedElt width should be 1 for scalars");
4506 }
4507#endif
4508
4509 // We return the minimum number of sign bits that are guaranteed to be present
4510 // in V, so for undef we have to conservatively return 1. We don't have the
4511 // same behavior for poison though -- that's a FIXME today.
4512
4513 Type *ScalarTy = Ty->getScalarType();
4514 unsigned TyBits = ScalarTy->isPointerTy() ?
4515 Q.DL.getPointerTypeSizeInBits(ScalarTy) :
4516 Q.DL.getTypeSizeInBits(Ty: ScalarTy);
4517
4518 unsigned Tmp, Tmp2;
4519 unsigned FirstAnswer = 1;
4520
4521 // Note that ConstantInt is handled by the general computeKnownBits case
4522 // below.
4523
4524 if (Depth == MaxAnalysisRecursionDepth)
4525 return 1;
4526
4527 if (auto *U = dyn_cast<Operator>(Val: V)) {
4528 switch (Operator::getOpcode(V)) {
4529 default: break;
4530 case Instruction::BitCast: {
4531 Value *Src = U->getOperand(i: 0);
4532 Type *SrcTy = Src->getType();
4533
4534 // Skip if the source type is not an integer or integer vector type
4535 // This ensures we only process integer-like types
4536 if (!SrcTy->isIntOrIntVectorTy())
4537 break;
4538
4539 unsigned SrcBits = SrcTy->getScalarSizeInBits();
4540
4541 // Bitcast 'large element' scalar/vector to 'small element' vector.
4542 if ((SrcBits % TyBits) != 0)
4543 break;
4544
4545 // Only proceed if the destination type is a fixed-size vector
4546 if (isa<FixedVectorType>(Val: Ty)) {
4547 // Fast case - sign splat can be simply split across the small elements.
4548 // This works for both vector and scalar sources
4549 Tmp = ComputeNumSignBits(V: Src, Q, Depth: Depth + 1);
4550 if (Tmp == SrcBits)
4551 return TyBits;
4552 }
4553 break;
4554 }
4555 case Instruction::SExt:
4556 Tmp = TyBits - U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4557 return ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1) +
4558 Tmp;
4559
4560 case Instruction::SDiv: {
4561 const APInt *Denominator;
4562 // sdiv X, C -> adds log(C) sign bits.
4563 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4564
4565 // Ignore non-positive denominator.
4566 if (!Denominator->isStrictlyPositive())
4567 break;
4568
4569 // Calculate the incoming numerator bits.
4570 unsigned NumBits =
4571 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4572
4573 // Add floor(log(C)) bits to the numerator bits.
4574 return std::min(a: TyBits, b: NumBits + Denominator->logBase2());
4575 }
4576 break;
4577 }
4578
4579 case Instruction::SRem: {
4580 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4581
4582 const APInt *Denominator;
4583 // srem X, C -> we know that the result is within [-C+1,C) when C is a
4584 // positive constant. This let us put a lower bound on the number of sign
4585 // bits.
4586 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: Denominator))) {
4587
4588 // Ignore non-positive denominator.
4589 if (Denominator->isStrictlyPositive()) {
4590 // Calculate the leading sign bit constraints by examining the
4591 // denominator. Given that the denominator is positive, there are two
4592 // cases:
4593 //
4594 // 1. The numerator is positive. The result range is [0,C) and
4595 // [0,C) u< (1 << ceilLogBase2(C)).
4596 //
4597 // 2. The numerator is negative. Then the result range is (-C,0] and
4598 // integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
4599 //
4600 // Thus a lower bound on the number of sign bits is `TyBits -
4601 // ceilLogBase2(C)`.
4602
4603 unsigned ResBits = TyBits - Denominator->ceilLogBase2();
4604 Tmp = std::max(a: Tmp, b: ResBits);
4605 }
4606 }
4607 return Tmp;
4608 }
4609
4610 case Instruction::AShr: {
4611 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4612 // ashr X, C -> adds C sign bits. Vectors too.
4613 const APInt *ShAmt;
4614 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4615 if (ShAmt->uge(RHS: TyBits))
4616 break; // Bad shift.
4617 unsigned ShAmtLimited = ShAmt->getZExtValue();
4618 Tmp += ShAmtLimited;
4619 if (Tmp > TyBits) Tmp = TyBits;
4620 }
4621 return Tmp;
4622 }
4623 case Instruction::Shl: {
4624 const APInt *ShAmt;
4625 Value *X = nullptr;
4626 if (match(V: U->getOperand(i: 1), P: m_APInt(Res&: ShAmt))) {
4627 // shl destroys sign bits.
4628 if (ShAmt->uge(RHS: TyBits))
4629 break; // Bad shift.
4630 // We can look through a zext (more or less treating it as a sext) if
4631 // all extended bits are shifted out.
4632 if (match(V: U->getOperand(i: 0), P: m_ZExt(Op: m_Value(V&: X))) &&
4633 ShAmt->uge(RHS: TyBits - X->getType()->getScalarSizeInBits())) {
4634 Tmp = ComputeNumSignBits(V: X, DemandedElts, Q, Depth: Depth + 1);
4635 Tmp += TyBits - X->getType()->getScalarSizeInBits();
4636 } else
4637 Tmp =
4638 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4639 if (ShAmt->uge(RHS: Tmp))
4640 break; // Shifted all sign bits out.
4641 Tmp2 = ShAmt->getZExtValue();
4642 return Tmp - Tmp2;
4643 }
4644 break;
4645 }
4646 case Instruction::And:
4647 case Instruction::Or:
4648 case Instruction::Xor: // NOT is handled here.
4649 // Logical binary ops preserve the number of sign bits at the worst.
4650 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4651 if (Tmp != 1) {
4652 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4653 FirstAnswer = std::min(a: Tmp, b: Tmp2);
4654 // We computed what we know about the sign bits as our first
4655 // answer. Now proceed to the generic code that uses
4656 // computeKnownBits, and pick whichever answer is better.
4657 }
4658 break;
4659
4660 case Instruction::Select: {
4661 // If we have a clamp pattern, we know that the number of sign bits will
4662 // be the minimum of the clamp min/max range.
4663 const Value *X;
4664 const APInt *CLow, *CHigh;
4665 if (isSignedMinMaxClamp(Select: U, In&: X, CLow, CHigh))
4666 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4667
4668 Tmp = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4669 if (Tmp == 1)
4670 break;
4671 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 2), DemandedElts, Q, Depth: Depth + 1);
4672 return std::min(a: Tmp, b: Tmp2);
4673 }
4674
4675 case Instruction::Add:
4676 // Add can have at most one carry bit. Thus we know that the output
4677 // is, at worst, one more bit than the inputs.
4678 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4679 if (Tmp == 1) break;
4680
4681 // Special case decrementing a value (ADD X, -1):
4682 if (const auto *CRHS = dyn_cast<Constant>(Val: U->getOperand(i: 1)))
4683 if (CRHS->isAllOnesValue()) {
4684 KnownBits Known(TyBits);
4685 computeKnownBits(V: U->getOperand(i: 0), DemandedElts, Known, Q, Depth: Depth + 1);
4686
4687 // If the input is known to be 0 or 1, the output is 0/-1, which is
4688 // all sign bits set.
4689 if ((Known.Zero | 1).isAllOnes())
4690 return TyBits;
4691
4692 // If we are subtracting one from a positive number, there is no carry
4693 // out of the result.
4694 if (Known.isNonNegative())
4695 return Tmp;
4696 }
4697
4698 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4699 if (Tmp2 == 1)
4700 break;
4701 return std::min(a: Tmp, b: Tmp2) - 1;
4702
4703 case Instruction::Sub:
4704 Tmp2 = ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4705 if (Tmp2 == 1)
4706 break;
4707
4708 // Handle NEG.
4709 if (const auto *CLHS = dyn_cast<Constant>(Val: U->getOperand(i: 0)))
4710 if (CLHS->isNullValue()) {
4711 KnownBits Known(TyBits);
4712 computeKnownBits(V: U->getOperand(i: 1), DemandedElts, Known, Q, Depth: Depth + 1);
4713 // If the input is known to be 0 or 1, the output is 0/-1, which is
4714 // all sign bits set.
4715 if ((Known.Zero | 1).isAllOnes())
4716 return TyBits;
4717
4718 // If the input is known to be positive (the sign bit is known clear),
4719 // the output of the NEG has the same number of sign bits as the
4720 // input.
4721 if (Known.isNonNegative())
4722 return Tmp2;
4723
4724 // Otherwise, we treat this like a SUB.
4725 }
4726
4727 // Sub can have at most one carry bit. Thus we know that the output
4728 // is, at worst, one more bit than the inputs.
4729 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4730 if (Tmp == 1)
4731 break;
4732 return std::min(a: Tmp, b: Tmp2) - 1;
4733
4734 case Instruction::Mul: {
4735 // The output of the Mul can be at most twice the valid bits in the
4736 // inputs.
4737 unsigned SignBitsOp0 =
4738 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4739 if (SignBitsOp0 == 1)
4740 break;
4741 unsigned SignBitsOp1 =
4742 ComputeNumSignBits(V: U->getOperand(i: 1), DemandedElts, Q, Depth: Depth + 1);
4743 if (SignBitsOp1 == 1)
4744 break;
4745 unsigned OutValidBits =
4746 (TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
4747 return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
4748 }
4749
4750 case Instruction::PHI: {
4751 const PHINode *PN = cast<PHINode>(Val: U);
4752 unsigned NumIncomingValues = PN->getNumIncomingValues();
4753 // Don't analyze large in-degree PHIs.
4754 if (NumIncomingValues > 4) break;
4755 // Unreachable blocks may have zero-operand PHI nodes.
4756 if (NumIncomingValues == 0) break;
4757
4758 // Take the minimum of all incoming values. This can't infinitely loop
4759 // because of our depth threshold.
4760 SimplifyQuery RecQ = Q.getWithoutCondContext();
4761 Tmp = TyBits;
4762 for (unsigned i = 0, e = NumIncomingValues; i != e; ++i) {
4763 if (Tmp == 1) return Tmp;
4764 RecQ.CtxI = PN->getIncomingBlock(i)->getTerminator();
4765 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(V: PN->getIncomingValue(i),
4766 DemandedElts, Q: RecQ, Depth: Depth + 1));
4767 }
4768 return Tmp;
4769 }
4770
4771 case Instruction::Trunc: {
4772 // If the input contained enough sign bits that some remain after the
4773 // truncation, then we can make use of that. Otherwise we don't know
4774 // anything.
4775 Tmp = ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4776 unsigned OperandTyBits = U->getOperand(i: 0)->getType()->getScalarSizeInBits();
4777 if (Tmp > (OperandTyBits - TyBits))
4778 return Tmp - (OperandTyBits - TyBits);
4779
4780 return 1;
4781 }
4782
4783 case Instruction::ExtractElement:
4784 // Look through extract element. At the moment we keep this simple and
4785 // skip tracking the specific element. But at least we might find
4786 // information valid for all elements of the vector (for example if vector
4787 // is sign extended, shifted, etc).
4788 return ComputeNumSignBits(V: U->getOperand(i: 0), Q, Depth: Depth + 1);
4789
4790 case Instruction::ShuffleVector: {
4791 // Collect the minimum number of sign bits that are shared by every vector
4792 // element referenced by the shuffle.
4793 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: U);
4794 if (!Shuf) {
4795 // FIXME: Add support for shufflevector constant expressions.
4796 return 1;
4797 }
4798 APInt DemandedLHS, DemandedRHS;
4799 // For undef elements, we don't know anything about the common state of
4800 // the shuffle result.
4801 if (!getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
4802 return 1;
4803 Tmp = std::numeric_limits<unsigned>::max();
4804 if (!!DemandedLHS) {
4805 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
4806 Tmp = ComputeNumSignBits(V: LHS, DemandedElts: DemandedLHS, Q, Depth: Depth + 1);
4807 }
4808 // If we don't know anything, early out and try computeKnownBits
4809 // fall-back.
4810 if (Tmp == 1)
4811 break;
4812 if (!!DemandedRHS) {
4813 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
4814 Tmp2 = ComputeNumSignBits(V: RHS, DemandedElts: DemandedRHS, Q, Depth: Depth + 1);
4815 Tmp = std::min(a: Tmp, b: Tmp2);
4816 }
4817 // If we don't know anything, early out and try computeKnownBits
4818 // fall-back.
4819 if (Tmp == 1)
4820 break;
4821 assert(Tmp <= TyBits && "Failed to determine minimum sign bits");
4822 return Tmp;
4823 }
4824 case Instruction::Call: {
4825 if (const auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
4826 switch (II->getIntrinsicID()) {
4827 default:
4828 break;
4829 case Intrinsic::abs:
4830 Tmp =
4831 ComputeNumSignBits(V: U->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
4832 if (Tmp == 1)
4833 break;
4834
4835 // Absolute value reduces number of sign bits by at most 1.
4836 return Tmp - 1;
4837 case Intrinsic::smin:
4838 case Intrinsic::smax: {
4839 const APInt *CLow, *CHigh;
4840 if (isSignedMinMaxIntrinsicClamp(II, CLow, CHigh))
4841 return std::min(a: CLow->getNumSignBits(), b: CHigh->getNumSignBits());
4842 }
4843 }
4844 }
4845 }
4846 }
4847 }
4848
4849 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4850 // use this information.
4851
4852 // If we can examine all elements of a vector constant successfully, we're
4853 // done (we can't do any better than that). If not, keep trying.
4854 if (unsigned VecSignBits =
4855 computeNumSignBitsVectorConstant(V, DemandedElts, TyBits))
4856 return VecSignBits;
4857
4858 KnownBits Known(TyBits);
4859 computeKnownBits(V, DemandedElts, Known, Q, Depth);
4860
4861 // If we know that the sign bit is either zero or one, determine the number of
4862 // identical bits in the top of the input value.
4863 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
4864}
4865
4866Intrinsic::ID llvm::getIntrinsicForCallSite(const CallBase &CB,
4867 const TargetLibraryInfo *TLI) {
4868 const Function *F = CB.getCalledFunction();
4869 if (!F)
4870 return Intrinsic::not_intrinsic;
4871
4872 if (F->isIntrinsic())
4873 return F->getIntrinsicID();
4874
4875 // We are going to infer semantics of a library function based on mapping it
4876 // to an LLVM intrinsic. Check that the library function is available from
4877 // this callbase and in this environment.
4878 if (F->hasLocalLinkage() || !TLI || !CB.onlyReadsMemory())
4879 return Intrinsic::not_intrinsic;
4880
4881 LibFunc Func = TLI->getLibFunc(CB);
4882 if (Func == NotLibFunc)
4883 return Intrinsic::not_intrinsic;
4884
4885 switch (Func) {
4886 default:
4887 break;
4888 case LibFunc_sin:
4889 case LibFunc_sinf:
4890 case LibFunc_sinl:
4891 return Intrinsic::sin;
4892 case LibFunc_cos:
4893 case LibFunc_cosf:
4894 case LibFunc_cosl:
4895 return Intrinsic::cos;
4896 case LibFunc_tan:
4897 case LibFunc_tanf:
4898 case LibFunc_tanl:
4899 return Intrinsic::tan;
4900 case LibFunc_asin:
4901 case LibFunc_asinf:
4902 case LibFunc_asinl:
4903 return Intrinsic::asin;
4904 case LibFunc_acos:
4905 case LibFunc_acosf:
4906 case LibFunc_acosl:
4907 return Intrinsic::acos;
4908 case LibFunc_atan:
4909 case LibFunc_atanf:
4910 case LibFunc_atanl:
4911 return Intrinsic::atan;
4912 case LibFunc_atan2:
4913 case LibFunc_atan2f:
4914 case LibFunc_atan2l:
4915 return Intrinsic::atan2;
4916 case LibFunc_sinh:
4917 case LibFunc_sinhf:
4918 case LibFunc_sinhl:
4919 return Intrinsic::sinh;
4920 case LibFunc_cosh:
4921 case LibFunc_coshf:
4922 case LibFunc_coshl:
4923 return Intrinsic::cosh;
4924 case LibFunc_tanh:
4925 case LibFunc_tanhf:
4926 case LibFunc_tanhl:
4927 return Intrinsic::tanh;
4928 case LibFunc_exp:
4929 case LibFunc_expf:
4930 case LibFunc_expl:
4931 return Intrinsic::exp;
4932 case LibFunc_exp2:
4933 case LibFunc_exp2f:
4934 case LibFunc_exp2l:
4935 return Intrinsic::exp2;
4936 case LibFunc_exp10:
4937 case LibFunc_exp10f:
4938 case LibFunc_exp10l:
4939 return Intrinsic::exp10;
4940 case LibFunc_log:
4941 case LibFunc_logf:
4942 case LibFunc_logl:
4943 return Intrinsic::log;
4944 case LibFunc_log10:
4945 case LibFunc_log10f:
4946 case LibFunc_log10l:
4947 return Intrinsic::log10;
4948 case LibFunc_log2:
4949 case LibFunc_log2f:
4950 case LibFunc_log2l:
4951 return Intrinsic::log2;
4952 case LibFunc_fabs:
4953 case LibFunc_fabsf:
4954 case LibFunc_fabsl:
4955 return Intrinsic::fabs;
4956 case LibFunc_fmin:
4957 case LibFunc_fminf:
4958 case LibFunc_fminl:
4959 return Intrinsic::minnum;
4960 case LibFunc_fmax:
4961 case LibFunc_fmaxf:
4962 case LibFunc_fmaxl:
4963 return Intrinsic::maxnum;
4964 case LibFunc_copysign:
4965 case LibFunc_copysignf:
4966 case LibFunc_copysignl:
4967 return Intrinsic::copysign;
4968 case LibFunc_floor:
4969 case LibFunc_floorf:
4970 case LibFunc_floorl:
4971 return Intrinsic::floor;
4972 case LibFunc_ceil:
4973 case LibFunc_ceilf:
4974 case LibFunc_ceill:
4975 return Intrinsic::ceil;
4976 case LibFunc_trunc:
4977 case LibFunc_truncf:
4978 case LibFunc_truncl:
4979 return Intrinsic::trunc;
4980 case LibFunc_rint:
4981 case LibFunc_rintf:
4982 case LibFunc_rintl:
4983 return Intrinsic::rint;
4984 case LibFunc_nearbyint:
4985 case LibFunc_nearbyintf:
4986 case LibFunc_nearbyintl:
4987 return Intrinsic::nearbyint;
4988 case LibFunc_round:
4989 case LibFunc_roundf:
4990 case LibFunc_roundl:
4991 return Intrinsic::round;
4992 case LibFunc_roundeven:
4993 case LibFunc_roundevenf:
4994 case LibFunc_roundevenl:
4995 return Intrinsic::roundeven;
4996 case LibFunc_pow:
4997 case LibFunc_powf:
4998 case LibFunc_powl:
4999 return Intrinsic::pow;
5000 case LibFunc_sqrt:
5001 case LibFunc_sqrtf:
5002 case LibFunc_sqrtl:
5003 return Intrinsic::sqrt;
5004 }
5005
5006 return Intrinsic::not_intrinsic;
5007}
5008
5009/// Given an exploded icmp instruction, return true if the comparison only
5010/// checks the sign bit. If it only checks the sign bit, set TrueIfSigned if
5011/// the result of the comparison is true when the input value is signed.
5012bool llvm::isSignBitCheck(ICmpInst::Predicate Pred, const APInt &RHS,
5013 bool &TrueIfSigned) {
5014 switch (Pred) {
5015 case ICmpInst::ICMP_SLT: // True if LHS s< 0
5016 TrueIfSigned = true;
5017 return RHS.isZero();
5018 case ICmpInst::ICMP_SLE: // True if LHS s<= -1
5019 TrueIfSigned = true;
5020 return RHS.isAllOnes();
5021 case ICmpInst::ICMP_SGT: // True if LHS s> -1
5022 TrueIfSigned = false;
5023 return RHS.isAllOnes();
5024 case ICmpInst::ICMP_SGE: // True if LHS s>= 0
5025 TrueIfSigned = false;
5026 return RHS.isZero();
5027 case ICmpInst::ICMP_UGT:
5028 // True if LHS u> RHS and RHS == sign-bit-mask - 1
5029 TrueIfSigned = true;
5030 return RHS.isMaxSignedValue();
5031 case ICmpInst::ICMP_UGE:
5032 // True if LHS u>= RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
5033 TrueIfSigned = true;
5034 return RHS.isMinSignedValue();
5035 case ICmpInst::ICMP_ULT:
5036 // True if LHS u< RHS and RHS == sign-bit-mask (2^7, 2^15, 2^31, etc)
5037 TrueIfSigned = false;
5038 return RHS.isMinSignedValue();
5039 case ICmpInst::ICMP_ULE:
5040 // True if LHS u<= RHS and RHS == sign-bit-mask - 1
5041 TrueIfSigned = false;
5042 return RHS.isMaxSignedValue();
5043 default:
5044 return false;
5045 }
5046}
5047
5048static void computeKnownFPClassFromCond(const Value *V, Value *Cond,
5049 bool CondIsTrue,
5050 const Instruction *CtxI,
5051 KnownFPClass &KnownFromContext,
5052 unsigned Depth = 0) {
5053 Value *A, *B;
5054 if (Depth < MaxAnalysisRecursionDepth &&
5055 (CondIsTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: A), R: m_Value(V&: B)))
5056 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: A), R: m_Value(V&: B))))) {
5057 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue, CtxI, KnownFromContext,
5058 Depth: Depth + 1);
5059 computeKnownFPClassFromCond(V, Cond: B, CondIsTrue, CtxI, KnownFromContext,
5060 Depth: Depth + 1);
5061 return;
5062 }
5063 if (Depth < MaxAnalysisRecursionDepth && match(V: Cond, P: m_Not(V: m_Value(V&: A)))) {
5064 computeKnownFPClassFromCond(V, Cond: A, CondIsTrue: !CondIsTrue, CtxI, KnownFromContext,
5065 Depth: Depth + 1);
5066 return;
5067 }
5068 CmpPredicate Pred;
5069 Value *LHS;
5070 uint64_t ClassVal = 0;
5071 const APFloat *CRHS;
5072 const APInt *RHS;
5073 if (match(V: Cond, P: m_FCmp(Pred, L: m_Value(V&: LHS), R: m_APFloat(Res&: CRHS)))) {
5074 auto [CmpVal, MaskIfTrue, MaskIfFalse] = fcmpImpliesClass(
5075 Pred, F: *cast<Instruction>(Val: Cond)->getParent()->getParent(), LHS, ConstRHS: *CRHS,
5076 LookThroughSrc: LHS != V);
5077 if (CmpVal == V)
5078 KnownFromContext.knownNot(RuleOut: ~(CondIsTrue ? MaskIfTrue : MaskIfFalse));
5079 } else if (match(V: Cond, P: m_Intrinsic<Intrinsic::is_fpclass>(
5080 Ops: m_Specific(V), Ops: m_ConstantInt(V&: ClassVal)))) {
5081 FPClassTest Mask = static_cast<FPClassTest>(ClassVal);
5082 KnownFromContext.knownNot(RuleOut: CondIsTrue ? ~Mask : Mask);
5083 } else if (match(V: Cond, P: m_ICmp(Pred, L: m_ElementWiseBitCast(Op: m_Specific(V)),
5084 R: m_APInt(Res&: RHS)))) {
5085 bool TrueIfSigned;
5086 if (!isSignBitCheck(Pred, RHS: *RHS, TrueIfSigned))
5087 return;
5088 if (TrueIfSigned == CondIsTrue)
5089 KnownFromContext.signBitMustBeOne();
5090 else
5091 KnownFromContext.signBitMustBeZero();
5092 }
5093}
5094
5095/// Compute the minimum and maximum values (inclusive) for the exponent of \p V,
5096/// assuming it is not nan. Returns {min, max, max-assuming-nonzero}. A value
5097/// frexp(0) = 0, so the tighter max-assuming-nonzero bound is only usable when
5098/// \p V is known not to be a logical zero (e.g., for fabs(x) < 0.25, the non-0
5099/// exponent range is [-149, -2], but the 0 edge case is above this range).
5100static std::tuple<int, int, int>
5101computeKnownExponentRangeFromContext(const Value *V, const SimplifyQuery &Q) {
5102 if (!Q.CtxI || !Q.DC || !Q.DT)
5103 return {APFloat::IEK_NaN, APFloat::IEK_Inf, APFloat::IEK_Inf};
5104
5105 // Intersect the bounds implied by every dominating condition, keeping the
5106 // tightest maximum. A value may participate in multiple compares
5107 // (e.g. fabs(x) < 2.0 and fabs(x) < 1.0), and the tighter one wins.
5108 int MaxExp = APFloat::IEK_Inf;
5109 int MaxExpNonZero = APFloat::IEK_Inf;
5110
5111 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
5112 CmpPredicate Pred;
5113 const APFloat *LimitC;
5114 if (!match(V: BI->getCondition(),
5115 P: m_FCmp(Pred, L: m_FAbs(Op0: m_Specific(V)), R: m_Finite(V&: LimitC))))
5116 continue;
5117
5118 if (Pred == FCmpInst::FCMP_ORD || Pred == FCmpInst::FCMP_UNO ||
5119 Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE)
5120 continue;
5121
5122 // If fabs(x) <= K, implies the exponent min exp range.
5123 // if fabs(x) >= K, swap the successor
5124 bool IsLessEqual =
5125 Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE ||
5126 Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE ||
5127 Pred == FCmpInst::FCMP_OEQ || Pred == FCmpInst::FCMP_UEQ;
5128
5129 bool KnownStrictlyLess =
5130 Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_ULT ||
5131 Pred == FCmpInst::FCMP_OGE || Pred == FCmpInst::FCMP_UGE;
5132
5133 BasicBlockEdge Edge1(BI->getParent(),
5134 BI->getSuccessor(i: IsLessEqual ? 0 : 1));
5135 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent())) {
5136 // frexp returns an exponent one greater than ilogb.
5137 int Exp = ilogb(Arg: *LimitC) + 1;
5138
5139 // A strict bound fabs(V) < 2^n forces ilogb(V) <= n - 1, so the max frexp
5140 // exponent drops by one when K is exact power of two.
5141 if (KnownStrictlyLess && LimitC->getExactLog2Abs() != INT_MIN)
5142 --Exp;
5143
5144 // frexp(0) = 0, which the bound above (assuming a normal nonzero value)
5145 // may exclude.
5146
5147 // TODO: Figure out lower bound to detect no-underflow.
5148 MaxExpNonZero = std::min(a: MaxExpNonZero, b: Exp);
5149 MaxExp = std::min(a: MaxExp, b: std::max(a: Exp, b: 0));
5150 }
5151 }
5152
5153 return {APFloat::IEK_NaN, MaxExp, MaxExpNonZero};
5154}
5155
5156static KnownFPClass computeKnownFPClassFromContext(const Value *V,
5157 const SimplifyQuery &Q) {
5158 KnownFPClass KnownFromContext;
5159
5160 if (Q.CC && Q.CC->AffectedValues.contains(Ptr: V))
5161 computeKnownFPClassFromCond(V, Cond: Q.CC->Cond, CondIsTrue: !Q.CC->Invert, CtxI: Q.CtxI,
5162 KnownFromContext);
5163
5164 if (!Q.CtxI)
5165 return KnownFromContext;
5166
5167 if (Q.DC && Q.DT) {
5168 // Handle dominating conditions.
5169 for (CondBrInst *BI : Q.DC->conditionsFor(V)) {
5170 Value *Cond = BI->getCondition();
5171
5172 BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(i: 0));
5173 if (Q.DT->dominates(BBE: Edge0, BB: Q.CtxI->getParent()))
5174 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/true, CtxI: Q.CtxI,
5175 KnownFromContext);
5176
5177 BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(i: 1));
5178 if (Q.DT->dominates(BBE: Edge1, BB: Q.CtxI->getParent()))
5179 computeKnownFPClassFromCond(V, Cond, /*CondIsTrue=*/false, CtxI: Q.CtxI,
5180 KnownFromContext);
5181 }
5182 }
5183
5184 if (!Q.AC)
5185 return KnownFromContext;
5186
5187 // Try to restrict the floating-point classes based on information from
5188 // assumptions.
5189 for (auto &AssumeVH : Q.AC->assumptionsFor(V)) {
5190 if (!AssumeVH)
5191 continue;
5192 CallInst *I = cast<CallInst>(Val&: AssumeVH);
5193
5194 assert(I->getFunction() == Q.CtxI->getParent()->getParent() &&
5195 "Got assumption for the wrong function!");
5196 assert(I->getIntrinsicID() == Intrinsic::assume &&
5197 "must be an assume intrinsic");
5198
5199 if (!isValidAssumeForContext(I, Q))
5200 continue;
5201
5202 computeKnownFPClassFromCond(V, Cond: I->getArgOperand(i: 0),
5203 /*CondIsTrue=*/true, CtxI: Q.CtxI, KnownFromContext);
5204 }
5205
5206 return KnownFromContext;
5207}
5208
5209void llvm::adjustKnownFPClassForSelectArm(KnownFPClass &Known, Value *Cond,
5210 Value *Arm, bool Invert,
5211 const SimplifyQuery &SQ,
5212 unsigned Depth) {
5213
5214 KnownFPClass KnownSrc;
5215 computeKnownFPClassFromCond(V: Arm, Cond,
5216 /*CondIsTrue=*/!Invert, CtxI: SQ.CtxI, KnownFromContext&: KnownSrc,
5217 Depth: Depth + 1);
5218 KnownSrc = KnownSrc.unionWith(RHS: Known);
5219 if (KnownSrc.isUnknown())
5220 return;
5221
5222 if (isGuaranteedNotToBeUndef(V: Arm, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT, Depth: Depth + 1))
5223 Known = KnownSrc;
5224}
5225
5226void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5227 FPClassTest InterestedClasses, KnownFPClass &Known,
5228 const SimplifyQuery &Q, unsigned Depth);
5229
5230static void computeKnownFPClass(const Value *V, KnownFPClass &Known,
5231 FPClassTest InterestedClasses,
5232 const SimplifyQuery &Q, unsigned Depth) {
5233 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5234 APInt DemandedElts =
5235 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
5236 computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Q, Depth);
5237}
5238
5239static void computeKnownFPClassForFPTrunc(const Operator *Op,
5240 const APInt &DemandedElts,
5241 FPClassTest InterestedClasses,
5242 KnownFPClass &Known,
5243 const SimplifyQuery &Q,
5244 unsigned Depth) {
5245 if ((InterestedClasses &
5246 (KnownFPClass::OrderedLessThanZeroMask | fcNan)) == fcNone)
5247 return;
5248
5249 KnownFPClass KnownSrc;
5250 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5251 Known&: KnownSrc, Q, Depth: Depth + 1);
5252 Known = KnownFPClass::fptrunc(KnownSrc);
5253}
5254
5255static constexpr KnownFPClass::MinMaxKind getMinMaxKind(Intrinsic::ID IID) {
5256 switch (IID) {
5257 case Intrinsic::minimum:
5258 return KnownFPClass::MinMaxKind::minimum;
5259 case Intrinsic::maximum:
5260 return KnownFPClass::MinMaxKind::maximum;
5261 case Intrinsic::minimumnum:
5262 return KnownFPClass::MinMaxKind::minimumnum;
5263 case Intrinsic::maximumnum:
5264 return KnownFPClass::MinMaxKind::maximumnum;
5265 case Intrinsic::minnum:
5266 return KnownFPClass::MinMaxKind::minnum;
5267 case Intrinsic::maxnum:
5268 return KnownFPClass::MinMaxKind::maxnum;
5269 default:
5270 llvm_unreachable("not a floating-point min-max intrinsic");
5271 }
5272}
5273
5274/// \return true if this is a floating point value that is known to have a
5275/// magnitude smaller than 1. i.e., fabs(X) <= 1.0 or is nan.
5276static bool isAbsoluteValueULEOne(const Value *V) {
5277 // TODO: Handle frexp
5278 // TODO: Other rounding intrinsics?
5279 // TODO: Try computeKnownExponentRangeFromContext
5280
5281 // fabs(x - floor(x)) <= 1
5282 const Value *SubFloorX;
5283 if (match(V, P: m_FSub(L: m_Value(V&: SubFloorX),
5284 R: m_Intrinsic<Intrinsic::floor>(Ops: m_Deferred(V: SubFloorX)))))
5285 return true;
5286
5287 return match(V, P: m_Intrinsic<Intrinsic::amdgcn_trig_preop>(Ops: m_Value())) ||
5288 match(V, P: m_Intrinsic<Intrinsic::amdgcn_fract>(Ops: m_Value()));
5289}
5290
5291void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
5292 FPClassTest InterestedClasses, KnownFPClass &Known,
5293 const SimplifyQuery &Q, unsigned Depth) {
5294 assert(Known.isUnknown() && "should not be called with known information");
5295
5296 if (!DemandedElts) {
5297 // No demanded elts, better to assume we don't know anything.
5298 Known.resetAll();
5299 return;
5300 }
5301
5302 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
5303
5304 if (auto *CFP = dyn_cast<ConstantFP>(Val: V)) {
5305 Known = KnownFPClass(CFP->getValueAPF());
5306 return;
5307 }
5308
5309 if (isa<ConstantAggregateZero>(Val: V)) {
5310 Known.setKnownFPClasses(fcPosZero);
5311 Known.setSignBit(false);
5312 return;
5313 }
5314
5315 if (isa<PoisonValue>(Val: V)) {
5316 Known.setKnownFPClasses(fcNone);
5317 Known.setSignBit(false);
5318 return;
5319 }
5320
5321 // Try to handle fixed width vector constants
5322 auto *VFVTy = dyn_cast<FixedVectorType>(Val: V->getType());
5323 const Constant *CV = dyn_cast<Constant>(Val: V);
5324 if (VFVTy && CV) {
5325 Known.setKnownFPClasses(fcNone);
5326 bool SignBitAllZero = true;
5327 bool SignBitAllOne = true;
5328
5329 // For vectors, verify that each element is not NaN.
5330 unsigned NumElts = VFVTy->getNumElements();
5331 for (unsigned i = 0; i != NumElts; ++i) {
5332 if (!DemandedElts[i])
5333 continue;
5334
5335 Constant *Elt = CV->getAggregateElement(Elt: i);
5336 if (!Elt) {
5337 Known = KnownFPClass();
5338 return;
5339 }
5340 if (isa<PoisonValue>(Val: Elt))
5341 continue;
5342 auto *CElt = dyn_cast<ConstantFP>(Val: Elt);
5343 if (!CElt) {
5344 Known = KnownFPClass();
5345 return;
5346 }
5347
5348 const APFloat &C = CElt->getValueAPF();
5349 Known.setKnownFPClasses(Known.getKnownFPClasses() | C.classify());
5350 if (C.isNegative())
5351 SignBitAllZero = false;
5352 else
5353 SignBitAllOne = false;
5354 }
5355 if (SignBitAllOne != SignBitAllZero)
5356 Known.setSignBit(SignBitAllOne);
5357 return;
5358 }
5359
5360 if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) {
5361 Known.setKnownFPClasses(fcNone);
5362 for (size_t I = 0, E = CDS->getNumElements(); I != E; ++I)
5363 Known |= CDS->getElementAsAPFloat(i: I).classify();
5364 return;
5365 }
5366
5367 if (const auto *CA = dyn_cast<ConstantAggregate>(Val: V)) {
5368 // TODO: Handle complex aggregates
5369 Known.setKnownFPClasses(fcNone);
5370 for (const Use &Op : CA->operands()) {
5371 auto *CFP = dyn_cast<ConstantFP>(Val: Op.get());
5372 if (!CFP) {
5373 Known = KnownFPClass();
5374 return;
5375 }
5376
5377 Known |= CFP->getValueAPF().classify();
5378 }
5379
5380 return;
5381 }
5382
5383 FPClassTest KnownNotFromFlags = fcNone;
5384 if (const auto *CB = dyn_cast<CallBase>(Val: V))
5385 KnownNotFromFlags |= CB->getRetNoFPClass();
5386 else if (const auto *Arg = dyn_cast<Argument>(Val: V))
5387 KnownNotFromFlags |= Arg->getNoFPClass();
5388
5389 const Operator *Op = dyn_cast<Operator>(Val: V);
5390 if (const FPMathOperator *FPOp = dyn_cast_or_null<FPMathOperator>(Val: Op)) {
5391 if (FPOp->hasNoNaNs())
5392 KnownNotFromFlags |= fcNan;
5393 if (FPOp->hasNoInfs())
5394 KnownNotFromFlags |= fcInf;
5395 }
5396
5397 KnownFPClass AssumedClasses = computeKnownFPClassFromContext(V, Q);
5398 KnownNotFromFlags |= ~AssumedClasses.getKnownFPClasses();
5399
5400 // We no longer need to find out about these bits from inputs if we can
5401 // assume this from flags/attributes.
5402 InterestedClasses &= ~KnownNotFromFlags;
5403
5404 llvm::scope_exit ClearClassesFromFlags([=, &Known] {
5405 Known.knownNot(RuleOut: KnownNotFromFlags);
5406 if (!Known.getSignBit() && AssumedClasses.getSignBit()) {
5407 if (*AssumedClasses.getSignBit())
5408 Known.signBitMustBeOne();
5409 else
5410 Known.signBitMustBeZero();
5411 }
5412 });
5413
5414 if (!Op)
5415 return;
5416
5417 // All recursive calls that increase depth must come after this.
5418 if (Depth == MaxAnalysisRecursionDepth)
5419 return;
5420
5421 const unsigned Opc = Op->getOpcode();
5422 switch (Opc) {
5423 case Instruction::FNeg: {
5424 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
5425 Known, Q, Depth: Depth + 1);
5426 Known.fneg();
5427 break;
5428 }
5429 case Instruction::Select: {
5430 auto ComputeForArm = [&](Value *Arm, bool Invert) {
5431 KnownFPClass Res;
5432 computeKnownFPClass(V: Arm, DemandedElts, InterestedClasses, Known&: Res, Q,
5433 Depth: Depth + 1);
5434 adjustKnownFPClassForSelectArm(Known&: Res, Cond: Op->getOperand(i: 0), Arm, Invert, SQ: Q,
5435 Depth);
5436 return Res;
5437 };
5438 // Only known if known in both the LHS and RHS.
5439 Known =
5440 ComputeForArm(Op->getOperand(i: 1), /*Invert=*/false)
5441 .intersectWith(RHS: ComputeForArm(Op->getOperand(i: 2), /*Invert=*/true));
5442 break;
5443 }
5444 case Instruction::Load: {
5445 const MDNode *NoFPClass =
5446 cast<LoadInst>(Val: Op)->getMetadata(KindID: LLVMContext::MD_nofpclass);
5447 if (!NoFPClass)
5448 break;
5449
5450 ConstantInt *MaskVal =
5451 mdconst::extract<ConstantInt>(MD: NoFPClass->getOperand(I: 0));
5452 Known.knownNot(RuleOut: static_cast<FPClassTest>(MaskVal->getZExtValue()));
5453 break;
5454 }
5455 case Instruction::Call: {
5456 const CallInst *II = cast<CallInst>(Val: Op);
5457 const Intrinsic::ID IID = II->getIntrinsicID();
5458 switch (IID) {
5459 case Intrinsic::fabs: {
5460 if ((InterestedClasses & (fcNan | fcPositive)) != fcNone) {
5461 // If we only care about the sign bit we don't need to inspect the
5462 // operand.
5463 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5464 InterestedClasses, Known, Q, Depth: Depth + 1);
5465 }
5466
5467 Known.fabs();
5468 break;
5469 }
5470 case Intrinsic::copysign: {
5471 KnownFPClass KnownSign;
5472
5473 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5474 Known, Q, Depth: Depth + 1);
5475 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5476 Known&: KnownSign, Q, Depth: Depth + 1);
5477 Known.copysign(Sign: KnownSign);
5478 break;
5479 }
5480 case Intrinsic::fma:
5481 case Intrinsic::fmuladd: {
5482 if ((InterestedClasses & fcNegative) == fcNone)
5483 break;
5484
5485 // FIXME: This should check isGuaranteedNotToBeUndef
5486 if (II->getArgOperand(i: 0) == II->getArgOperand(i: 1)) {
5487 KnownFPClass KnownSrc, KnownAddend;
5488 computeKnownFPClass(V: II->getArgOperand(i: 2), DemandedElts,
5489 InterestedClasses, Known&: KnownAddend, Q, Depth: Depth + 1);
5490 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5491 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5492
5493 const Function *F = II->getFunction();
5494 const fltSemantics &FltSem =
5495 II->getType()->getScalarType()->getFltSemantics();
5496 DenormalMode Mode =
5497 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5498
5499 if ((KnownNotFromFlags & fcNan) == fcNan) {
5500 KnownSrc.knownNot(RuleOut: fcNan);
5501 KnownAddend.knownNot(RuleOut: fcNan);
5502 }
5503
5504 if ((KnownNotFromFlags & fcInf) == fcInf) {
5505 KnownSrc.knownNot(RuleOut: fcInf);
5506 KnownAddend.knownNot(RuleOut: fcInf);
5507 }
5508
5509 Known = KnownFPClass::fma_square(Squared: KnownSrc, Addend: KnownAddend, Mode);
5510 break;
5511 }
5512
5513 KnownFPClass KnownSrc[3];
5514 for (int I = 0; I != 3; ++I) {
5515 computeKnownFPClass(V: II->getArgOperand(i: I), DemandedElts,
5516 InterestedClasses, Known&: KnownSrc[I], Q, Depth: Depth + 1);
5517 if (KnownSrc[I].isUnknown())
5518 return;
5519
5520 if ((KnownNotFromFlags & fcNan) == fcNan)
5521 KnownSrc[I].knownNot(RuleOut: fcNan);
5522 if ((KnownNotFromFlags & fcInf) == fcInf)
5523 KnownSrc[I].knownNot(RuleOut: fcInf);
5524 }
5525
5526 const Function *F = II->getFunction();
5527 const fltSemantics &FltSem =
5528 II->getType()->getScalarType()->getFltSemantics();
5529 DenormalMode Mode =
5530 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5531 Known = KnownFPClass::fma(LHS: KnownSrc[0], RHS: KnownSrc[1], Addend: KnownSrc[2], Mode);
5532 break;
5533 }
5534 case Intrinsic::sqrt:
5535 case Intrinsic::experimental_constrained_sqrt: {
5536 KnownFPClass KnownSrc;
5537 FPClassTest InterestedSrcs = InterestedClasses;
5538 if (InterestedClasses & fcNan)
5539 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
5540
5541 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5542 Known&: KnownSrc, Q, Depth: Depth + 1);
5543
5544 DenormalMode Mode = DenormalMode::getDynamic();
5545
5546 bool HasNSZ = Q.IIQ.hasNoSignedZeros(Op: II);
5547 if (!HasNSZ) {
5548 const Function *F = II->getFunction();
5549 const fltSemantics &FltSem =
5550 II->getType()->getScalarType()->getFltSemantics();
5551 Mode = F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
5552 }
5553
5554 Known = KnownFPClass::sqrt(Src: KnownSrc, Mode);
5555 if (HasNSZ)
5556 Known.knownNot(RuleOut: fcNegZero);
5557
5558 break;
5559 }
5560 case Intrinsic::sin: {
5561 KnownFPClass KnownSrc;
5562 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5563 Known&: KnownSrc, Q, Depth: Depth + 1);
5564 Known = KnownFPClass::sin(Src: KnownSrc);
5565 break;
5566 }
5567 case Intrinsic::cos: {
5568 KnownFPClass KnownSrc;
5569 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5570 Known&: KnownSrc, Q, Depth: Depth + 1);
5571 Known = KnownFPClass::cos(Src: KnownSrc);
5572 break;
5573 }
5574 case Intrinsic::tan: {
5575 KnownFPClass KnownSrc;
5576 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5577 Known&: KnownSrc, Q, Depth: Depth + 1);
5578 Known = KnownFPClass::tan(Src: KnownSrc);
5579 break;
5580 }
5581 case Intrinsic::sinh: {
5582 KnownFPClass KnownSrc;
5583 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5584 Known&: KnownSrc, Q, Depth: Depth + 1);
5585 Known = KnownFPClass::sinh(Src: KnownSrc);
5586 break;
5587 }
5588 case Intrinsic::cosh: {
5589 KnownFPClass KnownSrc;
5590 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5591 Known&: KnownSrc, Q, Depth: Depth + 1);
5592 Known = KnownFPClass::cosh(Src: KnownSrc);
5593 break;
5594 }
5595 case Intrinsic::tanh: {
5596 KnownFPClass KnownSrc;
5597 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5598 Known&: KnownSrc, Q, Depth: Depth + 1);
5599 Known = KnownFPClass::tanh(Src: KnownSrc);
5600 break;
5601 }
5602 case Intrinsic::asin: {
5603 KnownFPClass KnownSrc;
5604 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5605 Known&: KnownSrc, Q, Depth: Depth + 1);
5606 Known = KnownFPClass::asin(Src: KnownSrc);
5607 break;
5608 }
5609 case Intrinsic::acos: {
5610 KnownFPClass KnownSrc;
5611 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5612 Known&: KnownSrc, Q, Depth: Depth + 1);
5613 Known = KnownFPClass::acos(Src: KnownSrc);
5614 break;
5615 }
5616 case Intrinsic::atan: {
5617 KnownFPClass KnownSrc;
5618 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5619 Known&: KnownSrc, Q, Depth: Depth + 1);
5620 Known = KnownFPClass::atan(Src: KnownSrc);
5621 break;
5622 }
5623 case Intrinsic::atan2: {
5624 FPClassTest InterestedY = InterestedClasses;
5625 FPClassTest InterestedX = InterestedClasses;
5626
5627 // We can rule out negative values if y cannot have a negative value.
5628 if ((InterestedClasses & fcNegFinite) != fcNone)
5629 InterestedY |= fcNegative;
5630
5631 // We can rule out positive values if y cannot have a positive value.
5632 if ((InterestedClasses & fcPosFinite) != fcNone)
5633 InterestedY |= fcPositive | fcNegSubnormal;
5634
5635 // We can rule out zero and subnormal if x cannot have a positive value.
5636 if ((InterestedClasses & (fcZero | fcSubnormal)) != fcNone)
5637 InterestedX |= fcPositive | fcNegSubnormal;
5638
5639 KnownFPClass KnownY, KnownX;
5640 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedY,
5641 Known&: KnownY, Q, Depth: Depth + 1);
5642 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses: InterestedX,
5643 Known&: KnownX, Q, Depth: Depth + 1);
5644
5645 const Function *F = II->getFunction();
5646 DenormalMode Mode =
5647 F ? F->getDenormalMode(
5648 FPType: II->getType()->getScalarType()->getFltSemantics())
5649 : DenormalMode::getDynamic();
5650 Known = KnownFPClass::atan2(LHS: KnownY, RHS: KnownX, Mode);
5651 break;
5652 }
5653 case Intrinsic::maxnum:
5654 case Intrinsic::minnum:
5655 case Intrinsic::minimum:
5656 case Intrinsic::maximum:
5657 case Intrinsic::minimumnum:
5658 case Intrinsic::maximumnum: {
5659 KnownFPClass KnownLHS, KnownRHS;
5660 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5661 Known&: KnownLHS, Q, Depth: Depth + 1);
5662 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses,
5663 Known&: KnownRHS, Q, Depth: Depth + 1);
5664
5665 const Function *F = II->getFunction();
5666
5667 DenormalMode Mode =
5668 F ? F->getDenormalMode(
5669 FPType: II->getType()->getScalarType()->getFltSemantics())
5670 : DenormalMode::getDynamic();
5671
5672 Known = KnownFPClass::minMaxLike(LHS: KnownLHS, RHS: KnownRHS, Kind: getMinMaxKind(IID),
5673 DenormMode: Mode);
5674 break;
5675 }
5676 case Intrinsic::canonicalize: {
5677 KnownFPClass KnownSrc;
5678 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5679 Known&: KnownSrc, Q, Depth: Depth + 1);
5680
5681 const Function *F = II->getFunction();
5682 DenormalMode DenormMode =
5683 F ? F->getDenormalMode(
5684 FPType: II->getType()->getScalarType()->getFltSemantics())
5685 : DenormalMode::getDynamic();
5686 Known = KnownFPClass::canonicalize(Src: KnownSrc, DenormMode);
5687 break;
5688 }
5689 case Intrinsic::vector_reduce_fmax:
5690 case Intrinsic::vector_reduce_fmin:
5691 case Intrinsic::vector_reduce_fmaximum:
5692 case Intrinsic::vector_reduce_fminimum:
5693 case Intrinsic::vector_reduce_fmaximumnum:
5694 case Intrinsic::vector_reduce_fminimumnum: {
5695 // reduce min/max will choose an element from one of the vector elements,
5696 // so we can infer and class information that is common to all elements.
5697 Known = computeKnownFPClass(V: II->getArgOperand(i: 0), FMF: II->getFastMathFlags(),
5698 InterestedClasses, SQ: Q, Depth: Depth + 1);
5699 // Can only propagate sign if output is never NaN.
5700 if (!Known.isKnownNeverNaN())
5701 Known.setSignBit(std::nullopt);
5702 break;
5703 }
5704 // reverse preserves all characteristics of the input vec's element.
5705 case Intrinsic::vector_reverse:
5706 Known = computeKnownFPClass(
5707 V: II->getArgOperand(i: 0), DemandedElts: DemandedElts.reverseBits(),
5708 FMF: II->getFastMathFlags(), InterestedClasses, SQ: Q, Depth: Depth + 1);
5709 break;
5710 case Intrinsic::trunc:
5711 case Intrinsic::floor:
5712 case Intrinsic::ceil:
5713 case Intrinsic::rint:
5714 case Intrinsic::nearbyint:
5715 case Intrinsic::round:
5716 case Intrinsic::roundeven: {
5717 KnownFPClass KnownSrc;
5718 FPClassTest InterestedSrcs = InterestedClasses;
5719
5720 // Negative round ups towards zero produce negative zero.
5721 if (InterestedSrcs & fcNegFinite)
5722 InterestedSrcs |= fcNegFinite;
5723
5724 // Negative subnormals may flush to positive zero.
5725 if (InterestedSrcs & fcPosFinite)
5726 InterestedSrcs |= fcPosFinite | fcNegSubnormal;
5727
5728 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5729 Known&: KnownSrc, Q, Depth: Depth + 1);
5730
5731 const Function *F = II->getFunction();
5732 DenormalMode Mode =
5733 F ? F->getDenormalMode(
5734 FPType: II->getType()->getScalarType()->getFltSemantics())
5735 : DenormalMode::getDynamic();
5736 const bool IsMultiUnitFPType =
5737 V->getType()->getScalarType()->isMultiUnitFPType();
5738
5739 switch (IID) {
5740 case Intrinsic::trunc:
5741 Known = KnownFPClass::trunc(Src: KnownSrc, Mode);
5742 break;
5743 case Intrinsic::floor:
5744 Known = KnownFPClass::floor(Src: KnownSrc, IsMultiUnitFPType, Mode);
5745 break;
5746 case Intrinsic::ceil:
5747 Known = KnownFPClass::ceil(Src: KnownSrc, IsMultiUnitFPType, Mode);
5748 break;
5749 default:
5750 Known =
5751 KnownFPClass::roundToIntegral(Src: KnownSrc, IsMultiUnitFPType, Mode);
5752 break;
5753 }
5754 break;
5755 }
5756 case Intrinsic::exp:
5757 case Intrinsic::exp2:
5758 case Intrinsic::exp10:
5759 case Intrinsic::amdgcn_exp2: {
5760 KnownFPClass KnownSrc;
5761 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5762 Known&: KnownSrc, Q, Depth: Depth + 1);
5763
5764 Known = KnownFPClass::exp(Src: KnownSrc);
5765
5766 Type *EltTy = II->getType()->getScalarType();
5767 if (IID == Intrinsic::amdgcn_exp2 && EltTy->isFloatTy())
5768 Known.knownNot(RuleOut: fcSubnormal);
5769
5770 break;
5771 }
5772 case Intrinsic::fptrunc_round: {
5773 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
5774 Q, Depth);
5775 break;
5776 }
5777 case Intrinsic::log:
5778 case Intrinsic::log10:
5779 case Intrinsic::log2:
5780 case Intrinsic::experimental_constrained_log:
5781 case Intrinsic::experimental_constrained_log10:
5782 case Intrinsic::experimental_constrained_log2:
5783 case Intrinsic::amdgcn_log: {
5784 FPClassTest InterestedSrcs = fcNone;
5785
5786 // log(negative) produces NaN.
5787 if ((InterestedClasses & fcNan) != fcNone)
5788 InterestedSrcs |= fcNan | fcNegative;
5789
5790 // log(logical-zero) produces negative infinity.
5791 if ((InterestedClasses & fcNegInf) != fcNone)
5792 InterestedSrcs |= fcZero | fcSubnormal;
5793
5794 // log(x) < -0.0 if x < +1.0
5795 if ((InterestedClasses & fcNegNormal) != fcNone)
5796 InterestedSrcs |= fcPosSubnormal | fcPosNormal;
5797
5798 // log(x) >= +0.0 if x >= +1.0
5799 if ((InterestedClasses & (fcPosZero | fcPosNormal)) != fcNone)
5800 InterestedSrcs |= fcPosNormal;
5801
5802 // log(x) is positive infinity iff x is positive infinity.
5803 if ((InterestedClasses & fcPosInf) != fcNone)
5804 InterestedSrcs |= fcPosInf;
5805
5806 KnownFPClass KnownSrc;
5807 if (InterestedSrcs != fcNone)
5808 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5809 Known&: KnownSrc, Q, Depth: Depth + 1);
5810 const Function *F = II->getFunction();
5811 DenormalMode Mode =
5812 F ? F->getDenormalMode(
5813 FPType: II->getType()->getScalarType()->getFltSemantics())
5814 : DenormalMode::getDynamic();
5815 const bool IsMultiUnitFPType =
5816 V->getType()->getScalarType()->isMultiUnitFPType();
5817 Known = KnownFPClass::log(Src: KnownSrc, Mode, IsMultiUnitFPType);
5818 break;
5819 }
5820 case Intrinsic::pow: {
5821 const bool WantNaN = (InterestedClasses & fcNan) != fcNone;
5822 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
5823 if (!WantNaN && !WantNegative)
5824 break;
5825
5826 FPClassTest InterestedLHS = fcNone;
5827 FPClassTest InterestedRHS = fcNone;
5828 if (WantNaN) {
5829 // pow may return NaN if one of the arguments is NaN. NaN may also be
5830 // produced from a negative, non-zero finite base and a non-integer
5831 // exponent.
5832 InterestedLHS |= fcNan | fcNegNormal | fcNegSubnormal;
5833 InterestedRHS |= fcNan;
5834 }
5835 if (WantNegative) {
5836 // A negative value is returned when a negative base is raised to an odd
5837 // integer power. Only normal values can be odd integers.
5838 InterestedLHS |= fcNegative;
5839 InterestedRHS |= fcNormal;
5840 }
5841
5842 KnownFPClass KnownLHS;
5843 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedLHS,
5844 Known&: KnownLHS, Q, Depth: Depth + 1);
5845
5846 // If the LHS is unknown, then querying the RHS is only useful for rare
5847 // edge cases.
5848 if (KnownLHS.isUnknown())
5849 break;
5850
5851 KnownFPClass KnownRHS;
5852 computeKnownFPClass(V: II->getArgOperand(i: 1), DemandedElts, InterestedClasses: InterestedRHS,
5853 Known&: KnownRHS, Q, Depth: Depth + 1);
5854 Known = KnownFPClass::pow(LHS: KnownLHS, RHS: KnownRHS);
5855 break;
5856 }
5857 case Intrinsic::powi: {
5858 if ((InterestedClasses & (fcNan | fcInf | fcNegative)) == fcNone)
5859 break;
5860
5861 // The exponent is always a scalar, even when raising a vector to a power.
5862 const Value *Exp = II->getArgOperand(i: 1);
5863 unsigned BitWidth = Exp->getType()->getIntegerBitWidth();
5864 KnownBits ExponentKnownBits(BitWidth);
5865 computeKnownBits(V: Exp, DemandedElts: APInt(1, 1), Known&: ExponentKnownBits, Q, Depth: Depth + 1);
5866
5867 FPClassTest InterestedSrcs = fcNone;
5868 if (InterestedClasses & fcNan)
5869 InterestedSrcs |= fcNan;
5870 if (!ExponentKnownBits.isZero()) {
5871 if (InterestedClasses & fcInf)
5872 InterestedSrcs |= fcFinite | fcInf;
5873 if ((InterestedClasses & fcNegative) && !ExponentKnownBits.isEven())
5874 InterestedSrcs |= fcNegative;
5875 }
5876
5877 KnownFPClass KnownSrc;
5878 if (InterestedSrcs != fcNone)
5879 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
5880 Known&: KnownSrc, Q, Depth: Depth + 1);
5881
5882 Known = KnownFPClass::powi(Src: KnownSrc, N: ExponentKnownBits);
5883 break;
5884 }
5885 case Intrinsic::ldexp: {
5886 KnownFPClass KnownSrc;
5887 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5888 Known&: KnownSrc, Q, Depth: Depth + 1);
5889 // Can refine inf/zero handling based on the exponent operand.
5890 const FPClassTest ExpInfoMask = fcZero | fcSubnormal | fcInf;
5891
5892 const Value *ExpArg = II->getArgOperand(i: 1);
5893 ConstantRange ExpKnownRange =
5894 ((KnownSrc.getKnownFPClasses() & ExpInfoMask) != fcNone)
5895 ? computeConstantRange(V: ExpArg, /*ForSigned=*/true, SQ: Q, Depth: Depth + 1)
5896 : ConstantRange::getFull(
5897 BitWidth: ExpArg->getType()->getScalarSizeInBits());
5898
5899 const fltSemantics &Flt =
5900 II->getType()->getScalarType()->getFltSemantics();
5901
5902 const Function *F = II->getFunction();
5903 DenormalMode Mode =
5904 F ? F->getDenormalMode(FPType: Flt) : DenormalMode::getDynamic();
5905
5906 Known = KnownFPClass::ldexp(Src: KnownSrc, ConstantRangeMin: ExpKnownRange.getSignedMin(),
5907 ConstantRangeMax: ExpKnownRange.getSignedMax(), Flt, Mode);
5908 break;
5909 }
5910 case Intrinsic::arithmetic_fence: {
5911 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5912 Known, Q, Depth: Depth + 1);
5913 break;
5914 }
5915 case Intrinsic::experimental_constrained_sitofp:
5916 case Intrinsic::experimental_constrained_uitofp:
5917 // Cannot produce nan
5918 Known.knownNot(RuleOut: fcNan);
5919
5920 // sitofp and uitofp turn into +0.0 for zero.
5921 Known.knownNot(RuleOut: fcNegZero);
5922
5923 // Integers cannot be subnormal
5924 Known.knownNot(RuleOut: fcSubnormal);
5925
5926 if (IID == Intrinsic::experimental_constrained_uitofp)
5927 Known.signBitMustBeZero();
5928
5929 // TODO: Copy inf handling from instructions
5930 break;
5931
5932 case Intrinsic::amdgcn_fract: {
5933 Known.knownNot(RuleOut: fcInf);
5934
5935 if (InterestedClasses & fcNan) {
5936 KnownFPClass KnownSrc;
5937 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
5938 InterestedClasses, Known&: KnownSrc, Q, Depth: Depth + 1);
5939
5940 if (KnownSrc.isKnownNeverInfOrNaN())
5941 Known.knownNot(RuleOut: fcNan);
5942 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5943 Known.knownNot(RuleOut: fcSNan);
5944 }
5945
5946 break;
5947 }
5948 case Intrinsic::amdgcn_rcp: {
5949 KnownFPClass KnownSrc;
5950 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5951 Known&: KnownSrc, Q, Depth: Depth + 1);
5952
5953 Known.propagateNonNaN(Src: KnownSrc);
5954
5955 Type *EltTy = II->getType()->getScalarType();
5956
5957 // f32 denormal always flushed.
5958 if (EltTy->isFloatTy()) {
5959 Known.knownNot(RuleOut: fcSubnormal);
5960 KnownSrc.knownNot(RuleOut: fcSubnormal);
5961 }
5962
5963 if (KnownSrc.isKnownNever(Mask: fcNegative))
5964 Known.knownNot(RuleOut: fcNegative);
5965 if (KnownSrc.isKnownNever(Mask: fcPositive))
5966 Known.knownNot(RuleOut: fcPositive);
5967
5968 if (const Function *F = II->getFunction()) {
5969 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
5970 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
5971 Known.knownNot(RuleOut: fcPosInf);
5972 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
5973 Known.knownNot(RuleOut: fcNegInf);
5974 }
5975
5976 break;
5977 }
5978 case Intrinsic::amdgcn_rsq: {
5979 KnownFPClass KnownSrc;
5980 // The only negative value that can be returned is -inf for -0 inputs.
5981 Known.knownNot(RuleOut: fcNegZero | fcNegSubnormal | fcNegNormal);
5982
5983 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts, InterestedClasses,
5984 Known&: KnownSrc, Q, Depth: Depth + 1);
5985
5986 // Negative -> nan
5987 if (KnownSrc.isKnownNeverNaN() && KnownSrc.cannotBeOrderedLessThanZero())
5988 Known.knownNot(RuleOut: fcNan);
5989 else if (KnownSrc.isKnownNever(Mask: fcSNan))
5990 Known.knownNot(RuleOut: fcSNan);
5991
5992 // +inf -> +0
5993 if (KnownSrc.isKnownNeverPosInfinity())
5994 Known.knownNot(RuleOut: fcPosZero);
5995
5996 Type *EltTy = II->getType()->getScalarType();
5997
5998 // f32 denormal always flushed.
5999 if (EltTy->isFloatTy())
6000 Known.knownNot(RuleOut: fcPosSubnormal);
6001
6002 if (const Function *F = II->getFunction()) {
6003 DenormalMode Mode = F->getDenormalMode(FPType: EltTy->getFltSemantics());
6004
6005 // -0 -> -inf
6006 if (KnownSrc.isKnownNeverLogicalNegZero(Mode))
6007 Known.knownNot(RuleOut: fcNegInf);
6008
6009 // +0 -> +inf
6010 if (KnownSrc.isKnownNeverLogicalPosZero(Mode))
6011 Known.knownNot(RuleOut: fcPosInf);
6012 }
6013
6014 break;
6015 }
6016 case Intrinsic::amdgcn_trig_preop: {
6017 // Always returns a value [0, 1)
6018 Known.knownNot(RuleOut: fcNan | fcInf | fcNegative);
6019 break;
6020 }
6021 case Intrinsic::convert_from_arbitrary_fp: {
6022 auto *MD = cast<MetadataAsValue>(Val: II->getArgOperand(i: 1))->getMetadata();
6023 StringRef FormatStr = cast<MDString>(Val: MD)->getString();
6024
6025 const fltSemantics *SrcSemantics =
6026 APFloat::getArbitraryFPSemantics(Format: FormatStr);
6027 if (!SrcSemantics)
6028 break;
6029
6030 const fltSemantics DstSemantics =
6031 II->getType()->getScalarType()->getFltSemantics();
6032
6033 if (!APFloat::semanticsHasNaN(*SrcSemantics))
6034 Known.knownNot(RuleOut: fcNan);
6035
6036 // fcInf can only be cleared if the source format has no Inf encoding
6037 // and the dst max exp can accommodate src max exp.
6038 if (!APFloat::semanticsHasInf(*SrcSemantics) &&
6039 APFloat::semanticsMaxExponent(*SrcSemantics) <=
6040 APFloat::semanticsMaxExponent(DstSemantics))
6041 Known.knownNot(RuleOut: fcInf);
6042
6043 // Check and clear all neg flags for formats that do not have signed
6044 // representation.
6045 if (!APFloat::semanticsHasSignedRepr(*SrcSemantics))
6046 Known.knownNot(RuleOut: fcNegative);
6047
6048 // Check if format has no zero at all (Float8E8M0FNU), or no negative
6049 // zero.
6050 if (!APFloat::semanticsHasZero(*SrcSemantics))
6051 Known.knownNot(RuleOut: fcZero);
6052 else if (SrcSemantics->nanEncoding == fltNanEncoding::NegativeZero)
6053 Known.knownNot(RuleOut: fcNegZero);
6054
6055 // If src lands normally in dest, the result can never be subnormal.
6056 if (APFloat::isRepresentableAsNormalIn(Src: *SrcSemantics, Dst: DstSemantics))
6057 Known.knownNot(RuleOut: fcSubnormal);
6058 break;
6059 }
6060 default:
6061 break;
6062 }
6063
6064 break;
6065 }
6066 case Instruction::FAdd:
6067 case Instruction::FSub: {
6068 KnownFPClass KnownLHS, KnownRHS;
6069 bool WantNegative =
6070 Op->getOpcode() == Instruction::FAdd &&
6071 (InterestedClasses & KnownFPClass::OrderedLessThanZeroMask) != fcNone;
6072 bool WantNaN = (InterestedClasses & fcNan) != fcNone;
6073 bool WantNegZero = (InterestedClasses & fcNegZero) != fcNone;
6074
6075 if (!WantNaN && !WantNegative && !WantNegZero)
6076 break;
6077
6078 FPClassTest InterestedSrcs = InterestedClasses;
6079 if (WantNegative)
6080 InterestedSrcs |= KnownFPClass::OrderedLessThanZeroMask;
6081 if (InterestedClasses & fcNan)
6082 InterestedSrcs |= fcInf;
6083 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: InterestedSrcs,
6084 Known&: KnownRHS, Q, Depth: Depth + 1);
6085
6086 // Special case fadd x, x, which is the canonical form of fmul x, 2.
6087 bool Self = Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
6088 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT,
6089 Depth: Depth + 1);
6090 if (Self)
6091 KnownLHS = KnownRHS;
6092
6093 if ((WantNaN && KnownRHS.isKnownNeverNaN()) ||
6094 (WantNegative && KnownRHS.cannotBeOrderedLessThanZero()) ||
6095 WantNegZero || Opc == Instruction::FSub) {
6096
6097 // FIXME: Context function should always be passed in separately
6098 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6099 const fltSemantics &FltSem =
6100 Op->getType()->getScalarType()->getFltSemantics();
6101 DenormalMode Mode =
6102 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6103
6104 if (Self && Opc == Instruction::FAdd) {
6105 Known = KnownFPClass::fadd_self(Src: KnownLHS, Mode);
6106 } else {
6107 // RHS is canonically cheaper to compute. Skip inspecting the LHS if
6108 // there's no point.
6109
6110 if (!Self) {
6111 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
6112 Known&: KnownLHS, Q, Depth: Depth + 1);
6113 }
6114
6115 Known = Opc == Instruction::FAdd
6116 ? KnownFPClass::fadd(LHS: KnownLHS, RHS: KnownRHS, Mode)
6117 : KnownFPClass::fsub(LHS: KnownLHS, RHS: KnownRHS, Mode);
6118 }
6119 }
6120
6121 break;
6122 }
6123 case Instruction::FMul: {
6124 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6125 DenormalMode Mode =
6126 F ? F->getDenormalMode(
6127 FPType: Op->getType()->getScalarType()->getFltSemantics())
6128 : DenormalMode::getDynamic();
6129
6130 Value *LHS = Op->getOperand(i: 0);
6131 Value *RHS = Op->getOperand(i: 1);
6132 // X * X is always non-negative or a NaN.
6133 if (LHS == RHS &&
6134 isGuaranteedNotToBeUndef(V: LHS, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1)) {
6135 KnownFPClass KnownSrc;
6136 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownSrc, Q,
6137 Depth: Depth + 1);
6138 Known = KnownFPClass::square(Src: KnownSrc, Mode);
6139 break;
6140 }
6141
6142 KnownFPClass KnownLHS, KnownRHS;
6143
6144 const APFloat *CRHS;
6145 if (match(V: RHS, P: m_APFloat(Res&: CRHS))) {
6146 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
6147 Depth: Depth + 1);
6148 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: *CRHS, Mode);
6149 } else {
6150 computeKnownFPClass(V: RHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS, Q,
6151 Depth: Depth + 1);
6152 // TODO: Improve accuracy in unfused FMA pattern. We can prove an
6153 // additional not-nan if the addend is known-not negative infinity if the
6154 // multiply is known-not infinity.
6155
6156 computeKnownFPClass(V: LHS, DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS, Q,
6157 Depth: Depth + 1);
6158 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: KnownRHS, Mode);
6159 }
6160
6161 /// Propgate no-infs if the other source is known smaller than one, such
6162 /// that this cannot introduce overflow.
6163 if (KnownLHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: RHS))
6164 Known.knownNot(RuleOut: fcInf);
6165 else if (KnownRHS.isKnownNever(Mask: fcInf) && isAbsoluteValueULEOne(V: LHS))
6166 Known.knownNot(RuleOut: fcInf);
6167
6168 break;
6169 }
6170 case Instruction::FDiv: {
6171 const bool WantNan = (InterestedClasses & fcNan) != fcNone;
6172
6173 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6174 const fltSemantics &FltSem =
6175 Op->getType()->getScalarType()->getFltSemantics();
6176 DenormalMode Mode =
6177 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6178
6179 if (Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
6180 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT)) {
6181 // X / X is always exactly 1.0 or a NaN.
6182 Known.setKnownFPClasses(fcNan | fcPosNormal);
6183
6184 if (!WantNan)
6185 break;
6186
6187 KnownFPClass KnownSrc;
6188 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts,
6189 InterestedClasses: fcNan | fcInf | fcZero | fcSubnormal, Known&: KnownSrc, Q,
6190 Depth: Depth + 1);
6191
6192 Known = KnownFPClass::fdiv_self(Src: KnownSrc, Mode);
6193 break;
6194 }
6195
6196 const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
6197 const bool WantPositive = (InterestedClasses & fcPositive) != fcNone;
6198 if (!WantNan && !WantNegative && !WantPositive)
6199 break;
6200
6201 KnownFPClass KnownLHS, KnownRHS;
6202 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownRHS,
6203 Q, Depth: Depth + 1);
6204
6205 bool KnowSomethingUseful =
6206 KnownRHS.isKnownNeverNaN() ||
6207 KnownRHS.isKnownNever(Mask: fcNegNormal | fcNegSubnormal) ||
6208 KnownRHS.isKnownNever(Mask: fcPosNormal | fcPosSubnormal);
6209
6210 if (KnowSomethingUseful)
6211 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: fcAllFlags, Known&: KnownLHS,
6212 Q, Depth: Depth + 1);
6213
6214 Known = KnownFPClass::fdiv(LHS: KnownLHS, RHS: KnownRHS, Mode);
6215 break;
6216 }
6217 case Instruction::FRem: {
6218 FPClassTest InterestedLHS = fcNone;
6219 FPClassTest InterestedRHS = fcNone;
6220
6221 // NaN is also generated for frem(Inf, x) and frem(x, 0.0).
6222 if (InterestedClasses & fcNan) {
6223 InterestedLHS |= fcNan | fcInf;
6224 InterestedRHS |= fcNan | fcZero | fcSubnormal;
6225 }
6226
6227 // The sign for frem is the same as the first operand.
6228 if (InterestedClasses & (fcPosNormal | fcPosSubnormal))
6229 InterestedLHS |= fcPosNormal | fcPosSubnormal;
6230 if (InterestedClasses & (fcNegNormal | fcNegSubnormal))
6231 InterestedLHS |= fcNegNormal | fcNegSubnormal;
6232
6233 // A negative zero result requires a negative finite first operand.
6234 if (InterestedClasses & fcNegZero)
6235 InterestedLHS |= fcNegFinite;
6236
6237 // A positive zero result can additionally come from a negative finite
6238 // result being flushed to positive zero.
6239 if (InterestedClasses & fcPosZero)
6240 InterestedLHS |= fcPosFinite | fcNegNormal | fcNegSubnormal;
6241
6242 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6243 DenormalMode Mode =
6244 F ? F->getDenormalMode(
6245 FPType: Op->getType()->getScalarType()->getFltSemantics())
6246 : DenormalMode::getDynamic();
6247
6248 if (Op->getOperand(i: 0) == Op->getOperand(i: 1) &&
6249 isGuaranteedNotToBeUndef(V: Op->getOperand(i: 0), AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT)) {
6250 // X % X is always exactly [+-]0.0 or a NaN.
6251 FPClassTest InterestedSrcs = InterestedLHS | InterestedRHS;
6252 KnownFPClass KnownSrc;
6253 if (InterestedSrcs != fcNone)
6254 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedSrcs,
6255 Known&: KnownSrc, Q, Depth: Depth + 1);
6256 Known = KnownFPClass::frem_self(Src: KnownSrc, Mode);
6257 break;
6258 }
6259
6260 KnownFPClass KnownLHS;
6261 if (InterestedLHS != fcNone)
6262 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses: InterestedLHS,
6263 Known&: KnownLHS, Q, Depth: Depth + 1);
6264
6265 KnownFPClass KnownRHS;
6266 // RHS is only useful for refining NaN classes.
6267 if (InterestedRHS != fcNone && KnownLHS.isKnownNever(Mask: fcSNan))
6268 computeKnownFPClass(V: Op->getOperand(i: 1), DemandedElts, InterestedClasses: InterestedRHS,
6269 Known&: KnownRHS, Q, Depth: Depth + 1);
6270
6271 Known = KnownFPClass::frem(LHS: KnownLHS, RHS: KnownRHS, Mode);
6272
6273 break;
6274 }
6275 case Instruction::FPExt: {
6276 KnownFPClass KnownSrc;
6277 computeKnownFPClass(V: Op->getOperand(i: 0), DemandedElts, InterestedClasses,
6278 Known&: KnownSrc, Q, Depth: Depth + 1);
6279
6280 const fltSemantics &DstTy =
6281 Op->getType()->getScalarType()->getFltSemantics();
6282 const fltSemantics &SrcTy =
6283 Op->getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
6284
6285 Known = KnownFPClass::fpext(KnownSrc, DstTy, SrcTy);
6286 break;
6287 }
6288 case Instruction::FPTrunc: {
6289 computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known, Q,
6290 Depth);
6291 break;
6292 }
6293 case Instruction::SIToFP:
6294 case Instruction::UIToFP: {
6295 // Cannot produce nan
6296 Known.knownNot(RuleOut: fcNan);
6297
6298 // Integers cannot be subnormal
6299 Known.knownNot(RuleOut: fcSubnormal);
6300
6301 // sitofp and uitofp turn into +0.0 for zero.
6302 Known.knownNot(RuleOut: fcNegZero);
6303
6304 // UIToFP is always non-negative regardless of known bits.
6305 if (Op->getOpcode() == Instruction::UIToFP)
6306 Known.signBitMustBeZero();
6307
6308 // Only compute known bits if we can learn something useful from them.
6309 if (!(InterestedClasses & (fcPosZero | fcNormal | fcInf)))
6310 break;
6311
6312 KnownBits IntKnown =
6313 computeKnownBits(V: Op->getOperand(i: 0), DemandedElts, Q, Depth: Depth + 1);
6314
6315 // If the integer is non-zero, the result cannot be +0.0
6316 if (IntKnown.isNonZero())
6317 Known.knownNot(RuleOut: fcPosZero);
6318
6319 if (Op->getOpcode() == Instruction::SIToFP) {
6320 // If the signed integer is known non-negative, the result is
6321 // non-negative. If the signed integer is known negative, the result is
6322 // negative.
6323 if (IntKnown.isNonNegative()) {
6324 Known.signBitMustBeZero();
6325 } else if (IntKnown.isNegative()) {
6326 Known.signBitMustBeOne();
6327 }
6328 }
6329
6330 // Guard kept for ilogb()
6331 if (InterestedClasses & fcInf) {
6332 // Get width of largest magnitude integer known.
6333 // This still works for a signed minimum value because the largest FP
6334 // value is scaled by some fraction close to 2.0 (1.0 + 0.xxxx).
6335 int IntSize = IntKnown.getBitWidth();
6336 if (Op->getOpcode() == Instruction::UIToFP)
6337 IntSize -= IntKnown.countMinLeadingZeros();
6338 else if (Op->getOpcode() == Instruction::SIToFP)
6339 IntSize -= IntKnown.countMinSignBits();
6340
6341 // If the exponent of the largest finite FP value can hold the largest
6342 // integer, the result of the cast must be finite.
6343 Type *FPTy = Op->getType()->getScalarType();
6344 if (ilogb(Arg: APFloat::getLargest(Sem: FPTy->getFltSemantics())) >= IntSize)
6345 Known.knownNot(RuleOut: fcInf);
6346 }
6347
6348 break;
6349 }
6350 case Instruction::ExtractElement: {
6351 // Look through extract element. If the index is non-constant or
6352 // out-of-range demand all elements, otherwise just the extracted element.
6353 const Value *Vec = Op->getOperand(i: 0);
6354
6355 APInt DemandedVecElts;
6356 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType())) {
6357 unsigned NumElts = VecTy->getNumElements();
6358 DemandedVecElts = APInt::getAllOnes(numBits: NumElts);
6359 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1));
6360 if (CIdx && CIdx->getValue().ult(RHS: NumElts))
6361 DemandedVecElts = APInt::getOneBitSet(numBits: NumElts, BitNo: CIdx->getZExtValue());
6362 } else {
6363 DemandedVecElts = APInt(1, 1);
6364 }
6365
6366 return computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known,
6367 Q, Depth: Depth + 1);
6368 }
6369 case Instruction::InsertElement: {
6370 if (isa<ScalableVectorType>(Val: Op->getType()))
6371 return;
6372
6373 const Value *Vec = Op->getOperand(i: 0);
6374 const Value *Elt = Op->getOperand(i: 1);
6375 auto *CIdx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 2));
6376 unsigned NumElts = DemandedElts.getBitWidth();
6377 APInt DemandedVecElts = DemandedElts;
6378 bool NeedsElt = true;
6379 // If we know the index we are inserting to, clear it from Vec check.
6380 if (CIdx && CIdx->getValue().ult(RHS: NumElts)) {
6381 DemandedVecElts.clearBit(BitPosition: CIdx->getZExtValue());
6382 NeedsElt = DemandedElts[CIdx->getZExtValue()];
6383 }
6384
6385 // Do we demand the inserted element?
6386 if (NeedsElt) {
6387 computeKnownFPClass(V: Elt, Known, InterestedClasses, Q, Depth: Depth + 1);
6388 // If we don't know any bits, early out.
6389 if (Known.isUnknown())
6390 break;
6391 } else {
6392 Known.setKnownFPClasses(fcNone);
6393 }
6394
6395 // Do we need anymore elements from Vec?
6396 if (!DemandedVecElts.isZero()) {
6397 KnownFPClass Known2;
6398 computeKnownFPClass(V: Vec, DemandedElts: DemandedVecElts, InterestedClasses, Known&: Known2, Q,
6399 Depth: Depth + 1);
6400 Known |= Known2;
6401 }
6402
6403 break;
6404 }
6405 case Instruction::ShuffleVector: {
6406 // Handle vector splat idiom
6407 if (Value *Splat = getSplatValue(V)) {
6408 computeKnownFPClass(V: Splat, Known, InterestedClasses, Q, Depth: Depth + 1);
6409 break;
6410 }
6411
6412 // For undef elements, we don't know anything about the common state of
6413 // the shuffle result.
6414 APInt DemandedLHS, DemandedRHS;
6415 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: Op);
6416 if (!Shuf || !getShuffleDemandedElts(Shuf, DemandedElts, DemandedLHS, DemandedRHS))
6417 return;
6418
6419 if (!!DemandedLHS) {
6420 const Value *LHS = Shuf->getOperand(i_nocapture: 0);
6421 computeKnownFPClass(V: LHS, DemandedElts: DemandedLHS, InterestedClasses, Known, Q,
6422 Depth: Depth + 1);
6423
6424 // If we don't know any bits, early out.
6425 if (Known.isUnknown())
6426 break;
6427 } else {
6428 Known.setKnownFPClasses(fcNone);
6429 }
6430
6431 if (!!DemandedRHS) {
6432 KnownFPClass Known2;
6433 const Value *RHS = Shuf->getOperand(i_nocapture: 1);
6434 computeKnownFPClass(V: RHS, DemandedElts: DemandedRHS, InterestedClasses, Known&: Known2, Q,
6435 Depth: Depth + 1);
6436 Known |= Known2;
6437 }
6438
6439 break;
6440 }
6441 case Instruction::ExtractValue: {
6442 const ExtractValueInst *Extract = cast<ExtractValueInst>(Val: Op);
6443 ArrayRef<unsigned> Indices = Extract->getIndices();
6444 const Value *Src = Extract->getAggregateOperand();
6445 if (isa<StructType>(Val: Src->getType()) && Indices.size() == 1 &&
6446 Indices[0] == 0) {
6447 if (const auto *II = dyn_cast<IntrinsicInst>(Val: Src)) {
6448 switch (II->getIntrinsicID()) {
6449 case Intrinsic::frexp: {
6450 FPClassTest InterestedSrcs = InterestedClasses;
6451
6452 // Positive subnormals and negative subnormals could become positive
6453 // zero.
6454 if (InterestedClasses & fcPosZero)
6455 InterestedSrcs |= fcSubnormal;
6456
6457 // Negative subnormals could become negative zero.
6458 if (InterestedClasses & fcNegZero)
6459 InterestedSrcs |= fcNegSubnormal;
6460
6461 if (InterestedClasses & fcPosNormal)
6462 InterestedSrcs |= fcPosSubnormal;
6463
6464 if (InterestedClasses & fcNegNormal)
6465 InterestedSrcs |= fcNegSubnormal;
6466
6467 KnownFPClass KnownSrc;
6468 computeKnownFPClass(V: II->getArgOperand(i: 0), DemandedElts,
6469 InterestedClasses: InterestedSrcs, Known&: KnownSrc, Q, Depth: Depth + 1);
6470
6471 const Function *F = cast<Instruction>(Val: Op)->getFunction();
6472 const fltSemantics &FltSem =
6473 Op->getType()->getScalarType()->getFltSemantics();
6474
6475 DenormalMode Mode =
6476 F ? F->getDenormalMode(FPType: FltSem) : DenormalMode::getDynamic();
6477 Known = KnownFPClass::frexp_mant(Src: KnownSrc, Mode);
6478 return;
6479 }
6480 default:
6481 break;
6482 }
6483 }
6484 }
6485
6486 computeKnownFPClass(V: Src, DemandedElts, InterestedClasses, Known, Q,
6487 Depth: Depth + 1);
6488 break;
6489 }
6490 case Instruction::PHI: {
6491 const PHINode *P = cast<PHINode>(Val: Op);
6492 // Unreachable blocks may have zero-operand PHI nodes.
6493 if (P->getNumIncomingValues() == 0)
6494 break;
6495
6496 // Otherwise take the unions of the known bit sets of the operands,
6497 // taking conservative care to avoid excessive recursion.
6498 const unsigned PhiRecursionLimit = MaxAnalysisRecursionDepth - 2;
6499
6500 if (Depth < PhiRecursionLimit) {
6501 // Skip if every incoming value references to ourself.
6502 if (isa_and_nonnull<UndefValue>(Val: P->hasConstantValue()))
6503 break;
6504
6505 bool First = true;
6506
6507 for (const Use &U : P->operands()) {
6508 Value *IncValue;
6509 Instruction *CtxI;
6510 breakSelfRecursivePHI(U: &U, PHI: P, ValOut&: IncValue, CtxIOut&: CtxI);
6511 // Skip direct self references.
6512 if (IncValue == P)
6513 continue;
6514
6515 KnownFPClass KnownSrc;
6516 // Recurse, but cap the recursion to two levels, because we don't want
6517 // to waste time spinning around in loops. We need at least depth 2 to
6518 // detect known sign bits.
6519 computeKnownFPClass(V: IncValue, DemandedElts, InterestedClasses, Known&: KnownSrc,
6520 Q: Q.getWithoutCondContext().getWithInstruction(I: CtxI),
6521 Depth: PhiRecursionLimit);
6522
6523 if (First) {
6524 Known = KnownSrc;
6525 First = false;
6526 } else {
6527 Known |= KnownSrc;
6528 }
6529
6530 if (Known.getKnownFPClasses() == fcAllFlags)
6531 break;
6532 }
6533 }
6534
6535 // Look for the case of a for loop which has a positive
6536 // initial value and is incremented by a squared value.
6537 // This will propagate sign information out of such loops.
6538 if (P->getNumIncomingValues() != 2 || Known.cannotBeOrderedLessThanZero())
6539 break;
6540 for (unsigned I = 0; I < 2; I++) {
6541 Value *RecurValue = P->getIncomingValue(i: 1 - I);
6542 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: RecurValue);
6543 if (!II)
6544 continue;
6545 Value *R, *L, *Init;
6546 PHINode *PN;
6547 if (matchSimpleTernaryIntrinsicRecurrence(I: II, P&: PN, Init, OtherOp0&: L, OtherOp1&: R) &&
6548 PN == P) {
6549 switch (II->getIntrinsicID()) {
6550 case Intrinsic::fma:
6551 case Intrinsic::fmuladd: {
6552 KnownFPClass KnownStart;
6553 computeKnownFPClass(V: Init, DemandedElts, InterestedClasses, Known&: KnownStart,
6554 Q, Depth: Depth + 1);
6555 if (KnownStart.cannotBeOrderedLessThanZero() && L == R &&
6556 isGuaranteedNotToBeUndef(V: L, AC: Q.AC, CtxI: Q.CtxI, DT: Q.DT, Depth: Depth + 1))
6557 Known.knownNot(RuleOut: KnownFPClass::OrderedLessThanZeroMask);
6558 break;
6559 }
6560 }
6561 }
6562 }
6563 break;
6564 }
6565 case Instruction::BitCast: {
6566 const Value *Src;
6567 if (!match(V: Op, P: m_ElementWiseBitCast(Op: m_Value(V&: Src))) ||
6568 !Src->getType()->isIntOrIntVectorTy())
6569 break;
6570
6571 const Type *Ty = Op->getType();
6572
6573 Value *CastLHS, *CastRHS;
6574
6575 // Match bitcast(umax(bitcast(a), bitcast(b)))
6576 if (match(V: Src, P: m_c_MaxOrMin(L: m_BitCast(Op: m_Value(V&: CastLHS)),
6577 R: m_BitCast(Op: m_Value(V&: CastRHS)))) &&
6578 CastLHS->getType() == Ty && CastRHS->getType() == Ty) {
6579 KnownFPClass KnownLHS, KnownRHS;
6580 computeKnownFPClass(V: CastRHS, DemandedElts, InterestedClasses, Known&: KnownRHS, Q,
6581 Depth: Depth + 1);
6582 if (!KnownRHS.isUnknown()) {
6583 computeKnownFPClass(V: CastLHS, DemandedElts, InterestedClasses, Known&: KnownLHS,
6584 Q, Depth: Depth + 1);
6585 Known = KnownLHS | KnownRHS;
6586 }
6587
6588 return;
6589 }
6590
6591 const Type *EltTy = Ty->getScalarType();
6592 KnownBits Bits(EltTy->getPrimitiveSizeInBits());
6593 computeKnownBits(V: Src, DemandedElts, Known&: Bits, Q, Depth: Depth + 1);
6594
6595 Known = KnownFPClass::bitcast(FltSemantics: EltTy->getFltSemantics(), Bits);
6596 break;
6597 }
6598 default:
6599 break;
6600 }
6601}
6602
6603KnownFPClass llvm::computeKnownFPClass(const Value *V,
6604 const APInt &DemandedElts,
6605 FPClassTest InterestedClasses,
6606 const SimplifyQuery &SQ,
6607 unsigned Depth) {
6608 KnownFPClass KnownClasses;
6609 ::computeKnownFPClass(V, DemandedElts, InterestedClasses, Known&: KnownClasses, Q: SQ,
6610 Depth);
6611 return KnownClasses;
6612}
6613
6614KnownFPClass llvm::computeKnownFPClass(const Value *V,
6615 FPClassTest InterestedClasses,
6616 const SimplifyQuery &SQ,
6617 unsigned Depth) {
6618 KnownFPClass Known;
6619 ::computeKnownFPClass(V, Known, InterestedClasses, Q: SQ, Depth);
6620 return Known;
6621}
6622
6623KnownFPClass llvm::computeKnownFPClass(
6624 const Value *V, const DataLayout &DL, FPClassTest InterestedClasses,
6625 const TargetLibraryInfo *TLI, AssumptionCache *AC, const Instruction *CtxI,
6626 const DominatorTree *DT, bool UseInstrInfo, unsigned Depth) {
6627 return computeKnownFPClass(V, InterestedClasses,
6628 SQ: SimplifyQuery(DL, TLI, DT, AC, CtxI, UseInstrInfo),
6629 Depth);
6630}
6631
6632KnownFPClass
6633llvm::computeKnownFPClass(const Value *V, const APInt &DemandedElts,
6634 FastMathFlags FMF, FPClassTest InterestedClasses,
6635 const SimplifyQuery &SQ, unsigned Depth) {
6636 if (FMF.noNaNs())
6637 InterestedClasses &= ~fcNan;
6638 if (FMF.noInfs())
6639 InterestedClasses &= ~fcInf;
6640
6641 KnownFPClass Result =
6642 computeKnownFPClass(V, DemandedElts, InterestedClasses, SQ, Depth);
6643
6644 if (FMF.noNaNs())
6645 Result.setKnownFPClasses(Result.getKnownFPClasses() & ~fcNan);
6646 if (FMF.noInfs())
6647 Result.setKnownFPClasses(Result.getKnownFPClasses() & ~fcInf);
6648 return Result;
6649}
6650
6651KnownFPClass llvm::computeKnownFPClass(const Value *V, FastMathFlags FMF,
6652 FPClassTest InterestedClasses,
6653 const SimplifyQuery &SQ,
6654 unsigned Depth) {
6655 auto *FVTy = dyn_cast<FixedVectorType>(Val: V->getType());
6656 APInt DemandedElts =
6657 FVTy ? APInt::getAllOnes(numBits: FVTy->getNumElements()) : APInt(1, 1);
6658 return computeKnownFPClass(V, DemandedElts, FMF, InterestedClasses, SQ,
6659 Depth);
6660}
6661
6662bool llvm::cannotBeNegativeZero(const Value *V, const SimplifyQuery &SQ,
6663 unsigned Depth) {
6664 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNegZero, SQ, Depth);
6665 return Known.isKnownNeverNegZero();
6666}
6667
6668bool llvm::cannotBeOrderedLessThanZero(const Value *V, const SimplifyQuery &SQ,
6669 unsigned Depth) {
6670 KnownFPClass Known =
6671 computeKnownFPClass(V, InterestedClasses: KnownFPClass::OrderedLessThanZeroMask, SQ, Depth);
6672 return Known.cannotBeOrderedLessThanZero();
6673}
6674
6675bool llvm::isKnownNeverInfinity(const Value *V, const SimplifyQuery &SQ,
6676 unsigned Depth) {
6677 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf, SQ, Depth);
6678 return Known.isKnownNeverInfinity();
6679}
6680
6681/// Return true if the floating-point value can never contain a NaN or infinity.
6682bool llvm::isKnownNeverInfOrNaN(const Value *V, const SimplifyQuery &SQ,
6683 unsigned Depth) {
6684 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcInf | fcNan, SQ, Depth);
6685 return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity();
6686}
6687
6688/// Return true if the floating-point scalar value is not a NaN or if the
6689/// floating-point vector value has no NaN elements. Return false if a value
6690/// could ever be NaN.
6691bool llvm::isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ,
6692 unsigned Depth) {
6693 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcNan, SQ, Depth);
6694 return Known.isKnownNeverNaN();
6695}
6696
6697/// Return false if we can prove that the specified FP value's sign bit is 0.
6698/// Return true if we can prove that the specified FP value's sign bit is 1.
6699/// Otherwise return std::nullopt.
6700std::optional<bool> llvm::computeKnownFPSignBit(const Value *V,
6701 const SimplifyQuery &SQ,
6702 unsigned Depth) {
6703 KnownFPClass Known = computeKnownFPClass(V, InterestedClasses: fcAllFlags, SQ, Depth);
6704 return Known.getSignBit();
6705}
6706
6707bool llvm::canIgnoreSignBitOfZero(const Use &U) {
6708 auto *User = cast<Instruction>(Val: U.getUser());
6709 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6710 if (FPOp->hasNoSignedZeros())
6711 return true;
6712 }
6713
6714 switch (User->getOpcode()) {
6715 case Instruction::FPToSI:
6716 case Instruction::FPToUI:
6717 return true;
6718 case Instruction::FCmp:
6719 // fcmp treats both positive and negative zero as equal.
6720 return true;
6721 case Instruction::Call:
6722 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6723 switch (II->getIntrinsicID()) {
6724 case Intrinsic::fabs:
6725 return true;
6726 case Intrinsic::copysign:
6727 return U.getOperandNo() == 0;
6728 case Intrinsic::is_fpclass: {
6729 auto Test =
6730 static_cast<FPClassTest>(
6731 cast<ConstantInt>(Val: II->getArgOperand(i: 1))->getZExtValue()) &
6732 FPClassTest::fcZero;
6733 return Test == FPClassTest::fcZero || Test == FPClassTest::fcNone;
6734 }
6735 default:
6736 return false;
6737 }
6738 }
6739 return false;
6740 default:
6741 return false;
6742 }
6743}
6744
6745bool llvm::canIgnoreSignBitOfNaN(const Use &U) {
6746 auto *User = cast<Instruction>(Val: U.getUser());
6747 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: User)) {
6748 if (FPOp->hasNoNaNs())
6749 return true;
6750 }
6751
6752 switch (User->getOpcode()) {
6753 case Instruction::FPToSI:
6754 case Instruction::FPToUI:
6755 return true;
6756 // Proper FP math operations ignore the sign bit of NaN.
6757 case Instruction::FAdd:
6758 case Instruction::FSub:
6759 case Instruction::FMul:
6760 case Instruction::FDiv:
6761 case Instruction::FRem:
6762 case Instruction::FPTrunc:
6763 case Instruction::FPExt:
6764 case Instruction::FCmp:
6765 return true;
6766 // Bitwise FP operations should preserve the sign bit of NaN.
6767 case Instruction::FNeg:
6768 case Instruction::Select:
6769 case Instruction::PHI:
6770 return false;
6771 case Instruction::Ret:
6772 return User->getFunction()->getAttributes().getRetNoFPClass() &
6773 FPClassTest::fcNan;
6774 case Instruction::Call:
6775 case Instruction::Invoke: {
6776 if (auto *II = dyn_cast<IntrinsicInst>(Val: User)) {
6777 switch (II->getIntrinsicID()) {
6778 case Intrinsic::fabs:
6779 return true;
6780 case Intrinsic::copysign:
6781 return U.getOperandNo() == 0;
6782 // Other proper FP math intrinsics ignore the sign bit of NaN.
6783 case Intrinsic::maxnum:
6784 case Intrinsic::minnum:
6785 case Intrinsic::maximum:
6786 case Intrinsic::minimum:
6787 case Intrinsic::maximumnum:
6788 case Intrinsic::minimumnum:
6789 case Intrinsic::canonicalize:
6790 case Intrinsic::fma:
6791 case Intrinsic::fmuladd:
6792 case Intrinsic::sqrt:
6793 case Intrinsic::pow:
6794 case Intrinsic::powi:
6795 case Intrinsic::fptoui_sat:
6796 case Intrinsic::fptosi_sat:
6797 case Intrinsic::is_fpclass:
6798 return true;
6799 default:
6800 return false;
6801 }
6802 }
6803
6804 FPClassTest NoFPClass =
6805 cast<CallBase>(Val: User)->getParamNoFPClass(i: U.getOperandNo());
6806 return NoFPClass & FPClassTest::fcNan;
6807 }
6808 default:
6809 return false;
6810 }
6811}
6812
6813bool llvm::isKnownIntegral(const Value *V, const SimplifyQuery &SQ,
6814 FastMathFlags FMF) {
6815 if (isa<PoisonValue>(Val: V))
6816 return true;
6817 if (isa<UndefValue>(Val: V))
6818 return false;
6819
6820 if (match(V, P: m_CheckedFp(CheckFn: [](const APFloat &Val) { return Val.isInteger(); })))
6821 return true;
6822
6823 const Instruction *I = dyn_cast<Instruction>(Val: V);
6824 if (!I)
6825 return false;
6826
6827 switch (I->getOpcode()) {
6828 case Instruction::SIToFP:
6829 case Instruction::UIToFP:
6830 // TODO: Could check nofpclass(inf) on incoming argument
6831 if (FMF.noInfs())
6832 return true;
6833
6834 // Need to check int size cannot produce infinity, which computeKnownFPClass
6835 // knows how to do already.
6836 return isKnownNeverInfinity(V: I, SQ);
6837 case Instruction::Call: {
6838 const CallInst *CI = cast<CallInst>(Val: I);
6839 switch (CI->getIntrinsicID()) {
6840 case Intrinsic::trunc:
6841 case Intrinsic::floor:
6842 case Intrinsic::ceil:
6843 case Intrinsic::rint:
6844 case Intrinsic::nearbyint:
6845 case Intrinsic::round:
6846 case Intrinsic::roundeven:
6847 return (FMF.noInfs() && FMF.noNaNs()) || isKnownNeverInfOrNaN(V: I, SQ);
6848 default:
6849 break;
6850 }
6851
6852 break;
6853 }
6854 default:
6855 break;
6856 }
6857
6858 return false;
6859}
6860
6861Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
6862
6863 // All byte-wide stores are splatable, even of arbitrary variables.
6864 if (V->getType()->isIntegerTy(BitWidth: 8))
6865 return V;
6866
6867 LLVMContext &Ctx = V->getContext();
6868
6869 // Undef don't care.
6870 auto *UndefInt8 = UndefValue::get(T: Type::getInt8Ty(C&: Ctx));
6871 if (isa<UndefValue>(Val: V))
6872 return UndefInt8;
6873
6874 // Return poison for zero-sized type.
6875 if (DL.getTypeStoreSize(Ty: V->getType()).isZero())
6876 return PoisonValue::get(T: Type::getInt8Ty(C&: Ctx));
6877
6878 Constant *C = dyn_cast<Constant>(Val: V);
6879 if (!C) {
6880 // Conceptually, we could handle things like:
6881 // %a = zext i8 %X to i16
6882 // %b = shl i16 %a, 8
6883 // %c = or i16 %a, %b
6884 // but until there is an example that actually needs this, it doesn't seem
6885 // worth worrying about.
6886 return nullptr;
6887 }
6888
6889 // Handle 'null' ConstantArrayZero etc.
6890 if (C->isNullValue())
6891 return Constant::getNullValue(Ty: Type::getInt8Ty(C&: Ctx));
6892
6893 // Constant floating-point values can be handled as integer values if the
6894 // corresponding integer value is "byteable". An important case is 0.0.
6895 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
6896 Type *ScalarTy = CFP->getType()->getScalarType();
6897 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
6898 return isBytewiseValue(
6899 V: ConstantInt::get(Context&: Ctx, V: CFP->getValue().bitcastToAPInt()), DL);
6900
6901 // Don't handle long double formats, which have strange constraints.
6902 return nullptr;
6903 }
6904
6905 // We can handle constant integers that are multiple of 8 bits.
6906 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: C)) {
6907 if (CI->getBitWidth() % 8 == 0) {
6908 if (!CI->getValue().isSplat(SplatSizeInBits: 8))
6909 return nullptr;
6910 return ConstantInt::get(Context&: Ctx, V: CI->getValue().trunc(width: 8));
6911 }
6912 }
6913
6914 if (auto *CE = dyn_cast<ConstantExpr>(Val: C)) {
6915 if (CE->getOpcode() == Instruction::IntToPtr) {
6916 if (auto *PtrTy = dyn_cast<PointerType>(Val: CE->getType())) {
6917 unsigned BitWidth = DL.getPointerSizeInBits(AS: PtrTy->getAddressSpace());
6918 if (Constant *Op = ConstantFoldIntegerCast(
6919 C: CE->getOperand(i_nocapture: 0), DestTy: Type::getIntNTy(C&: Ctx, N: BitWidth), IsSigned: false, DL))
6920 return isBytewiseValue(V: Op, DL);
6921 }
6922 }
6923 }
6924
6925 auto Merge = [&](Value *LHS, Value *RHS) -> Value * {
6926 if (LHS == RHS)
6927 return LHS;
6928 if (!LHS || !RHS)
6929 return nullptr;
6930 if (LHS == UndefInt8)
6931 return RHS;
6932 if (RHS == UndefInt8)
6933 return LHS;
6934 return nullptr;
6935 };
6936
6937 if (ConstantDataSequential *CA = dyn_cast<ConstantDataSequential>(Val: C)) {
6938 Value *Val = UndefInt8;
6939 for (uint64_t I = 0, E = CA->getNumElements(); I != E; ++I)
6940 if (!(Val = Merge(Val, isBytewiseValue(V: CA->getElementAsConstant(i: I), DL))))
6941 return nullptr;
6942 return Val;
6943 }
6944
6945 if (isa<ConstantAggregate>(Val: C)) {
6946 Value *Val = UndefInt8;
6947 for (Value *Op : C->operands())
6948 if (!(Val = Merge(Val, isBytewiseValue(V: Op, DL))))
6949 return nullptr;
6950 return Val;
6951 }
6952
6953 // Don't try to handle the handful of other constants.
6954 return nullptr;
6955}
6956
6957// This is the recursive version of BuildSubAggregate. It takes a few different
6958// arguments. Idxs is the index within the nested struct From that we are
6959// looking at now (which is of type IndexedType). IdxSkip is the number of
6960// indices from Idxs that should be left out when inserting into the resulting
6961// struct. To is the result struct built so far, new insertvalue instructions
6962// build on that.
6963static Value *BuildSubAggregate(Value *From, Value *To, Type *IndexedType,
6964 SmallVectorImpl<unsigned> &Idxs,
6965 unsigned IdxSkip,
6966 BasicBlock::iterator InsertBefore) {
6967 StructType *STy = dyn_cast<StructType>(Val: IndexedType);
6968 if (STy) {
6969 // Save the original To argument so we can modify it
6970 Value *OrigTo = To;
6971 // General case, the type indexed by Idxs is a struct
6972 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6973 // Process each struct element recursively
6974 Idxs.push_back(Elt: i);
6975 Value *PrevTo = To;
6976 To = BuildSubAggregate(From, To, IndexedType: STy->getElementType(N: i), Idxs, IdxSkip,
6977 InsertBefore);
6978 Idxs.pop_back();
6979 if (!To) {
6980 // Couldn't find any inserted value for this index? Cleanup
6981 while (PrevTo != OrigTo) {
6982 InsertValueInst* Del = cast<InsertValueInst>(Val: PrevTo);
6983 PrevTo = Del->getAggregateOperand();
6984 Del->eraseFromParent();
6985 }
6986 // Stop processing elements
6987 break;
6988 }
6989 }
6990 // If we successfully found a value for each of our subaggregates
6991 if (To)
6992 return To;
6993 }
6994 // Base case, the type indexed by SourceIdxs is not a struct, or not all of
6995 // the struct's elements had a value that was inserted directly. In the latter
6996 // case, perhaps we can't determine each of the subelements individually, but
6997 // we might be able to find the complete struct somewhere.
6998
6999 // Find the value that is at that particular spot
7000 Value *V = FindInsertedValue(V: From, idx_range: Idxs);
7001
7002 if (!V)
7003 return nullptr;
7004
7005 // Insert the value in the new (sub) aggregate
7006 return InsertValueInst::Create(Agg: To, Val: V, Idxs: ArrayRef(Idxs).slice(N: IdxSkip), NameStr: "tmp",
7007 InsertBefore);
7008}
7009
7010// This helper takes a nested struct and extracts a part of it (which is again a
7011// struct) into a new value. For example, given the struct:
7012// { a, { b, { c, d }, e } }
7013// and the indices "1, 1" this returns
7014// { c, d }.
7015//
7016// It does this by inserting an insertvalue for each element in the resulting
7017// struct, as opposed to just inserting a single struct. This will only work if
7018// each of the elements of the substruct are known (ie, inserted into From by an
7019// insertvalue instruction somewhere).
7020//
7021// All inserted insertvalue instructions are inserted before InsertBefore
7022static Value *BuildSubAggregate(Value *From, ArrayRef<unsigned> idx_range,
7023 BasicBlock::iterator InsertBefore) {
7024 Type *IndexedType = ExtractValueInst::getIndexedType(Agg: From->getType(),
7025 Idxs: idx_range);
7026 Value *To = PoisonValue::get(T: IndexedType);
7027 SmallVector<unsigned, 10> Idxs(idx_range);
7028 unsigned IdxSkip = Idxs.size();
7029
7030 return BuildSubAggregate(From, To, IndexedType, Idxs, IdxSkip, InsertBefore);
7031}
7032
7033/// Given an aggregate and a sequence of indices, see if the scalar value
7034/// indexed is already around as a register, for example if it was inserted
7035/// directly into the aggregate.
7036///
7037/// If InsertBefore is not null, this function will duplicate (modified)
7038/// insertvalues when a part of a nested struct is extracted.
7039Value *
7040llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
7041 std::optional<BasicBlock::iterator> InsertBefore) {
7042 // Nothing to index? Just return V then (this is useful at the end of our
7043 // recursion).
7044 if (idx_range.empty())
7045 return V;
7046 // We have indices, so V should have an indexable type.
7047 assert((V->getType()->isStructTy() || V->getType()->isArrayTy()) &&
7048 "Not looking at a struct or array?");
7049 assert(ExtractValueInst::getIndexedType(V->getType(), idx_range) &&
7050 "Invalid indices for type?");
7051
7052 if (Constant *C = dyn_cast<Constant>(Val: V)) {
7053 C = C->getAggregateElement(Elt: idx_range[0]);
7054 if (!C) return nullptr;
7055 return FindInsertedValue(V: C, idx_range: idx_range.slice(N: 1), InsertBefore);
7056 }
7057
7058 if (InsertValueInst *I = dyn_cast<InsertValueInst>(Val: V)) {
7059 // Loop the indices for the insertvalue instruction in parallel with the
7060 // requested indices
7061 const unsigned *req_idx = idx_range.begin();
7062 for (const unsigned *i = I->idx_begin(), *e = I->idx_end();
7063 i != e; ++i, ++req_idx) {
7064 if (req_idx == idx_range.end()) {
7065 // We can't handle this without inserting insertvalues
7066 if (!InsertBefore)
7067 return nullptr;
7068
7069 // The requested index identifies a part of a nested aggregate. Handle
7070 // this specially. For example,
7071 // %A = insertvalue { i32, {i32, i32 } } undef, i32 10, 1, 0
7072 // %B = insertvalue { i32, {i32, i32 } } %A, i32 11, 1, 1
7073 // %C = extractvalue {i32, { i32, i32 } } %B, 1
7074 // This can be changed into
7075 // %A = insertvalue {i32, i32 } undef, i32 10, 0
7076 // %C = insertvalue {i32, i32 } %A, i32 11, 1
7077 // which allows the unused 0,0 element from the nested struct to be
7078 // removed.
7079 return BuildSubAggregate(From: V, idx_range: ArrayRef(idx_range.begin(), req_idx),
7080 InsertBefore: *InsertBefore);
7081 }
7082
7083 // This insert value inserts something else than what we are looking for.
7084 // See if the (aggregate) value inserted into has the value we are
7085 // looking for, then.
7086 if (*req_idx != *i)
7087 return FindInsertedValue(V: I->getAggregateOperand(), idx_range,
7088 InsertBefore);
7089 }
7090 // If we end up here, the indices of the insertvalue match with those
7091 // requested (though possibly only partially). Now we recursively look at
7092 // the inserted value, passing any remaining indices.
7093 return FindInsertedValue(V: I->getInsertedValueOperand(),
7094 idx_range: ArrayRef(req_idx, idx_range.end()), InsertBefore);
7095 }
7096
7097 if (ExtractValueInst *I = dyn_cast<ExtractValueInst>(Val: V)) {
7098 // If we're extracting a value from an aggregate that was extracted from
7099 // something else, we can extract from that something else directly instead.
7100 // However, we will need to chain I's indices with the requested indices.
7101
7102 // Calculate the number of indices required
7103 unsigned size = I->getNumIndices() + idx_range.size();
7104 // Allocate some space to put the new indices in
7105 SmallVector<unsigned, 5> Idxs;
7106 Idxs.reserve(N: size);
7107 // Add indices from the extract value instruction
7108 Idxs.append(in_start: I->idx_begin(), in_end: I->idx_end());
7109
7110 // Add requested indices
7111 Idxs.append(in_start: idx_range.begin(), in_end: idx_range.end());
7112
7113 assert(Idxs.size() == size
7114 && "Number of indices added not correct?");
7115
7116 return FindInsertedValue(V: I->getAggregateOperand(), idx_range: Idxs, InsertBefore);
7117 }
7118 // Otherwise, we don't know (such as, extracting from a function return value
7119 // or load instruction)
7120 return nullptr;
7121}
7122
7123// If V refers to an initialized global constant, set Slice either to
7124// its initializer if the size of its elements equals ElementSize, or,
7125// for ElementSize == 8, to its representation as an array of unsiged
7126// char. Return true on success.
7127// Offset is in the unit "nr of ElementSize sized elements".
7128bool llvm::getConstantDataArrayInfo(const Value *V,
7129 ConstantDataArraySlice &Slice,
7130 unsigned ElementSize, uint64_t Offset) {
7131 assert(V && "V should not be null.");
7132 assert((ElementSize % 8) == 0 &&
7133 "ElementSize expected to be a multiple of the size of a byte.");
7134 unsigned ElementSizeInBytes = ElementSize / 8;
7135
7136 // Drill down into the pointer expression V, ignoring any intervening
7137 // casts, and determine the identity of the object it references along
7138 // with the cumulative byte offset into it.
7139 const GlobalVariable *GV =
7140 dyn_cast<GlobalVariable>(Val: getUnderlyingObject(V));
7141 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
7142 // Fail if V is not based on constant global object.
7143 return false;
7144
7145 const DataLayout &DL = GV->getDataLayout();
7146 APInt Off(DL.getIndexTypeSizeInBits(Ty: V->getType()), 0);
7147
7148 if (GV != V->stripAndAccumulateConstantOffsets(DL, Offset&: Off,
7149 /*AllowNonInbounds*/ true))
7150 // Fail if a constant offset could not be determined.
7151 return false;
7152
7153 uint64_t StartIdx = Off.getLimitedValue();
7154 if (StartIdx == UINT64_MAX)
7155 // Fail if the constant offset is excessive.
7156 return false;
7157
7158 // Off/StartIdx is in the unit of bytes. So we need to convert to number of
7159 // elements. Simply bail out if that isn't possible.
7160 if ((StartIdx % ElementSizeInBytes) != 0)
7161 return false;
7162
7163 Offset += StartIdx / ElementSizeInBytes;
7164 ConstantDataArray *Array = nullptr;
7165 ArrayType *ArrayTy = nullptr;
7166
7167 if (GV->getInitializer()->isNullValue()) {
7168 Type *GVTy = GV->getValueType();
7169 uint64_t SizeInBytes = DL.getTypeStoreSize(Ty: GVTy).getFixedValue();
7170 uint64_t Length = SizeInBytes / ElementSizeInBytes;
7171
7172 Slice.Array = nullptr;
7173 Slice.Offset = 0;
7174 // Return an empty Slice for undersized constants to let callers
7175 // transform even undefined library calls into simpler, well-defined
7176 // expressions. This is preferable to making the calls although it
7177 // prevents sanitizers from detecting such calls.
7178 Slice.Length = Length < Offset ? 0 : Length - Offset;
7179 return true;
7180 }
7181
7182 auto *Init = const_cast<Constant *>(GV->getInitializer());
7183 if (auto *ArrayInit = dyn_cast<ConstantDataArray>(Val: Init)) {
7184 Type *InitElTy = ArrayInit->getElementType();
7185 if (InitElTy->isIntegerTy(BitWidth: ElementSize)) {
7186 // If Init is an initializer for an array of the expected type
7187 // and size, use it as is.
7188 Array = ArrayInit;
7189 ArrayTy = ArrayInit->getType();
7190 }
7191 }
7192
7193 if (!Array) {
7194 if (ElementSize != 8)
7195 // TODO: Handle conversions to larger integral types.
7196 return false;
7197
7198 // Otherwise extract the portion of the initializer starting
7199 // at Offset as an array of bytes, and reset Offset.
7200 Init = ReadByteArrayFromGlobal(GV, Offset);
7201 if (!Init)
7202 return false;
7203
7204 Offset = 0;
7205 Array = dyn_cast<ConstantDataArray>(Val: Init);
7206 ArrayTy = dyn_cast<ArrayType>(Val: Init->getType());
7207 }
7208
7209 uint64_t NumElts = ArrayTy->getArrayNumElements();
7210 if (Offset > NumElts)
7211 return false;
7212
7213 Slice.Array = Array;
7214 Slice.Offset = Offset;
7215 Slice.Length = NumElts - Offset;
7216 return true;
7217}
7218
7219/// Extract bytes from the initializer of the constant array V, which need
7220/// not be a nul-terminated string. On success, store the bytes in Str and
7221/// return true. When TrimAtNul is set, Str will contain only the bytes up
7222/// to but not including the first nul. Return false on failure.
7223bool llvm::getConstantStringInfo(const Value *V, StringRef &Str,
7224 bool TrimAtNul) {
7225 ConstantDataArraySlice Slice;
7226 if (!getConstantDataArrayInfo(V, Slice, ElementSize: 8))
7227 return false;
7228
7229 if (Slice.Array == nullptr) {
7230 if (TrimAtNul) {
7231 // Return a nul-terminated string even for an empty Slice. This is
7232 // safe because all existing SimplifyLibcalls callers require string
7233 // arguments and the behavior of the functions they fold is undefined
7234 // otherwise. Folding the calls this way is preferable to making
7235 // the undefined library calls, even though it prevents sanitizers
7236 // from reporting such calls.
7237 Str = StringRef();
7238 return true;
7239 }
7240 if (Slice.Length == 1) {
7241 Str = StringRef("", 1);
7242 return true;
7243 }
7244 // We cannot instantiate a StringRef as we do not have an appropriate string
7245 // of 0s at hand.
7246 return false;
7247 }
7248
7249 // Start out with the entire array in the StringRef.
7250 Str = Slice.Array->getAsString();
7251 // Skip over 'offset' bytes.
7252 Str = Str.substr(Start: Slice.Offset);
7253
7254 if (TrimAtNul) {
7255 // Trim off the \0 and anything after it. If the array is not nul
7256 // terminated, we just return the whole end of string. The client may know
7257 // some other way that the string is length-bound.
7258 Str = Str.substr(Start: 0, N: Str.find(C: '\0'));
7259 }
7260 return true;
7261}
7262
7263// These next two are very similar to the above, but also look through PHI
7264// nodes.
7265// TODO: See if we can integrate these two together.
7266
7267/// If we can compute the length of the string pointed to by
7268/// the specified pointer, return 'len+1'. If we can't, return 0.
7269static uint64_t GetStringLengthH(const Value *V,
7270 SmallPtrSetImpl<const PHINode*> &PHIs,
7271 unsigned CharSize) {
7272 // Look through noop bitcast instructions.
7273 V = V->stripPointerCasts();
7274
7275 // If this is a PHI node, there are two cases: either we have already seen it
7276 // or we haven't.
7277 if (const PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7278 if (!PHIs.insert(Ptr: PN).second)
7279 return ~0ULL; // already in the set.
7280
7281 // If it was new, see if all the input strings are the same length.
7282 uint64_t LenSoFar = ~0ULL;
7283 for (Value *IncValue : PN->incoming_values()) {
7284 uint64_t Len = GetStringLengthH(V: IncValue, PHIs, CharSize);
7285 if (Len == 0) return 0; // Unknown length -> unknown.
7286
7287 if (Len == ~0ULL) continue;
7288
7289 if (Len != LenSoFar && LenSoFar != ~0ULL)
7290 return 0; // Disagree -> unknown.
7291 LenSoFar = Len;
7292 }
7293
7294 // Success, all agree.
7295 return LenSoFar;
7296 }
7297
7298 // strlen(select(c,x,y)) -> strlen(x) ^ strlen(y)
7299 if (const SelectInst *SI = dyn_cast<SelectInst>(Val: V)) {
7300 uint64_t Len1 = GetStringLengthH(V: SI->getTrueValue(), PHIs, CharSize);
7301 if (Len1 == 0) return 0;
7302 uint64_t Len2 = GetStringLengthH(V: SI->getFalseValue(), PHIs, CharSize);
7303 if (Len2 == 0) return 0;
7304 if (Len1 == ~0ULL) return Len2;
7305 if (Len2 == ~0ULL) return Len1;
7306 if (Len1 != Len2) return 0;
7307 return Len1;
7308 }
7309
7310 // Otherwise, see if we can read the string.
7311 ConstantDataArraySlice Slice;
7312 if (!getConstantDataArrayInfo(V, Slice, ElementSize: CharSize))
7313 return 0;
7314
7315 if (Slice.Array == nullptr)
7316 // Zeroinitializer (including an empty one).
7317 return 1;
7318
7319 // Search for the first nul character. Return a conservative result even
7320 // when there is no nul. This is safe since otherwise the string function
7321 // being folded such as strlen is undefined, and can be preferable to
7322 // making the undefined library call.
7323 unsigned NullIndex = 0;
7324 for (unsigned E = Slice.Length; NullIndex < E; ++NullIndex) {
7325 if (Slice.Array->getElementAsInteger(i: Slice.Offset + NullIndex) == 0)
7326 break;
7327 }
7328
7329 return NullIndex + 1;
7330}
7331
7332/// If we can compute the length of the string pointed to by
7333/// the specified pointer, return 'len+1'. If we can't, return 0.
7334uint64_t llvm::GetStringLength(const Value *V, unsigned CharSize) {
7335 if (!V->getType()->isPointerTy())
7336 return 0;
7337
7338 SmallPtrSet<const PHINode*, 32> PHIs;
7339 uint64_t Len = GetStringLengthH(V, PHIs, CharSize);
7340 // If Len is ~0ULL, we had an infinite phi cycle: this is dead code, so return
7341 // an empty string as a length.
7342 return Len == ~0ULL ? 1 : Len;
7343}
7344
7345const Value *
7346llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call,
7347 bool MustPreserveOffset,
7348 bool MustPreserveProvenance) {
7349 assert(Call &&
7350 "getArgumentAliasingToReturnedPointer only works on nonnull calls");
7351 if (const Value *RV = Call->getReturnedArgOperand())
7352 return RV;
7353 // This can be used only as a aliasing property.
7354 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
7355 Call, MustPreserveOffset, MustPreserveProvenance))
7356 return Call->getArgOperand(i: 0);
7357 return nullptr;
7358}
7359
7360bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
7361 const CallBase *Call, bool MustPreserveOffset,
7362 bool MustPreserveProvenance) {
7363 switch (Call->getIntrinsicID()) {
7364 case Intrinsic::launder_invariant_group:
7365 case Intrinsic::aarch64_irg:
7366 case Intrinsic::aarch64_tagp:
7367 // The amdgcn_make_buffer_rsrc function does not alter the address of the
7368 // input pointer (and thus preserves the byte offset, which is the property
7369 // the MustPreserveOffset flag selects). However, it will not necessarily
7370 // map ptr addrspace(N) null to ptr addrspace(8) null, aka the "null
7371 // descriptor", which has "all loads return 0, all stores are dropped"
7372 // semantics. Given the context of this intrinsic list, no one should be
7373 // relying on such a strict bit-exact null mapping (and, at time of
7374 // writing, they are not), but we document this fact out of an abundance
7375 // of caution.
7376 case Intrinsic::amdgcn_make_buffer_rsrc:
7377 return !MustPreserveProvenance;
7378 case Intrinsic::ptrmask:
7379 return !MustPreserveOffset;
7380 case Intrinsic::threadlocal_address:
7381 // The underlying variable changes with thread ID. The Thread ID may change
7382 // at coroutine suspend points.
7383 return !Call->getParent()->getParent()->isPresplitCoroutine();
7384 default:
7385 return false;
7386 }
7387}
7388
7389/// \p PN defines a loop-variant pointer to an object. Check if the
7390/// previous iteration of the loop was referring to the same object as \p PN.
7391static bool isSameUnderlyingObjectInLoop(const PHINode *PN,
7392 const LoopInfo *LI) {
7393 // Find the loop-defined value.
7394 Loop *L = LI->getLoopFor(BB: PN->getParent());
7395 if (PN->getNumIncomingValues() != 2)
7396 return true;
7397
7398 // Find the value from previous iteration.
7399 auto *PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 0));
7400 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
7401 PrevValue = dyn_cast<Instruction>(Val: PN->getIncomingValue(i: 1));
7402 if (!PrevValue || LI->getLoopFor(BB: PrevValue->getParent()) != L)
7403 return true;
7404
7405 // If a new pointer is loaded in the loop, the pointer references a different
7406 // object in every iteration. E.g.:
7407 // for (i)
7408 // int *p = a[i];
7409 // ...
7410 if (auto *Load = dyn_cast<LoadInst>(Val: PrevValue))
7411 if (!L->isLoopInvariant(V: Load->getPointerOperand()))
7412 return false;
7413 return true;
7414}
7415
7416const Value *llvm::getUnderlyingObject(const Value *V, unsigned MaxLookup,
7417 bool MustPreserveProvenance) {
7418 for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
7419 if (auto *GEP = dyn_cast<GEPOperator>(Val: V)) {
7420 const Value *PtrOp = GEP->getPointerOperand();
7421 if (!PtrOp->getType()->isPointerTy()) // Only handle scalar pointer base.
7422 return V;
7423 V = PtrOp;
7424 } else if (Operator::getOpcode(V) == Instruction::BitCast ||
7425 Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
7426 Value *NewV = cast<Operator>(Val: V)->getOperand(i: 0);
7427 if (!NewV->getType()->isPointerTy())
7428 return V;
7429 V = NewV;
7430 } else if (auto *GA = dyn_cast<GlobalAlias>(Val: V)) {
7431 if (GA->isInterposable())
7432 return V;
7433 V = GA->getAliasee();
7434 } else {
7435 if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
7436 // Look through single-arg phi nodes created by LCSSA.
7437 if (PHI->getNumIncomingValues() == 1) {
7438 V = PHI->getIncomingValue(i: 0);
7439 continue;
7440 }
7441 } else if (auto *Call = dyn_cast<CallBase>(Val: V)) {
7442 // CaptureTracking can know about special capturing properties of some
7443 // intrinsics like launder.invariant.group, that can't be expressed with
7444 // the attributes, but have properties like returning aliasing pointer.
7445 // Because some analysis may assume that nocaptured pointer is not
7446 // returned from some special intrinsic (because function would have to
7447 // be marked with returns attribute), it is crucial to use this function
7448 // because it should be in sync with CaptureTracking. Not using it may
7449 // cause weird miscompilations where 2 aliasing pointers are assumed to
7450 // noalias.
7451 if (auto *RP = getArgumentAliasingToReturnedPointer(
7452 Call, /*MustPreserveOffset=*/false, MustPreserveProvenance)) {
7453 V = RP;
7454 continue;
7455 }
7456 }
7457
7458 return V;
7459 }
7460 assert(V->getType()->isPointerTy() && "Unexpected operand type!");
7461 }
7462 return V;
7463}
7464
7465void llvm::getUnderlyingObjects(const Value *V,
7466 SmallVectorImpl<const Value *> &Objects,
7467 const LoopInfo *LI, unsigned MaxLookup) {
7468 SmallPtrSet<const Value *, 4> Visited;
7469 SmallVector<const Value *, 4> Worklist;
7470 Worklist.push_back(Elt: V);
7471 do {
7472 const Value *P = Worklist.pop_back_val();
7473 P = getUnderlyingObject(V: P, MaxLookup);
7474
7475 if (!Visited.insert(Ptr: P).second)
7476 continue;
7477
7478 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7479 Worklist.push_back(Elt: SI->getTrueValue());
7480 Worklist.push_back(Elt: SI->getFalseValue());
7481 continue;
7482 }
7483
7484 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7485 // If this PHI changes the underlying object in every iteration of the
7486 // loop, don't look through it. Consider:
7487 // int **A;
7488 // for (i) {
7489 // Prev = Curr; // Prev = PHI (Prev_0, Curr)
7490 // Curr = A[i];
7491 // *Prev, *Curr;
7492 //
7493 // Prev is tracking Curr one iteration behind so they refer to different
7494 // underlying objects.
7495 if (!LI || !LI->isLoopHeader(BB: PN->getParent()) ||
7496 isSameUnderlyingObjectInLoop(PN, LI))
7497 append_range(C&: Worklist, R: PN->incoming_values());
7498 else
7499 Objects.push_back(Elt: P);
7500 continue;
7501 }
7502
7503 Objects.push_back(Elt: P);
7504 } while (!Worklist.empty());
7505}
7506
7507const Value *llvm::getUnderlyingObjectAggressive(const Value *V,
7508 bool MustPreserveProvenance) {
7509 const unsigned MaxVisited = 8;
7510
7511 SmallPtrSet<const Value *, 8> Visited;
7512 SmallVector<const Value *, 8> Worklist;
7513 Worklist.push_back(Elt: V);
7514 const Value *Object = nullptr;
7515 // Used as fallback if we can't find a common underlying object through
7516 // recursion.
7517 bool First = true;
7518 const Value *FirstObject =
7519 getUnderlyingObject(V, MaxLookup: MaxLookupSearchDepth, MustPreserveProvenance);
7520 do {
7521 const Value *P = Worklist.pop_back_val();
7522 P = First ? FirstObject
7523 : getUnderlyingObject(V: P, MaxLookup: MaxLookupSearchDepth,
7524 MustPreserveProvenance);
7525 First = false;
7526
7527 if (!Visited.insert(Ptr: P).second)
7528 continue;
7529
7530 if (Visited.size() == MaxVisited)
7531 return FirstObject;
7532
7533 if (auto *SI = dyn_cast<SelectInst>(Val: P)) {
7534 Worklist.push_back(Elt: SI->getTrueValue());
7535 Worklist.push_back(Elt: SI->getFalseValue());
7536 continue;
7537 }
7538
7539 if (auto *PN = dyn_cast<PHINode>(Val: P)) {
7540 append_range(C&: Worklist, R: PN->incoming_values());
7541 continue;
7542 }
7543
7544 if (!Object)
7545 Object = P;
7546 else if (Object != P)
7547 return FirstObject;
7548 } while (!Worklist.empty());
7549
7550 return Object ? Object : FirstObject;
7551}
7552
7553/// This is the function that does the work of looking through basic
7554/// ptrtoint+arithmetic+inttoptr sequences.
7555static const Value *getUnderlyingObjectFromInt(const Value *V) {
7556 do {
7557 if (const Operator *U = dyn_cast<Operator>(Val: V)) {
7558 // If we find a ptrtoint, we can transfer control back to the
7559 // regular getUnderlyingObjectFromInt.
7560 if (U->getOpcode() == Instruction::PtrToInt)
7561 return U->getOperand(i: 0);
7562 // If we find an add of a constant, a multiplied value, or a phi, it's
7563 // likely that the other operand will lead us to the base
7564 // object. We don't have to worry about the case where the
7565 // object address is somehow being computed by the multiply,
7566 // because our callers only care when the result is an
7567 // identifiable object.
7568 if (U->getOpcode() != Instruction::Add ||
7569 (!isa<ConstantInt>(Val: U->getOperand(i: 1)) &&
7570 Operator::getOpcode(V: U->getOperand(i: 1)) != Instruction::Mul &&
7571 !isa<PHINode>(Val: U->getOperand(i: 1))))
7572 return V;
7573 V = U->getOperand(i: 0);
7574 } else {
7575 return V;
7576 }
7577 assert(V->getType()->isIntegerTy() && "Unexpected operand type!");
7578 } while (true);
7579}
7580
7581/// This is a wrapper around getUnderlyingObjects and adds support for basic
7582/// ptrtoint+arithmetic+inttoptr sequences.
7583/// It returns false if unidentified object is found in getUnderlyingObjects.
7584bool llvm::getUnderlyingObjectsForCodeGen(const Value *V,
7585 SmallVectorImpl<Value *> &Objects) {
7586 SmallPtrSet<const Value *, 16> Visited;
7587 SmallVector<const Value *, 4> Working(1, V);
7588 bool AllObjectsIdentified = true;
7589 do {
7590 V = Working.pop_back_val();
7591
7592 SmallVector<const Value *, 4> Objs;
7593 getUnderlyingObjects(V, Objects&: Objs);
7594
7595 for (const Value *V : Objs) {
7596 if (!Visited.insert(Ptr: V).second)
7597 continue;
7598 if (Operator::getOpcode(V) == Instruction::IntToPtr) {
7599 const Value *O =
7600 getUnderlyingObjectFromInt(V: cast<User>(Val: V)->getOperand(i: 0));
7601 if (O->getType()->isPointerTy()) {
7602 Working.push_back(Elt: O);
7603 continue;
7604 }
7605 }
7606 AllObjectsIdentified &= isIdentifiedObject(V);
7607 Objects.push_back(Elt: const_cast<Value *>(V));
7608 }
7609 } while (!Working.empty());
7610 return AllObjectsIdentified;
7611}
7612
7613AllocaInst *llvm::findAllocaForValue(Value *V, bool OffsetZero) {
7614 AllocaInst *Result = nullptr;
7615 SmallPtrSet<Value *, 4> Visited;
7616 SmallVector<Value *, 4> Worklist;
7617
7618 auto AddWork = [&](Value *V) {
7619 if (Visited.insert(Ptr: V).second)
7620 Worklist.push_back(Elt: V);
7621 };
7622
7623 AddWork(V);
7624 do {
7625 V = Worklist.pop_back_val();
7626 assert(Visited.count(V));
7627
7628 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V)) {
7629 if (Result && Result != AI)
7630 return nullptr;
7631 Result = AI;
7632 } else if (CastInst *CI = dyn_cast<CastInst>(Val: V)) {
7633 AddWork(CI->getOperand(i_nocapture: 0));
7634 } else if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
7635 for (Value *IncValue : PN->incoming_values())
7636 AddWork(IncValue);
7637 } else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
7638 AddWork(SI->getTrueValue());
7639 AddWork(SI->getFalseValue());
7640 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: V)) {
7641 if (OffsetZero && !GEP->hasAllZeroIndices())
7642 return nullptr;
7643 AddWork(GEP->getPointerOperand());
7644 } else if (CallBase *CB = dyn_cast<CallBase>(Val: V)) {
7645 Value *Returned = CB->getReturnedArgOperand();
7646 if (Returned)
7647 AddWork(Returned);
7648 else
7649 return nullptr;
7650 } else {
7651 return nullptr;
7652 }
7653 } while (!Worklist.empty());
7654
7655 return Result;
7656}
7657
7658static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7659 const Value *V, bool AllowLifetime, bool AllowDroppable) {
7660 for (const User *U : V->users()) {
7661 const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
7662 if (!II)
7663 return false;
7664
7665 if (AllowLifetime && II->isLifetimeStartOrEnd())
7666 continue;
7667
7668 if (AllowDroppable && II->isDroppable())
7669 continue;
7670
7671 return false;
7672 }
7673 return true;
7674}
7675
7676bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
7677 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7678 V, /* AllowLifetime */ true, /* AllowDroppable */ false);
7679}
7680bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) {
7681 return onlyUsedByLifetimeMarkersOrDroppableInstsHelper(
7682 V, /* AllowLifetime */ true, /* AllowDroppable */ true);
7683}
7684
7685bool llvm::isNotCrossLaneOperation(const Instruction *I) {
7686 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
7687 return isTriviallyVectorizable(ID: II->getIntrinsicID());
7688 auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: I);
7689 return (!Shuffle || Shuffle->isSelect()) &&
7690 !isa<CallBase, BitCastInst, ExtractElementInst>(Val: I);
7691}
7692
7693bool llvm::isSafeToSpeculativelyExecute(
7694 const Instruction *Inst, const Instruction *CtxI, AssumptionCache *AC,
7695 const DominatorTree *DT, const TargetLibraryInfo *TLI, bool UseVariableInfo,
7696 bool IgnoreUBImplyingAttrs) {
7697 return isSafeToSpeculativelyExecuteWithOpcode(Opcode: Inst->getOpcode(), Inst, CtxI,
7698 AC, DT, TLI, UseVariableInfo,
7699 IgnoreUBImplyingAttrs);
7700}
7701
7702bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
7703 unsigned Opcode, const Instruction *Inst, const Instruction *CtxI,
7704 AssumptionCache *AC, const DominatorTree *DT, const TargetLibraryInfo *TLI,
7705 bool UseVariableInfo, bool IgnoreUBImplyingAttrs) {
7706#ifndef NDEBUG
7707 if (Inst->getOpcode() != Opcode) {
7708 // Check that the operands are actually compatible with the Opcode override.
7709 auto hasEqualReturnAndLeadingOperandTypes =
7710 [](const Instruction *Inst, unsigned NumLeadingOperands) {
7711 if (Inst->getNumOperands() < NumLeadingOperands)
7712 return false;
7713 const Type *ExpectedType = Inst->getType();
7714 for (unsigned ItOp = 0; ItOp < NumLeadingOperands; ++ItOp)
7715 if (Inst->getOperand(ItOp)->getType() != ExpectedType)
7716 return false;
7717 return true;
7718 };
7719 assert(!Instruction::isBinaryOp(Opcode) ||
7720 hasEqualReturnAndLeadingOperandTypes(Inst, 2));
7721 assert(!Instruction::isUnaryOp(Opcode) ||
7722 hasEqualReturnAndLeadingOperandTypes(Inst, 1));
7723 }
7724#endif
7725
7726 switch (Opcode) {
7727 default:
7728 return true;
7729 case Instruction::UDiv:
7730 case Instruction::URem: {
7731 // x / y is undefined if y == 0.
7732 const APInt *V;
7733 if (match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: V)))
7734 return *V != 0;
7735 return false;
7736 }
7737 case Instruction::SDiv:
7738 case Instruction::SRem: {
7739 // x / y is undefined if y == 0 or x == INT_MIN and y == -1
7740 const APInt *Numerator, *Denominator;
7741 if (!match(V: Inst->getOperand(i: 1), P: m_APInt(Res&: Denominator)))
7742 return false;
7743 // We cannot hoist this division if the denominator is 0.
7744 if (*Denominator == 0)
7745 return false;
7746 // It's safe to hoist if the denominator is not 0 or -1.
7747 if (!Denominator->isAllOnes())
7748 return true;
7749 // At this point we know that the denominator is -1. It is safe to hoist as
7750 // long we know that the numerator is not INT_MIN.
7751 if (match(V: Inst->getOperand(i: 0), P: m_APInt(Res&: Numerator)))
7752 return !Numerator->isMinSignedValue();
7753 // The numerator *might* be MinSignedValue.
7754 return false;
7755 }
7756 case Instruction::Load: {
7757 if (!UseVariableInfo)
7758 return false;
7759
7760 const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst);
7761 if (!LI)
7762 return false;
7763 if (mustSuppressSpeculation(LI: *LI))
7764 return false;
7765 const DataLayout &DL = LI->getDataLayout();
7766 return isDereferenceableAndAlignedPointer(
7767 V: LI->getPointerOperand(), Ty: LI->getType(), Alignment: LI->getAlign(),
7768 Q: SimplifyQuery(DL, TLI, DT, AC, CtxI));
7769 }
7770 case Instruction::Call: {
7771 auto *CI = dyn_cast<const CallInst>(Val: Inst);
7772 if (!CI)
7773 return false;
7774 const Function *Callee = CI->getCalledFunction();
7775
7776 // The called function could have undefined behavior or side-effects, even
7777 // if marked readnone nounwind.
7778 if (!Callee || !Callee->isSpeculatable())
7779 return false;
7780 // Since the operands may be changed after hoisting, undefined behavior may
7781 // be triggered by some UB-implying attributes.
7782 return IgnoreUBImplyingAttrs || !CI->hasUBImplyingAttrs();
7783 }
7784 case Instruction::VAArg:
7785 case Instruction::Alloca:
7786 case Instruction::Invoke:
7787 case Instruction::CallBr:
7788 case Instruction::PHI:
7789 case Instruction::Store:
7790 case Instruction::Ret:
7791 case Instruction::UncondBr:
7792 case Instruction::CondBr:
7793 case Instruction::IndirectBr:
7794 case Instruction::Switch:
7795 case Instruction::Unreachable:
7796 case Instruction::Fence:
7797 case Instruction::AtomicRMW:
7798 case Instruction::AtomicCmpXchg:
7799 case Instruction::LandingPad:
7800 case Instruction::Resume:
7801 case Instruction::CatchSwitch:
7802 case Instruction::CatchPad:
7803 case Instruction::CatchRet:
7804 case Instruction::CleanupPad:
7805 case Instruction::CleanupRet:
7806 return false; // Misc instructions which have effects
7807 }
7808}
7809
7810bool llvm::mayHaveNonDefUseDependency(const Instruction &I) {
7811 if (I.mayReadOrWriteMemory())
7812 // Memory dependency possible
7813 return true;
7814 if (!isSafeToSpeculativelyExecute(Inst: &I))
7815 // Can't move above a maythrow call or infinite loop. Or if an
7816 // inalloca alloca, above a stacksave call.
7817 return true;
7818 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7819 // 1) Can't reorder two inf-loop calls, even if readonly
7820 // 2) Also can't reorder an inf-loop call below a instruction which isn't
7821 // safe to speculative execute. (Inverse of above)
7822 return true;
7823 return false;
7824}
7825
7826/// Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
7827static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) {
7828 switch (OR) {
7829 case ConstantRange::OverflowResult::MayOverflow:
7830 return OverflowResult::MayOverflow;
7831 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
7832 return OverflowResult::AlwaysOverflowsLow;
7833 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
7834 return OverflowResult::AlwaysOverflowsHigh;
7835 case ConstantRange::OverflowResult::NeverOverflows:
7836 return OverflowResult::NeverOverflows;
7837 }
7838 llvm_unreachable("Unknown OverflowResult");
7839}
7840
7841/// Combine constant ranges from computeConstantRange() and computeKnownBits().
7842ConstantRange
7843llvm::computeConstantRangeIncludingKnownBits(const WithCache<const Value *> &V,
7844 bool ForSigned,
7845 const SimplifyQuery &SQ) {
7846 ConstantRange CR1 =
7847 ConstantRange::fromKnownBits(Known: V.getKnownBits(Q: SQ), IsSigned: ForSigned);
7848 ConstantRange CR2 = computeConstantRange(V, ForSigned, SQ);
7849 ConstantRange::PreferredRangeType RangeType =
7850 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
7851 return CR1.intersectWith(CR: CR2, Type: RangeType);
7852}
7853
7854OverflowResult llvm::computeOverflowForUnsignedMul(const Value *LHS,
7855 const Value *RHS,
7856 const SimplifyQuery &SQ,
7857 bool IsNSW) {
7858 ConstantRange LHSRange =
7859 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7860 ConstantRange RHSRange =
7861 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7862
7863 // mul nsw of two non-negative numbers is also nuw.
7864 if (IsNSW && LHSRange.isAllNonNegative() && RHSRange.isAllNonNegative())
7865 return OverflowResult::NeverOverflows;
7866
7867 return mapOverflowResult(OR: LHSRange.unsignedMulMayOverflow(Other: RHSRange));
7868}
7869
7870OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
7871 const Value *RHS,
7872 const SimplifyQuery &SQ) {
7873 // Multiplying n * m significant bits yields a result of n + m significant
7874 // bits. If the total number of significant bits does not exceed the
7875 // result bit width (minus 1), there is no overflow.
7876 // This means if we have enough leading sign bits in the operands
7877 // we can guarantee that the result does not overflow.
7878 // Ref: "Hacker's Delight" by Henry Warren
7879 unsigned BitWidth = LHS->getType()->getScalarSizeInBits();
7880
7881 // Note that underestimating the number of sign bits gives a more
7882 // conservative answer.
7883 unsigned SignBits =
7884 ::ComputeNumSignBits(V: LHS, Q: SQ) + ::ComputeNumSignBits(V: RHS, Q: SQ);
7885
7886 // First handle the easy case: if we have enough sign bits there's
7887 // definitely no overflow.
7888 if (SignBits > BitWidth + 1)
7889 return OverflowResult::NeverOverflows;
7890
7891 // There are two ambiguous cases where there can be no overflow:
7892 // SignBits == BitWidth + 1 and
7893 // SignBits == BitWidth
7894 // The second case is difficult to check, therefore we only handle the
7895 // first case.
7896 if (SignBits == BitWidth + 1) {
7897 // It overflows only when both arguments are negative and the true
7898 // product is exactly the minimum negative number.
7899 // E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
7900 // For simplicity we just check if at least one side is not negative.
7901 KnownBits LHSKnown = computeKnownBits(V: LHS, Q: SQ);
7902 KnownBits RHSKnown = computeKnownBits(V: RHS, Q: SQ);
7903 if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
7904 return OverflowResult::NeverOverflows;
7905 }
7906 return OverflowResult::MayOverflow;
7907}
7908
7909OverflowResult
7910llvm::computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
7911 const WithCache<const Value *> &RHS,
7912 const SimplifyQuery &SQ) {
7913 ConstantRange LHSRange =
7914 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
7915 ConstantRange RHSRange =
7916 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
7917 return mapOverflowResult(OR: LHSRange.unsignedAddMayOverflow(Other: RHSRange));
7918}
7919
7920static OverflowResult
7921computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
7922 const WithCache<const Value *> &RHS,
7923 const AddOperator *Add, const SimplifyQuery &SQ) {
7924 if (Add && Add->hasNoSignedWrap()) {
7925 return OverflowResult::NeverOverflows;
7926 }
7927
7928 // If LHS and RHS each have at least two sign bits, the addition will look
7929 // like
7930 //
7931 // XX..... +
7932 // YY.....
7933 //
7934 // If the carry into the most significant position is 0, X and Y can't both
7935 // be 1 and therefore the carry out of the addition is also 0.
7936 //
7937 // If the carry into the most significant position is 1, X and Y can't both
7938 // be 0 and therefore the carry out of the addition is also 1.
7939 //
7940 // Since the carry into the most significant position is always equal to
7941 // the carry out of the addition, there is no signed overflow.
7942 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
7943 return OverflowResult::NeverOverflows;
7944
7945 ConstantRange LHSRange =
7946 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
7947 ConstantRange RHSRange =
7948 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
7949 OverflowResult OR =
7950 mapOverflowResult(OR: LHSRange.signedAddMayOverflow(Other: RHSRange));
7951 if (OR != OverflowResult::MayOverflow)
7952 return OR;
7953
7954 // The remaining code needs Add to be available. Early returns if not so.
7955 if (!Add)
7956 return OverflowResult::MayOverflow;
7957
7958 // If the sign of Add is the same as at least one of the operands, this add
7959 // CANNOT overflow. If this can be determined from the known bits of the
7960 // operands the above signedAddMayOverflow() check will have already done so.
7961 // The only other way to improve on the known bits is from an assumption, so
7962 // call computeKnownBitsFromContext() directly.
7963 bool LHSOrRHSKnownNonNegative =
7964 (LHSRange.isAllNonNegative() || RHSRange.isAllNonNegative());
7965 bool LHSOrRHSKnownNegative =
7966 (LHSRange.isAllNegative() || RHSRange.isAllNegative());
7967 if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) {
7968 KnownBits AddKnown(LHSRange.getBitWidth());
7969 computeKnownBitsFromContext(V: Add, Known&: AddKnown, Q: SQ);
7970 if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) ||
7971 (AddKnown.isNegative() && LHSOrRHSKnownNegative))
7972 return OverflowResult::NeverOverflows;
7973 }
7974
7975 return OverflowResult::MayOverflow;
7976}
7977
7978OverflowResult llvm::computeOverflowForUnsignedSub(const Value *LHS,
7979 const Value *RHS,
7980 const SimplifyQuery &SQ) {
7981 // X - (X % ?)
7982 // The remainder of a value can't have greater magnitude than itself,
7983 // so the subtraction can't overflow.
7984
7985 // X - (X -nuw ?)
7986 // In the minimal case, this would simplify to "?", so there's no subtract
7987 // at all. But if this analysis is used to peek through casts, for example,
7988 // then determining no-overflow may allow other transforms.
7989
7990 // TODO: There are other patterns like this.
7991 // See simplifyICmpWithBinOpOnLHS() for candidates.
7992 if (match(V: RHS, P: m_URem(L: m_Specific(V: LHS), R: m_Value())) ||
7993 match(V: RHS, P: m_NUWSub(L: m_Specific(V: LHS), R: m_Value())))
7994 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT))
7995 return OverflowResult::NeverOverflows;
7996
7997 if (auto C = isImpliedByDomCondition(Pred: CmpInst::ICMP_UGE, LHS, RHS, ContextI: SQ.CtxI,
7998 DL: SQ.DL)) {
7999 if (*C)
8000 return OverflowResult::NeverOverflows;
8001 return OverflowResult::AlwaysOverflowsLow;
8002 }
8003
8004 ConstantRange LHSRange =
8005 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/false, SQ);
8006 ConstantRange RHSRange =
8007 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/false, SQ);
8008 return mapOverflowResult(OR: LHSRange.unsignedSubMayOverflow(Other: RHSRange));
8009}
8010
8011OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS,
8012 const Value *RHS,
8013 const SimplifyQuery &SQ) {
8014 // X - (X % ?)
8015 // The remainder of a value can't have greater magnitude than itself,
8016 // so the subtraction can't overflow.
8017
8018 // X - (X -nsw ?)
8019 // In the minimal case, this would simplify to "?", so there's no subtract
8020 // at all. But if this analysis is used to peek through casts, for example,
8021 // then determining no-overflow may allow other transforms.
8022 if (match(V: RHS, P: m_SRem(L: m_Specific(V: LHS), R: m_Value())) ||
8023 match(V: RHS, P: m_NSWSub(L: m_Specific(V: LHS), R: m_Value())))
8024 if (isGuaranteedNotToBeUndef(V: LHS, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT))
8025 return OverflowResult::NeverOverflows;
8026
8027 // If LHS and RHS each have at least two sign bits, the subtraction
8028 // cannot overflow.
8029 if (::ComputeNumSignBits(V: LHS, Q: SQ) > 1 && ::ComputeNumSignBits(V: RHS, Q: SQ) > 1)
8030 return OverflowResult::NeverOverflows;
8031
8032 ConstantRange LHSRange =
8033 computeConstantRangeIncludingKnownBits(V: LHS, /*ForSigned=*/true, SQ);
8034 ConstantRange RHSRange =
8035 computeConstantRangeIncludingKnownBits(V: RHS, /*ForSigned=*/true, SQ);
8036 return mapOverflowResult(OR: LHSRange.signedSubMayOverflow(Other: RHSRange));
8037}
8038
8039bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO,
8040 const DominatorTree &DT) {
8041 SmallVector<const CondBrInst *, 2> GuardingBranches;
8042 SmallVector<const ExtractValueInst *, 2> Results;
8043
8044 for (const User *U : WO->users()) {
8045 if (const auto *EVI = dyn_cast<ExtractValueInst>(Val: U)) {
8046 assert(EVI->getNumIndices() == 1 && "Obvious from CI's type");
8047
8048 if (EVI->getIndices()[0] == 0)
8049 Results.push_back(Elt: EVI);
8050 else {
8051 assert(EVI->getIndices()[0] == 1 && "Obvious from CI's type");
8052
8053 for (const auto *U : EVI->users())
8054 if (const auto *B = dyn_cast<CondBrInst>(Val: U))
8055 GuardingBranches.push_back(Elt: B);
8056 }
8057 } else {
8058 // We are using the aggregate directly in a way we don't want to analyze
8059 // here (storing it to a global, say).
8060 return false;
8061 }
8062 }
8063
8064 auto AllUsesGuardedByBranch = [&](const CondBrInst *BI) {
8065 BasicBlockEdge NoWrapEdge(BI->getParent(), BI->getSuccessor(i: 1));
8066
8067 // Check if all users of the add are provably no-wrap.
8068 for (const auto *Result : Results) {
8069 // If the extractvalue itself is not executed on overflow, the we don't
8070 // need to check each use separately, since domination is transitive.
8071 if (DT.dominates(BBE: NoWrapEdge, BB: Result->getParent()))
8072 continue;
8073
8074 for (const auto &RU : Result->uses())
8075 if (!DT.dominates(BBE: NoWrapEdge, U: RU))
8076 return false;
8077 }
8078
8079 return true;
8080 };
8081
8082 return llvm::any_of(Range&: GuardingBranches, P: AllUsesGuardedByBranch);
8083}
8084
8085/// Shifts return poison if shiftwidth is larger than the bitwidth.
8086static bool shiftAmountKnownInRange(const Value *ShiftAmount) {
8087 auto *C = dyn_cast<Constant>(Val: ShiftAmount);
8088 if (!C)
8089 return false;
8090
8091 // Shifts return poison if shiftwidth is larger than the bitwidth.
8092 SmallVector<const Constant *, 4> ShiftAmounts;
8093 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: C->getType())) {
8094 unsigned NumElts = FVTy->getNumElements();
8095 for (unsigned i = 0; i < NumElts; ++i)
8096 ShiftAmounts.push_back(Elt: C->getAggregateElement(Elt: i));
8097 } else if (isa<ScalableVectorType>(Val: C->getType()))
8098 return false; // Can't tell, just return false to be safe
8099 else
8100 ShiftAmounts.push_back(Elt: C);
8101
8102 bool Safe = llvm::all_of(Range&: ShiftAmounts, P: [](const Constant *C) {
8103 auto *CI = dyn_cast_or_null<ConstantInt>(Val: C);
8104 return CI && CI->getValue().ult(RHS: C->getType()->getIntegerBitWidth());
8105 });
8106
8107 return Safe;
8108}
8109
8110static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
8111 bool ConsiderFlagsAndMetadata) {
8112
8113 if (ConsiderFlagsAndMetadata && includesPoison(Kind) &&
8114 Op->hasPoisonGeneratingAnnotations())
8115 return true;
8116
8117 unsigned Opcode = Op->getOpcode();
8118
8119 // Check whether opcode is a poison/undef-generating operation
8120 switch (Opcode) {
8121 case Instruction::Shl:
8122 case Instruction::AShr:
8123 case Instruction::LShr:
8124 return includesPoison(Kind) && !shiftAmountKnownInRange(ShiftAmount: Op->getOperand(i: 1));
8125 case Instruction::FPToSI:
8126 case Instruction::FPToUI:
8127 // fptosi/ui yields poison if the resulting value does not fit in the
8128 // destination type.
8129 return true;
8130 case Instruction::Call:
8131 if (auto *II = dyn_cast<IntrinsicInst>(Val: Op)) {
8132 switch (II->getIntrinsicID()) {
8133 // NOTE: Use IntrNoCreateUndefOrPoison when possible.
8134 case Intrinsic::ctlz:
8135 case Intrinsic::cttz:
8136 case Intrinsic::abs:
8137 // We're not considering flags so it is safe to just return false.
8138 return false;
8139 case Intrinsic::sshl_sat:
8140 case Intrinsic::ushl_sat:
8141 if (!includesPoison(Kind) ||
8142 shiftAmountKnownInRange(ShiftAmount: II->getArgOperand(i: 1)))
8143 return false;
8144 break;
8145 }
8146 }
8147 [[fallthrough]];
8148 case Instruction::CallBr:
8149 case Instruction::Invoke: {
8150 const auto *CB = cast<CallBase>(Val: Op);
8151 return !CB->hasRetAttr(Kind: Attribute::NoUndef) &&
8152 !CB->hasFnAttr(Kind: Attribute::NoCreateUndefOrPoison);
8153 }
8154 case Instruction::InsertElement:
8155 case Instruction::ExtractElement: {
8156 // If index exceeds the length of the vector, it returns poison
8157 auto *VTy = cast<VectorType>(Val: Op->getOperand(i: 0)->getType());
8158 unsigned IdxOp = Op->getOpcode() == Instruction::InsertElement ? 2 : 1;
8159 auto *Idx = dyn_cast<ConstantInt>(Val: Op->getOperand(i: IdxOp));
8160 if (includesPoison(Kind))
8161 return !Idx ||
8162 Idx->getValue().uge(RHS: VTy->getElementCount().getKnownMinValue());
8163 return false;
8164 }
8165 case Instruction::ShuffleVector: {
8166 ArrayRef<int> Mask = isa<ConstantExpr>(Val: Op)
8167 ? cast<ConstantExpr>(Val: Op)->getShuffleMask()
8168 : cast<ShuffleVectorInst>(Val: Op)->getShuffleMask();
8169 return includesPoison(Kind) && is_contained(Range&: Mask, Element: PoisonMaskElem);
8170 }
8171 case Instruction::FNeg:
8172 case Instruction::PHI:
8173 case Instruction::Select:
8174 case Instruction::ExtractValue:
8175 case Instruction::InsertValue:
8176 case Instruction::Freeze:
8177 case Instruction::ICmp:
8178 case Instruction::FCmp:
8179 case Instruction::GetElementPtr:
8180 return false;
8181 case Instruction::AddrSpaceCast:
8182 return true;
8183 default: {
8184 const auto *CE = dyn_cast<ConstantExpr>(Val: Op);
8185 if (isa<CastInst>(Val: Op) || (CE && CE->isCast()))
8186 return false;
8187 else if (Instruction::isBinaryOp(Opcode))
8188 return false;
8189 // Be conservative and return true.
8190 return true;
8191 }
8192 }
8193}
8194
8195bool llvm::canCreateUndefOrPoison(const Operator *Op,
8196 bool ConsiderFlagsAndMetadata) {
8197 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::UndefOrPoison,
8198 ConsiderFlagsAndMetadata);
8199}
8200
8201bool llvm::canCreatePoison(const Operator *Op, bool ConsiderFlagsAndMetadata) {
8202 return ::canCreateUndefOrPoison(Op, Kind: UndefPoisonKind::PoisonOnly,
8203 ConsiderFlagsAndMetadata);
8204}
8205
8206static bool directlyImpliesPoison(const Value *ValAssumedPoison, const Value *V,
8207 unsigned Depth) {
8208 if (ValAssumedPoison == V)
8209 return true;
8210
8211 const unsigned MaxDepth = 2;
8212 if (Depth >= MaxDepth)
8213 return false;
8214
8215 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
8216 if (any_of(Range: I->operands(), P: [=](const Use &Op) {
8217 return propagatesPoison(PoisonOp: Op) &&
8218 directlyImpliesPoison(ValAssumedPoison, V: Op, Depth: Depth + 1);
8219 }))
8220 return true;
8221
8222 // V = extractvalue V0, idx
8223 // V2 = extractvalue V0, idx2
8224 // V0's elements are all poison or not. (e.g., add_with_overflow)
8225 const WithOverflowInst *II;
8226 if (match(V: I, P: m_ExtractValue(V: m_WithOverflowInst(I&: II))) &&
8227 (match(V: ValAssumedPoison, P: m_ExtractValue(V: m_Specific(V: II))) ||
8228 llvm::is_contained(Range: II->args(), Element: ValAssumedPoison)))
8229 return true;
8230 }
8231 return false;
8232}
8233
8234static bool impliesPoison(const Value *ValAssumedPoison, const Value *V,
8235 unsigned Depth) {
8236 if (isGuaranteedNotToBePoison(V: ValAssumedPoison))
8237 return true;
8238
8239 if (directlyImpliesPoison(ValAssumedPoison, V, /* Depth */ 0))
8240 return true;
8241
8242 const unsigned MaxDepth = 2;
8243 if (Depth >= MaxDepth)
8244 return false;
8245
8246 const auto *I = dyn_cast<Instruction>(Val: ValAssumedPoison);
8247 if (I && !canCreatePoison(Op: cast<Operator>(Val: I))) {
8248 return all_of(Range: I->operands(), P: [=](const Value *Op) {
8249 return impliesPoison(ValAssumedPoison: Op, V, Depth: Depth + 1);
8250 });
8251 }
8252 return false;
8253}
8254
8255bool llvm::impliesPoison(const Value *ValAssumedPoison, const Value *V) {
8256 return ::impliesPoison(ValAssumedPoison, V, /* Depth */ 0);
8257}
8258
8259static bool programUndefinedIfUndefOrPoison(const Value *V, bool PoisonOnly);
8260
8261static bool isGuaranteedNotToBeUndefOrPoison(
8262 const Value *V, AssumptionCache *AC, const Instruction *CtxI,
8263 const DominatorTree *DT, unsigned Depth, UndefPoisonKind Kind) {
8264 if (Depth >= MaxAnalysisRecursionDepth)
8265 return false;
8266
8267 if (isa<MetadataAsValue>(Val: V))
8268 return false;
8269
8270 if (const auto *A = dyn_cast<Argument>(Val: V)) {
8271 if (A->hasAttribute(Kind: Attribute::NoUndef) ||
8272 A->hasAttribute(Kind: Attribute::Dereferenceable) ||
8273 A->hasAttribute(Kind: Attribute::DereferenceableOrNull))
8274 return true;
8275 }
8276
8277 if (auto *C = dyn_cast<Constant>(Val: V)) {
8278 if (isa<PoisonValue>(Val: C))
8279 return !includesPoison(Kind);
8280
8281 if (isa<UndefValue>(Val: C))
8282 return !includesUndef(Kind);
8283
8284 if (isa<ConstantInt>(Val: C) || isa<GlobalVariable>(Val: C) || isa<ConstantFP>(Val: C) ||
8285 isa<ConstantPointerNull>(Val: C) || isa<Function>(Val: C) || isa<ConstantData>(Val: C))
8286 return true;
8287
8288 if (C->getType()->isVectorTy() || C->getType()->isAggregateType()) {
8289 if (isa<ConstantExpr>(Val: C)) {
8290 // Scalable vectors can use a ConstantExpr to build a splat.
8291 if (Constant *SplatC = C->getSplatValue())
8292 if (isa<ConstantInt>(Val: SplatC) || isa<ConstantFP>(Val: SplatC))
8293 return true;
8294 } else if (auto *CA = dyn_cast<ConstantAggregate>(Val: C)) {
8295 return all_of(Range: CA->operands(), P: [&](const Use &Op) {
8296 return isGuaranteedNotToBeUndefOrPoison(V: Op.get(), AC, CtxI, DT,
8297 Depth: Depth + 1, Kind);
8298 });
8299 }
8300 }
8301 }
8302
8303 // Strip cast operations from a pointer value.
8304 // Note that stripPointerCastsSameRepresentation can strip off getelementptr
8305 // inbounds with zero offset. To guarantee that the result isn't poison, the
8306 // stripped pointer is checked as it has to be pointing into an allocated
8307 // object or be null `null` to ensure `inbounds` getelement pointers with a
8308 // zero offset could not produce poison.
8309 // It can strip off addrspacecast that do not change bit representation as
8310 // well. We believe that such addrspacecast is equivalent to no-op.
8311 auto *StrippedV = V->stripPointerCastsSameRepresentation();
8312 if (isa<AllocaInst>(Val: StrippedV) || isa<GlobalVariable>(Val: StrippedV) ||
8313 isa<Function>(Val: StrippedV) || isa<ConstantPointerNull>(Val: StrippedV))
8314 return true;
8315
8316 auto OpCheck = [&](const Value *V) {
8317 return isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth: Depth + 1, Kind);
8318 };
8319
8320 if (auto *Opr = dyn_cast<Operator>(Val: V)) {
8321 // If the value is a freeze instruction, then it can never
8322 // be undef or poison.
8323 if (isa<FreezeInst>(Val: V))
8324 return true;
8325
8326 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
8327 if (CB->hasRetAttr(Kind: Attribute::NoUndef) ||
8328 CB->hasRetAttr(Kind: Attribute::Dereferenceable) ||
8329 CB->hasRetAttr(Kind: Attribute::DereferenceableOrNull))
8330 return true;
8331 }
8332
8333 if (!::canCreateUndefOrPoison(Op: Opr, Kind,
8334 /*ConsiderFlagsAndMetadata=*/true)) {
8335 if (const auto *PN = dyn_cast<PHINode>(Val: V)) {
8336 unsigned Num = PN->getNumIncomingValues();
8337 bool IsWellDefined = true;
8338 for (unsigned i = 0; i < Num; ++i) {
8339 if (PN == PN->getIncomingValue(i))
8340 continue;
8341 auto *TI = PN->getIncomingBlock(i)->getTerminator();
8342 if (!isGuaranteedNotToBeUndefOrPoison(V: PN->getIncomingValue(i), AC, CtxI: TI,
8343 DT, Depth: Depth + 1, Kind)) {
8344 IsWellDefined = false;
8345 break;
8346 }
8347 }
8348 if (IsWellDefined)
8349 return true;
8350 } else if (auto *Splat = isa<ShuffleVectorInst>(Val: Opr) ? getSplatValue(V: Opr)
8351 : nullptr) {
8352 // For splats we only need to check the value being splatted.
8353 if (OpCheck(Splat))
8354 return true;
8355 } else if (all_of(Range: Opr->operands(), P: OpCheck))
8356 return true;
8357 }
8358 }
8359
8360 if (auto *I = dyn_cast<LoadInst>(Val: V))
8361 if (I->hasMetadata(KindID: LLVMContext::MD_noundef) ||
8362 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable) ||
8363 I->hasMetadata(KindID: LLVMContext::MD_dereferenceable_or_null))
8364 return true;
8365
8366 if (programUndefinedIfUndefOrPoison(V, PoisonOnly: !includesUndef(Kind)))
8367 return true;
8368
8369 // CtxI may be null or a cloned instruction.
8370 if (!CtxI || !CtxI->getParent() || !DT)
8371 return false;
8372
8373 auto *DNode = DT->getNode(BB: CtxI->getParent());
8374 if (!DNode)
8375 // Unreachable block
8376 return false;
8377
8378 // If V is used as a branch condition before reaching CtxI, V cannot be
8379 // undef or poison.
8380 // br V, BB1, BB2
8381 // BB1:
8382 // CtxI ; V cannot be undef or poison here
8383 auto *Dominator = DNode->getIDom();
8384 // This check is purely for compile time reasons: we can skip the IDom walk
8385 // if what we are checking for includes undef and the value is not an integer.
8386 if (!includesUndef(Kind) || V->getType()->isIntegerTy())
8387 while (Dominator) {
8388 auto *TI = Dominator->getBlock()->getTerminatorOrNull();
8389
8390 Value *Cond = nullptr;
8391 if (auto BI = dyn_cast_or_null<CondBrInst>(Val: TI)) {
8392 Cond = BI->getCondition();
8393 } else if (auto SI = dyn_cast_or_null<SwitchInst>(Val: TI)) {
8394 Cond = SI->getCondition();
8395 }
8396
8397 if (Cond) {
8398 if (Cond == V)
8399 return true;
8400 else if (!includesUndef(Kind) && isa<Operator>(Val: Cond)) {
8401 // For poison, we can analyze further
8402 auto *Opr = cast<Operator>(Val: Cond);
8403 if (any_of(Range: Opr->operands(), P: [V](const Use &U) {
8404 return V == U && propagatesPoison(PoisonOp: U);
8405 }))
8406 return true;
8407 }
8408 }
8409
8410 Dominator = Dominator->getIDom();
8411 }
8412
8413 if (AC && getKnowledgeValidInContext(V, AttrKinds: {Attribute::NoUndef}, AC&: *AC, CtxI, DT))
8414 return true;
8415
8416 return false;
8417}
8418
8419bool llvm::isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC,
8420 const Instruction *CtxI,
8421 const DominatorTree *DT,
8422 unsigned Depth) {
8423 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8424 Kind: UndefPoisonKind::UndefOrPoison);
8425}
8426
8427bool llvm::isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC,
8428 const Instruction *CtxI,
8429 const DominatorTree *DT, unsigned Depth) {
8430 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8431 Kind: UndefPoisonKind::PoisonOnly);
8432}
8433
8434bool llvm::isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC,
8435 const Instruction *CtxI,
8436 const DominatorTree *DT, unsigned Depth) {
8437 return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth,
8438 Kind: UndefPoisonKind::UndefOnly);
8439}
8440
8441/// Return true if undefined behavior would provably be executed on the path to
8442/// OnPathTo if Root produced a posion result. Note that this doesn't say
8443/// anything about whether OnPathTo is actually executed or whether Root is
8444/// actually poison. This can be used to assess whether a new use of Root can
8445/// be added at a location which is control equivalent with OnPathTo (such as
8446/// immediately before it) without introducing UB which didn't previously
8447/// exist. Note that a false result conveys no information.
8448bool llvm::mustExecuteUBIfPoisonOnPathTo(Instruction *Root,
8449 Instruction *OnPathTo,
8450 DominatorTree *DT) {
8451 // Basic approach is to assume Root is poison, propagate poison forward
8452 // through all users we can easily track, and then check whether any of those
8453 // users are provable UB and must execute before out exiting block might
8454 // exit.
8455
8456 // The set of all recursive users we've visited (which are assumed to all be
8457 // poison because of said visit)
8458 SmallPtrSet<const Value *, 16> KnownPoison;
8459 SmallVector<const Instruction*, 16> Worklist;
8460 Worklist.push_back(Elt: Root);
8461 while (!Worklist.empty()) {
8462 const Instruction *I = Worklist.pop_back_val();
8463
8464 // If we know this must trigger UB on a path leading our target.
8465 if (mustTriggerUB(I, KnownPoison) && DT->dominates(Def: I, User: OnPathTo))
8466 return true;
8467
8468 // If we can't analyze propagation through this instruction, just skip it
8469 // and transitive users. Safe as false is a conservative result.
8470 if (I != Root && !any_of(Range: I->operands(), P: [&KnownPoison](const Use &U) {
8471 return KnownPoison.contains(Ptr: U) && propagatesPoison(PoisonOp: U);
8472 }))
8473 continue;
8474
8475 if (KnownPoison.insert(Ptr: I).second)
8476 for (const User *User : I->users())
8477 Worklist.push_back(Elt: cast<Instruction>(Val: User));
8478 }
8479
8480 // Might be non-UB, or might have a path we couldn't prove must execute on
8481 // way to exiting bb.
8482 return false;
8483}
8484
8485OverflowResult llvm::computeOverflowForSignedAdd(const AddOperator *Add,
8486 const SimplifyQuery &SQ) {
8487 return ::computeOverflowForSignedAdd(LHS: Add->getOperand(i_nocapture: 0), RHS: Add->getOperand(i_nocapture: 1),
8488 Add, SQ);
8489}
8490
8491OverflowResult
8492llvm::computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
8493 const WithCache<const Value *> &RHS,
8494 const SimplifyQuery &SQ) {
8495 return ::computeOverflowForSignedAdd(LHS, RHS, Add: nullptr, SQ);
8496}
8497
8498bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
8499 // Note: An atomic operation isn't guaranteed to return in a reasonable amount
8500 // of time because it's possible for another thread to interfere with it for an
8501 // arbitrary length of time, but programs aren't allowed to rely on that.
8502
8503 // If there is no successor, then execution can't transfer to it.
8504 if (isa<ReturnInst>(Val: I))
8505 return false;
8506 if (isa<UnreachableInst>(Val: I))
8507 return false;
8508
8509 // Note: Do not add new checks here; instead, change Instruction::mayThrow or
8510 // Instruction::willReturn.
8511 //
8512 // FIXME: Move this check into Instruction::willReturn.
8513 if (isa<CatchPadInst>(Val: I)) {
8514 switch (classifyEHPersonality(Pers: I->getFunction()->getPersonalityFn())) {
8515 default:
8516 // A catchpad may invoke exception object constructors and such, which
8517 // in some languages can be arbitrary code, so be conservative by default.
8518 return false;
8519 case EHPersonality::CoreCLR:
8520 // For CoreCLR, it just involves a type test.
8521 return true;
8522 }
8523 }
8524
8525 // An instruction that returns without throwing must transfer control flow
8526 // to a successor.
8527 return !I->mayThrow() && I->willReturn();
8528}
8529
8530bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) {
8531 // TODO: This is slightly conservative for invoke instruction since exiting
8532 // via an exception *is* normal control for them.
8533 for (const Instruction &I : *BB)
8534 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8535 return false;
8536 return true;
8537}
8538
8539bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8540 BasicBlock::const_iterator Begin, BasicBlock::const_iterator End,
8541 unsigned ScanLimit) {
8542 return isGuaranteedToTransferExecutionToSuccessor(Range: make_range(x: Begin, y: End),
8543 ScanLimit);
8544}
8545
8546bool llvm::isGuaranteedToTransferExecutionToSuccessor(
8547 iterator_range<BasicBlock::const_iterator> Range, unsigned ScanLimit) {
8548 assert(ScanLimit && "scan limit must be non-zero");
8549 for (const Instruction &I : Range) {
8550 if (--ScanLimit == 0)
8551 return false;
8552 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8553 return false;
8554 }
8555 return true;
8556}
8557
8558bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
8559 const Loop *L) {
8560 // The loop header is guaranteed to be executed for every iteration.
8561 //
8562 // FIXME: Relax this constraint to cover all basic blocks that are
8563 // guaranteed to be executed at every iteration.
8564 if (I->getParent() != L->getHeader()) return false;
8565
8566 for (const Instruction &LI : *L->getHeader()) {
8567 if (&LI == I) return true;
8568 if (!isGuaranteedToTransferExecutionToSuccessor(I: &LI)) return false;
8569 }
8570 llvm_unreachable("Instruction not contained in its own parent basic block.");
8571}
8572
8573bool llvm::intrinsicPropagatesPoison(Intrinsic::ID IID) {
8574 switch (IID) {
8575 // TODO: Add more intrinsics.
8576 case Intrinsic::sadd_with_overflow:
8577 case Intrinsic::ssub_with_overflow:
8578 case Intrinsic::smul_with_overflow:
8579 case Intrinsic::uadd_with_overflow:
8580 case Intrinsic::usub_with_overflow:
8581 case Intrinsic::umul_with_overflow:
8582 // If an input is a vector containing a poison element, the
8583 // two output vectors (calculated results, overflow bits)'
8584 // corresponding lanes are poison.
8585 return true;
8586 case Intrinsic::ctpop:
8587 case Intrinsic::ctlz:
8588 case Intrinsic::cttz:
8589 case Intrinsic::abs:
8590 case Intrinsic::smax:
8591 case Intrinsic::smin:
8592 case Intrinsic::umax:
8593 case Intrinsic::umin:
8594 case Intrinsic::scmp:
8595 case Intrinsic::smulh:
8596 case Intrinsic::umulh:
8597 case Intrinsic::is_fpclass:
8598 case Intrinsic::ptrmask:
8599 case Intrinsic::ucmp:
8600 case Intrinsic::bitreverse:
8601 case Intrinsic::bswap:
8602 case Intrinsic::sadd_sat:
8603 case Intrinsic::ssub_sat:
8604 case Intrinsic::sshl_sat:
8605 case Intrinsic::uadd_sat:
8606 case Intrinsic::usub_sat:
8607 case Intrinsic::ushl_sat:
8608 case Intrinsic::smul_fix:
8609 case Intrinsic::smul_fix_sat:
8610 case Intrinsic::umul_fix:
8611 case Intrinsic::umul_fix_sat:
8612 case Intrinsic::pow:
8613 case Intrinsic::powi:
8614 case Intrinsic::sin:
8615 case Intrinsic::sinh:
8616 case Intrinsic::cos:
8617 case Intrinsic::cosh:
8618 case Intrinsic::sincos:
8619 case Intrinsic::sincospi:
8620 case Intrinsic::tan:
8621 case Intrinsic::tanh:
8622 case Intrinsic::asin:
8623 case Intrinsic::acos:
8624 case Intrinsic::atan:
8625 case Intrinsic::atan2:
8626 case Intrinsic::canonicalize:
8627 case Intrinsic::sqrt:
8628 case Intrinsic::fma:
8629 case Intrinsic::fmuladd:
8630 case Intrinsic::exp:
8631 case Intrinsic::exp2:
8632 case Intrinsic::exp10:
8633 case Intrinsic::log:
8634 case Intrinsic::log2:
8635 case Intrinsic::log10:
8636 case Intrinsic::modf:
8637 case Intrinsic::floor:
8638 case Intrinsic::ceil:
8639 case Intrinsic::trunc:
8640 case Intrinsic::rint:
8641 case Intrinsic::nearbyint:
8642 case Intrinsic::round:
8643 case Intrinsic::roundeven:
8644 case Intrinsic::lrint:
8645 case Intrinsic::llrint:
8646 case Intrinsic::fshl:
8647 case Intrinsic::fshr:
8648 case Intrinsic::frexp:
8649 case Intrinsic::get_active_lane_mask:
8650 return true;
8651 default:
8652 return false;
8653 }
8654}
8655
8656bool llvm::propagatesPoison(const Use &PoisonOp) {
8657 const Operator *I = cast<Operator>(Val: PoisonOp.getUser());
8658 switch (I->getOpcode()) {
8659 case Instruction::Freeze:
8660 case Instruction::PHI:
8661 case Instruction::Invoke:
8662 return false;
8663 case Instruction::Select:
8664 return PoisonOp.getOperandNo() == 0;
8665 case Instruction::Call:
8666 if (auto *II = dyn_cast<IntrinsicInst>(Val: I))
8667 return intrinsicPropagatesPoison(IID: II->getIntrinsicID());
8668 return false;
8669 case Instruction::ICmp:
8670 case Instruction::FCmp:
8671 case Instruction::GetElementPtr:
8672 return true;
8673 default:
8674 if (isa<BinaryOperator>(Val: I) || isa<UnaryOperator>(Val: I) || isa<CastInst>(Val: I))
8675 return true;
8676
8677 // Be conservative and return false.
8678 return false;
8679 }
8680}
8681
8682/// Enumerates all operands of \p I that are guaranteed to not be undef or
8683/// poison. If the callback \p Handle returns true, stop processing and return
8684/// true. Otherwise, return false.
8685template <typename CallableT>
8686static bool handleGuaranteedWellDefinedOps(const Instruction *I,
8687 const CallableT &Handle) {
8688 switch (I->getOpcode()) {
8689 case Instruction::Store:
8690 if (Handle(cast<StoreInst>(Val: I)->getPointerOperand()))
8691 return true;
8692 break;
8693
8694 case Instruction::Load:
8695 if (Handle(cast<LoadInst>(Val: I)->getPointerOperand()))
8696 return true;
8697 break;
8698
8699 // Since dereferenceable attribute imply noundef, atomic operations
8700 // also implicitly have noundef pointers too
8701 case Instruction::AtomicCmpXchg:
8702 if (Handle(cast<AtomicCmpXchgInst>(Val: I)->getPointerOperand()))
8703 return true;
8704 break;
8705
8706 case Instruction::AtomicRMW:
8707 if (Handle(cast<AtomicRMWInst>(Val: I)->getPointerOperand()))
8708 return true;
8709 break;
8710
8711 case Instruction::Call:
8712 case Instruction::Invoke: {
8713 const CallBase *CB = cast<CallBase>(Val: I);
8714 if (CB->isIndirectCall() && Handle(CB->getCalledOperand()))
8715 return true;
8716 for (unsigned i = 0; i < CB->arg_size(); ++i)
8717 if ((CB->paramHasAttr(ArgNo: i, Kind: Attribute::NoUndef) ||
8718 CB->paramHasAttr(ArgNo: i, Kind: Attribute::Dereferenceable) ||
8719 CB->paramHasAttr(ArgNo: i, Kind: Attribute::DereferenceableOrNull)) &&
8720 Handle(CB->getArgOperand(i)))
8721 return true;
8722 break;
8723 }
8724 case Instruction::Ret:
8725 if (I->getFunction()->hasRetAttribute(Kind: Attribute::NoUndef) &&
8726 Handle(I->getOperand(i: 0)))
8727 return true;
8728 break;
8729 case Instruction::Switch:
8730 if (Handle(cast<SwitchInst>(Val: I)->getCondition()))
8731 return true;
8732 break;
8733 case Instruction::CondBr:
8734 if (Handle(cast<CondBrInst>(Val: I)->getCondition()))
8735 return true;
8736 break;
8737 default:
8738 break;
8739 }
8740
8741 return false;
8742}
8743
8744/// Enumerates all operands of \p I that are guaranteed to not be poison.
8745template <typename CallableT>
8746static bool handleGuaranteedNonPoisonOps(const Instruction *I,
8747 const CallableT &Handle) {
8748 if (handleGuaranteedWellDefinedOps(I, Handle))
8749 return true;
8750 switch (I->getOpcode()) {
8751 // Divisors of these operations are allowed to be partially undef.
8752 case Instruction::UDiv:
8753 case Instruction::SDiv:
8754 case Instruction::URem:
8755 case Instruction::SRem:
8756 return Handle(I->getOperand(i: 1));
8757 default:
8758 return false;
8759 }
8760}
8761
8762bool llvm::mustTriggerUB(const Instruction *I,
8763 const SmallPtrSetImpl<const Value *> &KnownPoison) {
8764 return handleGuaranteedNonPoisonOps(
8765 I, Handle: [&](const Value *V) { return KnownPoison.count(Ptr: V); });
8766}
8767
8768static bool programUndefinedIfUndefOrPoison(const Value *V,
8769 bool PoisonOnly) {
8770 // We currently only look for uses of values within the same basic
8771 // block, as that makes it easier to guarantee that the uses will be
8772 // executed given that Inst is executed.
8773 //
8774 // FIXME: Expand this to consider uses beyond the same basic block. To do
8775 // this, look out for the distinction between post-dominance and strong
8776 // post-dominance.
8777 const BasicBlock *BB = nullptr;
8778 BasicBlock::const_iterator Begin;
8779 if (const auto *Inst = dyn_cast<Instruction>(Val: V)) {
8780 BB = Inst->getParent();
8781 Begin = Inst->getIterator();
8782 Begin++;
8783 } else if (const auto *Arg = dyn_cast<Argument>(Val: V)) {
8784 if (Arg->getParent()->isDeclaration())
8785 return false;
8786 BB = &Arg->getParent()->getEntryBlock();
8787 Begin = BB->begin();
8788 } else {
8789 return false;
8790 }
8791
8792 // Limit number of instructions we look at, to avoid scanning through large
8793 // blocks. The current limit is chosen arbitrarily.
8794 unsigned ScanLimit = 32;
8795 BasicBlock::const_iterator End = BB->end();
8796
8797 if (!PoisonOnly) {
8798 // Since undef does not propagate eagerly, be conservative & just check
8799 // whether a value is directly passed to an instruction that must take
8800 // well-defined operands.
8801
8802 for (const auto &I : make_range(x: Begin, y: End)) {
8803 if (--ScanLimit == 0)
8804 break;
8805
8806 if (handleGuaranteedWellDefinedOps(I: &I, Handle: [V](const Value *WellDefinedOp) {
8807 return WellDefinedOp == V;
8808 }))
8809 return true;
8810
8811 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8812 break;
8813 }
8814 return false;
8815 }
8816
8817 // Set of instructions that we have proved will yield poison if Inst
8818 // does.
8819 SmallPtrSet<const Value *, 16> YieldsPoison;
8820 SmallPtrSet<const BasicBlock *, 4> Visited;
8821
8822 YieldsPoison.insert(Ptr: V);
8823 Visited.insert(Ptr: BB);
8824
8825 while (true) {
8826 for (const auto &I : make_range(x: Begin, y: End)) {
8827 if (--ScanLimit == 0)
8828 return false;
8829 if (mustTriggerUB(I: &I, KnownPoison: YieldsPoison))
8830 return true;
8831 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
8832 return false;
8833
8834 // If an operand is poison and propagates it, mark I as yielding poison.
8835 for (const Use &Op : I.operands()) {
8836 if (YieldsPoison.count(Ptr: Op) && propagatesPoison(PoisonOp: Op)) {
8837 YieldsPoison.insert(Ptr: &I);
8838 break;
8839 }
8840 }
8841
8842 // Special handling for select, which returns poison if its operand 0 is
8843 // poison (handled in the loop above) *or* if both its true/false operands
8844 // are poison (handled here).
8845 if (I.getOpcode() == Instruction::Select &&
8846 YieldsPoison.count(Ptr: I.getOperand(i: 1)) &&
8847 YieldsPoison.count(Ptr: I.getOperand(i: 2))) {
8848 YieldsPoison.insert(Ptr: &I);
8849 }
8850 }
8851
8852 BB = BB->getSingleSuccessor();
8853 if (!BB || !Visited.insert(Ptr: BB).second)
8854 break;
8855
8856 Begin = BB->getFirstNonPHIIt();
8857 End = BB->end();
8858 }
8859 return false;
8860}
8861
8862bool llvm::programUndefinedIfUndefOrPoison(const Instruction *Inst) {
8863 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: false);
8864}
8865
8866bool llvm::programUndefinedIfPoison(const Instruction *Inst) {
8867 return ::programUndefinedIfUndefOrPoison(V: Inst, PoisonOnly: true);
8868}
8869
8870static bool isKnownNonNaN(const Value *V, FastMathFlags FMF) {
8871 if (FMF.noNaNs())
8872 return true;
8873
8874 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8875 return !C->isNaN();
8876
8877 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8878 if (!C->getElementType()->isFloatingPointTy())
8879 return false;
8880 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8881 if (C->getElementAsAPFloat(i: I).isNaN())
8882 return false;
8883 }
8884 return true;
8885 }
8886
8887 if (isa<ConstantAggregateZero>(Val: V))
8888 return true;
8889
8890 return false;
8891}
8892
8893static bool isKnownNonZero(const Value *V) {
8894 if (auto *C = dyn_cast<ConstantFP>(Val: V))
8895 return !C->isZero();
8896
8897 if (auto *C = dyn_cast<ConstantDataVector>(Val: V)) {
8898 if (!C->getElementType()->isFloatingPointTy())
8899 return false;
8900 for (unsigned I = 0, E = C->getNumElements(); I < E; ++I) {
8901 if (C->getElementAsAPFloat(i: I).isZero())
8902 return false;
8903 }
8904 return true;
8905 }
8906
8907 return false;
8908}
8909
8910/// Match clamp pattern for float types without care about NaNs or signed zeros.
8911/// Given non-min/max outer cmp/select from the clamp pattern this
8912/// function recognizes if it can be substitued by a "canonical" min/max
8913/// pattern.
8914static SelectPatternResult matchFastFloatClamp(CmpInst::Predicate Pred,
8915 Value *CmpLHS, Value *CmpRHS,
8916 Value *TrueVal, Value *FalseVal,
8917 Value *&LHS, Value *&RHS) {
8918 // Try to match
8919 // X < C1 ? C1 : Min(X, C2) --> Max(C1, Min(X, C2))
8920 // X > C1 ? C1 : Max(X, C2) --> Min(C1, Max(X, C2))
8921 // and return description of the outer Max/Min.
8922
8923 // First, check if select has inverse order:
8924 if (CmpRHS == FalseVal) {
8925 std::swap(a&: TrueVal, b&: FalseVal);
8926 Pred = CmpInst::getInversePredicate(pred: Pred);
8927 }
8928
8929 // Assume success now. If there's no match, callers should not use these anyway.
8930 LHS = TrueVal;
8931 RHS = FalseVal;
8932
8933 const APFloat *FC1;
8934 if (CmpRHS != TrueVal || !match(V: CmpRHS, P: m_APFloat(Res&: FC1)) || !FC1->isFinite())
8935 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8936
8937 const APFloat *FC2;
8938 switch (Pred) {
8939 case CmpInst::FCMP_OLT:
8940 case CmpInst::FCMP_OLE:
8941 case CmpInst::FCMP_ULT:
8942 case CmpInst::FCMP_ULE:
8943 if (match(V: FalseVal, P: m_OrdOrUnordFMin(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8944 *FC1 < *FC2)
8945 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8946 if (match(V: FalseVal, P: m_FMinNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
8947 *FC1 < *FC2)
8948 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8949 break;
8950 case CmpInst::FCMP_OGT:
8951 case CmpInst::FCMP_OGE:
8952 case CmpInst::FCMP_UGT:
8953 case CmpInst::FCMP_UGE:
8954 if (match(V: FalseVal, P: m_OrdOrUnordFMax(L: m_Specific(V: CmpLHS), R: m_APFloat(Res&: FC2))) &&
8955 *FC1 > *FC2)
8956 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8957 if (match(V: FalseVal, P: m_FMaxNum(Op0: m_Specific(V: CmpLHS), Op1: m_APFloat(Res&: FC2))) &&
8958 *FC1 > *FC2)
8959 return {.Flavor: SPF_FMINNUM, .NaNBehavior: SPNB_RETURNS_ANY, .Ordered: false};
8960 break;
8961 default:
8962 break;
8963 }
8964
8965 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
8966}
8967
8968/// Recognize variations of:
8969/// CLAMP(v,l,h) ==> ((v) < (l) ? (l) : ((v) > (h) ? (h) : (v)))
8970static SelectPatternResult matchClamp(CmpInst::Predicate Pred,
8971 Value *CmpLHS, Value *CmpRHS,
8972 Value *TrueVal, Value *FalseVal) {
8973 // Swap the select operands and predicate to match the patterns below.
8974 if (CmpRHS != TrueVal) {
8975 Pred = ICmpInst::getSwappedPredicate(pred: Pred);
8976 std::swap(a&: TrueVal, b&: FalseVal);
8977 }
8978 const APInt *C1;
8979 if (CmpRHS == TrueVal && match(V: CmpRHS, P: m_APInt(Res&: C1))) {
8980 const APInt *C2;
8981 // (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
8982 if (match(V: FalseVal, P: m_SMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8983 C1->slt(RHS: *C2) && Pred == CmpInst::ICMP_SLT)
8984 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8985
8986 // (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
8987 if (match(V: FalseVal, P: m_SMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8988 C1->sgt(RHS: *C2) && Pred == CmpInst::ICMP_SGT)
8989 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
8990
8991 // (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
8992 if (match(V: FalseVal, P: m_UMin(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8993 C1->ult(RHS: *C2) && Pred == CmpInst::ICMP_ULT)
8994 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
8995
8996 // (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
8997 if (match(V: FalseVal, P: m_UMax(Op0: m_Specific(V: CmpLHS), Op1: m_APInt(Res&: C2))) &&
8998 C1->ugt(RHS: *C2) && Pred == CmpInst::ICMP_UGT)
8999 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9000 }
9001 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9002}
9003
9004/// If the input value is the result of a 'not' op, constant integer, or vector
9005/// splat of a constant integer, return the bitwise-not source value.
9006/// TODO: This could be extended to handle non-splat vector integer constants.
9007static Value *getNotValue(Value *V) {
9008 Value *NotV;
9009 if (match(V, P: m_Not(V: m_Value(V&: NotV))))
9010 return NotV;
9011
9012 const APInt *C;
9013 if (match(V, P: m_APInt(Res&: C)))
9014 return ConstantInt::get(Ty: V->getType(), V: ~(*C));
9015
9016 return nullptr;
9017}
9018
9019/// Match non-obvious integer minimum and maximum sequences.
9020static SelectPatternResult matchMinMax(CmpInst::Predicate Pred, Value *CmpLHS,
9021 Value *CmpRHS, Value *TrueVal,
9022 Value *FalseVal, Value *&LHS,
9023 Value *&RHS) {
9024 // Assume success. If there's no match, callers should not use these anyway.
9025 LHS = TrueVal;
9026 RHS = FalseVal;
9027
9028 SelectPatternResult SPR = matchClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal);
9029 if (SPR.Flavor != SelectPatternFlavor::SPF_UNKNOWN)
9030 return SPR;
9031
9032 // Look through 'not' ops to find disguised min/max.
9033 // (X > Y) ? ~X : ~Y ==> (~X < ~Y) ? ~X : ~Y ==> MIN(~X, ~Y)
9034 // (X < Y) ? ~X : ~Y ==> (~X > ~Y) ? ~X : ~Y ==> MAX(~X, ~Y)
9035 if (CmpLHS == getNotValue(V: TrueVal) && CmpRHS == getNotValue(V: FalseVal)) {
9036 switch (Pred) {
9037 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9038 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9039 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9040 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9041 default: break;
9042 }
9043 }
9044
9045 // (X > Y) ? ~Y : ~X ==> (~X < ~Y) ? ~Y : ~X ==> MAX(~Y, ~X)
9046 // (X < Y) ? ~Y : ~X ==> (~X > ~Y) ? ~Y : ~X ==> MIN(~Y, ~X)
9047 if (CmpLHS == getNotValue(V: FalseVal) && CmpRHS == getNotValue(V: TrueVal)) {
9048 switch (Pred) {
9049 case CmpInst::ICMP_SGT: return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9050 case CmpInst::ICMP_SLT: return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9051 case CmpInst::ICMP_UGT: return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9052 case CmpInst::ICMP_ULT: return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9053 default: break;
9054 }
9055 }
9056
9057 if (Pred != CmpInst::ICMP_SGT && Pred != CmpInst::ICMP_SLT)
9058 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9059
9060 const APInt *C1;
9061 if (!match(V: CmpRHS, P: m_APInt(Res&: C1)))
9062 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9063
9064 // An unsigned min/max can be written with a signed compare.
9065 const APInt *C2;
9066 if ((CmpLHS == TrueVal && match(V: FalseVal, P: m_APInt(Res&: C2))) ||
9067 (CmpLHS == FalseVal && match(V: TrueVal, P: m_APInt(Res&: C2)))) {
9068 // Is the sign bit set?
9069 // (X <s 0) ? X : MAXVAL ==> (X >u MAXVAL) ? X : MAXVAL ==> UMAX
9070 // (X <s 0) ? MAXVAL : X ==> (X >u MAXVAL) ? MAXVAL : X ==> UMIN
9071 if (Pred == CmpInst::ICMP_SLT && C1->isZero() && C2->isMaxSignedValue())
9072 return {.Flavor: CmpLHS == TrueVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9073
9074 // Is the sign bit clear?
9075 // (X >s -1) ? MINVAL : X ==> (X <u MINVAL) ? MINVAL : X ==> UMAX
9076 // (X >s -1) ? X : MINVAL ==> (X <u MINVAL) ? X : MINVAL ==> UMIN
9077 if (Pred == CmpInst::ICMP_SGT && C1->isAllOnes() && C2->isMinSignedValue())
9078 return {.Flavor: CmpLHS == FalseVal ? SPF_UMAX : SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9079 }
9080
9081 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9082}
9083
9084bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW,
9085 bool AllowPoison) {
9086 assert(X && Y && "Invalid operand");
9087
9088 auto IsNegationOf = [&](const Value *X, const Value *Y) {
9089 if (!match(V: X, P: m_Neg(V: m_Specific(V: Y))))
9090 return false;
9091
9092 auto *BO = cast<BinaryOperator>(Val: X);
9093 if (NeedNSW && !BO->hasNoSignedWrap())
9094 return false;
9095
9096 auto *Zero = cast<Constant>(Val: BO->getOperand(i_nocapture: 0));
9097 if (!AllowPoison && !Zero->isNullValue())
9098 return false;
9099
9100 return true;
9101 };
9102
9103 // X = -Y or Y = -X
9104 if (IsNegationOf(X, Y) || IsNegationOf(Y, X))
9105 return true;
9106
9107 // X = sub (A, B), Y = sub (B, A) || X = sub nsw (A, B), Y = sub nsw (B, A)
9108 Value *A, *B;
9109 return (!NeedNSW && (match(V: X, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9110 match(V: Y, P: m_Sub(L: m_Specific(V: B), R: m_Specific(V: A))))) ||
9111 (NeedNSW && (match(V: X, P: m_NSWSub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
9112 match(V: Y, P: m_NSWSub(L: m_Specific(V: B), R: m_Specific(V: A)))));
9113}
9114
9115bool llvm::isKnownInversion(const Value *X, const Value *Y) {
9116 // Handle X = icmp pred A, B, Y = icmp pred A, C.
9117 Value *A, *B, *C;
9118 CmpPredicate Pred1, Pred2;
9119 if (!match(V: X, P: m_ICmp(Pred&: Pred1, L: m_Value(V&: A), R: m_Value(V&: B))) ||
9120 !match(V: Y, P: m_c_ICmp(Pred&: Pred2, L: m_Specific(V: A), R: m_Value(V&: C))))
9121 return false;
9122
9123 // They must both have samesign flag or not.
9124 if (Pred1.hasSameSign() != Pred2.hasSameSign())
9125 return false;
9126
9127 if (B == C)
9128 return Pred1 == ICmpInst::getInversePredicate(pred: Pred2);
9129
9130 // Try to infer the relationship from constant ranges.
9131 const APInt *RHSC1, *RHSC2;
9132 if (!match(V: B, P: m_APInt(Res&: RHSC1)) || !match(V: C, P: m_APInt(Res&: RHSC2)))
9133 return false;
9134
9135 // Sign bits of two RHSCs should match.
9136 if (Pred1.hasSameSign() && RHSC1->isNonNegative() != RHSC2->isNonNegative())
9137 return false;
9138
9139 const auto CR1 = ConstantRange::makeExactICmpRegion(Pred: Pred1, Other: *RHSC1);
9140 const auto CR2 = ConstantRange::makeExactICmpRegion(Pred: Pred2, Other: *RHSC2);
9141
9142 return CR1.inverse() == CR2;
9143}
9144
9145SelectPatternResult llvm::getSelectPattern(CmpInst::Predicate Pred,
9146 SelectPatternNaNBehavior NaNBehavior,
9147 bool Ordered) {
9148 switch (Pred) {
9149 default:
9150 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false}; // Equality.
9151 case ICmpInst::ICMP_UGT:
9152 case ICmpInst::ICMP_UGE:
9153 return {.Flavor: SPF_UMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9154 case ICmpInst::ICMP_SGT:
9155 case ICmpInst::ICMP_SGE:
9156 return {.Flavor: SPF_SMAX, .NaNBehavior: SPNB_NA, .Ordered: false};
9157 case ICmpInst::ICMP_ULT:
9158 case ICmpInst::ICMP_ULE:
9159 return {.Flavor: SPF_UMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9160 case ICmpInst::ICMP_SLT:
9161 case ICmpInst::ICMP_SLE:
9162 return {.Flavor: SPF_SMIN, .NaNBehavior: SPNB_NA, .Ordered: false};
9163 case FCmpInst::FCMP_UGT:
9164 case FCmpInst::FCMP_UGE:
9165 case FCmpInst::FCMP_OGT:
9166 case FCmpInst::FCMP_OGE:
9167 return {.Flavor: SPF_FMAXNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
9168 case FCmpInst::FCMP_ULT:
9169 case FCmpInst::FCMP_ULE:
9170 case FCmpInst::FCMP_OLT:
9171 case FCmpInst::FCMP_OLE:
9172 return {.Flavor: SPF_FMINNUM, .NaNBehavior: NaNBehavior, .Ordered: Ordered};
9173 }
9174}
9175
9176std::optional<std::pair<CmpPredicate, Constant *>>
9177llvm::getFlippedStrictnessPredicateAndConstant(CmpPredicate Pred, Constant *C) {
9178 assert(ICmpInst::isRelational(Pred) && ICmpInst::isIntPredicate(Pred) &&
9179 "Only for relational integer predicates.");
9180 if (isa<UndefValue>(Val: C))
9181 return std::nullopt;
9182
9183 Type *Type = C->getType();
9184 bool IsSigned = ICmpInst::isSigned(Pred);
9185
9186 CmpInst::Predicate UnsignedPred = ICmpInst::getUnsignedPredicate(Pred);
9187 bool WillIncrement =
9188 UnsignedPred == ICmpInst::ICMP_ULE || UnsignedPred == ICmpInst::ICMP_UGT;
9189
9190 // Check if the constant operand can be safely incremented/decremented
9191 // without overflowing/underflowing.
9192 auto ConstantIsOk = [Pred, WillIncrement, IsSigned](ConstantInt *C) {
9193 if (WillIncrement ? C->isMaxValue(IsSigned) : C->isMinValue(IsSigned))
9194 return false;
9195
9196 if (!Pred.hasSameSign())
9197 return true;
9198
9199 // Crossing the corresponding boundary in the other ordering changes the
9200 // sign bit, and therefore changes the poison domain.
9201 return WillIncrement ? !C->isMaxValue(IsSigned: !IsSigned)
9202 : !C->isMinValue(IsSigned: !IsSigned);
9203 };
9204
9205 Constant *SafeReplacementConstant = nullptr;
9206 if (auto *CI = dyn_cast<ConstantInt>(Val: C)) {
9207 // Bail out if the constant can't be safely incremented/decremented.
9208 if (!ConstantIsOk(CI))
9209 return std::nullopt;
9210 } else if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Type)) {
9211 unsigned NumElts = FVTy->getNumElements();
9212 for (unsigned i = 0; i != NumElts; ++i) {
9213 Constant *Elt = C->getAggregateElement(Elt: i);
9214 if (!Elt)
9215 return std::nullopt;
9216
9217 if (isa<UndefValue>(Val: Elt))
9218 continue;
9219
9220 // Bail out if we can't determine if this constant is min/max or if we
9221 // know that this constant is min/max.
9222 auto *CI = dyn_cast<ConstantInt>(Val: Elt);
9223 if (!CI || !ConstantIsOk(CI))
9224 return std::nullopt;
9225
9226 if (!SafeReplacementConstant)
9227 SafeReplacementConstant = CI;
9228 }
9229 } else if (isa<VectorType>(Val: C->getType())) {
9230 // Handle scalable splat
9231 Value *SplatC = C->getSplatValue();
9232 auto *CI = dyn_cast_or_null<ConstantInt>(Val: SplatC);
9233 // Bail out if the constant can't be safely incremented/decremented.
9234 if (!CI || !ConstantIsOk(CI))
9235 return std::nullopt;
9236 } else {
9237 // ConstantExpr?
9238 return std::nullopt;
9239 }
9240
9241 // It may not be safe to change a compare predicate in the presence of
9242 // undefined elements, so replace those elements with the first safe constant
9243 // that we found.
9244 // TODO: in case of poison, it is safe; let's replace undefs only.
9245 if (C->containsUndefOrPoisonElement()) {
9246 assert(SafeReplacementConstant && "Replacement constant not set");
9247 C = Constant::replaceUndefsWith(C, Replacement: SafeReplacementConstant);
9248 }
9249
9250 CmpPredicate NewPred(CmpInst::getFlippedStrictnessPredicate(pred: Pred),
9251 Pred.hasSameSign());
9252
9253 // Increment or decrement the constant.
9254 Constant *OneOrNegOne = ConstantInt::get(Ty: Type, V: WillIncrement ? 1 : -1, IsSigned: true);
9255 Constant *NewC = ConstantExpr::getAdd(C1: C, C2: OneOrNegOne);
9256
9257 return std::make_pair(x&: NewPred, y&: NewC);
9258}
9259
9260static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred,
9261 FastMathFlags FMF, Value *CmpLHS,
9262 Value *CmpRHS, Value *TrueVal,
9263 Value *FalseVal, Value *&LHS,
9264 Value *&RHS) {
9265 if (CmpInst::isFPPredicate(P: Pred)) {
9266 // IEEE-754 ignores the sign of 0.0 in comparisons. So if the select has one
9267 // 0.0 operand, set the compare's 0.0 operands to that same value for the
9268 // purpose of identifying min/max. Disregard vector constants with undefined
9269 // elements because those can not be back-propagated for analysis.
9270 Value *OutputZeroVal = nullptr;
9271 if (match(V: TrueVal, P: m_AnyZeroFP()) && !match(V: FalseVal, P: m_AnyZeroFP()) &&
9272 !cast<Constant>(Val: TrueVal)->containsUndefOrPoisonElement())
9273 OutputZeroVal = TrueVal;
9274 else if (match(V: FalseVal, P: m_AnyZeroFP()) && !match(V: TrueVal, P: m_AnyZeroFP()) &&
9275 !cast<Constant>(Val: FalseVal)->containsUndefOrPoisonElement())
9276 OutputZeroVal = FalseVal;
9277
9278 if (OutputZeroVal) {
9279 if (match(V: CmpLHS, P: m_AnyZeroFP()) && CmpLHS != OutputZeroVal)
9280 CmpLHS = OutputZeroVal;
9281 if (match(V: CmpRHS, P: m_AnyZeroFP()) && CmpRHS != OutputZeroVal)
9282 CmpRHS = OutputZeroVal;
9283 }
9284 }
9285
9286 LHS = CmpLHS;
9287 RHS = CmpRHS;
9288
9289 // Signed zero may return inconsistent results between implementations.
9290 // (0.0 <= -0.0) ? 0.0 : -0.0 // Returns 0.0
9291 // minNum(0.0, -0.0) // May return -0.0 or 0.0 (IEEE 754-2008 5.3.1)
9292 // Therefore, we behave conservatively and only proceed if at least one of the
9293 // operands is known to not be zero or if we don't care about signed zero.
9294 if (CmpInst::isFPPredicate(P: Pred)) {
9295 if (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9296 !isKnownNonZero(V: CmpRHS))
9297 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9298 }
9299
9300 SelectPatternNaNBehavior NaNBehavior = SPNB_NA;
9301 bool Ordered = false;
9302
9303 // When given one NaN and one non-NaN input:
9304 // - maxnum/minnum (C99 fmaxf()/fminf()) return the non-NaN input.
9305 // - A simple C99 (a < b ? a : b) construction will return 'b' (as the
9306 // ordered comparison fails), which could be NaN or non-NaN.
9307 // so here we discover exactly what NaN behavior is required/accepted.
9308 if (CmpInst::isFPPredicate(P: Pred)) {
9309 bool LHSSafe = isKnownNonNaN(V: CmpLHS, FMF);
9310 bool RHSSafe = isKnownNonNaN(V: CmpRHS, FMF);
9311
9312 if (LHSSafe && RHSSafe) {
9313 // Both operands are known non-NaN.
9314 NaNBehavior = SPNB_RETURNS_ANY;
9315 Ordered = CmpInst::isOrdered(predicate: Pred);
9316 } else if (CmpInst::isOrdered(predicate: Pred)) {
9317 // An ordered comparison will return false when given a NaN, so it
9318 // returns the RHS.
9319 Ordered = true;
9320 if (LHSSafe)
9321 // LHS is non-NaN, so if RHS is NaN then NaN will be returned.
9322 NaNBehavior = SPNB_RETURNS_NAN;
9323 else if (RHSSafe)
9324 NaNBehavior = SPNB_RETURNS_OTHER;
9325 else
9326 // Completely unsafe.
9327 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9328 } else {
9329 Ordered = false;
9330 // An unordered comparison will return true when given a NaN, so it
9331 // returns the LHS.
9332 if (LHSSafe)
9333 // LHS is non-NaN, so if RHS is NaN then non-NaN will be returned.
9334 NaNBehavior = SPNB_RETURNS_OTHER;
9335 else if (RHSSafe)
9336 NaNBehavior = SPNB_RETURNS_NAN;
9337 else
9338 // Completely unsafe.
9339 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9340 }
9341 }
9342
9343 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
9344 std::swap(a&: CmpLHS, b&: CmpRHS);
9345 Pred = CmpInst::getSwappedPredicate(pred: Pred);
9346 if (NaNBehavior == SPNB_RETURNS_NAN)
9347 NaNBehavior = SPNB_RETURNS_OTHER;
9348 else if (NaNBehavior == SPNB_RETURNS_OTHER)
9349 NaNBehavior = SPNB_RETURNS_NAN;
9350 Ordered = !Ordered;
9351 }
9352
9353 // ([if]cmp X, Y) ? X : Y
9354 if (TrueVal == CmpLHS && FalseVal == CmpRHS)
9355 return getSelectPattern(Pred, NaNBehavior, Ordered);
9356
9357 if (isKnownNegation(X: TrueVal, Y: FalseVal)) {
9358 // Sign-extending LHS does not change its sign, so TrueVal/FalseVal can
9359 // match against either LHS or sign-preserving operations on LHS, like
9360 // sext(LHS), or binary ops that do not wrap in signed sense.
9361 auto CmpLHSOrSExt =
9362 m_CombineOr(Ps: m_Specific(V: CmpLHS), Ps: m_SExt(Op: m_Specific(V: CmpLHS)));
9363 auto MaybeSExtOrMulCmpLHS =
9364 m_CombineOr(Ps: CmpLHSOrSExt, Ps: m_NSWMul(L: CmpLHSOrSExt, R: m_StrictlyPositive()),
9365 Ps: m_NSWShl(L: CmpLHSOrSExt, R: m_Value()));
9366 auto ZeroOrAllOnes = m_CombineOr(Ps: m_ZeroInt(), Ps: m_AllOnes());
9367 auto ZeroOrOne = m_CombineOr(Ps: m_ZeroInt(), Ps: m_One());
9368 if (match(V: TrueVal, P: MaybeSExtOrMulCmpLHS)) {
9369 // Set the return values. If the compare uses the negated value (-X >s 0),
9370 // swap the return values because the negated value is always 'RHS'.
9371 LHS = TrueVal;
9372 RHS = FalseVal;
9373 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: FalseVal))))
9374 std::swap(a&: LHS, b&: RHS);
9375
9376 // (X >s 0) ? X : -X or (X >s -1) ? X : -X --> ABS(X)
9377 // (-X >s 0) ? -X : X or (-X >s -1) ? -X : X --> ABS(X)
9378 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9379 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9380
9381 // (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X)
9382 if (Pred == ICmpInst::ICMP_SGE && match(V: CmpRHS, P: ZeroOrOne))
9383 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9384
9385 // (X <s 0) ? X : -X or (X <s 1) ? X : -X --> NABS(X)
9386 // (-X <s 0) ? -X : X or (-X <s 1) ? -X : X --> NABS(X)
9387 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9388 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9389 } else if (match(V: FalseVal, P: MaybeSExtOrMulCmpLHS)) {
9390 // Set the return values. If the compare uses the negated value (-X >s 0),
9391 // swap the return values because the negated value is always 'RHS'.
9392 LHS = FalseVal;
9393 RHS = TrueVal;
9394 if (match(V: CmpLHS, P: m_Neg(V: m_Specific(V: TrueVal))))
9395 std::swap(a&: LHS, b&: RHS);
9396
9397 // (X >s 0) ? -X : X or (X >s -1) ? -X : X --> NABS(X)
9398 // (-X >s 0) ? X : -X or (-X >s -1) ? X : -X --> NABS(X)
9399 if (Pred == ICmpInst::ICMP_SGT && match(V: CmpRHS, P: ZeroOrAllOnes))
9400 return {.Flavor: SPF_NABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9401
9402 // (X <s 0) ? -X : X or (X <s 1) ? -X : X --> ABS(X)
9403 // (-X <s 0) ? X : -X or (-X <s 1) ? X : -X --> ABS(X)
9404 if (Pred == ICmpInst::ICMP_SLT && match(V: CmpRHS, P: ZeroOrOne))
9405 return {.Flavor: SPF_ABS, .NaNBehavior: SPNB_NA, .Ordered: false};
9406 }
9407 }
9408
9409 if (CmpInst::isIntPredicate(P: Pred))
9410 return matchMinMax(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
9411
9412 // According to (IEEE 754-2008 5.3.1), minNum(0.0, -0.0) and similar
9413 // may return either -0.0 or 0.0, so fcmp/select pair has stricter
9414 // semantics than minNum. Be conservative in such case.
9415 if (NaNBehavior != SPNB_RETURNS_ANY ||
9416 (!FMF.noSignedZeros() && !isKnownNonZero(V: CmpLHS) &&
9417 !isKnownNonZero(V: CmpRHS)))
9418 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9419
9420 return matchFastFloatClamp(Pred, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS, RHS);
9421}
9422
9423static Value *lookThroughCastConst(CmpInst *CmpI, Type *SrcTy, Constant *C,
9424 Instruction::CastOps *CastOp) {
9425 const DataLayout &DL = CmpI->getDataLayout();
9426
9427 Constant *CastedTo = nullptr;
9428 switch (*CastOp) {
9429 case Instruction::ZExt:
9430 if (CmpI->isUnsigned())
9431 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy);
9432 break;
9433 case Instruction::SExt:
9434 if (CmpI->isSigned())
9435 CastedTo = ConstantExpr::getTrunc(C, Ty: SrcTy, OnlyIfReduced: true);
9436 break;
9437 case Instruction::Trunc:
9438 Constant *CmpConst;
9439 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_Constant(C&: CmpConst)) &&
9440 CmpConst->getType() == SrcTy) {
9441 // Here we have the following case:
9442 //
9443 // %cond = cmp iN %x, CmpConst
9444 // %tr = trunc iN %x to iK
9445 // %narrowsel = select i1 %cond, iK %t, iK C
9446 //
9447 // We can always move trunc after select operation:
9448 //
9449 // %cond = cmp iN %x, CmpConst
9450 // %widesel = select i1 %cond, iN %x, iN CmpConst
9451 // %tr = trunc iN %widesel to iK
9452 //
9453 // Note that C could be extended in any way because we don't care about
9454 // upper bits after truncation. It can't be abs pattern, because it would
9455 // look like:
9456 //
9457 // select i1 %cond, x, -x.
9458 //
9459 // So only min/max pattern could be matched. Such match requires widened C
9460 // == CmpConst. That is why set widened C = CmpConst, condition trunc
9461 // CmpConst == C is checked below.
9462 CastedTo = CmpConst;
9463 } else {
9464 unsigned ExtOp = CmpI->isSigned() ? Instruction::SExt : Instruction::ZExt;
9465 CastedTo = ConstantFoldCastOperand(Opcode: ExtOp, C, DestTy: SrcTy, DL);
9466 }
9467 break;
9468 case Instruction::FPTrunc:
9469 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPExt, C, DestTy: SrcTy, DL);
9470 break;
9471 case Instruction::FPExt:
9472 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPTrunc, C, DestTy: SrcTy, DL);
9473 break;
9474 case Instruction::FPToUI:
9475 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::UIToFP, C, DestTy: SrcTy, DL);
9476 break;
9477 case Instruction::FPToSI:
9478 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::SIToFP, C, DestTy: SrcTy, DL);
9479 break;
9480 case Instruction::UIToFP:
9481 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToUI, C, DestTy: SrcTy, DL);
9482 break;
9483 case Instruction::SIToFP:
9484 CastedTo = ConstantFoldCastOperand(Opcode: Instruction::FPToSI, C, DestTy: SrcTy, DL);
9485 break;
9486 default:
9487 break;
9488 }
9489
9490 if (!CastedTo)
9491 return nullptr;
9492
9493 // Make sure the cast doesn't lose any information.
9494 Constant *CastedBack =
9495 ConstantFoldCastOperand(Opcode: *CastOp, C: CastedTo, DestTy: C->getType(), DL);
9496 if (CastedBack && CastedBack != C)
9497 return nullptr;
9498
9499 return CastedTo;
9500}
9501
9502/// Helps to match a select pattern in case of a type mismatch.
9503///
9504/// The function processes the case when type of true and false values of a
9505/// select instruction differs from type of the cmp instruction operands because
9506/// of a cast instruction. The function checks if it is legal to move the cast
9507/// operation after "select". If yes, it returns the new second value of
9508/// "select" (with the assumption that cast is moved):
9509/// 1. As operand of cast instruction when both values of "select" are same cast
9510/// instructions.
9511/// 2. As restored constant (by applying reverse cast operation) when the first
9512/// value of the "select" is a cast operation and the second value is a
9513/// constant. It is implemented in lookThroughCastConst().
9514/// 3. As one operand is cast instruction and the other is not. The operands in
9515/// sel(cmp) are in different type integer.
9516/// NOTE: We return only the new second value because the first value could be
9517/// accessed as operand of cast instruction.
9518static Value *lookThroughCast(CmpInst *CmpI, Value *V1, Value *V2,
9519 Instruction::CastOps *CastOp) {
9520 auto *Cast1 = dyn_cast<CastInst>(Val: V1);
9521 if (!Cast1)
9522 return nullptr;
9523
9524 *CastOp = Cast1->getOpcode();
9525 Type *SrcTy = Cast1->getSrcTy();
9526 if (auto *Cast2 = dyn_cast<CastInst>(Val: V2)) {
9527 // If V1 and V2 are both the same cast from the same type, look through V1.
9528 if (*CastOp == Cast2->getOpcode() && SrcTy == Cast2->getSrcTy())
9529 return Cast2->getOperand(i_nocapture: 0);
9530 return nullptr;
9531 }
9532
9533 auto *C = dyn_cast<Constant>(Val: V2);
9534 if (C)
9535 return lookThroughCastConst(CmpI, SrcTy, C, CastOp);
9536
9537 Value *CastedTo = nullptr;
9538 if (*CastOp == Instruction::Trunc) {
9539 if (match(V: CmpI->getOperand(i_nocapture: 1), P: m_ZExtOrSExt(Op: m_Specific(V: V2)))) {
9540 // Here we have the following case:
9541 // %y_ext = sext iK %y to iN
9542 // %cond = cmp iN %x, %y_ext
9543 // %tr = trunc iN %x to iK
9544 // %narrowsel = select i1 %cond, iK %tr, iK %y
9545 //
9546 // We can always move trunc after select operation:
9547 // %y_ext = sext iK %y to iN
9548 // %cond = cmp iN %x, %y_ext
9549 // %widesel = select i1 %cond, iN %x, iN %y_ext
9550 // %tr = trunc iN %widesel to iK
9551 assert(V2->getType() == Cast1->getType() &&
9552 "V2 and Cast1 should be the same type.");
9553 CastedTo = CmpI->getOperand(i_nocapture: 1);
9554 }
9555 }
9556
9557 return CastedTo;
9558}
9559SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
9560 Instruction::CastOps *CastOp) {
9561 SelectInst *SI = dyn_cast<SelectInst>(Val: V);
9562 if (!SI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9563
9564 CmpInst *CmpI = dyn_cast<CmpInst>(Val: SI->getCondition());
9565 if (!CmpI) return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9566
9567 Value *TrueVal = SI->getTrueValue();
9568 Value *FalseVal = SI->getFalseValue();
9569
9570 return llvm::matchDecomposedSelectPattern(
9571 CmpI, TrueVal, FalseVal, LHS, RHS, FMF: SI->getFastMathFlagsOrNone(), CastOp);
9572}
9573
9574SelectPatternResult llvm::matchDecomposedSelectPattern(
9575 CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS,
9576 FastMathFlags FMF, Instruction::CastOps *CastOp) {
9577 CmpInst::Predicate Pred = CmpI->getPredicate();
9578 Value *CmpLHS = CmpI->getOperand(i_nocapture: 0);
9579 Value *CmpRHS = CmpI->getOperand(i_nocapture: 1);
9580 if (isa<FPMathOperator>(Val: CmpI) && CmpI->hasNoNaNs())
9581 FMF.setNoNaNs();
9582
9583 // Bail out early.
9584 if (CmpI->isEquality())
9585 return {.Flavor: SPF_UNKNOWN, .NaNBehavior: SPNB_NA, .Ordered: false};
9586
9587 // Deal with type mismatches.
9588 if (CastOp && CmpLHS->getType() != TrueVal->getType()) {
9589 if (Value *C = lookThroughCast(CmpI, V1: TrueVal, V2: FalseVal, CastOp)) {
9590 // If this is a potential fmin/fmax with a cast to integer, then ignore
9591 // -0.0 because there is no corresponding integer value.
9592 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9593 FMF.setNoSignedZeros();
9594 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS,
9595 TrueVal: cast<CastInst>(Val: TrueVal)->getOperand(i_nocapture: 0), FalseVal: C,
9596 LHS, RHS);
9597 }
9598 if (Value *C = lookThroughCast(CmpI, V1: FalseVal, V2: TrueVal, CastOp)) {
9599 // If this is a potential fmin/fmax with a cast to integer, then ignore
9600 // -0.0 because there is no corresponding integer value.
9601 if (*CastOp == Instruction::FPToSI || *CastOp == Instruction::FPToUI)
9602 FMF.setNoSignedZeros();
9603 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal: C,
9604 FalseVal: cast<CastInst>(Val: FalseVal)->getOperand(i_nocapture: 0), LHS,
9605 RHS);
9606 }
9607 }
9608 return ::matchSelectPattern(Pred, FMF, CmpLHS, CmpRHS, TrueVal, FalseVal, LHS,
9609 RHS);
9610}
9611
9612CmpInst::Predicate llvm::getMinMaxPred(SelectPatternFlavor SPF, bool Ordered) {
9613 if (SPF == SPF_SMIN) return ICmpInst::ICMP_SLT;
9614 if (SPF == SPF_UMIN) return ICmpInst::ICMP_ULT;
9615 if (SPF == SPF_SMAX) return ICmpInst::ICMP_SGT;
9616 if (SPF == SPF_UMAX) return ICmpInst::ICMP_UGT;
9617 if (SPF == SPF_FMINNUM)
9618 return Ordered ? FCmpInst::FCMP_OLT : FCmpInst::FCMP_ULT;
9619 if (SPF == SPF_FMAXNUM)
9620 return Ordered ? FCmpInst::FCMP_OGT : FCmpInst::FCMP_UGT;
9621 llvm_unreachable("unhandled!");
9622}
9623
9624Intrinsic::ID llvm::getMinMaxIntrinsic(SelectPatternFlavor SPF) {
9625 switch (SPF) {
9626 case SelectPatternFlavor::SPF_UMIN:
9627 return Intrinsic::umin;
9628 case SelectPatternFlavor::SPF_UMAX:
9629 return Intrinsic::umax;
9630 case SelectPatternFlavor::SPF_SMIN:
9631 return Intrinsic::smin;
9632 case SelectPatternFlavor::SPF_SMAX:
9633 return Intrinsic::smax;
9634 default:
9635 llvm_unreachable("Unexpected SPF");
9636 }
9637}
9638
9639SelectPatternFlavor llvm::getInverseMinMaxFlavor(SelectPatternFlavor SPF) {
9640 if (SPF == SPF_SMIN) return SPF_SMAX;
9641 if (SPF == SPF_UMIN) return SPF_UMAX;
9642 if (SPF == SPF_SMAX) return SPF_SMIN;
9643 if (SPF == SPF_UMAX) return SPF_UMIN;
9644 llvm_unreachable("unhandled!");
9645}
9646
9647Intrinsic::ID llvm::getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID) {
9648 switch (MinMaxID) {
9649 case Intrinsic::smax: return Intrinsic::smin;
9650 case Intrinsic::smin: return Intrinsic::smax;
9651 case Intrinsic::umax: return Intrinsic::umin;
9652 case Intrinsic::umin: return Intrinsic::umax;
9653 // Please note that next four intrinsics may produce the same result for
9654 // original and inverted case even if X != Y due to NaN is handled specially.
9655 case Intrinsic::maximum: return Intrinsic::minimum;
9656 case Intrinsic::minimum: return Intrinsic::maximum;
9657 case Intrinsic::maxnum: return Intrinsic::minnum;
9658 case Intrinsic::minnum: return Intrinsic::maxnum;
9659 case Intrinsic::maximumnum:
9660 return Intrinsic::minimumnum;
9661 case Intrinsic::minimumnum:
9662 return Intrinsic::maximumnum;
9663 default: llvm_unreachable("Unexpected intrinsic");
9664 }
9665}
9666
9667APInt llvm::getMinMaxLimit(SelectPatternFlavor SPF, unsigned BitWidth) {
9668 switch (SPF) {
9669 case SPF_SMAX: return APInt::getSignedMaxValue(numBits: BitWidth);
9670 case SPF_SMIN: return APInt::getSignedMinValue(numBits: BitWidth);
9671 case SPF_UMAX: return APInt::getMaxValue(numBits: BitWidth);
9672 case SPF_UMIN: return APInt::getMinValue(numBits: BitWidth);
9673 default: llvm_unreachable("Unexpected flavor");
9674 }
9675}
9676
9677std::pair<Intrinsic::ID, bool>
9678llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
9679 // Check if VL contains select instructions that can be folded into a min/max
9680 // vector intrinsic and return the intrinsic if it is possible.
9681 // TODO: Support floating point min/max.
9682 bool AllCmpSingleUse = true;
9683 SelectPatternResult SelectPattern;
9684 SelectPattern.Flavor = SPF_UNKNOWN;
9685 if (all_of(Range&: VL, P: [&SelectPattern, &AllCmpSingleUse](Value *I) {
9686 Value *LHS, *RHS;
9687 auto CurrentPattern = matchSelectPattern(V: I, LHS, RHS);
9688 if (!SelectPatternResult::isMinOrMax(SPF: CurrentPattern.Flavor))
9689 return false;
9690 if (SelectPattern.Flavor != SPF_UNKNOWN &&
9691 SelectPattern.Flavor != CurrentPattern.Flavor)
9692 return false;
9693 SelectPattern = CurrentPattern;
9694 AllCmpSingleUse &=
9695 match(V: I, P: m_Select(C: m_OneUse(SubPattern: m_Value()), L: m_Value(), R: m_Value()));
9696 return true;
9697 })) {
9698 switch (SelectPattern.Flavor) {
9699 case SPF_SMIN:
9700 return {Intrinsic::smin, AllCmpSingleUse};
9701 case SPF_UMIN:
9702 return {Intrinsic::umin, AllCmpSingleUse};
9703 case SPF_SMAX:
9704 return {Intrinsic::smax, AllCmpSingleUse};
9705 case SPF_UMAX:
9706 return {Intrinsic::umax, AllCmpSingleUse};
9707 case SPF_FMAXNUM:
9708 return {Intrinsic::maxnum, AllCmpSingleUse};
9709 case SPF_FMINNUM:
9710 return {Intrinsic::minnum, AllCmpSingleUse};
9711 default:
9712 llvm_unreachable("unexpected select pattern flavor");
9713 }
9714 }
9715 return {Intrinsic::not_intrinsic, false};
9716}
9717
9718template <typename InstTy>
9719static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
9720 Value *&Init, Value *&OtherOp) {
9721 // Handle the case of a simple two-predecessor recurrence PHI.
9722 // There's a lot more that could theoretically be done here, but
9723 // this is sufficient to catch some interesting cases.
9724 // TODO: Expand list -- gep, uadd.sat etc.
9725 if (PN->getNumIncomingValues() != 2)
9726 return false;
9727
9728 for (unsigned I = 0; I != 2; ++I) {
9729 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9730 Operation && Operation->getNumOperands() >= 2) {
9731 Value *LHS = Operation->getOperand(0);
9732 Value *RHS = Operation->getOperand(1);
9733 if (LHS != PN && RHS != PN)
9734 continue;
9735
9736 Inst = Operation;
9737 Init = PN->getIncomingValue(i: !I);
9738 OtherOp = (LHS == PN) ? RHS : LHS;
9739 return true;
9740 }
9741 }
9742 return false;
9743}
9744
9745template <typename InstTy>
9746static bool matchThreeInputRecurrence(const PHINode *PN, InstTy *&Inst,
9747 Value *&Init, Value *&OtherOp0,
9748 Value *&OtherOp1) {
9749 if (PN->getNumIncomingValues() != 2)
9750 return false;
9751
9752 for (unsigned I = 0; I != 2; ++I) {
9753 if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(i: I));
9754 Operation && Operation->getNumOperands() >= 3) {
9755 Value *Op0 = Operation->getOperand(0);
9756 Value *Op1 = Operation->getOperand(1);
9757 Value *Op2 = Operation->getOperand(2);
9758
9759 if (Op0 != PN && Op1 != PN && Op2 != PN)
9760 continue;
9761
9762 Inst = Operation;
9763 Init = PN->getIncomingValue(i: !I);
9764 if (Op0 == PN) {
9765 OtherOp0 = Op1;
9766 OtherOp1 = Op2;
9767 } else if (Op1 == PN) {
9768 OtherOp0 = Op0;
9769 OtherOp1 = Op2;
9770 } else {
9771 OtherOp0 = Op0;
9772 OtherOp1 = Op1;
9773 }
9774 return true;
9775 }
9776 }
9777 return false;
9778}
9779bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
9780 Value *&Start, Value *&Step) {
9781 // We try to match a recurrence of the form:
9782 // %iv = [Start, %entry], [%iv.next, %backedge]
9783 // %iv.next = binop %iv, Step
9784 // Or:
9785 // %iv = [Start, %entry], [%iv.next, %backedge]
9786 // %iv.next = binop Step, %iv
9787 return matchTwoInputRecurrence(PN: P, Inst&: BO, Init&: Start, OtherOp&: Step);
9788}
9789
9790bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
9791 Value *&Start, Value *&Step) {
9792 BinaryOperator *BO = nullptr;
9793 return match(V: I, P: m_c_BinOp(L: m_Phi(PN&: P), R: m_Value())) &&
9794 matchSimpleRecurrence(P, BO, Start, Step) && BO == I;
9795}
9796
9797bool llvm::matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I,
9798 PHINode *&P, Value *&Init,
9799 Value *&OtherOp) {
9800 // Binary intrinsics only supported for now.
9801 if (I->arg_size() != 2 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9802 I->getType() != I->getArgOperand(i: 1)->getType())
9803 return false;
9804
9805 IntrinsicInst *II = nullptr;
9806 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9807 if (!P)
9808 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9809
9810 return P && matchTwoInputRecurrence(PN: P, Inst&: II, Init, OtherOp) && II == I;
9811}
9812
9813bool llvm::matchSimpleTernaryIntrinsicRecurrence(const IntrinsicInst *I,
9814 PHINode *&P, Value *&Init,
9815 Value *&OtherOp0,
9816 Value *&OtherOp1) {
9817 if (I->arg_size() != 3 || I->getType() != I->getArgOperand(i: 0)->getType() ||
9818 I->getType() != I->getArgOperand(i: 1)->getType() ||
9819 I->getType() != I->getArgOperand(i: 2)->getType())
9820 return false;
9821 IntrinsicInst *II = nullptr;
9822 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 0));
9823 if (!P) {
9824 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 1));
9825 if (!P)
9826 P = dyn_cast<PHINode>(Val: I->getArgOperand(i: 2));
9827 }
9828 return P && matchThreeInputRecurrence(PN: P, Inst&: II, Init, OtherOp0, OtherOp1) &&
9829 II == I;
9830}
9831
9832/// Return true if "icmp Pred LHS RHS" is always true.
9833static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
9834 const Value *RHS) {
9835 if (ICmpInst::isTrueWhenEqual(predicate: Pred) && LHS == RHS)
9836 return true;
9837
9838 switch (Pred) {
9839 default:
9840 return false;
9841
9842 case CmpInst::ICMP_SLE: {
9843 const APInt *C;
9844
9845 // LHS s<= LHS +_{nsw} C if C >= 0
9846 // LHS s<= LHS | C if C >= 0
9847 if (match(V: RHS, P: m_NSWAdd(L: m_Specific(V: LHS), R: m_APInt(Res&: C))) ||
9848 match(V: RHS, P: m_Or(L: m_Specific(V: LHS), R: m_APInt(Res&: C))))
9849 return !C->isNegative();
9850
9851 // LHS s<= smax(LHS, V) for any V
9852 if (match(V: RHS, P: m_c_SMax(L: m_Specific(V: LHS), R: m_Value())))
9853 return true;
9854
9855 // smin(RHS, V) s<= RHS for any V
9856 if (match(V: LHS, P: m_c_SMin(L: m_Specific(V: RHS), R: m_Value())))
9857 return true;
9858
9859 // Match A to (X +_{nsw} CA) and B to (X +_{nsw} CB)
9860 const Value *X;
9861 const APInt *CLHS, *CRHS;
9862 if (match(V: LHS, P: m_NSWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9863 match(V: RHS, P: m_NSWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9864 return CLHS->sle(RHS: *CRHS);
9865
9866 return false;
9867 }
9868
9869 case CmpInst::ICMP_ULE: {
9870 // LHS u<= LHS +_{nuw} V for any V
9871 if (match(V: RHS, P: m_c_Add(L: m_Specific(V: LHS), R: m_Value())) &&
9872 cast<OverflowingBinaryOperator>(Val: RHS)->hasNoUnsignedWrap())
9873 return true;
9874
9875 // LHS u<= LHS | V for any V
9876 if (match(V: RHS, P: m_c_Or(L: m_Specific(V: LHS), R: m_Value())))
9877 return true;
9878
9879 // LHS u<= umax(LHS, V) for any V
9880 if (match(V: RHS, P: m_c_UMax(L: m_Specific(V: LHS), R: m_Value())))
9881 return true;
9882
9883 // RHS >> V u<= RHS for any V
9884 if (match(V: LHS, P: m_LShr(L: m_Specific(V: RHS), R: m_Value())))
9885 return true;
9886
9887 // RHS u/ C_ugt_1 u<= RHS
9888 const APInt *C;
9889 if (match(V: LHS, P: m_UDiv(L: m_Specific(V: RHS), R: m_APInt(Res&: C))) && C->ugt(RHS: 1))
9890 return true;
9891
9892 // RHS & V u<= RHS for any V
9893 if (match(V: LHS, P: m_c_And(L: m_Specific(V: RHS), R: m_Value())))
9894 return true;
9895
9896 // umin(RHS, V) u<= RHS for any V
9897 if (match(V: LHS, P: m_c_UMin(L: m_Specific(V: RHS), R: m_Value())))
9898 return true;
9899
9900 // Match A to (X +_{nuw} CA) and B to (X +_{nuw} CB)
9901 const Value *X;
9902 const APInt *CLHS, *CRHS;
9903 if (match(V: LHS, P: m_NUWAddLike(L: m_Value(V&: X), R: m_APInt(Res&: CLHS))) &&
9904 match(V: RHS, P: m_NUWAddLike(L: m_Specific(V: X), R: m_APInt(Res&: CRHS))))
9905 return CLHS->ule(RHS: *CRHS);
9906
9907 return false;
9908 }
9909 }
9910}
9911
9912/// Return true if "icmp Pred BLHS BRHS" is true whenever "icmp Pred
9913/// ALHS ARHS" is true. Otherwise, return std::nullopt.
9914static std::optional<bool>
9915isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS,
9916 const Value *ARHS, const Value *BLHS, const Value *BRHS) {
9917 switch (Pred) {
9918 default:
9919 return std::nullopt;
9920
9921 case CmpInst::ICMP_SLT:
9922 case CmpInst::ICMP_SLE:
9923 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BLHS, RHS: ALHS) &&
9924 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ARHS, RHS: BRHS))
9925 return true;
9926 return std::nullopt;
9927
9928 case CmpInst::ICMP_SGT:
9929 case CmpInst::ICMP_SGE:
9930 if (isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: ALHS, RHS: BLHS) &&
9931 isTruePredicate(Pred: CmpInst::ICMP_SLE, LHS: BRHS, RHS: ARHS))
9932 return true;
9933 return std::nullopt;
9934
9935 case CmpInst::ICMP_ULT:
9936 case CmpInst::ICMP_ULE:
9937 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BLHS, RHS: ALHS) &&
9938 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ARHS, RHS: BRHS))
9939 return true;
9940 return std::nullopt;
9941
9942 case CmpInst::ICMP_UGT:
9943 case CmpInst::ICMP_UGE:
9944 if (isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: ALHS, RHS: BLHS) &&
9945 isTruePredicate(Pred: CmpInst::ICMP_ULE, LHS: BRHS, RHS: ARHS))
9946 return true;
9947 return std::nullopt;
9948 }
9949}
9950
9951/// Return true if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is true.
9952/// Return false if "icmp LPred X, LCR" implies "icmp RPred X, RCR" is false.
9953/// Otherwise, return std::nullopt if we can't infer anything.
9954static std::optional<bool>
9955isImpliedCondCommonOperandWithCR(CmpPredicate LPred, const ConstantRange &LCR,
9956 CmpPredicate RPred, const ConstantRange &RCR) {
9957 auto CRImpliesPred = [&](ConstantRange CR,
9958 CmpInst::Predicate Pred) -> std::optional<bool> {
9959 // If all true values for lhs and true for rhs, lhs implies rhs
9960 if (CR.icmp(Pred, Other: RCR))
9961 return true;
9962
9963 // If there is no overlap, lhs implies not rhs
9964 if (CR.icmp(Pred: CmpInst::getInversePredicate(pred: Pred), Other: RCR))
9965 return false;
9966
9967 return std::nullopt;
9968 };
9969 if (auto Res = CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
9970 RPred))
9971 return Res;
9972 if (LPred.hasSameSign() ^ RPred.hasSameSign()) {
9973 LPred = LPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: LPred)
9974 : LPred.dropSameSign();
9975 RPred = RPred.hasSameSign() ? ICmpInst::getFlippedSignednessPredicate(Pred: RPred)
9976 : RPred.dropSameSign();
9977 return CRImpliesPred(ConstantRange::makeAllowedICmpRegion(Pred: LPred, Other: LCR),
9978 RPred);
9979 }
9980 return std::nullopt;
9981}
9982
9983/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
9984/// is true. Return false if LHS implies RHS is false. Otherwise, return
9985/// std::nullopt if we can't infer anything.
9986static std::optional<bool>
9987isImpliedCondICmps(CmpPredicate LPred, const Value *L0, const Value *L1,
9988 CmpPredicate RPred, const Value *R0, const Value *R1,
9989 const DataLayout &DL, bool LHSIsTrue) {
9990 // The rest of the logic assumes the LHS condition is true. If that's not the
9991 // case, invert the predicate to make it so.
9992 if (!LHSIsTrue)
9993 LPred = ICmpInst::getInverseCmpPredicate(Pred: LPred);
9994
9995 // We can have non-canonical operands, so try to normalize any common operand
9996 // to L0/R0.
9997 if (L0 == R1) {
9998 std::swap(a&: R0, b&: R1);
9999 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10000 }
10001 if (R0 == L1) {
10002 std::swap(a&: L0, b&: L1);
10003 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
10004 }
10005 if (L1 == R1) {
10006 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
10007 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
10008 std::swap(a&: L0, b&: L1);
10009 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
10010 std::swap(a&: R0, b&: R1);
10011 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10012 }
10013 }
10014
10015 // See if we can infer anything if operand-0 matches and we have at least one
10016 // constant.
10017 const APInt *Unused;
10018 if (L0 == R0 && (match(V: L1, P: m_APInt(Res&: Unused)) || match(V: R1, P: m_APInt(Res&: Unused)))) {
10019 // Potential TODO: We could also further use the constant range of L0/R0 to
10020 // further constraint the constant ranges. At the moment this leads to
10021 // several regressions related to not transforming `multi_use(A + C0) eq/ne
10022 // C1` (see discussion: D58633).
10023 SimplifyQuery SQ(DL);
10024 ConstantRange LCR = computeConstantRange(V: L1, ForSigned: ICmpInst::isSigned(Pred: LPred), SQ,
10025 Depth: MaxAnalysisRecursionDepth - 1);
10026 ConstantRange RCR = computeConstantRange(V: R1, ForSigned: ICmpInst::isSigned(Pred: RPred), SQ,
10027 Depth: MaxAnalysisRecursionDepth - 1);
10028
10029 // Even if L1/R1 are not both constant, we can still sometimes deduce
10030 // relationship from a single constant. For example X u> Y implies X != 0.
10031 if (auto R = isImpliedCondCommonOperandWithCR(LPred, LCR, RPred, RCR))
10032 return R;
10033 // If both L1/R1 were exact constant ranges and we didn't get anything
10034 // here, we won't be able to deduce this.
10035 if (match(V: L1, P: m_APInt(Res&: Unused)) && match(V: R1, P: m_APInt(Res&: Unused)))
10036 return std::nullopt;
10037 }
10038
10039 // Can we infer anything when the two compares have matching operands?
10040 if (L0 == R0 && L1 == R1)
10041 return ICmpInst::isImpliedByMatchingCmp(Pred1: LPred, Pred2: RPred);
10042
10043 // It only really makes sense in the context of signed comparison for "X - Y
10044 // must be positive if X >= Y and no overflow".
10045 // Take SGT as an example: L0:x > L1:y and C >= 0
10046 // ==> R0:(x -nsw y) < R1:(-C) is false
10047 CmpInst::Predicate SignedLPred = LPred.getPreferredSignedPredicate();
10048 if ((SignedLPred == ICmpInst::ICMP_SGT ||
10049 SignedLPred == ICmpInst::ICMP_SGE) &&
10050 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
10051 if (match(V: R1, P: m_NonPositive()) &&
10052 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == false)
10053 return false;
10054 }
10055
10056 // Take SLT as an example: L0:x < L1:y and C <= 0
10057 // ==> R0:(x -nsw y) < R1:(-C) is true
10058 if ((SignedLPred == ICmpInst::ICMP_SLT ||
10059 SignedLPred == ICmpInst::ICMP_SLE) &&
10060 match(V: R0, P: m_NSWSub(L: m_Specific(V: L0), R: m_Specific(V: L1)))) {
10061 if (match(V: R1, P: m_NonNegative()) &&
10062 ICmpInst::isImpliedByMatchingCmp(Pred1: SignedLPred, Pred2: RPred) == true)
10063 return true;
10064 }
10065
10066 // (a - b) pred C -> a != b, if 0 pred C is false
10067 // (ptrtoint(a) - ptrtoint(b)) pred C -> a != b, if 0 pred C is false
10068 Value *A, *B;
10069 if (ICmpInst::isEquality(P: RPred) && cmpExcludesZero(Pred: LPred, RHS: L1) &&
10070 match(V: L0, P: m_Sub(L: m_Value(V&: A), R: m_Value(V&: B))) &&
10071 ((A == R0 && B == R1) || (A == R1 && B == R0) ||
10072 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0))) &&
10073 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1)))) ||
10074 (match(V: A, P: m_PtrToIntOrAddr(Op: m_Specific(V: R1))) &&
10075 match(V: B, P: m_PtrToIntOrAddr(Op: m_Specific(V: R0)))))) {
10076 return RPred.dropSameSign() == ICmpInst::ICMP_NE;
10077 }
10078
10079 // L0 = R0 = L1 + R1, L0 >=u L1 implies R0 >=u R1, L0 <u L1 implies R0 <u R1
10080 if (L0 == R0 &&
10081 (LPred == ICmpInst::ICMP_ULT || LPred == ICmpInst::ICMP_UGE) &&
10082 (RPred == ICmpInst::ICMP_ULT || RPred == ICmpInst::ICMP_UGE) &&
10083 match(V: L0, P: m_c_Add(L: m_Specific(V: L1), R: m_Specific(V: R1))))
10084 return CmpPredicate::getMatching(A: LPred, B: RPred).has_value();
10085
10086 if (auto P = CmpPredicate::getMatching(A: LPred, B: RPred))
10087 return isImpliedCondOperands(Pred: *P, ALHS: L0, ARHS: L1, BLHS: R0, BRHS: R1);
10088
10089 // L0 u< C sets limits to L0's bits which may imply (L0 & Mask) pred RC
10090 // Example: L0 u< 13 => (L0 & 16) == 0
10091 const APInt *LC, *RC, *MaskC;
10092 if (match(V: L1, P: m_APInt(Res&: LC)) && match(V: R1, P: m_APInt(Res&: RC)) &&
10093 match(V: R0, P: m_And(L: m_Specific(V: L0), R: m_APInt(Res&: MaskC)))) {
10094 ConstantRange LCRange = ConstantRange::makeExactICmpRegion(Pred: LPred, Other: *LC);
10095 ConstantRange MaskedCRange = LCRange.binaryAnd(Other: *MaskC);
10096 if (MaskedCRange.icmp(Pred: RPred, Other: ConstantRange(*RC)))
10097 return true;
10098 if (MaskedCRange.icmp(Pred: ICmpInst::getInversePredicate(pred: RPred),
10099 Other: ConstantRange(*RC)))
10100 return false;
10101 }
10102
10103 return std::nullopt;
10104}
10105
10106/// Return true if LHS implies RHS (expanded to its components as "R0 RPred R1")
10107/// is true. Return false if LHS implies RHS is false. Otherwise, return
10108/// std::nullopt if we can't infer anything.
10109static std::optional<bool>
10110isImpliedCondFCmps(FCmpInst::Predicate LPred, const Value *L0, const Value *L1,
10111 FCmpInst::Predicate RPred, const Value *R0, const Value *R1,
10112 const DataLayout &DL, bool LHSIsTrue) {
10113 // The rest of the logic assumes the LHS condition is true. If that's not the
10114 // case, invert the predicate to make it so.
10115 if (!LHSIsTrue)
10116 LPred = FCmpInst::getInversePredicate(pred: LPred);
10117
10118 // We can have non-canonical operands, so try to normalize any common operand
10119 // to L0/R0.
10120 if (L0 == R1) {
10121 std::swap(a&: R0, b&: R1);
10122 RPred = FCmpInst::getSwappedPredicate(pred: RPred);
10123 }
10124 if (R0 == L1) {
10125 std::swap(a&: L0, b&: L1);
10126 LPred = FCmpInst::getSwappedPredicate(pred: LPred);
10127 }
10128 if (L1 == R1) {
10129 // If we have L0 == R0 and L1 == R1, then make L1/R1 the constants.
10130 if (L0 != R0 || match(V: L0, P: m_ImmConstant())) {
10131 std::swap(a&: L0, b&: L1);
10132 LPred = ICmpInst::getSwappedCmpPredicate(Pred: LPred);
10133 std::swap(a&: R0, b&: R1);
10134 RPred = ICmpInst::getSwappedCmpPredicate(Pred: RPred);
10135 }
10136 }
10137
10138 // Can we infer anything when the two compares have matching operands?
10139 if (L0 == R0 && L1 == R1) {
10140 if ((LPred & RPred) == LPred)
10141 return true;
10142 if ((LPred & ~RPred) == LPred)
10143 return false;
10144 }
10145
10146 // See if we can infer anything if operand-0 matches and we have at least one
10147 // constant.
10148 const APFloat *L1C, *R1C;
10149 if (L0 == R0 && match(V: L1, P: m_APFloat(Res&: L1C)) && match(V: R1, P: m_APFloat(Res&: R1C))) {
10150 if (std::optional<ConstantFPRange> DomCR =
10151 ConstantFPRange::makeExactFCmpRegion(Pred: LPred, Other: *L1C)) {
10152 if (std::optional<ConstantFPRange> ImpliedCR =
10153 ConstantFPRange::makeExactFCmpRegion(Pred: RPred, Other: *R1C)) {
10154 if (ImpliedCR->contains(CR: *DomCR))
10155 return true;
10156 }
10157 if (std::optional<ConstantFPRange> ImpliedCR =
10158 ConstantFPRange::makeExactFCmpRegion(
10159 Pred: FCmpInst::getInversePredicate(pred: RPred), Other: *R1C)) {
10160 if (ImpliedCR->contains(CR: *DomCR))
10161 return false;
10162 }
10163 }
10164 }
10165
10166 return std::nullopt;
10167}
10168
10169/// Return true if LHS implies RHS is true. Return false if LHS implies RHS is
10170/// false. Otherwise, return std::nullopt if we can't infer anything. We
10171/// expect the RHS to be an icmp and the LHS to be an 'and', 'or', or a 'select'
10172/// instruction.
10173static std::optional<bool>
10174isImpliedCondAndOr(const Instruction *LHS, CmpPredicate RHSPred,
10175 const Value *RHSOp0, const Value *RHSOp1,
10176 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
10177 // The LHS must be an 'or', 'and', or a 'select' instruction.
10178 assert((LHS->getOpcode() == Instruction::And ||
10179 LHS->getOpcode() == Instruction::Or ||
10180 LHS->getOpcode() == Instruction::Select) &&
10181 "Expected LHS to be 'and', 'or', or 'select'.");
10182
10183 assert(Depth <= MaxAnalysisRecursionDepth && "Hit recursion limit");
10184
10185 // If the result of an 'or' is false, then we know both legs of the 'or' are
10186 // false. Similarly, if the result of an 'and' is true, then we know both
10187 // legs of the 'and' are true.
10188 const Value *ALHS, *ARHS;
10189 if ((!LHSIsTrue && match(V: LHS, P: m_LogicalOr(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS)))) ||
10190 (LHSIsTrue && match(V: LHS, P: m_LogicalAnd(L: m_Value(V&: ALHS), R: m_Value(V&: ARHS))))) {
10191 // FIXME: Make this non-recursion.
10192 if (std::optional<bool> Implication = isImpliedCondition(
10193 LHS: ALHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
10194 return Implication;
10195 if (std::optional<bool> Implication = isImpliedCondition(
10196 LHS: ARHS, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue, Depth: Depth + 1))
10197 return Implication;
10198 return std::nullopt;
10199 }
10200 return std::nullopt;
10201}
10202
10203std::optional<bool>
10204llvm::isImpliedCondition(const Value *LHS, CmpPredicate RHSPred,
10205 const Value *RHSOp0, const Value *RHSOp1,
10206 const DataLayout &DL, bool LHSIsTrue, unsigned Depth) {
10207 // Bail out when we hit the limit.
10208 if (Depth == MaxAnalysisRecursionDepth)
10209 return std::nullopt;
10210
10211 // A mismatch occurs when we compare a scalar cmp to a vector cmp, for
10212 // example.
10213 if (RHSOp0->getType()->isVectorTy() != LHS->getType()->isVectorTy())
10214 return std::nullopt;
10215
10216 assert(LHS->getType()->isIntOrIntVectorTy(1) &&
10217 "Expected integer type only!");
10218
10219 // Match not
10220 if (match(V: LHS, P: m_Not(V: m_Value(V&: LHS))))
10221 LHSIsTrue = !LHSIsTrue;
10222
10223 // Both LHS and RHS are icmps.
10224 if (RHSOp0->getType()->getScalarType()->isIntOrPtrTy()) {
10225 CmpPredicate LHSPred;
10226 Value *LHSOp0, *LHSOp1;
10227 if (match(V: LHS, P: m_ICmpLike(Pred&: LHSPred, L: m_Value(V&: LHSOp0), R: m_Value(V&: LHSOp1))))
10228 return isImpliedCondICmps(LPred: LHSPred, L0: LHSOp0, L1: LHSOp1, RPred: RHSPred, R0: RHSOp0,
10229 R1: RHSOp1, DL, LHSIsTrue);
10230 } else {
10231 assert(RHSOp0->getType()->isFPOrFPVectorTy() &&
10232 "Expected floating point type only!");
10233 if (const auto *LHSCmp = dyn_cast<FCmpInst>(Val: LHS))
10234 return isImpliedCondFCmps(LPred: LHSCmp->getPredicate(), L0: LHSCmp->getOperand(i_nocapture: 0),
10235 L1: LHSCmp->getOperand(i_nocapture: 1), RPred: RHSPred, R0: RHSOp0, R1: RHSOp1,
10236 DL, LHSIsTrue);
10237 }
10238
10239 /// The LHS should be an 'or', 'and', or a 'select' instruction. We expect
10240 /// the RHS to be an icmp.
10241 /// FIXME: Add support for and/or/select on the RHS.
10242 if (const Instruction *LHSI = dyn_cast<Instruction>(Val: LHS)) {
10243 if ((LHSI->getOpcode() == Instruction::And ||
10244 LHSI->getOpcode() == Instruction::Or ||
10245 LHSI->getOpcode() == Instruction::Select))
10246 return isImpliedCondAndOr(LHS: LHSI, RHSPred, RHSOp0, RHSOp1, DL, LHSIsTrue,
10247 Depth);
10248 }
10249 return std::nullopt;
10250}
10251
10252std::optional<bool> llvm::isImpliedCondition(const Value *LHS, const Value *RHS,
10253 const DataLayout &DL,
10254 bool LHSIsTrue, unsigned Depth) {
10255 // LHS ==> RHS by definition
10256 if (LHS == RHS)
10257 return LHSIsTrue;
10258
10259 // Match not
10260 bool InvertRHS = false;
10261 if (match(V: RHS, P: m_Not(V: m_Value(V&: RHS)))) {
10262 if (LHS == RHS)
10263 return !LHSIsTrue;
10264 InvertRHS = true;
10265 }
10266
10267 CmpPredicate RHSPred;
10268 Value *RHSOp0, *RHSOp1;
10269 if (match(V: RHS, P: m_ICmpLike(Pred&: RHSPred, L: m_Value(V&: RHSOp0), R: m_Value(V&: RHSOp1)))) {
10270 if (auto Implied = isImpliedCondition(LHS, RHSPred, RHSOp0, RHSOp1, DL,
10271 LHSIsTrue, Depth))
10272 return InvertRHS ? !*Implied : *Implied;
10273 return std::nullopt;
10274 }
10275 if (const FCmpInst *RHSCmp = dyn_cast<FCmpInst>(Val: RHS)) {
10276 if (auto Implied = isImpliedCondition(
10277 LHS, RHSPred: RHSCmp->getPredicate(), RHSOp0: RHSCmp->getOperand(i_nocapture: 0),
10278 RHSOp1: RHSCmp->getOperand(i_nocapture: 1), DL, LHSIsTrue, Depth))
10279 return InvertRHS ? !*Implied : *Implied;
10280 return std::nullopt;
10281 }
10282
10283 if (Depth == MaxAnalysisRecursionDepth)
10284 return std::nullopt;
10285
10286 // LHS ==> (RHS1 || RHS2) if LHS ==> RHS1 or LHS ==> RHS2
10287 // LHS ==> !(RHS1 && RHS2) if LHS ==> !RHS1 or LHS ==> !RHS2
10288 const Value *RHS1, *RHS2;
10289 if (match(V: RHS, P: m_LogicalOr(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
10290 if (std::optional<bool> Imp =
10291 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
10292 if (*Imp == true)
10293 return !InvertRHS;
10294 if (std::optional<bool> Imp =
10295 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
10296 if (*Imp == true)
10297 return !InvertRHS;
10298 }
10299 if (match(V: RHS, P: m_LogicalAnd(L: m_Value(V&: RHS1), R: m_Value(V&: RHS2)))) {
10300 if (std::optional<bool> Imp =
10301 isImpliedCondition(LHS, RHS: RHS1, DL, LHSIsTrue, Depth: Depth + 1))
10302 if (*Imp == false)
10303 return InvertRHS;
10304 if (std::optional<bool> Imp =
10305 isImpliedCondition(LHS, RHS: RHS2, DL, LHSIsTrue, Depth: Depth + 1))
10306 if (*Imp == false)
10307 return InvertRHS;
10308 }
10309
10310 return std::nullopt;
10311}
10312
10313// Returns a pair (Condition, ConditionIsTrue), where Condition is a branch
10314// condition dominating ContextI or nullptr, if no condition is found.
10315static std::pair<Value *, bool>
10316getDomPredecessorCondition(const Instruction *ContextI) {
10317 if (!ContextI || !ContextI->getParent())
10318 return {nullptr, false};
10319
10320 // TODO: This is a poor/cheap way to determine dominance. Should we use a
10321 // dominator tree (eg, from a SimplifyQuery) instead?
10322 const BasicBlock *ContextBB = ContextI->getParent();
10323 const BasicBlock *PredBB = ContextBB->getSinglePredecessor();
10324 if (!PredBB)
10325 return {nullptr, false};
10326
10327 // We need a conditional branch in the predecessor.
10328 Value *PredCond;
10329 BasicBlock *TrueBB, *FalseBB;
10330 if (!match(V: PredBB->getTerminator(), P: m_Br(C: m_Value(V&: PredCond), T&: TrueBB, F&: FalseBB)))
10331 return {nullptr, false};
10332
10333 // The branch should get simplified. Don't bother simplifying this condition.
10334 if (TrueBB == FalseBB)
10335 return {nullptr, false};
10336
10337 assert((TrueBB == ContextBB || FalseBB == ContextBB) &&
10338 "Predecessor block does not point to successor?");
10339
10340 // Is this condition implied by the predecessor condition?
10341 return {PredCond, TrueBB == ContextBB};
10342}
10343
10344std::optional<bool> llvm::isImpliedByDomCondition(const Value *Cond,
10345 const Instruction *ContextI,
10346 const DataLayout &DL) {
10347 assert(Cond->getType()->isIntOrIntVectorTy(1) && "Condition must be bool");
10348 auto PredCond = getDomPredecessorCondition(ContextI);
10349 if (PredCond.first)
10350 return isImpliedCondition(LHS: PredCond.first, RHS: Cond, DL, LHSIsTrue: PredCond.second);
10351 return std::nullopt;
10352}
10353
10354std::optional<bool> llvm::isImpliedByDomCondition(CmpPredicate Pred,
10355 const Value *LHS,
10356 const Value *RHS,
10357 const Instruction *ContextI,
10358 const DataLayout &DL) {
10359 auto PredCond = getDomPredecessorCondition(ContextI);
10360 if (PredCond.first)
10361 return isImpliedCondition(LHS: PredCond.first, RHSPred: Pred, RHSOp0: LHS, RHSOp1: RHS, DL,
10362 LHSIsTrue: PredCond.second);
10363 return std::nullopt;
10364}
10365
10366static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
10367 APInt &Upper, const InstrInfoQuery &IIQ,
10368 bool PreferSignedRange) {
10369 unsigned Width = Lower.getBitWidth();
10370 const APInt *C;
10371 switch (BO.getOpcode()) {
10372 case Instruction::Sub:
10373 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10374 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10375 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10376
10377 // If the caller expects a signed compare, then try to use a signed range.
10378 // Otherwise if both no-wraps are set, use the unsigned range because it
10379 // is never larger than the signed range. Example:
10380 // "sub nuw nsw i8 -2, x" is unsigned [0, 254] vs. signed [-128, 126].
10381 // "sub nuw nsw i8 2, x" is unsigned [0, 2] vs. signed [-125, 127].
10382 if (PreferSignedRange && HasNSW && HasNUW)
10383 HasNUW = false;
10384
10385 if (HasNUW) {
10386 // 'sub nuw c, x' produces [0, C].
10387 Upper = *C + 1;
10388 } else if (HasNSW) {
10389 if (C->isNegative()) {
10390 // 'sub nsw -C, x' produces [SINT_MIN, -C - SINT_MIN].
10391 Lower = APInt::getSignedMinValue(numBits: Width);
10392 Upper = *C - APInt::getSignedMaxValue(numBits: Width);
10393 } else {
10394 // Note that sub 0, INT_MIN is not NSW. It techically is a signed wrap
10395 // 'sub nsw C, x' produces [C - SINT_MAX, SINT_MAX].
10396 Lower = *C - APInt::getSignedMaxValue(numBits: Width);
10397 Upper = APInt::getSignedMinValue(numBits: Width);
10398 }
10399 }
10400 }
10401 break;
10402 case Instruction::Add:
10403 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10404 bool HasNSW = IIQ.hasNoSignedWrap(Op: &BO);
10405 bool HasNUW = IIQ.hasNoUnsignedWrap(Op: &BO);
10406
10407 // If the caller expects a signed compare, then try to use a signed
10408 // range. Otherwise if both no-wraps are set, use the unsigned range
10409 // because it is never larger than the signed range. Example: "add nuw
10410 // nsw i8 X, -2" is unsigned [254,255] vs. signed [-128, 125].
10411 if (PreferSignedRange && HasNSW && HasNUW)
10412 HasNUW = false;
10413
10414 if (HasNUW) {
10415 // 'add nuw x, C' produces [C, UINT_MAX].
10416 Lower = *C;
10417 } else if (HasNSW) {
10418 if (C->isNegative()) {
10419 // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C].
10420 Lower = APInt::getSignedMinValue(numBits: Width);
10421 Upper = APInt::getSignedMaxValue(numBits: Width) + *C + 1;
10422 } else {
10423 // 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX].
10424 Lower = APInt::getSignedMinValue(numBits: Width) + *C;
10425 Upper = APInt::getSignedMaxValue(numBits: Width) + 1;
10426 }
10427 }
10428 }
10429 break;
10430
10431 case Instruction::And:
10432 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10433 // 'and x, C' produces [0, C].
10434 Upper = *C + 1;
10435 // X & -X is a power of two or zero. So we can cap the value at max power of
10436 // two.
10437 if (match(V: BO.getOperand(i_nocapture: 0), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 1)))) ||
10438 match(V: BO.getOperand(i_nocapture: 1), P: m_Neg(V: m_Specific(V: BO.getOperand(i_nocapture: 0)))))
10439 Upper = APInt::getSignedMinValue(numBits: Width) + 1;
10440 break;
10441
10442 case Instruction::Or:
10443 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10444 // 'or x, C' produces [C, UINT_MAX].
10445 Lower = *C;
10446 break;
10447
10448 case Instruction::AShr:
10449 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10450 // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C].
10451 Lower = APInt::getSignedMinValue(numBits: Width).ashr(ShiftAmt: *C);
10452 Upper = APInt::getSignedMaxValue(numBits: Width).ashr(ShiftAmt: *C) + 1;
10453 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10454 unsigned ShiftAmount = Width - 1;
10455 if (!C->isZero() && IIQ.isExact(Op: &BO))
10456 ShiftAmount = C->countr_zero();
10457 if (C->isNegative()) {
10458 // 'ashr C, x' produces [C, C >> (Width-1)]
10459 Lower = *C;
10460 Upper = C->ashr(ShiftAmt: ShiftAmount) + 1;
10461 } else {
10462 // 'ashr C, x' produces [C >> (Width-1), C]
10463 Lower = C->ashr(ShiftAmt: ShiftAmount);
10464 Upper = *C + 1;
10465 }
10466 }
10467 break;
10468
10469 case Instruction::LShr:
10470 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10471 // 'lshr x, C' produces [0, UINT_MAX >> C].
10472 Upper = APInt::getAllOnes(numBits: Width).lshr(ShiftAmt: *C) + 1;
10473 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10474 // 'lshr C, x' produces [C >> (Width-1), C].
10475 unsigned ShiftAmount = Width - 1;
10476 if (!C->isZero() && IIQ.isExact(Op: &BO))
10477 ShiftAmount = C->countr_zero();
10478 Lower = C->lshr(shiftAmt: ShiftAmount);
10479 Upper = *C + 1;
10480 }
10481 break;
10482
10483 case Instruction::Shl:
10484 if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10485 if (IIQ.hasNoUnsignedWrap(Op: &BO)) {
10486 // 'shl nuw C, x' produces [C, C << CLZ(C)]
10487 Lower = *C;
10488 Upper = Lower.shl(shiftAmt: Lower.countl_zero()) + 1;
10489 } else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw?
10490 if (C->isNegative()) {
10491 // 'shl nsw C, x' produces [C << CLO(C)-1, C]
10492 unsigned ShiftAmount = C->countl_one() - 1;
10493 Lower = C->shl(shiftAmt: ShiftAmount);
10494 Upper = *C + 1;
10495 } else {
10496 // 'shl nsw C, x' produces [C, C << CLZ(C)-1]
10497 unsigned ShiftAmount = C->countl_zero() - 1;
10498 Lower = *C;
10499 Upper = C->shl(shiftAmt: ShiftAmount) + 1;
10500 }
10501 } else {
10502 // If lowbit is set, value can never be zero.
10503 if ((*C)[0])
10504 Lower = APInt::getOneBitSet(numBits: Width, BitNo: 0);
10505 // If we are shifting a constant the largest it can be is if the longest
10506 // sequence of consecutive ones is shifted to the highbits (breaking
10507 // ties for which sequence is higher). At the moment we take a liberal
10508 // upper bound on this by just popcounting the constant.
10509 // TODO: There may be a bitwise trick for it longest/highest
10510 // consecutative sequence of ones (naive method is O(Width) loop).
10511 Upper = APInt::getHighBitsSet(numBits: Width, hiBitsSet: C->popcount()) + 1;
10512 }
10513 } else if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && C->ult(RHS: Width)) {
10514 Upper = APInt::getBitsSetFrom(numBits: Width, loBit: C->getZExtValue()) + 1;
10515 }
10516 break;
10517
10518 case Instruction::SDiv:
10519 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10520 APInt IntMin = APInt::getSignedMinValue(numBits: Width);
10521 APInt IntMax = APInt::getSignedMaxValue(numBits: Width);
10522 if (C->isAllOnes()) {
10523 // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX]
10524 // where C != -1 and C != 0 and C != 1
10525 Lower = IntMin + 1;
10526 Upper = IntMax + 1;
10527 } else if (C->countl_zero() < Width - 1) {
10528 // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C]
10529 // where C != -1 and C != 0 and C != 1
10530 Lower = IntMin.sdiv(RHS: *C);
10531 Upper = IntMax.sdiv(RHS: *C);
10532 if (Lower.sgt(RHS: Upper))
10533 std::swap(a&: Lower, b&: Upper);
10534 Upper = Upper + 1;
10535 assert(Upper != Lower && "Upper part of range has wrapped!");
10536 }
10537 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10538 if (C->isMinSignedValue()) {
10539 // 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2].
10540 Lower = *C;
10541 Upper = Lower.lshr(shiftAmt: 1) + 1;
10542 } else {
10543 // 'sdiv C, x' produces [-|C|, |C|].
10544 Upper = C->abs() + 1;
10545 Lower = (-Upper) + 1;
10546 }
10547 }
10548 break;
10549
10550 case Instruction::UDiv:
10551 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)) && !C->isZero()) {
10552 // 'udiv x, C' produces [0, UINT_MAX / C].
10553 Upper = APInt::getMaxValue(numBits: Width).udiv(RHS: *C) + 1;
10554 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10555 // 'udiv C, x' produces [0, C].
10556 Upper = *C + 1;
10557 }
10558 break;
10559
10560 case Instruction::SRem:
10561 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10562 // 'srem x, C' produces (-|C|, |C|).
10563 Upper = C->abs();
10564 Lower = (-Upper) + 1;
10565 } else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10566 if (C->isNegative()) {
10567 // 'srem -|C|, x' produces [-|C|, 0].
10568 Upper = 1;
10569 Lower = *C;
10570 } else {
10571 // 'srem |C|, x' produces [0, |C|].
10572 Upper = *C + 1;
10573 }
10574 }
10575 break;
10576
10577 case Instruction::URem:
10578 if (match(V: BO.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10579 // 'urem x, C' produces [0, C).
10580 Upper = *C;
10581 else if (match(V: BO.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10582 // 'urem C, x' produces [0, C].
10583 Upper = *C + 1;
10584 break;
10585
10586 default:
10587 break;
10588 }
10589}
10590
10591static ConstantRange getRangeForIntrinsic(const IntrinsicInst &II,
10592 bool UseInstrInfo) {
10593 unsigned Width = II.getType()->getScalarSizeInBits();
10594 const APInt *C;
10595 switch (II.getIntrinsicID()) {
10596 case Intrinsic::ctlz:
10597 case Intrinsic::cttz: {
10598 APInt Upper(Width, Width);
10599 if (!UseInstrInfo || !match(V: II.getArgOperand(i: 1), P: m_One()))
10600 Upper += 1;
10601 // Maximum of set/clear bits is the bit width.
10602 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper);
10603 }
10604 case Intrinsic::ctpop:
10605 // Maximum of set/clear bits is the bit width.
10606 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10607 Upper: APInt(Width, Width) + 1);
10608 case Intrinsic::uadd_sat:
10609 // uadd.sat(x, C) produces [C, UINT_MAX].
10610 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10611 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10612 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10613 break;
10614 case Intrinsic::sadd_sat:
10615 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) ||
10616 match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10617 if (C->isNegative())
10618 // sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)].
10619 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10620 Upper: APInt::getSignedMaxValue(numBits: Width) + *C +
10621 1);
10622
10623 // sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX].
10624 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) + *C,
10625 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10626 }
10627 break;
10628 case Intrinsic::usub_sat:
10629 // usub.sat(C, x) produces [0, C].
10630 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)))
10631 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10632
10633 // usub.sat(x, C) produces [0, UINT_MAX - C].
10634 if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10635 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10636 Upper: APInt::getMaxValue(numBits: Width) - *C + 1);
10637 break;
10638 case Intrinsic::ssub_sat:
10639 if (match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C))) {
10640 if (C->isNegative())
10641 // ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)].
10642 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10643 Upper: *C - APInt::getSignedMinValue(numBits: Width) +
10644 1);
10645
10646 // ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX].
10647 return ConstantRange::getNonEmpty(Lower: *C - APInt::getSignedMaxValue(numBits: Width),
10648 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10649 } else if (match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C))) {
10650 if (C->isNegative())
10651 // ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]:
10652 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width) - *C,
10653 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10654
10655 // ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C].
10656 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10657 Upper: APInt::getSignedMaxValue(numBits: Width) - *C +
10658 1);
10659 }
10660 break;
10661 case Intrinsic::umin:
10662 case Intrinsic::umax:
10663 case Intrinsic::smin:
10664 case Intrinsic::smax:
10665 if (!match(V: II.getOperand(i_nocapture: 0), P: m_APInt(Res&: C)) &&
10666 !match(V: II.getOperand(i_nocapture: 1), P: m_APInt(Res&: C)))
10667 break;
10668
10669 switch (II.getIntrinsicID()) {
10670 case Intrinsic::umin:
10671 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width), Upper: *C + 1);
10672 case Intrinsic::umax:
10673 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: Width));
10674 case Intrinsic::smin:
10675 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: Width),
10676 Upper: *C + 1);
10677 case Intrinsic::smax:
10678 return ConstantRange::getNonEmpty(Lower: *C,
10679 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10680 default:
10681 llvm_unreachable("Must be min/max intrinsic");
10682 }
10683 break;
10684 case Intrinsic::abs:
10685 // If abs of SIGNED_MIN is poison, then the result is [0..SIGNED_MAX],
10686 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10687 if (match(V: II.getOperand(i_nocapture: 1), P: m_One()))
10688 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10689 Upper: APInt::getSignedMaxValue(numBits: Width) + 1);
10690
10691 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: Width),
10692 Upper: APInt::getSignedMinValue(numBits: Width) + 1);
10693 case Intrinsic::vscale:
10694 if (!II.getParent() || !II.getFunction())
10695 break;
10696 return getVScaleRange(F: II.getFunction(), BitWidth: Width);
10697 case Intrinsic::read_register:
10698 case Intrinsic::read_volatile_register: {
10699 const Module *M = II.getModule();
10700 if (!M || !M->getTargetTriple().isRISCV())
10701 break;
10702 if (II.getFunction() && isReadVLENB(II))
10703 return getRISCVVLENBRange(II, Width);
10704 break;
10705 }
10706 default:
10707 break;
10708 }
10709
10710 return ConstantRange::getFull(BitWidth: Width);
10711}
10712
10713static ConstantRange getRangeForSelectPattern(const SelectInst &SI,
10714 const InstrInfoQuery &IIQ) {
10715 unsigned BitWidth = SI.getType()->getScalarSizeInBits();
10716 const Value *LHS = nullptr, *RHS = nullptr;
10717 SelectPatternResult R = matchSelectPattern(V: &SI, LHS, RHS);
10718 if (R.Flavor == SPF_UNKNOWN)
10719 return ConstantRange::getFull(BitWidth);
10720
10721 if (R.Flavor == SelectPatternFlavor::SPF_ABS) {
10722 // If the negation part of the abs (in RHS) has the NSW flag,
10723 // then the result of abs(X) is [0..SIGNED_MAX],
10724 // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN.
10725 if (match(V: RHS, P: m_Neg(V: m_Specific(V: LHS))) &&
10726 IIQ.hasNoSignedWrap(Op: cast<Instruction>(Val: RHS)))
10727 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10728 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10729
10730 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth),
10731 Upper: APInt::getSignedMinValue(numBits: BitWidth) + 1);
10732 }
10733
10734 if (R.Flavor == SelectPatternFlavor::SPF_NABS) {
10735 // The result of -abs(X) is <= 0.
10736 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10737 Upper: APInt(BitWidth, 1));
10738 }
10739
10740 const APInt *C;
10741 if (!match(V: LHS, P: m_APInt(Res&: C)) && !match(V: RHS, P: m_APInt(Res&: C)))
10742 return ConstantRange::getFull(BitWidth);
10743
10744 switch (R.Flavor) {
10745 case SPF_UMIN:
10746 return ConstantRange::getNonEmpty(Lower: APInt::getZero(numBits: BitWidth), Upper: *C + 1);
10747 case SPF_UMAX:
10748 return ConstantRange::getNonEmpty(Lower: *C, Upper: APInt::getZero(numBits: BitWidth));
10749 case SPF_SMIN:
10750 return ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
10751 Upper: *C + 1);
10752 case SPF_SMAX:
10753 return ConstantRange::getNonEmpty(Lower: *C,
10754 Upper: APInt::getSignedMaxValue(numBits: BitWidth) + 1);
10755 default:
10756 return ConstantRange::getFull(BitWidth);
10757 }
10758}
10759
10760static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) {
10761 // The maximum representable value of a half is 65504. For floats the maximum
10762 // value is 3.4e38 which requires roughly 129 bits.
10763 unsigned BitWidth = I->getType()->getScalarSizeInBits();
10764 if (!I->getOperand(i: 0)->getType()->getScalarType()->isHalfTy())
10765 return;
10766 if (isa<FPToSIInst>(Val: I) && BitWidth >= 17) {
10767 Lower = APInt(BitWidth, -65504, true);
10768 Upper = APInt(BitWidth, 65505);
10769 }
10770
10771 if (isa<FPToUIInst>(Val: I) && BitWidth >= 16) {
10772 // For a fptoui the lower limit is left as 0.
10773 Upper = APInt(BitWidth, 65505);
10774 }
10775}
10776
10777ConstantRange llvm::computeConstantRange(const Value *V, bool ForSigned,
10778 const SimplifyQuery &SQ,
10779 unsigned Depth) {
10780 assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction");
10781
10782 if (Depth == MaxAnalysisRecursionDepth)
10783 return ConstantRange::getFull(BitWidth: V->getType()->getScalarSizeInBits());
10784
10785 if (auto *C = dyn_cast<Constant>(Val: V))
10786 return C->toConstantRange();
10787
10788 unsigned BitWidth = V->getType()->getScalarSizeInBits();
10789 ConstantRange CR = ConstantRange::getFull(BitWidth);
10790 if (auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
10791 APInt Lower = APInt(BitWidth, 0);
10792 APInt Upper = APInt(BitWidth, 0);
10793 // TODO: Return ConstantRange.
10794 setLimitsForBinOp(BO: *BO, Lower, Upper, IIQ: SQ.IIQ, PreferSignedRange: ForSigned);
10795 CR = ConstantRange::getNonEmpty(Lower, Upper);
10796 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
10797 CR = getRangeForIntrinsic(II: *II, UseInstrInfo: SQ.IIQ.UseInstrInfo);
10798 else if (auto *SI = dyn_cast<SelectInst>(Val: V)) {
10799 ConstantRange CRTrue =
10800 computeConstantRange(V: SI->getTrueValue(), ForSigned, SQ, Depth: Depth + 1);
10801 ConstantRange CRFalse =
10802 computeConstantRange(V: SI->getFalseValue(), ForSigned, SQ, Depth: Depth + 1);
10803 CR = CRTrue.unionWith(CR: CRFalse);
10804 CR = CR.intersectWith(CR: getRangeForSelectPattern(SI: *SI, IIQ: SQ.IIQ));
10805 } else if (auto *TI = dyn_cast<TruncInst>(Val: V)) {
10806 ConstantRange SrcCR =
10807 computeConstantRange(V: TI->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
10808 CR = SrcCR.truncate(BitWidth);
10809 } else if (auto *ZExt = dyn_cast<ZExtInst>(Val: V)) {
10810 ConstantRange SrcCR =
10811 computeConstantRange(V: ZExt->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
10812 CR = SrcCR.zeroExtend(BitWidth);
10813 } else if (auto *SExt = dyn_cast<SExtInst>(Val: V)) {
10814 ConstantRange SrcCR =
10815 computeConstantRange(V: SExt->getOperand(i_nocapture: 0), ForSigned, SQ, Depth: Depth + 1);
10816 CR = SrcCR.signExtend(BitWidth);
10817 } else if (isa<FPToUIInst>(Val: V) || isa<FPToSIInst>(Val: V)) {
10818 APInt Lower = APInt(BitWidth, 0);
10819 APInt Upper = APInt(BitWidth, 0);
10820 // TODO: Return ConstantRange.
10821 setLimitForFPToI(I: cast<Instruction>(Val: V), Lower, Upper);
10822 CR = ConstantRange::getNonEmpty(Lower, Upper);
10823 } else if (const auto *A = dyn_cast<Argument>(Val: V))
10824 if (std::optional<ConstantRange> Range = A->getRange())
10825 CR = *Range;
10826
10827 if (auto *I = dyn_cast<Instruction>(Val: V)) {
10828 if (auto *Range = SQ.IIQ.getMetadata(I, KindID: LLVMContext::MD_range))
10829 CR = CR.intersectWith(CR: getConstantRangeFromMetadata(RangeMD: *Range));
10830
10831 Value *FrexpSrc;
10832 if (const auto *CB = dyn_cast<CallBase>(Val: V)) {
10833 if (std::optional<ConstantRange> Range = CB->getRange())
10834 CR = CR.intersectWith(CR: *Range);
10835 } else if (match(V: I, P: m_ExtractValue<1>(V: m_Intrinsic<Intrinsic::frexp>(
10836 Ops: m_Value(V&: FrexpSrc))))) {
10837 const fltSemantics &FltSem =
10838 FrexpSrc->getType()->getScalarType()->getFltSemantics();
10839 // It should be possible to implement this for any type, but this logic
10840 // only computes the range assuming standard subnormal handling.
10841 if (APFloat::isIEEELikeFP(FltSem)) {
10842 KnownFPClass KnownSrc = computeKnownFPClass(
10843 V: FrexpSrc, InterestedClasses: fcSubnormal | fcZero | fcNan | fcInf, SQ, Depth: Depth + 1);
10844
10845 // The exponent of frexp(NaN) and frexp(Inf) is unspecified. Only
10846 // constrain its range when the source can be neither.
10847 if (KnownSrc.isKnownNeverInfOrNaN()) {
10848 int MinExp = APFloat::semanticsMinExponent(FltSem) + 1;
10849
10850 // Offset to find the true minimum exponent value for a denormal.
10851 if (!KnownSrc.isKnownNeverSubnormal())
10852 MinExp -= (APFloat::semanticsPrecision(FltSem) - 1);
10853
10854 int MaxExp = APFloat::semanticsMaxExponent(FltSem) + 1;
10855
10856 auto [AdjustedMin, AdjustedMax, AdjustedMaxNonZero] =
10857 computeKnownExponentRangeFromContext(V: FrexpSrc, Q: SQ);
10858
10859 DenormalMode Mode = I->getFunction()->getDenormalMode(FPType: FltSem);
10860 bool NeverLogicalZero = KnownSrc.isKnownNeverLogicalZero(Mode);
10861
10862 MinExp = std::max(a: AdjustedMin, b: MinExp);
10863 MaxExp = std::min(a: NeverLogicalZero ? AdjustedMaxNonZero : AdjustedMax,
10864 b: MaxExp);
10865
10866 CR = ConstantRange::getNonEmpty(
10867 Lower: APInt(BitWidth, static_cast<int64_t>(MinExp), /*isSigned=*/true),
10868 Upper: APInt(BitWidth, static_cast<int64_t>(MaxExp) + 1,
10869 /*isSigned=*/true));
10870 }
10871 }
10872 }
10873 }
10874
10875 if (SQ.CtxI && SQ.AC) {
10876 // Try to restrict the range based on information from assumptions.
10877 for (auto &AssumeVH : SQ.AC->assumptionsFor(V)) {
10878 if (!AssumeVH)
10879 continue;
10880 CallInst *I = cast<CallInst>(Val&: AssumeVH);
10881 assert(I->getParent()->getParent() == SQ.CtxI->getParent()->getParent() &&
10882 "Got assumption for the wrong function!");
10883 assert(I->getIntrinsicID() == Intrinsic::assume &&
10884 "must be an assume intrinsic");
10885
10886 if (!isValidAssumeForContext(I, Q: SQ))
10887 continue;
10888 Value *Arg = I->getArgOperand(i: 0);
10889 ICmpInst *Cmp = dyn_cast<ICmpInst>(Val: Arg);
10890 // Currently we just use information from comparisons.
10891 if (!Cmp || Cmp->getOperand(i_nocapture: 0) != V)
10892 continue;
10893 // TODO: Set "ForSigned" parameter via Cmp->isSigned()?
10894 ConstantRange RHS =
10895 computeConstantRange(V: Cmp->getOperand(i_nocapture: 1), /*ForSigned=*/false,
10896 SQ: SQ.getWithInstruction(I), Depth: Depth + 1);
10897 CR = CR.intersectWith(
10898 CR: ConstantRange::makeAllowedICmpRegion(Pred: Cmp->getCmpPredicate(), Other: RHS));
10899 }
10900 }
10901
10902 return CR;
10903}
10904
10905static void
10906addValueAffectedByCondition(Value *V,
10907 function_ref<void(Value *)> InsertAffected) {
10908 assert(V != nullptr);
10909 if (isa<Argument>(Val: V) || isa<GlobalValue>(Val: V)) {
10910 InsertAffected(V);
10911 } else if (auto *I = dyn_cast<Instruction>(Val: V)) {
10912 InsertAffected(V);
10913
10914 // Peek through unary operators to find the source of the condition.
10915 Value *Op;
10916 if (match(V: I, P: m_CombineOr(Ps: m_PtrToIntOrAddr(Op: m_Value(V&: Op)),
10917 Ps: m_Trunc(Op: m_Value(V&: Op))))) {
10918 if (isa<Instruction>(Val: Op) || isa<Argument>(Val: Op))
10919 InsertAffected(Op);
10920 }
10921 }
10922}
10923
10924void llvm::findValuesAffectedByCondition(
10925 Value *Cond, bool IsAssume, function_ref<void(Value *)> InsertAffected) {
10926 auto AddAffected = [&InsertAffected](Value *V) {
10927 addValueAffectedByCondition(V, InsertAffected);
10928 };
10929
10930 auto AddCmpOperands = [&AddAffected, IsAssume](Value *LHS, Value *RHS) {
10931 if (IsAssume) {
10932 AddAffected(LHS);
10933 AddAffected(RHS);
10934 } else if (match(V: RHS, P: m_Constant()))
10935 AddAffected(LHS);
10936 };
10937
10938 SmallVector<Value *, 8> Worklist;
10939 SmallPtrSet<Value *, 8> Visited;
10940 Worklist.push_back(Elt: Cond);
10941 while (!Worklist.empty()) {
10942 Value *V = Worklist.pop_back_val();
10943 if (!Visited.insert(Ptr: V).second)
10944 continue;
10945
10946 CmpPredicate Pred;
10947 Value *A, *B, *X;
10948
10949 if (IsAssume) {
10950 AddAffected(V);
10951 if (match(V, P: m_Not(V: m_Value(V&: X))))
10952 AddAffected(X);
10953 }
10954
10955 if (match(V, P: m_LogicalOp(L: m_Value(V&: A), R: m_Value(V&: B)))) {
10956 // assume(A && B) is split to -> assume(A); assume(B);
10957 // assume(!(A || B)) is split to -> assume(!A); assume(!B);
10958 // Finally, assume(A || B) / assume(!(A && B)) generally don't provide
10959 // enough information to be worth handling (intersection of information as
10960 // opposed to union).
10961 if (!IsAssume) {
10962 Worklist.push_back(Elt: A);
10963 Worklist.push_back(Elt: B);
10964 }
10965 } else if (match(V, P: m_ICmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
10966 bool HasRHSC = match(V: B, P: m_ConstantInt());
10967 if (ICmpInst::isEquality(P: Pred)) {
10968 AddAffected(A);
10969 if (IsAssume)
10970 AddAffected(B);
10971 if (HasRHSC) {
10972 Value *Y;
10973 // (X << C) or (X >>_s C) or (X >>_u C).
10974 if (match(V: A, P: m_Shift(L: m_Value(V&: X), R: m_ConstantInt())))
10975 AddAffected(X);
10976 // (X & C) or (X | C).
10977 else if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
10978 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10979 AddAffected(X);
10980 AddAffected(Y);
10981 }
10982 // X - Y
10983 else if (match(V: A, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10984 AddAffected(X);
10985 AddAffected(Y);
10986 }
10987 }
10988 } else {
10989 AddCmpOperands(A, B);
10990 if (HasRHSC) {
10991 Value *Y;
10992 // Handle (A + C1) u< C2, which is the canonical form of
10993 // A > C3 && A < C4.
10994 if (match(V: A, P: m_AddLike(L: m_Value(V&: X), R: m_ConstantInt())))
10995 AddAffected(X);
10996 // X - Y pred C, which implies X != Y if 0 pred C is false.
10997 else if (match(V: A, P: m_Sub(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
10998 AddAffected(X);
10999 AddAffected(Y);
11000 }
11001
11002 if (ICmpInst::isUnsigned(Pred)) {
11003 // X & Y u> C -> X >u C && Y >u C
11004 // X | Y u< C -> X u< C && Y u< C
11005 // X nuw+ Y u< C -> X u< C && Y u< C
11006 if (match(V: A, P: m_And(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
11007 match(V: A, P: m_Or(L: m_Value(V&: X), R: m_Value(V&: Y))) ||
11008 match(V: A, P: m_NUWAdd(L: m_Value(V&: X), R: m_Value(V&: Y)))) {
11009 AddAffected(X);
11010 AddAffected(Y);
11011 }
11012 }
11013 }
11014
11015 // Handle icmp slt/sgt (bitcast X to int), 0/-1, which is supported
11016 // by computeKnownFPClass().
11017 if (match(V: A, P: m_ElementWiseBitCast(Op: m_Value(V&: X)))) {
11018 if (Pred == ICmpInst::ICMP_SLT && match(V: B, P: m_Zero()))
11019 InsertAffected(X);
11020 else if (Pred == ICmpInst::ICMP_SGT && match(V: B, P: m_AllOnes()))
11021 InsertAffected(X);
11022 }
11023 }
11024
11025 auto AddNuwSquareOperand = [&AddAffected](Value *Op) {
11026 Value *SquareOp = nullptr;
11027 if (match(V: Op, P: m_NUWMul(L: m_Value(V&: SquareOp), R: m_Deferred(V: SquareOp))))
11028 AddAffected(SquareOp);
11029 };
11030 AddNuwSquareOperand(A);
11031 AddNuwSquareOperand(B);
11032
11033 if (HasRHSC && match(V: A, P: m_Ctpop(Op0: m_Value(V&: X))))
11034 AddAffected(X);
11035 } else if (match(V, P: m_FCmp(Pred, L: m_Value(V&: A), R: m_Value(V&: B)))) {
11036 AddCmpOperands(A, B);
11037
11038 // fcmp fneg(x), y
11039 // fcmp fabs(x), y
11040 // fcmp fneg(fabs(x)), y
11041 if (match(V: A, P: m_FNeg(X: m_Value(V&: A))))
11042 AddAffected(A);
11043 if (match(V: A, P: m_FAbs(Op0: m_Value(V&: A))))
11044 AddAffected(A);
11045
11046 } else if (match(V, P: m_Intrinsic<Intrinsic::is_fpclass>(Ops: m_Value(V&: A),
11047 Ops: m_Value()))) {
11048 // Handle patterns that computeKnownFPClass() support.
11049 AddAffected(A);
11050 } else if (!IsAssume && match(V, P: m_Trunc(Op: m_Value(V&: X)))) {
11051 // Assume is checked here as X is already added above for assumes in
11052 // addValueAffectedByCondition
11053 AddAffected(X);
11054 } else if (!IsAssume && match(V, P: m_Not(V: m_Value(V&: X)))) {
11055 // Assume is checked here to avoid issues with ephemeral values
11056 Worklist.push_back(Elt: X);
11057 }
11058 }
11059}
11060
11061const Value *llvm::stripNullTest(const Value *V) {
11062 // (X >> C) or/add (X & mask(C) != 0)
11063 if (const auto *BO = dyn_cast<BinaryOperator>(Val: V)) {
11064 if (BO->getOpcode() == Instruction::Add ||
11065 BO->getOpcode() == Instruction::Or) {
11066 const Value *X;
11067 const APInt *C1, *C2;
11068 if (match(V: BO, P: m_c_BinOp(L: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: C1)),
11069 R: m_ZExt(Op: m_SpecificICmp(
11070 MatchPred: ICmpInst::ICMP_NE,
11071 L: m_And(L: m_Deferred(V: X), R: m_LowBitMask(V&: C2)),
11072 R: m_Zero())))) &&
11073 C2->popcount() == C1->getZExtValue())
11074 return X;
11075 }
11076 }
11077 return nullptr;
11078}
11079
11080Value *llvm::stripNullTest(Value *V) {
11081 return const_cast<Value *>(stripNullTest(V: const_cast<const Value *>(V)));
11082}
11083
11084bool llvm::collectPossibleValues(const Value *V,
11085 SmallPtrSetImpl<const Constant *> &Constants,
11086 unsigned MaxCount, bool AllowUndefOrPoison) {
11087 SmallPtrSet<const Instruction *, 8> Visited;
11088 SmallVector<const Instruction *, 8> Worklist;
11089 auto Push = [&](const Value *V) -> bool {
11090 Constant *C;
11091 if (match(V: const_cast<Value *>(V), P: m_ImmConstant(C))) {
11092 if (!AllowUndefOrPoison && !isGuaranteedNotToBeUndefOrPoison(V: C))
11093 return false;
11094 // Check existence first to avoid unnecessary allocations.
11095 if (Constants.contains(Ptr: C))
11096 return true;
11097 if (Constants.size() == MaxCount)
11098 return false;
11099 Constants.insert(Ptr: C);
11100 return true;
11101 }
11102
11103 if (auto *Inst = dyn_cast<Instruction>(Val: V)) {
11104 if (Visited.insert(Ptr: Inst).second)
11105 Worklist.push_back(Elt: Inst);
11106 return true;
11107 }
11108 return false;
11109 };
11110 if (!Push(V))
11111 return false;
11112 while (!Worklist.empty()) {
11113 const Instruction *CurInst = Worklist.pop_back_val();
11114 switch (CurInst->getOpcode()) {
11115 case Instruction::Select:
11116 if (!Push(CurInst->getOperand(i: 1)))
11117 return false;
11118 if (!Push(CurInst->getOperand(i: 2)))
11119 return false;
11120 break;
11121 case Instruction::PHI:
11122 for (Value *IncomingValue : cast<PHINode>(Val: CurInst)->incoming_values()) {
11123 // Fast path for recurrence PHI.
11124 if (IncomingValue == CurInst)
11125 continue;
11126 if (!Push(IncomingValue))
11127 return false;
11128 }
11129 break;
11130 default:
11131 return false;
11132 }
11133 }
11134 return true;
11135}
11136