1//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the scalar evolution analysis
10// engine, which is used primarily to analyze expressions involving induction
11// variables in loops.
12//
13// There are several aspects to this library. First is the representation of
14// scalar expressions, which are represented as subclasses of the SCEV class.
15// These classes are used to represent certain types of subexpressions that we
16// can handle. We only create one SCEV of a particular shape, so
17// pointer-comparisons for equality are legal.
18//
19// One important aspect of the SCEV objects is that they are never cyclic, even
20// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
21// the PHI node is one of the idioms that we can represent (e.g., a polynomial
22// recurrence) then we represent it directly as a recurrence node, otherwise we
23// represent it as a SCEVUnknown node.
24//
25// In addition to being able to represent expressions of various types, we also
26// have folders that are used to build the *canonical* representation for a
27// particular expression. These folders are capable of using a variety of
28// rewrite rules to simplify the expressions.
29//
30// Once the folders are defined, we can implement the more interesting
31// higher-level code, such as the code that recognizes PHI nodes of various
32// types, computes the execution count of a loop, etc.
33//
34// TODO: We should use these routines and value representations to implement
35// dependence analysis!
36//
37//===----------------------------------------------------------------------===//
38//
39// There are several good references for the techniques used in this analysis.
40//
41// Chains of recurrences -- a method to expedite the evaluation
42// of closed-form functions
43// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
44//
45// On computational properties of chains of recurrences
46// Eugene V. Zima
47//
48// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
49// Robert A. van Engelen
50//
51// Efficient Symbolic Analysis for Optimizing Compilers
52// Robert A. van Engelen
53//
54// Using the chains of recurrences algebra for data dependence testing and
55// induction variable substitution
56// MS Thesis, Johnie Birch
57//
58//===----------------------------------------------------------------------===//
59
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/ADT/APInt.h"
62#include "llvm/ADT/ArrayRef.h"
63#include "llvm/ADT/DenseMap.h"
64#include "llvm/ADT/DepthFirstIterator.h"
65#include "llvm/ADT/FoldingSet.h"
66#include "llvm/ADT/STLExtras.h"
67#include "llvm/ADT/ScopeExit.h"
68#include "llvm/ADT/Sequence.h"
69#include "llvm/ADT/SmallPtrSet.h"
70#include "llvm/ADT/SmallVector.h"
71#include "llvm/ADT/Statistic.h"
72#include "llvm/ADT/StringExtras.h"
73#include "llvm/ADT/StringRef.h"
74#include "llvm/Analysis/AssumptionCache.h"
75#include "llvm/Analysis/ConstantFolding.h"
76#include "llvm/Analysis/InstructionSimplify.h"
77#include "llvm/Analysis/LoopInfo.h"
78#include "llvm/Analysis/MemoryBuiltins.h"
79#include "llvm/Analysis/ScalarEvolutionExpressions.h"
80#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
81#include "llvm/Analysis/TargetLibraryInfo.h"
82#include "llvm/Analysis/ValueTracking.h"
83#include "llvm/Config/llvm-config.h"
84#include "llvm/IR/Argument.h"
85#include "llvm/IR/BasicBlock.h"
86#include "llvm/IR/CFG.h"
87#include "llvm/IR/Constant.h"
88#include "llvm/IR/ConstantRange.h"
89#include "llvm/IR/Constants.h"
90#include "llvm/IR/DataLayout.h"
91#include "llvm/IR/DerivedTypes.h"
92#include "llvm/IR/Dominators.h"
93#include "llvm/IR/Function.h"
94#include "llvm/IR/GlobalAlias.h"
95#include "llvm/IR/GlobalValue.h"
96#include "llvm/IR/InstIterator.h"
97#include "llvm/IR/InstrTypes.h"
98#include "llvm/IR/Instruction.h"
99#include "llvm/IR/Instructions.h"
100#include "llvm/IR/IntrinsicInst.h"
101#include "llvm/IR/Intrinsics.h"
102#include "llvm/IR/LLVMContext.h"
103#include "llvm/IR/Operator.h"
104#include "llvm/IR/PatternMatch.h"
105#include "llvm/IR/Type.h"
106#include "llvm/IR/Use.h"
107#include "llvm/IR/User.h"
108#include "llvm/IR/Value.h"
109#include "llvm/IR/Verifier.h"
110#include "llvm/InitializePasses.h"
111#include "llvm/Pass.h"
112#include "llvm/Support/Casting.h"
113#include "llvm/Support/CommandLine.h"
114#include "llvm/Support/Compiler.h"
115#include "llvm/Support/Debug.h"
116#include "llvm/Support/ErrorHandling.h"
117#include "llvm/Support/InterleavedRange.h"
118#include "llvm/Support/KnownBits.h"
119#include "llvm/Support/SaveAndRestore.h"
120#include "llvm/Support/raw_ostream.h"
121#include <algorithm>
122#include <cassert>
123#include <climits>
124#include <cstdint>
125#include <cstdlib>
126#include <map>
127#include <memory>
128#include <numeric>
129#include <optional>
130#include <tuple>
131#include <utility>
132#include <vector>
133
134using namespace llvm;
135using namespace PatternMatch;
136using namespace SCEVPatternMatch;
137
138#define DEBUG_TYPE "scalar-evolution"
139
140STATISTIC(NumExitCountsComputed,
141 "Number of loop exits with predictable exit counts");
142STATISTIC(NumExitCountsNotComputed,
143 "Number of loop exits without predictable exit counts");
144STATISTIC(NumBruteForceTripCountsComputed,
145 "Number of loops with trip counts computed by force");
146
147#ifdef EXPENSIVE_CHECKS
148bool llvm::VerifySCEV = true;
149#else
150bool llvm::VerifySCEV = false;
151#endif
152
153static cl::opt<unsigned>
154 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
155 cl::desc("Maximum number of iterations SCEV will "
156 "symbolically execute a constant "
157 "derived loop"),
158 cl::init(Val: 100));
159
160static cl::opt<bool, true> VerifySCEVOpt(
161 "verify-scev", cl::Hidden, cl::location(L&: VerifySCEV),
162 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
163static cl::opt<bool> VerifySCEVStrict(
164 "verify-scev-strict", cl::Hidden,
165 cl::desc("Enable stricter verification with -verify-scev is passed"));
166
167static cl::opt<bool> VerifyIR(
168 "scev-verify-ir", cl::Hidden,
169 cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"),
170 cl::init(Val: false));
171
172static cl::opt<unsigned> MulOpsInlineThreshold(
173 "scev-mulops-inline-threshold", cl::Hidden,
174 cl::desc("Threshold for inlining multiplication operands into a SCEV"),
175 cl::init(Val: 32));
176
177static cl::opt<unsigned> AddOpsInlineThreshold(
178 "scev-addops-inline-threshold", cl::Hidden,
179 cl::desc("Threshold for inlining addition operands into a SCEV"),
180 cl::init(Val: 500));
181
182static cl::opt<unsigned> MaxSCEVCompareDepth(
183 "scalar-evolution-max-scev-compare-depth", cl::Hidden,
184 cl::desc("Maximum depth of recursive SCEV complexity comparisons"),
185 cl::init(Val: 32));
186
187static cl::opt<unsigned> MaxSCEVOperationsImplicationDepth(
188 "scalar-evolution-max-scev-operations-implication-depth", cl::Hidden,
189 cl::desc("Maximum depth of recursive SCEV operations implication analysis"),
190 cl::init(Val: 2));
191
192static cl::opt<unsigned> MaxValueCompareDepth(
193 "scalar-evolution-max-value-compare-depth", cl::Hidden,
194 cl::desc("Maximum depth of recursive value complexity comparisons"),
195 cl::init(Val: 2));
196
197static cl::opt<unsigned>
198 MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden,
199 cl::desc("Maximum depth of recursive arithmetics"),
200 cl::init(Val: 32));
201
202static cl::opt<unsigned> MaxConstantEvolvingDepth(
203 "scalar-evolution-max-constant-evolving-depth", cl::Hidden,
204 cl::desc("Maximum depth of recursive constant evolving"), cl::init(Val: 32));
205
206static cl::opt<unsigned>
207 MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden,
208 cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"),
209 cl::init(Val: 8));
210
211static cl::opt<unsigned>
212 MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden,
213 cl::desc("Max coefficients in AddRec during evolving"),
214 cl::init(Val: 8));
215
216static cl::opt<unsigned>
217 HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden,
218 cl::desc("Size of the expression which is considered huge"),
219 cl::init(Val: 4096));
220
221static cl::opt<unsigned> RangeIterThreshold(
222 "scev-range-iter-threshold", cl::Hidden,
223 cl::desc("Threshold for switching to iteratively computing SCEV ranges"),
224 cl::init(Val: 32));
225
226static cl::opt<unsigned> MaxLoopGuardCollectionDepth(
227 "scalar-evolution-max-loop-guard-collection-depth", cl::Hidden,
228 cl::desc("Maximum depth for recursive loop guard collection"), cl::init(Val: 1));
229
230static cl::opt<bool>
231ClassifyExpressions("scalar-evolution-classify-expressions",
232 cl::Hidden, cl::init(Val: true),
233 cl::desc("When printing analysis, include information on every instruction"));
234
235static cl::opt<bool> UseExpensiveRangeSharpening(
236 "scalar-evolution-use-expensive-range-sharpening", cl::Hidden,
237 cl::init(Val: false),
238 cl::desc("Use more powerful methods of sharpening expression ranges. May "
239 "be costly in terms of compile time"));
240
241static cl::opt<bool>
242 EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden,
243 cl::desc("Handle <= and >= in finite loops"),
244 cl::init(Val: true));
245
246static cl::opt<bool> UseContextForNoWrapFlagInference(
247 "scalar-evolution-use-context-for-no-wrap-flag-strenghening", cl::Hidden,
248 cl::desc("Infer nuw/nsw flags using context where suitable"),
249 cl::init(Val: true));
250
251//===----------------------------------------------------------------------===//
252// SCEV class definitions
253//===----------------------------------------------------------------------===//
254
255void SCEV::computeAndSetCanonical(ScalarEvolution &SE) {
256 // Leaf nodes are always their own canonical.
257 switch (getSCEVType()) {
258 case scConstant:
259 case scVScale:
260 case scUnknown:
261 CanonicalSCEV = this;
262 return;
263 default:
264 break;
265 }
266
267 // For all other expressions, check whether any immediate operand has a
268 // different canonical. Since operands are always created before their parent,
269 // their canonical pointers are already set — no recursion needed.
270 bool Changed = false;
271 SmallVector<SCEVUse, 4> CanonOps;
272 for (SCEVUse Op : operands()) {
273 CanonOps.push_back(Elt: Op->getCanonical());
274 Changed |= CanonOps.back() != Op;
275 }
276
277 if (!Changed) {
278 CanonicalSCEV = this;
279 return;
280 }
281
282 // Rebuild the expression from the canonical operands, stripping use flags.
283 CanonicalSCEV = SE.getWithOperands(S: this, NewOps&: CanonOps);
284}
285
286//===----------------------------------------------------------------------===//
287// Implementation of the SCEV class.
288//
289
290#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
291LLVM_DUMP_METHOD void SCEV::dump() const {
292 print(dbgs());
293 dbgs() << '\n';
294}
295#endif
296
297void SCEV::print(raw_ostream &OS) const {
298 switch (getSCEVType()) {
299 case scConstant:
300 cast<SCEVConstant>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
301 return;
302 case scVScale:
303 OS << "vscale";
304 return;
305 case scPtrToAddr: {
306 const SCEVCastExpr *PtrCast = cast<SCEVCastExpr>(Val: this);
307 SCEVUse Op = PtrCast->getOperand();
308 OS << "(ptrtoaddr " << *Op->getType() << " " << Op << " to "
309 << *PtrCast->getType() << ")";
310 return;
311 }
312 case scTruncate: {
313 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: this);
314 SCEVUse Op = Trunc->getOperand();
315 OS << "(trunc " << *Op->getType() << " " << Op << " to "
316 << *Trunc->getType() << ")";
317 return;
318 }
319 case scZeroExtend: {
320 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: this);
321 SCEVUse Op = ZExt->getOperand();
322 OS << "(zext " << *Op->getType() << " " << Op << " to " << *ZExt->getType()
323 << ")";
324 return;
325 }
326 case scSignExtend: {
327 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: this);
328 SCEVUse Op = SExt->getOperand();
329 OS << "(sext " << *Op->getType() << " " << Op << " to " << *SExt->getType()
330 << ")";
331 return;
332 }
333 case scAddRecExpr: {
334 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: this);
335 OS << "{" << AR->getOperand(i: 0);
336 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
337 OS << ",+," << AR->getOperand(i);
338 OS << "}<";
339 if (AR->hasNoUnsignedWrap())
340 OS << "nuw><";
341 if (AR->hasNoSignedWrap())
342 OS << "nsw><";
343 if (AR->hasNoSelfWrap() && !AR->hasNoUnsignedWrap() &&
344 !AR->hasNoSignedWrap())
345 OS << "nw><";
346 AR->getLoop()->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
347 OS << ">";
348 return;
349 }
350 case scAddExpr:
351 case scMulExpr:
352 case scUMaxExpr:
353 case scSMaxExpr:
354 case scUMinExpr:
355 case scSMinExpr:
356 case scSequentialUMinExpr: {
357 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(Val: this);
358 const char *OpStr = nullptr;
359 switch (NAry->getSCEVType()) {
360 case scAddExpr: OpStr = " + "; break;
361 case scMulExpr: OpStr = " * "; break;
362 case scUMaxExpr: OpStr = " umax "; break;
363 case scSMaxExpr: OpStr = " smax "; break;
364 case scUMinExpr:
365 OpStr = " umin ";
366 break;
367 case scSMinExpr:
368 OpStr = " smin ";
369 break;
370 case scSequentialUMinExpr:
371 OpStr = " umin_seq ";
372 break;
373 default:
374 llvm_unreachable("There are no other nary expression types.");
375 }
376 OS << "(" << llvm::interleaved(R: NAry->operands(), Separator: OpStr) << ")";
377 switch (NAry->getSCEVType()) {
378 case scAddExpr:
379 case scMulExpr:
380 if (NAry->hasNoUnsignedWrap())
381 OS << "<nuw>";
382 if (NAry->hasNoSignedWrap())
383 OS << "<nsw>";
384 break;
385 default:
386 // Nothing to print for other nary expressions.
387 break;
388 }
389 return;
390 }
391 case scUDivExpr: {
392 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: this);
393 OS << "(" << UDiv->getLHS() << " /u " << UDiv->getRHS() << ")";
394 return;
395 }
396 case scUnknown:
397 cast<SCEVUnknown>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
398 return;
399 case scCouldNotCompute:
400 OS << "***COULDNOTCOMPUTE***";
401 return;
402 }
403 llvm_unreachable("Unknown SCEV kind!");
404}
405
406ArrayRef<SCEVUse> SCEV::operands() const {
407 switch (getSCEVType()) {
408 case scConstant:
409 case scVScale:
410 case scUnknown:
411 return {};
412 case scPtrToAddr:
413 case scTruncate:
414 case scZeroExtend:
415 case scSignExtend:
416 return cast<SCEVCastExpr>(Val: this)->operands();
417 case scAddRecExpr:
418 case scAddExpr:
419 case scMulExpr:
420 case scUMaxExpr:
421 case scSMaxExpr:
422 case scUMinExpr:
423 case scSMinExpr:
424 case scSequentialUMinExpr:
425 return cast<SCEVNAryExpr>(Val: this)->operands();
426 case scUDivExpr:
427 return cast<SCEVUDivExpr>(Val: this)->operands();
428 case scCouldNotCompute:
429 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
430 }
431 llvm_unreachable("Unknown SCEV kind!");
432}
433
434bool SCEV::isZero() const { return match(S: this, P: m_scev_Zero()); }
435
436bool SCEV::isOne() const { return match(S: this, P: m_scev_One()); }
437
438bool SCEV::isAllOnesValue() const { return match(S: this, P: m_scev_AllOnes()); }
439
440bool SCEV::isNonConstantNegative() const {
441 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: this);
442 if (!Mul) return false;
443
444 // If there is a constant factor, it will be first.
445 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: 0));
446 if (!SC) return false;
447
448 // Return true if the value is negative, this matches things like (-42 * V).
449 return SC->getAPInt().isNegative();
450}
451
452SCEVCouldNotCompute::SCEVCouldNotCompute()
453 : SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0, nullptr) {}
454
455bool SCEVCouldNotCompute::classof(const SCEV *S) {
456 return S->getSCEVType() == scCouldNotCompute;
457}
458
459const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
460 auto &Entry = ConstantSCEVs[V];
461 if (Entry)
462 return Entry;
463
464 FoldingSetNodeID ID;
465 ID.AddInteger(I: scConstant);
466 ID.AddPointer(Ptr: V);
467 FoldingSetInsertToken Token;
468 if (SCEVConstant *S =
469 static_cast<SCEVConstant *>(UniqueSCEVs.lookup(ID, Token)))
470 return Entry = S;
471 SCEVConstant *S =
472 new (SCEVAllocator) SCEVConstant(ID.Intern(Allocator&: SCEVAllocator), V);
473 UniqueSCEVs.insert(N: S, Token);
474 S->computeAndSetCanonical(SE&: *this);
475 return Entry = S;
476}
477
478const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
479 return getConstant(V: ConstantInt::get(Context&: getContext(), V: Val));
480}
481
482const SCEV *
483ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
484 IntegerType *ITy = cast<IntegerType>(Val: getEffectiveSCEVType(Ty));
485 // TODO: Avoid implicit trunc?
486 // See https://github.com/llvm/llvm-project/issues/112510.
487 return getConstant(
488 V: ConstantInt::get(Ty: ITy, V, IsSigned: isSigned, /*ImplicitTrunc=*/true));
489}
490
491const SCEV *ScalarEvolution::getVScale(Type *Ty) {
492 FoldingSetNodeID ID;
493 ID.AddInteger(I: scVScale);
494 ID.AddPointer(Ptr: Ty);
495 FoldingSetInsertToken Token;
496 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
497 return S;
498 SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(Allocator&: SCEVAllocator), Ty);
499 UniqueSCEVs.insert(N: S, Token);
500 S->computeAndSetCanonical(SE&: *this);
501 return S;
502}
503
504const SCEV *ScalarEvolution::getElementCount(Type *Ty, ElementCount EC,
505 SCEV::NoWrapFlags Flags) {
506 const SCEV *Res = getConstant(Ty, V: EC.getKnownMinValue());
507 if (EC.isScalable())
508 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty), Flags);
509 return Res;
510}
511
512SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
513 SCEVUse op, Type *ty)
514 : SCEV(ID, SCEVTy, computeExpressionSize(Args: op), ty), Op(op) {}
515
516SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID,
517 const SCEV *Op, Type *ITy)
518 : SCEVCastExpr(ID, scPtrToAddr, Op, ITy) {
519 assert(getOperand()->getType()->isPointerTy() && getType()->isIntegerTy() &&
520 "Must be a non-bit-width-changing pointer-to-integer cast!");
521}
522
523SCEVIntegralCastExpr::SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID,
524 SCEVTypes SCEVTy, SCEVUse op,
525 Type *ty)
526 : SCEVCastExpr(ID, SCEVTy, op, ty) {}
527
528SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
529 Type *ty)
530 : SCEVIntegralCastExpr(ID, scTruncate, op, ty) {
531 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
532 "Cannot truncate non-integer value!");
533}
534
535SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
536 Type *ty)
537 : SCEVIntegralCastExpr(ID, scZeroExtend, op, ty) {
538 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
539 "Cannot zero extend non-integer value!");
540}
541
542SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
543 Type *ty)
544 : SCEVIntegralCastExpr(ID, scSignExtend, op, ty) {
545 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
546 "Cannot sign extend non-integer value!");
547}
548
549void SCEVUnknown::deleted() {
550 // Clear this SCEVUnknown from various maps.
551 SE->forgetMemoizedResults(SCEVs: {this});
552
553 // Remove this SCEVUnknown from the uniquing map.
554 SE->UniqueSCEVs.erase(N: this);
555
556 // Release the value.
557 setValPtr(nullptr);
558}
559
560void SCEVUnknown::allUsesReplacedWith(Value *New) {
561 // Clear this SCEVUnknown from various maps.
562 SE->forgetMemoizedResults(SCEVs: {this});
563
564 // Remove this SCEVUnknown from the uniquing map.
565 SE->UniqueSCEVs.erase(N: this);
566
567 // Replace the value pointer in case someone is still using this SCEVUnknown.
568 setValPtr(New);
569}
570
571//===----------------------------------------------------------------------===//
572// SCEV Utilities
573//===----------------------------------------------------------------------===//
574
575/// Compare the two values \p LV and \p RV in terms of their "complexity" where
576/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
577/// operands in SCEV expressions.
578static int CompareValueComplexity(const LoopInfo *const LI, Value *LV,
579 Value *RV, unsigned Depth) {
580 if (Depth > MaxValueCompareDepth)
581 return 0;
582
583 // Order pointer values after integer values. This helps SCEVExpander form
584 // GEPs.
585 bool LIsPointer = LV->getType()->isPointerTy(),
586 RIsPointer = RV->getType()->isPointerTy();
587 if (LIsPointer != RIsPointer)
588 return (int)LIsPointer - (int)RIsPointer;
589
590 // Compare getValueID values.
591 unsigned LID = LV->getValueID(), RID = RV->getValueID();
592 if (LID != RID)
593 return (int)LID - (int)RID;
594
595 // Sort arguments by their position.
596 if (const auto *LA = dyn_cast<Argument>(Val: LV)) {
597 const auto *RA = cast<Argument>(Val: RV);
598 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
599 return (int)LArgNo - (int)RArgNo;
600 }
601
602 if (const auto *LGV = dyn_cast<GlobalValue>(Val: LV)) {
603 const auto *RGV = cast<GlobalValue>(Val: RV);
604
605 if (auto L = LGV->getLinkage() - RGV->getLinkage())
606 return L;
607
608 const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
609 auto LT = GV->getLinkage();
610 return !(GlobalValue::isPrivateLinkage(Linkage: LT) ||
611 GlobalValue::isInternalLinkage(Linkage: LT));
612 };
613
614 // Use the names to distinguish the two values, but only if the
615 // names are semantically important.
616 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
617 return LGV->getName().compare(RHS: RGV->getName());
618 }
619
620 // For instructions, compare their loop depth, and their operand count. This
621 // is pretty loose.
622 if (const auto *LInst = dyn_cast<Instruction>(Val: LV)) {
623 const auto *RInst = cast<Instruction>(Val: RV);
624
625 // Compare loop depths.
626 const BasicBlock *LParent = LInst->getParent(),
627 *RParent = RInst->getParent();
628 if (LParent != RParent) {
629 unsigned LDepth = LI->getLoopDepth(BB: LParent),
630 RDepth = LI->getLoopDepth(BB: RParent);
631 if (LDepth != RDepth)
632 return (int)LDepth - (int)RDepth;
633 }
634
635 // Compare the number of operands.
636 unsigned LNumOps = LInst->getNumOperands(),
637 RNumOps = RInst->getNumOperands();
638 if (LNumOps != RNumOps)
639 return (int)LNumOps - (int)RNumOps;
640
641 for (unsigned Idx : seq(Size: LNumOps)) {
642 int Result = CompareValueComplexity(LI, LV: LInst->getOperand(i: Idx),
643 RV: RInst->getOperand(i: Idx), Depth: Depth + 1);
644 if (Result != 0)
645 return Result;
646 }
647 }
648
649 return 0;
650}
651
652// Return negative, zero, or positive, if LHS is less than, equal to, or greater
653// than RHS, respectively. A three-way result allows recursive comparisons to be
654// more efficient.
655// If the max analysis depth was reached, return std::nullopt, assuming we do
656// not know if they are equivalent for sure.
657static std::optional<int>
658CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
659 const SCEV *RHS, DominatorTree &DT, unsigned Depth = 0) {
660 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
661 if (LHS == RHS)
662 return 0;
663
664 // Primarily, sort the SCEVs by their getSCEVType().
665 SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
666 if (LType != RType)
667 return (int)LType - (int)RType;
668
669 if (Depth > MaxSCEVCompareDepth)
670 return std::nullopt;
671
672 // Aside from the getSCEVType() ordering, the particular ordering
673 // isn't very important except that it's beneficial to be consistent,
674 // so that (a + b) and (b + a) don't end up as different expressions.
675 switch (LType) {
676 case scUnknown: {
677 const SCEVUnknown *LU = cast<SCEVUnknown>(Val: LHS);
678 const SCEVUnknown *RU = cast<SCEVUnknown>(Val: RHS);
679
680 int X =
681 CompareValueComplexity(LI, LV: LU->getValue(), RV: RU->getValue(), Depth: Depth + 1);
682 return X;
683 }
684
685 case scConstant: {
686 const SCEVConstant *LC = cast<SCEVConstant>(Val: LHS);
687 const SCEVConstant *RC = cast<SCEVConstant>(Val: RHS);
688
689 // Compare constant values.
690 const APInt &LA = LC->getAPInt();
691 const APInt &RA = RC->getAPInt();
692 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
693 if (LBitWidth != RBitWidth)
694 return (int)LBitWidth - (int)RBitWidth;
695 return LA.ult(RHS: RA) ? -1 : 1;
696 }
697
698 case scVScale: {
699 const auto *LTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: LHS)->getType());
700 const auto *RTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: RHS)->getType());
701 return LTy->getBitWidth() - RTy->getBitWidth();
702 }
703
704 case scAddRecExpr: {
705 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(Val: LHS);
706 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(Val: RHS);
707
708 // There is always a dominance between two recs that are used by one SCEV,
709 // so we can safely sort recs by loop header dominance. We require such
710 // order in getAddExpr.
711 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
712 if (LLoop != RLoop) {
713 const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
714 assert(LHead != RHead && "Two loops share the same header?");
715 if (DT.dominates(A: LHead, B: RHead))
716 return 1;
717 assert(DT.dominates(RHead, LHead) &&
718 "No dominance between recurrences used by one SCEV?");
719 return -1;
720 }
721
722 [[fallthrough]];
723 }
724
725 case scTruncate:
726 case scZeroExtend:
727 case scSignExtend:
728 case scPtrToAddr:
729 case scAddExpr:
730 case scMulExpr:
731 case scUDivExpr:
732 case scSMaxExpr:
733 case scUMaxExpr:
734 case scSMinExpr:
735 case scUMinExpr:
736 case scSequentialUMinExpr: {
737 ArrayRef<SCEVUse> LOps = LHS->operands();
738 ArrayRef<SCEVUse> ROps = RHS->operands();
739
740 // Lexicographically compare n-ary-like expressions.
741 unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
742 if (LNumOps != RNumOps)
743 return (int)LNumOps - (int)RNumOps;
744
745 for (unsigned i = 0; i != LNumOps; ++i) {
746 auto X = CompareSCEVComplexity(LI, LHS: LOps[i].getPointer(),
747 RHS: ROps[i].getPointer(), DT, Depth: Depth + 1);
748 if (X != 0)
749 return X;
750 }
751 return 0;
752 }
753
754 case scCouldNotCompute:
755 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
756 }
757 llvm_unreachable("Unknown SCEV kind!");
758}
759
760/// Given a list of SCEV objects, order them by their complexity, and group
761/// objects of the same complexity together by value. When this routine is
762/// finished, we know that any duplicates in the vector are consecutive and that
763/// complexity is monotonically increasing.
764///
765/// Note that we go take special precautions to ensure that we get deterministic
766/// results from this routine. In other words, we don't want the results of
767/// this to depend on where the addresses of various SCEV objects happened to
768/// land in memory.
769static void GroupByComplexity(SmallVectorImpl<SCEVUse> &Ops, LoopInfo *LI,
770 DominatorTree &DT) {
771 if (Ops.size() < 2) return; // Noop
772
773 // Whether LHS has provably less complexity than RHS.
774 auto IsLessComplex = [&](SCEVUse LHS, SCEVUse RHS) {
775 auto Complexity = CompareSCEVComplexity(LI, LHS, RHS, DT);
776 return Complexity && *Complexity < 0;
777 };
778 if (Ops.size() == 2) {
779 // This is the common case, which also happens to be trivially simple.
780 // Special case it.
781 SCEVUse &LHS = Ops[0], &RHS = Ops[1];
782 if (IsLessComplex(RHS, LHS))
783 std::swap(a&: LHS, b&: RHS);
784 return;
785 }
786
787 // Do the rough sort by complexity.
788 llvm::stable_sort(
789 Range&: Ops, C: [&](SCEVUse LHS, SCEVUse RHS) { return IsLessComplex(LHS, RHS); });
790
791 // Now that we are sorted by complexity, group elements of the same
792 // complexity. Note that this is, at worst, N^2, but the vector is likely to
793 // be extremely short in practice. Note that we take this approach because we
794 // do not want to depend on the addresses of the objects we are grouping.
795 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
796 const SCEV *S = Ops[i];
797 unsigned Complexity = S->getSCEVType();
798
799 // If there are any objects of the same complexity and same value as this
800 // one, group them.
801 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
802 if (Ops[j] == S) { // Found a duplicate.
803 // Move it to immediately after i'th element.
804 std::swap(a&: Ops[i+1], b&: Ops[j]);
805 ++i; // no need to rescan it.
806 if (i == e-2) return; // Done!
807 }
808 }
809 }
810}
811
812/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
813/// least HugeExprThreshold nodes).
814static bool hasHugeExpression(ArrayRef<SCEVUse> Ops) {
815 return any_of(Range&: Ops, P: [](const SCEV *S) {
816 return S->getExpressionSize() >= HugeExprThreshold;
817 });
818}
819
820/// Performs a number of common optimizations on the passed \p Ops. If the
821/// whole expression reduces down to a single operand, it will be returned.
822///
823/// The following optimizations are performed:
824/// * Fold constants using the \p Fold function.
825/// * Remove identity constants satisfying \p IsIdentity.
826/// * If a constant satisfies \p IsAbsorber, return it.
827/// * Sort operands by complexity.
828template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
829static const SCEV *
830constantFoldAndGroupOps(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT,
831 SmallVectorImpl<SCEVUse> &Ops, FoldT Fold,
832 IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
833 const SCEVConstant *Folded = nullptr;
834 for (unsigned Idx = 0; Idx < Ops.size();) {
835 const SCEV *Op = Ops[Idx];
836 if (const auto *C = dyn_cast<SCEVConstant>(Val: Op)) {
837 if (!Folded)
838 Folded = C;
839 else
840 Folded = cast<SCEVConstant>(
841 SE.getConstant(Fold(Folded->getAPInt(), C->getAPInt())));
842 Ops.erase(CI: Ops.begin() + Idx);
843 continue;
844 }
845 ++Idx;
846 }
847
848 if (Ops.empty()) {
849 assert(Folded && "Must have folded value");
850 return Folded;
851 }
852
853 if (Folded && IsAbsorber(Folded->getAPInt()))
854 return Folded;
855
856 GroupByComplexity(Ops, LI: &LI, DT);
857 if (Folded && !IsIdentity(Folded->getAPInt()))
858 Ops.insert(I: Ops.begin(), Elt: Folded);
859
860 return Ops.size() == 1 ? Ops[0] : nullptr;
861}
862
863//===----------------------------------------------------------------------===//
864// Simple SCEV method implementations
865//===----------------------------------------------------------------------===//
866
867/// Compute BC(It, K). The result has width W. Assume, K > 0.
868static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
869 ScalarEvolution &SE,
870 Type *ResultTy) {
871 // Handle the simplest case efficiently.
872 if (K == 1)
873 return SE.getTruncateOrZeroExtend(V: It, Ty: ResultTy);
874
875 // We are using the following formula for BC(It, K):
876 //
877 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
878 //
879 // Suppose, W is the bitwidth of the return value. We must be prepared for
880 // overflow. Hence, we must assure that the result of our computation is
881 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
882 // safe in modular arithmetic.
883 //
884 // However, this code doesn't use exactly that formula; the formula it uses
885 // is something like the following, where T is the number of factors of 2 in
886 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
887 // exponentiation:
888 //
889 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
890 //
891 // This formula is trivially equivalent to the previous formula. However,
892 // this formula can be implemented much more efficiently. The trick is that
893 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
894 // arithmetic. To do exact division in modular arithmetic, all we have
895 // to do is multiply by the inverse. Therefore, this step can be done at
896 // width W.
897 //
898 // The next issue is how to safely do the division by 2^T. The way this
899 // is done is by doing the multiplication step at a width of at least W + T
900 // bits. This way, the bottom W+T bits of the product are accurate. Then,
901 // when we perform the division by 2^T (which is equivalent to a right shift
902 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
903 // truncated out after the division by 2^T.
904 //
905 // In comparison to just directly using the first formula, this technique
906 // is much more efficient; using the first formula requires W * K bits,
907 // but this formula less than W + K bits. Also, the first formula requires
908 // a division step, whereas this formula only requires multiplies and shifts.
909 //
910 // It doesn't matter whether the subtraction step is done in the calculation
911 // width or the input iteration count's width; if the subtraction overflows,
912 // the result must be zero anyway. We prefer here to do it in the width of
913 // the induction variable because it helps a lot for certain cases; CodeGen
914 // isn't smart enough to ignore the overflow, which leads to much less
915 // efficient code if the width of the subtraction is wider than the native
916 // register width.
917 //
918 // (It's possible to not widen at all by pulling out factors of 2 before
919 // the multiplication; for example, K=2 can be calculated as
920 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
921 // extra arithmetic, so it's not an obvious win, and it gets
922 // much more complicated for K > 3.)
923
924 // Protection from insane SCEVs; this bound is conservative,
925 // but it probably doesn't matter.
926 if (K > 1000)
927 return SE.getCouldNotCompute();
928
929 unsigned W = SE.getTypeSizeInBits(Ty: ResultTy);
930
931 // Calculate K! / 2^T and T; we divide out the factors of two before
932 // multiplying for calculating K! / 2^T to avoid overflow.
933 // Other overflow doesn't matter because we only care about the bottom
934 // W bits of the result.
935 APInt OddFactorial(W, 1);
936 unsigned T = 1;
937 for (unsigned i = 3; i <= K; ++i) {
938 unsigned TwoFactors = countr_zero(Val: i);
939 T += TwoFactors;
940 OddFactorial *= (i >> TwoFactors);
941 }
942
943 // We need at least W + T bits for the multiplication step
944 unsigned CalculationBits = W + T;
945
946 // Calculate 2^T, at width T+W.
947 APInt DivFactor = APInt::getOneBitSet(numBits: CalculationBits, BitNo: T);
948
949 // Calculate the multiplicative inverse of K! / 2^T;
950 // this multiplication factor will perform the exact division by
951 // K! / 2^T.
952 APInt MultiplyFactor = OddFactorial.multiplicativeInverse();
953
954 // Calculate the product, at width T+W
955 IntegerType *CalculationTy = IntegerType::get(C&: SE.getContext(),
956 NumBits: CalculationBits);
957 const SCEV *Dividend = SE.getTruncateOrZeroExtend(V: It, Ty: CalculationTy);
958 for (unsigned i = 1; i != K; ++i) {
959 const SCEV *S = SE.getMinusSCEV(LHS: It, RHS: SE.getConstant(Ty: It->getType(), V: i));
960 Dividend = SE.getMulExpr(LHS: Dividend,
961 RHS: SE.getTruncateOrZeroExtend(V: S, Ty: CalculationTy));
962 }
963
964 // Divide by 2^T
965 const SCEV *DivResult = SE.getUDivExpr(LHS: Dividend, RHS: SE.getConstant(Val: DivFactor));
966
967 // Truncate the result, and divide by K! / 2^T.
968
969 return SE.getMulExpr(LHS: SE.getConstant(Val: MultiplyFactor),
970 RHS: SE.getTruncateOrZeroExtend(V: DivResult, Ty: ResultTy));
971}
972
973/// Return the value of this chain of recurrences at the specified iteration
974/// number. We can evaluate this recurrence by multiplying each element in the
975/// chain by the binomial coefficient corresponding to it. In other words, we
976/// can evaluate {A,+,B,+,C,+,D} as:
977///
978/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
979///
980/// where BC(It, k) stands for binomial coefficient.
981const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
982 ScalarEvolution &SE) const {
983 return evaluateAtIteration(Operands: operands(), It, SE);
984}
985
986const SCEV *SCEVAddRecExpr::evaluateAtIteration(ArrayRef<SCEVUse> Operands,
987 const SCEV *It,
988 ScalarEvolution &SE) {
989 assert(Operands.size() > 0);
990 const SCEV *Result = Operands[0].getPointer();
991 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
992 // The computation is correct in the face of overflow provided that the
993 // multiplication is performed _after_ the evaluation of the binomial
994 // coefficient.
995 const SCEV *Coeff = BinomialCoefficient(It, K: i, SE, ResultTy: Result->getType());
996 if (isa<SCEVCouldNotCompute>(Val: Coeff))
997 return Coeff;
998
999 Result =
1000 SE.getAddExpr(LHS: Result, RHS: SE.getMulExpr(LHS: Operands[i].getPointer(), RHS: Coeff));
1001 }
1002 return Result;
1003}
1004
1005//===----------------------------------------------------------------------===//
1006// SCEV Expression folder implementations
1007//===----------------------------------------------------------------------===//
1008
1009/// The SCEVCastSinkingRewriter takes a scalar evolution expression,
1010/// which computes a pointer-typed value, and rewrites the whole expression
1011/// tree so that *all* the computations are done on integers, and the only
1012/// pointer-typed operands in the expression are SCEVUnknown.
1013/// The CreatePtrCast callback is invoked to create the actual conversion
1014/// (ptrtoint or ptrtoaddr) at the SCEVUnknown leaves.
1015class SCEVCastSinkingRewriter
1016 : public SCEVRewriteVisitor<SCEVCastSinkingRewriter> {
1017 using Base = SCEVRewriteVisitor<SCEVCastSinkingRewriter>;
1018 using ConversionFn = function_ref<const SCEV *(const SCEVUnknown *)>;
1019 Type *TargetTy;
1020 ConversionFn CreatePtrCast;
1021
1022public:
1023 SCEVCastSinkingRewriter(ScalarEvolution &SE, Type *TargetTy,
1024 ConversionFn CreatePtrCast)
1025 : Base(SE), TargetTy(TargetTy), CreatePtrCast(std::move(CreatePtrCast)) {}
1026
1027 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE,
1028 Type *TargetTy, ConversionFn CreatePtrCast) {
1029 SCEVCastSinkingRewriter Rewriter(SE, TargetTy, std::move(CreatePtrCast));
1030 return Rewriter.visit(S: Scev);
1031 }
1032
1033 const SCEV *visit(const SCEV *S) {
1034 Type *STy = S->getType();
1035 // If the expression is not pointer-typed, just keep it as-is.
1036 if (!STy->isPointerTy())
1037 return S;
1038 // Else, recursively sink the cast down into it.
1039 return Base::visit(S);
1040 }
1041
1042 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
1043 // Preserve wrap flags on rewritten SCEVAddExpr, which the default
1044 // implementation drops.
1045 SmallVector<SCEVUse, 2> Operands;
1046 bool Changed = false;
1047 for (SCEVUse Op : Expr->operands()) {
1048 Operands.push_back(Elt: visit(S: Op.getPointer()));
1049 Changed |= Op.getPointer() != Operands.back();
1050 }
1051 return !Changed ? Expr : SE.getAddExpr(Ops&: Operands, Flags: Expr->getNoWrapFlags());
1052 }
1053
1054 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
1055 assert(Expr->getType()->isPointerTy() &&
1056 "Should only reach pointer-typed SCEVUnknown's.");
1057 // Perform some basic constant folding. If the operand of the cast is a
1058 // null pointer, don't create a cast SCEV expression (that will be left
1059 // as-is), but produce a zero constant.
1060 if (isa<ConstantPointerNull>(Val: Expr->getValue()))
1061 return SE.getZero(Ty: TargetTy);
1062 return CreatePtrCast(Expr);
1063 }
1064};
1065
1066const SCEV *ScalarEvolution::getPtrToAddrExpr(const SCEV *Op) {
1067 assert(Op->getType()->isPointerTy() && "Op must be a pointer");
1068
1069 // Treat pointers with unstable representation conservatively, since the
1070 // address bits may change.
1071 if (DL.hasUnstableRepresentation(Ty: Op->getType()))
1072 return getCouldNotCompute();
1073
1074 Type *Ty = DL.getAddressType(PtrTy: Op->getType());
1075
1076 // Use the rewriter to sink the cast down to SCEVUnknown leaves.
1077 // The rewriter handles null pointer constant folding.
1078 const SCEV *IntOp = SCEVCastSinkingRewriter::rewrite(
1079 Scev: Op, SE&: *this, TargetTy: Ty, CreatePtrCast: [this, Ty](const SCEVUnknown *U) {
1080 FoldingSetNodeID ID;
1081 ID.AddInteger(I: scPtrToAddr);
1082 ID.AddPointer(Ptr: U);
1083 ID.AddPointer(Ptr: Ty);
1084 FoldingSetInsertToken Token;
1085 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1086 return S;
1087 SCEV *S = new (SCEVAllocator)
1088 SCEVPtrToAddrExpr(ID.Intern(Allocator&: SCEVAllocator), U, Ty);
1089 UniqueSCEVs.insert(N: S, Token);
1090 S->computeAndSetCanonical(SE&: *this);
1091 registerUser(User: S, Ops: U);
1092 return static_cast<const SCEV *>(S);
1093 });
1094 assert(IntOp->getType()->isIntegerTy() &&
1095 "We must have succeeded in sinking the cast, "
1096 "and ending up with an integer-typed expression!");
1097 return IntOp;
1098}
1099
1100const SCEV *ScalarEvolution::getTruncateExpr(SCEVUse Op, Type *Ty,
1101 unsigned Depth) {
1102 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
1103 "This is not a truncating conversion!");
1104 assert(isSCEVable(Ty) &&
1105 "This is not a conversion to a SCEVable type!");
1106 assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!");
1107 Ty = getEffectiveSCEVType(Ty);
1108
1109 FoldingSetNodeID ID;
1110 ID.AddInteger(I: scTruncate);
1111 ID.AddPointer(Ptr: Op.getOpaqueValue());
1112 ID.AddPointer(Ptr: Ty);
1113 FoldingSetInsertToken Token;
1114 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1115 return S;
1116
1117 // Fold if the operand is constant.
1118 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1119 return getConstant(
1120 V: cast<ConstantInt>(Val: ConstantExpr::getTrunc(C: SC->getValue(), Ty)));
1121
1122 // trunc(trunc(x)) --> trunc(x)
1123 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op))
1124 return getTruncateExpr(Op: ST->getOperand(), Ty, Depth: Depth + 1);
1125
1126 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
1127 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val&: Op))
1128 return getTruncateOrSignExtend(V: SS->getOperand(), Ty, Depth: Depth + 1);
1129
1130 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
1131 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1132 return getTruncateOrZeroExtend(V: SZ->getOperand(), Ty, Depth: Depth + 1);
1133
1134 if (Depth > MaxCastDepth) {
1135 SCEV *S =
1136 new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator), Op, Ty);
1137 UniqueSCEVs.insert(N: S, Token);
1138 S->computeAndSetCanonical(SE&: *this);
1139 registerUser(User: S, Ops: Op);
1140 return S;
1141 }
1142
1143 // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
1144 // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
1145 // if after transforming we have at most one truncate, not counting truncates
1146 // that replace other casts.
1147 if (isa<SCEVAddExpr>(Val: Op) || isa<SCEVMulExpr>(Val: Op)) {
1148 auto *CommOp = cast<SCEVCommutativeExpr>(Val&: Op);
1149 SmallVector<SCEVUse, 4> Operands;
1150 unsigned numTruncs = 0;
1151 for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1152 ++i) {
1153 const SCEV *S = getTruncateExpr(Op: CommOp->getOperand(i), Ty, Depth: Depth + 1);
1154 if (!isa<SCEVIntegralCastExpr>(Val: CommOp->getOperand(i)) &&
1155 isa<SCEVTruncateExpr>(Val: S))
1156 numTruncs++;
1157 Operands.push_back(Elt: S);
1158 }
1159 if (numTruncs < 2) {
1160 if (isa<SCEVAddExpr>(Val: Op))
1161 return getAddExpr(Ops&: Operands);
1162 if (isa<SCEVMulExpr>(Val: Op))
1163 return getMulExpr(Ops&: Operands);
1164 llvm_unreachable("Unexpected SCEV type for Op.");
1165 }
1166 // Although we checked in the beginning that ID is not in the cache, it is
1167 // possible that during recursion and different modification ID was inserted
1168 // into the cache. So if we find it, just return it.
1169 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1170 return S;
1171 }
1172
1173 // If the input value is a chrec scev, truncate the chrec's operands.
1174 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: Op)) {
1175 SmallVector<SCEVUse, 4> Operands;
1176 for (const SCEV *Op : AddRec->operands())
1177 Operands.push_back(Elt: getTruncateExpr(Op, Ty, Depth: Depth + 1));
1178 return getAddRecExpr(Operands, L: AddRec->getLoop(), Flags: SCEV::FlagAnyWrap);
1179 }
1180
1181 // Return zero if truncating to known zeros.
1182 uint32_t MinTrailingZeros = getMinTrailingZeros(S: Op);
1183 if (MinTrailingZeros >= getTypeSizeInBits(Ty))
1184 return getZero(Ty);
1185
1186 // The cast wasn't folded; create an explicit cast node. We can reuse
1187 // the existing insert position since if we get here, we won't have
1188 // made any changes which would invalidate it.
1189 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator),
1190 Op, Ty);
1191 UniqueSCEVs.insert(N: S, Token);
1192 S->computeAndSetCanonical(SE&: *this);
1193 registerUser(User: S, Ops: Op);
1194 return S;
1195}
1196
1197// Get the limit of a recurrence such that incrementing by Step cannot cause
1198// signed overflow as long as the value of the recurrence within the
1199// loop does not exceed this limit before incrementing.
1200static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
1201 ICmpInst::Predicate *Pred,
1202 ScalarEvolution *SE) {
1203 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1204 if (SE->isKnownPositive(S: Step)) {
1205 *Pred = ICmpInst::ICMP_SLT;
1206 return SE->getConstant(Val: APInt::getSignedMinValue(numBits: BitWidth) -
1207 SE->getSignedRangeMax(S: Step));
1208 }
1209 if (SE->isKnownNegative(S: Step)) {
1210 *Pred = ICmpInst::ICMP_SGT;
1211 return SE->getConstant(Val: APInt::getSignedMaxValue(numBits: BitWidth) -
1212 SE->getSignedRangeMin(S: Step));
1213 }
1214 return nullptr;
1215}
1216
1217// Get the limit of a recurrence such that incrementing by Step cannot cause
1218// unsigned overflow as long as the value of the recurrence within the loop does
1219// not exceed this limit before incrementing.
1220static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step,
1221 ICmpInst::Predicate *Pred,
1222 ScalarEvolution *SE) {
1223 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1224 *Pred = ICmpInst::ICMP_ULT;
1225
1226 return SE->getConstant(Val: APInt::getMinValue(numBits: BitWidth) -
1227 SE->getUnsignedRangeMax(S: Step));
1228}
1229
1230namespace {
1231
1232struct ExtendOpTraitsBase {
1233 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(SCEVUse, Type *,
1234 unsigned);
1235};
1236
1237// Used to make code generic over signed and unsigned overflow.
1238template <typename ExtendOp> struct ExtendOpTraits {
1239 // Members present:
1240 //
1241 // static const SCEV::NoWrapFlags WrapType;
1242 //
1243 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
1244 //
1245 // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1246 // ICmpInst::Predicate *Pred,
1247 // ScalarEvolution *SE);
1248};
1249
1250template <>
1251struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
1252 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW;
1253
1254 static const GetExtendExprTy GetExtendExpr;
1255
1256 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1257 ICmpInst::Predicate *Pred,
1258 ScalarEvolution *SE) {
1259 return getSignedOverflowLimitForStep(Step, Pred, SE);
1260 }
1261};
1262
1263const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1264 SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr;
1265
1266template <>
1267struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
1268 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW;
1269
1270 static const GetExtendExprTy GetExtendExpr;
1271
1272 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1273 ICmpInst::Predicate *Pred,
1274 ScalarEvolution *SE) {
1275 return getUnsignedOverflowLimitForStep(Step, Pred, SE);
1276 }
1277};
1278
1279const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1280 SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr;
1281
1282} // end anonymous namespace
1283
1284// The recurrence AR has been shown to have no signed/unsigned wrap or something
1285// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
1286// easily prove NSW/NUW for its preincrement or postincrement sibling. This
1287// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
1288// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
1289// expression "Step + sext/zext(PreIncAR)" is congruent with
1290// "sext/zext(PostIncAR)"
1291template <typename ExtendOpTy>
1292static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR,
1293 ScalarEvolution *SE, unsigned Depth) {
1294 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1295 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1296
1297 const Loop *L = AR->getLoop();
1298 const SCEV *Start = AR->getStart();
1299 const SCEV *Step = AR->getStepRecurrence(SE&: *SE);
1300
1301 // Check for a simple looking step prior to loop entry.
1302 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Val: Start);
1303 if (!SA)
1304 return nullptr;
1305
1306 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
1307 // subtraction is expensive. For this purpose, perform a quick and dirty
1308 // difference, by checking for Step in the operand list. Note, that
1309 // SA might have repeated ops, like %a + %a + ..., so only remove one.
1310 SmallVector<SCEVUse, 4> DiffOps(SA->operands());
1311 for (auto It = DiffOps.begin(); It != DiffOps.end(); ++It)
1312 if (*It == Step) {
1313 DiffOps.erase(CI: It);
1314 break;
1315 }
1316
1317 if (DiffOps.size() == SA->getNumOperands())
1318 return nullptr;
1319
1320 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
1321 // `Step`:
1322
1323 // 1. NSW/NUW flags on the step increment.
1324 auto PreStartFlags =
1325 ScalarEvolution::maskFlags(Flags: SA->getNoWrapFlags(), Mask: SCEV::FlagNUW);
1326 const SCEV *PreStart = SE->getAddExpr(Ops&: DiffOps, Flags: PreStartFlags);
1327 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
1328 Val: SE->getAddRecExpr(Start: PreStart, Step, L, Flags: SCEV::FlagAnyWrap));
1329
1330 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
1331 // "S+X does not sign/unsign-overflow".
1332 //
1333
1334 const SCEV *BECount = SE->getBackedgeTakenCount(L);
1335 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType)) &&
1336 !isa<SCEVCouldNotCompute>(Val: BECount) && SE->isKnownPositive(S: BECount))
1337 return PreStart;
1338
1339 // 2. Direct overflow check on the step operation's expression.
1340 unsigned BitWidth = SE->getTypeSizeInBits(Ty: AR->getType());
1341 Type *WideTy = IntegerType::get(C&: SE->getContext(), NumBits: BitWidth * 2);
1342 const SCEV *OperandExtendedStart =
1343 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
1344 (SE->*GetExtendExpr)(Step, WideTy, Depth));
1345 if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
1346 if (PreAR && any(AR->getNoWrapFlags(Mask: WrapType))) {
1347 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
1348 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
1349 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
1350 SE->setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(PreAR), Flags: WrapType);
1351 }
1352 return PreStart;
1353 }
1354
1355 // 3. Loop precondition.
1356 ICmpInst::Predicate Pred;
1357 const SCEV *OverflowLimit =
1358 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1359
1360 if (OverflowLimit &&
1361 SE->isLoopEntryGuardedByCond(L, Pred, LHS: PreStart, RHS: OverflowLimit))
1362 return PreStart;
1363
1364 return nullptr;
1365}
1366
1367// Get the normalized zero or sign extended expression for this AddRec's Start.
1368template <typename ExtendOpTy>
1369static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
1370 ScalarEvolution *SE,
1371 unsigned Depth) {
1372 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1373
1374 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, SE, Depth);
1375 if (!PreStart)
1376 return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
1377
1378 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(SE&: *SE), Ty,
1379 Depth),
1380 (SE->*GetExtendExpr)(PreStart, Ty, Depth));
1381}
1382
1383// Try to prove away overflow by looking at "nearby" add recurrences. A
1384// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
1385// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
1386//
1387// Formally:
1388//
1389// {S,+,X} == {S-T,+,X} + T
1390// => Ext({S,+,X}) == Ext({S-T,+,X} + T)
1391//
1392// If ({S-T,+,X} + T) does not overflow ... (1)
1393//
1394// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
1395//
1396// If {S-T,+,X} does not overflow ... (2)
1397//
1398// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
1399// == {Ext(S-T)+Ext(T),+,Ext(X)}
1400//
1401// If (S-T)+T does not overflow ... (3)
1402//
1403// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
1404// == {Ext(S),+,Ext(X)} == LHS
1405//
1406// Thus, if (1), (2) and (3) are true for some T, then
1407// Ext({S,+,X}) == {Ext(S),+,Ext(X)}
1408//
1409// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
1410// does not overflow" restricted to the 0th iteration. Therefore we only need
1411// to check for (1) and (2).
1412//
1413// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
1414// is `Delta` (defined below).
1415template <typename ExtendOpTy>
1416bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
1417 const SCEV *Step,
1418 const Loop *L) {
1419 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1420
1421 // We restrict `Start` to a constant to prevent SCEV from spending too much
1422 // time here. It is correct (but more expensive) to continue with a
1423 // non-constant `Start` and do a general SCEV subtraction to compute
1424 // `PreStart` below.
1425 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Val: Start);
1426 if (!StartC)
1427 return false;
1428
1429 APInt StartAI = StartC->getAPInt();
1430
1431 for (unsigned Delta : {-2, -1, 1, 2}) {
1432 const SCEV *PreStart = getConstant(Val: StartAI - Delta);
1433
1434 FoldingSetNodeID ID;
1435 ID.AddInteger(I: scAddRecExpr);
1436 ID.AddPointer(Ptr: PreStart);
1437 ID.AddPointer(Ptr: Step);
1438 ID.AddPointer(Ptr: L);
1439 FoldingSetInsertToken Token;
1440 const auto *PreAR =
1441 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.lookup(ID, Token));
1442
1443 // Give up if we don't already have the add recurrence we need because
1444 // actually constructing an add recurrence is relatively expensive.
1445 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType))) { // proves (2)
1446 const SCEV *DeltaS = getConstant(Ty: StartC->getType(), V: Delta);
1447 ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
1448 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1449 DeltaS, &Pred, this);
1450 if (Limit && isKnownPredicate(Pred, LHS: PreAR, RHS: Limit)) // proves (1)
1451 return true;
1452 }
1453 }
1454
1455 return false;
1456}
1457
1458// Finds an integer D for an expression (C + x + y + ...) such that the top
1459// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
1460// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
1461// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
1462// the (C + x + y + ...) expression is \p WholeAddExpr.
1463static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1464 const SCEVConstant *ConstantTerm,
1465 const SCEVAddExpr *WholeAddExpr) {
1466 const APInt &C = ConstantTerm->getAPInt();
1467 const unsigned BitWidth = C.getBitWidth();
1468 // Find number of trailing zeros of (x + y + ...) w/o the C first:
1469 uint32_t TZ = BitWidth;
1470 for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
1471 TZ = std::min(a: TZ, b: SE.getMinTrailingZeros(S: WholeAddExpr->getOperand(i: I)));
1472 if (TZ) {
1473 // Set D to be as many least significant bits of C as possible while still
1474 // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
1475 return TZ < BitWidth ? C.trunc(width: TZ).zext(width: BitWidth) : C;
1476 }
1477 return APInt(BitWidth, 0);
1478}
1479
1480// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
1481// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
1482// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
1483// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
1484static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1485 const APInt &ConstantStart,
1486 const SCEV *Step) {
1487 const unsigned BitWidth = ConstantStart.getBitWidth();
1488 const uint32_t TZ = SE.getMinTrailingZeros(S: Step);
1489 if (TZ)
1490 return TZ < BitWidth ? ConstantStart.trunc(width: TZ).zext(width: BitWidth)
1491 : ConstantStart;
1492 return APInt(BitWidth, 0);
1493}
1494
1495static void insertFoldCacheEntry(
1496 const ScalarEvolution::FoldID &ID, const SCEV *S,
1497 DenseMap<ScalarEvolution::FoldID, const SCEV *> &FoldCache,
1498 DenseMap<const SCEV *, SmallVector<ScalarEvolution::FoldID, 2>>
1499 &FoldCacheUser) {
1500 auto I = FoldCache.insert(KV: {ID, S});
1501 if (!I.second) {
1502 // Remove FoldCacheUser entry for ID when replacing an existing FoldCache
1503 // entry.
1504 auto &UserIDs = FoldCacheUser[I.first->second];
1505 assert(count(UserIDs, ID) == 1 && "unexpected duplicates in UserIDs");
1506 for (unsigned I = 0; I != UserIDs.size(); ++I)
1507 if (UserIDs[I] == ID) {
1508 std::swap(a&: UserIDs[I], b&: UserIDs.back());
1509 break;
1510 }
1511 UserIDs.pop_back();
1512 I.first->second = S;
1513 }
1514 FoldCacheUser[S].push_back(Elt: ID);
1515}
1516
1517const SCEV *ScalarEvolution::getZeroExtendExpr(SCEVUse Op, Type *Ty,
1518 unsigned Depth) {
1519 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1520 "This is not an extending conversion!");
1521 assert(isSCEVable(Ty) &&
1522 "This is not a conversion to a SCEVable type!");
1523 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1524 Ty = getEffectiveSCEVType(Ty);
1525
1526 FoldID ID(scZeroExtend, Op, Ty);
1527 if (const SCEV *S = FoldCache.lookup(Val: ID))
1528 return S;
1529
1530 const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
1531 if (!isa<SCEVZeroExtendExpr>(Val: S))
1532 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1533 return S;
1534}
1535
1536const SCEV *ScalarEvolution::getZeroExtendExprImpl(SCEVUse Op, Type *Ty,
1537 unsigned Depth) {
1538 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1539 "This is not an extending conversion!");
1540 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1541 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1542
1543 // Fold if the operand is constant.
1544 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1545 return getConstant(Val: SC->getAPInt().zext(width: getTypeSizeInBits(Ty)));
1546
1547 // zext(zext(x)) --> zext(x)
1548 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1549 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1550
1551 // If the operand is an affine AddRec with the no-unsigned-wrap flag, the
1552 // zero-extension distributes over the recurrence.
1553 const SCEV *Start, *Step;
1554 const Loop *L;
1555 if (Depth <= MaxCastDepth &&
1556 match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1557 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1558 if (AR->hasNoUnsignedWrap()) {
1559 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1560 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1561 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1562 }
1563 }
1564
1565 // Before doing any expensive analysis, check to see if we've already
1566 // computed a SCEV for this Op and Ty.
1567 FoldingSetNodeID ID;
1568 ID.AddInteger(I: scZeroExtend);
1569 ID.AddPointer(Ptr: Op.getOpaqueValue());
1570 ID.AddPointer(Ptr: Ty);
1571 FoldingSetInsertToken Token;
1572 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1573 return S;
1574 if (Depth > MaxCastDepth) {
1575 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1576 Op, Ty);
1577 UniqueSCEVs.insert(N: S, Token);
1578 S->computeAndSetCanonical(SE&: *this);
1579 registerUser(User: S, Ops: Op);
1580 return S;
1581 }
1582
1583 // zext(trunc(x)) --> zext(x) or x or trunc(x)
1584 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
1585 // It's possible the bits taken off by the truncate were all zero bits. If
1586 // so, we should be able to simplify this further.
1587 const SCEV *X = ST->getOperand();
1588 ConstantRange CR = getUnsignedRange(S: X);
1589 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1590 unsigned NewBits = getTypeSizeInBits(Ty);
1591 if (CR.truncate(BitWidth: TruncBits).zeroExtend(BitWidth: NewBits).contains(
1592 CR: CR.zextOrTrunc(BitWidth: NewBits)))
1593 return getTruncateOrZeroExtend(V: X, Ty, Depth);
1594 }
1595
1596 // If the input value is a chrec scev, and we can prove that the value
1597 // did not overflow the old, smaller, value, we can zero extend all of the
1598 // operands (often constants). This allows analysis of something like
1599 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
1600 if (match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1601 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1602 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1603
1604 // The no-unsigned-wrap case is handled before the uniquing lookup above.
1605
1606 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1607 // Note that this serves two purposes: It filters out loops that are
1608 // simply not analyzable, and it covers the case where this code is
1609 // being called from within backedge-taken count analysis, such that
1610 // attempting to ask for the backedge-taken count would likely result
1611 // in infinite recursion. In the later case, the analysis code will
1612 // cope with a conservative value, and it will take care to purge
1613 // that value once it has finished.
1614 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1615 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
1616 // Manually compute the final value for AR, checking for overflow.
1617
1618 // Check whether the backedge-taken count can be losslessly casted to
1619 // the addrec's type. The count is always unsigned.
1620 const SCEV *CastedMaxBECount =
1621 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
1622 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1623 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
1624 if (MaxBECount == RecastedMaxBECount) {
1625 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
1626 // Check whether Start+Step*MaxBECount has no unsigned overflow.
1627 const SCEV *ZMul =
1628 getMulExpr(LHS: CastedMaxBECount, RHS: Step, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1629 const SCEV *ZAdd = getZeroExtendExpr(
1630 Op: getAddExpr(LHS: Start, RHS: ZMul, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1), Ty: WideTy,
1631 Depth: Depth + 1);
1632 const SCEV *WideStart = getZeroExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
1633 const SCEV *WideMaxBECount =
1634 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
1635 const SCEV *OperandExtendedAdd =
1636 getAddExpr(LHS: WideStart,
1637 RHS: getMulExpr(LHS: WideMaxBECount,
1638 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1639 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
1640 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1641 if (ZAdd == OperandExtendedAdd) {
1642 // Cache knowledge of AR NUW, which is propagated to this AddRec.
1643 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1644 // Return the expression with the addrec on the outside.
1645 Start =
1646 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1647 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1648 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1649 }
1650 // Similar to above, only this time treat the step value as signed.
1651 // This covers loops that count down.
1652 OperandExtendedAdd =
1653 getAddExpr(LHS: WideStart,
1654 RHS: getMulExpr(LHS: WideMaxBECount,
1655 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1656 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
1657 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1658 if (ZAdd == OperandExtendedAdd) {
1659 // Cache knowledge of AR NW, which is propagated to this AddRec.
1660 // Negative step causes unsigned wrap, but it still can't self-wrap.
1661 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1662 // Return the expression with the addrec on the outside.
1663 Start =
1664 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1665 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1666 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1667 }
1668 }
1669 }
1670
1671 // Normally, in the cases we can prove no-overflow via a
1672 // backedge guarding condition, we can also compute a backedge
1673 // taken count for the loop. The exceptions are assumptions and
1674 // guards present in the loop -- SCEV is not great at exploiting
1675 // these to compute max backedge taken counts, but can still use
1676 // these to prove lack of overflow. Use this fact to avoid
1677 // doing extra work that may not pay off.
1678 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount) || HasGuards ||
1679 !AC.assumptions().empty()) {
1680
1681 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1682 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
1683 if (AR->hasNoUnsignedWrap()) {
1684 // Same as nuw case above - duplicated here to avoid a compile time
1685 // issue. It's not clear that the order of checks does matter, but
1686 // it's one of two issue possible causes for a change which was
1687 // reverted. Be conservative for the moment.
1688 Start =
1689 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1690 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1691 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1692 }
1693
1694 // For a negative step, we can extend the operands iff doing so only
1695 // traverses values in the range zext([0,UINT_MAX]).
1696 if (isKnownNegative(S: Step)) {
1697 const SCEV *N =
1698 getConstant(Val: APInt::getMaxValue(numBits: BitWidth) - getSignedRangeMin(S: Step));
1699 if (isLoopBackedgeGuardedByCond(L, Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N) ||
1700 isKnownOnEveryIteration(Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N)) {
1701 // Cache knowledge of AR NW, which is propagated to this
1702 // AddRec. Negative step causes unsigned wrap, but it
1703 // still can't self-wrap.
1704 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1705 // Return the expression with the addrec on the outside.
1706 Start =
1707 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1708 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1709 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1710 }
1711 }
1712 }
1713
1714 // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
1715 // if D + (C - D + Step * n) could be proven to not unsigned wrap
1716 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1717 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
1718 const APInt &C = SC->getAPInt();
1719 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
1720 if (D != 0) {
1721 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1722 const SCEV *SResidual =
1723 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
1724 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1725 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: SCEV::FlagNSW | SCEV::FlagNUW,
1726 Depth: Depth + 1);
1727 }
1728 }
1729
1730 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1731 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1732 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1733 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1734 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1735 }
1736 }
1737
1738 // zext(A % B) --> zext(A) % zext(B)
1739 {
1740 const SCEV *LHS;
1741 const SCEV *RHS;
1742 if (match(U: Op, P: m_scev_URem(LHS: m_SCEV(V&: LHS), RHS: m_SCEV(V&: RHS), SE&: *this)))
1743 return getURemExpr(LHS: getZeroExtendExpr(Op: LHS, Ty, Depth: Depth + 1),
1744 RHS: getZeroExtendExpr(Op: RHS, Ty, Depth: Depth + 1));
1745 }
1746
1747 // zext(A / B) --> zext(A) / zext(B).
1748 if (auto *Div = dyn_cast<SCEVUDivExpr>(Val&: Op))
1749 return getUDivExpr(LHS: getZeroExtendExpr(Op: Div->getLHS(), Ty, Depth: Depth + 1),
1750 RHS: getZeroExtendExpr(Op: Div->getRHS(), Ty, Depth: Depth + 1));
1751
1752 if (auto *SA = dyn_cast<SCEVAddExpr>(Val&: Op)) {
1753 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
1754 if (SA->hasNoUnsignedWrap()) {
1755 // If the addition does not unsign overflow then we can, by definition,
1756 // commute the zero extension with the addition operation.
1757 SmallVector<SCEVUse, 4> Ops;
1758 for (SCEVUse Op : SA->operands())
1759 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1760 return getAddExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1761 }
1762
1763 const APInt *C, *C2;
1764 // zext (C + A)<nsw> -> (sext(C) + sext(A))<nsw> if zext (C + A)<nsw> >=s 0.
1765 // Currently the non-negative check is done manually, as isKnownNonNegative
1766 // is too expensive.
1767 if (SA->hasNoSignedWrap() &&
1768 match(V: SA, P: m_scev_Add(Op0: m_scev_APInt(C),
1769 Op1: m_scev_SMax(Op0: m_scev_APInt(C&: C2), Op1: m_SCEV()))) &&
1770 C->isNegative() && !C->isMinSignedValue() && C2->sge(RHS: C->abs())) {
1771 assert(isKnownNonNegative(SA) && "incorrectly determined non-negative");
1772 return getAddExpr(LHS: getSignExtendExpr(Op: SA->getOperand(i: 0), Ty, Depth: Depth + 1),
1773 RHS: getSignExtendExpr(Op: SA->getOperand(i: 1), Ty, Depth: Depth + 1),
1774 Flags: SCEV::FlagNSW, Depth: Depth + 1);
1775 }
1776
1777 // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
1778 // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
1779 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1780 //
1781 // Often address arithmetics contain expressions like
1782 // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
1783 // This transformation is useful while proving that such expressions are
1784 // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
1785 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1786 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1787 if (D != 0) {
1788 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1789 const SCEV *SResidual =
1790 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagAnyWrap, Depth);
1791 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1792 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1793 Depth: Depth + 1);
1794 }
1795 }
1796 }
1797
1798 if (auto *SM = dyn_cast<SCEVMulExpr>(Val&: Op)) {
1799 // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
1800 if (SM->hasNoUnsignedWrap()) {
1801 // If the multiply does not unsign overflow then we can, by definition,
1802 // commute the zero extension with the multiply operation.
1803 SmallVector<SCEVUse, 4> Ops;
1804 for (SCEVUse Op : SM->operands())
1805 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1806 return getMulExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1807 }
1808
1809 // zext(2^K * (trunc X to iN)) to iM ->
1810 // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
1811 //
1812 // Proof:
1813 //
1814 // zext(2^K * (trunc X to iN)) to iM
1815 // = zext((trunc X to iN) << K) to iM
1816 // = zext((trunc X to i{N-K}) << K)<nuw> to iM
1817 // (because shl removes the top K bits)
1818 // = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
1819 // = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
1820 //
1821 const APInt *C;
1822 const SCEV *TruncRHS;
1823 if (match(V: SM,
1824 P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_scev_Trunc(Op0: m_SCEV(V&: TruncRHS)))) &&
1825 C->isPowerOf2()) {
1826 int NewTruncBits =
1827 getTypeSizeInBits(Ty: SM->getOperand(i: 1)->getType()) - C->logBase2();
1828 Type *NewTruncTy = IntegerType::get(C&: getContext(), NumBits: NewTruncBits);
1829 return getMulExpr(
1830 LHS: getZeroExtendExpr(Op: SM->getOperand(i: 0), Ty),
1831 RHS: getZeroExtendExpr(Op: getTruncateExpr(Op: TruncRHS, Ty: NewTruncTy), Ty),
1832 Flags: SCEV::FlagNUW, Depth: Depth + 1);
1833 }
1834 }
1835
1836 // zext(umin(x, y)) -> umin(zext(x), zext(y))
1837 // zext(umax(x, y)) -> umax(zext(x), zext(y))
1838 if (isa<SCEVUMinExpr>(Val: Op) || isa<SCEVUMaxExpr>(Val: Op)) {
1839 auto *MinMax = cast<SCEVMinMaxExpr>(Val&: Op);
1840 SmallVector<SCEVUse, 4> Operands;
1841 for (SCEVUse Operand : MinMax->operands())
1842 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1843 if (isa<SCEVUMinExpr>(Val: MinMax))
1844 return getUMinExpr(Operands);
1845 return getUMaxExpr(Operands);
1846 }
1847
1848 // zext(umin_seq(x, y)) -> umin_seq(zext(x), zext(y))
1849 if (auto *MinMax = dyn_cast<SCEVSequentialMinMaxExpr>(Val&: Op)) {
1850 assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
1851 SmallVector<SCEVUse, 4> Operands;
1852 for (SCEVUse Operand : MinMax->operands())
1853 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1854 return getUMinExpr(Operands, /*Sequential*/ true);
1855 }
1856
1857 // The cast wasn't folded; create an explicit cast node.
1858 // Recompute the insert position, as it may have been invalidated.
1859 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1860 return S;
1861 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1862 Op, Ty);
1863 UniqueSCEVs.insert(N: S, Token);
1864 S->computeAndSetCanonical(SE&: *this);
1865 registerUser(User: S, Ops: Op);
1866 return S;
1867}
1868
1869const SCEV *ScalarEvolution::getSignExtendExpr(SCEVUse Op, Type *Ty,
1870 unsigned Depth) {
1871 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1872 "This is not an extending conversion!");
1873 assert(isSCEVable(Ty) &&
1874 "This is not a conversion to a SCEVable type!");
1875 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1876 Ty = getEffectiveSCEVType(Ty);
1877
1878 FoldID ID(scSignExtend, Op, Ty);
1879 if (const SCEV *S = FoldCache.lookup(Val: ID))
1880 return S;
1881
1882 const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
1883 if (!isa<SCEVSignExtendExpr>(Val: S))
1884 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1885 return S;
1886}
1887
1888const SCEV *ScalarEvolution::getSignExtendExprImpl(SCEVUse Op, Type *Ty,
1889 unsigned Depth) {
1890 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1891 "This is not an extending conversion!");
1892 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1893 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1894 Ty = getEffectiveSCEVType(Ty);
1895
1896 // Fold if the operand is constant.
1897 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1898 return getConstant(Val: SC->getAPInt().sext(width: getTypeSizeInBits(Ty)));
1899
1900 // sext(sext(x)) --> sext(x)
1901 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val&: Op))
1902 return getSignExtendExpr(Op: SS->getOperand(), Ty, Depth: Depth + 1);
1903
1904 // sext(zext(x)) --> zext(x)
1905 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1906 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1907
1908 // If the operand is an affine AddRec with the no-signed-wrap flag, the
1909 // sign-extension distributes over the recurrence.
1910 const SCEV *Start, *Step;
1911 const Loop *L;
1912 if (Depth <= MaxCastDepth &&
1913 match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1914 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1915 if (AR->hasNoSignedWrap()) {
1916 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1917 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1918 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1919 }
1920 }
1921
1922 // Before doing any expensive analysis, check to see if we've already
1923 // computed a SCEV for this Op and Ty.
1924 FoldingSetNodeID ID;
1925 ID.AddInteger(I: scSignExtend);
1926 ID.AddPointer(Ptr: Op.getOpaqueValue());
1927 ID.AddPointer(Ptr: Ty);
1928 FoldingSetInsertToken Token;
1929 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1930 return S;
1931 // Limit recursion depth.
1932 if (Depth > MaxCastDepth) {
1933 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1934 Op, Ty);
1935 UniqueSCEVs.insert(N: S, Token);
1936 S->computeAndSetCanonical(SE&: *this);
1937 registerUser(User: S, Ops: Op);
1938 return S;
1939 }
1940
1941 // sext(trunc(x)) --> sext(x) or x or trunc(x)
1942 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
1943 // It's possible the bits taken off by the truncate were all sign bits. If
1944 // so, we should be able to simplify this further.
1945 const SCEV *X = ST->getOperand();
1946 ConstantRange CR = getSignedRange(S: X);
1947 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1948 unsigned NewBits = getTypeSizeInBits(Ty);
1949 if (CR.truncate(BitWidth: TruncBits).signExtend(BitWidth: NewBits).contains(
1950 CR: CR.sextOrTrunc(BitWidth: NewBits)))
1951 return getTruncateOrSignExtend(V: X, Ty, Depth);
1952 }
1953
1954 if (auto *SA = dyn_cast<SCEVAddExpr>(Val&: Op)) {
1955 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
1956 if (SA->hasNoSignedWrap()) {
1957 // If the addition does not sign overflow then we can, by definition,
1958 // commute the sign extension with the addition operation.
1959 SmallVector<SCEVUse, 4> Ops;
1960 for (SCEVUse Op : SA->operands())
1961 Ops.push_back(Elt: getSignExtendExpr(Op, Ty, Depth: Depth + 1));
1962 return getAddExpr(Ops, Flags: SCEV::FlagNSW, Depth: Depth + 1);
1963 }
1964
1965 // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
1966 // if D + (C - D + x + y + ...) could be proven to not signed wrap
1967 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1968 //
1969 // For instance, this will bring two seemingly different expressions:
1970 // 1 + sext(5 + 20 * %x + 24 * %y) and
1971 // sext(6 + 20 * %x + 24 * %y)
1972 // to the same form:
1973 // 2 + sext(4 + 20 * %x + 24 * %y)
1974 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1975 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1976 if (D != 0) {
1977 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1978 const SCEV *SResidual =
1979 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagAnyWrap, Depth);
1980 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1981 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1982 Depth: Depth + 1);
1983 }
1984 }
1985 }
1986 // If the input value is a chrec scev, and we can prove that the value
1987 // did not overflow the old, smaller, value, we can sign extend all of the
1988 // operands (often constants). This allows analysis of something like
1989 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
1990 if (match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1991 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1992 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1993
1994 // The no-signed-wrap case is handled before the uniquing lookup above.
1995
1996 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1997 // Note that this serves two purposes: It filters out loops that are
1998 // simply not analyzable, and it covers the case where this code is
1999 // being called from within backedge-taken count analysis, such that
2000 // attempting to ask for the backedge-taken count would likely result
2001 // in infinite recursion. In the later case, the analysis code will
2002 // cope with a conservative value, and it will take care to purge
2003 // that value once it has finished.
2004 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
2005 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
2006 // Manually compute the final value for AR, checking for
2007 // overflow.
2008
2009 // Check whether the backedge-taken count can be losslessly casted to
2010 // the addrec's type. The count is always unsigned.
2011 const SCEV *CastedMaxBECount =
2012 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
2013 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
2014 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
2015 if (MaxBECount == RecastedMaxBECount) {
2016 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
2017 // Check whether Start+Step*MaxBECount has no signed overflow.
2018 const SCEV *SMul =
2019 getMulExpr(LHS: CastedMaxBECount, RHS: Step, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2020 const SCEV *SAdd = getSignExtendExpr(
2021 Op: getAddExpr(LHS: Start, RHS: SMul, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1), Ty: WideTy,
2022 Depth: Depth + 1);
2023 const SCEV *WideStart = getSignExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
2024 const SCEV *WideMaxBECount =
2025 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
2026 const SCEV *OperandExtendedAdd =
2027 getAddExpr(LHS: WideStart,
2028 RHS: getMulExpr(LHS: WideMaxBECount,
2029 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2030 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2031 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2032 if (SAdd == OperandExtendedAdd) {
2033 // Cache knowledge of AR NSW, which is propagated to this AddRec.
2034 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2035 // Return the expression with the addrec on the outside.
2036 Start =
2037 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2038 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2039 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2040 }
2041 // Similar to above, only this time treat the step value as unsigned.
2042 // This covers loops that count up with an unsigned step.
2043 OperandExtendedAdd =
2044 getAddExpr(LHS: WideStart,
2045 RHS: getMulExpr(LHS: WideMaxBECount,
2046 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2047 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2048 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2049 if (SAdd == OperandExtendedAdd) {
2050 // If AR wraps around then
2051 //
2052 // abs(Step) * MaxBECount > unsigned-max(AR->getType())
2053 // => SAdd != OperandExtendedAdd
2054 //
2055 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
2056 // (SAdd == OperandExtendedAdd => AR is NW)
2057
2058 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
2059
2060 // Return the expression with the addrec on the outside.
2061 Start =
2062 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2063 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2064 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2065 }
2066 }
2067 }
2068
2069 auto NewFlags = proveNoSignedWrapViaInduction(AR);
2070 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
2071 if (AR->hasNoSignedWrap()) {
2072 // Same as nsw case above - duplicated here to avoid a compile time
2073 // issue. It's not clear that the order of checks does matter, but
2074 // it's one of two issue possible causes for a change which was
2075 // reverted. Be conservative for the moment.
2076 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2077 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2078 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2079 }
2080
2081 // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
2082 // if D + (C - D + Step * n) could be proven to not signed wrap
2083 // where D maximizes the number of trailing zeros of (C - D + Step * n)
2084 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
2085 const APInt &C = SC->getAPInt();
2086 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
2087 if (D != 0) {
2088 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
2089 const SCEV *SResidual =
2090 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
2091 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
2092 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
2093 Depth: Depth + 1);
2094 }
2095 }
2096
2097 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
2098 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2099 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2100 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2101 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2102 }
2103 }
2104
2105 // If the input value is provably positive and we could not simplify
2106 // away the sext build a zext instead.
2107 if (isKnownNonNegative(S: Op))
2108 return getZeroExtendExpr(Op, Ty, Depth: Depth + 1);
2109
2110 // sext(smin(x, y)) -> smin(sext(x), sext(y))
2111 // sext(smax(x, y)) -> smax(sext(x), sext(y))
2112 if (isa<SCEVSMinExpr>(Val: Op) || isa<SCEVSMaxExpr>(Val: Op)) {
2113 auto *MinMax = cast<SCEVMinMaxExpr>(Val&: Op);
2114 SmallVector<SCEVUse, 4> Operands;
2115 for (SCEVUse Operand : MinMax->operands())
2116 Operands.push_back(Elt: getSignExtendExpr(Op: Operand, Ty));
2117 if (isa<SCEVSMinExpr>(Val: MinMax))
2118 return getSMinExpr(Operands);
2119 return getSMaxExpr(Operands);
2120 }
2121
2122 // The cast wasn't folded; create an explicit cast node.
2123 // Recompute the insert position, as it may have been invalidated.
2124 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
2125 return S;
2126 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2127 Op, Ty);
2128 UniqueSCEVs.insert(N: S, Token);
2129 S->computeAndSetCanonical(SE&: *this);
2130 registerUser(User: S, Ops: Op);
2131 return S;
2132}
2133
2134const SCEV *ScalarEvolution::getCastExpr(SCEVTypes Kind, SCEVUse Op, Type *Ty) {
2135 switch (Kind) {
2136 case scTruncate:
2137 return getTruncateExpr(Op, Ty);
2138 case scZeroExtend:
2139 return getZeroExtendExpr(Op, Ty);
2140 case scSignExtend:
2141 return getSignExtendExpr(Op, Ty);
2142 case scPtrToAddr: {
2143 const SCEV *Expr = getPtrToAddrExpr(Op);
2144 assert(Expr->getType() == Ty && "requested type must match");
2145 return Expr;
2146 }
2147 default:
2148 llvm_unreachable("Not a SCEV cast expression!");
2149 }
2150}
2151
2152/// getAnyExtendExpr - Return a SCEV for the given operand extended with
2153/// unspecified bits out to the given type.
2154const SCEV *ScalarEvolution::getAnyExtendExpr(SCEVUse Op, Type *Ty) {
2155 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2156 "This is not an extending conversion!");
2157 assert(isSCEVable(Ty) &&
2158 "This is not a conversion to a SCEVable type!");
2159 Ty = getEffectiveSCEVType(Ty);
2160
2161 // Sign-extend negative constants.
2162 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
2163 if (SC->getAPInt().isNegative())
2164 return getSignExtendExpr(Op, Ty);
2165
2166 // Peel off a truncate cast.
2167 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
2168 const SCEV *NewOp = T->getOperand();
2169 if (getTypeSizeInBits(Ty: NewOp->getType()) < getTypeSizeInBits(Ty))
2170 return getAnyExtendExpr(Op: NewOp, Ty);
2171 return getTruncateOrNoop(V: NewOp, Ty);
2172 }
2173
2174 // Next try a zext cast. If the cast is folded, use it.
2175 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
2176 if (!isa<SCEVZeroExtendExpr>(Val: ZExt))
2177 return ZExt;
2178
2179 // Next try a sext cast. If the cast is folded, use it.
2180 const SCEV *SExt = getSignExtendExpr(Op, Ty);
2181 if (!isa<SCEVSignExtendExpr>(Val: SExt))
2182 return SExt;
2183
2184 // Force the cast to be folded into the operands of an addrec.
2185 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: Op)) {
2186 SmallVector<SCEVUse, 4> Ops;
2187 for (const SCEV *Op : AR->operands())
2188 Ops.push_back(Elt: getAnyExtendExpr(Op, Ty));
2189 return getAddRecExpr(Operands&: Ops, L: AR->getLoop(), Flags: SCEV::FlagNW);
2190 }
2191
2192 // If the expression is obviously signed, use the sext cast value.
2193 if (isa<SCEVSMaxExpr>(Val: Op))
2194 return SExt;
2195
2196 // Absent any other information, use the zext cast value.
2197 return ZExt;
2198}
2199
2200/// Process the given Ops list, which is a list of operands to be added under
2201/// the given scale, update the given map. This is a helper function for
2202/// getAddRecExpr. As an example of what it does, given a sequence of operands
2203/// that would form an add expression like this:
2204///
2205/// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
2206///
2207/// where A and B are constants, update the map with these values:
2208///
2209/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
2210///
2211/// and add 13 + A*B*29 to AccumulatedConstant.
2212/// This will allow getAddRecExpr to produce this:
2213///
2214/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
2215///
2216/// This form often exposes folding opportunities that are hidden in
2217/// the original operand list.
2218///
2219/// Return true iff it appears that any interesting folding opportunities
2220/// may be exposed. This helps getAddRecExpr short-circuit extra work in
2221/// the common case where no interesting opportunities are present, and
2222/// is also used as a check to avoid infinite recursion.
2223static bool CollectAddOperandsWithScales(SmallDenseMap<SCEVUse, APInt, 16> &M,
2224 SmallVectorImpl<SCEVUse> &NewOps,
2225 APInt &AccumulatedConstant,
2226 ArrayRef<SCEVUse> Ops,
2227 const APInt &Scale,
2228 ScalarEvolution &SE) {
2229 bool Interesting = false;
2230
2231 // Iterate over the add operands. They are sorted, with constants first.
2232 unsigned i = 0;
2233 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Ops[i])) {
2234 ++i;
2235 // Pull a buried constant out to the outside.
2236 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
2237 Interesting = true;
2238 AccumulatedConstant += Scale * C->getAPInt();
2239 }
2240
2241 // Next comes everything else. We're especially interested in multiplies
2242 // here, but they're in the middle, so just visit the rest with one loop.
2243 for (; i != Ops.size(); ++i) {
2244 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: Ops[i]);
2245 if (Mul && isa<SCEVConstant>(Val: Mul->getOperand(i: 0))) {
2246 APInt NewScale =
2247 Scale * cast<SCEVConstant>(Val: Mul->getOperand(i: 0))->getAPInt();
2248 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Val: Mul->getOperand(i: 1))) {
2249 // A multiplication of a constant with another add; recurse.
2250 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: Mul->getOperand(i: 1));
2251 Interesting |= CollectAddOperandsWithScales(
2252 M, NewOps, AccumulatedConstant, Ops: Add->operands(), Scale: NewScale, SE);
2253 } else {
2254 // A multiplication of a constant with some other value. Update
2255 // the map.
2256 SmallVector<SCEVUse, 4> MulOps(drop_begin(RangeOrContainer: Mul->operands()));
2257 const SCEV *Key = SE.getMulExpr(Ops&: MulOps);
2258 auto Pair = M.insert(KV: {Key, NewScale});
2259 if (Pair.second) {
2260 NewOps.push_back(Elt: Pair.first->first);
2261 } else {
2262 Pair.first->second += NewScale;
2263 // The map already had an entry for this value, which may indicate
2264 // a folding opportunity.
2265 Interesting = true;
2266 }
2267 }
2268 } else {
2269 // An ordinary operand. Update the map.
2270 auto Pair = M.insert(KV: {Ops[i], Scale});
2271 if (Pair.second) {
2272 NewOps.push_back(Elt: Pair.first->first);
2273 } else {
2274 Pair.first->second += Scale;
2275 // The map already had an entry for this value, which may indicate
2276 // a folding opportunity.
2277 Interesting = true;
2278 }
2279 }
2280 }
2281
2282 return Interesting;
2283}
2284
2285bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed,
2286 const SCEV *LHS, const SCEV *RHS,
2287 const Instruction *CtxI) {
2288 const SCEV *(ScalarEvolution::*Operation)(SCEVUse, SCEVUse, SCEV::NoWrapFlags,
2289 unsigned);
2290 switch (BinOp) {
2291 default:
2292 llvm_unreachable("Unsupported binary op");
2293 case Instruction::Add:
2294 Operation = &ScalarEvolution::getAddExpr;
2295 break;
2296 case Instruction::Sub:
2297 Operation = &ScalarEvolution::getMinusSCEV;
2298 break;
2299 case Instruction::Mul:
2300 Operation = &ScalarEvolution::getMulExpr;
2301 break;
2302 }
2303
2304 const SCEV *(ScalarEvolution::*Extension)(SCEVUse, Type *, unsigned) =
2305 Signed ? &ScalarEvolution::getSignExtendExpr
2306 : &ScalarEvolution::getZeroExtendExpr;
2307
2308 // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
2309 auto *NarrowTy = cast<IntegerType>(Val: LHS->getType());
2310 auto *WideTy =
2311 IntegerType::get(C&: NarrowTy->getContext(), NumBits: NarrowTy->getBitWidth() * 2);
2312
2313 const SCEV *A = (this->*Extension)(
2314 (this->*Operation)(LHS, RHS, SCEV::FlagAnyWrap, 0), WideTy, 0);
2315 const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
2316 const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
2317 const SCEV *B = (this->*Operation)(LHSB, RHSB, SCEV::FlagAnyWrap, 0);
2318 if (A == B)
2319 return true;
2320 // Can we use context to prove the fact we need?
2321 if (!CtxI)
2322 return false;
2323 // TODO: Support mul.
2324 if (BinOp == Instruction::Mul)
2325 return false;
2326 auto *RHSC = dyn_cast<SCEVConstant>(Val: RHS);
2327 // TODO: Lift this limitation.
2328 if (!RHSC)
2329 return false;
2330 APInt C = RHSC->getAPInt();
2331 unsigned NumBits = C.getBitWidth();
2332 bool IsSub = (BinOp == Instruction::Sub);
2333 bool IsNegativeConst = (Signed && C.isNegative());
2334 // Compute the direction and magnitude by which we need to check overflow.
2335 bool OverflowDown = IsSub ^ IsNegativeConst;
2336 APInt Magnitude = C;
2337 if (IsNegativeConst) {
2338 if (C == APInt::getSignedMinValue(numBits: NumBits))
2339 // TODO: SINT_MIN on inversion gives the same negative value, we don't
2340 // want to deal with that.
2341 return false;
2342 Magnitude = -C;
2343 }
2344
2345 ICmpInst::Predicate Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2346 if (OverflowDown) {
2347 // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
2348 APInt Min = Signed ? APInt::getSignedMinValue(numBits: NumBits)
2349 : APInt::getMinValue(numBits: NumBits);
2350 APInt Limit = Min + Magnitude;
2351 return isKnownPredicateAt(Pred, LHS: getConstant(Val: Limit), RHS: LHS, CtxI);
2352 } else {
2353 // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
2354 APInt Max = Signed ? APInt::getSignedMaxValue(numBits: NumBits)
2355 : APInt::getMaxValue(numBits: NumBits);
2356 APInt Limit = Max - Magnitude;
2357 return isKnownPredicateAt(Pred, LHS, RHS: getConstant(Val: Limit), CtxI);
2358 }
2359}
2360
2361std::optional<SCEV::NoWrapFlags>
2362ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp(
2363 const OverflowingBinaryOperator *OBO) {
2364 // It cannot be done any better.
2365 if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
2366 return std::nullopt;
2367
2368 SCEV::NoWrapFlags Flags = SCEV::NoWrapFlags::FlagAnyWrap;
2369
2370 if (OBO->hasNoUnsignedWrap())
2371 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2372 if (OBO->hasNoSignedWrap())
2373 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2374
2375 bool Deduced = false;
2376
2377 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)OBO->getOpcode();
2378 const SCEV *LHS = getSCEV(V: OBO->getOperand(i_nocapture: 0));
2379 const SCEV *RHS = getSCEV(V: OBO->getOperand(i_nocapture: 1));
2380
2381 bool CanUseNSW = true;
2382 const APInt *ShiftAmt;
2383 // Treat `shl %a, C` as `mul %a, 1 << C`.
2384 if (match(V: OBO, P: m_Shl(L: m_Value(), R: m_APInt(Res&: ShiftAmt)))) {
2385 unsigned BitWidth = ShiftAmt->getBitWidth();
2386 if (ShiftAmt->uge(RHS: BitWidth))
2387 return std::nullopt;
2388 // NSW only transfers if the shift amount is < BitWidth - 1, as INT_MIN * -1
2389 // overflows.
2390 CanUseNSW = ShiftAmt->ult(RHS: BitWidth - 1);
2391 Opcode = Instruction::Mul;
2392 RHS = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: ShiftAmt->getZExtValue()));
2393 } else if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
2394 Opcode != Instruction::Mul) {
2395 return std::nullopt;
2396 }
2397
2398 const Instruction *CtxI =
2399 UseContextForNoWrapFlagInference ? dyn_cast<Instruction>(Val: OBO) : nullptr;
2400 if (!OBO->hasNoUnsignedWrap() &&
2401 willNotOverflow(BinOp: Opcode, /* Signed */ false, LHS, RHS, CtxI)) {
2402 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2403 Deduced = true;
2404 }
2405
2406 if (CanUseNSW && !OBO->hasNoSignedWrap() &&
2407 willNotOverflow(BinOp: Opcode, /* Signed */ true, LHS, RHS, CtxI)) {
2408 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2409 Deduced = true;
2410 }
2411
2412 if (Deduced)
2413 return Flags;
2414 return std::nullopt;
2415}
2416
2417// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
2418// `OldFlags' as can't-wrap behavior. Infer a more aggressive set of
2419// can't-overflow flags for the operation if possible.
2420static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE,
2421 SCEVTypes Type,
2422 ArrayRef<SCEVUse> Ops,
2423 SCEV::NoWrapFlags Flags) {
2424 using namespace std::placeholders;
2425
2426 using OBO = OverflowingBinaryOperator;
2427
2428 bool CanAnalyze =
2429 Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr;
2430 (void)CanAnalyze;
2431 assert(CanAnalyze && "don't call from other places!");
2432
2433 SCEV::NoWrapFlags SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
2434 SCEV::NoWrapFlags SignOrUnsignWrap =
2435 ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2436
2437 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
2438 auto IsKnownNonNegative = [&](SCEVUse U) {
2439 return SE->isKnownNonNegative(S: U);
2440 };
2441
2442 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Range&: Ops, P: IsKnownNonNegative))
2443 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SignOrUnsignMask);
2444
2445 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2446
2447 if (SignOrUnsignWrap != SignOrUnsignMask &&
2448 (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
2449 isa<SCEVConstant>(Val: Ops[0])) {
2450
2451 auto Opcode = [&] {
2452 switch (Type) {
2453 case scAddExpr:
2454 return Instruction::Add;
2455 case scMulExpr:
2456 return Instruction::Mul;
2457 default:
2458 llvm_unreachable("Unexpected SCEV op.");
2459 }
2460 }();
2461
2462 const APInt &C = cast<SCEVConstant>(Val: Ops[0])->getAPInt();
2463
2464 // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
2465 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
2466 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2467 BinOp: Opcode, Other: C, NoWrapKind: OBO::NoSignedWrap);
2468 if (NSWRegion.contains(CR: SE->getSignedRange(S: Ops[1])))
2469 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2470 }
2471
2472 // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
2473 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
2474 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2475 BinOp: Opcode, Other: C, NoWrapKind: OBO::NoUnsignedWrap);
2476 if (NUWRegion.contains(CR: SE->getUnsignedRange(S: Ops[1])))
2477 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2478 }
2479 }
2480
2481 // <0,+,nonnegative><nw> is also nuw
2482 // TODO: Add corresponding nsw case
2483 if (Type == scAddRecExpr && ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNW) &&
2484 !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) && Ops.size() == 2 &&
2485 Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
2486 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2487
2488 // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
2489 if (Type == scMulExpr && !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) &&
2490 Ops.size() == 2) {
2491 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[0]))
2492 if (UDiv->getOperand(i: 1) == Ops[1])
2493 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2494 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[1]))
2495 if (UDiv->getOperand(i: 1) == Ops[0])
2496 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2497 }
2498
2499 return Flags;
2500}
2501
2502bool ScalarEvolution::isAvailableAtLoopEntry(const SCEV *S, const Loop *L) {
2503 return isLoopInvariant(S, L) && properlyDominates(S, BB: L->getHeader());
2504}
2505
2506/// Get a canonical add expression, or something simpler if possible.
2507const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<SCEVUse> &Ops,
2508 SCEV::NoWrapFlags OrigFlags,
2509 unsigned Depth) {
2510 assert(!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2511 "only nuw or nsw allowed");
2512 assert(!Ops.empty() && "Cannot get empty add!");
2513 if (Ops.size() == 1) return Ops[0];
2514#ifndef NDEBUG
2515 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
2516 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
2517 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
2518 "SCEVAddExpr operand types don't match!");
2519 unsigned NumPtrs = count_if(
2520 Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
2521 assert(NumPtrs <= 1 && "add has at most one pointer operand");
2522#endif
2523
2524 const SCEV *Folded = constantFoldAndGroupOps(
2525 SE&: *this, LI, DT, Ops,
2526 Fold: [](const APInt &C1, const APInt &C2) { return C1 + C2; },
2527 IsIdentity: [](const APInt &C) { return C.isZero(); }, // identity
2528 IsAbsorber: [](const APInt &C) { return false; }); // absorber
2529 if (Folded)
2530 return Folded;
2531
2532 unsigned Idx = isa<SCEVConstant>(Val: Ops[0]) ? 1 : 0;
2533
2534 // Delay expensive flag strengthening until necessary.
2535 auto ComputeFlags = [this, OrigFlags](ArrayRef<SCEVUse> Ops) {
2536 return StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops, Flags: OrigFlags);
2537 };
2538
2539 // Limit recursion calls depth.
2540 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
2541 return getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops));
2542
2543 if (SCEV *S = findExistingSCEVInCache(SCEVType: scAddExpr, Ops)) {
2544 // Don't strengthen flags if we have no new information.
2545 SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
2546 if (Add->getNoWrapFlags(Mask: OrigFlags) != OrigFlags)
2547 Add->setNoWrapFlags(ComputeFlags(Ops));
2548 return S;
2549 }
2550
2551 // Okay, check to see if the same value occurs in the operand list more than
2552 // once. If so, merge them together into an multiply expression. Since we
2553 // sorted the list, these values are required to be adjacent.
2554 Type *Ty = Ops[0]->getType();
2555 bool FoundMatch = false;
2556 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
2557 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2
2558 // Scan ahead to count how many equal operands there are.
2559 unsigned Count = 2;
2560 while (i+Count != e && Ops[i+Count] == Ops[i])
2561 ++Count;
2562 // Merge the values into a multiply.
2563 SCEVUse Scale = getConstant(Ty, V: Count);
2564 const SCEV *Mul = getMulExpr(LHS: Scale, RHS: Ops[i], Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2565 if (Ops.size() == Count)
2566 return Mul;
2567 Ops[i] = Mul;
2568 Ops.erase(CS: Ops.begin()+i+1, CE: Ops.begin()+i+Count);
2569 --i; e -= Count - 1;
2570 FoundMatch = true;
2571 }
2572 if (FoundMatch)
2573 return getAddExpr(Ops, OrigFlags, Depth: Depth + 1);
2574
2575 // Check for truncates. If all the operands are truncated from the same
2576 // type, see if factoring out the truncate would permit the result to be
2577 // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
2578 // if the contents of the resulting outer trunc fold to something simple.
2579 auto FindTruncSrcType = [&]() -> Type * {
2580 // We're ultimately looking to fold an addrec of truncs and muls of only
2581 // constants and truncs, so if we find any other types of SCEV
2582 // as operands of the addrec then we bail and return nullptr here.
2583 // Otherwise, we return the type of the operand of a trunc that we find.
2584 if (auto *T = dyn_cast<SCEVTruncateExpr>(Val&: Ops[Idx]))
2585 return T->getOperand()->getType();
2586 if (const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
2587 SCEVUse LastOp = Mul->getOperand(i: Mul->getNumOperands() - 1);
2588 if (const auto *T = dyn_cast<SCEVTruncateExpr>(Val&: LastOp))
2589 return T->getOperand()->getType();
2590 }
2591 return nullptr;
2592 };
2593 if (auto *SrcType = FindTruncSrcType()) {
2594 SmallVector<SCEVUse, 8> LargeOps;
2595 bool Ok = true;
2596 // Check all the operands to see if they can be represented in the
2597 // source type of the truncate.
2598 for (const SCEV *Op : Ops) {
2599 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2600 if (T->getOperand()->getType() != SrcType) {
2601 Ok = false;
2602 break;
2603 }
2604 LargeOps.push_back(Elt: T->getOperand());
2605 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Op)) {
2606 LargeOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2607 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val: Op)) {
2608 SmallVector<SCEVUse, 8> LargeMulOps;
2609 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2610 if (const SCEVTruncateExpr *T =
2611 dyn_cast<SCEVTruncateExpr>(Val: M->getOperand(i: j))) {
2612 if (T->getOperand()->getType() != SrcType) {
2613 Ok = false;
2614 break;
2615 }
2616 LargeMulOps.push_back(Elt: T->getOperand());
2617 } else if (const auto *C = dyn_cast<SCEVConstant>(Val: M->getOperand(i: j))) {
2618 LargeMulOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2619 } else {
2620 Ok = false;
2621 break;
2622 }
2623 }
2624 if (Ok)
2625 LargeOps.push_back(Elt: getMulExpr(Ops&: LargeMulOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2626 } else {
2627 Ok = false;
2628 break;
2629 }
2630 }
2631 if (Ok) {
2632 // Evaluate the expression in the larger type.
2633 const SCEV *Fold = getAddExpr(Ops&: LargeOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2634 // If it folds to something simple, use it. Otherwise, don't.
2635 if (isa<SCEVConstant>(Val: Fold) || isa<SCEVUnknown>(Val: Fold))
2636 return getTruncateExpr(Op: Fold, Ty);
2637 }
2638 }
2639
2640 if (Ops.size() == 2) {
2641 // Check if we have an expression of the form ((X + C1) - C2), where C1 and
2642 // C2 can be folded in a way that allows retaining wrapping flags of (X +
2643 // C1).
2644 const SCEV *A = Ops[0];
2645 const SCEV *B = Ops[1];
2646 auto *AddExpr = dyn_cast<SCEVAddExpr>(Val: B);
2647 auto *C = dyn_cast<SCEVConstant>(Val: A);
2648 if (AddExpr && C && isa<SCEVConstant>(Val: AddExpr->getOperand(i: 0))) {
2649 auto C1 = cast<SCEVConstant>(Val: AddExpr->getOperand(i: 0))->getAPInt();
2650 auto C2 = C->getAPInt();
2651 SCEV::NoWrapFlags PreservedFlags = SCEV::FlagAnyWrap;
2652
2653 APInt ConstAdd = C1 + C2;
2654 auto AddFlags = AddExpr->getNoWrapFlags();
2655 // Adding a smaller constant is NUW if the original AddExpr was NUW.
2656 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNUW) &&
2657 ConstAdd.ule(RHS: C1)) {
2658 PreservedFlags =
2659 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNUW);
2660 }
2661
2662 // Adding a constant with the same sign and small magnitude is NSW, if the
2663 // original AddExpr was NSW.
2664 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNSW) &&
2665 C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
2666 ConstAdd.abs().ule(RHS: C1.abs())) {
2667 PreservedFlags =
2668 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNSW);
2669 }
2670
2671 if (PreservedFlags != SCEV::FlagAnyWrap) {
2672 SmallVector<SCEVUse, 4> NewOps(AddExpr->operands());
2673 NewOps[0] = getConstant(Val: ConstAdd);
2674 return getAddExpr(Ops&: NewOps, OrigFlags: PreservedFlags);
2675 }
2676 }
2677
2678 // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext
2679 // (B), if trunc (A) + -A + B does not unsigned-wrap.
2680 const SCEVAddExpr *InnerAdd;
2681 if (match(S: B, P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
2682 const SCEV *NarrowA = getTruncateExpr(Op: A, Ty: InnerAdd->getType());
2683 if (NarrowA == getNegativeSCEV(V: InnerAdd->getOperand(i: 0)) &&
2684 getZeroExtendExpr(Op: NarrowA, Ty: B->getType()) == A &&
2685 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops: {NarrowA, InnerAdd},
2686 Flags: SCEV::FlagAnyWrap),
2687 TestFlags: SCEV::FlagNUW)) {
2688 return getZeroExtendExpr(Op: getAddExpr(LHS: NarrowA, RHS: InnerAdd), Ty: B->getType());
2689 }
2690 }
2691 }
2692
2693 // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
2694 const SCEV *Y;
2695 if (Ops.size() == 2 &&
2696 match(U: Ops[0],
2697 P: m_scev_Mul(Op0: m_scev_AllOnes(),
2698 Op1: m_scev_URem(LHS: m_scev_Specific(S: Ops[1]), RHS: m_SCEV(V&: Y), SE&: *this))))
2699 return getMulExpr(LHS: Y, RHS: getUDivExpr(LHS: Ops[1], RHS: Y));
2700
2701 // Skip past any other cast SCEVs.
2702 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
2703 ++Idx;
2704
2705 // If there are add operands they would be next.
2706 if (Idx < Ops.size()) {
2707 bool DeletedAdd = false;
2708 // If the original flags and all inlined SCEVAddExprs are NUW, use the
2709 // common NUW flag for expression after inlining. Other flags cannot be
2710 // preserved, because they may depend on the original order of operations.
2711 SCEV::NoWrapFlags CommonFlags = maskFlags(Flags: OrigFlags, Mask: SCEV::FlagNUW);
2712 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[Idx])) {
2713 if (Ops.size() > AddOpsInlineThreshold ||
2714 Add->getNumOperands() > AddOpsInlineThreshold)
2715 break;
2716 // If we have an add, expand the add operands onto the end of the operands
2717 // list.
2718 Ops.erase(CI: Ops.begin()+Idx);
2719 append_range(C&: Ops, R: Add->operands());
2720 DeletedAdd = true;
2721 CommonFlags = maskFlags(Flags: CommonFlags, Mask: Add->getNoWrapFlags());
2722 }
2723
2724 // If we deleted at least one add, we added operands to the end of the list,
2725 // and they are not necessarily sorted. Recurse to resort and resimplify
2726 // any operands we just acquired.
2727 if (DeletedAdd)
2728 return getAddExpr(Ops, OrigFlags: CommonFlags, Depth: Depth + 1);
2729 }
2730
2731 // Skip over the add expression until we get to a multiply.
2732 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
2733 ++Idx;
2734
2735 // Check to see if there are any folding opportunities present with
2736 // operands multiplied by constant values.
2737 if (Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx])) {
2738 uint64_t BitWidth = getTypeSizeInBits(Ty);
2739 SmallDenseMap<SCEVUse, APInt, 16> M;
2740 SmallVector<SCEVUse, 8> NewOps;
2741 APInt AccumulatedConstant(BitWidth, 0);
2742 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2743 Ops, Scale: APInt(BitWidth, 1), SE&: *this)) {
2744 struct APIntCompare {
2745 bool operator()(const APInt &LHS, const APInt &RHS) const {
2746 return LHS.ult(RHS);
2747 }
2748 };
2749
2750 // Some interesting folding opportunity is present, so its worthwhile to
2751 // re-generate the operands list. Group the operands by constant scale,
2752 // to avoid multiplying by the same constant scale multiple times.
2753 std::map<APInt, SmallVector<SCEVUse, 4>, APIntCompare> MulOpLists;
2754 for (const SCEV *NewOp : NewOps)
2755 MulOpLists[M.find(Val: NewOp)->second].push_back(Elt: NewOp);
2756 // Re-generate the operands list.
2757 Ops.clear();
2758 if (AccumulatedConstant != 0)
2759 Ops.push_back(Elt: getConstant(Val: AccumulatedConstant));
2760 for (auto &MulOp : MulOpLists) {
2761 if (MulOp.first == 1) {
2762 Ops.push_back(Elt: getAddExpr(Ops&: MulOp.second, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2763 } else if (MulOp.first != 0) {
2764 Ops.push_back(Elt: getMulExpr(
2765 LHS: getConstant(Val: MulOp.first),
2766 RHS: getAddExpr(Ops&: MulOp.second, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2767 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2768 }
2769 }
2770 if (Ops.empty())
2771 return getZero(Ty);
2772 if (Ops.size() == 1)
2773 return Ops[0];
2774 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2775 }
2776 }
2777
2778 // Given a SCEVMulExpr and an operand index, return the product of all
2779 // operands except the one at OpIdx.
2780 auto StripFactor = [&](const SCEVMulExpr *M, unsigned OpIdx) -> SCEVUse {
2781 if (M->getNumOperands() == 2)
2782 return M->getOperand(i: OpIdx == 0);
2783 SmallVector<SCEVUse, 4> Remaining(M->operands().take_front(N: OpIdx));
2784 append_range(C&: Remaining, R: M->operands().drop_front(N: OpIdx + 1));
2785 return getMulExpr(Ops&: Remaining, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2786 };
2787
2788 // If we are adding something to a multiply expression, make sure the
2789 // something is not already an operand of the multiply. If so, merge it into
2790 // the multiply.
2791 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx]); ++Idx) {
2792 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val&: Ops[Idx]);
2793 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
2794 // Scan all terms to find every occurrence of common factor MulOpSCEV
2795 // and fold them in one shot:
2796 // A1*X + A2*X + ... + An*X --> X * (A1 + A2 + ... + An)
2797 const SCEV *MulOpSCEV = Mul->getOperand(i: MulOp);
2798 if (isa<SCEVConstant>(Val: MulOpSCEV))
2799 continue;
2800
2801 // Cofactors: 1 for bare addends matching MulOpSCEV, or the
2802 // remaining product for multiply terms containing MulOpSCEV.
2803 SmallVector<SCEVUse, 4> Cofactors;
2804 SmallVector<unsigned, 4> DeadIndices;
2805 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) {
2806 if (MulOpSCEV == Ops[AddOp]) {
2807 // W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
2808 Cofactors.push_back(Elt: getOne(Ty));
2809 DeadIndices.push_back(Elt: AddOp);
2810 continue;
2811 }
2812
2813 if (AddOp <= Idx || !isa<SCEVMulExpr>(Val: Ops[AddOp]))
2814 continue;
2815
2816 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Val&: Ops[AddOp]);
2817 for (unsigned OMulOp = 0, OE = OtherMul->getNumOperands(); OMulOp != OE;
2818 ++OMulOp) {
2819 if (OtherMul->getOperand(i: OMulOp) == MulOpSCEV) {
2820 // (A*B*C) + (A*D*E) --> A * (B*C + D*E)
2821 Cofactors.push_back(Elt: StripFactor(OtherMul, OMulOp));
2822 DeadIndices.push_back(Elt: AddOp);
2823 break;
2824 }
2825 }
2826 }
2827
2828 // Fold all collected cofactors with the anchor multiply's cofactor:
2829 // MulOpSCEV * (Cofactor_1 + ... + Cofactor_n + AnchorCofactor)
2830 if (!Cofactors.empty()) {
2831 Cofactors.push_back(Elt: StripFactor(Mul, MulOp));
2832
2833 SCEVUse InnerSum = getAddExpr(Ops&: Cofactors, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2834 SCEVUse OuterMul =
2835 getMulExpr(LHS: MulOpSCEV, RHS: InnerSum, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2836
2837 // DeadIndices does not include Idx (the anchor), hence +1.
2838 if (Ops.size() == DeadIndices.size() + 1)
2839 return OuterMul;
2840
2841 // Erase Ops[Idx] first, then erase DeadIndices in reverse order.
2842 // The -1 adjustment accounts for the shift from removing Idx;
2843 // reverse order means each erasure only shifts later positions,
2844 // which have already been processed.
2845 Ops.erase(CI: Ops.begin() + Idx);
2846 for (unsigned Dead : reverse(C&: DeadIndices))
2847 Ops.erase(CI: Ops.begin() + (Dead > Idx ? Dead - 1 : Dead));
2848
2849 Ops.push_back(Elt: OuterMul);
2850 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2851 }
2852 }
2853 }
2854
2855 // If there are any add recurrences in the operands list, see if any other
2856 // added values are loop invariant. If so, we can fold them into the
2857 // recurrence.
2858 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
2859 ++Idx;
2860
2861 // Scan over all recurrences, trying to fold loop invariants into them.
2862 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
2863 // Scan all of the other operands to this add and add them to the vector if
2864 // they are loop invariant w.r.t. the recurrence.
2865 SmallVector<SCEVUse, 8> LIOps;
2866 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
2867 const Loop *AddRecLoop = AddRec->getLoop();
2868 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2869 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRecLoop)) {
2870 LIOps.push_back(Elt: Ops[i]);
2871 Ops.erase(CI: Ops.begin()+i);
2872 --i; --e;
2873 }
2874
2875 // If we found some loop invariants, fold them into the recurrence.
2876 if (!LIOps.empty()) {
2877 // Compute nowrap flags for the addition of the loop-invariant ops and
2878 // the addrec. Temporarily push it as an operand for that purpose. These
2879 // flags are valid in the scope of the addrec only.
2880 LIOps.push_back(Elt: AddRec);
2881 SCEV::NoWrapFlags Flags = ComputeFlags(LIOps);
2882 LIOps.pop_back();
2883
2884 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
2885 LIOps.push_back(Elt: AddRec->getStart());
2886
2887 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
2888
2889 // It is not in general safe to propagate flags valid on an add within
2890 // the addrec scope to one outside it. We must prove that the inner
2891 // scope is guaranteed to execute if the outer one does to be able to
2892 // safely propagate. We know the program is undefined if poison is
2893 // produced on the inner scoped addrec. We also know that *for this use*
2894 // the outer scoped add can't overflow (because of the flags we just
2895 // computed for the inner scoped add) without the program being undefined.
2896 // Proving that entry to the outer scope neccesitates entry to the inner
2897 // scope, thus proves the program undefined if the flags would be violated
2898 // in the outer scope.
2899 SCEV::NoWrapFlags AddFlags = Flags;
2900 if (AddFlags != SCEV::FlagAnyWrap) {
2901 auto *DefI = getDefiningScopeBound(Ops: LIOps);
2902 auto *ReachI = &*AddRecLoop->getHeader()->begin();
2903 if (!isGuaranteedToTransferExecutionTo(A: DefI, B: ReachI))
2904 AddFlags = SCEV::FlagAnyWrap;
2905 }
2906 AddRecOps[0] = getAddExpr(Ops&: LIOps, OrigFlags: AddFlags, Depth: Depth + 1);
2907
2908 // Build the new addrec. Propagate the NUW and NSW flags if both the
2909 // outer add and the inner addrec are guaranteed to have no overflow.
2910 // Always propagate NW.
2911 Flags = AddRec->getNoWrapFlags(Mask: setFlags(Flags, OnFlags: SCEV::FlagNW));
2912 const SCEV *NewRec = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags);
2913
2914 // If all of the other operands were loop invariant, we are done.
2915 if (Ops.size() == 1) return NewRec;
2916
2917 // Otherwise, add the folded AddRec by the non-invariant parts.
2918 for (unsigned i = 0;; ++i)
2919 if (Ops[i] == AddRec) {
2920 Ops[i] = NewRec;
2921 break;
2922 }
2923 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2924 }
2925
2926 // Okay, if there weren't any loop invariants to be folded, check to see if
2927 // there are multiple AddRec's with the same loop induction variable being
2928 // added together. If so, we can fold them.
2929 for (unsigned OtherIdx = Idx+1;
2930 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2931 ++OtherIdx) {
2932 // We expect the AddRecExpr's to be sorted in reverse dominance order,
2933 // so that the 1st found AddRecExpr is dominated by all others.
2934 assert(DT.dominates(
2935 cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),
2936 AddRec->getLoop()->getHeader()) &&
2937 "AddRecExprs are not sorted in reverse dominance order?");
2938 if (AddRecLoop == cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx])->getLoop()) {
2939 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L>
2940 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
2941 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2942 ++OtherIdx) {
2943 const auto *OtherAddRec = cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
2944 if (OtherAddRec->getLoop() == AddRecLoop) {
2945 for (unsigned i = 0, e = OtherAddRec->getNumOperands();
2946 i != e; ++i) {
2947 if (i >= AddRecOps.size()) {
2948 append_range(C&: AddRecOps, R: OtherAddRec->operands().drop_front(N: i));
2949 break;
2950 }
2951 AddRecOps[i] =
2952 getAddExpr(LHS: AddRecOps[i], RHS: OtherAddRec->getOperand(i),
2953 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2954 }
2955 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
2956 }
2957 }
2958 // Step size has changed, so we cannot guarantee no self-wraparound.
2959 Ops[Idx] = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags: SCEV::FlagAnyWrap);
2960 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2961 }
2962 }
2963
2964 // Otherwise couldn't fold anything into this recurrence. Move onto the
2965 // next one.
2966 }
2967
2968 // Okay, it looks like we really DO need an add expr. Check to see if we
2969 // already have one, otherwise create a new one.
2970 return getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops));
2971}
2972
2973const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
2974 SCEV::NoWrapFlags Flags) {
2975 FoldingSetNodeID ID;
2976 ID.AddInteger(I: scAddExpr);
2977 for (SCEVUse Op : Ops)
2978 ID.AddPointer(Ptr: Op.getOpaqueValue());
2979 FoldingSetInsertToken Token;
2980 SCEVAddExpr *S = static_cast<SCEVAddExpr *>(UniqueSCEVs.lookup(ID, Token));
2981 if (!S) {
2982 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
2983 llvm::uninitialized_copy(Src&: Ops, Dst: O);
2984 S = new (SCEVAllocator)
2985 SCEVAddExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size());
2986 UniqueSCEVs.insert(N: S, Token);
2987 S->computeAndSetCanonical(SE&: *this);
2988 registerUser(User: S, Ops);
2989 }
2990 S->setNoWrapFlags(Flags);
2991 return S;
2992}
2993
2994const SCEV *ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<SCEVUse> Ops,
2995 const Loop *L,
2996 SCEV::NoWrapFlags Flags) {
2997 FoldingSetNodeID ID;
2998 ID.AddInteger(I: scAddRecExpr);
2999 for (SCEVUse Op : Ops)
3000 ID.AddPointer(Ptr: Op.getOpaqueValue());
3001 ID.AddPointer(Ptr: L);
3002 FoldingSetInsertToken Token;
3003 SCEVAddRecExpr *S =
3004 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.lookup(ID, Token));
3005 if (!S) {
3006 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3007 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3008 S = new (SCEVAllocator)
3009 SCEVAddRecExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size(), L);
3010 UniqueSCEVs.insert(N: S, Token);
3011 S->computeAndSetCanonical(SE&: *this);
3012 LoopUsers[L].push_back(Elt: S);
3013 registerUser(User: S, Ops);
3014 }
3015 setNoWrapFlags(AddRec: S, Flags);
3016 return S;
3017}
3018
3019const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
3020 SCEV::NoWrapFlags Flags) {
3021 FoldingSetNodeID ID;
3022 ID.AddInteger(I: scMulExpr);
3023 for (SCEVUse Op : Ops)
3024 ID.AddPointer(Ptr: Op.getOpaqueValue());
3025 FoldingSetInsertToken Token;
3026 SCEVMulExpr *S = static_cast<SCEVMulExpr *>(UniqueSCEVs.lookup(ID, Token));
3027 if (!S) {
3028 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3029 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3030 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(Allocator&: SCEVAllocator),
3031 O, Ops.size());
3032 UniqueSCEVs.insert(N: S, Token);
3033 S->computeAndSetCanonical(SE&: *this);
3034 registerUser(User: S, Ops);
3035 }
3036 S->setNoWrapFlags(Flags);
3037 return S;
3038}
3039
3040const SCEV *ScalarEvolution::getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS) {
3041 FoldingSetNodeID ID;
3042 ID.AddInteger(I: scUDivExpr);
3043 ID.AddPointer(Ptr: LHS.getOpaqueValue());
3044 ID.AddPointer(Ptr: RHS.getOpaqueValue());
3045 FoldingSetInsertToken Token;
3046 SCEV *S = UniqueSCEVs.lookup(ID, Token);
3047 if (!S) {
3048 S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(Allocator&: SCEVAllocator), LHS, RHS);
3049 UniqueSCEVs.insert(N: S, Token);
3050 S->computeAndSetCanonical(SE&: *this);
3051 registerUser(User: S, Ops: ArrayRef<SCEVUse>({LHS, RHS}));
3052 }
3053 return S;
3054}
3055
3056static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
3057 uint64_t k = i*j;
3058 if (j > 1 && k / j != i) Overflow = true;
3059 return k;
3060}
3061
3062/// Compute the result of "n choose k", the binomial coefficient. If an
3063/// intermediate computation overflows, Overflow will be set and the return will
3064/// be garbage. Overflow is not cleared on absence of overflow.
3065static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
3066 // We use the multiplicative formula:
3067 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
3068 // At each iteration, we take the n-th term of the numeral and divide by the
3069 // (k-n)th term of the denominator. This division will always produce an
3070 // integral result, and helps reduce the chance of overflow in the
3071 // intermediate computations. However, we can still overflow even when the
3072 // final result would fit.
3073
3074 if (n == 0 || n == k) return 1;
3075 if (k > n) return 0;
3076
3077 if (k > n/2)
3078 k = n-k;
3079
3080 uint64_t r = 1;
3081 for (uint64_t i = 1; i <= k; ++i) {
3082 r = umul_ov(i: r, j: n-(i-1), Overflow);
3083 r /= i;
3084 }
3085 return r;
3086}
3087
3088/// Determine if any of the operands in this SCEV are a constant or if
3089/// any of the add or multiply expressions in this SCEV contain a constant.
3090static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
3091 struct FindConstantInAddMulChain {
3092 bool FoundConstant = false;
3093
3094 bool follow(const SCEV *S) {
3095 FoundConstant |= isa<SCEVConstant>(Val: S);
3096 return isa<SCEVAddExpr>(Val: S) || isa<SCEVMulExpr>(Val: S);
3097 }
3098
3099 bool isDone() const {
3100 return FoundConstant;
3101 }
3102 };
3103
3104 FindConstantInAddMulChain F;
3105 SCEVTraversal<FindConstantInAddMulChain> ST(F);
3106 ST.visitAll(Root: StartExpr);
3107 return F.FoundConstant;
3108}
3109
3110/// Get a canonical multiply expression, or something simpler if possible.
3111const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
3112 SCEV::NoWrapFlags OrigFlags,
3113 unsigned Depth) {
3114 assert(OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3115 "only nuw or nsw allowed");
3116 assert(!Ops.empty() && "Cannot get empty mul!");
3117 if (Ops.size() == 1) return Ops[0];
3118#ifndef NDEBUG
3119 Type *ETy = Ops[0]->getType();
3120 assert(!ETy->isPointerTy());
3121 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
3122 assert(Ops[i]->getType() == ETy &&
3123 "SCEVMulExpr operand types don't match!");
3124#endif
3125
3126 const SCEV *Folded = constantFoldAndGroupOps(
3127 SE&: *this, LI, DT, Ops,
3128 Fold: [](const APInt &C1, const APInt &C2) { return C1 * C2; },
3129 IsIdentity: [](const APInt &C) { return C.isOne(); }, // identity
3130 IsAbsorber: [](const APInt &C) { return C.isZero(); }); // absorber
3131 if (Folded)
3132 return Folded;
3133
3134 // Delay expensive flag strengthening until necessary.
3135 auto ComputeFlags = [this, OrigFlags](const ArrayRef<SCEVUse> Ops) {
3136 return StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops, Flags: OrigFlags);
3137 };
3138
3139 // Limit recursion calls depth.
3140 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
3141 return getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops));
3142
3143 if (SCEV *S = findExistingSCEVInCache(SCEVType: scMulExpr, Ops)) {
3144 // Don't strengthen flags if we have no new information.
3145 SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
3146 if (Mul->getNoWrapFlags(Mask: OrigFlags) != OrigFlags)
3147 Mul->setNoWrapFlags(ComputeFlags(Ops));
3148 return S;
3149 }
3150
3151 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: Ops[0])) {
3152 if (Ops.size() == 2) {
3153 // C1*(C2+V) -> C1*C2 + C1*V
3154 // If any of Add's ops are Adds or Muls with a constant, apply this
3155 // transformation as well.
3156 //
3157 // TODO: There are some cases where this transformation is not
3158 // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of
3159 // this transformation should be narrowed down.
3160 const SCEV *Op0, *Op1;
3161 if (match(U: Ops[1], P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))) &&
3162 containsConstantInAddMulChain(StartExpr: Ops[1])) {
3163 const SCEV *LHS = getMulExpr(LHS: LHSC, RHS: Op0, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3164 const SCEV *RHS = getMulExpr(LHS: LHSC, RHS: Op1, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3165 return getAddExpr(LHS, RHS, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3166 }
3167
3168 if (Ops[0]->isAllOnesValue()) {
3169 // If we have a mul by -1 of an add, try distributing the -1 among the
3170 // add operands.
3171 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[1])) {
3172 SmallVector<SCEVUse, 4> NewOps;
3173 bool AnyFolded = false;
3174 for (const SCEV *AddOp : Add->operands()) {
3175 const SCEV *Mul = getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddOp),
3176 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3177 if (!isa<SCEVMulExpr>(Val: Mul)) AnyFolded = true;
3178 NewOps.push_back(Elt: Mul);
3179 }
3180 if (AnyFolded)
3181 return getAddExpr(Ops&: NewOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3182 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: Ops[1])) {
3183 // Negation preserves a recurrence's no self-wrap property.
3184 SmallVector<SCEVUse, 4> Operands;
3185 for (const SCEV *AddRecOp : AddRec->operands())
3186 Operands.push_back(Elt: getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddRecOp),
3187 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3188 // Let M be the minimum representable signed value. AddRec with nsw
3189 // multiplied by -1 can have signed overflow if and only if it takes a
3190 // value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
3191 // maximum signed value. In all other cases signed overflow is
3192 // impossible.
3193 auto FlagsMask = SCEV::FlagNW;
3194 if (AddRec->hasNoSignedWrap()) {
3195 auto MinInt =
3196 APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: AddRec->getType()));
3197 if (getSignedRangeMin(S: AddRec) != MinInt)
3198 FlagsMask = setFlags(Flags: FlagsMask, OnFlags: SCEV::FlagNSW);
3199 }
3200 return getAddRecExpr(Operands, L: AddRec->getLoop(),
3201 Flags: AddRec->getNoWrapFlags(Mask: FlagsMask));
3202 }
3203 }
3204
3205 // Try to push the constant operand into a ZExt: C * zext (A + B) ->
3206 // zext (C*A + C*B) if trunc (C) * (A + B) does not unsigned-wrap.
3207 const SCEVAddExpr *InnerAdd;
3208 if (match(U: Ops[1], P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
3209 const SCEV *NarrowC = getTruncateExpr(Op: LHSC, Ty: InnerAdd->getType());
3210 if (isa<SCEVConstant>(Val: InnerAdd->getOperand(i: 0)) &&
3211 getZeroExtendExpr(Op: NarrowC, Ty: Ops[1]->getType()) == LHSC &&
3212 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops: {NarrowC, InnerAdd},
3213 Flags: SCEV::FlagAnyWrap),
3214 TestFlags: SCEV::FlagNUW)) {
3215 auto *Res = getMulExpr(LHS: NarrowC, RHS: InnerAdd, Flags: SCEV::FlagNUW, Depth: Depth + 1);
3216 return getZeroExtendExpr(Op: Res, Ty: Ops[1]->getType(), Depth: Depth + 1);
3217 };
3218 }
3219
3220 // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2,
3221 // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple
3222 // of C1, fold to (D /u (C2 /u C1)).
3223 const SCEV *D;
3224 APInt C1V = LHSC->getAPInt();
3225 // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1
3226 // as -1 * 1, as it won't enable additional folds.
3227 if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes())
3228 C1V = C1V.abs();
3229 const SCEVConstant *C2;
3230 if (C1V.isPowerOf2() &&
3231 match(U: Ops[1], P: m_scev_UDiv(Op0: m_SCEV(V&: D), Op1: m_SCEVConstant(V&: C2))) &&
3232 C2->getAPInt().isPowerOf2() &&
3233 C1V.logBase2() <= getMinTrailingZeros(S: D)) {
3234 const SCEV *NewMul = nullptr;
3235 if (C1V.uge(RHS: C2->getAPInt())) {
3236 NewMul = getMulExpr(LHS: getUDivExpr(LHS: getConstant(Val: C1V), RHS: C2), RHS: D);
3237 } else if (C2->getAPInt().logBase2() <= getMinTrailingZeros(S: D)) {
3238 assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier");
3239 NewMul = getUDivExpr(LHS: D, RHS: getUDivExpr(LHS: C2, RHS: getConstant(Val: C1V)));
3240 }
3241 if (NewMul)
3242 return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(V: NewMul);
3243 }
3244 }
3245 }
3246
3247 // Skip over the add expression until we get to a multiply.
3248 unsigned Idx = 0;
3249 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
3250 ++Idx;
3251
3252 // If there are mul operands inline them all into this expression.
3253 if (Idx < Ops.size()) {
3254 bool DeletedMul = false;
3255 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
3256 if (Ops.size() > MulOpsInlineThreshold)
3257 break;
3258 // If we have an mul, expand the mul operands onto the end of the
3259 // operands list.
3260 Ops.erase(CI: Ops.begin()+Idx);
3261 append_range(C&: Ops, R: Mul->operands());
3262 DeletedMul = true;
3263 }
3264
3265 // If we deleted at least one mul, we added operands to the end of the
3266 // list, and they are not necessarily sorted. Recurse to resort and
3267 // resimplify any operands we just acquired.
3268 if (DeletedMul)
3269 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3270 }
3271
3272 // If there are any add recurrences in the operands list, see if any other
3273 // added values are loop invariant. If so, we can fold them into the
3274 // recurrence.
3275 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3276 ++Idx;
3277
3278 // Scan over all recurrences, trying to fold loop invariants into them.
3279 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
3280 // Scan all of the other operands to this mul and add them to the vector
3281 // if they are loop invariant w.r.t. the recurrence.
3282 SmallVector<SCEVUse, 8> LIOps;
3283 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
3284 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3285 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRec->getLoop())) {
3286 LIOps.push_back(Elt: Ops[i]);
3287 Ops.erase(CI: Ops.begin()+i);
3288 --i; --e;
3289 }
3290
3291 // If we found some loop invariants, fold them into the recurrence.
3292 if (!LIOps.empty()) {
3293 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
3294 SmallVector<SCEVUse, 4> NewOps;
3295 NewOps.reserve(N: AddRec->getNumOperands());
3296 const SCEV *Scale = getMulExpr(Ops&: LIOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3297
3298 // If both the mul and addrec are nuw, we can preserve nuw.
3299 // If both the mul and addrec are nsw, we can only preserve nsw if either
3300 // a) they are also nuw, or
3301 // b) all multiplications of addrec operands with scale are nsw.
3302 SCEV::NoWrapFlags Flags =
3303 AddRec->getNoWrapFlags(Mask: ComputeFlags({Scale, AddRec}));
3304
3305 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
3306 NewOps.push_back(Elt: getMulExpr(LHS: Scale, RHS: AddRec->getOperand(i),
3307 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3308
3309 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW) && !hasFlags(Flags, TestFlags: SCEV::FlagNUW)) {
3310 ConstantRange NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
3311 BinOp: Instruction::Mul, Other: getSignedRange(S: Scale),
3312 NoWrapKind: OverflowingBinaryOperator::NoSignedWrap);
3313 if (!NSWRegion.contains(CR: getSignedRange(S: AddRec->getOperand(i))))
3314 Flags = clearFlags(Flags, OffFlags: SCEV::FlagNSW);
3315 }
3316 }
3317
3318 const SCEV *NewRec = getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags);
3319
3320 // If all of the other operands were loop invariant, we are done.
3321 if (Ops.size() == 1) return NewRec;
3322
3323 // Otherwise, multiply the folded AddRec by the non-invariant parts.
3324 for (unsigned i = 0;; ++i)
3325 if (Ops[i] == AddRec) {
3326 Ops[i] = NewRec;
3327 break;
3328 }
3329 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3330 }
3331
3332 // Okay, if there weren't any loop invariants to be folded, check to see
3333 // if there are multiple AddRec's with the same loop induction variable
3334 // being multiplied together. If so, we can fold them.
3335
3336 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
3337 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
3338 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
3339 // ]]],+,...up to x=2n}.
3340 // Note that the arguments to choose() are always integers with values
3341 // known at compile time, never SCEV objects.
3342 //
3343 // The implementation avoids pointless extra computations when the two
3344 // addrec's are of different length (mathematically, it's equivalent to
3345 // an infinite stream of zeros on the right).
3346 bool OpsModified = false;
3347 for (unsigned OtherIdx = Idx+1;
3348 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3349 ++OtherIdx) {
3350 const SCEVAddRecExpr *OtherAddRec =
3351 dyn_cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
3352 if (!OtherAddRec || OtherAddRec->getLoop() != AddRec->getLoop())
3353 continue;
3354
3355 // Limit max number of arguments to avoid creation of unreasonably big
3356 // SCEVAddRecs with very complex operands.
3357 if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
3358 MaxAddRecSize || hasHugeExpression(Ops: {AddRec, OtherAddRec}))
3359 continue;
3360
3361 bool Overflow = false;
3362 Type *Ty = AddRec->getType();
3363 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
3364 SmallVector<SCEVUse, 7> AddRecOps;
3365 for (int x = 0, xe = AddRec->getNumOperands() +
3366 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
3367 SmallVector<SCEVUse, 7> SumOps;
3368 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3369 uint64_t Coeff1 = Choose(n: x, k: 2*x - y, Overflow);
3370 for (int z = std::max(a: y-x, b: y-(int)AddRec->getNumOperands()+1),
3371 ze = std::min(a: x+1, b: (int)OtherAddRec->getNumOperands());
3372 z < ze && !Overflow; ++z) {
3373 uint64_t Coeff2 = Choose(n: 2*x - y, k: x-z, Overflow);
3374 uint64_t Coeff;
3375 if (LargerThan64Bits)
3376 Coeff = umul_ov(i: Coeff1, j: Coeff2, Overflow);
3377 else
3378 Coeff = Coeff1*Coeff2;
3379 const SCEV *CoeffTerm = getConstant(Ty, V: Coeff);
3380 const SCEV *Term1 = AddRec->getOperand(i: y-z);
3381 const SCEV *Term2 = OtherAddRec->getOperand(i: z);
3382 SumOps.push_back(Elt: getMulExpr(Op0: CoeffTerm, Op1: Term1, Op2: Term2,
3383 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3384 }
3385 }
3386 if (SumOps.empty())
3387 SumOps.push_back(Elt: getZero(Ty));
3388 AddRecOps.push_back(Elt: getAddExpr(Ops&: SumOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3389 }
3390 if (!Overflow) {
3391 const SCEV *NewAddRec = getAddRecExpr(Operands&: AddRecOps, L: AddRec->getLoop(),
3392 Flags: SCEV::FlagAnyWrap);
3393 if (Ops.size() == 2) return NewAddRec;
3394 Ops[Idx] = NewAddRec;
3395 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
3396 OpsModified = true;
3397 AddRec = dyn_cast<SCEVAddRecExpr>(Val: NewAddRec);
3398 if (!AddRec)
3399 break;
3400 }
3401 }
3402 if (OpsModified)
3403 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3404
3405 // Otherwise couldn't fold anything into this recurrence. Move onto the
3406 // next one.
3407 }
3408
3409 // Okay, it looks like we really DO need an mul expr. Check to see if we
3410 // already have one, otherwise create a new one.
3411 return getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops));
3412}
3413
3414/// Represents an unsigned remainder expression based on unsigned division.
3415const SCEV *ScalarEvolution::getURemExpr(SCEVUse LHS, SCEVUse RHS) {
3416 assert(getEffectiveSCEVType(LHS->getType()) ==
3417 getEffectiveSCEVType(RHS->getType()) &&
3418 "SCEVURemExpr operand types don't match!");
3419
3420 // Short-circuit easy cases
3421 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3422 // If constant is one, the result is trivial
3423 if (RHSC->getValue()->isOne())
3424 return getZero(Ty: LHS->getType()); // X urem 1 --> 0
3425
3426 // If constant is a power of two, fold into a zext(trunc(LHS)).
3427 if (RHSC->getAPInt().isPowerOf2()) {
3428 Type *FullTy = LHS->getType();
3429 Type *TruncTy =
3430 IntegerType::get(C&: getContext(), NumBits: RHSC->getAPInt().logBase2());
3431 return getZeroExtendExpr(Op: getTruncateExpr(Op: LHS, Ty: TruncTy), Ty: FullTy);
3432 }
3433 }
3434
3435 // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
3436 const SCEV *UDiv = getUDivExpr(LHS, RHS);
3437 const SCEV *Mult = getMulExpr(LHS: UDiv, RHS, Flags: SCEV::FlagNUW);
3438 return getMinusSCEV(LHS, RHS: Mult, Flags: SCEV::FlagNUW);
3439}
3440
3441/// Get a canonical unsigned division expression, or something simpler if
3442/// possible.
3443const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
3444 assert(!LHS->getType()->isPointerTy() &&
3445 "SCEVUDivExpr operand can't be pointer!");
3446 assert(LHS->getType() == RHS->getType() &&
3447 "SCEVUDivExpr operand types don't match!");
3448
3449 if (SCEV *S =
3450 findExistingSCEVInCache(SCEVType: scUDivExpr, Ops: ArrayRef<SCEVUse>({LHS, RHS})))
3451 return S;
3452
3453 // 0 udiv Y == 0
3454 if (match(U: LHS, P: m_scev_Zero()))
3455 return LHS;
3456
3457 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3458 if (RHSC->getValue()->isOne())
3459 return LHS; // X udiv 1 --> x
3460 // If the denominator is zero, the result of the udiv is undefined. Don't
3461 // try to analyze it, because the resolution chosen here may differ from
3462 // the resolution chosen in other parts of the compiler.
3463 if (!RHSC->getValue()->isZero()) {
3464 // Determine if the division can be folded into the operands of
3465 // its operands.
3466 // TODO: Generalize this to non-constants by using known-bits information.
3467 Type *Ty = LHS->getType();
3468 unsigned LZ = RHSC->getAPInt().countl_zero();
3469 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
3470 // For non-power-of-two values, effectively round the value up to the
3471 // nearest power of two.
3472 if (!RHSC->getAPInt().isPowerOf2())
3473 ++MaxShiftAmt;
3474 IntegerType *ExtTy =
3475 IntegerType::get(C&: getContext(), NumBits: getTypeSizeInBits(Ty) + MaxShiftAmt);
3476 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
3477 if (const SCEVConstant *Step =
3478 dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE&: *this))) {
3479 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
3480 const APInt &StepInt = Step->getAPInt();
3481 const APInt &DivInt = RHSC->getAPInt();
3482 if (!StepInt.urem(RHS: DivInt) &&
3483 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3484 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3485 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy),
3486 L: AR->getLoop(), Flags: SCEV::FlagAnyWrap)) {
3487 SmallVector<SCEVUse, 4> Operands;
3488 for (const SCEV *Op : AR->operands())
3489 Operands.push_back(Elt: getUDivExpr(LHS: Op, RHS));
3490 return getAddRecExpr(Operands, L: AR->getLoop(), Flags: SCEV::FlagNW);
3491 }
3492 /// Get a canonical UDivExpr for a recurrence.
3493 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
3494 const APInt *StartRem;
3495 if (!DivInt.urem(RHS: StepInt) && match(S: getURemExpr(LHS: AR->getStart(), RHS: Step),
3496 P: m_scev_APInt(C&: StartRem))) {
3497 bool NoWrap =
3498 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3499 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3500 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy), L: AR->getLoop(),
3501 Flags: SCEV::FlagAnyWrap);
3502
3503 // With N <= C and both N, C as powers-of-2, the transformation
3504 // {X,+,N}/C => {(X - X%N),+,N}/C preserves division results even
3505 // if wrapping occurs, as the division results remain equivalent for
3506 // all offsets in [[(X - X%N), X).
3507 bool CanFoldWithWrap = StepInt.ule(RHS: DivInt) && // N <= C
3508 StepInt.isPowerOf2() && DivInt.isPowerOf2();
3509 // Only fold if the subtraction can be folded in the start
3510 // expression.
3511 const SCEV *NewStart =
3512 getMinusSCEV(LHS: AR->getStart(), RHS: getConstant(Val: *StartRem));
3513 if (*StartRem != 0 && (NoWrap || CanFoldWithWrap) &&
3514 !isa<SCEVAddExpr>(Val: NewStart)) {
3515 const SCEV *NewLHS =
3516 getAddRecExpr(Start: NewStart, Step, L: AR->getLoop(),
3517 Flags: NoWrap ? SCEV::FlagNW : SCEV::FlagAnyWrap);
3518 if (LHS != NewLHS)
3519 return getUDivExpr(LHS: NewLHS, RHS);
3520 }
3521 }
3522 }
3523 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
3524 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val&: LHS)) {
3525 if (M->hasNoUnsignedWrap()) {
3526 // Find an operand that's safely divisible.
3527 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3528 const SCEV *Op = M->getOperand(i);
3529 const SCEV *Div = getUDivExpr(LHS: Op, RHS: RHSC);
3530 if (!isa<SCEVUDivExpr>(Val: Div) && getMulExpr(LHS: Div, RHS: RHSC) == Op) {
3531 SmallVector<SCEVUse, 4> Operands(M->operands());
3532 Operands[i] = Div;
3533 return getMulExpr(Ops&: Operands);
3534 }
3535 }
3536
3537 // Even if it's not divisible, try to remove a common factor.
3538 if (const auto *LHSC = dyn_cast<SCEVConstant>(Val: M->getOperand(i: 0))) {
3539 APInt Factor = APIntOps::GreatestCommonDivisor(A: LHSC->getAPInt(),
3540 B: RHSC->getAPInt());
3541 if (!Factor.isIntN(N: 1)) {
3542 SmallVector<SCEVUse, 2> NewOperands;
3543 NewOperands.push_back(Elt: getConstant(Val: LHSC->getAPInt().udiv(RHS: Factor)));
3544 append_range(C&: NewOperands, R: M->operands().drop_front());
3545 const SCEV *NewMul = getMulExpr(Ops&: NewOperands);
3546 return getUDivExpr(LHS: NewMul,
3547 RHS: getConstant(Val: RHSC->getAPInt().udiv(RHS: Factor)));
3548 }
3549 }
3550 }
3551 }
3552
3553 // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
3554 if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(Val&: LHS)) {
3555 if (auto *DivisorConstant =
3556 dyn_cast<SCEVConstant>(Val: OtherDiv->getRHS())) {
3557 bool Overflow = false;
3558 APInt NewRHS =
3559 DivisorConstant->getAPInt().umul_ov(RHS: RHSC->getAPInt(), Overflow);
3560 if (Overflow) {
3561 return getConstant(Ty: RHSC->getType(), V: 0, isSigned: false);
3562 }
3563 return getUDivExpr(LHS: OtherDiv->getLHS(), RHS: getConstant(Val: NewRHS));
3564 }
3565 }
3566
3567 // (A+B)/C --> (A/C + B/C) if the add does not unsigned wrap and A/C and
3568 // B/C can be folded.
3569 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(Val&: LHS)) {
3570 if (A->hasNoUnsignedWrap()) {
3571 SmallVector<SCEVUse, 4> Operands;
3572 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
3573 const SCEV *Op = getUDivExpr(LHS: A->getOperand(i), RHS);
3574 if (isa<SCEVUDivExpr>(Val: Op) ||
3575 getMulExpr(LHS: Op, RHS) != A->getOperand(i))
3576 break;
3577 Operands.push_back(Elt: Op);
3578 }
3579 if (Operands.size() == A->getNumOperands())
3580 return getAddExpr(Ops&: Operands);
3581 }
3582 }
3583
3584 // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
3585 // This is an idiom for rounding A up to the next multiple of N, where A
3586 // is aready known to be a multiple of M. In this case, instcombine can
3587 // see that some low bits of the added constant are unused, so can clear
3588 // them, but we want to canonicalise to set the low bits. This makes the
3589 // pattern easier to match, without needing to check for known bits in
3590 // A*M.
3591 const APInt &N = RHSC->getAPInt();
3592 const APInt *NMinusM, *M;
3593 const SCEV *A;
3594 if (match(U: LHS, P: m_scev_Add(Op0: m_scev_APInt(C&: NMinusM),
3595 Op1: m_scev_Mul(Op0: m_scev_APInt(C&: M), Op1: m_SCEV(V&: A))))) {
3596 if (N.isPowerOf2() && M->isPowerOf2() && M->ult(RHS: N) &&
3597 *NMinusM == N - *M) {
3598 return getUDivExpr(
3599 LHS: getAddExpr(LHS: getConstant(Val: N - 1), RHS: getMulExpr(LHS: getConstant(Val: *M), RHS: A)),
3600 RHS);
3601 }
3602 }
3603
3604 // Fold if both operands are constant.
3605 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS))
3606 return getConstant(Val: LHSC->getAPInt().udiv(RHS: RHSC->getAPInt()));
3607 }
3608 }
3609
3610 // ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
3611 const APInt *NegC, *C;
3612 if (match(U: LHS,
3613 P: m_scev_Add(Op0: m_scev_APInt(C&: NegC),
3614 Op1: m_scev_SMax(Op0: m_scev_APInt(C), Op1: m_scev_Specific(S: RHS)))) &&
3615 NegC->isNegative() && !NegC->isMinSignedValue() && *C == -*NegC)
3616 return getZero(Ty: LHS->getType());
3617
3618 // (%a * %b)<nuw> / %b -> %a
3619 const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: LHS);
3620 if (Mul && Mul->hasNoUnsignedWrap()) {
3621 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
3622 if (Mul->getOperand(i) == RHS) {
3623 SmallVector<SCEVUse, 2> Operands;
3624 append_range(C&: Operands, R: Mul->operands().take_front(N: i));
3625 append_range(C&: Operands, R: Mul->operands().drop_front(N: i + 1));
3626 return getMulExpr(Ops&: Operands);
3627 }
3628 }
3629 }
3630
3631 // TODO: Generalize to handle any common factors.
3632 // udiv (mul nuw a, vscale), (mul nuw b, vscale) --> udiv a, b
3633 const SCEV *NewLHS, *NewRHS;
3634 if (match(U: LHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
3635 match(U: RHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale())))
3636 return getUDivExpr(LHS: NewLHS, RHS: NewRHS);
3637
3638 return getOrCreateUDivExpr(LHS, RHS);
3639}
3640
3641/// Get a canonical unsigned division expression, or something simpler if
3642/// possible. There is no representation for an exact udiv in SCEV IR, but we
3643/// can attempt to optimize it prior to construction.
3644const SCEV *ScalarEvolution::getUDivExactExpr(SCEVUse LHS, SCEVUse RHS) {
3645 // Currently there is no exact specific logic.
3646
3647 return getUDivExpr(LHS, RHS);
3648}
3649
3650/// Get an add recurrence expression for the specified loop. Simplify the
3651/// expression as much as possible.
3652const SCEV *ScalarEvolution::getAddRecExpr(SCEVUse Start, SCEVUse Step,
3653 const Loop *L,
3654 SCEV::NoWrapFlags Flags) {
3655 SmallVector<SCEVUse, 4> Operands;
3656 Operands.push_back(Elt: Start);
3657 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Val&: Step))
3658 if (StepChrec->getLoop() == L) {
3659 append_range(C&: Operands, R: StepChrec->operands());
3660 return getAddRecExpr(Operands, L, Flags: maskFlags(Flags, Mask: SCEV::FlagNW));
3661 }
3662
3663 Operands.push_back(Elt: Step);
3664 return getAddRecExpr(Operands, L, Flags);
3665}
3666
3667/// Get an add recurrence expression for the specified loop. Simplify the
3668/// expression as much as possible.
3669const SCEV *ScalarEvolution::getAddRecExpr(SmallVectorImpl<SCEVUse> &Operands,
3670 const Loop *L,
3671 SCEV::NoWrapFlags Flags) {
3672 if (Operands.size() == 1) return Operands[0];
3673#ifndef NDEBUG
3674 Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
3675 for (const SCEV *Op : llvm::drop_begin(Operands)) {
3676 assert(getEffectiveSCEVType(Op->getType()) == ETy &&
3677 "SCEVAddRecExpr operand types don't match!");
3678 assert(!Op->getType()->isPointerTy() && "Step must be integer");
3679 }
3680 for (const SCEV *Op : Operands)
3681 assert(isAvailableAtLoopEntry(Op, L) &&
3682 "SCEVAddRecExpr operand is not available at loop entry!");
3683#endif
3684
3685 if (Operands.back()->isZero()) {
3686 Operands.pop_back();
3687 return getAddRecExpr(Operands, L, Flags: SCEV::FlagAnyWrap); // {X,+,0} --> X
3688 }
3689
3690 // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
3691 // use that information to infer NUW and NSW flags. However, computing a
3692 // BE count requires calling getAddRecExpr, so we may not yet have a
3693 // meaningful BE count at this point (and if we don't, we'd be stuck
3694 // with a SCEVCouldNotCompute as the cached BE count).
3695
3696 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
3697
3698 // Canonicalize nested AddRecs in by nesting them in order of loop depth.
3699 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Val&: Operands[0])) {
3700 const Loop *NestedLoop = NestedAR->getLoop();
3701 if (L->contains(L: NestedLoop)
3702 ? (L->getLoopDepth() < NestedLoop->getLoopDepth())
3703 : (!NestedLoop->contains(L) &&
3704 DT.dominates(A: L->getHeader(), B: NestedLoop->getHeader()))) {
3705 SmallVector<SCEVUse, 4> NestedOperands(NestedAR->operands());
3706 Operands[0] = NestedAR->getStart();
3707 // AddRecs require their operands be loop-invariant with respect to their
3708 // loops. Don't perform this transformation if it would break this
3709 // requirement.
3710 bool AllInvariant = all_of(
3711 Range&: Operands, P: [&](const SCEV *Op) { return isLoopInvariant(S: Op, L); });
3712
3713 if (AllInvariant) {
3714 // Create a recurrence for the outer loop with the same step size.
3715 //
3716 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
3717 // inner recurrence has the same property.
3718 SCEV::NoWrapFlags OuterFlags =
3719 maskFlags(Flags, Mask: SCEV::FlagNW | NestedAR->getNoWrapFlags());
3720
3721 NestedOperands[0] = getAddRecExpr(Operands, L, Flags: OuterFlags);
3722 AllInvariant = all_of(Range&: NestedOperands, P: [&](const SCEV *Op) {
3723 return isLoopInvariant(S: Op, L: NestedLoop);
3724 });
3725
3726 if (AllInvariant) {
3727 // Ok, both add recurrences are valid after the transformation.
3728 //
3729 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
3730 // the outer recurrence has the same property.
3731 SCEV::NoWrapFlags InnerFlags =
3732 maskFlags(Flags: NestedAR->getNoWrapFlags(), Mask: SCEV::FlagNW | Flags);
3733 return getAddRecExpr(Operands&: NestedOperands, L: NestedLoop, Flags: InnerFlags);
3734 }
3735 }
3736 // Reset Operands to its original state.
3737 Operands[0] = NestedAR;
3738 }
3739 }
3740
3741 // Okay, it looks like we really DO need an addrec expr. Check to see if we
3742 // already have one, otherwise create a new one.
3743 return getOrCreateAddRecExpr(Ops: Operands, L, Flags);
3744}
3745
3746const SCEV *ScalarEvolution::getGEPExpr(GEPOperator *GEP,
3747 ArrayRef<SCEVUse> IndexExprs) {
3748 const SCEV *BaseExpr = getSCEV(V: GEP->getPointerOperand());
3749 // getSCEV(Base)->getType() has the same address space as Base->getType()
3750 // because SCEV::getType() preserves the address space.
3751 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
3752 if (NW != GEPNoWrapFlags::none()) {
3753 // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
3754 // but to do that, we have to ensure that said flag is valid in the entire
3755 // defined scope of the SCEV.
3756 // TODO: non-instructions have global scope. We might be able to prove
3757 // some global scope cases
3758 auto *GEPI = dyn_cast<Instruction>(Val: GEP);
3759 if (!GEPI || !isSCEVExprNeverPoison(I: GEPI))
3760 NW = GEPNoWrapFlags::none();
3761 }
3762
3763 return getGEPExpr(BaseExpr, IndexExprs, SrcElementTy: GEP->getSourceElementType(), NW);
3764}
3765
3766const SCEV *ScalarEvolution::getGEPExpr(SCEVUse BaseExpr,
3767 ArrayRef<SCEVUse> IndexExprs,
3768 Type *SrcElementTy, GEPNoWrapFlags NW) {
3769 SCEV::NoWrapFlags OffsetWrap = SCEV::FlagAnyWrap;
3770 if (NW.hasNoUnsignedSignedWrap())
3771 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNSW);
3772 if (NW.hasNoUnsignedWrap())
3773 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNUW);
3774
3775 Type *CurTy = BaseExpr->getType();
3776 Type *IntIdxTy = getEffectiveSCEVType(Ty: BaseExpr->getType());
3777 bool FirstIter = true;
3778 SmallVector<SCEVUse, 4> Offsets;
3779 for (SCEVUse IndexExpr : IndexExprs) {
3780 // Compute the (potentially symbolic) offset in bytes for this index.
3781 if (StructType *STy = dyn_cast<StructType>(Val: CurTy)) {
3782 // For a struct, add the member offset.
3783 ConstantInt *Index = cast<SCEVConstant>(Val&: IndexExpr)->getValue();
3784 unsigned FieldNo = Index->getZExtValue();
3785 const SCEV *FieldOffset = getOffsetOfExpr(IntTy: IntIdxTy, STy, FieldNo);
3786 Offsets.push_back(Elt: FieldOffset);
3787
3788 // Update CurTy to the type of the field at Index.
3789 CurTy = STy->getTypeAtIndex(V: Index);
3790 } else {
3791 // Update CurTy to its element type.
3792 if (FirstIter) {
3793 assert(isa<PointerType>(CurTy) &&
3794 "The first index of a GEP indexes a pointer");
3795 CurTy = SrcElementTy;
3796 FirstIter = false;
3797 } else {
3798 CurTy = GetElementPtrInst::getTypeAtIndex(Ty: CurTy, Idx: (uint64_t)0);
3799 }
3800 // For an array, add the element offset, explicitly scaled.
3801 const SCEV *ElementSize = getSizeOfExpr(IntTy: IntIdxTy, AllocTy: CurTy);
3802 // Getelementptr indices are signed.
3803 IndexExpr = getTruncateOrSignExtend(V: IndexExpr, Ty: IntIdxTy);
3804
3805 // Multiply the index by the element size to compute the element offset.
3806 const SCEV *LocalOffset = getMulExpr(LHS: IndexExpr, RHS: ElementSize, Flags: OffsetWrap);
3807 Offsets.push_back(Elt: LocalOffset);
3808 }
3809 }
3810
3811 // Handle degenerate case of GEP without offsets.
3812 if (Offsets.empty())
3813 return BaseExpr;
3814
3815 // Add the offsets together, assuming nsw if inbounds.
3816 const SCEV *Offset = getAddExpr(Ops&: Offsets, OrigFlags: OffsetWrap);
3817 // Add the base address and the offset. We cannot use the nsw flag, as the
3818 // base address is unsigned. However, if we know that the offset is
3819 // non-negative, we can use nuw.
3820 bool NUW = NW.hasNoUnsignedWrap() ||
3821 (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Offset));
3822 SCEV::NoWrapFlags BaseWrap = NUW ? SCEV::FlagNUW : SCEV::FlagAnyWrap;
3823 auto *GEPExpr = getAddExpr(LHS: BaseExpr, RHS: Offset, Flags: BaseWrap);
3824 assert(BaseExpr->getType() == GEPExpr->getType() &&
3825 "GEP should not change type mid-flight.");
3826 return GEPExpr;
3827}
3828
3829SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
3830 ArrayRef<SCEVUse> Ops) {
3831 FoldingSetNodeID ID;
3832 ID.AddInteger(I: SCEVType);
3833 for (SCEVUse Op : Ops)
3834 ID.AddPointer(Ptr: Op.getOpaqueValue());
3835 FoldingSetInsertToken Token;
3836 return UniqueSCEVs.lookup(ID, Token);
3837}
3838
3839const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
3840 SCEV::NoWrapFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
3841 return getSMaxExpr(LHS: Op, RHS: getNegativeSCEV(V: Op, Flags));
3842}
3843
3844const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
3845 SmallVectorImpl<SCEVUse> &Ops) {
3846 assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
3847 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
3848 if (Ops.size() == 1) return Ops[0];
3849#ifndef NDEBUG
3850 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
3851 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
3852 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
3853 "Operand types don't match!");
3854 assert(Ops[0]->getType()->isPointerTy() ==
3855 Ops[i]->getType()->isPointerTy() &&
3856 "min/max should be consistently pointerish");
3857 }
3858#endif
3859
3860 bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
3861 bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
3862
3863 const SCEV *Folded = constantFoldAndGroupOps(
3864 SE&: *this, LI, DT, Ops,
3865 Fold: [&](const APInt &C1, const APInt &C2) {
3866 switch (Kind) {
3867 case scSMaxExpr:
3868 return APIntOps::smax(A: C1, B: C2);
3869 case scSMinExpr:
3870 return APIntOps::smin(A: C1, B: C2);
3871 case scUMaxExpr:
3872 return APIntOps::umax(A: C1, B: C2);
3873 case scUMinExpr:
3874 return APIntOps::umin(A: C1, B: C2);
3875 default:
3876 llvm_unreachable("Unknown SCEV min/max opcode");
3877 }
3878 },
3879 IsIdentity: [&](const APInt &C) {
3880 // identity
3881 if (IsMax)
3882 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3883 else
3884 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3885 },
3886 IsAbsorber: [&](const APInt &C) {
3887 // absorber
3888 if (IsMax)
3889 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3890 else
3891 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3892 });
3893 if (Folded)
3894 return Folded;
3895
3896 // Check if we have created the same expression before.
3897 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops)) {
3898 return S;
3899 }
3900
3901 // Find the first operation of the same kind
3902 unsigned Idx = 0;
3903 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
3904 ++Idx;
3905
3906 // Check to see if one of the operands is of the same kind. If so, expand its
3907 // operands onto our operand list, and recurse to simplify.
3908 if (Idx < Ops.size()) {
3909 bool DeletedAny = false;
3910 while (Ops[Idx]->getSCEVType() == Kind) {
3911 const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Val&: Ops[Idx]);
3912 Ops.erase(CI: Ops.begin()+Idx);
3913 append_range(C&: Ops, R: SMME->operands());
3914 DeletedAny = true;
3915 }
3916
3917 if (DeletedAny)
3918 return getMinMaxExpr(Kind, Ops);
3919 }
3920
3921 // Okay, check to see if the same value occurs in the operand list twice. If
3922 // so, delete one. Since we sorted the list, these values are required to
3923 // be adjacent.
3924 llvm::CmpInst::Predicate GEPred =
3925 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
3926 llvm::CmpInst::Predicate LEPred =
3927 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
3928 llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
3929 llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
3930 for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
3931 if (Ops[i] == Ops[i + 1] ||
3932 isKnownViaNonRecursiveReasoning(Pred: FirstPred, LHS: Ops[i], RHS: Ops[i + 1])) {
3933 // X op Y op Y --> X op Y
3934 // X op Y --> X, if we know X, Y are ordered appropriately
3935 Ops.erase(CS: Ops.begin() + i + 1, CE: Ops.begin() + i + 2);
3936 --i;
3937 --e;
3938 } else if (isKnownViaNonRecursiveReasoning(Pred: SecondPred, LHS: Ops[i],
3939 RHS: Ops[i + 1])) {
3940 // X op Y --> Y, if we know X, Y are ordered appropriately
3941 Ops.erase(CS: Ops.begin() + i, CE: Ops.begin() + i + 1);
3942 --i;
3943 --e;
3944 }
3945 }
3946
3947 if (Ops.size() == 1) return Ops[0];
3948
3949 assert(!Ops.empty() && "Reduced smax down to nothing!");
3950
3951 // Okay, it looks like we really DO need an expr. Check to see if we
3952 // already have one, otherwise create a new one.
3953 FoldingSetNodeID ID;
3954 ID.AddInteger(I: Kind);
3955 for (SCEVUse Op : Ops)
3956 ID.AddPointer(Ptr: Op.getOpaqueValue());
3957 FoldingSetInsertToken Token;
3958 const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
3959 if (ExistingSCEV)
3960 return ExistingSCEV;
3961 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3962 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3963 SCEV *S = new (SCEVAllocator)
3964 SCEVMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
3965
3966 UniqueSCEVs.insert(N: S, Token);
3967 S->computeAndSetCanonical(SE&: *this);
3968 registerUser(User: S, Ops);
3969 return S;
3970}
3971
3972namespace {
3973
3974class SCEVSequentialMinMaxDeduplicatingVisitor final
3975 : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
3976 std::optional<const SCEV *>> {
3977 using RetVal = std::optional<const SCEV *>;
3978 using Base = SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor, RetVal>;
3979
3980 ScalarEvolution &SE;
3981 const SCEVTypes RootKind; // Must be a sequential min/max expression.
3982 const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
3983 SmallPtrSet<const SCEV *, 16> SeenOps;
3984
3985 bool canRecurseInto(SCEVTypes Kind) const {
3986 // We can only recurse into the SCEV expression of the same effective type
3987 // as the type of our root SCEV expression.
3988 return RootKind == Kind || NonSequentialRootKind == Kind;
3989 };
3990
3991 RetVal visitAnyMinMaxExpr(const SCEV *S) {
3992 assert((isa<SCEVMinMaxExpr>(S) || isa<SCEVSequentialMinMaxExpr>(S)) &&
3993 "Only for min/max expressions.");
3994 SCEVTypes Kind = S->getSCEVType();
3995
3996 if (!canRecurseInto(Kind))
3997 return S;
3998
3999 auto *NAry = cast<SCEVNAryExpr>(Val: S);
4000 SmallVector<SCEVUse> NewOps;
4001 bool Changed = visit(Kind, OrigOps: NAry->operands(), NewOps);
4002
4003 if (!Changed)
4004 return S;
4005 if (NewOps.empty())
4006 return std::nullopt;
4007
4008 return isa<SCEVSequentialMinMaxExpr>(Val: S)
4009 ? SE.getSequentialMinMaxExpr(Kind, Operands&: NewOps)
4010 : SE.getMinMaxExpr(Kind, Ops&: NewOps);
4011 }
4012
4013 RetVal visit(const SCEV *S) {
4014 // Has the whole operand been seen already?
4015 if (!SeenOps.insert(Ptr: S).second)
4016 return std::nullopt;
4017 return Base::visit(S);
4018 }
4019
4020public:
4021 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
4022 SCEVTypes RootKind)
4023 : SE(SE), RootKind(RootKind),
4024 NonSequentialRootKind(
4025 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
4026 Ty: RootKind)) {}
4027
4028 bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<SCEVUse> OrigOps,
4029 SmallVectorImpl<SCEVUse> &NewOps) {
4030 bool Changed = false;
4031 SmallVector<SCEVUse> Ops;
4032 Ops.reserve(N: OrigOps.size());
4033
4034 for (const SCEV *Op : OrigOps) {
4035 RetVal NewOp = visit(S: Op);
4036 if (NewOp != Op)
4037 Changed = true;
4038 if (NewOp)
4039 Ops.emplace_back(Args&: *NewOp);
4040 }
4041
4042 if (Changed)
4043 NewOps = std::move(Ops);
4044 return Changed;
4045 }
4046
4047 RetVal visitConstant(const SCEVConstant *Constant) { return Constant; }
4048
4049 RetVal visitVScale(const SCEVVScale *VScale) { return VScale; }
4050
4051 RetVal visitPtrToAddrExpr(const SCEVPtrToAddrExpr *Expr) { return Expr; }
4052
4053 RetVal visitTruncateExpr(const SCEVTruncateExpr *Expr) { return Expr; }
4054
4055 RetVal visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { return Expr; }
4056
4057 RetVal visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { return Expr; }
4058
4059 RetVal visitAddExpr(const SCEVAddExpr *Expr) { return Expr; }
4060
4061 RetVal visitMulExpr(const SCEVMulExpr *Expr) { return Expr; }
4062
4063 RetVal visitUDivExpr(const SCEVUDivExpr *Expr) { return Expr; }
4064
4065 RetVal visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
4066
4067 RetVal visitSMaxExpr(const SCEVSMaxExpr *Expr) {
4068 return visitAnyMinMaxExpr(S: Expr);
4069 }
4070
4071 RetVal visitUMaxExpr(const SCEVUMaxExpr *Expr) {
4072 return visitAnyMinMaxExpr(S: Expr);
4073 }
4074
4075 RetVal visitSMinExpr(const SCEVSMinExpr *Expr) {
4076 return visitAnyMinMaxExpr(S: Expr);
4077 }
4078
4079 RetVal visitUMinExpr(const SCEVUMinExpr *Expr) {
4080 return visitAnyMinMaxExpr(S: Expr);
4081 }
4082
4083 RetVal visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Expr) {
4084 return visitAnyMinMaxExpr(S: Expr);
4085 }
4086
4087 RetVal visitUnknown(const SCEVUnknown *Expr) { return Expr; }
4088
4089 RetVal visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { return Expr; }
4090};
4091
4092} // namespace
4093
4094static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind) {
4095 switch (Kind) {
4096 case scConstant:
4097 case scVScale:
4098 case scTruncate:
4099 case scZeroExtend:
4100 case scSignExtend:
4101 case scPtrToAddr:
4102 case scAddExpr:
4103 case scMulExpr:
4104 case scUDivExpr:
4105 case scAddRecExpr:
4106 case scUMaxExpr:
4107 case scSMaxExpr:
4108 case scUMinExpr:
4109 case scSMinExpr:
4110 case scUnknown:
4111 // If any operand is poison, the whole expression is poison.
4112 return true;
4113 case scSequentialUMinExpr:
4114 // FIXME: if the *first* operand is poison, the whole expression is poison.
4115 return false; // Pessimistically, say that it does not propagate poison.
4116 case scCouldNotCompute:
4117 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
4118 }
4119 llvm_unreachable("Unknown SCEV kind!");
4120}
4121
4122namespace {
4123// The only way poison may be introduced in a SCEV expression is from a
4124// poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown,
4125// not SCEVConstant). Notably, nowrap flags in SCEV nodes can *not*
4126// introduce poison -- they encode guaranteed, non-speculated knowledge.
4127//
4128// Additionally, all SCEV nodes propagate poison from inputs to outputs,
4129// with the notable exception of umin_seq, where only poison from the first
4130// operand is (unconditionally) propagated.
4131struct SCEVPoisonCollector {
4132 bool LookThroughMaybePoisonBlocking;
4133 SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
4134 SCEVPoisonCollector(bool LookThroughMaybePoisonBlocking)
4135 : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
4136
4137 bool follow(const SCEV *S) {
4138 if (!LookThroughMaybePoisonBlocking &&
4139 !scevUnconditionallyPropagatesPoisonFromOperands(Kind: S->getSCEVType()))
4140 return false;
4141
4142 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
4143 if (!isGuaranteedNotToBePoison(V: SU->getValue()))
4144 MaybePoison.insert(Ptr: SU);
4145 }
4146 return true;
4147 }
4148 bool isDone() const { return false; }
4149};
4150} // namespace
4151
4152/// Return true if V is poison given that AssumedPoison is already poison.
4153static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) {
4154 // First collect all SCEVs that might result in AssumedPoison to be poison.
4155 // We need to look through potentially poison-blocking operations here,
4156 // because we want to find all SCEVs that *might* result in poison, not only
4157 // those that are *required* to.
4158 SCEVPoisonCollector PC1(/* LookThroughMaybePoisonBlocking */ true);
4159 visitAll(Root: AssumedPoison, Visitor&: PC1);
4160
4161 // AssumedPoison is never poison. As the assumption is false, the implication
4162 // is true. Don't bother walking the other SCEV in this case.
4163 if (PC1.MaybePoison.empty())
4164 return true;
4165
4166 // Collect all SCEVs in S that, if poison, *will* result in S being poison
4167 // as well. We cannot look through potentially poison-blocking operations
4168 // here, as their arguments only *may* make the result poison.
4169 SCEVPoisonCollector PC2(/* LookThroughMaybePoisonBlocking */ false);
4170 visitAll(Root: S, Visitor&: PC2);
4171
4172 // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison,
4173 // it will also make S poison by being part of PC2.MaybePoison.
4174 return llvm::set_is_subset(S1: PC1.MaybePoison, S2: PC2.MaybePoison);
4175}
4176
4177void ScalarEvolution::getPoisonGeneratingValues(
4178 SmallPtrSetImpl<const Value *> &Result, const SCEV *S) {
4179 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ false);
4180 visitAll(Root: S, Visitor&: PC);
4181 for (const SCEVUnknown *SU : PC.MaybePoison)
4182 Result.insert(Ptr: SU->getValue());
4183}
4184
4185bool ScalarEvolution::canReuseInstruction(
4186 const SCEV *S, Instruction *I,
4187 SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts) {
4188 // If the instruction cannot be poison, it's always safe to reuse.
4189 if (programUndefinedIfPoison(Inst: I))
4190 return true;
4191
4192 // Otherwise, it is possible that I is more poisonous that S. Collect the
4193 // poison-contributors of S, and then check whether I has any additional
4194 // poison-contributors. Poison that is contributed through poison-generating
4195 // flags is handled by dropping those flags instead.
4196 SmallPtrSet<const Value *, 8> PoisonVals;
4197 getPoisonGeneratingValues(Result&: PoisonVals, S);
4198
4199 SmallVector<Value *> Worklist;
4200 SmallPtrSet<Value *, 8> Visited;
4201 Worklist.push_back(Elt: I);
4202 while (!Worklist.empty()) {
4203 Value *V = Worklist.pop_back_val();
4204 if (!Visited.insert(Ptr: V).second)
4205 continue;
4206
4207 // Avoid walking large instruction graphs.
4208 if (Visited.size() > 16)
4209 return false;
4210
4211 // Either the value can't be poison, or the S would also be poison if it
4212 // is.
4213 if (PoisonVals.contains(Ptr: V) || ::isGuaranteedNotToBePoison(V))
4214 continue;
4215
4216 auto *I = dyn_cast<Instruction>(Val: V);
4217 if (!I)
4218 return false;
4219
4220 // Disjoint or instructions are interpreted as adds by SCEV. However, we
4221 // can't replace an arbitrary add with disjoint or, even if we drop the
4222 // flag. We would need to convert the or into an add.
4223 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: I))
4224 if (PDI->isDisjoint())
4225 return false;
4226
4227 // FIXME: Ignore vscale, even though it technically could be poison. Do this
4228 // because SCEV currently assumes it can't be poison. Remove this special
4229 // case once we proper model when vscale can be poison.
4230 if (auto *II = dyn_cast<IntrinsicInst>(Val: I);
4231 II && II->getIntrinsicID() == Intrinsic::vscale)
4232 continue;
4233
4234 if (canCreatePoison(Op: cast<Operator>(Val: I), /*ConsiderFlagsAndMetadata*/ false))
4235 return false;
4236
4237 // If the instruction can't create poison, we can recurse to its operands.
4238 if (I->hasPoisonGeneratingAnnotations())
4239 DropPoisonGeneratingInsts.push_back(Elt: I);
4240
4241 llvm::append_range(C&: Worklist, R: I->operands());
4242 }
4243 return true;
4244}
4245
4246const SCEV *
4247ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
4248 SmallVectorImpl<SCEVUse> &Ops) {
4249 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
4250 "Not a SCEVSequentialMinMaxExpr!");
4251 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4252 if (Ops.size() == 1)
4253 return Ops[0];
4254#ifndef NDEBUG
4255 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4256 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4257 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4258 "Operand types don't match!");
4259 assert(Ops[0]->getType()->isPointerTy() ==
4260 Ops[i]->getType()->isPointerTy() &&
4261 "min/max should be consistently pointerish");
4262 }
4263#endif
4264
4265 // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
4266 // so we can *NOT* do any kind of sorting of the expressions!
4267
4268 // Check if we have created the same expression before.
4269 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops))
4270 return S;
4271
4272 // FIXME: there are *some* simplifications that we can do here.
4273
4274 // Keep only the first instance of an operand.
4275 {
4276 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
4277 bool Changed = Deduplicator.visit(Kind, OrigOps: Ops, NewOps&: Ops);
4278 if (Changed)
4279 return getSequentialMinMaxExpr(Kind, Ops);
4280 }
4281
4282 // Check to see if one of the operands is of the same kind. If so, expand its
4283 // operands onto our operand list, and recurse to simplify.
4284 {
4285 unsigned Idx = 0;
4286 bool DeletedAny = false;
4287 while (Idx < Ops.size()) {
4288 if (Ops[Idx]->getSCEVType() != Kind) {
4289 ++Idx;
4290 continue;
4291 }
4292 const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Val&: Ops[Idx]);
4293 Ops.erase(CI: Ops.begin() + Idx);
4294 Ops.insert(I: Ops.begin() + Idx, From: SMME->operands().begin(),
4295 To: SMME->operands().end());
4296 DeletedAny = true;
4297 }
4298
4299 if (DeletedAny)
4300 return getSequentialMinMaxExpr(Kind, Ops);
4301 }
4302
4303 const SCEV *SaturationPoint;
4304 ICmpInst::Predicate Pred;
4305 switch (Kind) {
4306 case scSequentialUMinExpr:
4307 SaturationPoint = getZero(Ty: Ops[0]->getType());
4308 Pred = ICmpInst::ICMP_ULE;
4309 break;
4310 default:
4311 llvm_unreachable("Not a sequential min/max type.");
4312 }
4313
4314 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4315 if (!isGuaranteedNotToCauseUB(Op: Ops[i]))
4316 continue;
4317 // We can replace %x umin_seq %y with %x umin %y if either:
4318 // * %y being poison implies %x is also poison.
4319 // * %x cannot be the saturating value (e.g. zero for umin).
4320 if (::impliesPoison(AssumedPoison: Ops[i], S: Ops[i - 1]) ||
4321 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_NE, LHS: Ops[i - 1],
4322 RHS: SaturationPoint)) {
4323 SmallVector<SCEVUse, 2> SeqOps = {Ops[i - 1], Ops[i]};
4324 Ops[i - 1] = getMinMaxExpr(
4325 Kind: SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Ty: Kind),
4326 Ops&: SeqOps);
4327 Ops.erase(CI: Ops.begin() + i);
4328 return getSequentialMinMaxExpr(Kind, Ops);
4329 }
4330 // Fold %x umin_seq %y to %x if %x ule %y.
4331 // TODO: We might be able to prove the predicate for a later operand.
4332 if (isKnownViaNonRecursiveReasoning(Pred, LHS: Ops[i - 1], RHS: Ops[i])) {
4333 Ops.erase(CI: Ops.begin() + i);
4334 return getSequentialMinMaxExpr(Kind, Ops);
4335 }
4336 }
4337
4338 // Okay, it looks like we really DO need an expr. Check to see if we
4339 // already have one, otherwise create a new one.
4340 FoldingSetNodeID ID;
4341 ID.AddInteger(I: Kind);
4342 for (SCEVUse Op : Ops)
4343 ID.AddPointer(Ptr: Op.getOpaqueValue());
4344 FoldingSetInsertToken Token;
4345 const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
4346 if (ExistingSCEV)
4347 return ExistingSCEV;
4348
4349 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
4350 llvm::uninitialized_copy(Src&: Ops, Dst: O);
4351 SCEV *S = new (SCEVAllocator)
4352 SCEVSequentialMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
4353
4354 UniqueSCEVs.insert(N: S, Token);
4355 S->computeAndSetCanonical(SE&: *this);
4356 registerUser(User: S, Ops);
4357 return S;
4358}
4359
4360const SCEV *ScalarEvolution::getSMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4361 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4362 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4363}
4364
4365const SCEV *ScalarEvolution::getSMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4366 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4367}
4368
4369const SCEV *ScalarEvolution::getUMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4370 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4371 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4372}
4373
4374const SCEV *ScalarEvolution::getUMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4375 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4376}
4377
4378const SCEV *ScalarEvolution::getSMinExpr(SCEVUse LHS, SCEVUse RHS) {
4379 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4380 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4381}
4382
4383const SCEV *ScalarEvolution::getSMinExpr(SmallVectorImpl<SCEVUse> &Ops) {
4384 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4385}
4386
4387const SCEV *ScalarEvolution::getUMinExpr(SCEVUse LHS, SCEVUse RHS,
4388 bool Sequential) {
4389 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4390 return getUMinExpr(Operands&: Ops, Sequential);
4391}
4392
4393const SCEV *ScalarEvolution::getUMinExpr(SmallVectorImpl<SCEVUse> &Ops,
4394 bool Sequential) {
4395 return Sequential ? getSequentialMinMaxExpr(Kind: scSequentialUMinExpr, Ops)
4396 : getMinMaxExpr(Kind: scUMinExpr, Ops);
4397}
4398
4399const SCEV *
4400ScalarEvolution::getSizeOfExpr(Type *IntTy, TypeSize Size) {
4401 const SCEV *Res = getConstant(Ty: IntTy, V: Size.getKnownMinValue());
4402 if (Size.isScalable())
4403 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty: IntTy));
4404 return Res;
4405}
4406
4407const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
4408 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeAllocSize(Ty: AllocTy));
4409}
4410
4411const SCEV *ScalarEvolution::getStoreSizeOfExpr(Type *IntTy, Type *StoreTy) {
4412 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeStoreSize(Ty: StoreTy));
4413}
4414
4415const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
4416 StructType *STy,
4417 unsigned FieldNo) {
4418 // We can bypass creating a target-independent constant expression and then
4419 // folding it back into a ConstantInt. This is just a compile-time
4420 // optimization.
4421 const StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
4422 assert(!SL->getSizeInBits().isScalable() &&
4423 "Cannot get offset for structure containing scalable vector types");
4424 return getConstant(Ty: IntTy, V: SL->getElementOffset(Idx: FieldNo));
4425}
4426
4427const SCEV *ScalarEvolution::getUnknown(Value *V) {
4428 // Don't attempt to do anything other than create a SCEVUnknown object
4429 // here. createSCEV only calls getUnknown after checking for all other
4430 // interesting possibilities, and any other code that calls getUnknown
4431 // is doing so in order to hide a value from SCEV canonicalization.
4432
4433 FoldingSetNodeID ID;
4434 ID.AddInteger(I: scUnknown);
4435 ID.AddPointer(Ptr: V);
4436 FoldingSetInsertToken Token;
4437 if (SCEV *S = UniqueSCEVs.lookup(ID, Token)) {
4438 assert(cast<SCEVUnknown>(S)->getValue() == V &&
4439 "Stale SCEVUnknown in uniquing map!");
4440 return S;
4441 }
4442 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(Allocator&: SCEVAllocator), V, this,
4443 FirstUnknown);
4444 FirstUnknown = cast<SCEVUnknown>(Val: S);
4445 UniqueSCEVs.insert(N: S, Token);
4446 S->computeAndSetCanonical(SE&: *this);
4447 return S;
4448}
4449
4450//===----------------------------------------------------------------------===//
4451// Basic SCEV Analysis and PHI Idiom Recognition Code
4452//
4453
4454/// Test if values of the given type are analyzable within the SCEV
4455/// framework. This primarily includes integer types, and it can optionally
4456/// include pointer types if the ScalarEvolution class has access to
4457/// target-specific information.
4458bool ScalarEvolution::isSCEVable(Type *Ty) const {
4459 // Integers and pointers are always SCEVable.
4460 return Ty->isIntOrPtrTy();
4461}
4462
4463/// Return the size in bits of the specified type, for which isSCEVable must
4464/// return true.
4465uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
4466 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4467 if (Ty->isPointerTy())
4468 return getDataLayout().getIndexTypeSizeInBits(Ty);
4469 return getDataLayout().getTypeSizeInBits(Ty);
4470}
4471
4472/// Return a type with the same bitwidth as the given type and which represents
4473/// how SCEV will treat the given type, for which isSCEVable must return
4474/// true. For pointer types, this is the pointer index sized integer type.
4475Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
4476 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4477
4478 if (Ty->isIntegerTy())
4479 return Ty;
4480
4481 // The only other support type is pointer.
4482 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
4483 return getDataLayout().getIndexType(PtrTy: Ty);
4484}
4485
4486Type *ScalarEvolution::getWiderType(Type *T1, Type *T2) const {
4487 return getTypeSizeInBits(Ty: T1) >= getTypeSizeInBits(Ty: T2) ? T1 : T2;
4488}
4489
4490bool ScalarEvolution::instructionCouldExistWithOperands(const SCEV *A,
4491 const SCEV *B) {
4492 /// For a valid use point to exist, the defining scope of one operand
4493 /// must dominate the other.
4494 bool PreciseA, PreciseB;
4495 auto *ScopeA = getDefiningScopeBound(Ops: {A}, Precise&: PreciseA);
4496 auto *ScopeB = getDefiningScopeBound(Ops: {B}, Precise&: PreciseB);
4497 if (!PreciseA || !PreciseB)
4498 // Can't tell.
4499 return false;
4500 return (ScopeA == ScopeB) || DT.dominates(Def: ScopeA, User: ScopeB) ||
4501 DT.dominates(Def: ScopeB, User: ScopeA);
4502}
4503
4504const SCEV *ScalarEvolution::getCouldNotCompute() {
4505 return CouldNotCompute.get();
4506}
4507
4508bool ScalarEvolution::checkValidity(const SCEV *S) const {
4509 bool ContainsNulls = SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
4510 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
4511 return SU && SU->getValue() == nullptr;
4512 });
4513
4514 return !ContainsNulls;
4515}
4516
4517bool ScalarEvolution::containsAddRecurrence(const SCEV *S) {
4518 HasRecMapType::iterator I = HasRecMap.find(Val: S);
4519 if (I != HasRecMap.end())
4520 return I->second;
4521
4522 bool FoundAddRec =
4523 SCEVExprContains(Root: S, Pred: [](const SCEV *S) { return isa<SCEVAddRecExpr>(Val: S); });
4524 HasRecMap.insert(KV: {S, FoundAddRec});
4525 return FoundAddRec;
4526}
4527
4528/// Return the ValueOffsetPair set for \p S. \p S can be represented
4529/// by the value and offset from any ValueOffsetPair in the set.
4530ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) {
4531 ExprValueMapType::iterator SI = ExprValueMap.find_as(Val: S);
4532 if (SI == ExprValueMap.end())
4533 return {};
4534 return SI->second.getArrayRef();
4535}
4536
4537/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
4538/// cannot be used separately. eraseValueFromMap should be used to remove
4539/// V from ValueExprMap and ExprValueMap at the same time.
4540void ScalarEvolution::eraseValueFromMap(Value *V) {
4541 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4542 if (I != ValueExprMap.end()) {
4543 auto EVIt = ExprValueMap.find(Val: I->second);
4544 bool Removed = EVIt->second.remove(X: V);
4545 (void) Removed;
4546 assert(Removed && "Value not in ExprValueMap?");
4547 ValueExprMap.erase(I);
4548 }
4549}
4550
4551void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
4552 // A recursive query may have already computed the SCEV. It should be
4553 // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
4554 // inferred nowrap flags.
4555 auto It = ValueExprMap.find_as(Val: V);
4556 if (It == ValueExprMap.end()) {
4557 ValueExprMap.insert(KV: {SCEVCallbackVH(V, this), S});
4558 ExprValueMap[S].insert(X: V);
4559 }
4560}
4561
4562/// Return an existing SCEV if it exists, otherwise analyze the expression and
4563/// create a new one.
4564const SCEV *ScalarEvolution::getSCEV(Value *V) {
4565 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4566
4567 if (const SCEV *S = getExistingSCEV(V))
4568 return S;
4569 return createSCEVIter(V);
4570}
4571
4572const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
4573 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4574
4575 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4576 if (I != ValueExprMap.end()) {
4577 const SCEV *S = I->second;
4578 assert(checkValidity(S) &&
4579 "existing SCEV has not been properly invalidated");
4580 return S;
4581 }
4582 return nullptr;
4583}
4584
4585/// Return a SCEV corresponding to -V = -1*V
4586const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V,
4587 SCEV::NoWrapFlags Flags) {
4588 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4589 return getConstant(
4590 V: cast<ConstantInt>(Val: ConstantExpr::getNeg(C: VC->getValue())));
4591
4592 Type *Ty = V->getType();
4593 Ty = getEffectiveSCEVType(Ty);
4594 return getMulExpr(LHS: V, RHS: getMinusOne(Ty), Flags);
4595}
4596
4597/// If Expr computes ~A, return A else return nullptr
4598static const SCEV *MatchNotExpr(const SCEV *Expr) {
4599 const SCEV *MulOp;
4600 if (match(S: Expr, P: m_scev_Add(Op0: m_scev_AllOnes(),
4601 Op1: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: MulOp)))))
4602 return MulOp;
4603 return nullptr;
4604}
4605
4606/// Return a SCEV corresponding to ~V = -1-V
4607const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
4608 assert(!V->getType()->isPointerTy() && "Can't negate pointer");
4609
4610 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4611 return getConstant(
4612 V: cast<ConstantInt>(Val: ConstantExpr::getNot(C: VC->getValue())));
4613
4614 // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
4615 if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(Val: V)) {
4616 auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
4617 SmallVector<SCEVUse, 2> MatchedOperands;
4618 for (const SCEV *Operand : MME->operands()) {
4619 const SCEV *Matched = MatchNotExpr(Expr: Operand);
4620 if (!Matched)
4621 return (const SCEV *)nullptr;
4622 MatchedOperands.push_back(Elt: Matched);
4623 }
4624 return getMinMaxExpr(Kind: SCEVMinMaxExpr::negate(T: MME->getSCEVType()),
4625 Ops&: MatchedOperands);
4626 };
4627 if (const SCEV *Replaced = MatchMinMaxNegation(MME))
4628 return Replaced;
4629 }
4630
4631 Type *Ty = V->getType();
4632 Ty = getEffectiveSCEVType(Ty);
4633 return getMinusSCEV(LHS: getMinusOne(Ty), RHS: V);
4634}
4635
4636const SCEV *ScalarEvolution::removePointerBase(const SCEV *P) {
4637 assert(P->getType()->isPointerTy());
4638
4639 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: P)) {
4640 // The base of an AddRec is the first operand.
4641 SmallVector<SCEVUse> Ops{AddRec->operands()};
4642 Ops[0] = removePointerBase(P: Ops[0]);
4643 // Don't try to transfer nowrap flags for now. We could in some cases
4644 // (for example, if pointer operand of the AddRec is a SCEVUnknown).
4645 return getAddRecExpr(Operands&: Ops, L: AddRec->getLoop(), Flags: SCEV::FlagAnyWrap);
4646 }
4647 if (auto *Add = dyn_cast<SCEVAddExpr>(Val: P)) {
4648 // The base of an Add is the pointer operand.
4649 SmallVector<SCEVUse> Ops{Add->operands()};
4650 SCEVUse *PtrOp = nullptr;
4651 for (SCEVUse &AddOp : Ops) {
4652 if (AddOp->getType()->isPointerTy()) {
4653 assert(!PtrOp && "Cannot have multiple pointer ops");
4654 PtrOp = &AddOp;
4655 }
4656 }
4657 *PtrOp = removePointerBase(P: *PtrOp);
4658 // Don't try to transfer nowrap flags for now. We could in some cases
4659 // (for example, if the pointer operand of the Add is a SCEVUnknown).
4660 return getAddExpr(Ops);
4661 }
4662 // Any other expression must be a pointer base.
4663 return getZero(Ty: P->getType());
4664}
4665
4666const SCEV *ScalarEvolution::getMinusSCEV(SCEVUse LHS, SCEVUse RHS,
4667 SCEV::NoWrapFlags Flags,
4668 unsigned Depth) {
4669 // Fast path: X - X --> 0.
4670 if (LHS == RHS)
4671 return getZero(Ty: LHS->getType());
4672
4673 // If we subtract two pointers with different pointer bases, bail.
4674 // Eventually, we're going to add an assertion to getMulExpr that we
4675 // can't multiply by a pointer.
4676 if (RHS->getType()->isPointerTy()) {
4677 if (!LHS->getType()->isPointerTy() ||
4678 getPointerBase(V: LHS) != getPointerBase(V: RHS))
4679 return getCouldNotCompute();
4680 LHS = removePointerBase(P: LHS);
4681 RHS = removePointerBase(P: RHS);
4682 }
4683
4684 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
4685 // makes it so that we cannot make much use of NUW.
4686 auto AddFlags = SCEV::FlagAnyWrap;
4687 const bool RHSIsNotMinSigned =
4688 !getSignedRangeMin(S: RHS).isMinSignedValue();
4689 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW)) {
4690 // Let M be the minimum representable signed value. Then (-1)*RHS
4691 // signed-wraps if and only if RHS is M. That can happen even for
4692 // a NSW subtraction because e.g. (-1)*M signed-wraps even though
4693 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
4694 // (-1)*RHS, we need to prove that RHS != M.
4695 //
4696 // If LHS is non-negative and we know that LHS - RHS does not
4697 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
4698 // either by proving that RHS > M or that LHS >= 0.
4699 if (RHSIsNotMinSigned || isKnownNonNegative(S: LHS)) {
4700 AddFlags = SCEV::FlagNSW;
4701 }
4702 }
4703
4704 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
4705 // RHS is NSW and LHS >= 0.
4706 //
4707 // The difficulty here is that the NSW flag may have been proven
4708 // relative to a loop that is to be found in a recurrence in LHS and
4709 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
4710 // larger scope than intended.
4711 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
4712
4713 return getAddExpr(LHS, RHS: getNegativeSCEV(V: RHS, Flags: NegFlags), Flags: AddFlags, Depth);
4714}
4715
4716const SCEV *ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty,
4717 unsigned Depth) {
4718 Type *SrcTy = V->getType();
4719 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4720 "Cannot truncate or zero extend with non-integer arguments!");
4721 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4722 return V; // No conversion
4723 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4724 return getTruncateExpr(Op: V, Ty, Depth);
4725 return getZeroExtendExpr(Op: V, Ty, Depth);
4726}
4727
4728const SCEV *ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, Type *Ty,
4729 unsigned Depth) {
4730 Type *SrcTy = V->getType();
4731 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4732 "Cannot truncate or zero extend with non-integer arguments!");
4733 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4734 return V; // No conversion
4735 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4736 return getTruncateExpr(Op: V, Ty, Depth);
4737 return getSignExtendExpr(Op: V, Ty, Depth);
4738}
4739
4740const SCEV *ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
4741 Type *SrcTy = V->getType();
4742 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4743 "Cannot noop or zero extend with non-integer arguments!");
4744 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4745 "getNoopOrZeroExtend cannot truncate!");
4746 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4747 return V; // No conversion
4748 return getZeroExtendExpr(Op: V, Ty);
4749}
4750
4751const SCEV *ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
4752 Type *SrcTy = V->getType();
4753 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4754 "Cannot noop or sign extend with non-integer arguments!");
4755 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4756 "getNoopOrSignExtend cannot truncate!");
4757 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4758 return V; // No conversion
4759 return getSignExtendExpr(Op: V, Ty);
4760}
4761
4762const SCEV *ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
4763 Type *SrcTy = V->getType();
4764 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4765 "Cannot noop or any extend with non-integer arguments!");
4766 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4767 "getNoopOrAnyExtend cannot truncate!");
4768 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4769 return V; // No conversion
4770 return getAnyExtendExpr(Op: V, Ty);
4771}
4772
4773const SCEV *ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
4774 Type *SrcTy = V->getType();
4775 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4776 "Cannot truncate or noop with non-integer arguments!");
4777 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
4778 "getTruncateOrNoop cannot extend!");
4779 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4780 return V; // No conversion
4781 return getTruncateExpr(Op: V, Ty);
4782}
4783
4784const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
4785 const SCEV *RHS) {
4786 const SCEV *PromotedLHS = LHS;
4787 const SCEV *PromotedRHS = RHS;
4788
4789 if (getTypeSizeInBits(Ty: LHS->getType()) > getTypeSizeInBits(Ty: RHS->getType()))
4790 PromotedRHS = getZeroExtendExpr(Op: RHS, Ty: LHS->getType());
4791 else
4792 PromotedLHS = getNoopOrZeroExtend(V: LHS, Ty: RHS->getType());
4793
4794 return getUMaxExpr(LHS: PromotedLHS, RHS: PromotedRHS);
4795}
4796
4797const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
4798 const SCEV *RHS,
4799 bool Sequential) {
4800 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4801 return getUMinFromMismatchedTypes(Ops, Sequential);
4802}
4803
4804const SCEV *
4805ScalarEvolution::getUMinFromMismatchedTypes(SmallVectorImpl<SCEVUse> &Ops,
4806 bool Sequential) {
4807 assert(!Ops.empty() && "At least one operand must be!");
4808 // Trivial case.
4809 if (Ops.size() == 1)
4810 return Ops[0];
4811
4812 // Find the max type first.
4813 Type *MaxType = nullptr;
4814 for (SCEVUse S : Ops)
4815 if (MaxType)
4816 MaxType = getWiderType(T1: MaxType, T2: S->getType());
4817 else
4818 MaxType = S->getType();
4819 assert(MaxType && "Failed to find maximum type!");
4820
4821 // Extend all ops to max type.
4822 SmallVector<SCEVUse, 2> PromotedOps;
4823 for (SCEVUse S : Ops)
4824 PromotedOps.push_back(Elt: getNoopOrZeroExtend(V: S, Ty: MaxType));
4825
4826 // Generate umin.
4827 return getUMinExpr(Ops&: PromotedOps, Sequential);
4828}
4829
4830const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
4831 // A pointer operand may evaluate to a nonpointer expression, such as null.
4832 if (!V->getType()->isPointerTy())
4833 return V;
4834
4835 while (true) {
4836 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: V)) {
4837 V = AddRec->getStart();
4838 } else if (auto *Add = dyn_cast<SCEVAddExpr>(Val: V)) {
4839 const SCEV *PtrOp = nullptr;
4840 for (const SCEV *AddOp : Add->operands()) {
4841 if (AddOp->getType()->isPointerTy()) {
4842 assert(!PtrOp && "Cannot have multiple pointer ops");
4843 PtrOp = AddOp;
4844 }
4845 }
4846 assert(PtrOp && "Must have pointer op");
4847 V = PtrOp;
4848 } else // Not something we can look further into.
4849 return V;
4850 }
4851}
4852
4853/// Push users of the given Instruction onto the given Worklist.
4854static void PushDefUseChildren(Instruction *I,
4855 SmallVectorImpl<Instruction *> &Worklist,
4856 SmallPtrSetImpl<Instruction *> &Visited) {
4857 // Push the def-use children onto the Worklist stack.
4858 for (User *U : I->users()) {
4859 auto *UserInsn = cast<Instruction>(Val: U);
4860 if (Visited.insert(Ptr: UserInsn).second)
4861 Worklist.push_back(Elt: UserInsn);
4862 }
4863}
4864
4865namespace {
4866
4867/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
4868/// expression in case its Loop is L. If it is not L then
4869/// if IgnoreOtherLoops is true then use AddRec itself
4870/// otherwise rewrite cannot be done.
4871/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4872class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
4873public:
4874 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
4875 bool IgnoreOtherLoops = true) {
4876 SCEVInitRewriter Rewriter(L, SE);
4877 const SCEV *Result = Rewriter.visit(S);
4878 if (Rewriter.hasSeenLoopVariantSCEVUnknown())
4879 return SE.getCouldNotCompute();
4880 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
4881 ? SE.getCouldNotCompute()
4882 : Result;
4883 }
4884
4885 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4886 if (!SE.isLoopInvariant(S: Expr, L))
4887 SeenLoopVariantSCEVUnknown = true;
4888 return Expr;
4889 }
4890
4891 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4892 // Only re-write AddRecExprs for this loop.
4893 if (Expr->getLoop() == L)
4894 return Expr->getStart();
4895 SeenOtherLoops = true;
4896 return Expr;
4897 }
4898
4899 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4900
4901 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4902
4903private:
4904 explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
4905 : SCEVRewriteVisitor(SE), L(L) {}
4906
4907 const Loop *L;
4908 bool SeenLoopVariantSCEVUnknown = false;
4909 bool SeenOtherLoops = false;
4910};
4911
4912/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
4913/// increment expression in case its Loop is L. If it is not L then
4914/// use AddRec itself.
4915/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4916class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
4917public:
4918 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
4919 SCEVPostIncRewriter Rewriter(L, SE);
4920 const SCEV *Result = Rewriter.visit(S);
4921 return Rewriter.hasSeenLoopVariantSCEVUnknown()
4922 ? SE.getCouldNotCompute()
4923 : Result;
4924 }
4925
4926 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4927 if (!SE.isLoopInvariant(S: Expr, L))
4928 SeenLoopVariantSCEVUnknown = true;
4929 return Expr;
4930 }
4931
4932 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4933 // Only re-write AddRecExprs for this loop.
4934 if (Expr->getLoop() == L)
4935 return Expr->getPostIncExpr(SE);
4936 SeenOtherLoops = true;
4937 return Expr;
4938 }
4939
4940 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4941
4942 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4943
4944private:
4945 explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
4946 : SCEVRewriteVisitor(SE), L(L) {}
4947
4948 const Loop *L;
4949 bool SeenLoopVariantSCEVUnknown = false;
4950 bool SeenOtherLoops = false;
4951};
4952
4953/// This class evaluates the compare condition by matching it against the
4954/// condition of loop latch. If there is a match we assume a true value
4955/// for the condition while building SCEV nodes.
4956class SCEVBackedgeConditionFolder
4957 : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
4958public:
4959 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4960 ScalarEvolution &SE) {
4961 bool IsPosBECond = false;
4962 Value *BECond = nullptr;
4963 if (BasicBlock *Latch = L->getLoopLatch()) {
4964 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Latch->getTerminator())) {
4965 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
4966 "Both outgoing branches should not target same header!");
4967 BECond = BI->getCondition();
4968 IsPosBECond = BI->getSuccessor(i: 0) == L->getHeader();
4969 } else {
4970 return S;
4971 }
4972 }
4973 SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
4974 return Rewriter.visit(S);
4975 }
4976
4977 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4978 const SCEV *Result = Expr;
4979 bool InvariantF = SE.isLoopInvariant(S: Expr, L);
4980
4981 if (!InvariantF) {
4982 Instruction *I = cast<Instruction>(Val: Expr->getValue());
4983 switch (I->getOpcode()) {
4984 case Instruction::Select: {
4985 SelectInst *SI = cast<SelectInst>(Val: I);
4986 std::optional<const SCEV *> Res =
4987 compareWithBackedgeCondition(IC: SI->getCondition());
4988 if (Res) {
4989 bool IsOne = cast<SCEVConstant>(Val: *Res)->getValue()->isOne();
4990 Result = SE.getSCEV(V: IsOne ? SI->getTrueValue() : SI->getFalseValue());
4991 }
4992 break;
4993 }
4994 default: {
4995 std::optional<const SCEV *> Res = compareWithBackedgeCondition(IC: I);
4996 if (Res)
4997 Result = *Res;
4998 break;
4999 }
5000 }
5001 }
5002 return Result;
5003 }
5004
5005private:
5006 explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
5007 bool IsPosBECond, ScalarEvolution &SE)
5008 : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
5009 IsPositiveBECond(IsPosBECond) {}
5010
5011 std::optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
5012
5013 const Loop *L;
5014 /// Loop back condition.
5015 Value *BackedgeCond = nullptr;
5016 /// Set to true if loop back is on positive branch condition.
5017 bool IsPositiveBECond;
5018};
5019
5020std::optional<const SCEV *>
5021SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
5022
5023 // If value matches the backedge condition for loop latch,
5024 // then return a constant evolution node based on loopback
5025 // branch taken.
5026 if (BackedgeCond == IC)
5027 return IsPositiveBECond ? SE.getOne(Ty: Type::getInt1Ty(C&: SE.getContext()))
5028 : SE.getZero(Ty: Type::getInt1Ty(C&: SE.getContext()));
5029 return std::nullopt;
5030}
5031
5032class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
5033public:
5034 static const SCEV *rewrite(const SCEV *S, const Loop *L,
5035 ScalarEvolution &SE) {
5036 SCEVShiftRewriter Rewriter(L, SE);
5037 const SCEV *Result = Rewriter.visit(S);
5038 return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
5039 }
5040
5041 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5042 // Only allow AddRecExprs for this loop.
5043 if (!SE.isLoopInvariant(S: Expr, L))
5044 Valid = false;
5045 return Expr;
5046 }
5047
5048 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5049 if (Expr->getLoop() == L && Expr->isAffine())
5050 return SE.getMinusSCEV(LHS: Expr, RHS: Expr->getStepRecurrence(SE));
5051 Valid = false;
5052 return Expr;
5053 }
5054
5055 bool isValid() { return Valid; }
5056
5057private:
5058 explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
5059 : SCEVRewriteVisitor(SE), L(L) {}
5060
5061 const Loop *L;
5062 bool Valid = true;
5063};
5064
5065} // end anonymous namespace
5066
5067void ScalarEvolution::inferNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
5068 if (!AR->isAffine())
5069 return;
5070
5071 // Force computation of ranges, which will also perform range-based flag
5072 // inference.
5073 if (!AR->hasNoSignedWrap())
5074 (void)getSignedRange(S: AR);
5075
5076 if (!AR->hasNoUnsignedWrap())
5077 (void)getUnsignedRange(S: AR);
5078
5079 if (!AR->hasNoSelfWrap()) {
5080 const SCEV *BECount = getConstantMaxBackedgeTakenCount(L: AR->getLoop());
5081 if (const SCEVConstant *BECountMax = dyn_cast<SCEVConstant>(Val: BECount)) {
5082 ConstantRange StepCR = getSignedRange(S: AR->getStepRecurrence(SE&: *this));
5083 const APInt &BECountAP = BECountMax->getAPInt();
5084 unsigned NoOverflowBitWidth =
5085 BECountAP.getActiveBits() + StepCR.getMinSignedBits();
5086 if (NoOverflowBitWidth <= getTypeSizeInBits(Ty: AR->getType()))
5087 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
5088 }
5089 }
5090}
5091
5092SCEV::NoWrapFlags
5093ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5094 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
5095
5096 if (AR->hasNoSignedWrap())
5097 return Result;
5098
5099 if (!AR->isAffine())
5100 return Result;
5101
5102 // This function can be expensive, only try to prove NSW once per AddRec.
5103 if (!SignedWrapViaInductionTried.insert(Ptr: AR).second)
5104 return Result;
5105
5106 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5107 const Loop *L = AR->getLoop();
5108
5109 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5110 // Note that this serves two purposes: It filters out loops that are
5111 // simply not analyzable, and it covers the case where this code is
5112 // being called from within backedge-taken count analysis, such that
5113 // attempting to ask for the backedge-taken count would likely result
5114 // in infinite recursion. In the later case, the analysis code will
5115 // cope with a conservative value, and it will take care to purge
5116 // that value once it has finished.
5117 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5118
5119 // Normally, in the cases we can prove no-overflow via a
5120 // backedge guarding condition, we can also compute a backedge
5121 // taken count for the loop. The exceptions are assumptions and
5122 // guards present in the loop -- SCEV is not great at exploiting
5123 // these to compute max backedge taken counts, but can still use
5124 // these to prove lack of overflow. Use this fact to avoid
5125 // doing extra work that may not pay off.
5126
5127 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5128 AC.assumptions().empty())
5129 return Result;
5130
5131 // If the backedge is guarded by a comparison with the pre-inc value the
5132 // addrec is safe. Also, if the entry is guarded by a comparison with the
5133 // start value and the backedge is guarded by a comparison with the post-inc
5134 // value, the addrec is safe.
5135 ICmpInst::Predicate Pred;
5136 const SCEV *OverflowLimit =
5137 getSignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
5138 if (OverflowLimit &&
5139 (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
5140 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))) {
5141 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNSW);
5142 }
5143 return Result;
5144}
5145SCEV::NoWrapFlags
5146ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5147 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
5148
5149 if (AR->hasNoUnsignedWrap())
5150 return Result;
5151
5152 if (!AR->isAffine())
5153 return Result;
5154
5155 // This function can be expensive, only try to prove NUW once per AddRec.
5156 if (!UnsignedWrapViaInductionTried.insert(Ptr: AR).second)
5157 return Result;
5158
5159 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5160 const Loop *L = AR->getLoop();
5161
5162 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5163 // Note that this serves two purposes: It filters out loops that are
5164 // simply not analyzable, and it covers the case where this code is
5165 // being called from within backedge-taken count analysis, such that
5166 // attempting to ask for the backedge-taken count would likely result
5167 // in infinite recursion. In the later case, the analysis code will
5168 // cope with a conservative value, and it will take care to purge
5169 // that value once it has finished.
5170 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5171
5172 // Normally, in the cases we can prove no-overflow via a
5173 // backedge guarding condition, we can also compute a backedge
5174 // taken count for the loop. The exceptions are assumptions and
5175 // guards present in the loop -- SCEV is not great at exploiting
5176 // these to compute max backedge taken counts, but can still use
5177 // these to prove lack of overflow. Use this fact to avoid
5178 // doing extra work that may not pay off.
5179
5180 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5181 AC.assumptions().empty())
5182 return Result;
5183
5184 // If the backedge is guarded by a comparison with the pre-inc value the
5185 // addrec is safe. Also, if the entry is guarded by a comparison with the
5186 // start value and the backedge is guarded by a comparison with the post-inc
5187 // value, the addrec is safe.
5188 if (isKnownPositive(S: Step)) {
5189 ICmpInst::Predicate Pred;
5190 const SCEV *OverflowLimit =
5191 getUnsignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
5192 if (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
5193 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))
5194 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNUW);
5195 }
5196 return Result;
5197}
5198
5199namespace {
5200
5201/// Represents an abstract binary operation. This may exist as a
5202/// normal instruction or constant expression, or may have been
5203/// derived from an expression tree.
5204struct BinaryOp {
5205 unsigned Opcode;
5206 Value *LHS;
5207 Value *RHS;
5208 bool IsNSW = false;
5209 bool IsNUW = false;
5210
5211 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
5212 /// constant expression.
5213 Operator *Op = nullptr;
5214
5215 explicit BinaryOp(Operator *Op)
5216 : Opcode(Op->getOpcode()), LHS(Op->getOperand(i: 0)), RHS(Op->getOperand(i: 1)),
5217 Op(Op) {
5218 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: Op)) {
5219 IsNSW = OBO->hasNoSignedWrap();
5220 IsNUW = OBO->hasNoUnsignedWrap();
5221 }
5222 }
5223
5224 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
5225 bool IsNUW = false)
5226 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5227};
5228
5229} // end anonymous namespace
5230
5231/// Try to map \p V into a BinaryOp, and return \c std::nullopt on failure.
5232static std::optional<BinaryOp> MatchBinaryOp(Value *V, const DataLayout &DL,
5233 AssumptionCache &AC,
5234 const DominatorTree &DT,
5235 const Instruction *CxtI) {
5236 auto *Op = dyn_cast<Operator>(Val: V);
5237 if (!Op)
5238 return std::nullopt;
5239
5240 // Implementation detail: all the cleverness here should happen without
5241 // creating new SCEV expressions -- our caller knowns tricks to avoid creating
5242 // SCEV expressions when possible, and we should not break that.
5243
5244 switch (Op->getOpcode()) {
5245 case Instruction::Add:
5246 case Instruction::Sub:
5247 case Instruction::Mul:
5248 case Instruction::UDiv:
5249 case Instruction::URem:
5250 case Instruction::And:
5251 case Instruction::AShr:
5252 case Instruction::Shl:
5253 return BinaryOp(Op);
5254
5255 case Instruction::Or: {
5256 // Convert or disjoint into add nuw nsw.
5257 if (cast<PossiblyDisjointInst>(Val: Op)->isDisjoint()) {
5258 BinaryOp BinOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1),
5259 /*IsNSW=*/true, /*IsNUW=*/true);
5260 // Keep the reference to the original instruction so that we can later
5261 // check whether it can produce poison value or not.
5262 BinOp.Op = Op;
5263 return BinOp;
5264 }
5265 return BinaryOp(Op);
5266 }
5267
5268 case Instruction::Xor:
5269 if (auto *RHSC = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1)))
5270 // If the RHS of the xor is a signmask, then this is just an add.
5271 // Instcombine turns add of signmask into xor as a strength reduction step.
5272 if (RHSC->getValue().isSignMask())
5273 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5274 // Binary `xor` is a bit-wise `add`.
5275 if (V->getType()->isIntegerTy(BitWidth: 1))
5276 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5277 return BinaryOp(Op);
5278
5279 case Instruction::LShr:
5280 // Turn logical shift right of a constant into a unsigned divide.
5281 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1))) {
5282 uint32_t BitWidth = cast<IntegerType>(Val: Op->getType())->getBitWidth();
5283
5284 // If the shift count is not less than the bitwidth, the result of
5285 // the shift is undefined. Don't try to analyze it, because the
5286 // resolution chosen here may differ from the resolution chosen in
5287 // other parts of the compiler.
5288 if (SA->getValue().ult(RHS: BitWidth)) {
5289 Constant *X =
5290 ConstantInt::get(Context&: SA->getContext(),
5291 V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
5292 return BinaryOp(Instruction::UDiv, Op->getOperand(i: 0), X);
5293 }
5294 }
5295 return BinaryOp(Op);
5296
5297 case Instruction::ExtractValue: {
5298 auto *EVI = cast<ExtractValueInst>(Val: Op);
5299 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5300 break;
5301
5302 auto *WO = dyn_cast<WithOverflowInst>(Val: EVI->getAggregateOperand());
5303 if (!WO)
5304 break;
5305
5306 Instruction::BinaryOps BinOp = WO->getBinaryOp();
5307 bool Signed = WO->isSigned();
5308 // TODO: Should add nuw/nsw flags for mul as well.
5309 if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
5310 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5311
5312 // Now that we know that all uses of the arithmetic-result component of
5313 // CI are guarded by the overflow check, we can go ahead and pretend
5314 // that the arithmetic is non-overflowing.
5315 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5316 /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
5317 }
5318
5319 default:
5320 break;
5321 }
5322
5323 // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
5324 // semantics as a Sub, return a binary sub expression.
5325 if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
5326 if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5327 return BinaryOp(Instruction::Sub, II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1));
5328
5329 return std::nullopt;
5330}
5331
5332/// Helper function to createAddRecFromPHIWithCasts. We have a phi
5333/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
5334/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
5335/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
5336/// follows one of the following patterns:
5337/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5338/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5339/// If the SCEV expression of \p Op conforms with one of the expected patterns
5340/// we return the type of the truncation operation, and indicate whether the
5341/// truncated type should be treated as signed/unsigned by setting
5342/// \p Signed to true/false, respectively.
5343static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
5344 bool &Signed, ScalarEvolution &SE) {
5345 // The case where Op == SymbolicPHI (that is, with no type conversions on
5346 // the way) is handled by the regular add recurrence creating logic and
5347 // would have already been triggered in createAddRecForPHI. Reaching it here
5348 // means that createAddRecFromPHI had failed for this PHI before (e.g.,
5349 // because one of the other operands of the SCEVAddExpr updating this PHI is
5350 // not invariant).
5351 //
5352 // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
5353 // this case predicates that allow us to prove that Op == SymbolicPHI will
5354 // be added.
5355 if (Op == SymbolicPHI)
5356 return nullptr;
5357
5358 unsigned SourceBits = SE.getTypeSizeInBits(Ty: SymbolicPHI->getType());
5359 unsigned NewBits = SE.getTypeSizeInBits(Ty: Op->getType());
5360 if (SourceBits != NewBits)
5361 return nullptr;
5362
5363 if (match(S: Op, P: m_scev_SExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5364 Signed = true;
5365 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5366 }
5367 if (match(S: Op, P: m_scev_ZExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5368 Signed = false;
5369 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5370 }
5371 return nullptr;
5372}
5373
5374static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
5375 if (!PN->getType()->isIntegerTy())
5376 return nullptr;
5377 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5378 if (!L || L->getHeader() != PN->getParent())
5379 return nullptr;
5380 return L;
5381}
5382
5383// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
5384// computation that updates the phi follows the following pattern:
5385// (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
5386// which correspond to a phi->trunc->sext/zext->add->phi update chain.
5387// If so, try to see if it can be rewritten as an AddRecExpr under some
5388// Predicates. If successful, return them as a pair. Also cache the results
5389// of the analysis.
5390//
5391// Example usage scenario:
5392// Say the Rewriter is called for the following SCEV:
5393// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5394// where:
5395// %X = phi i64 (%Start, %BEValue)
5396// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
5397// and call this function with %SymbolicPHI = %X.
5398//
5399// The analysis will find that the value coming around the backedge has
5400// the following SCEV:
5401// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5402// Upon concluding that this matches the desired pattern, the function
5403// will return the pair {NewAddRec, SmallPredsVec} where:
5404// NewAddRec = {%Start,+,%Step}
5405// SmallPredsVec = {P1, P2, P3} as follows:
5406// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
5407// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
5408// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
5409// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
5410// under the predicates {P1,P2,P3}.
5411// This predicated rewrite will be cached in PredicatedSCEVRewrites:
5412// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
5413//
5414// TODO's:
5415//
5416// 1) Extend the Induction descriptor to also support inductions that involve
5417// casts: When needed (namely, when we are called in the context of the
5418// vectorizer induction analysis), a Set of cast instructions will be
5419// populated by this method, and provided back to isInductionPHI. This is
5420// needed to allow the vectorizer to properly record them to be ignored by
5421// the cost model and to avoid vectorizing them (otherwise these casts,
5422// which are redundant under the runtime overflow checks, will be
5423// vectorized, which can be costly).
5424//
5425// 2) Support additional induction/PHISCEV patterns: We also want to support
5426// inductions where the sext-trunc / zext-trunc operations (partly) occur
5427// after the induction update operation (the induction increment):
5428//
5429// (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
5430// which correspond to a phi->add->trunc->sext/zext->phi update chain.
5431//
5432// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
5433// which correspond to a phi->trunc->add->sext/zext->phi update chain.
5434//
5435// 3) Outline common code with createAddRecFromPHI to avoid duplication.
5436std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5437ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
5438 SmallVector<const SCEVPredicate *, 3> Predicates;
5439
5440 // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
5441 // return an AddRec expression under some predicate.
5442
5443 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5444 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5445 assert(L && "Expecting an integer loop header phi");
5446
5447 // The loop may have multiple entrances or multiple exits; we can analyze
5448 // this phi as an addrec if it has a unique entry value and a unique
5449 // backedge value.
5450 Value *BEValueV = nullptr, *StartValueV = nullptr;
5451 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5452 Value *V = PN->getIncomingValue(i);
5453 if (L->contains(BB: PN->getIncomingBlock(i))) {
5454 if (!BEValueV) {
5455 BEValueV = V;
5456 } else if (BEValueV != V) {
5457 BEValueV = nullptr;
5458 break;
5459 }
5460 } else if (!StartValueV) {
5461 StartValueV = V;
5462 } else if (StartValueV != V) {
5463 StartValueV = nullptr;
5464 break;
5465 }
5466 }
5467 if (!BEValueV || !StartValueV)
5468 return std::nullopt;
5469
5470 const SCEV *BEValue = getSCEV(V: BEValueV);
5471
5472 // If the value coming around the backedge is an add with the symbolic
5473 // value we just inserted, possibly with casts that we can ignore under
5474 // an appropriate runtime guard, then we found a simple induction variable!
5475 const auto *Add = dyn_cast<SCEVAddExpr>(Val: BEValue);
5476 if (!Add)
5477 return std::nullopt;
5478
5479 // If there is a single occurrence of the symbolic value, possibly
5480 // casted, replace it with a recurrence.
5481 unsigned FoundIndex = Add->getNumOperands();
5482 Type *TruncTy = nullptr;
5483 bool Signed;
5484 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5485 if ((TruncTy =
5486 isSimpleCastedPHI(Op: Add->getOperand(i), SymbolicPHI, Signed, SE&: *this)))
5487 if (FoundIndex == e) {
5488 FoundIndex = i;
5489 break;
5490 }
5491
5492 if (FoundIndex == Add->getNumOperands())
5493 return std::nullopt;
5494
5495 // Create an add with everything but the specified operand.
5496 SmallVector<SCEVUse, 8> Ops;
5497 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5498 if (i != FoundIndex)
5499 Ops.push_back(Elt: Add->getOperand(i));
5500 const SCEV *Accum = getAddExpr(Ops);
5501
5502 // The runtime checks will not be valid if the step amount is
5503 // varying inside the loop.
5504 if (!isLoopInvariant(S: Accum, L))
5505 return std::nullopt;
5506
5507 // *** Part2: Create the predicates
5508
5509 // Analysis was successful: we have a phi-with-cast pattern for which we
5510 // can return an AddRec expression under the following predicates:
5511 //
5512 // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
5513 // fits within the truncated type (does not overflow) for i = 0 to n-1.
5514 // P2: An Equal predicate that guarantees that
5515 // Start = (Ext ix (Trunc iy (Start) to ix) to iy)
5516 // P3: An Equal predicate that guarantees that
5517 // Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
5518 //
5519 // As we next prove, the above predicates guarantee that:
5520 // Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
5521 //
5522 //
5523 // More formally, we want to prove that:
5524 // Expr(i+1) = Start + (i+1) * Accum
5525 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5526 //
5527 // Given that:
5528 // 1) Expr(0) = Start
5529 // 2) Expr(1) = Start + Accum
5530 // = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
5531 // 3) Induction hypothesis (step i):
5532 // Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
5533 //
5534 // Proof:
5535 // Expr(i+1) =
5536 // = Start + (i+1)*Accum
5537 // = (Start + i*Accum) + Accum
5538 // = Expr(i) + Accum
5539 // = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
5540 // :: from step i
5541 //
5542 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
5543 //
5544 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
5545 // + (Ext ix (Trunc iy (Accum) to ix) to iy)
5546 // + Accum :: from P3
5547 //
5548 // = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
5549 // + Accum :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
5550 //
5551 // = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
5552 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5553 //
5554 // By induction, the same applies to all iterations 1<=i<n:
5555 //
5556
5557 // Create a truncated addrec for which we will add a no overflow check (P1).
5558 const SCEV *StartVal = getSCEV(V: StartValueV);
5559 const SCEV *PHISCEV =
5560 getAddRecExpr(Start: getTruncateExpr(Op: StartVal, Ty: TruncTy),
5561 Step: getTruncateExpr(Op: Accum, Ty: TruncTy), L, Flags: SCEV::FlagAnyWrap);
5562
5563 // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
5564 // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
5565 // will be constant.
5566 //
5567 // If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
5568 // add P1.
5569 if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV)) {
5570 SCEVWrapPredicate::IncrementWrapFlags AddedFlags =
5571 Signed ? SCEVWrapPredicate::IncrementNSSW
5572 : SCEVWrapPredicate::IncrementNUSW;
5573 const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
5574 Predicates.push_back(Elt: AddRecPred);
5575 }
5576
5577 // Create the Equal Predicates P2,P3:
5578
5579 // It is possible that the predicates P2 and/or P3 are computable at
5580 // compile time due to StartVal and/or Accum being constants.
5581 // If either one is, then we can check that now and escape if either P2
5582 // or P3 is false.
5583
5584 // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
5585 // for each of StartVal and Accum
5586 auto getExtendedExpr = [&](const SCEV *Expr,
5587 bool CreateSignExtend) -> const SCEV * {
5588 assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant");
5589 const SCEV *TruncatedExpr = getTruncateExpr(Op: Expr, Ty: TruncTy);
5590 const SCEV *ExtendedExpr =
5591 CreateSignExtend ? getSignExtendExpr(Op: TruncatedExpr, Ty: Expr->getType())
5592 : getZeroExtendExpr(Op: TruncatedExpr, Ty: Expr->getType());
5593 return ExtendedExpr;
5594 };
5595
5596 // Given:
5597 // ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
5598 // = getExtendedExpr(Expr)
5599 // Determine whether the predicate P: Expr == ExtendedExpr
5600 // is known to be false at compile time
5601 auto PredIsKnownFalse = [&](const SCEV *Expr,
5602 const SCEV *ExtendedExpr) -> bool {
5603 return Expr != ExtendedExpr &&
5604 isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: Expr, RHS: ExtendedExpr);
5605 };
5606
5607 const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
5608 if (PredIsKnownFalse(StartVal, StartExtended)) {
5609 LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";);
5610 return std::nullopt;
5611 }
5612
5613 // The Step is always Signed (because the overflow checks are either
5614 // NSSW or NUSW)
5615 const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
5616 if (PredIsKnownFalse(Accum, AccumExtended)) {
5617 LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";);
5618 return std::nullopt;
5619 }
5620
5621 auto AppendPredicate = [&](const SCEV *Expr,
5622 const SCEV *ExtendedExpr) -> void {
5623 if (Expr != ExtendedExpr &&
5624 !isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: Expr, RHS: ExtendedExpr)) {
5625 const SCEVPredicate *Pred = getEqualPredicate(LHS: Expr, RHS: ExtendedExpr);
5626 LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred);
5627 Predicates.push_back(Elt: Pred);
5628 }
5629 };
5630
5631 AppendPredicate(StartVal, StartExtended);
5632 AppendPredicate(Accum, AccumExtended);
5633
5634 // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
5635 // which the casts had been folded away. The caller can rewrite SymbolicPHI
5636 // into NewAR if it will also add the runtime overflow checks specified in
5637 // Predicates.
5638 auto *NewAR = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags: SCEV::FlagAnyWrap);
5639
5640 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5641 std::make_pair(x&: NewAR, y&: Predicates);
5642 // Remember the result of the analysis for this SCEV at this locayyytion.
5643 PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
5644 return PredRewrite;
5645}
5646
5647std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5648ScalarEvolution::createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI) {
5649 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5650 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5651 if (!L)
5652 return std::nullopt;
5653
5654 // Check to see if we already analyzed this PHI.
5655 auto I = PredicatedSCEVRewrites.find(Val: {SymbolicPHI, L});
5656 if (I != PredicatedSCEVRewrites.end()) {
5657 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5658 I->second;
5659 // Analysis was done before and failed to create an AddRec:
5660 if (Rewrite.first == SymbolicPHI)
5661 return std::nullopt;
5662 // Analysis was done before and succeeded to create an AddRec under
5663 // a predicate:
5664 assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec");
5665 assert(!(Rewrite.second).empty() && "Expected to find Predicates");
5666 return Rewrite;
5667 }
5668
5669 std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5670 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5671
5672 // Record in the cache that the analysis failed
5673 if (!Rewrite) {
5674 SmallVector<const SCEVPredicate *, 3> Predicates;
5675 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5676 return std::nullopt;
5677 }
5678
5679 return Rewrite;
5680}
5681
5682// FIXME: This utility is currently required because the Rewriter currently
5683// does not rewrite this expression:
5684// {0, +, (sext ix (trunc iy to ix) to iy)}
5685// into {0, +, %step},
5686// even when the following Equal predicate exists:
5687// "%step == (sext ix (trunc iy to ix) to iy)".
5688bool PredicatedScalarEvolution::areAddRecsEqualWithPreds(
5689 const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2,
5690 ArrayRef<const SCEVPredicate *> NoWrapPreds) const {
5691 if (AR1 == AR2)
5692 return true;
5693
5694 SCEVUnionPredicate NoWrapUnionPred(NoWrapPreds, SE);
5695 SCEVUnionPredicate AllPreds = Preds->getUnionWith(N: &NoWrapUnionPred, SE);
5696 auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
5697 if (Expr1 != Expr2 &&
5698 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr1, RHS: Expr2), SE) &&
5699 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr2, RHS: Expr1), SE))
5700 return false;
5701 return true;
5702 };
5703
5704 if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
5705 !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
5706 return false;
5707 return true;
5708}
5709
5710/// A helper function for createAddRecFromPHI to handle simple cases.
5711///
5712/// This function tries to find an AddRec expression for the simplest (yet most
5713/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
5714/// If it fails, createAddRecFromPHI will use a more general, but slow,
5715/// technique for finding the AddRec expression.
5716const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
5717 Value *BEValueV,
5718 Value *StartValueV) {
5719 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5720 assert(L && L->getHeader() == PN->getParent());
5721 assert(BEValueV && StartValueV);
5722
5723 auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CxtI: PN);
5724 if (!BO)
5725 return nullptr;
5726
5727 if (BO->Opcode != Instruction::Add)
5728 return nullptr;
5729
5730 const SCEV *Accum = nullptr;
5731 if (BO->LHS == PN && L->isLoopInvariant(V: BO->RHS))
5732 Accum = getSCEV(V: BO->RHS);
5733 else if (BO->RHS == PN && L->isLoopInvariant(V: BO->LHS))
5734 Accum = getSCEV(V: BO->LHS);
5735
5736 if (!Accum)
5737 return nullptr;
5738
5739 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5740 if (BO->IsNUW)
5741 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5742 if (BO->IsNSW)
5743 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5744
5745 const SCEV *StartVal = getSCEV(V: StartValueV);
5746 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5747 insertValueToMap(V: PN, S: PHISCEV);
5748
5749 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
5750 inferNoWrapViaConstantRanges(AR);
5751
5752 // We can add Flags to the post-inc expression only if we
5753 // know that it is *undefined behavior* for BEValueV to
5754 // overflow.
5755 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV)) {
5756 assert(isLoopInvariant(Accum, L) &&
5757 "Accum is defined outside L, but is not invariant?");
5758 if (isAddRecNeverPoison(I: BEInst, L))
5759 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5760 }
5761
5762 return PHISCEV;
5763}
5764
5765const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
5766 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5767 if (!L || L->getHeader() != PN->getParent())
5768 return nullptr;
5769
5770 // The loop may have multiple entrances or multiple exits; we can analyze
5771 // this phi as an addrec if it has a unique entry value and a unique
5772 // backedge value.
5773 Value *BEValueV = nullptr, *StartValueV = nullptr;
5774 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5775 Value *V = PN->getIncomingValue(i);
5776 if (L->contains(BB: PN->getIncomingBlock(i))) {
5777 if (!BEValueV) {
5778 BEValueV = V;
5779 } else if (BEValueV != V) {
5780 BEValueV = nullptr;
5781 break;
5782 }
5783 } else if (!StartValueV) {
5784 StartValueV = V;
5785 } else if (StartValueV != V) {
5786 StartValueV = nullptr;
5787 break;
5788 }
5789 }
5790 if (!BEValueV || !StartValueV)
5791 return nullptr;
5792
5793 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
5794 "PHI node already processed?");
5795
5796 // First, try to find AddRec expression without creating a fictituos symbolic
5797 // value for PN.
5798 if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5799 return S;
5800
5801 // Handle PHI node value symbolically.
5802 const SCEV *SymbolicName = getUnknown(V: PN);
5803 insertValueToMap(V: PN, S: SymbolicName);
5804
5805 // Using this symbolic name for the PHI, analyze the value coming around
5806 // the back-edge.
5807 const SCEV *BEValue = getSCEV(V: BEValueV);
5808
5809 // NOTE: If BEValue is loop invariant, we know that the PHI node just
5810 // has a special value for the first iteration of the loop.
5811
5812 // If the value coming around the backedge is an add with the symbolic
5813 // value we just inserted, then we found a simple induction variable!
5814 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: BEValue)) {
5815 // If there is a single occurrence of the symbolic value, replace it
5816 // with a recurrence.
5817 unsigned FoundIndex = Add->getNumOperands();
5818 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5819 if (Add->getOperand(i) == SymbolicName)
5820 if (FoundIndex == e) {
5821 FoundIndex = i;
5822 break;
5823 }
5824
5825 if (FoundIndex != Add->getNumOperands()) {
5826 // Create an add with everything but the specified operand.
5827 SmallVector<SCEVUse, 8> Ops;
5828 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5829 if (i != FoundIndex)
5830 Ops.push_back(Elt: SCEVBackedgeConditionFolder::rewrite(S: Add->getOperand(i),
5831 L, SE&: *this));
5832 const SCEV *Accum = getAddExpr(Ops);
5833
5834 // This is not a valid addrec if the step amount is varying each
5835 // loop iteration, but is not itself an addrec in this loop.
5836 if (isLoopInvariant(S: Accum, L) ||
5837 (isa<SCEVAddRecExpr>(Val: Accum) &&
5838 cast<SCEVAddRecExpr>(Val: Accum)->getLoop() == L)) {
5839 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5840
5841 if (auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CxtI: PN)) {
5842 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
5843 if (BO->IsNUW)
5844 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5845 if (BO->IsNSW)
5846 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5847 }
5848 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: BEValueV)) {
5849 if (GEP->getOperand(i_nocapture: 0) == PN) {
5850 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
5851 // If the increment has any nowrap flags, then we know the address
5852 // space cannot be wrapped around.
5853 if (NW != GEPNoWrapFlags::none())
5854 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
5855 // If the GEP is nuw or nusw with non-negative offset, we know that
5856 // no unsigned wrap occurs. We cannot set the nsw flag as only the
5857 // offset is treated as signed, while the base is unsigned.
5858 if (NW.hasNoUnsignedWrap() ||
5859 (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Accum)))
5860 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5861 }
5862
5863 // We cannot transfer nuw and nsw flags from subtraction
5864 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
5865 // for instance.
5866 }
5867
5868 const SCEV *StartVal = getSCEV(V: StartValueV);
5869 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5870
5871 // Okay, for the entire analysis of this edge we assumed the PHI
5872 // to be symbolic. We now need to go back and purge all of the
5873 // entries for the scalars that use the symbolic expression.
5874 forgetMemoizedResults(SCEVs: {SymbolicName});
5875 insertValueToMap(V: PN, S: PHISCEV);
5876
5877 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
5878 inferNoWrapViaConstantRanges(AR);
5879
5880 // We can add Flags to the post-inc expression only if we
5881 // know that it is *undefined behavior* for BEValueV to
5882 // overflow.
5883 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV))
5884 if (isLoopInvariant(S: Accum, L) && isAddRecNeverPoison(I: BEInst, L))
5885 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5886
5887 return PHISCEV;
5888 }
5889 }
5890 } else {
5891 // Otherwise, this could be a loop like this:
5892 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
5893 // In this case, j = {1,+,1} and BEValue is j.
5894 // Because the other in-value of i (0) fits the evolution of BEValue
5895 // i really is an addrec evolution.
5896 //
5897 // We can generalize this saying that i is the shifted value of BEValue
5898 // by one iteration:
5899 // PHI(f(0), f({1,+,1})) --> f({0,+,1})
5900
5901 // Do not allow refinement in rewriting of BEValue.
5902 const SCEV *Shifted = SCEVShiftRewriter::rewrite(S: BEValue, L, SE&: *this);
5903 const SCEV *Start = SCEVInitRewriter::rewrite(S: Shifted, L, SE&: *this, IgnoreOtherLoops: false);
5904 if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
5905 isGuaranteedNotToCauseUB(Op: Shifted) && ::impliesPoison(AssumedPoison: Shifted, S: Start)) {
5906 const SCEV *StartVal = getSCEV(V: StartValueV);
5907 if (Start == StartVal) {
5908 // Okay, for the entire analysis of this edge we assumed the PHI
5909 // to be symbolic. We now need to go back and purge all of the
5910 // entries for the scalars that use the symbolic expression.
5911 forgetMemoizedResults(SCEVs: {SymbolicName});
5912 insertValueToMap(V: PN, S: Shifted);
5913 return Shifted;
5914 }
5915 }
5916 }
5917
5918 // Remove the temporary PHI node SCEV that has been inserted while intending
5919 // to create an AddRecExpr for this PHI node. We can not keep this temporary
5920 // as it will prevent later (possibly simpler) SCEV expressions to be added
5921 // to the ValueExprMap.
5922 eraseValueFromMap(V: PN);
5923
5924 return nullptr;
5925}
5926
5927// Try to match a control flow sequence that branches out at BI and merges back
5928// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
5929// match.
5930static bool BrPHIToSelect(DominatorTree &DT, CondBrInst *BI, PHINode *Merge,
5931 Value *&C, Value *&LHS, Value *&RHS) {
5932 C = BI->getCondition();
5933
5934 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(i: 0));
5935 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(i: 1));
5936
5937 Use &LeftUse = Merge->getOperandUse(i: 0);
5938 Use &RightUse = Merge->getOperandUse(i: 1);
5939
5940 if (DT.dominates(BBE: LeftEdge, U: LeftUse) && DT.dominates(BBE: RightEdge, U: RightUse)) {
5941 LHS = LeftUse;
5942 RHS = RightUse;
5943 return true;
5944 }
5945
5946 if (DT.dominates(BBE: LeftEdge, U: RightUse) && DT.dominates(BBE: RightEdge, U: LeftUse)) {
5947 LHS = RightUse;
5948 RHS = LeftUse;
5949 return true;
5950 }
5951
5952 return false;
5953}
5954
5955static bool getOperandsForSelectLikePHI(DominatorTree &DT, PHINode *PN,
5956 Value *&Cond, Value *&LHS,
5957 Value *&RHS) {
5958 auto IsReachable =
5959 [&](BasicBlock *BB) { return DT.isReachableFromEntry(A: BB); };
5960 if (PN->getNumIncomingValues() == 2 && all_of(Range: PN->blocks(), P: IsReachable)) {
5961 // Try to match
5962 //
5963 // br %cond, label %left, label %right
5964 // left:
5965 // br label %merge
5966 // right:
5967 // br label %merge
5968 // merge:
5969 // V = phi [ %x, %left ], [ %y, %right ]
5970 //
5971 // as "select %cond, %x, %y"
5972
5973 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
5974 assert(IDom && "At least the entry block should dominate PN");
5975
5976 auto *BI = dyn_cast<CondBrInst>(Val: IDom->getTerminator());
5977 return BI && BrPHIToSelect(DT, BI, Merge: PN, C&: Cond, LHS, RHS);
5978 }
5979 return false;
5980}
5981
5982const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
5983 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
5984 if (getOperandsForSelectLikePHI(DT, PN, Cond, LHS, RHS) &&
5985 properlyDominates(S: getSCEV(V: LHS), BB: PN->getParent()) &&
5986 properlyDominates(S: getSCEV(V: RHS), BB: PN->getParent()))
5987 return createNodeForSelectOrPHI(V: PN, Cond, TrueVal: LHS, FalseVal: RHS);
5988
5989 return nullptr;
5990}
5991
5992static BinaryOperator *getCommonInstForPHI(PHINode *PN) {
5993 BinaryOperator *CommonInst = nullptr;
5994 // Check if instructions are identical.
5995 for (Value *Incoming : PN->incoming_values()) {
5996 auto *IncomingInst = dyn_cast<BinaryOperator>(Val: Incoming);
5997 if (!IncomingInst)
5998 return nullptr;
5999 if (CommonInst) {
6000 if (!CommonInst->isIdenticalToWhenDefined(I: IncomingInst))
6001 return nullptr; // Not identical, give up
6002 } else {
6003 // Remember binary operator
6004 CommonInst = IncomingInst;
6005 }
6006 }
6007 return CommonInst;
6008}
6009
6010/// Returns SCEV for the first operand of a phi if all phi operands have
6011/// identical opcodes and operands
6012/// eg.
6013/// a: %add = %a + %b
6014/// br %c
6015/// b: %add1 = %a + %b
6016/// br %c
6017/// c: %phi = phi [%add, a], [%add1, b]
6018/// scev(%phi) => scev(%add)
6019const SCEV *
6020ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
6021 BinaryOperator *CommonInst = getCommonInstForPHI(PN);
6022 if (!CommonInst)
6023 return nullptr;
6024
6025 // Check if SCEV exprs for instructions are identical.
6026 const SCEV *CommonSCEV = getSCEV(V: CommonInst);
6027 bool SCEVExprsIdentical =
6028 all_of(Range: drop_begin(RangeOrContainer: PN->incoming_values()),
6029 P: [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
6030 return SCEVExprsIdentical ? CommonSCEV : nullptr;
6031}
6032
6033const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
6034 if (const SCEV *S = createAddRecFromPHI(PN))
6035 return S;
6036
6037 // We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
6038 // phi node for X.
6039 if (Value *V = simplifyInstruction(
6040 I: PN, Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
6041 /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
6042 return getSCEV(V);
6043
6044 if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
6045 return S;
6046
6047 if (const SCEV *S = createNodeFromSelectLikePHI(PN))
6048 return S;
6049
6050 // If it's not a loop phi, we can't handle it yet.
6051 return getUnknown(V: PN);
6052}
6053
6054bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
6055 SCEVTypes RootKind) {
6056 struct FindClosure {
6057 const SCEV *OperandToFind;
6058 const SCEVTypes RootKind; // Must be a sequential min/max expression.
6059 const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind.
6060
6061 bool Found = false;
6062
6063 bool canRecurseInto(SCEVTypes Kind) const {
6064 // We can only recurse into the SCEV expression of the same effective type
6065 // as the type of our root SCEV expression, and into zero-extensions.
6066 return RootKind == Kind || NonSequentialRootKind == Kind ||
6067 scZeroExtend == Kind;
6068 };
6069
6070 FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind)
6071 : OperandToFind(OperandToFind), RootKind(RootKind),
6072 NonSequentialRootKind(
6073 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
6074 Ty: RootKind)) {}
6075
6076 bool follow(const SCEV *S) {
6077 Found = S == OperandToFind;
6078
6079 return !isDone() && canRecurseInto(Kind: S->getSCEVType());
6080 }
6081
6082 bool isDone() const { return Found; }
6083 };
6084
6085 FindClosure FC(OperandToFind, RootKind);
6086 visitAll(Root, Visitor&: FC);
6087 return FC.Found;
6088}
6089
6090std::optional<const SCEV *>
6091ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(Type *Ty,
6092 ICmpInst *Cond,
6093 Value *TrueVal,
6094 Value *FalseVal) {
6095 // Try to match some simple smax or umax patterns.
6096 auto *ICI = Cond;
6097
6098 Value *LHS = ICI->getOperand(i_nocapture: 0);
6099 Value *RHS = ICI->getOperand(i_nocapture: 1);
6100
6101 switch (ICI->getPredicate()) {
6102 case ICmpInst::ICMP_SLT:
6103 case ICmpInst::ICMP_SLE:
6104 case ICmpInst::ICMP_ULT:
6105 case ICmpInst::ICMP_ULE:
6106 std::swap(a&: LHS, b&: RHS);
6107 [[fallthrough]];
6108 case ICmpInst::ICMP_SGT:
6109 case ICmpInst::ICMP_SGE:
6110 case ICmpInst::ICMP_UGT:
6111 case ICmpInst::ICMP_UGE:
6112 // a > b ? a+x : b+x -> max(a, b)+x
6113 // a > b ? b+x : a+x -> min(a, b)+x
6114 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty)) {
6115 bool Signed = ICI->isSigned();
6116 const SCEV *LA = getSCEV(V: TrueVal);
6117 const SCEV *RA = getSCEV(V: FalseVal);
6118 const SCEV *LS = getSCEV(V: LHS);
6119 const SCEV *RS = getSCEV(V: RHS);
6120 if (LA->getType()->isPointerTy()) {
6121 // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
6122 // Need to make sure we can't produce weird expressions involving
6123 // negated pointers.
6124 if (LA == LS && RA == RS)
6125 return Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS);
6126 if (LA == RS && RA == LS)
6127 return Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS);
6128 }
6129 auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
6130 if (Op->getType()->isPointerTy()) {
6131 Op = getPtrToAddrExpr(Op);
6132 if (isa<SCEVCouldNotCompute>(Val: Op))
6133 return Op;
6134 }
6135 if (Signed)
6136 Op = getNoopOrSignExtend(V: Op, Ty);
6137 else
6138 Op = getNoopOrZeroExtend(V: Op, Ty);
6139 return Op;
6140 };
6141 LS = CoerceOperand(LS);
6142 RS = CoerceOperand(RS);
6143 if (isa<SCEVCouldNotCompute>(Val: LS) || isa<SCEVCouldNotCompute>(Val: RS))
6144 break;
6145 const SCEV *LDiff = getMinusSCEV(LHS: LA, RHS: LS);
6146 const SCEV *RDiff = getMinusSCEV(LHS: RA, RHS: RS);
6147 if (LDiff == RDiff)
6148 return getAddExpr(LHS: Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS),
6149 RHS: LDiff);
6150 LDiff = getMinusSCEV(LHS: LA, RHS: RS);
6151 RDiff = getMinusSCEV(LHS: RA, RHS: LS);
6152 if (LDiff == RDiff)
6153 return getAddExpr(LHS: Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS),
6154 RHS: LDiff);
6155 }
6156 break;
6157 case ICmpInst::ICMP_NE:
6158 // x != 0 ? x+y : C+y -> x == 0 ? C+y : x+y
6159 std::swap(a&: TrueVal, b&: FalseVal);
6160 [[fallthrough]];
6161 case ICmpInst::ICMP_EQ:
6162 // x == 0 ? C+y : x+y -> umax(x, C)+y iff C u<= 1
6163 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty) &&
6164 isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()) {
6165 const SCEV *X = getNoopOrZeroExtend(V: getSCEV(V: LHS), Ty);
6166 const SCEV *TrueValExpr = getSCEV(V: TrueVal); // C+y
6167 const SCEV *FalseValExpr = getSCEV(V: FalseVal); // x+y
6168 const SCEV *Y = getMinusSCEV(LHS: FalseValExpr, RHS: X); // y = (x+y)-x
6169 const SCEV *C = getMinusSCEV(LHS: TrueValExpr, RHS: Y); // C = (C+y)-y
6170 if (isa<SCEVConstant>(Val: C) && cast<SCEVConstant>(Val: C)->getAPInt().ule(RHS: 1))
6171 return getAddExpr(LHS: getUMaxExpr(LHS: X, RHS: C), RHS: Y);
6172 }
6173 // x == 0 ? 0 : umin (..., x, ...) -> umin_seq(x, umin (...))
6174 // x == 0 ? 0 : umin_seq(..., x, ...) -> umin_seq(x, umin_seq(...))
6175 // x == 0 ? 0 : umin (..., umin_seq(..., x, ...), ...)
6176 // -> umin_seq(x, umin (..., umin_seq(...), ...))
6177 if (isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero() &&
6178 isa<ConstantInt>(Val: TrueVal) && cast<ConstantInt>(Val: TrueVal)->isZero()) {
6179 const SCEV *X = getSCEV(V: LHS);
6180 while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: X))
6181 X = ZExt->getOperand();
6182 if (getTypeSizeInBits(Ty: X->getType()) <= getTypeSizeInBits(Ty)) {
6183 const SCEV *FalseValExpr = getSCEV(V: FalseVal);
6184 if (SCEVMinMaxExprContains(Root: FalseValExpr, OperandToFind: X, RootKind: scSequentialUMinExpr))
6185 return getUMinExpr(LHS: getNoopOrZeroExtend(V: X, Ty), RHS: FalseValExpr,
6186 /*Sequential=*/true);
6187 }
6188 }
6189 break;
6190 default:
6191 break;
6192 }
6193
6194 return std::nullopt;
6195}
6196
6197static std::optional<const SCEV *>
6198createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr,
6199 const SCEV *TrueExpr, const SCEV *FalseExpr) {
6200 assert(CondExpr->getType()->isIntegerTy(1) &&
6201 TrueExpr->getType() == FalseExpr->getType() &&
6202 TrueExpr->getType()->isIntegerTy(1) &&
6203 "Unexpected operands of a select.");
6204
6205 // i1 cond ? i1 x : i1 C --> C + (i1 cond ? (i1 x - i1 C) : i1 0)
6206 // --> C + (umin_seq cond, x - C)
6207 //
6208 // i1 cond ? i1 C : i1 x --> C + (i1 cond ? i1 0 : (i1 x - i1 C))
6209 // --> C + (i1 ~cond ? (i1 x - i1 C) : i1 0)
6210 // --> C + (umin_seq ~cond, x - C)
6211
6212 // FIXME: while we can't legally model the case where both of the hands
6213 // are fully variable, we only require that the *difference* is constant.
6214 if (!isa<SCEVConstant>(Val: TrueExpr) && !isa<SCEVConstant>(Val: FalseExpr))
6215 return std::nullopt;
6216
6217 const SCEV *X, *C;
6218 if (isa<SCEVConstant>(Val: TrueExpr)) {
6219 CondExpr = SE->getNotSCEV(V: CondExpr);
6220 X = FalseExpr;
6221 C = TrueExpr;
6222 } else {
6223 X = TrueExpr;
6224 C = FalseExpr;
6225 }
6226 return SE->getAddExpr(LHS: C, RHS: SE->getUMinExpr(LHS: CondExpr, RHS: SE->getMinusSCEV(LHS: X, RHS: C),
6227 /*Sequential=*/true));
6228}
6229
6230static std::optional<const SCEV *>
6231createNodeForSelectViaUMinSeq(ScalarEvolution *SE, Value *Cond, Value *TrueVal,
6232 Value *FalseVal) {
6233 if (!isa<ConstantInt>(Val: TrueVal) && !isa<ConstantInt>(Val: FalseVal))
6234 return std::nullopt;
6235
6236 const auto *SECond = SE->getSCEV(V: Cond);
6237 const auto *SETrue = SE->getSCEV(V: TrueVal);
6238 const auto *SEFalse = SE->getSCEV(V: FalseVal);
6239 return createNodeForSelectViaUMinSeq(SE, CondExpr: SECond, TrueExpr: SETrue, FalseExpr: SEFalse);
6240}
6241
6242const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
6243 Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) {
6244 assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?");
6245 assert(TrueVal->getType() == FalseVal->getType() &&
6246 V->getType() == TrueVal->getType() &&
6247 "Types of select hands and of the result must match.");
6248
6249 // For now, only deal with i1-typed `select`s.
6250 if (!V->getType()->isIntegerTy(BitWidth: 1))
6251 return getUnknown(V);
6252
6253 if (std::optional<const SCEV *> S =
6254 createNodeForSelectViaUMinSeq(SE: this, Cond, TrueVal, FalseVal))
6255 return *S;
6256
6257 return getUnknown(V);
6258}
6259
6260const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond,
6261 Value *TrueVal,
6262 Value *FalseVal) {
6263 // Handle "constant" branch or select. This can occur for instance when a
6264 // loop pass transforms an inner loop and moves on to process the outer loop.
6265 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond))
6266 return getSCEV(V: CI->isOne() ? TrueVal : FalseVal);
6267
6268 if (auto *I = dyn_cast<Instruction>(Val: V)) {
6269 if (auto *ICI = dyn_cast<ICmpInst>(Val: Cond)) {
6270 if (std::optional<const SCEV *> S =
6271 createNodeForSelectOrPHIInstWithICmpInstCond(Ty: I->getType(), Cond: ICI,
6272 TrueVal, FalseVal))
6273 return *S;
6274 }
6275 }
6276
6277 return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal);
6278}
6279
6280/// Expand GEP instructions into add and multiply operations. This allows them
6281/// to be analyzed by regular SCEV code.
6282const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
6283 assert(GEP->getSourceElementType()->isSized() &&
6284 "GEP source element type must be sized");
6285
6286 SmallVector<SCEVUse, 4> IndexExprs;
6287 for (Value *Index : GEP->indices())
6288 IndexExprs.push_back(Elt: getSCEV(V: Index));
6289 return getGEPExpr(GEP, IndexExprs);
6290}
6291
6292APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S,
6293 const Instruction *CtxI) {
6294 uint64_t BitWidth = getTypeSizeInBits(Ty: S->getType());
6295 auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
6296 return TrailingZeros >= BitWidth
6297 ? APInt::getZero(numBits: BitWidth)
6298 : APInt::getOneBitSet(numBits: BitWidth, BitNo: TrailingZeros);
6299 };
6300 auto GetGCDMultiple = [this, CtxI](const SCEVNAryExpr *N) {
6301 // The result is GCD of all operands results.
6302 APInt Res = getConstantMultiple(S: N->getOperand(i: 0), CtxI);
6303 for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
6304 Res = APIntOps::GreatestCommonDivisor(
6305 A: Res, B: getConstantMultiple(S: N->getOperand(i: I), CtxI));
6306 return Res;
6307 };
6308
6309 switch (S->getSCEVType()) {
6310 case scConstant:
6311 return cast<SCEVConstant>(Val: S)->getAPInt();
6312 case scPtrToAddr:
6313 return getConstantMultiple(S: cast<SCEVCastExpr>(Val: S)->getOperand());
6314 case scUDivExpr:
6315 case scVScale:
6316 return APInt(BitWidth, 1);
6317 case scTruncate: {
6318 // Only multiples that are a power of 2 will hold after truncation.
6319 const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(Val: S);
6320 uint32_t TZ = getMinTrailingZeros(S: T->getOperand(), CtxI);
6321 return GetShiftedByZeros(TZ);
6322 }
6323 case scZeroExtend: {
6324 const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(Val: S);
6325 return getConstantMultiple(S: Z->getOperand(), CtxI).zext(width: BitWidth);
6326 }
6327 case scSignExtend: {
6328 // Only multiples that are a power of 2 will hold after sext.
6329 const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(Val: S);
6330 uint32_t TZ = getMinTrailingZeros(S: E->getOperand(), CtxI);
6331 return GetShiftedByZeros(TZ);
6332 }
6333 case scMulExpr: {
6334 const SCEVMulExpr *M = cast<SCEVMulExpr>(Val: S);
6335 if (M->hasNoUnsignedWrap()) {
6336 // The result is the product of all operand results.
6337 APInt Res = getConstantMultiple(S: M->getOperand(i: 0), CtxI);
6338 for (const SCEV *Operand : M->operands().drop_front())
6339 Res = Res * getConstantMultiple(S: Operand, CtxI);
6340 return Res;
6341 }
6342
6343 // If there are no wrap guarentees, find the trailing zeros, which is the
6344 // sum of trailing zeros for all its operands.
6345 uint32_t TZ = 0;
6346 for (const SCEV *Operand : M->operands())
6347 TZ += getMinTrailingZeros(S: Operand, CtxI);
6348 return GetShiftedByZeros(TZ);
6349 }
6350 case scAddExpr:
6351 case scAddRecExpr: {
6352 const SCEVNAryExpr *N = cast<SCEVNAryExpr>(Val: S);
6353 if (N->hasNoUnsignedWrap())
6354 return GetGCDMultiple(N);
6355 // Find the trailing bits, which is the minimum of its operands.
6356 uint32_t TZ = getMinTrailingZeros(S: N->getOperand(i: 0), CtxI);
6357 for (const SCEV *Operand : N->operands().drop_front())
6358 TZ = std::min(a: TZ, b: getMinTrailingZeros(S: Operand, CtxI));
6359 return GetShiftedByZeros(TZ);
6360 }
6361 case scUMaxExpr:
6362 case scSMaxExpr:
6363 case scUMinExpr:
6364 case scSMinExpr:
6365 case scSequentialUMinExpr:
6366 return GetGCDMultiple(cast<SCEVNAryExpr>(Val: S));
6367 case scUnknown: {
6368 // Ask ValueTracking for known bits. SCEVUnknown only become available at
6369 // the point their underlying IR instruction has been defined. If CtxI was
6370 // not provided, use:
6371 // * the first instruction in the entry block if it is an argument
6372 // * the instruction itself otherwise.
6373 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6374 if (!CtxI) {
6375 if (isa<Argument>(Val: U->getValue()))
6376 CtxI = &*F.getEntryBlock().begin();
6377 else if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
6378 CtxI = I;
6379 }
6380 unsigned Known =
6381 computeKnownBits(V: U->getValue(),
6382 Q: SimplifyQuery(getDataLayout(), &DT, &AC, CtxI)
6383 .allowEphemerals(AllowEphemerals: true))
6384 .countMinTrailingZeros();
6385 return GetShiftedByZeros(Known);
6386 }
6387 case scCouldNotCompute:
6388 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6389 }
6390 llvm_unreachable("Unknown SCEV kind!");
6391}
6392
6393APInt ScalarEvolution::getConstantMultiple(const SCEV *S,
6394 const Instruction *CtxI) {
6395 // Skip looking up and updating the cache if there is a context instruction,
6396 // as the result will only be valid in the specified context.
6397 if (CtxI)
6398 return getConstantMultipleImpl(S, CtxI);
6399
6400 auto I = ConstantMultipleCache.find(Val: S);
6401 if (I != ConstantMultipleCache.end())
6402 return I->second;
6403
6404 APInt Result = getConstantMultipleImpl(S, CtxI);
6405 auto InsertPair = ConstantMultipleCache.insert(KV: {S, Result});
6406 assert(InsertPair.second && "Should insert a new key");
6407 return InsertPair.first->second;
6408}
6409
6410APInt ScalarEvolution::getNonZeroConstantMultiple(const SCEV *S) {
6411 APInt Multiple = getConstantMultiple(S);
6412 return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
6413}
6414
6415uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S,
6416 const Instruction *CtxI) {
6417 return std::min(a: getConstantMultiple(S, CtxI).countTrailingZeros(),
6418 b: (unsigned)getTypeSizeInBits(Ty: S->getType()));
6419}
6420
6421/// Helper method to assign a range to V from metadata present in the IR.
6422static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
6423 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
6424 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_range))
6425 return getConstantRangeFromMetadata(RangeMD: *MD);
6426 if (const auto *CB = dyn_cast<CallBase>(Val: V))
6427 if (std::optional<ConstantRange> Range = CB->getRange())
6428 return Range;
6429 }
6430 if (auto *A = dyn_cast<Argument>(Val: V))
6431 if (std::optional<ConstantRange> Range = A->getRange())
6432 return Range;
6433
6434 return std::nullopt;
6435}
6436
6437void ScalarEvolution::setNoWrapFlags(SCEVAddRecExpr *AddRec,
6438 SCEV::NoWrapFlags Flags) {
6439 if (AddRec->getNoWrapFlags(Mask: Flags) != Flags) {
6440 AddRec->setNoWrapFlags(Flags);
6441 UnsignedRanges.erase(Val: AddRec);
6442 SignedRanges.erase(Val: AddRec);
6443 ConstantMultipleCache.erase(Val: AddRec);
6444 }
6445}
6446
6447ConstantRange ScalarEvolution::
6448getRangeForUnknownRecurrence(const SCEVUnknown *U) {
6449 const DataLayout &DL = getDataLayout();
6450
6451 unsigned BitWidth = getTypeSizeInBits(Ty: U->getType());
6452 const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
6453
6454 // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
6455 // use information about the trip count to improve our available range. Note
6456 // that the trip count independent cases are already handled by known bits.
6457 // WARNING: The definition of recurrence used here is subtly different than
6458 // the one used by AddRec (and thus most of this file). Step is allowed to
6459 // be arbitrarily loop varying here, where AddRec allows only loop invariant
6460 // and other addrecs in the same loop (for non-affine addrecs). The code
6461 // below intentionally handles the case where step is not loop invariant.
6462 auto *P = dyn_cast<PHINode>(Val: U->getValue());
6463 if (!P)
6464 return FullSet;
6465
6466 // Make sure that no Phi input comes from an unreachable block. Otherwise,
6467 // even the values that are not available in these blocks may come from them,
6468 // and this leads to false-positive recurrence test.
6469 for (auto *Pred : predecessors(BB: P->getParent()))
6470 if (!DT.isReachableFromEntry(A: Pred))
6471 return FullSet;
6472
6473 BinaryOperator *BO;
6474 Value *Start, *Step;
6475 if (!matchSimpleRecurrence(P, BO, Start, Step))
6476 return FullSet;
6477
6478 // If we found a recurrence in reachable code, we must be in a loop. Note
6479 // that BO might be in some subloop of L, and that's completely okay.
6480 auto *L = LI.getLoopFor(BB: P->getParent());
6481 assert(L && L->getHeader() == P->getParent());
6482 if (!L->contains(BB: BO->getParent()))
6483 // NOTE: This bailout should be an assert instead. However, asserting
6484 // the condition here exposes a case where LoopFusion is querying SCEV
6485 // with malformed loop information during the midst of the transform.
6486 // There doesn't appear to be an obvious fix, so for the moment bailout
6487 // until the caller issue can be fixed. PR49566 tracks the bug.
6488 return FullSet;
6489
6490 // TODO: Extend to other opcodes such as mul, and div
6491 switch (BO->getOpcode()) {
6492 default:
6493 return FullSet;
6494 case Instruction::AShr:
6495 case Instruction::LShr:
6496 case Instruction::Shl:
6497 break;
6498 };
6499
6500 if (BO->getOperand(i_nocapture: 0) != P)
6501 // TODO: Handle the power function forms some day.
6502 return FullSet;
6503
6504 unsigned TC = getSmallConstantMaxTripCount(L);
6505 if (!TC || TC >= BitWidth)
6506 return FullSet;
6507
6508 auto KnownStart = computeKnownBits(V: Start, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6509 auto KnownStep = computeKnownBits(V: Step, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6510 assert(KnownStart.getBitWidth() == BitWidth &&
6511 KnownStep.getBitWidth() == BitWidth);
6512
6513 // Compute total shift amount, being careful of overflow and bitwidths.
6514 auto MaxShiftAmt = KnownStep.getMaxValue();
6515 APInt TCAP(BitWidth, TC-1);
6516 bool Overflow = false;
6517 auto TotalShift = MaxShiftAmt.umul_ov(RHS: TCAP, Overflow);
6518 if (Overflow)
6519 return FullSet;
6520
6521 switch (BO->getOpcode()) {
6522 default:
6523 llvm_unreachable("filtered out above");
6524 case Instruction::AShr: {
6525 // For each ashr, three cases:
6526 // shift = 0 => unchanged value
6527 // saturation => 0 or -1
6528 // other => a value closer to zero (of the same sign)
6529 // Thus, the end value is closer to zero than the start.
6530 auto KnownEnd = KnownBits::ashr(LHS: KnownStart,
6531 RHS: KnownBits::makeConstant(C: TotalShift));
6532 if (KnownStart.isNonNegative())
6533 // Analogous to lshr (simply not yet canonicalized)
6534 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6535 Upper: KnownStart.getMaxValue() + 1);
6536 if (KnownStart.isNegative())
6537 // End >=u Start && End <=s Start
6538 return ConstantRange::getNonEmpty(Lower: KnownStart.getMinValue(),
6539 Upper: KnownEnd.getMaxValue() + 1);
6540 break;
6541 }
6542 case Instruction::LShr: {
6543 // For each lshr, three cases:
6544 // shift = 0 => unchanged value
6545 // saturation => 0
6546 // other => a smaller positive number
6547 // Thus, the low end of the unsigned range is the last value produced.
6548 auto KnownEnd = KnownBits::lshr(LHS: KnownStart,
6549 RHS: KnownBits::makeConstant(C: TotalShift));
6550 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6551 Upper: KnownStart.getMaxValue() + 1);
6552 }
6553 case Instruction::Shl: {
6554 // Iff no bits are shifted out, value increases on every shift.
6555 auto KnownEnd = KnownBits::shl(LHS: KnownStart,
6556 RHS: KnownBits::makeConstant(C: TotalShift));
6557 if (TotalShift.ult(RHS: KnownStart.countMinLeadingZeros()))
6558 return ConstantRange(KnownStart.getMinValue(),
6559 KnownEnd.getMaxValue() + 1);
6560 break;
6561 }
6562 };
6563 return FullSet;
6564}
6565
6566// The goal of this function is to check if recursively visiting the operands
6567// of this PHI might lead to an infinite loop. If we do see such a loop,
6568// there's no good way to break it, so we avoid analyzing such cases.
6569//
6570// getRangeRef previously used a visited set to avoid infinite loops, but this
6571// caused other issues: the result was dependent on the order of getRangeRef
6572// calls, and the interaction with createSCEVIter could cause a stack overflow
6573// in some cases (see issue #148253).
6574//
6575// FIXME: The way this is implemented is overly conservative; this checks
6576// for a few obviously safe patterns, but anything that doesn't lead to
6577// recursion is fine.
6578static bool RangeRefPHIAllowedOperands(DominatorTree &DT, PHINode *PHI) {
6579 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
6580 if (getOperandsForSelectLikePHI(DT, PN: PHI, Cond, LHS, RHS))
6581 return true;
6582
6583 if (all_of(Range: PHI->operands(),
6584 P: [&](Value *Operand) { return DT.dominates(Def: Operand, User: PHI); }))
6585 return true;
6586
6587 return false;
6588}
6589
6590const ConstantRange &
6591ScalarEvolution::getRangeRefIter(const SCEV *S,
6592 ScalarEvolution::RangeSignHint SignHint) {
6593 DenseMap<const SCEV *, ConstantRange> &Cache =
6594 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6595 : SignedRanges;
6596 SmallVector<SCEVUse> WorkList;
6597 SmallPtrSet<const SCEV *, 8> Seen;
6598
6599 // Add Expr to the worklist, if Expr is either an N-ary expression or a
6600 // SCEVUnknown PHI node.
6601 auto AddToWorklist = [&WorkList, &Seen, &Cache](const SCEV *Expr) {
6602 if (!Seen.insert(Ptr: Expr).second)
6603 return;
6604 if (Cache.contains(Val: Expr))
6605 return;
6606 switch (Expr->getSCEVType()) {
6607 case scUnknown:
6608 if (!isa<PHINode>(Val: cast<SCEVUnknown>(Val: Expr)->getValue()))
6609 break;
6610 [[fallthrough]];
6611 case scConstant:
6612 case scVScale:
6613 case scTruncate:
6614 case scZeroExtend:
6615 case scSignExtend:
6616 case scPtrToAddr:
6617 case scAddExpr:
6618 case scMulExpr:
6619 case scUDivExpr:
6620 case scAddRecExpr:
6621 case scUMaxExpr:
6622 case scSMaxExpr:
6623 case scUMinExpr:
6624 case scSMinExpr:
6625 case scSequentialUMinExpr:
6626 WorkList.push_back(Elt: Expr);
6627 break;
6628 case scCouldNotCompute:
6629 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6630 }
6631 };
6632 AddToWorklist(S);
6633
6634 // Build worklist by queuing operands of N-ary expressions and phi nodes.
6635 for (unsigned I = 0; I != WorkList.size(); ++I) {
6636 const SCEV *P = WorkList[I];
6637 auto *UnknownS = dyn_cast<SCEVUnknown>(Val: P);
6638 // If it is not a `SCEVUnknown`, just recurse into operands.
6639 if (!UnknownS) {
6640 for (const SCEV *Op : P->operands())
6641 AddToWorklist(Op);
6642 continue;
6643 }
6644 // `SCEVUnknown`'s require special treatment.
6645 if (PHINode *P = dyn_cast<PHINode>(Val: UnknownS->getValue())) {
6646 if (!RangeRefPHIAllowedOperands(DT, PHI: P))
6647 continue;
6648 for (auto &Op : reverse(C: P->operands()))
6649 AddToWorklist(getSCEV(V: Op));
6650 }
6651 }
6652
6653 if (!WorkList.empty()) {
6654 // Use getRangeRef to compute ranges for items in the worklist in reverse
6655 // order. This will force ranges for earlier operands to be computed before
6656 // their users in most cases.
6657 for (const SCEV *P : reverse(C: drop_begin(RangeOrContainer&: WorkList))) {
6658 getRangeRef(S: P, Hint: SignHint);
6659 }
6660 }
6661
6662 return getRangeRef(S, Hint: SignHint, Depth: 0);
6663}
6664
6665const APInt *ScalarEvolution::getConstantAPIntOrNull(const SCEV *S) {
6666 if (const auto *C = dyn_cast<SCEVConstant>(Val: S))
6667 return &C->getAPInt();
6668 return nullptr;
6669}
6670
6671/// Determine the range for a particular SCEV. If SignHint is
6672/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
6673/// with a "cleaner" unsigned (resp. signed) representation.
6674const ConstantRange &ScalarEvolution::getRangeRef(
6675 const SCEV *S, ScalarEvolution::RangeSignHint SignHint, unsigned Depth) {
6676 DenseMap<const SCEV *, ConstantRange> &Cache =
6677 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6678 : SignedRanges;
6679 ConstantRange::PreferredRangeType RangeType =
6680 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? ConstantRange::Unsigned
6681 : ConstantRange::Signed;
6682
6683 // See if we've computed this range already.
6684 auto I = Cache.find(Val: S);
6685 if (I != Cache.end())
6686 return I->second;
6687
6688 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: S))
6689 return setRange(S: C, Hint: SignHint, CR: ConstantRange(C->getAPInt()));
6690
6691 // Switch to iteratively computing the range for S, if it is part of a deeply
6692 // nested expression.
6693 if (Depth > RangeIterThreshold)
6694 return getRangeRefIter(S, SignHint);
6695
6696 unsigned BitWidth = getTypeSizeInBits(Ty: S->getType());
6697 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
6698 using OBO = OverflowingBinaryOperator;
6699
6700 // If the value has known zeros, the maximum value will have those known zeros
6701 // as well.
6702 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
6703 APInt Multiple = getNonZeroConstantMultiple(S);
6704 APInt Remainder = APInt::getMaxValue(numBits: BitWidth).urem(RHS: Multiple);
6705 if (!Remainder.isZero())
6706 ConservativeResult =
6707 ConstantRange(APInt::getMinValue(numBits: BitWidth),
6708 APInt::getMaxValue(numBits: BitWidth) - Remainder + 1);
6709 }
6710 else {
6711 uint32_t TZ = getMinTrailingZeros(S);
6712 if (TZ != 0) {
6713 ConservativeResult = ConstantRange(
6714 APInt::getSignedMinValue(numBits: BitWidth),
6715 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: TZ).shl(shiftAmt: TZ) + 1);
6716 }
6717 }
6718
6719 switch (S->getSCEVType()) {
6720 case scConstant:
6721 llvm_unreachable("Already handled above.");
6722 case scVScale:
6723 return setRange(S, Hint: SignHint, CR: getVScaleRange(F: &F, BitWidth));
6724 case scTruncate: {
6725 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: S);
6726 ConstantRange X = getRangeRef(S: Trunc->getOperand(), SignHint, Depth: Depth + 1);
6727 return setRange(
6728 S: Trunc, Hint: SignHint,
6729 CR: ConservativeResult.intersectWith(CR: X.truncate(BitWidth), Type: RangeType));
6730 }
6731 case scZeroExtend: {
6732 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: S);
6733 ConstantRange X = getRangeRef(S: ZExt->getOperand(), SignHint, Depth: Depth + 1);
6734 return setRange(
6735 S: ZExt, Hint: SignHint,
6736 CR: ConservativeResult.intersectWith(CR: X.zeroExtend(BitWidth), Type: RangeType));
6737 }
6738 case scSignExtend: {
6739 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: S);
6740 ConstantRange X = getRangeRef(S: SExt->getOperand(), SignHint, Depth: Depth + 1);
6741 return setRange(
6742 S: SExt, Hint: SignHint,
6743 CR: ConservativeResult.intersectWith(CR: X.signExtend(BitWidth), Type: RangeType));
6744 }
6745 case scPtrToAddr: {
6746 const SCEVCastExpr *Cast = cast<SCEVCastExpr>(Val: S);
6747 ConstantRange X = getRangeRef(S: Cast->getOperand(), SignHint, Depth: Depth + 1);
6748 return setRange(S: Cast, Hint: SignHint, CR: X);
6749 }
6750 case scAddExpr: {
6751 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: S);
6752 // Check if this is a URem pattern: A - (A / B) * B, which is always < B.
6753 const SCEV *URemLHS = nullptr, *URemRHS = nullptr;
6754 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED &&
6755 match(S, P: m_scev_URem(LHS: m_SCEV(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE&: *this))) {
6756 ConstantRange LHSRange = getRangeRef(S: URemLHS, SignHint, Depth: Depth + 1);
6757 ConstantRange RHSRange = getRangeRef(S: URemRHS, SignHint, Depth: Depth + 1);
6758 ConservativeResult =
6759 ConservativeResult.intersectWith(CR: LHSRange.urem(Other: RHSRange), Type: RangeType);
6760 }
6761 ConstantRange X = getRangeRef(S: Add->getOperand(i: 0), SignHint, Depth: Depth + 1);
6762 unsigned WrapType = OBO::AnyWrap;
6763 if (Add->hasNoSignedWrap())
6764 WrapType |= OBO::NoSignedWrap;
6765 if (Add->hasNoUnsignedWrap())
6766 WrapType |= OBO::NoUnsignedWrap;
6767 for (const SCEV *Op : drop_begin(RangeOrContainer: Add->operands()))
6768 X = X.addWithNoWrap(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1), NoWrapKind: WrapType,
6769 RangeType);
6770 return setRange(S: Add, Hint: SignHint,
6771 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6772 }
6773 case scMulExpr: {
6774 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val: S);
6775 ConstantRange X = getRangeRef(S: Mul->getOperand(i: 0), SignHint, Depth: Depth + 1);
6776 for (const SCEV *Op : drop_begin(RangeOrContainer: Mul->operands()))
6777 X = X.multiply(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1));
6778 return setRange(S: Mul, Hint: SignHint,
6779 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6780 }
6781 case scUDivExpr: {
6782 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: S);
6783 ConstantRange X = getRangeRef(S: UDiv->getLHS(), SignHint, Depth: Depth + 1);
6784 ConstantRange Y = getRangeRef(S: UDiv->getRHS(), SignHint, Depth: Depth + 1);
6785 return setRange(S: UDiv, Hint: SignHint,
6786 CR: ConservativeResult.intersectWith(CR: X.udiv(Other: Y), Type: RangeType));
6787 }
6788 case scAddRecExpr: {
6789 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: S);
6790 // If there's no unsigned wrap, the value will never be less than its
6791 // initial value.
6792 if (AddRec->hasNoUnsignedWrap()) {
6793 APInt UnsignedMinValue = getUnsignedRangeMin(S: AddRec->getStart());
6794 if (!UnsignedMinValue.isZero())
6795 ConservativeResult = ConservativeResult.intersectWith(
6796 CR: ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), Type: RangeType);
6797 }
6798
6799 // If there's no signed wrap, and all the operands except initial value have
6800 // the same sign or zero, the value won't ever be:
6801 // 1: smaller than initial value if operands are non negative,
6802 // 2: bigger than initial value if operands are non positive.
6803 // For both cases, value can not cross signed min/max boundary.
6804 if (AddRec->hasNoSignedWrap()) {
6805 bool AllNonNeg = true;
6806 bool AllNonPos = true;
6807 for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
6808 if (!isKnownNonNegative(S: AddRec->getOperand(i)))
6809 AllNonNeg = false;
6810 if (!isKnownNonPositive(S: AddRec->getOperand(i)))
6811 AllNonPos = false;
6812 }
6813 if (AllNonNeg)
6814 ConservativeResult = ConservativeResult.intersectWith(
6815 CR: ConstantRange::getNonEmpty(Lower: getSignedRangeMin(S: AddRec->getStart()),
6816 Upper: APInt::getSignedMinValue(numBits: BitWidth)),
6817 Type: RangeType);
6818 else if (AllNonPos)
6819 ConservativeResult = ConservativeResult.intersectWith(
6820 CR: ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
6821 Upper: getSignedRangeMax(S: AddRec->getStart()) +
6822 1),
6823 Type: RangeType);
6824 }
6825
6826 // TODO: non-affine addrec
6827 if (AddRec->isAffine()) {
6828 const SCEV *MaxBEScev =
6829 getConstantMaxBackedgeTakenCount(L: AddRec->getLoop());
6830 if (!isa<SCEVCouldNotCompute>(Val: MaxBEScev)) {
6831 APInt MaxBECount = cast<SCEVConstant>(Val: MaxBEScev)->getAPInt();
6832
6833 // Adjust MaxBECount to the same bitwidth as AddRec. We can truncate if
6834 // MaxBECount's active bits are all <= AddRec's bit width.
6835 if (MaxBECount.getBitWidth() > BitWidth &&
6836 MaxBECount.getActiveBits() <= BitWidth)
6837 MaxBECount = MaxBECount.trunc(width: BitWidth);
6838 else if (MaxBECount.getBitWidth() < BitWidth)
6839 MaxBECount = MaxBECount.zext(width: BitWidth);
6840
6841 if (MaxBECount.getBitWidth() == BitWidth) {
6842 auto [RangeFromAffine, Flags] = getRangeForAffineAR(
6843 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
6844 ConservativeResult =
6845 ConservativeResult.intersectWith(CR: RangeFromAffine, Type: RangeType);
6846 const_cast<SCEVAddRecExpr *>(AddRec)->setNoWrapFlags(Flags);
6847
6848 auto RangeFromFactoring = getRangeViaFactoring(
6849 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
6850 ConservativeResult =
6851 ConservativeResult.intersectWith(CR: RangeFromFactoring, Type: RangeType);
6852 }
6853 }
6854
6855 // Now try symbolic BE count and more powerful methods.
6856 if (UseExpensiveRangeSharpening) {
6857 const SCEV *SymbolicMaxBECount =
6858 getSymbolicMaxBackedgeTakenCount(L: AddRec->getLoop());
6859 if (!isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount) &&
6860 getTypeSizeInBits(Ty: MaxBEScev->getType()) <= BitWidth &&
6861 AddRec->hasNoSelfWrap()) {
6862 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
6863 AddRec, MaxBECount: SymbolicMaxBECount, BitWidth, SignHint);
6864 ConservativeResult =
6865 ConservativeResult.intersectWith(CR: RangeFromAffineNew, Type: RangeType);
6866 }
6867 }
6868 }
6869
6870 return setRange(S: AddRec, Hint: SignHint, CR: std::move(ConservativeResult));
6871 }
6872 case scUMaxExpr:
6873 case scSMaxExpr:
6874 case scUMinExpr:
6875 case scSMinExpr:
6876 case scSequentialUMinExpr: {
6877 Intrinsic::ID ID;
6878 switch (S->getSCEVType()) {
6879 case scUMaxExpr:
6880 ID = Intrinsic::umax;
6881 break;
6882 case scSMaxExpr:
6883 ID = Intrinsic::smax;
6884 break;
6885 case scUMinExpr:
6886 case scSequentialUMinExpr:
6887 ID = Intrinsic::umin;
6888 break;
6889 case scSMinExpr:
6890 ID = Intrinsic::smin;
6891 break;
6892 default:
6893 llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.");
6894 }
6895
6896 const auto *NAry = cast<SCEVNAryExpr>(Val: S);
6897 ConstantRange X = getRangeRef(S: NAry->getOperand(i: 0), SignHint, Depth: Depth + 1);
6898 for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
6899 X = X.intrinsic(
6900 IntrinsicID: ID, Ops: {X, getRangeRef(S: NAry->getOperand(i), SignHint, Depth: Depth + 1)});
6901 return setRange(S, Hint: SignHint,
6902 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6903 }
6904 case scUnknown: {
6905 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6906 Value *V = U->getValue();
6907
6908 // Check if the IR explicitly contains !range metadata.
6909 std::optional<ConstantRange> MDRange = GetRangeFromMetadata(V);
6910 if (MDRange)
6911 ConservativeResult =
6912 ConservativeResult.intersectWith(CR: *MDRange, Type: RangeType);
6913
6914 // Use facts about recurrences in the underlying IR. Note that add
6915 // recurrences are AddRecExprs and thus don't hit this path. This
6916 // primarily handles shift recurrences.
6917 auto CR = getRangeForUnknownRecurrence(U);
6918 ConservativeResult = ConservativeResult.intersectWith(CR);
6919
6920 // See if ValueTracking can give us a useful range.
6921 const DataLayout &DL = getDataLayout();
6922 KnownBits Known = computeKnownBits(V, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6923 if (Known.getBitWidth() != BitWidth)
6924 Known = Known.zextOrTrunc(BitWidth);
6925
6926 // ValueTracking may be able to compute a tighter result for the number of
6927 // sign bits than for the value of those sign bits.
6928 unsigned NS = ComputeNumSignBits(Op: V, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6929 if (U->getType()->isPointerTy()) {
6930 // If the pointer size is larger than the index size type, this can cause
6931 // NS to be larger than BitWidth. So compensate for this.
6932 unsigned ptrSize = DL.getPointerTypeSizeInBits(U->getType());
6933 int ptrIdxDiff = ptrSize - BitWidth;
6934 if (ptrIdxDiff > 0 && ptrSize > BitWidth && NS > (unsigned)ptrIdxDiff)
6935 NS -= ptrIdxDiff;
6936 }
6937
6938 if (NS > 1) {
6939 // If we know any of the sign bits, we know all of the sign bits.
6940 if (!Known.Zero.getHiBits(numBits: NS).isZero())
6941 Known.Zero.setHighBits(NS);
6942 if (!Known.One.getHiBits(numBits: NS).isZero())
6943 Known.One.setHighBits(NS);
6944 }
6945
6946 if (Known.getMinValue() != Known.getMaxValue() + 1)
6947 ConservativeResult = ConservativeResult.intersectWith(
6948 CR: ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
6949 Type: RangeType);
6950 if (NS > 1)
6951 ConservativeResult = ConservativeResult.intersectWith(
6952 CR: ConstantRange(APInt::getSignedMinValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1),
6953 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1) + 1),
6954 Type: RangeType);
6955
6956 if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
6957 // Strengthen the range if the underlying IR value is a
6958 // global/alloca/heap allocation using the size of the object.
6959 bool CanBeNull;
6960 uint64_t DerefBytes = V->getPointerDereferenceableBytes(
6961 DL, CanBeNull, /*CanBeFreed=*/nullptr);
6962 if (DerefBytes > 1 && isUIntN(N: BitWidth, x: DerefBytes)) {
6963 // The highest address the object can start is DerefBytes bytes before
6964 // the end (unsigned max value). If this value is not a multiple of the
6965 // alignment, the last possible start value is the next lowest multiple
6966 // of the alignment. Note: The computations below cannot overflow,
6967 // because if they would there's no possible start address for the
6968 // object.
6969 APInt MaxVal =
6970 APInt::getMaxValue(numBits: BitWidth) - APInt(BitWidth, DerefBytes);
6971 uint64_t Align = U->getValue()->getPointerAlignment(DL).value();
6972 uint64_t Rem = MaxVal.urem(RHS: Align);
6973 MaxVal -= APInt(BitWidth, Rem);
6974 APInt MinVal = APInt::getZero(numBits: BitWidth);
6975 if (llvm::isKnownNonZero(V, Q: DL))
6976 MinVal = Align;
6977 ConservativeResult = ConservativeResult.intersectWith(
6978 CR: ConstantRange::getNonEmpty(Lower: MinVal, Upper: MaxVal + 1), Type: RangeType);
6979 }
6980 }
6981
6982 // A range of Phi is a subset of union of all ranges of its input.
6983 if (PHINode *Phi = dyn_cast<PHINode>(Val: V)) {
6984 // SCEVExpander sometimes creates SCEVUnknowns that are secretly
6985 // AddRecs; return the range for the corresponding AddRec.
6986 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: getSCEV(V)))
6987 return getRangeRef(S: AR, SignHint, Depth: Depth + 1);
6988
6989 // Make sure that we do not run over cycled Phis.
6990 if (RangeRefPHIAllowedOperands(DT, PHI: Phi)) {
6991 ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
6992
6993 for (const auto &Op : Phi->operands()) {
6994 auto OpRange = getRangeRef(S: getSCEV(V: Op), SignHint, Depth: Depth + 1);
6995 RangeFromOps = RangeFromOps.unionWith(CR: OpRange);
6996 // No point to continue if we already have a full set.
6997 if (RangeFromOps.isFullSet())
6998 break;
6999 }
7000 ConservativeResult =
7001 ConservativeResult.intersectWith(CR: RangeFromOps, Type: RangeType);
7002 }
7003 }
7004
7005 // vscale can't be equal to zero
7006 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V))
7007 if (II->getIntrinsicID() == Intrinsic::vscale) {
7008 ConstantRange Disallowed = APInt::getZero(numBits: BitWidth);
7009 ConservativeResult = ConservativeResult.difference(CR: Disallowed);
7010 }
7011
7012 return setRange(S: U, Hint: SignHint, CR: std::move(ConservativeResult));
7013 }
7014 case scCouldNotCompute:
7015 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
7016 }
7017
7018 return setRange(S, Hint: SignHint, CR: std::move(ConservativeResult));
7019}
7020
7021// Given a StartRange, Step and MaxBECount for an expression compute a range of
7022// values that the expression can take. Initially, the expression has a value
7023// from StartRange and then is changed by Step up to MaxBECount times. Signed
7024// argument defines if we treat Step as signed or unsigned. The second return
7025// value indicates that no wrapping occurred.
7026static std::pair<ConstantRange, bool>
7027getRangeForAffineARHelper(APInt Step, const ConstantRange &StartRange,
7028 const APInt &MaxBECount, bool Signed) {
7029 unsigned BitWidth = Step.getBitWidth();
7030 assert(BitWidth == StartRange.getBitWidth() &&
7031 BitWidth == MaxBECount.getBitWidth() && "mismatched bit widths");
7032 // If either Step or MaxBECount is 0, then the expression won't change, and we
7033 // just need to return the initial range.
7034 if (Step == 0 || MaxBECount == 0)
7035 return {StartRange, true};
7036
7037 // If we don't know anything about the initial value (i.e. StartRange is
7038 // FullRange), then we don't know anything about the final range either.
7039 // Return FullRange.
7040 if (StartRange.isFullSet())
7041 return {ConstantRange::getFull(BitWidth), false};
7042
7043 // If Step is signed and negative, then we use its absolute value, but we also
7044 // note that we're moving in the opposite direction.
7045 bool Descending = Signed && Step.isNegative();
7046
7047 if (Signed)
7048 // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
7049 // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
7050 // This equations hold true due to the well-defined wrap-around behavior of
7051 // APInt.
7052 Step = Step.abs();
7053
7054 // Check if Offset is more than full span of BitWidth. If it is, the
7055 // expression is guaranteed to overflow.
7056 if (APInt::getMaxValue(numBits: StartRange.getBitWidth()).udiv(RHS: Step).ult(RHS: MaxBECount))
7057 return {ConstantRange::getFull(BitWidth), false};
7058
7059 // Offset is by how much the expression can change. Checks above guarantee no
7060 // overflow here.
7061 APInt Offset = Step * MaxBECount;
7062
7063 // Minimum value of the final range will match the minimal value of StartRange
7064 // if the expression is increasing and will be decreased by Offset otherwise.
7065 // Maximum value of the final range will match the maximal value of StartRange
7066 // if the expression is decreasing and will be increased by Offset otherwise.
7067 APInt StartLower = StartRange.getLower();
7068 APInt StartUpper = StartRange.getUpper() - 1;
7069 bool Overflow;
7070 APInt MovedBoundary;
7071 if (Signed) {
7072 // This does not use sadd_ov, as we want to check overflow for a signed
7073 // start with an unsigned offset.
7074 if (Descending) {
7075 MovedBoundary = StartLower - std::move(Offset);
7076 Overflow = MovedBoundary.sgt(RHS: StartLower) || StartRange.isSignWrappedSet();
7077 } else {
7078 MovedBoundary = StartUpper + std::move(Offset);
7079 Overflow = MovedBoundary.slt(RHS: StartUpper) || StartRange.isSignWrappedSet();
7080 }
7081 } else {
7082 MovedBoundary = StartUpper.uadd_ov(RHS: std::move(Offset), Overflow);
7083 Overflow |= StartRange.isWrappedSet();
7084 }
7085
7086 // It's possible that the new minimum/maximum value will fall into the initial
7087 // range (due to wrap around). This means that the expression can take any
7088 // value in this bitwidth, and we have to return full range.
7089 if (StartRange.contains(Val: MovedBoundary))
7090 return {ConstantRange::getFull(BitWidth), false};
7091
7092 APInt NewLower =
7093 Descending ? std::move(MovedBoundary) : std::move(StartLower);
7094 APInt NewUpper =
7095 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
7096 NewUpper += 1;
7097
7098 // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
7099 return {ConstantRange::getNonEmpty(Lower: std::move(NewLower), Upper: std::move(NewUpper)),
7100 !Overflow};
7101}
7102
7103std::pair<ConstantRange, SCEV::NoWrapFlags>
7104ScalarEvolution::getRangeForAffineAR(const SCEV *Start, const SCEV *Step,
7105 const APInt &MaxBECount) {
7106 assert(getTypeSizeInBits(Start->getType()) ==
7107 getTypeSizeInBits(Step->getType()) &&
7108 getTypeSizeInBits(Start->getType()) == MaxBECount.getBitWidth() &&
7109 "mismatched bit widths");
7110
7111 // First, consider step signed.
7112 ConstantRange StartSRange = getSignedRange(S: Start);
7113 ConstantRange StepSRange = getSignedRange(S: Step);
7114
7115 // If Step can be both positive and negative, we need to find ranges for the
7116 // maximum absolute step values in both directions and union them.
7117 auto [SR1, NSW1] = getRangeForAffineARHelper(
7118 Step: StepSRange.getSignedMin(), StartRange: StartSRange, MaxBECount, /*Signed=*/true);
7119 auto [SR2, NSW2] = getRangeForAffineARHelper(Step: StepSRange.getSignedMax(),
7120 StartRange: StartSRange, MaxBECount,
7121 /*Signed=*/true);
7122 ConstantRange SR = SR1.unionWith(CR: SR2);
7123
7124 // Next, consider step unsigned.
7125 auto [UR, NUW] = getRangeForAffineARHelper(
7126 Step: getUnsignedRangeMax(S: Step), StartRange: getUnsignedRange(S: Start), MaxBECount,
7127 /*Signed=*/false);
7128
7129 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7130 if (NUW)
7131 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7132 if (NSW1 && NSW2)
7133 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7134
7135 // Finally, intersect signed and unsigned ranges.
7136 return {SR.intersectWith(CR: UR, Type: ConstantRange::Smallest), Flags};
7137}
7138
7139ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
7140 const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
7141 ScalarEvolution::RangeSignHint SignHint) {
7142 assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n");
7143 assert(AddRec->hasNoSelfWrap() &&
7144 "This only works for non-self-wrapping AddRecs!");
7145 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
7146 const SCEV *Step = AddRec->getStepRecurrence(SE&: *this);
7147 // Only deal with constant step to save compile time.
7148 if (!isa<SCEVConstant>(Val: Step))
7149 return ConstantRange::getFull(BitWidth);
7150 // Let's make sure that we can prove that we do not self-wrap during
7151 // MaxBECount iterations. We need this because MaxBECount is a maximum
7152 // iteration count estimate, and we might infer nw from some exit for which we
7153 // do not know max exit count (or any other side reasoning).
7154 // TODO: Turn into assert at some point.
7155 if (getTypeSizeInBits(Ty: MaxBECount->getType()) >
7156 getTypeSizeInBits(Ty: AddRec->getType()))
7157 return ConstantRange::getFull(BitWidth);
7158 MaxBECount = getNoopOrZeroExtend(V: MaxBECount, Ty: AddRec->getType());
7159 const SCEV *RangeWidth = getMinusOne(Ty: AddRec->getType());
7160 const SCEV *StepAbs = getUMinExpr(LHS: Step, RHS: getNegativeSCEV(V: Step));
7161 const SCEV *MaxItersWithoutWrap = getUDivExpr(LHS: RangeWidth, RHS: StepAbs);
7162 if (!isKnownPredicateViaConstantRanges(Pred: ICmpInst::ICMP_ULE, LHS: MaxBECount,
7163 RHS: MaxItersWithoutWrap))
7164 return ConstantRange::getFull(BitWidth);
7165
7166 ICmpInst::Predicate LEPred =
7167 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
7168 ICmpInst::Predicate GEPred =
7169 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
7170 const SCEV *End = AddRec->evaluateAtIteration(It: MaxBECount, SE&: *this);
7171
7172 // We know that there is no self-wrap. Let's take Start and End values and
7173 // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
7174 // the iteration. They either lie inside the range [Min(Start, End),
7175 // Max(Start, End)] or outside it:
7176 //
7177 // Case 1: RangeMin ... Start V1 ... VN End ... RangeMax;
7178 // Case 2: RangeMin Vk ... V1 Start ... End Vn ... Vk + 1 RangeMax;
7179 //
7180 // No self wrap flag guarantees that the intermediate values cannot be BOTH
7181 // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
7182 // knowledge, let's try to prove that we are dealing with Case 1. It is so if
7183 // Start <= End and step is positive, or Start >= End and step is negative.
7184 const SCEV *Start = applyLoopGuards(Expr: AddRec->getStart(), L: AddRec->getLoop());
7185 ConstantRange StartRange = getRangeRef(S: Start, SignHint);
7186 ConstantRange EndRange = getRangeRef(S: End, SignHint);
7187 ConstantRange RangeBetween = StartRange.unionWith(CR: EndRange);
7188 // If they already cover full iteration space, we will know nothing useful
7189 // even if we prove what we want to prove.
7190 if (RangeBetween.isFullSet())
7191 return RangeBetween;
7192 // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
7193 bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
7194 : RangeBetween.isWrappedSet();
7195 if (IsWrappedSet)
7196 return ConstantRange::getFull(BitWidth);
7197
7198 if (isKnownPositive(S: Step) &&
7199 isKnownPredicateViaConstantRanges(Pred: LEPred, LHS: Start, RHS: End))
7200 return RangeBetween;
7201 if (isKnownNegative(S: Step) &&
7202 isKnownPredicateViaConstantRanges(Pred: GEPred, LHS: Start, RHS: End))
7203 return RangeBetween;
7204 return ConstantRange::getFull(BitWidth);
7205}
7206
7207ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
7208 const SCEV *Step,
7209 const APInt &MaxBECount) {
7210 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
7211 // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
7212
7213 unsigned BitWidth = MaxBECount.getBitWidth();
7214 assert(getTypeSizeInBits(Start->getType()) == BitWidth &&
7215 getTypeSizeInBits(Step->getType()) == BitWidth &&
7216 "mismatched bit widths");
7217
7218 struct SelectPattern {
7219 Value *Condition = nullptr;
7220 APInt TrueValue;
7221 APInt FalseValue;
7222
7223 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
7224 const SCEV *S) {
7225 std::optional<unsigned> CastOp;
7226 APInt Offset(BitWidth, 0);
7227
7228 assert(SE.getTypeSizeInBits(S->getType()) == BitWidth &&
7229 "Should be!");
7230
7231 // Peel off a constant offset. In the future we could consider being
7232 // smarter here and handle {Start+Step,+,Step} too.
7233 const APInt *Off;
7234 if (match(S, P: m_scev_Add(Op0: m_scev_APInt(C&: Off), Op1: m_SCEV(V&: S))))
7235 Offset = *Off;
7236
7237 // Peel off a cast operation
7238 if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(Val: S)) {
7239 CastOp = SCast->getSCEVType();
7240 S = SCast->getOperand();
7241 }
7242
7243 using namespace llvm::PatternMatch;
7244
7245 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
7246 const APInt *TrueVal, *FalseVal;
7247 if (!SU ||
7248 !match(V: SU->getValue(), P: m_Select(C: m_Value(V&: Condition), L: m_APInt(Res&: TrueVal),
7249 R: m_APInt(Res&: FalseVal)))) {
7250 Condition = nullptr;
7251 return;
7252 }
7253
7254 TrueValue = *TrueVal;
7255 FalseValue = *FalseVal;
7256
7257 // Re-apply the cast we peeled off earlier
7258 if (CastOp)
7259 switch (*CastOp) {
7260 default:
7261 llvm_unreachable("Unknown SCEV cast type!");
7262
7263 case scTruncate:
7264 TrueValue = TrueValue.trunc(width: BitWidth);
7265 FalseValue = FalseValue.trunc(width: BitWidth);
7266 break;
7267 case scZeroExtend:
7268 TrueValue = TrueValue.zext(width: BitWidth);
7269 FalseValue = FalseValue.zext(width: BitWidth);
7270 break;
7271 case scSignExtend:
7272 TrueValue = TrueValue.sext(width: BitWidth);
7273 FalseValue = FalseValue.sext(width: BitWidth);
7274 break;
7275 }
7276
7277 // Re-apply the constant offset we peeled off earlier
7278 TrueValue += Offset;
7279 FalseValue += Offset;
7280 }
7281
7282 bool isRecognized() { return Condition != nullptr; }
7283 };
7284
7285 SelectPattern StartPattern(*this, BitWidth, Start);
7286 if (!StartPattern.isRecognized())
7287 return ConstantRange::getFull(BitWidth);
7288
7289 SelectPattern StepPattern(*this, BitWidth, Step);
7290 if (!StepPattern.isRecognized())
7291 return ConstantRange::getFull(BitWidth);
7292
7293 if (StartPattern.Condition != StepPattern.Condition) {
7294 // We don't handle this case today; but we could, by considering four
7295 // possibilities below instead of two. I'm not sure if there are cases where
7296 // that will help over what getRange already does, though.
7297 return ConstantRange::getFull(BitWidth);
7298 }
7299
7300 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
7301 // construct arbitrary general SCEV expressions here. This function is called
7302 // from deep in the call stack, and calling getSCEV (on a sext instruction,
7303 // say) can end up caching a suboptimal value.
7304
7305 // FIXME: without the explicit `this` receiver below, MSVC errors out with
7306 // C2352 and C2512 (otherwise it isn't needed).
7307
7308 const SCEV *TrueStart = this->getConstant(Val: StartPattern.TrueValue);
7309 const SCEV *TrueStep = this->getConstant(Val: StepPattern.TrueValue);
7310 const SCEV *FalseStart = this->getConstant(Val: StartPattern.FalseValue);
7311 const SCEV *FalseStep = this->getConstant(Val: StepPattern.FalseValue);
7312
7313 ConstantRange TrueRange =
7314 this->getRangeForAffineAR(Start: TrueStart, Step: TrueStep, MaxBECount).first;
7315 ConstantRange FalseRange =
7316 this->getRangeForAffineAR(Start: FalseStart, Step: FalseStep, MaxBECount).first;
7317
7318 return TrueRange.unionWith(CR: FalseRange);
7319}
7320
7321SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
7322 if (isa<ConstantExpr>(Val: V)) return SCEV::FlagAnyWrap;
7323 const BinaryOperator *BinOp = cast<BinaryOperator>(Val: V);
7324
7325 // Return early if there are no flags to propagate to the SCEV.
7326 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7327 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: BinOp);
7328 PDI && PDI->isDisjoint()) {
7329 Flags = ScalarEvolution::setFlags(Flags: SCEV::FlagNUW, OnFlags: SCEV::FlagNSW);
7330 } else {
7331 if (BinOp->hasNoUnsignedWrap())
7332 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7333 if (BinOp->hasNoSignedWrap())
7334 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7335 }
7336 if (Flags == SCEV::FlagAnyWrap)
7337 return SCEV::FlagAnyWrap;
7338
7339 return isSCEVExprNeverPoison(I: BinOp) ? Flags : SCEV::FlagAnyWrap;
7340}
7341
7342const Instruction *
7343ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
7344 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
7345 return &*AddRec->getLoop()->getHeader()->begin();
7346 if (auto *U = dyn_cast<SCEVUnknown>(Val: S))
7347 if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
7348 return I;
7349 return nullptr;
7350}
7351
7352const Instruction *ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops,
7353 bool &Precise) {
7354 Precise = true;
7355 // Do a bounded search of the def relation of the requested SCEVs.
7356 SmallPtrSet<const SCEV *, 16> Visited;
7357 SmallVector<SCEVUse> Worklist;
7358 auto pushOp = [&](const SCEV *S) {
7359 if (!Visited.insert(Ptr: S).second)
7360 return;
7361 // Threshold of 30 here is arbitrary.
7362 if (Visited.size() > 30) {
7363 Precise = false;
7364 return;
7365 }
7366 Worklist.push_back(Elt: S);
7367 };
7368
7369 for (SCEVUse S : Ops)
7370 pushOp(S);
7371
7372 const Instruction *Bound = nullptr;
7373 while (!Worklist.empty()) {
7374 SCEVUse S = Worklist.pop_back_val();
7375 if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
7376 if (!Bound || DT.dominates(Def: Bound, User: DefI))
7377 Bound = DefI;
7378 } else {
7379 for (SCEVUse Op : S->operands())
7380 pushOp(Op);
7381 }
7382 }
7383 return Bound ? Bound : &*F.getEntryBlock().begin();
7384}
7385
7386const Instruction *
7387ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops) {
7388 bool Discard;
7389 return getDefiningScopeBound(Ops, Precise&: Discard);
7390}
7391
7392bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
7393 const Instruction *B) {
7394 if (A->getParent() == B->getParent() &&
7395 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7396 End: B->getIterator()))
7397 return true;
7398
7399 auto *BLoop = LI.getLoopFor(BB: B->getParent());
7400 if (BLoop && BLoop->getHeader() == B->getParent() &&
7401 BLoop->getLoopPreheader() == A->getParent() &&
7402 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7403 End: A->getParent()->end()) &&
7404 isGuaranteedToTransferExecutionToSuccessor(Begin: B->getParent()->begin(),
7405 End: B->getIterator()))
7406 return true;
7407 return false;
7408}
7409
7410bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
7411 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
7412 visitAll(Root: Op, Visitor&: PC);
7413 return PC.MaybePoison.empty();
7414}
7415
7416bool ScalarEvolution::isGuaranteedNotToCauseUB(const SCEV *Op) {
7417 return !SCEVExprContains(Root: Op, Pred: [this](const SCEV *S) {
7418 const SCEV *Op1;
7419 bool M = match(S, P: m_scev_UDiv(Op0: m_SCEV(), Op1: m_SCEV(V&: Op1)));
7420 // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
7421 // is a non-zero constant, we have to assume the UDiv may be UB.
7422 return M && (!isKnownNonZero(S: Op1) || !isGuaranteedNotToBePoison(Op: Op1));
7423 });
7424}
7425
7426bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
7427 // Only proceed if we can prove that I does not yield poison.
7428 if (!programUndefinedIfPoison(Inst: I))
7429 return false;
7430
7431 // At this point we know that if I is executed, then it does not wrap
7432 // according to at least one of NSW or NUW. If I is not executed, then we do
7433 // not know if the calculation that I represents would wrap. Multiple
7434 // instructions can map to the same SCEV. If we apply NSW or NUW from I to
7435 // the SCEV, we must guarantee no wrapping for that SCEV also when it is
7436 // derived from other instructions that map to the same SCEV. We cannot make
7437 // that guarantee for cases where I is not executed. So we need to find a
7438 // upper bound on the defining scope for the SCEV, and prove that I is
7439 // executed every time we enter that scope. When the bounding scope is a
7440 // loop (the common case), this is equivalent to proving I executes on every
7441 // iteration of that loop.
7442 SmallVector<SCEVUse> SCEVOps;
7443 for (const Use &Op : I->operands()) {
7444 // I could be an extractvalue from a call to an overflow intrinsic.
7445 // TODO: We can do better here in some cases.
7446 if (isSCEVable(Ty: Op->getType()))
7447 SCEVOps.push_back(Elt: getSCEV(V: Op));
7448 }
7449 auto *DefI = getDefiningScopeBound(Ops: SCEVOps);
7450 return isGuaranteedToTransferExecutionTo(A: DefI, B: I);
7451}
7452
7453bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
7454 // If we know that \c I can never be poison period, then that's enough.
7455 if (isSCEVExprNeverPoison(I))
7456 return true;
7457
7458 // If the loop only has one exit, then we know that, if the loop is entered,
7459 // any instruction dominating that exit will be executed. If any such
7460 // instruction would result in UB, the addrec cannot be poison.
7461 //
7462 // This is basically the same reasoning as in isSCEVExprNeverPoison(), but
7463 // also handles uses outside the loop header (they just need to dominate the
7464 // single exit).
7465
7466 auto *ExitingBB = L->getExitingBlock();
7467 if (!ExitingBB || !loopHasNoAbnormalExits(L))
7468 return false;
7469
7470 SmallPtrSet<const Value *, 16> KnownPoison;
7471 SmallVector<const Instruction *, 8> Worklist;
7472
7473 // We start by assuming \c I, the post-inc add recurrence, is poison. Only
7474 // things that are known to be poison under that assumption go on the
7475 // Worklist.
7476 KnownPoison.insert(Ptr: I);
7477 Worklist.push_back(Elt: I);
7478
7479 while (!Worklist.empty()) {
7480 const Instruction *Poison = Worklist.pop_back_val();
7481
7482 for (const Use &U : Poison->uses()) {
7483 const Instruction *PoisonUser = cast<Instruction>(Val: U.getUser());
7484 if (mustTriggerUB(I: PoisonUser, KnownPoison) &&
7485 DT.dominates(A: PoisonUser->getParent(), B: ExitingBB))
7486 return true;
7487
7488 if (propagatesPoison(PoisonOp: U) && L->contains(Inst: PoisonUser))
7489 if (KnownPoison.insert(Ptr: PoisonUser).second)
7490 Worklist.push_back(Elt: PoisonUser);
7491 }
7492 }
7493
7494 return false;
7495}
7496
7497ScalarEvolution::LoopProperties
7498ScalarEvolution::getLoopProperties(const Loop *L) {
7499 using LoopProperties = ScalarEvolution::LoopProperties;
7500
7501 auto Itr = LoopPropertiesCache.find(Val: L);
7502 if (Itr == LoopPropertiesCache.end()) {
7503 auto HasSideEffects = [](Instruction *I) {
7504 if (auto *SI = dyn_cast<StoreInst>(Val: I))
7505 return !SI->isSimple();
7506
7507 if (I->mayThrow())
7508 return true;
7509
7510 // Non-volatile memset / memcpy do not count as side-effect for forward
7511 // progress.
7512 if (isa<MemIntrinsic>(Val: I) && !I->isVolatile())
7513 return false;
7514
7515 return I->mayWriteToMemory();
7516 };
7517
7518 LoopProperties LP = {/* HasNoAbnormalExits */ true,
7519 /*HasNoSideEffects*/ true};
7520
7521 for (auto *BB : L->getBlocks())
7522 for (auto &I : *BB) {
7523 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7524 LP.HasNoAbnormalExits = false;
7525 if (HasSideEffects(&I))
7526 LP.HasNoSideEffects = false;
7527 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7528 break; // We're already as pessimistic as we can get.
7529 }
7530
7531 auto InsertPair = LoopPropertiesCache.insert(KV: {L, LP});
7532 assert(InsertPair.second && "We just checked!");
7533 Itr = InsertPair.first;
7534 }
7535
7536 return Itr->second;
7537}
7538
7539bool ScalarEvolution::loopIsFiniteByAssumption(const Loop *L) {
7540 // A mustprogress loop without side effects must be finite.
7541 // TODO: The check used here is very conservative. It's only *specific*
7542 // side effects which are well defined in infinite loops.
7543 return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L));
7544}
7545
7546const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
7547 // Worklist item with a Value and a bool indicating whether all operands have
7548 // been visited already.
7549 using PointerTy = PointerIntPair<Value *, 1, bool>;
7550 SmallVector<PointerTy> Stack;
7551
7552 Stack.emplace_back(Args&: V, Args: false);
7553 while (!Stack.empty()) {
7554 auto E = Stack.back();
7555 Value *CurV = E.getPointer();
7556
7557 if (getExistingSCEV(V: CurV)) {
7558 Stack.pop_back();
7559 continue;
7560 }
7561
7562 SmallVector<Value *> Ops;
7563 const SCEV *CreatedSCEV = nullptr;
7564 // If all operands have been visited already, create the SCEV.
7565 if (E.getInt()) {
7566 CreatedSCEV = createSCEV(V: CurV);
7567 } else {
7568 // Otherwise get the operands we need to create SCEV's for before creating
7569 // the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
7570 // just use it.
7571 CreatedSCEV = getOperandsToCreate(V: CurV, Ops);
7572 }
7573
7574 if (CreatedSCEV) {
7575 insertValueToMap(V: CurV, S: CreatedSCEV);
7576 Stack.pop_back();
7577 } else {
7578 Stack.back().setInt(true);
7579 // Queue its operands which need to be constructed.
7580 for (Value *Op : Ops)
7581 Stack.emplace_back(Args&: Op, Args: false);
7582 }
7583 }
7584
7585 return getExistingSCEV(V);
7586}
7587
7588const SCEV *
7589ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
7590 if (!isSCEVable(Ty: V->getType()))
7591 return getUnknown(V);
7592
7593 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7594 // Don't attempt to analyze instructions in blocks that aren't
7595 // reachable. Such instructions don't matter, and they aren't required
7596 // to obey basic rules for definitions dominating uses which this
7597 // analysis depends on.
7598 if (!DT.isReachableFromEntry(A: I->getParent()))
7599 return getUnknown(V: PoisonValue::get(T: V->getType()));
7600 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7601 return getConstant(V: CI);
7602 else if (isa<GlobalAlias>(Val: V))
7603 return getUnknown(V);
7604 else if (!isa<ConstantExpr>(Val: V))
7605 return getUnknown(V);
7606
7607 Operator *U = cast<Operator>(Val: V);
7608 if (auto BO =
7609 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CxtI: dyn_cast<Instruction>(Val: V))) {
7610 bool IsConstArg = isa<ConstantInt>(Val: BO->RHS);
7611 switch (BO->Opcode) {
7612 case Instruction::Add:
7613 case Instruction::Mul: {
7614 // For additions and multiplications, traverse add/mul chains for which we
7615 // can potentially create a single SCEV, to reduce the number of
7616 // get{Add,Mul}Expr calls.
7617 do {
7618 if (BO->Op) {
7619 if (BO->Op != V && getExistingSCEV(V: BO->Op)) {
7620 Ops.push_back(Elt: BO->Op);
7621 break;
7622 }
7623 }
7624 Ops.push_back(Elt: BO->RHS);
7625 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7626 CxtI: dyn_cast<Instruction>(Val: V));
7627 if (!NewBO ||
7628 (BO->Opcode == Instruction::Add &&
7629 (NewBO->Opcode != Instruction::Add &&
7630 NewBO->Opcode != Instruction::Sub)) ||
7631 (BO->Opcode == Instruction::Mul &&
7632 NewBO->Opcode != Instruction::Mul)) {
7633 Ops.push_back(Elt: BO->LHS);
7634 break;
7635 }
7636 // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
7637 // requires a SCEV for the LHS.
7638 if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
7639 auto *I = dyn_cast<Instruction>(Val: BO->Op);
7640 if (I && programUndefinedIfPoison(Inst: I)) {
7641 Ops.push_back(Elt: BO->LHS);
7642 break;
7643 }
7644 }
7645 BO = NewBO;
7646 } while (true);
7647 return nullptr;
7648 }
7649 case Instruction::Sub:
7650 case Instruction::UDiv:
7651 case Instruction::URem:
7652 break;
7653 case Instruction::AShr:
7654 case Instruction::Shl:
7655 case Instruction::Xor:
7656 if (!IsConstArg)
7657 return nullptr;
7658 break;
7659 case Instruction::And:
7660 case Instruction::Or:
7661 if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(BitWidth: 1))
7662 return nullptr;
7663 break;
7664 case Instruction::LShr:
7665 return getUnknown(V);
7666 default:
7667 llvm_unreachable("Unhandled binop");
7668 break;
7669 }
7670
7671 Ops.push_back(Elt: BO->LHS);
7672 Ops.push_back(Elt: BO->RHS);
7673 return nullptr;
7674 }
7675
7676 switch (U->getOpcode()) {
7677 case Instruction::Trunc:
7678 case Instruction::ZExt:
7679 case Instruction::SExt:
7680 case Instruction::PtrToAddr:
7681 case Instruction::PtrToInt:
7682 Ops.push_back(Elt: U->getOperand(i: 0));
7683 return nullptr;
7684
7685 case Instruction::BitCast:
7686 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType())) {
7687 Ops.push_back(Elt: U->getOperand(i: 0));
7688 return nullptr;
7689 }
7690 return getUnknown(V);
7691
7692 case Instruction::SDiv:
7693 case Instruction::SRem:
7694 Ops.push_back(Elt: U->getOperand(i: 0));
7695 Ops.push_back(Elt: U->getOperand(i: 1));
7696 return nullptr;
7697
7698 case Instruction::GetElementPtr:
7699 assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() &&
7700 "GEP source element type must be sized");
7701 llvm::append_range(C&: Ops, R: U->operands());
7702 return nullptr;
7703
7704 case Instruction::IntToPtr:
7705 return getUnknown(V);
7706
7707 case Instruction::PHI:
7708 // getNodeForPHI has four ways to turn a PHI into a SCEV; retrieve the
7709 // relevant nodes for each of them.
7710 //
7711 // The first is just to call simplifyInstruction, and get something back
7712 // that isn't a PHI.
7713 if (Value *V = simplifyInstruction(
7714 I: cast<PHINode>(Val: U),
7715 Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
7716 /*UseInstrInfo=*/true, /*CanUseUndef=*/false})) {
7717 assert(V);
7718 Ops.push_back(Elt: V);
7719 return nullptr;
7720 }
7721 // The second is createNodeForPHIWithIdenticalOperands: this looks for
7722 // operands which all perform the same operation, but haven't been
7723 // CSE'ed for whatever reason.
7724 if (BinaryOperator *BO = getCommonInstForPHI(PN: cast<PHINode>(Val: U))) {
7725 assert(BO);
7726 Ops.push_back(Elt: BO);
7727 return nullptr;
7728 }
7729 // The third is createNodeFromSelectLikePHI; this takes a PHI which
7730 // is equivalent to a select, and analyzes it like a select.
7731 {
7732 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
7733 if (getOperandsForSelectLikePHI(DT, PN: cast<PHINode>(Val: U), Cond, LHS, RHS)) {
7734 assert(Cond);
7735 assert(LHS);
7736 assert(RHS);
7737 if (auto *CondICmp = dyn_cast<ICmpInst>(Val: Cond)) {
7738 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 0));
7739 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 1));
7740 }
7741 Ops.push_back(Elt: Cond);
7742 Ops.push_back(Elt: LHS);
7743 Ops.push_back(Elt: RHS);
7744 return nullptr;
7745 }
7746 }
7747 // The fourth way is createAddRecFromPHI. It's complicated to handle here,
7748 // so just construct it recursively.
7749 //
7750 // In addition to getNodeForPHI, also construct nodes which might be needed
7751 // by getRangeRef.
7752 if (RangeRefPHIAllowedOperands(DT, PHI: cast<PHINode>(Val: U))) {
7753 for (Value *V : cast<PHINode>(Val: U)->operands())
7754 Ops.push_back(Elt: V);
7755 return nullptr;
7756 }
7757 return nullptr;
7758
7759 case Instruction::Select: {
7760 // Check if U is a select that can be simplified to a SCEVUnknown.
7761 auto CanSimplifyToUnknown = [this, U]() {
7762 if (U->getType()->isIntegerTy(BitWidth: 1) || isa<ConstantInt>(Val: U->getOperand(i: 0)))
7763 return false;
7764
7765 auto *ICI = dyn_cast<ICmpInst>(Val: U->getOperand(i: 0));
7766 if (!ICI)
7767 return false;
7768 Value *LHS = ICI->getOperand(i_nocapture: 0);
7769 Value *RHS = ICI->getOperand(i_nocapture: 1);
7770 if (ICI->getPredicate() == CmpInst::ICMP_EQ ||
7771 ICI->getPredicate() == CmpInst::ICMP_NE) {
7772 if (!(isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()))
7773 return true;
7774 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
7775 getTypeSizeInBits(Ty: U->getType()))
7776 return true;
7777 return false;
7778 };
7779 if (CanSimplifyToUnknown())
7780 return getUnknown(V: U);
7781
7782 llvm::append_range(C&: Ops, R: U->operands());
7783 return nullptr;
7784 break;
7785 }
7786 case Instruction::Call:
7787 case Instruction::Invoke:
7788 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand()) {
7789 Ops.push_back(Elt: RV);
7790 return nullptr;
7791 }
7792
7793 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
7794 switch (II->getIntrinsicID()) {
7795 case Intrinsic::abs:
7796 Ops.push_back(Elt: II->getArgOperand(i: 0));
7797 return nullptr;
7798 case Intrinsic::umax:
7799 case Intrinsic::umin:
7800 case Intrinsic::smax:
7801 case Intrinsic::smin:
7802 case Intrinsic::usub_sat:
7803 case Intrinsic::uadd_sat:
7804 Ops.push_back(Elt: II->getArgOperand(i: 0));
7805 Ops.push_back(Elt: II->getArgOperand(i: 1));
7806 return nullptr;
7807 case Intrinsic::start_loop_iterations:
7808 case Intrinsic::annotation:
7809 case Intrinsic::ptr_annotation:
7810 Ops.push_back(Elt: II->getArgOperand(i: 0));
7811 return nullptr;
7812 default:
7813 break;
7814 }
7815 }
7816 break;
7817 }
7818
7819 return nullptr;
7820}
7821
7822const SCEV *ScalarEvolution::createSCEV(Value *V) {
7823 if (!isSCEVable(Ty: V->getType()))
7824 return getUnknown(V);
7825
7826 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7827 // Don't attempt to analyze instructions in blocks that aren't
7828 // reachable. Such instructions don't matter, and they aren't required
7829 // to obey basic rules for definitions dominating uses which this
7830 // analysis depends on.
7831 if (!DT.isReachableFromEntry(A: I->getParent()))
7832 return getUnknown(V: PoisonValue::get(T: V->getType()));
7833 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7834 return getConstant(V: CI);
7835 else if (isa<GlobalAlias>(Val: V))
7836 return getUnknown(V);
7837 else if (!isa<ConstantExpr>(Val: V))
7838 return getUnknown(V);
7839
7840 const SCEV *LHS;
7841 const SCEV *RHS;
7842
7843 Operator *U = cast<Operator>(Val: V);
7844 if (auto BO =
7845 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CxtI: dyn_cast<Instruction>(Val: V))) {
7846 switch (BO->Opcode) {
7847 case Instruction::Add: {
7848 // The simple thing to do would be to just call getSCEV on both operands
7849 // and call getAddExpr with the result. However if we're looking at a
7850 // bunch of things all added together, this can be quite inefficient,
7851 // because it leads to N-1 getAddExpr calls for N ultimate operands.
7852 // Instead, gather up all the operands and make a single getAddExpr call.
7853 // LLVM IR canonical form means we need only traverse the left operands.
7854 SmallVector<SCEVUse, 4> AddOps;
7855 do {
7856 if (BO->Op) {
7857 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
7858 AddOps.push_back(Elt: OpSCEV);
7859 break;
7860 }
7861
7862 // If a NUW or NSW flag can be applied to the SCEV for this
7863 // addition, then compute the SCEV for this addition by itself
7864 // with a separate call to getAddExpr. We need to do that
7865 // instead of pushing the operands of the addition onto AddOps,
7866 // since the flags are only known to apply to this particular
7867 // addition - they may not apply to other additions that can be
7868 // formed with operands from AddOps.
7869 const SCEV *RHS = getSCEV(V: BO->RHS);
7870 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
7871 if (Flags != SCEV::FlagAnyWrap) {
7872 const SCEV *LHS = getSCEV(V: BO->LHS);
7873 if (BO->Opcode == Instruction::Sub)
7874 AddOps.push_back(Elt: getMinusSCEV(LHS, RHS, Flags));
7875 else
7876 AddOps.push_back(Elt: getAddExpr(LHS, RHS, Flags));
7877 break;
7878 }
7879 }
7880
7881 if (BO->Opcode == Instruction::Sub)
7882 AddOps.push_back(Elt: getNegativeSCEV(V: getSCEV(V: BO->RHS)));
7883 else
7884 AddOps.push_back(Elt: getSCEV(V: BO->RHS));
7885
7886 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7887 CxtI: dyn_cast<Instruction>(Val: V));
7888 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
7889 NewBO->Opcode != Instruction::Sub)) {
7890 AddOps.push_back(Elt: getSCEV(V: BO->LHS));
7891 break;
7892 }
7893 BO = NewBO;
7894 } while (true);
7895
7896 return getAddExpr(Ops&: AddOps);
7897 }
7898
7899 case Instruction::Mul: {
7900 SmallVector<SCEVUse, 4> MulOps;
7901 do {
7902 if (BO->Op) {
7903 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
7904 MulOps.push_back(Elt: OpSCEV);
7905 break;
7906 }
7907
7908 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
7909 if (Flags != SCEV::FlagAnyWrap) {
7910 LHS = getSCEV(V: BO->LHS);
7911 RHS = getSCEV(V: BO->RHS);
7912 MulOps.push_back(Elt: getMulExpr(LHS, RHS, Flags));
7913 break;
7914 }
7915 }
7916
7917 MulOps.push_back(Elt: getSCEV(V: BO->RHS));
7918 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7919 CxtI: dyn_cast<Instruction>(Val: V));
7920 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
7921 MulOps.push_back(Elt: getSCEV(V: BO->LHS));
7922 break;
7923 }
7924 BO = NewBO;
7925 } while (true);
7926
7927 return getMulExpr(Ops&: MulOps);
7928 }
7929 case Instruction::UDiv:
7930 LHS = getSCEV(V: BO->LHS);
7931 RHS = getSCEV(V: BO->RHS);
7932 return getUDivExpr(LHS, RHS);
7933 case Instruction::URem:
7934 LHS = getSCEV(V: BO->LHS);
7935 RHS = getSCEV(V: BO->RHS);
7936 return getURemExpr(LHS, RHS);
7937 case Instruction::Sub: {
7938 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7939 if (BO->Op)
7940 Flags = getNoWrapFlagsFromUB(V: BO->Op);
7941
7942 // Try to use ptrtoaddr for subtracts with at least one ptrtoint
7943 // operand. While we don't model ptrtoint directly in SCEV, the
7944 // difference between two pointer addresses is well-defined.
7945 Value *PtrLHS = nullptr, *PtrRHS = nullptr;
7946 bool HasPtrLHS = match(V: BO->LHS, P: m_PtrToInt(Op: m_Value(V&: PtrLHS)));
7947 bool HasPtrRHS = match(V: BO->RHS, P: m_PtrToInt(Op: m_Value(V&: PtrRHS)));
7948 if (HasPtrLHS || HasPtrRHS) {
7949 // Convert a ptrtoint operand (OrigOp) to ptrtoaddr of its pointer
7950 // PtrOp. When only one side is ptrtoint (BothPtr is false), skip
7951 // SCEVUnknown pointers since wrapping them in ptrtoaddr adds no
7952 // useful structure.
7953 auto GetOp = [&](bool HasPtr, Value *PtrOp, Value *OrigOp,
7954 bool BothPtr) -> const SCEV * {
7955 if (!HasPtr)
7956 return getSCEV(V: OrigOp);
7957 const SCEV *PtrSCEV = getSCEV(V: PtrOp);
7958 if (BothPtr || !isa<SCEVUnknown>(Val: PtrSCEV)) {
7959 const SCEV *Addr = getPtrToAddrExpr(Op: PtrSCEV);
7960 if (!isa<SCEVCouldNotCompute>(Val: Addr) &&
7961 getTypeSizeInBits(Ty: OrigOp->getType()) <=
7962 getTypeSizeInBits(Ty: Addr->getType()))
7963 return getTruncateOrNoop(V: Addr, Ty: OrigOp->getType());
7964 }
7965 return getSCEV(V: OrigOp);
7966 };
7967 const SCEV *L = GetOp(HasPtrLHS, PtrLHS, BO->LHS, HasPtrRHS);
7968 const SCEV *R = GetOp(HasPtrRHS, PtrRHS, BO->RHS, HasPtrLHS);
7969 return getMinusSCEV(LHS: L, RHS: R, Flags);
7970 }
7971
7972 LHS = getSCEV(V: BO->LHS);
7973 RHS = getSCEV(V: BO->RHS);
7974 return getMinusSCEV(LHS, RHS, Flags);
7975 }
7976 case Instruction::And:
7977 // For an expression like x&255 that merely masks off the high bits,
7978 // use zext(trunc(x)) as the SCEV expression.
7979 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
7980 if (CI->isZero())
7981 return getSCEV(V: BO->RHS);
7982 if (CI->isMinusOne())
7983 return getSCEV(V: BO->LHS);
7984 const APInt &A = CI->getValue();
7985
7986 // Instcombine's ShrinkDemandedConstant may strip bits out of
7987 // constants, obscuring what would otherwise be a low-bits mask.
7988 // Use computeKnownBits to compute what ShrinkDemandedConstant
7989 // knew about to reconstruct a low-bits mask value.
7990 unsigned LZ = A.countl_zero();
7991 unsigned TZ = A.countr_zero();
7992 unsigned BitWidth = A.getBitWidth();
7993 KnownBits Known(BitWidth);
7994 computeKnownBits(V: BO->LHS, Known, DL: getDataLayout(), AC: &AC, CxtI: nullptr, DT: &DT);
7995
7996 APInt EffectiveMask =
7997 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - LZ - TZ).shl(shiftAmt: TZ);
7998 if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
7999 const SCEV *MulCount = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ));
8000 const SCEV *LHS = getSCEV(V: BO->LHS);
8001 const SCEV *ShiftedLHS = nullptr;
8002 if (auto *LHSMul = dyn_cast<SCEVMulExpr>(Val: LHS)) {
8003 if (auto *OpC = dyn_cast<SCEVConstant>(Val: LHSMul->getOperand(i: 0))) {
8004 // For an expression like (x * 8) & 8, simplify the multiply.
8005 unsigned MulZeros = OpC->getAPInt().countr_zero();
8006 unsigned GCD = std::min(a: MulZeros, b: TZ);
8007 APInt DivAmt = APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ - GCD);
8008 SmallVector<SCEVUse, 4> MulOps;
8009 MulOps.push_back(Elt: getConstant(Val: OpC->getAPInt().ashr(ShiftAmt: GCD)));
8010 append_range(C&: MulOps, R: LHSMul->operands().drop_front());
8011 auto *NewMul = getMulExpr(Ops&: MulOps, OrigFlags: LHSMul->getNoWrapFlags());
8012 ShiftedLHS = getUDivExpr(LHS: NewMul, RHS: getConstant(Val: DivAmt));
8013 }
8014 }
8015 if (!ShiftedLHS)
8016 ShiftedLHS = getUDivExpr(LHS, RHS: MulCount);
8017 return getMulExpr(
8018 LHS: getZeroExtendExpr(
8019 Op: getTruncateExpr(Op: ShiftedLHS,
8020 Ty: IntegerType::get(C&: getContext(), NumBits: BitWidth - LZ - TZ)),
8021 Ty: BO->LHS->getType()),
8022 RHS: MulCount);
8023 }
8024 }
8025 // Binary `and` is a bit-wise `umin`.
8026 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
8027 LHS = getSCEV(V: BO->LHS);
8028 RHS = getSCEV(V: BO->RHS);
8029 return getUMinExpr(LHS, RHS);
8030 }
8031 break;
8032
8033 case Instruction::Or:
8034 // Binary `or` is a bit-wise `umax`.
8035 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
8036 LHS = getSCEV(V: BO->LHS);
8037 RHS = getSCEV(V: BO->RHS);
8038 return getUMaxExpr(LHS, RHS);
8039 }
8040 break;
8041
8042 case Instruction::Xor:
8043 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8044 // If the RHS of xor is -1, then this is a not operation.
8045 if (CI->isMinusOne())
8046 return getNotSCEV(V: getSCEV(V: BO->LHS));
8047
8048 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
8049 // This is a variant of the check for xor with -1, and it handles
8050 // the case where instcombine has trimmed non-demanded bits out
8051 // of an xor with -1.
8052 if (auto *LBO = dyn_cast<BinaryOperator>(Val: BO->LHS))
8053 if (ConstantInt *LCI = dyn_cast<ConstantInt>(Val: LBO->getOperand(i_nocapture: 1)))
8054 if (LBO->getOpcode() == Instruction::And &&
8055 LCI->getValue() == CI->getValue())
8056 if (const SCEVZeroExtendExpr *Z =
8057 dyn_cast<SCEVZeroExtendExpr>(Val: getSCEV(V: BO->LHS))) {
8058 Type *UTy = BO->LHS->getType();
8059 const SCEV *Z0 = Z->getOperand();
8060 Type *Z0Ty = Z0->getType();
8061 unsigned Z0TySize = getTypeSizeInBits(Ty: Z0Ty);
8062
8063 // If C is a low-bits mask, the zero extend is serving to
8064 // mask off the high bits. Complement the operand and
8065 // re-apply the zext.
8066 if (CI->getValue().isMask(numBits: Z0TySize))
8067 return getZeroExtendExpr(Op: getNotSCEV(V: Z0), Ty: UTy);
8068
8069 // If C is a single bit, it may be in the sign-bit position
8070 // before the zero-extend. In this case, represent the xor
8071 // using an add, which is equivalent, and re-apply the zext.
8072 APInt Trunc = CI->getValue().trunc(width: Z0TySize);
8073 if (Trunc.zext(width: getTypeSizeInBits(Ty: UTy)) == CI->getValue() &&
8074 Trunc.isSignMask())
8075 return getZeroExtendExpr(Op: getAddExpr(LHS: Z0, RHS: getConstant(Val: Trunc)),
8076 Ty: UTy);
8077 }
8078 }
8079 break;
8080
8081 case Instruction::Shl:
8082 // Turn shift left of a constant amount into a multiply.
8083 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8084 uint32_t BitWidth = cast<IntegerType>(Val: SA->getType())->getBitWidth();
8085
8086 // If the shift count is not less than the bitwidth, the result of
8087 // the shift is undefined. Don't try to analyze it, because the
8088 // resolution chosen here may differ from the resolution chosen in
8089 // other parts of the compiler.
8090 if (SA->getValue().uge(RHS: BitWidth))
8091 break;
8092
8093 // We can safely preserve the nuw flag in all cases. It's also safe to
8094 // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
8095 // requires special handling. It can be preserved as long as we're not
8096 // left shifting by bitwidth - 1.
8097 auto Flags = SCEV::FlagAnyWrap;
8098 if (BO->Op) {
8099 auto MulFlags = getNoWrapFlagsFromUB(V: BO->Op);
8100 if (any(Val: MulFlags & SCEV::FlagNSW) &&
8101 (any(Val: MulFlags & SCEV::FlagNUW) ||
8102 SA->getValue().ult(RHS: BitWidth - 1)))
8103 Flags = Flags | SCEV::FlagNSW;
8104 if (any(Val: MulFlags & SCEV::FlagNUW))
8105 Flags = Flags | SCEV::FlagNUW;
8106 }
8107
8108 ConstantInt *X = ConstantInt::get(
8109 Context&: getContext(), V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
8110 return getMulExpr(LHS: getSCEV(V: BO->LHS), RHS: getConstant(V: X), Flags);
8111 }
8112 break;
8113
8114 case Instruction::AShr:
8115 // AShr X, C, where C is a constant.
8116 ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS);
8117 if (!CI)
8118 break;
8119
8120 Type *OuterTy = BO->LHS->getType();
8121 uint64_t BitWidth = getTypeSizeInBits(Ty: OuterTy);
8122 // If the shift count is not less than the bitwidth, the result of
8123 // the shift is undefined. Don't try to analyze it, because the
8124 // resolution chosen here may differ from the resolution chosen in
8125 // other parts of the compiler.
8126 if (CI->getValue().uge(RHS: BitWidth))
8127 break;
8128
8129 if (CI->isZero())
8130 return getSCEV(V: BO->LHS); // shift by zero --> noop
8131
8132 uint64_t AShrAmt = CI->getZExtValue();
8133 Type *TruncTy = IntegerType::get(C&: getContext(), NumBits: BitWidth - AShrAmt);
8134
8135 Operator *L = dyn_cast<Operator>(Val: BO->LHS);
8136 const SCEV *AddTruncateExpr = nullptr;
8137 ConstantInt *ShlAmtCI = nullptr;
8138 const SCEV *AddConstant = nullptr;
8139
8140 if (L && L->getOpcode() == Instruction::Add) {
8141 // X = Shl A, n
8142 // Y = Add X, c
8143 // Z = AShr Y, m
8144 // n, c and m are constants.
8145
8146 Operator *LShift = dyn_cast<Operator>(Val: L->getOperand(i: 0));
8147 ConstantInt *AddOperandCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8148 if (LShift && LShift->getOpcode() == Instruction::Shl) {
8149 if (AddOperandCI) {
8150 const SCEV *ShlOp0SCEV = getSCEV(V: LShift->getOperand(i: 0));
8151 ShlAmtCI = dyn_cast<ConstantInt>(Val: LShift->getOperand(i: 1));
8152 // since we truncate to TruncTy, the AddConstant should be of the
8153 // same type, so create a new Constant with type same as TruncTy.
8154 // Also, the Add constant should be shifted right by AShr amount.
8155 APInt AddOperand = AddOperandCI->getValue().ashr(ShiftAmt: AShrAmt);
8156 AddConstant = getConstant(Val: AddOperand.trunc(width: BitWidth - AShrAmt));
8157 // we model the expression as sext(add(trunc(A), c << n)), since the
8158 // sext(trunc) part is already handled below, we create a
8159 // AddExpr(TruncExp) which will be used later.
8160 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8161 }
8162 }
8163 } else if (L && L->getOpcode() == Instruction::Shl) {
8164 // X = Shl A, n
8165 // Y = AShr X, m
8166 // Both n and m are constant.
8167
8168 const SCEV *ShlOp0SCEV = getSCEV(V: L->getOperand(i: 0));
8169 ShlAmtCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8170 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8171 }
8172
8173 if (AddTruncateExpr && ShlAmtCI) {
8174 // We can merge the two given cases into a single SCEV statement,
8175 // incase n = m, the mul expression will be 2^0, so it gets resolved to
8176 // a simpler case. The following code handles the two cases:
8177 //
8178 // 1) For a two-shift sext-inreg, i.e. n = m,
8179 // use sext(trunc(x)) as the SCEV expression.
8180 //
8181 // 2) When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
8182 // expression. We already checked that ShlAmt < BitWidth, so
8183 // the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
8184 // ShlAmt - AShrAmt < Amt.
8185 const APInt &ShlAmt = ShlAmtCI->getValue();
8186 if (ShlAmt.ult(RHS: BitWidth) && ShlAmt.uge(RHS: AShrAmt)) {
8187 APInt Mul = APInt::getOneBitSet(numBits: BitWidth - AShrAmt,
8188 BitNo: ShlAmtCI->getZExtValue() - AShrAmt);
8189 const SCEV *CompositeExpr =
8190 getMulExpr(LHS: AddTruncateExpr, RHS: getConstant(Val: Mul));
8191 if (L->getOpcode() != Instruction::Shl)
8192 CompositeExpr = getAddExpr(LHS: CompositeExpr, RHS: AddConstant);
8193
8194 return getSignExtendExpr(Op: CompositeExpr, Ty: OuterTy);
8195 }
8196 }
8197 break;
8198 }
8199 }
8200
8201 switch (U->getOpcode()) {
8202 case Instruction::Trunc:
8203 return getTruncateExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8204
8205 case Instruction::ZExt:
8206 return getZeroExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8207
8208 case Instruction::SExt:
8209 if (auto BO = MatchBinaryOp(V: U->getOperand(i: 0), DL: getDataLayout(), AC, DT,
8210 CxtI: dyn_cast<Instruction>(Val: V))) {
8211 // The NSW flag of a subtract does not always survive the conversion to
8212 // A + (-1)*B. By pushing sign extension onto its operands we are much
8213 // more likely to preserve NSW and allow later AddRec optimisations.
8214 //
8215 // NOTE: This is effectively duplicating this logic from getSignExtend:
8216 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
8217 // but by that point the NSW information has potentially been lost.
8218 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
8219 Type *Ty = U->getType();
8220 auto *V1 = getSignExtendExpr(Op: getSCEV(V: BO->LHS), Ty);
8221 auto *V2 = getSignExtendExpr(Op: getSCEV(V: BO->RHS), Ty);
8222 return getMinusSCEV(LHS: V1, RHS: V2, Flags: SCEV::FlagNSW);
8223 }
8224 }
8225 return getSignExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8226
8227 case Instruction::BitCast:
8228 // BitCasts are no-op casts so we just eliminate the cast.
8229 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType()))
8230 return getSCEV(V: U->getOperand(i: 0));
8231 break;
8232
8233 case Instruction::PtrToAddr: {
8234 const SCEV *IntOp = getPtrToAddrExpr(Op: getSCEV(V: U->getOperand(i: 0)));
8235 if (isa<SCEVCouldNotCompute>(Val: IntOp))
8236 return getUnknown(V);
8237 return IntOp;
8238 }
8239
8240 case Instruction::PtrToInt:
8241 // SCEV only models ptrtoaddr.
8242 return getUnknown(V);
8243
8244 case Instruction::IntToPtr:
8245 // Just don't deal with inttoptr casts.
8246 return getUnknown(V);
8247
8248 case Instruction::SDiv:
8249 // If both operands are non-negative, this is just an udiv.
8250 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8251 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8252 return getUDivExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8253 break;
8254
8255 case Instruction::SRem:
8256 // If both operands are non-negative, this is just an urem.
8257 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8258 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8259 return getURemExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8260 break;
8261
8262 case Instruction::GetElementPtr:
8263 return createNodeForGEP(GEP: cast<GEPOperator>(Val: U));
8264
8265 case Instruction::PHI:
8266 return createNodeForPHI(PN: cast<PHINode>(Val: U));
8267
8268 case Instruction::Select:
8269 return createNodeForSelectOrPHI(V: U, Cond: U->getOperand(i: 0), TrueVal: U->getOperand(i: 1),
8270 FalseVal: U->getOperand(i: 2));
8271
8272 case Instruction::Call:
8273 case Instruction::Invoke:
8274 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand())
8275 return getSCEV(V: RV);
8276
8277 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
8278 switch (II->getIntrinsicID()) {
8279 case Intrinsic::abs:
8280 return getAbsExpr(
8281 Op: getSCEV(V: II->getArgOperand(i: 0)),
8282 /*IsNSW=*/cast<ConstantInt>(Val: II->getArgOperand(i: 1))->isOne());
8283 case Intrinsic::umax:
8284 LHS = getSCEV(V: II->getArgOperand(i: 0));
8285 RHS = getSCEV(V: II->getArgOperand(i: 1));
8286 return getUMaxExpr(LHS, RHS);
8287 case Intrinsic::umin:
8288 LHS = getSCEV(V: II->getArgOperand(i: 0));
8289 RHS = getSCEV(V: II->getArgOperand(i: 1));
8290 return getUMinExpr(LHS, RHS);
8291 case Intrinsic::smax:
8292 LHS = getSCEV(V: II->getArgOperand(i: 0));
8293 RHS = getSCEV(V: II->getArgOperand(i: 1));
8294 return getSMaxExpr(LHS, RHS);
8295 case Intrinsic::smin:
8296 LHS = getSCEV(V: II->getArgOperand(i: 0));
8297 RHS = getSCEV(V: II->getArgOperand(i: 1));
8298 return getSMinExpr(LHS, RHS);
8299 case Intrinsic::usub_sat: {
8300 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8301 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8302 const SCEV *ClampedY = getUMinExpr(LHS: X, RHS: Y);
8303 return getMinusSCEV(LHS: X, RHS: ClampedY, Flags: SCEV::FlagNUW);
8304 }
8305 case Intrinsic::uadd_sat: {
8306 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8307 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8308 const SCEV *ClampedX = getUMinExpr(LHS: X, RHS: getNotSCEV(V: Y));
8309 return getAddExpr(LHS: ClampedX, RHS: Y, Flags: SCEV::FlagNUW);
8310 }
8311 case Intrinsic::start_loop_iterations:
8312 case Intrinsic::annotation:
8313 case Intrinsic::ptr_annotation:
8314 // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is
8315 // just eqivalent to the first operand for SCEV purposes.
8316 return getSCEV(V: II->getArgOperand(i: 0));
8317 case Intrinsic::vscale:
8318 return getVScale(Ty: II->getType());
8319 default:
8320 break;
8321 }
8322 }
8323 break;
8324 }
8325
8326 return getUnknown(V);
8327}
8328
8329//===----------------------------------------------------------------------===//
8330// Iteration Count Computation Code
8331//
8332
8333const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount) {
8334 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8335 return getCouldNotCompute();
8336
8337 auto *ExitCountType = ExitCount->getType();
8338 assert(ExitCountType->isIntegerTy());
8339 auto *EvalTy = Type::getIntNTy(C&: ExitCountType->getContext(),
8340 N: 1 + ExitCountType->getScalarSizeInBits());
8341 return getTripCountFromExitCount(ExitCount, EvalTy, L: nullptr);
8342}
8343
8344const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
8345 Type *EvalTy,
8346 const Loop *L) {
8347 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8348 return getCouldNotCompute();
8349
8350 unsigned ExitCountSize = getTypeSizeInBits(Ty: ExitCount->getType());
8351 unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
8352
8353 auto CanAddOneWithoutOverflow = [&]() {
8354 ConstantRange ExitCountRange =
8355 getRangeRef(S: ExitCount, SignHint: RangeSignHint::HINT_RANGE_UNSIGNED);
8356 if (!ExitCountRange.contains(Val: APInt::getMaxValue(numBits: ExitCountSize)))
8357 return true;
8358
8359 return L && isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: ExitCount,
8360 RHS: getMinusOne(Ty: ExitCount->getType()));
8361 };
8362
8363 // If we need to zero extend the backedge count, check if we can add one to
8364 // it prior to zero extending without overflow. Provided this is safe, it
8365 // allows better simplification of the +1.
8366 if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
8367 return getZeroExtendExpr(
8368 Op: getAddExpr(LHS: ExitCount, RHS: getOne(Ty: ExitCount->getType())), Ty: EvalTy);
8369
8370 // Get the total trip count from the count by adding 1. This may wrap.
8371 return getAddExpr(LHS: getTruncateOrZeroExtend(V: ExitCount, Ty: EvalTy), RHS: getOne(Ty: EvalTy));
8372}
8373
8374static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
8375 if (!ExitCount)
8376 return 0;
8377
8378 ConstantInt *ExitConst = ExitCount->getValue();
8379
8380 // Guard against huge trip counts.
8381 if (ExitConst->getValue().getActiveBits() > 32)
8382 return 0;
8383
8384 // In case of integer overflow, this returns 0, which is correct.
8385 return ((unsigned)ExitConst->getZExtValue()) + 1;
8386}
8387
8388unsigned ScalarEvolution::getSmallConstantTripCount(const Loop *L) {
8389 auto *ExitCount = dyn_cast<SCEVConstant>(Val: getBackedgeTakenCount(L, Kind: Exact));
8390 return getConstantTripCount(ExitCount);
8391}
8392
8393unsigned
8394ScalarEvolution::getSmallConstantTripCount(const Loop *L,
8395 const BasicBlock *ExitingBlock) {
8396 assert(ExitingBlock && "Must pass a non-null exiting block!");
8397 assert(L->isLoopExiting(ExitingBlock) &&
8398 "Exiting block must actually branch out of the loop!");
8399 const SCEVConstant *ExitCount =
8400 dyn_cast<SCEVConstant>(Val: getExitCount(L, ExitingBlock));
8401 return getConstantTripCount(ExitCount);
8402}
8403
8404unsigned ScalarEvolution::getSmallConstantMaxTripCount(
8405 const Loop *L, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8406
8407 const auto *MaxExitCount =
8408 Predicates ? getPredicatedConstantMaxBackedgeTakenCount(L, Predicates&: *Predicates)
8409 : getConstantMaxBackedgeTakenCount(L);
8410 return getConstantTripCount(ExitCount: dyn_cast<SCEVConstant>(Val: MaxExitCount));
8411}
8412
8413unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L) {
8414 SmallVector<BasicBlock *, 8> ExitingBlocks;
8415 L->getExitingBlocks(ExitingBlocks);
8416
8417 std::optional<unsigned> Res;
8418 for (auto *ExitingBB : ExitingBlocks) {
8419 unsigned Multiple = getSmallConstantTripMultiple(L, ExitingBlock: ExitingBB);
8420 if (!Res)
8421 Res = Multiple;
8422 Res = std::gcd(m: *Res, n: Multiple);
8423 }
8424 return Res.value_or(u: 1);
8425}
8426
8427unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8428 const SCEV *ExitCount) {
8429 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8430 return 1;
8431
8432 // Get the trip count
8433 const SCEV *TCExpr = getTripCountFromExitCount(ExitCount: applyLoopGuards(Expr: ExitCount, L));
8434
8435 APInt Multiple = getNonZeroConstantMultiple(S: TCExpr);
8436 // If a trip multiple is huge (>=2^32), the trip count is still divisible by
8437 // the greatest power of 2 divisor less than 2^32.
8438 return Multiple.getActiveBits() > 32
8439 ? 1U << std::min(a: 31U, b: Multiple.countTrailingZeros())
8440 : (unsigned)Multiple.getZExtValue();
8441}
8442
8443/// Returns the largest constant divisor of the trip count of this loop as a
8444/// normal unsigned value, if possible. This means that the actual trip count is
8445/// always a multiple of the returned value (don't forget the trip count could
8446/// very well be zero as well!).
8447///
8448/// Returns 1 if the trip count is unknown or not guaranteed to be the
8449/// multiple of a constant (which is also the case if the trip count is simply
8450/// constant, use getSmallConstantTripCount for that case), Will also return 1
8451/// if the trip count is very large (>= 2^32).
8452///
8453/// As explained in the comments for getSmallConstantTripCount, this assumes
8454/// that control exits the loop via ExitingBlock.
8455unsigned
8456ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8457 const BasicBlock *ExitingBlock) {
8458 assert(ExitingBlock && "Must pass a non-null exiting block!");
8459 assert(L->isLoopExiting(ExitingBlock) &&
8460 "Exiting block must actually branch out of the loop!");
8461 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
8462 return getSmallConstantTripMultiple(L, ExitCount);
8463}
8464
8465const SCEV *ScalarEvolution::getExitCount(const Loop *L,
8466 const BasicBlock *ExitingBlock,
8467 ExitCountKind Kind) {
8468 switch (Kind) {
8469 case Exact:
8470 return getBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this);
8471 case SymbolicMaximum:
8472 return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this);
8473 case ConstantMaximum:
8474 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this);
8475 };
8476 llvm_unreachable("Invalid ExitCountKind!");
8477}
8478
8479const SCEV *ScalarEvolution::getPredicatedExitCount(
8480 const Loop *L, const BasicBlock *ExitingBlock,
8481 SmallVectorImpl<const SCEVPredicate *> *Predicates, ExitCountKind Kind) {
8482 switch (Kind) {
8483 case Exact:
8484 return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this,
8485 Predicates);
8486 case SymbolicMaximum:
8487 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this,
8488 Predicates);
8489 case ConstantMaximum:
8490 return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this,
8491 Predicates);
8492 };
8493 llvm_unreachable("Invalid ExitCountKind!");
8494}
8495
8496const SCEV *ScalarEvolution::getPredicatedBackedgeTakenCount(
8497 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8498 return getPredicatedBackedgeTakenInfo(L).getExact(L, SE: this, Predicates: &Preds);
8499}
8500
8501const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
8502 ExitCountKind Kind) {
8503 switch (Kind) {
8504 case Exact:
8505 return getBackedgeTakenInfo(L).getExact(L, SE: this);
8506 case ConstantMaximum:
8507 return getBackedgeTakenInfo(L).getConstantMax(SE: this);
8508 case SymbolicMaximum:
8509 return getBackedgeTakenInfo(L).getSymbolicMax(L, SE: this);
8510 };
8511 llvm_unreachable("Invalid ExitCountKind!");
8512}
8513
8514const SCEV *ScalarEvolution::getPredicatedSymbolicMaxBackedgeTakenCount(
8515 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8516 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, SE: this, Predicates: &Preds);
8517}
8518
8519const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
8520 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8521 return getPredicatedBackedgeTakenInfo(L).getConstantMax(SE: this, Predicates: &Preds);
8522}
8523
8524bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
8525 return getBackedgeTakenInfo(L).isConstantMaxOrZero(SE: this);
8526}
8527
8528/// Push PHI nodes in the header of the given loop onto the given Worklist.
8529static void PushLoopPHIs(const Loop *L,
8530 SmallVectorImpl<Instruction *> &Worklist,
8531 SmallPtrSetImpl<Instruction *> &Visited) {
8532 BasicBlock *Header = L->getHeader();
8533
8534 // Push all Loop-header PHIs onto the Worklist stack.
8535 for (PHINode &PN : Header->phis())
8536 if (Visited.insert(Ptr: &PN).second)
8537 Worklist.push_back(Elt: &PN);
8538}
8539
8540ScalarEvolution::BackedgeTakenInfo &
8541ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
8542 auto &BTI = getBackedgeTakenInfo(L);
8543 if (BTI.hasFullInfo())
8544 return BTI;
8545
8546 auto Pair = PredicatedBackedgeTakenCounts.try_emplace(Key: L);
8547
8548 if (!Pair.second)
8549 return Pair.first->second;
8550
8551 BackedgeTakenInfo Result =
8552 computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
8553
8554 return PredicatedBackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8555}
8556
8557ScalarEvolution::BackedgeTakenInfo &
8558ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
8559 // Initially insert an invalid entry for this loop. If the insertion
8560 // succeeds, proceed to actually compute a backedge-taken count and
8561 // update the value. The temporary CouldNotCompute value tells SCEV
8562 // code elsewhere that it shouldn't attempt to request a new
8563 // backedge-taken count, which could result in infinite recursion.
8564 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
8565 BackedgeTakenCounts.try_emplace(Key: L);
8566 if (!Pair.second)
8567 return Pair.first->second;
8568
8569 // computeBackedgeTakenCount may allocate memory for its result. Inserting it
8570 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
8571 // must be cleared in this scope.
8572 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
8573
8574 // Now that we know more about the trip count for this loop, forget any
8575 // existing SCEV values for PHI nodes in this loop since they are only
8576 // conservative estimates made without the benefit of trip count
8577 // information. This invalidation is not necessary for correctness, and is
8578 // only done to produce more precise results.
8579 if (Result.hasAnyInfo()) {
8580 // Invalidate any expression using an addrec in this loop.
8581 SmallVector<SCEVUse, 8> ToForget;
8582 auto LoopUsersIt = LoopUsers.find(Val: L);
8583 if (LoopUsersIt != LoopUsers.end())
8584 append_range(C&: ToForget, R&: LoopUsersIt->second);
8585 forgetMemoizedResults(SCEVs: ToForget);
8586
8587 // Invalidate constant-evolved loop header phis.
8588 for (PHINode &PN : L->getHeader()->phis())
8589 ConstantEvolutionLoopExitValue.erase(Val: &PN);
8590 }
8591
8592 // Re-lookup the insert position, since the call to
8593 // computeBackedgeTakenCount above could result in a
8594 // recusive call to getBackedgeTakenInfo (on a different
8595 // loop), which would invalidate the iterator computed
8596 // earlier.
8597 return BackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8598}
8599
8600void ScalarEvolution::forgetAllLoops() {
8601 // This method is intended to forget all info about loops. It should
8602 // invalidate caches as if the following happened:
8603 // - The trip counts of all loops have changed arbitrarily
8604 // - Every llvm::Value has been updated in place to produce a different
8605 // result.
8606 BackedgeTakenCounts.clear();
8607 PredicatedBackedgeTakenCounts.clear();
8608 BECountUsers.clear();
8609 LoopPropertiesCache.clear();
8610 ConstantEvolutionLoopExitValue.clear();
8611 ValueExprMap.clear();
8612 ValuesAtScopes.clear();
8613 ValuesAtScopesUsers.clear();
8614 LoopDispositions.clear();
8615 BlockDispositions.clear();
8616 UnsignedRanges.clear();
8617 SignedRanges.clear();
8618 ExprValueMap.clear();
8619 HasRecMap.clear();
8620 ConstantMultipleCache.clear();
8621 PredicatedSCEVRewrites.clear();
8622 FoldCache.clear();
8623 FoldCacheUser.clear();
8624}
8625void ScalarEvolution::visitAndClearUsers(
8626 SmallVectorImpl<Instruction *> &Worklist,
8627 SmallPtrSetImpl<Instruction *> &Visited,
8628 SmallVectorImpl<SCEVUse> &ToForget) {
8629 while (!Worklist.empty()) {
8630 Instruction *I = Worklist.pop_back_val();
8631 if (!isSCEVable(Ty: I->getType()) && !isa<WithOverflowInst>(Val: I))
8632 continue;
8633
8634 ValueExprMapType::iterator It =
8635 ValueExprMap.find_as(Val: static_cast<Value *>(I));
8636 if (It != ValueExprMap.end()) {
8637 ToForget.push_back(Elt: It->second);
8638 eraseValueFromMap(V: It->first);
8639 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
8640 ConstantEvolutionLoopExitValue.erase(Val: PN);
8641 }
8642
8643 PushDefUseChildren(I, Worklist, Visited);
8644 }
8645}
8646
8647void ScalarEvolution::forgetLoop(const Loop *L) {
8648 SmallVector<const Loop *, 16> LoopWorklist(1, L);
8649 SmallVector<Instruction *, 32> Worklist;
8650 SmallPtrSet<Instruction *, 16> Visited;
8651 SmallVector<SCEVUse, 16> ToForget;
8652
8653 // Iterate over all the loops and sub-loops to drop SCEV information.
8654 while (!LoopWorklist.empty()) {
8655 auto *CurrL = LoopWorklist.pop_back_val();
8656
8657 // Drop any stored trip count value.
8658 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ false);
8659 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ true);
8660
8661 // Drop information about predicated SCEV rewrites for this loop.
8662 PredicatedSCEVRewrites.remove_if(
8663 Pred: [&](const auto &Entry) { return Entry.first.second == CurrL; });
8664
8665 auto LoopUsersItr = LoopUsers.find(Val: CurrL);
8666 if (LoopUsersItr != LoopUsers.end())
8667 llvm::append_range(C&: ToForget, R&: LoopUsersItr->second);
8668
8669 // Drop information about expressions based on loop-header PHIs.
8670 PushLoopPHIs(L: CurrL, Worklist, Visited);
8671 visitAndClearUsers(Worklist, Visited, ToForget);
8672
8673 LoopPropertiesCache.erase(Val: CurrL);
8674 // Forget all contained loops too, to avoid dangling entries in the
8675 // ValuesAtScopes map.
8676 LoopWorklist.append(in_start: CurrL->begin(), in_end: CurrL->end());
8677 }
8678 forgetMemoizedResults(SCEVs: ToForget);
8679}
8680
8681void ScalarEvolution::forgetTopmostLoop(const Loop *L) {
8682 forgetLoop(L: L->getOutermostLoop());
8683}
8684
8685void ScalarEvolution::forgetValue(Value *V) {
8686 Instruction *I = dyn_cast<Instruction>(Val: V);
8687 if (!I) return;
8688
8689 // Drop information about expressions based on loop-header PHIs.
8690 SmallVector<Instruction *, 16> Worklist;
8691 SmallPtrSet<Instruction *, 8> Visited;
8692 SmallVector<SCEVUse, 8> ToForget;
8693 Worklist.push_back(Elt: I);
8694 Visited.insert(Ptr: I);
8695 visitAndClearUsers(Worklist, Visited, ToForget);
8696
8697 forgetMemoizedResults(SCEVs: ToForget);
8698}
8699
8700void ScalarEvolution::forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V) {
8701 // If SCEV looked through a trivial LCSSA phi node, we might have SCEV's
8702 // directly using a SCEVUnknown/SCEVAddRec defined in the loop. After an
8703 // extra predecessor is added, this is no longer valid. Find all Unknowns and
8704 // AddRecs defined in the loop and invalidate any SCEV's making use of them.
8705 auto InvalidateValue = [&](Value *Val) {
8706 if (!isSCEVable(Ty: Val->getType()))
8707 return;
8708 if (const SCEV *S = getExistingSCEV(V: Val)) {
8709 struct InvalidationRootCollector {
8710 Loop *L;
8711 SmallVector<SCEVUse, 8> Roots;
8712
8713 InvalidationRootCollector(Loop *L) : L(L) {}
8714
8715 bool follow(const SCEV *S) {
8716 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
8717 if (auto *I = dyn_cast<Instruction>(Val: SU->getValue()))
8718 if (L->contains(Inst: I))
8719 Roots.push_back(Elt: S);
8720 } else if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S)) {
8721 if (L->contains(L: AddRec->getLoop()))
8722 Roots.push_back(Elt: S);
8723 }
8724 return true;
8725 }
8726 bool isDone() const { return false; }
8727 };
8728
8729 InvalidationRootCollector C(L);
8730 visitAll(Root: S, Visitor&: C);
8731 forgetMemoizedResults(SCEVs: C.Roots);
8732 }
8733 };
8734
8735 InvalidateValue(V);
8736
8737 // If V has a non-SCEV-able type (e.g. {i64, i1} from a with.overflow
8738 // intrinsic), its users (e.g. extractvalue) may have stale SCEV
8739 // expressions referencing loop-internal values.
8740 if (!isSCEVable(Ty: V->getType()) && any_of(Range: V->incoming_values(), P: [](Value *Inc) {
8741 return isa<WithOverflowInst>(Val: Inc);
8742 }))
8743 for (User *U : V->users())
8744 InvalidateValue(U);
8745 // Also perform the normal invalidation.
8746 forgetValue(V);
8747}
8748
8749void ScalarEvolution::forgetLoopDispositions() { LoopDispositions.clear(); }
8750
8751void ScalarEvolution::forgetBlockAndLoopDispositions(Value *V) {
8752 // Unless a specific value is passed to invalidation, completely clear both
8753 // caches.
8754 if (!V) {
8755 BlockDispositions.clear();
8756 LoopDispositions.clear();
8757 return;
8758 }
8759
8760 if (!isSCEVable(Ty: V->getType()))
8761 return;
8762
8763 const SCEV *S = getExistingSCEV(V);
8764 if (!S)
8765 return;
8766
8767 // Invalidate the block and loop dispositions cached for S. Dispositions of
8768 // S's users may change if S's disposition changes (i.e. a user may change to
8769 // loop-invariant, if S changes to loop invariant), so also invalidate
8770 // dispositions of S's users recursively.
8771 SmallVector<SCEVUse, 8> Worklist = {S};
8772 SmallPtrSet<const SCEV *, 8> Seen = {S};
8773 while (!Worklist.empty()) {
8774 const SCEV *Curr = Worklist.pop_back_val();
8775 bool LoopDispoRemoved = LoopDispositions.erase(Val: Curr);
8776 bool BlockDispoRemoved = BlockDispositions.erase(Val: Curr);
8777 if (!LoopDispoRemoved && !BlockDispoRemoved)
8778 continue;
8779 auto Users = SCEVUsers.find(Val: Curr);
8780 if (Users != SCEVUsers.end())
8781 for (const auto *User : Users->second)
8782 if (Seen.insert(Ptr: User).second)
8783 Worklist.push_back(Elt: User);
8784 }
8785}
8786
8787/// Get the exact loop backedge taken count considering all loop exits. A
8788/// computable result can only be returned for loops with all exiting blocks
8789/// dominating the latch. howFarToZero assumes that the limit of each loop test
8790/// is never skipped. This is a valid assumption as long as the loop exits via
8791/// that test. For precise results, it is the caller's responsibility to specify
8792/// the relevant loop exiting block using getExact(ExitingBlock, SE).
8793const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
8794 const Loop *L, ScalarEvolution *SE,
8795 SmallVectorImpl<const SCEVPredicate *> *Preds) const {
8796 // If any exits were not computable, the loop is not computable.
8797 if (!isComplete() || ExitNotTaken.empty())
8798 return SE->getCouldNotCompute();
8799
8800 const BasicBlock *Latch = L->getLoopLatch();
8801 // All exiting blocks we have collected must dominate the only backedge.
8802 if (!Latch)
8803 return SE->getCouldNotCompute();
8804
8805 // All exiting blocks we have gathered dominate loop's latch, so exact trip
8806 // count is simply a minimum out of all these calculated exit counts.
8807 SmallVector<SCEVUse, 2> Ops;
8808 for (const auto &ENT : ExitNotTaken) {
8809 const SCEV *BECount = ENT.ExactNotTaken;
8810 assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!");
8811 assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&
8812 "We should only have known counts for exiting blocks that dominate "
8813 "latch!");
8814
8815 Ops.push_back(Elt: BECount);
8816
8817 if (Preds)
8818 append_range(C&: *Preds, R: ENT.Predicates);
8819
8820 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
8821 "Predicate should be always true!");
8822 }
8823
8824 // If an earlier exit exits on the first iteration (exit count zero), then
8825 // a later poison exit count should not propagate into the result. This are
8826 // exactly the semantics provided by umin_seq.
8827 return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true);
8828}
8829
8830const ScalarEvolution::ExitNotTakenInfo *
8831ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
8832 const BasicBlock *ExitingBlock,
8833 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8834 for (const auto &ENT : ExitNotTaken)
8835 if (ENT.ExitingBlock == ExitingBlock) {
8836 if (ENT.hasAlwaysTruePredicate())
8837 return &ENT;
8838 else if (Predicates) {
8839 append_range(C&: *Predicates, R: ENT.Predicates);
8840 return &ENT;
8841 }
8842 }
8843
8844 return nullptr;
8845}
8846
8847/// getConstantMax - Get the constant max backedge taken count for the loop.
8848const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
8849 ScalarEvolution *SE,
8850 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8851 if (!getConstantMax())
8852 return SE->getCouldNotCompute();
8853
8854 for (const auto &ENT : ExitNotTaken)
8855 if (!ENT.hasAlwaysTruePredicate()) {
8856 if (!Predicates)
8857 return SE->getCouldNotCompute();
8858 append_range(C&: *Predicates, R: ENT.Predicates);
8859 }
8860
8861 assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||
8862 isa<SCEVConstant>(getConstantMax())) &&
8863 "No point in having a non-constant max backedge taken count!");
8864 return getConstantMax();
8865}
8866
8867const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
8868 const Loop *L, ScalarEvolution *SE,
8869 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8870 if (!SymbolicMax) {
8871 // Form an expression for the maximum exit count possible for this loop. We
8872 // merge the max and exact information to approximate a version of
8873 // getConstantMaxBackedgeTakenCount which isn't restricted to just
8874 // constants.
8875 SmallVector<SCEVUse, 4> ExitCounts;
8876
8877 for (const auto &ENT : ExitNotTaken) {
8878 const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
8879 if (!isa<SCEVCouldNotCompute>(Val: ExitCount)) {
8880 assert(SE->DT.dominates(ENT.ExitingBlock, L->getLoopLatch()) &&
8881 "We should only have known counts for exiting blocks that "
8882 "dominate latch!");
8883 ExitCounts.push_back(Elt: ExitCount);
8884 if (Predicates)
8885 append_range(C&: *Predicates, R: ENT.Predicates);
8886
8887 assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
8888 "Predicate should be always true!");
8889 }
8890 }
8891 if (ExitCounts.empty())
8892 SymbolicMax = SE->getCouldNotCompute();
8893 else
8894 SymbolicMax =
8895 SE->getUMinFromMismatchedTypes(Ops&: ExitCounts, /*Sequential*/ true);
8896 }
8897 return SymbolicMax;
8898}
8899
8900bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
8901 ScalarEvolution *SE) const {
8902 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
8903 return !ENT.hasAlwaysTruePredicate();
8904 };
8905 return MaxOrZero && !any_of(Range: ExitNotTaken, P: PredicateNotAlwaysTrue);
8906}
8907
8908ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E)
8909 : ExitLimit(E, E, E, false) {}
8910
8911ScalarEvolution::ExitLimit::ExitLimit(
8912 const SCEV *E, const SCEV *ConstantMaxNotTaken,
8913 const SCEV *SymbolicMaxNotTaken, bool MaxOrZero,
8914 ArrayRef<ArrayRef<const SCEVPredicate *>> PredLists)
8915 : ExactNotTaken(E), ConstantMaxNotTaken(ConstantMaxNotTaken),
8916 SymbolicMaxNotTaken(SymbolicMaxNotTaken), MaxOrZero(MaxOrZero) {
8917 // If we prove the max count is zero, so is the symbolic bound. This happens
8918 // in practice due to differences in a) how context sensitive we've chosen
8919 // to be and b) how we reason about bounds implied by UB.
8920 if (ConstantMaxNotTaken->isZero()) {
8921 this->ExactNotTaken = E = ConstantMaxNotTaken;
8922 this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
8923 }
8924
8925 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
8926 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
8927 "Exact is not allowed to be less precise than Constant Max");
8928 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
8929 !isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken)) &&
8930 "Exact is not allowed to be less precise than Symbolic Max");
8931 assert((isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken) ||
8932 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
8933 "Symbolic Max is not allowed to be less precise than Constant Max");
8934 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
8935 isa<SCEVConstant>(ConstantMaxNotTaken)) &&
8936 "No point in having a non-constant max backedge taken count!");
8937 SmallPtrSet<const SCEVPredicate *, 4> SeenPreds;
8938 for (const auto PredList : PredLists)
8939 for (const auto *P : PredList) {
8940 if (SeenPreds.contains(Ptr: P))
8941 continue;
8942 assert(!isa<SCEVUnionPredicate>(P) && "Only add leaf predicates here!");
8943 SeenPreds.insert(Ptr: P);
8944 Predicates.push_back(Elt: P);
8945 }
8946 assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&
8947 "Backedge count should be int");
8948 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
8949 !ConstantMaxNotTaken->getType()->isPointerTy()) &&
8950 "Max backedge count should be int");
8951}
8952
8953ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E,
8954 const SCEV *ConstantMaxNotTaken,
8955 const SCEV *SymbolicMaxNotTaken,
8956 bool MaxOrZero,
8957 ArrayRef<const SCEVPredicate *> PredList)
8958 : ExitLimit(E, ConstantMaxNotTaken, SymbolicMaxNotTaken, MaxOrZero,
8959 ArrayRef({PredList})) {}
8960
8961/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
8962/// computable exit into a persistent ExitNotTakenInfo array.
8963ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
8964 ArrayRef<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo> ExitCounts,
8965 bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
8966 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
8967 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8968
8969 ExitNotTaken.reserve(N: ExitCounts.size());
8970 std::transform(first: ExitCounts.begin(), last: ExitCounts.end(),
8971 result: std::back_inserter(x&: ExitNotTaken),
8972 unary_op: [&](const EdgeExitInfo &EEI) {
8973 BasicBlock *ExitBB = EEI.first;
8974 const ExitLimit &EL = EEI.second;
8975 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
8976 EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
8977 EL.Predicates);
8978 });
8979 assert((isa<SCEVCouldNotCompute>(ConstantMax) ||
8980 isa<SCEVConstant>(ConstantMax)) &&
8981 "No point in having a non-constant max backedge taken count!");
8982}
8983
8984/// Compute the number of times the backedge of the specified loop will execute.
8985ScalarEvolution::BackedgeTakenInfo
8986ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
8987 bool AllowPredicates) {
8988 SmallVector<BasicBlock *, 8> ExitingBlocks;
8989 L->getExitingBlocks(ExitingBlocks);
8990
8991 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8992
8993 SmallVector<EdgeExitInfo, 4> ExitCounts;
8994 bool CouldComputeBECount = true;
8995 BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
8996 const SCEV *MustExitMaxBECount = nullptr;
8997 const SCEV *MayExitMaxBECount = nullptr;
8998 bool MustExitMaxOrZero = false;
8999 bool IsOnlyExit = ExitingBlocks.size() == 1;
9000
9001 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
9002 // and compute maxBECount.
9003 // Do a union of all the predicates here.
9004 for (BasicBlock *ExitBB : ExitingBlocks) {
9005 // We canonicalize untaken exits to br (constant), ignore them so that
9006 // proving an exit untaken doesn't negatively impact our ability to reason
9007 // about the loop as whole.
9008 if (auto *BI = dyn_cast<CondBrInst>(Val: ExitBB->getTerminator()))
9009 if (auto *CI = dyn_cast<ConstantInt>(Val: BI->getCondition())) {
9010 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9011 if (ExitIfTrue == CI->isZero())
9012 continue;
9013 }
9014
9015 ExitLimit EL = computeExitLimit(L, ExitingBlock: ExitBB, IsOnlyExit, AllowPredicates);
9016
9017 assert((AllowPredicates || EL.Predicates.empty()) &&
9018 "Predicated exit limit when predicates are not allowed!");
9019
9020 // 1. For each exit that can be computed, add an entry to ExitCounts.
9021 // CouldComputeBECount is true only if all exits can be computed.
9022 if (EL.ExactNotTaken != getCouldNotCompute())
9023 ++NumExitCountsComputed;
9024 else
9025 // We couldn't compute an exact value for this exit, so
9026 // we won't be able to compute an exact value for the loop.
9027 CouldComputeBECount = false;
9028 // Remember exit count if either exact or symbolic is known. Because
9029 // Exact always implies symbolic, only check symbolic.
9030 if (EL.SymbolicMaxNotTaken != getCouldNotCompute())
9031 ExitCounts.emplace_back(Args&: ExitBB, Args&: EL);
9032 else {
9033 assert(EL.ExactNotTaken == getCouldNotCompute() &&
9034 "Exact is known but symbolic isn't?");
9035 ++NumExitCountsNotComputed;
9036 }
9037
9038 // 2. Derive the loop's MaxBECount from each exit's max number of
9039 // non-exiting iterations. Partition the loop exits into two kinds:
9040 // LoopMustExits and LoopMayExits.
9041 //
9042 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
9043 // is a LoopMayExit. If any computable LoopMustExit is found, then
9044 // MaxBECount is the minimum EL.ConstantMaxNotTaken of computable
9045 // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
9046 // EL.ConstantMaxNotTaken, where CouldNotCompute is considered greater than
9047 // any
9048 // computable EL.ConstantMaxNotTaken.
9049 if (EL.ConstantMaxNotTaken != getCouldNotCompute() && Latch &&
9050 DT.dominates(A: ExitBB, B: Latch)) {
9051 if (!MustExitMaxBECount) {
9052 MustExitMaxBECount = EL.ConstantMaxNotTaken;
9053 MustExitMaxOrZero = EL.MaxOrZero;
9054 } else {
9055 MustExitMaxBECount = getUMinFromMismatchedTypes(LHS: MustExitMaxBECount,
9056 RHS: EL.ConstantMaxNotTaken);
9057 }
9058 } else if (MayExitMaxBECount != getCouldNotCompute()) {
9059 if (!MayExitMaxBECount || EL.ConstantMaxNotTaken == getCouldNotCompute())
9060 MayExitMaxBECount = EL.ConstantMaxNotTaken;
9061 else {
9062 MayExitMaxBECount = getUMaxFromMismatchedTypes(LHS: MayExitMaxBECount,
9063 RHS: EL.ConstantMaxNotTaken);
9064 }
9065 }
9066 }
9067 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
9068 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
9069 // The loop backedge will be taken the maximum or zero times if there's
9070 // a single exit that must be taken the maximum or zero times.
9071 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
9072
9073 // Remember which SCEVs are used in exit limits for invalidation purposes.
9074 // We only care about non-constant SCEVs here, so we can ignore
9075 // EL.ConstantMaxNotTaken
9076 // and MaxBECount, which must be SCEVConstant.
9077 for (const auto &Pair : ExitCounts) {
9078 if (!isa<SCEVConstant>(Val: Pair.second.ExactNotTaken))
9079 BECountUsers[Pair.second.ExactNotTaken].insert(Ptr: {L, AllowPredicates});
9080 if (!isa<SCEVConstant>(Val: Pair.second.SymbolicMaxNotTaken))
9081 BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
9082 Ptr: {L, AllowPredicates});
9083 }
9084 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
9085 MaxBECount, MaxOrZero);
9086}
9087
9088ScalarEvolution::ExitLimit
9089ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
9090 bool IsOnlyExit, bool AllowPredicates) {
9091 assert(L->contains(ExitingBlock) && "Exit count for non-loop block?");
9092 // If our exiting block does not dominate the latch, then its connection with
9093 // loop's exit limit may be far from trivial.
9094 const BasicBlock *Latch = L->getLoopLatch();
9095 if (!Latch || !DT.dominates(A: ExitingBlock, B: Latch))
9096 return getCouldNotCompute();
9097
9098 Instruction *Term = ExitingBlock->getTerminator();
9099 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Term)) {
9100 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9101 assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&
9102 "It should have one successor in loop and one exit block!");
9103 // Proceed to the next level to examine the exit condition expression.
9104 return computeExitLimitFromCond(L, ExitCond: BI->getCondition(), ExitIfTrue,
9105 /*ControlsOnlyExit=*/IsOnlyExit,
9106 AllowPredicates);
9107 }
9108
9109 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: Term)) {
9110 // For switch, make sure that there is a single exit from the loop.
9111 BasicBlock *Exit = nullptr;
9112 for (auto *SBB : successors(BB: ExitingBlock))
9113 if (!L->contains(BB: SBB)) {
9114 if (Exit) // Multiple exit successors.
9115 return getCouldNotCompute();
9116 Exit = SBB;
9117 }
9118 assert(Exit && "Exiting block must have at least one exit");
9119 return computeExitLimitFromSingleExitSwitch(
9120 L, Switch: SI, ExitingBB: Exit, /*ControlsOnlyExit=*/IsSubExpr: IsOnlyExit);
9121 }
9122
9123 return getCouldNotCompute();
9124}
9125
9126ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCond(
9127 const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9128 bool AllowPredicates) {
9129 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
9130 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
9131 ControlsOnlyExit, AllowPredicates);
9132}
9133
9134std::optional<ScalarEvolution::ExitLimit>
9135ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
9136 bool ExitIfTrue, bool ControlsOnlyExit,
9137 bool AllowPredicates) {
9138 (void)this->L;
9139 (void)this->ExitIfTrue;
9140 (void)this->AllowPredicates;
9141
9142 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9143 this->AllowPredicates == AllowPredicates &&
9144 "Variance in assumed invariant key components!");
9145 auto Itr = TripCountMap.find(Val: {ExitCond, ControlsOnlyExit});
9146 if (Itr == TripCountMap.end())
9147 return std::nullopt;
9148 return Itr->second;
9149}
9150
9151void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond,
9152 bool ExitIfTrue,
9153 bool ControlsOnlyExit,
9154 bool AllowPredicates,
9155 const ExitLimit &EL) {
9156 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9157 this->AllowPredicates == AllowPredicates &&
9158 "Variance in assumed invariant key components!");
9159
9160 auto InsertResult = TripCountMap.insert(KV: {{ExitCond, ControlsOnlyExit}, EL});
9161 assert(InsertResult.second && "Expected successful insertion!");
9162 (void)InsertResult;
9163 (void)ExitIfTrue;
9164}
9165
9166ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
9167 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9168 bool ControlsOnlyExit, bool AllowPredicates) {
9169
9170 if (auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit,
9171 AllowPredicates))
9172 return *MaybeEL;
9173
9174 ExitLimit EL = computeExitLimitFromCondImpl(
9175 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates);
9176 Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL);
9177 return EL;
9178}
9179
9180ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
9181 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9182 bool ControlsOnlyExit, bool AllowPredicates) {
9183 // Handle BinOp conditions (And, Or).
9184 if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
9185 Cache, L, ExitCond, ExitIfTrue, AllowPredicates))
9186 return *LimitFromBinOp;
9187
9188 // With an icmp, it may be feasible to compute an exact backedge-taken count.
9189 // Proceed to the next level to examine the icmp.
9190 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(Val: ExitCond)) {
9191 ExitLimit EL =
9192 computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue, IsSubExpr: ControlsOnlyExit);
9193 if (EL.hasFullInfo() || !AllowPredicates)
9194 return EL;
9195
9196 // Try again, but use SCEV predicates this time.
9197 return computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue,
9198 IsSubExpr: ControlsOnlyExit,
9199 /*AllowPredicates=*/true);
9200 }
9201
9202 // Check for a constant condition. These are normally stripped out by
9203 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
9204 // preserve the CFG and is temporarily leaving constant conditions
9205 // in place.
9206 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: ExitCond)) {
9207 if (ExitIfTrue == !CI->getZExtValue())
9208 // The backedge is always taken.
9209 return getCouldNotCompute();
9210 // The backedge is never taken.
9211 return getZero(Ty: CI->getType());
9212 }
9213
9214 // If we're exiting based on the overflow flag of an x.with.overflow intrinsic
9215 // with a constant step, we can form an equivalent icmp predicate and figure
9216 // out how many iterations will be taken before we exit.
9217 const WithOverflowInst *WO;
9218 const APInt *C;
9219 if (match(V: ExitCond, P: m_ExtractValue<1>(V: m_WithOverflowInst(I&: WO))) &&
9220 match(V: WO->getRHS(), P: m_APInt(Res&: C))) {
9221 ConstantRange NWR =
9222 ConstantRange::makeExactNoWrapRegion(BinOp: WO->getBinaryOp(), Other: *C,
9223 NoWrapKind: WO->getNoWrapKind());
9224 CmpInst::Predicate Pred;
9225 APInt NewRHSC, Offset;
9226 NWR.getEquivalentICmp(Pred, RHS&: NewRHSC, Offset);
9227 if (!ExitIfTrue)
9228 Pred = ICmpInst::getInversePredicate(pred: Pred);
9229 auto *LHS = getSCEV(V: WO->getLHS());
9230 if (Offset != 0)
9231 LHS = getAddExpr(LHS, RHS: getConstant(Val: Offset));
9232 auto EL = computeExitLimitFromICmp(L, Pred, LHS, RHS: getConstant(Val: NewRHSC),
9233 IsSubExpr: ControlsOnlyExit, AllowPredicates);
9234 if (EL.hasAnyInfo())
9235 return EL;
9236 }
9237
9238 // If it's not an integer or pointer comparison then compute it the hard way.
9239 return computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9240}
9241
9242std::optional<ScalarEvolution::ExitLimit>
9243ScalarEvolution::computeExitLimitFromCondFromBinOp(ExitLimitCacheTy &Cache,
9244 const Loop *L,
9245 Value *ExitCond,
9246 bool ExitIfTrue,
9247 bool AllowPredicates) {
9248 // Check if the controlling expression for this loop is an And or Or.
9249 Value *Op0, *Op1;
9250 bool IsAnd;
9251 if (match(V: ExitCond, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9252 IsAnd = true;
9253 else if (match(V: ExitCond, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9254 IsAnd = false;
9255 else
9256 return std::nullopt;
9257
9258 // A sub-condition of a non-trivial binop never solely controls the exit,
9259 // whether we exit always depends on both conditions.
9260 ExitLimit EL0 = computeExitLimitFromCondCached(
9261 Cache, L, ExitCond: Op0, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9262 ExitLimit EL1 = computeExitLimitFromCondCached(
9263 Cache, L, ExitCond: Op1, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9264
9265 // EitherMayExit is true in these two cases:
9266 // br (and Op0 Op1), loop, exit
9267 // br (or Op0 Op1), exit, loop
9268 bool EitherMayExit = IsAnd ^ ExitIfTrue;
9269
9270 const SCEV *BECount = getCouldNotCompute();
9271 const SCEV *ConstantMaxBECount = getCouldNotCompute();
9272 const SCEV *SymbolicMaxBECount = getCouldNotCompute();
9273 if (EitherMayExit) {
9274 bool UseSequentialUMin = !isa<BinaryOperator>(Val: ExitCond);
9275 // Both conditions must be same for the loop to continue executing.
9276 // Choose the less conservative count.
9277 if (EL0.ExactNotTaken != getCouldNotCompute() &&
9278 EL1.ExactNotTaken != getCouldNotCompute()) {
9279 BECount = getUMinFromMismatchedTypes(LHS: EL0.ExactNotTaken, RHS: EL1.ExactNotTaken,
9280 Sequential: UseSequentialUMin);
9281 }
9282 if (EL0.ConstantMaxNotTaken == getCouldNotCompute())
9283 ConstantMaxBECount = EL1.ConstantMaxNotTaken;
9284 else if (EL1.ConstantMaxNotTaken == getCouldNotCompute())
9285 ConstantMaxBECount = EL0.ConstantMaxNotTaken;
9286 else
9287 ConstantMaxBECount = getUMinFromMismatchedTypes(LHS: EL0.ConstantMaxNotTaken,
9288 RHS: EL1.ConstantMaxNotTaken);
9289 if (EL0.SymbolicMaxNotTaken == getCouldNotCompute())
9290 SymbolicMaxBECount = EL1.SymbolicMaxNotTaken;
9291 else if (EL1.SymbolicMaxNotTaken == getCouldNotCompute())
9292 SymbolicMaxBECount = EL0.SymbolicMaxNotTaken;
9293 else
9294 SymbolicMaxBECount = getUMinFromMismatchedTypes(
9295 LHS: EL0.SymbolicMaxNotTaken, RHS: EL1.SymbolicMaxNotTaken, Sequential: UseSequentialUMin);
9296 } else {
9297 // Both conditions must be same at the same time for the loop to exit.
9298 // For now, be conservative.
9299 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
9300 BECount = EL0.ExactNotTaken;
9301 }
9302
9303 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able
9304 // to be more aggressive when computing BECount than when computing
9305 // ConstantMaxBECount. In these cases it is possible for EL0.ExactNotTaken
9306 // and
9307 // EL1.ExactNotTaken to match, but for EL0.ConstantMaxNotTaken and
9308 // EL1.ConstantMaxNotTaken to not.
9309 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
9310 !isa<SCEVCouldNotCompute>(Val: BECount))
9311 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
9312 if (isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount))
9313 SymbolicMaxBECount =
9314 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
9315 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
9316 {ArrayRef(EL0.Predicates), ArrayRef(EL1.Predicates)});
9317}
9318
9319ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9320 const Loop *L, ICmpInst *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9321 bool AllowPredicates) {
9322 // If the condition was exit on true, convert the condition to exit on false
9323 CmpPredicate Pred;
9324 if (!ExitIfTrue)
9325 Pred = ExitCond->getCmpPredicate();
9326 else
9327 Pred = ExitCond->getInverseCmpPredicate();
9328 const ICmpInst::Predicate OriginalPred = Pred;
9329
9330 const SCEV *LHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 0));
9331 const SCEV *RHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 1));
9332
9333 ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, IsSubExpr: ControlsOnlyExit,
9334 AllowPredicates);
9335 if (EL.hasAnyInfo())
9336 return EL;
9337
9338 auto *ExhaustiveCount =
9339 computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9340
9341 if (!isa<SCEVCouldNotCompute>(Val: ExhaustiveCount))
9342 return ExhaustiveCount;
9343
9344 return computeShiftCompareExitLimit(LHS: ExitCond->getOperand(i_nocapture: 0),
9345 RHS: ExitCond->getOperand(i_nocapture: 1), L, Pred: OriginalPred);
9346}
9347ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9348 const Loop *L, CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS,
9349 bool ControlsOnlyExit, bool AllowPredicates) {
9350
9351 // Try to evaluate any dependencies out of the loop.
9352 LHS = getSCEVAtScope(S: LHS, L);
9353 RHS = getSCEVAtScope(S: RHS, L);
9354
9355 // At this point, we would like to compute how many iterations of the
9356 // loop the predicate will return true for these inputs.
9357 if (isLoopInvariant(S: LHS, L) && !isLoopInvariant(S: RHS, L)) {
9358 // If there is a loop-invariant, force it into the RHS.
9359 std::swap(a&: LHS, b&: RHS);
9360 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
9361 }
9362
9363 bool ControllingFiniteLoop = ControlsOnlyExit && loopHasNoAbnormalExits(L) &&
9364 loopIsFiniteByAssumption(L);
9365 // Simplify the operands before analyzing them.
9366 (void)SimplifyICmpOperands(Pred, LHS, RHS, /*Depth=*/0);
9367
9368 // If we have a comparison of a chrec against a constant, try to use value
9369 // ranges to answer this query.
9370 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS))
9371 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
9372 if (AddRec->getLoop() == L) {
9373 // Form the constant range.
9374 ConstantRange CompRange =
9375 ConstantRange::makeExactICmpRegion(Pred, Other: RHSC->getAPInt());
9376
9377 const SCEV *Ret = AddRec->getNumIterationsInRange(Range: CompRange, SE&: *this);
9378 if (!isa<SCEVCouldNotCompute>(Val: Ret)) return Ret;
9379 }
9380
9381 // If this loop must exit based on this condition (or execute undefined
9382 // behaviour), see if we can improve wrap flags. This is essentially
9383 // a must execute style proof.
9384 if (ControllingFiniteLoop && isLoopInvariant(S: RHS, L)) {
9385 // If we can prove the test sequence produced must repeat the same values
9386 // on self-wrap of the IV, then we can infer that IV doesn't self wrap
9387 // because if it did, we'd have an infinite (undefined) loop.
9388 // TODO: We can peel off any functions which are invertible *in L*. Loop
9389 // invariant terms are effectively constants for our purposes here.
9390 SCEVUse InnerLHS = LHS;
9391 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val&: LHS))
9392 InnerLHS = ZExt->getOperand();
9393 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: InnerLHS);
9394 AR && !AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() &&
9395 isKnownToBeAPowerOfTwo(S: AR->getStepRecurrence(SE&: *this), /*OrZero=*/true,
9396 /*OrNegative=*/true)) {
9397 auto Flags = AR->getNoWrapFlags();
9398 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
9399 SmallVector<SCEVUse> Operands{AR->operands()};
9400 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9401 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9402 }
9403
9404 // For a slt/ult condition with a positive step, can we prove nsw/nuw?
9405 // From no-self-wrap, this follows trivially from the fact that every
9406 // (un)signed-wrapped, but not self-wrapped value must be LT than the
9407 // last value before (un)signed wrap. Since we know that last value
9408 // didn't exit, nor will any smaller one.
9409 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT) {
9410 auto WrapType = Pred == ICmpInst::ICMP_SLT ? SCEV::FlagNSW : SCEV::FlagNUW;
9411 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS);
9412 AR && AR->getLoop() == L && AR->isAffine() &&
9413 !AR->getNoWrapFlags(Mask: WrapType) && AR->hasNoSelfWrap() &&
9414 isKnownPositive(S: AR->getStepRecurrence(SE&: *this))) {
9415 auto Flags = AR->getNoWrapFlags();
9416 Flags = setFlags(Flags, OnFlags: WrapType);
9417 SmallVector<SCEVUse> Operands{AR->operands()};
9418 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9419 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9420 }
9421 }
9422 }
9423
9424 switch (Pred) {
9425 case ICmpInst::ICMP_NE: { // while (X != Y)
9426 // Convert to: while (X-Y != 0)
9427 if (LHS->getType()->isPointerTy()) {
9428 LHS = getPtrToAddrExpr(Op: LHS);
9429 if (isa<SCEVCouldNotCompute>(Val: LHS))
9430 return LHS;
9431 }
9432 if (RHS->getType()->isPointerTy()) {
9433 RHS = getPtrToAddrExpr(Op: RHS);
9434 if (isa<SCEVCouldNotCompute>(Val: RHS))
9435 return RHS;
9436 }
9437 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit,
9438 AllowPredicates);
9439 if (EL.hasAnyInfo())
9440 return EL;
9441 break;
9442 }
9443 case ICmpInst::ICMP_EQ: { // while (X == Y)
9444 // Convert to: while (X-Y == 0)
9445 if (LHS->getType()->isPointerTy()) {
9446 LHS = getPtrToAddrExpr(Op: LHS);
9447 if (isa<SCEVCouldNotCompute>(Val: LHS))
9448 return LHS;
9449 }
9450 if (RHS->getType()->isPointerTy()) {
9451 RHS = getPtrToAddrExpr(Op: RHS);
9452 if (isa<SCEVCouldNotCompute>(Val: RHS))
9453 return RHS;
9454 }
9455 ExitLimit EL = howFarToNonZero(V: getMinusSCEV(LHS, RHS), L);
9456 if (EL.hasAnyInfo()) return EL;
9457 break;
9458 }
9459 case ICmpInst::ICMP_SLE:
9460 case ICmpInst::ICMP_ULE:
9461 // Since the loop is finite, an invariant RHS cannot include the boundary
9462 // value, otherwise it would loop forever.
9463 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9464 !isLoopInvariant(S: RHS, L)) {
9465 // Otherwise, perform the addition in a wider type, to avoid overflow.
9466 // If the LHS is an addrec with the appropriate nowrap flag, the
9467 // extension will be sunk into it and the exit count can be analyzed.
9468 auto *OldType = dyn_cast<IntegerType>(Val: LHS->getType());
9469 if (!OldType)
9470 break;
9471 // Prefer doubling the bitwidth over adding a single bit to make it more
9472 // likely that we use a legal type.
9473 auto *NewType =
9474 Type::getIntNTy(C&: OldType->getContext(), N: OldType->getBitWidth() * 2);
9475 if (ICmpInst::isSigned(Pred)) {
9476 LHS = getSignExtendExpr(Op: LHS, Ty: NewType);
9477 RHS = getSignExtendExpr(Op: RHS, Ty: NewType);
9478 } else {
9479 LHS = getZeroExtendExpr(Op: LHS, Ty: NewType);
9480 RHS = getZeroExtendExpr(Op: RHS, Ty: NewType);
9481 }
9482 }
9483 RHS = getAddExpr(LHS: getOne(Ty: RHS->getType()), RHS);
9484 [[fallthrough]];
9485 case ICmpInst::ICMP_SLT:
9486 case ICmpInst::ICMP_ULT: { // while (X < Y)
9487 bool IsSigned = ICmpInst::isSigned(Pred);
9488 ExitLimit EL = howManyLessThans(LHS, RHS, L, isSigned: IsSigned, ControlsOnlyExit,
9489 AllowPredicates);
9490 if (EL.hasAnyInfo())
9491 return EL;
9492 break;
9493 }
9494 case ICmpInst::ICMP_SGE:
9495 case ICmpInst::ICMP_UGE:
9496 // Since the loop is finite, an invariant RHS cannot include the boundary
9497 // value, otherwise it would loop forever.
9498 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9499 !isLoopInvariant(S: RHS, L))
9500 break;
9501 RHS = getAddExpr(LHS: getMinusOne(Ty: RHS->getType()), RHS);
9502 [[fallthrough]];
9503 case ICmpInst::ICMP_SGT:
9504 case ICmpInst::ICMP_UGT: { // while (X > Y)
9505 bool IsSigned = ICmpInst::isSigned(Pred);
9506 ExitLimit EL = howManyGreaterThans(LHS, RHS, L, isSigned: IsSigned, IsSubExpr: ControlsOnlyExit,
9507 AllowPredicates);
9508 if (EL.hasAnyInfo())
9509 return EL;
9510 break;
9511 }
9512 default:
9513 break;
9514 }
9515
9516 return getCouldNotCompute();
9517}
9518
9519ScalarEvolution::ExitLimit
9520ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L,
9521 SwitchInst *Switch,
9522 BasicBlock *ExitingBlock,
9523 bool ControlsOnlyExit) {
9524 assert(!L->contains(ExitingBlock) && "Not an exiting block!");
9525
9526 // Give up if the exit is the default dest of a switch.
9527 if (Switch->getDefaultDest() == ExitingBlock)
9528 return getCouldNotCompute();
9529
9530 assert(L->contains(Switch->getDefaultDest()) &&
9531 "Default case must not exit the loop!");
9532 const SCEV *LHS = getSCEVAtScope(V: Switch->getCondition(), L);
9533 const SCEV *RHS = getConstant(V: Switch->findCaseDest(BB: ExitingBlock));
9534
9535 // while (X != Y) --> while (X-Y != 0)
9536 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit);
9537 if (EL.hasAnyInfo())
9538 return EL;
9539
9540 return getCouldNotCompute();
9541}
9542
9543static ConstantInt *
9544EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
9545 ScalarEvolution &SE) {
9546 const SCEV *InVal = SE.getConstant(V: C);
9547 const SCEV *Val = AddRec->evaluateAtIteration(It: InVal, SE);
9548 assert(isa<SCEVConstant>(Val) &&
9549 "Evaluation of SCEV at constant didn't fold correctly?");
9550 return cast<SCEVConstant>(Val)->getValue();
9551}
9552
9553ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
9554 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) {
9555 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: RHSV);
9556 if (!RHS)
9557 return getCouldNotCompute();
9558
9559 const BasicBlock *Latch = L->getLoopLatch();
9560 if (!Latch)
9561 return getCouldNotCompute();
9562
9563 const BasicBlock *Predecessor = L->getLoopPredecessor();
9564 if (!Predecessor)
9565 return getCouldNotCompute();
9566
9567 // Return true if V is of the form "LHS `shift_op` <positive constant>".
9568 // Return LHS in OutLHS, shift_op in OutOpCode, and the shift amount in
9569 // OutShiftAmt.
9570 auto MatchPositiveShift = [](Value *V, Value *&OutLHS,
9571 Instruction::BinaryOps &OutOpCode,
9572 unsigned &OutShiftAmt) {
9573 using namespace PatternMatch;
9574
9575 ConstantInt *ShiftAmt;
9576 if (match(V, P: m_LShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9577 OutOpCode = Instruction::LShr;
9578 else if (match(V, P: m_AShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9579 OutOpCode = Instruction::AShr;
9580 else if (match(V, P: m_Shl(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9581 OutOpCode = Instruction::Shl;
9582 else
9583 return false;
9584
9585 uint64_t Amt = ShiftAmt->getValue().getLimitedValue();
9586 if (Amt == 0 || Amt >= OutLHS->getType()->getScalarSizeInBits())
9587 return false;
9588 OutShiftAmt = Amt;
9589 return true;
9590 };
9591
9592 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in
9593 //
9594 // loop:
9595 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ]
9596 // %iv.shifted = lshr i32 %iv, <positive constant>
9597 //
9598 // Return true on a successful match. Return the corresponding PHI node (%iv
9599 // above) in PNOut, the opcode of the shift operation in OpCodeOut, and the
9600 // shift amount in ShiftAmtOut.
9601 auto MatchShiftRecurrence = [&](Value *V, PHINode *&PNOut,
9602 Instruction::BinaryOps &OpCodeOut,
9603 unsigned &ShiftAmtOut) {
9604 std::optional<Instruction::BinaryOps> PostShiftOpCode;
9605
9606 {
9607 Instruction::BinaryOps OpC;
9608 Value *V;
9609 unsigned Amt;
9610
9611 // If we encounter a shift instruction, "peel off" the shift operation,
9612 // and remember that we did so. Later when we inspect %iv's backedge
9613 // value, we will make sure that the backedge value uses the same
9614 // operation.
9615 //
9616 // Note: the peeled shift operation does not have to be the same
9617 // instruction as the one feeding into the PHI's backedge value. We only
9618 // really care about it being the same *kind* of shift instruction --
9619 // that's all that is required for our later inferences to hold.
9620 if (MatchPositiveShift(LHS, V, OpC, Amt)) {
9621 PostShiftOpCode = OpC;
9622 LHS = V;
9623 }
9624 }
9625
9626 PNOut = dyn_cast<PHINode>(Val: LHS);
9627 if (!PNOut || PNOut->getParent() != L->getHeader())
9628 return false;
9629
9630 Value *BEValue = PNOut->getIncomingValueForBlock(BB: Latch);
9631 Value *OpLHS;
9632
9633 return
9634 // The backedge value for the PHI node must be a shift by a positive
9635 // amount
9636 MatchPositiveShift(BEValue, OpLHS, OpCodeOut, ShiftAmtOut) &&
9637
9638 // of the PHI node itself
9639 OpLHS == PNOut &&
9640
9641 // and the kind of shift should be match the kind of shift we peeled
9642 // off, if any.
9643 (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut);
9644 };
9645
9646 PHINode *PN;
9647 Instruction::BinaryOps OpCode;
9648 unsigned ShiftAmt;
9649 if (!MatchShiftRecurrence(LHS, PN, OpCode, ShiftAmt))
9650 return getCouldNotCompute();
9651
9652 const DataLayout &DL = getDataLayout();
9653
9654 // The key rationale for this optimization is that for some kinds of shift
9655 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1
9656 // within a finite number of iterations. If the condition guarding the
9657 // backedge (in the sense that the backedge is taken if the condition is true)
9658 // is false for the value the shift recurrence stabilizes to, then we know
9659 // that the backedge is taken only a finite number of times.
9660
9661 ConstantInt *StableValue = nullptr;
9662 switch (OpCode) {
9663 default:
9664 llvm_unreachable("Impossible case!");
9665
9666 case Instruction::AShr: {
9667 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
9668 // bitwidth(K) iterations.
9669 Value *FirstValue = PN->getIncomingValueForBlock(BB: Predecessor);
9670 KnownBits Known = computeKnownBits(V: FirstValue, DL, AC: &AC,
9671 CxtI: Predecessor->getTerminator(), DT: &DT);
9672 auto *Ty = cast<IntegerType>(Val: RHS->getType());
9673 if (Known.isNonNegative())
9674 StableValue = ConstantInt::get(Ty, V: 0);
9675 else if (Known.isNegative())
9676 StableValue = ConstantInt::get(Ty, V: -1, IsSigned: true);
9677 else
9678 return getCouldNotCompute();
9679
9680 break;
9681 }
9682 case Instruction::LShr:
9683 case Instruction::Shl:
9684 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>}
9685 // stabilize to 0 in at most bitwidth(K) iterations.
9686 StableValue = ConstantInt::get(Ty: cast<IntegerType>(Val: RHS->getType()), V: 0);
9687 break;
9688 }
9689
9690 auto *Result =
9691 ConstantFoldCompareInstOperands(Predicate: Pred, LHS: StableValue, RHS, DL, TLI: &TLI);
9692 assert(Result->getType()->isIntegerTy(1) &&
9693 "Otherwise cannot be an operand to a branch instruction");
9694
9695 if (Result->isNullValue()) {
9696 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
9697 unsigned MaxBTC = BitWidth;
9698
9699 // For right-shift recurrences (lshr/ashr with non-negative start), we can
9700 // compute a tighter max backedge-taken count from the range of the start
9701 // value. After k shifts of ShiftAmt, value = start >> (k * ShiftAmt).
9702 // The value reaches 0 (the stable value) when k * ShiftAmt >=
9703 // activeBits(start), so max BTC = ceil(activeBits(maxStart) / ShiftAmt).
9704 if (OpCode == Instruction::LShr || OpCode == Instruction::AShr) {
9705 Value *StartValue = PN->getIncomingValueForBlock(BB: Predecessor);
9706 const SCEV *StartSCEV = getSCEV(V: StartValue);
9707 APInt MaxStart = getUnsignedRangeMax(S: StartSCEV);
9708 if (MaxStart.isStrictlyPositive()) {
9709 unsigned ActiveBits = MaxStart.getActiveBits();
9710 unsigned RangeBTC = divideCeil(Numerator: ActiveBits, Denominator: ShiftAmt);
9711 MaxBTC = std::min(a: MaxBTC, b: RangeBTC);
9712 }
9713 }
9714
9715 const SCEV *UpperBound =
9716 getConstant(Ty: getEffectiveSCEVType(Ty: RHS->getType()), V: MaxBTC);
9717 return ExitLimit(getCouldNotCompute(), UpperBound, UpperBound, false);
9718 }
9719
9720 return getCouldNotCompute();
9721}
9722
9723/// Return true if we can constant fold an instruction of the specified type,
9724/// assuming that all operands were constants.
9725static bool CanConstantFold(const Instruction *I) {
9726 if (isa<BinaryOperator>(Val: I) || isa<CmpInst>(Val: I) ||
9727 isa<SelectInst>(Val: I) || isa<CastInst>(Val: I) || isa<GetElementPtrInst>(Val: I) ||
9728 isa<LoadInst>(Val: I) || isa<ExtractValueInst>(Val: I))
9729 return true;
9730
9731 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
9732 if (const Function *F = CI->getCalledFunction())
9733 return canConstantFoldCallTo(Call: CI, F);
9734 return false;
9735}
9736
9737/// Determine whether this instruction can constant evolve within this loop
9738/// assuming its operands can all constant evolve.
9739static bool canConstantEvolve(Instruction *I, const Loop *L) {
9740 // An instruction outside of the loop can't be derived from a loop PHI.
9741 if (!L->contains(Inst: I)) return false;
9742
9743 if (isa<PHINode>(Val: I)) {
9744 // We don't currently keep track of the control flow needed to evaluate
9745 // PHIs, so we cannot handle PHIs inside of loops.
9746 return L->getHeader() == I->getParent();
9747 }
9748
9749 // If we won't be able to constant fold this expression even if the operands
9750 // are constants, bail early.
9751 return CanConstantFold(I);
9752}
9753
9754/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
9755/// recursing through each instruction operand until reaching a loop header phi.
9756static PHINode *
9757getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
9758 DenseMap<Instruction *, PHINode *> &PHIMap,
9759 unsigned Depth) {
9760 if (Depth > MaxConstantEvolvingDepth)
9761 return nullptr;
9762
9763 // Otherwise, we can evaluate this instruction if all of its operands are
9764 // constant or derived from a PHI node themselves.
9765 PHINode *PHI = nullptr;
9766 for (Value *Op : UseInst->operands()) {
9767 if (isa<Constant>(Val: Op)) continue;
9768
9769 Instruction *OpInst = dyn_cast<Instruction>(Val: Op);
9770 if (!OpInst || !canConstantEvolve(I: OpInst, L)) return nullptr;
9771
9772 PHINode *P = dyn_cast<PHINode>(Val: OpInst);
9773 if (!P)
9774 // If this operand is already visited, reuse the prior result.
9775 // We may have P != PHI if this is the deepest point at which the
9776 // inconsistent paths meet.
9777 P = PHIMap.lookup(Val: OpInst);
9778 if (!P) {
9779 // Recurse and memoize the results, whether a phi is found or not.
9780 // This recursive call invalidates pointers into PHIMap.
9781 P = getConstantEvolvingPHIOperands(UseInst: OpInst, L, PHIMap, Depth: Depth + 1);
9782 PHIMap[OpInst] = P;
9783 }
9784 if (!P)
9785 return nullptr; // Not evolving from PHI
9786 if (PHI && PHI != P)
9787 return nullptr; // Evolving from multiple different PHIs.
9788 PHI = P;
9789 }
9790 // This is a expression evolving from a constant PHI!
9791 return PHI;
9792}
9793
9794/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
9795/// in the loop that V is derived from. We allow arbitrary operations along the
9796/// way, but the operands of an operation must either be constants or a value
9797/// derived from a constant PHI. If this expression does not fit with these
9798/// constraints, return null.
9799static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
9800 Instruction *I = dyn_cast<Instruction>(Val: V);
9801 if (!I || !canConstantEvolve(I, L)) return nullptr;
9802
9803 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
9804 return PN;
9805
9806 // Record non-constant instructions contained by the loop.
9807 DenseMap<Instruction *, PHINode *> PHIMap;
9808 return getConstantEvolvingPHIOperands(UseInst: I, L, PHIMap, Depth: 0);
9809}
9810
9811/// EvaluateExpression - Given an expression that passes the
9812/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
9813/// in the loop has the value PHIVal. If we can't fold this expression for some
9814/// reason, return null.
9815static Constant *EvaluateExpression(Value *V, const Loop *L,
9816 DenseMap<Instruction *, Constant *> &Vals,
9817 const DataLayout &DL,
9818 const TargetLibraryInfo *TLI) {
9819 // Convenient constant check, but redundant for recursive calls.
9820 if (Constant *C = dyn_cast<Constant>(Val: V)) return C;
9821 Instruction *I = dyn_cast<Instruction>(Val: V);
9822 if (!I) return nullptr;
9823
9824 if (Constant *C = Vals.lookup(Val: I)) return C;
9825
9826 // An instruction inside the loop depends on a value outside the loop that we
9827 // weren't given a mapping for, or a value such as a call inside the loop.
9828 if (!canConstantEvolve(I, L)) return nullptr;
9829
9830 // An unmapped PHI can be due to a branch or another loop inside this loop,
9831 // or due to this not being the initial iteration through a loop where we
9832 // couldn't compute the evolution of this particular PHI last time.
9833 if (isa<PHINode>(Val: I)) return nullptr;
9834
9835 std::vector<Constant*> Operands(I->getNumOperands());
9836
9837 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
9838 Instruction *Operand = dyn_cast<Instruction>(Val: I->getOperand(i));
9839 if (!Operand) {
9840 Operands[i] = dyn_cast<Constant>(Val: I->getOperand(i));
9841 if (!Operands[i]) return nullptr;
9842 continue;
9843 }
9844 Constant *C = EvaluateExpression(V: Operand, L, Vals, DL, TLI);
9845 Vals[Operand] = C;
9846 if (!C) return nullptr;
9847 Operands[i] = C;
9848 }
9849
9850 return ConstantFoldInstOperands(I, Ops: Operands, DL, TLI,
9851 /*AllowNonDeterministic=*/false);
9852}
9853
9854
9855// If every incoming value to PN except the one for BB is a specific Constant,
9856// return that, else return nullptr.
9857static Constant *getOtherIncomingValue(PHINode *PN, BasicBlock *BB) {
9858 Constant *IncomingVal = nullptr;
9859
9860 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
9861 if (PN->getIncomingBlock(i) == BB)
9862 continue;
9863
9864 auto *CurrentVal = dyn_cast<Constant>(Val: PN->getIncomingValue(i));
9865 if (!CurrentVal)
9866 return nullptr;
9867
9868 if (IncomingVal != CurrentVal) {
9869 if (IncomingVal)
9870 return nullptr;
9871 IncomingVal = CurrentVal;
9872 }
9873 }
9874
9875 return IncomingVal;
9876}
9877
9878/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
9879/// in the header of its containing loop, we know the loop executes a
9880/// constant number of times, and the PHI node is just a recurrence
9881/// involving constants, fold it.
9882Constant *
9883ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
9884 const APInt &BEs,
9885 const Loop *L) {
9886 auto [I, Inserted] = ConstantEvolutionLoopExitValue.try_emplace(Key: PN);
9887 if (!Inserted)
9888 return I->second;
9889
9890 if (BEs.ugt(RHS: MaxBruteForceIterations))
9891 return nullptr; // Not going to evaluate it.
9892
9893 Constant *&RetVal = I->second;
9894
9895 DenseMap<Instruction *, Constant *> CurrentIterVals;
9896 BasicBlock *Header = L->getHeader();
9897 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9898
9899 BasicBlock *Latch = L->getLoopLatch();
9900 if (!Latch)
9901 return nullptr;
9902
9903 for (PHINode &PHI : Header->phis()) {
9904 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
9905 CurrentIterVals[&PHI] = StartCST;
9906 }
9907 if (!CurrentIterVals.count(Val: PN))
9908 return RetVal = nullptr;
9909
9910 Value *BEValue = PN->getIncomingValueForBlock(BB: Latch);
9911
9912 // Execute the loop symbolically to determine the exit value.
9913 assert(BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) &&
9914 "BEs is <= MaxBruteForceIterations which is an 'unsigned'!");
9915
9916 unsigned NumIterations = BEs.getZExtValue(); // must be in range
9917 unsigned IterationNum = 0;
9918 const DataLayout &DL = getDataLayout();
9919 for (; ; ++IterationNum) {
9920 if (IterationNum == NumIterations)
9921 return RetVal = CurrentIterVals[PN]; // Got exit value!
9922
9923 // Compute the value of the PHIs for the next iteration.
9924 // EvaluateExpression adds non-phi values to the CurrentIterVals map.
9925 DenseMap<Instruction *, Constant *> NextIterVals;
9926 Constant *NextPHI =
9927 EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9928 if (!NextPHI)
9929 return nullptr; // Couldn't evaluate!
9930 NextIterVals[PN] = NextPHI;
9931
9932 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
9933
9934 // Also evaluate the other PHI nodes. However, we don't get to stop if we
9935 // cease to be able to evaluate one of them or if they stop evolving,
9936 // because that doesn't necessarily prevent us from computing PN.
9937 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
9938 for (const auto &I : CurrentIterVals) {
9939 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
9940 if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
9941 PHIsToCompute.emplace_back(Args&: PHI, Args: I.second);
9942 }
9943 // We use two distinct loops because EvaluateExpression may invalidate any
9944 // iterators into CurrentIterVals.
9945 for (const auto &I : PHIsToCompute) {
9946 PHINode *PHI = I.first;
9947 Constant *&NextPHI = NextIterVals[PHI];
9948 if (!NextPHI) { // Not already computed.
9949 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
9950 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9951 }
9952 if (NextPHI != I.second)
9953 StoppedEvolving = false;
9954 }
9955
9956 // If all entries in CurrentIterVals == NextIterVals then we can stop
9957 // iterating, the loop can't continue to change.
9958 if (StoppedEvolving)
9959 return RetVal = CurrentIterVals[PN];
9960
9961 CurrentIterVals.swap(RHS&: NextIterVals);
9962 }
9963}
9964
9965const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L,
9966 Value *Cond,
9967 bool ExitWhen) {
9968 PHINode *PN = getConstantEvolvingPHI(V: Cond, L);
9969 if (!PN) return getCouldNotCompute();
9970
9971 // If the loop is canonicalized, the PHI will have exactly two entries.
9972 // That's the only form we support here.
9973 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
9974
9975 DenseMap<Instruction *, Constant *> CurrentIterVals;
9976 BasicBlock *Header = L->getHeader();
9977 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9978
9979 BasicBlock *Latch = L->getLoopLatch();
9980 assert(Latch && "Should follow from NumIncomingValues == 2!");
9981
9982 for (PHINode &PHI : Header->phis()) {
9983 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
9984 CurrentIterVals[&PHI] = StartCST;
9985 }
9986 if (!CurrentIterVals.count(Val: PN))
9987 return getCouldNotCompute();
9988
9989 // Okay, we find a PHI node that defines the trip count of this loop. Execute
9990 // the loop symbolically to determine when the condition gets a value of
9991 // "ExitWhen".
9992 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis.
9993 const DataLayout &DL = getDataLayout();
9994 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
9995 auto *CondVal = dyn_cast_or_null<ConstantInt>(
9996 Val: EvaluateExpression(V: Cond, L, Vals&: CurrentIterVals, DL, TLI: &TLI));
9997
9998 // Couldn't symbolically evaluate.
9999 if (!CondVal) return getCouldNotCompute();
10000
10001 if (CondVal->getValue() == uint64_t(ExitWhen)) {
10002 ++NumBruteForceTripCountsComputed;
10003 return getConstant(Ty: Type::getInt32Ty(C&: getContext()), V: IterationNum);
10004 }
10005
10006 // Update all the PHI nodes for the next iteration.
10007 DenseMap<Instruction *, Constant *> NextIterVals;
10008
10009 // Create a list of which PHIs we need to compute. We want to do this before
10010 // calling EvaluateExpression on them because that may invalidate iterators
10011 // into CurrentIterVals.
10012 SmallVector<PHINode *, 8> PHIsToCompute;
10013 for (const auto &I : CurrentIterVals) {
10014 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
10015 if (!PHI || PHI->getParent() != Header) continue;
10016 PHIsToCompute.push_back(Elt: PHI);
10017 }
10018 for (PHINode *PHI : PHIsToCompute) {
10019 Constant *&NextPHI = NextIterVals[PHI];
10020 if (NextPHI) continue; // Already computed!
10021
10022 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
10023 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
10024 }
10025 CurrentIterVals.swap(RHS&: NextIterVals);
10026 }
10027
10028 // Too many iterations were needed to evaluate.
10029 return getCouldNotCompute();
10030}
10031
10032const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
10033 SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values =
10034 ValuesAtScopes[V];
10035 // Check to see if we've folded this expression at this loop before.
10036 for (auto &LS : Values)
10037 if (LS.first == L)
10038 return LS.second ? LS.second : V;
10039
10040 Values.emplace_back(Args&: L, Args: nullptr);
10041
10042 // Otherwise compute it.
10043 const SCEV *C = computeSCEVAtScope(S: V, L);
10044 for (auto &LS : reverse(C&: ValuesAtScopes[V]))
10045 if (LS.first == L) {
10046 LS.second = C;
10047 if (!isa<SCEVConstant>(Val: C))
10048 ValuesAtScopesUsers[C].push_back(Elt: {L, V});
10049 break;
10050 }
10051 return C;
10052}
10053
10054/// This builds up a Constant using the ConstantExpr interface. That way, we
10055/// will return Constants for objects which aren't represented by a
10056/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
10057/// Returns NULL if the SCEV isn't representable as a Constant.
10058static Constant *BuildConstantFromSCEV(const SCEV *V) {
10059 switch (V->getSCEVType()) {
10060 case scCouldNotCompute:
10061 case scAddRecExpr:
10062 case scVScale:
10063 return nullptr;
10064 case scConstant:
10065 return cast<SCEVConstant>(Val: V)->getValue();
10066 case scUnknown:
10067 return dyn_cast<Constant>(Val: cast<SCEVUnknown>(Val: V)->getValue());
10068 case scPtrToAddr: {
10069 const SCEVPtrToAddrExpr *P2I = cast<SCEVPtrToAddrExpr>(Val: V);
10070 if (Constant *CastOp = BuildConstantFromSCEV(V: P2I->getOperand()))
10071 return ConstantExpr::getPtrToAddr(C: CastOp, Ty: P2I->getType());
10072
10073 return nullptr;
10074 }
10075 case scTruncate: {
10076 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(Val: V);
10077 if (Constant *CastOp = BuildConstantFromSCEV(V: ST->getOperand()))
10078 return ConstantExpr::getTrunc(C: CastOp, Ty: ST->getType());
10079 return nullptr;
10080 }
10081 case scAddExpr: {
10082 const SCEVAddExpr *SA = cast<SCEVAddExpr>(Val: V);
10083 Constant *C = nullptr;
10084 for (const SCEV *Op : SA->operands()) {
10085 Constant *OpC = BuildConstantFromSCEV(V: Op);
10086 if (!OpC)
10087 return nullptr;
10088 if (!C) {
10089 C = OpC;
10090 continue;
10091 }
10092 assert(!C->getType()->isPointerTy() &&
10093 "Can only have one pointer, and it must be last");
10094 if (OpC->getType()->isPointerTy()) {
10095 // The offsets have been converted to bytes. We can add bytes using
10096 // an i8 GEP.
10097 C = ConstantExpr::getPtrAdd(Ptr: OpC, Offset: C);
10098 } else {
10099 C = ConstantExpr::getAdd(C1: C, C2: OpC);
10100 }
10101 }
10102 return C;
10103 }
10104 case scMulExpr:
10105 case scSignExtend:
10106 case scZeroExtend:
10107 case scUDivExpr:
10108 case scSMaxExpr:
10109 case scUMaxExpr:
10110 case scSMinExpr:
10111 case scUMinExpr:
10112 case scSequentialUMinExpr:
10113 return nullptr;
10114 }
10115 llvm_unreachable("Unknown SCEV kind!");
10116}
10117
10118const SCEV *ScalarEvolution::getWithOperands(const SCEV *S,
10119 SmallVectorImpl<SCEVUse> &NewOps) {
10120 switch (S->getSCEVType()) {
10121 case scTruncate:
10122 case scZeroExtend:
10123 case scSignExtend:
10124 case scPtrToAddr:
10125 return getCastExpr(Kind: S->getSCEVType(), Op: NewOps[0], Ty: S->getType());
10126 case scAddRecExpr: {
10127 auto *AddRec = cast<SCEVAddRecExpr>(Val: S);
10128 return getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags: AddRec->getNoWrapFlags());
10129 }
10130 case scAddExpr:
10131 return getAddExpr(Ops&: NewOps, OrigFlags: cast<SCEVAddExpr>(Val: S)->getNoWrapFlags());
10132 case scMulExpr:
10133 return getMulExpr(Ops&: NewOps, OrigFlags: cast<SCEVMulExpr>(Val: S)->getNoWrapFlags());
10134 case scUDivExpr:
10135 return getUDivExpr(LHS: NewOps[0], RHS: NewOps[1]);
10136 case scUMaxExpr:
10137 case scSMaxExpr:
10138 case scUMinExpr:
10139 case scSMinExpr:
10140 return getMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10141 case scSequentialUMinExpr:
10142 return getSequentialMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10143 case scConstant:
10144 case scVScale:
10145 case scUnknown:
10146 return S;
10147 case scCouldNotCompute:
10148 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10149 }
10150 llvm_unreachable("Unknown SCEV kind!");
10151}
10152
10153const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
10154 switch (V->getSCEVType()) {
10155 case scConstant:
10156 case scVScale:
10157 return V;
10158 case scAddRecExpr: {
10159 // If this is a loop recurrence for a loop that does not contain L, then we
10160 // are dealing with the final value computed by the loop.
10161 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: V);
10162 // First, attempt to evaluate each operand.
10163 // Avoid performing the look-up in the common case where the specified
10164 // expression has no loop-variant portions.
10165 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
10166 const SCEV *OpAtScope = getSCEVAtScope(V: AddRec->getOperand(i), L);
10167 if (OpAtScope == AddRec->getOperand(i))
10168 continue;
10169
10170 // Okay, at least one of these operands is loop variant but might be
10171 // foldable. Build a new instance of the folded commutative expression.
10172 SmallVector<SCEVUse, 8> NewOps;
10173 NewOps.reserve(N: AddRec->getNumOperands());
10174 append_range(C&: NewOps, R: AddRec->operands().take_front(N: i));
10175 NewOps.push_back(Elt: OpAtScope);
10176 for (++i; i != e; ++i)
10177 NewOps.push_back(Elt: getSCEVAtScope(V: AddRec->getOperand(i), L));
10178
10179 const SCEV *FoldedRec = getAddRecExpr(
10180 Operands&: NewOps, L: AddRec->getLoop(), Flags: AddRec->getNoWrapFlags(Mask: SCEV::FlagNW));
10181 AddRec = dyn_cast<SCEVAddRecExpr>(Val: FoldedRec);
10182 // The addrec may be folded to a nonrecurrence, for example, if the
10183 // induction variable is multiplied by zero after constant folding. Go
10184 // ahead and return the folded value.
10185 if (!AddRec)
10186 return FoldedRec;
10187 break;
10188 }
10189
10190 // If the scope is outside the addrec's loop, evaluate it by using the
10191 // loop exit value of the addrec.
10192 if (!AddRec->getLoop()->contains(L)) {
10193 // To evaluate this recurrence, we need to know how many times the AddRec
10194 // loop iterates. Compute this now.
10195 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: AddRec->getLoop());
10196 if (BackedgeTakenCount == getCouldNotCompute())
10197 return AddRec;
10198
10199 // Then, evaluate the AddRec.
10200 return AddRec->evaluateAtIteration(It: BackedgeTakenCount, SE&: *this);
10201 }
10202
10203 return AddRec;
10204 }
10205 case scTruncate:
10206 case scZeroExtend:
10207 case scSignExtend:
10208 case scPtrToAddr:
10209 case scAddExpr:
10210 case scMulExpr:
10211 case scUDivExpr:
10212 case scUMaxExpr:
10213 case scSMaxExpr:
10214 case scUMinExpr:
10215 case scSMinExpr:
10216 case scSequentialUMinExpr: {
10217 ArrayRef<SCEVUse> Ops = V->operands();
10218 // Avoid performing the look-up in the common case where the specified
10219 // expression has no loop-variant portions.
10220 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
10221 const SCEV *OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10222 if (OpAtScope != Ops[i].getPointer()) {
10223 // Okay, at least one of these operands is loop variant but might be
10224 // foldable. Build a new instance of the folded commutative expression.
10225 SmallVector<SCEVUse, 8> NewOps;
10226 NewOps.reserve(N: Ops.size());
10227 append_range(C&: NewOps, R: Ops.take_front(N: i));
10228 NewOps.push_back(Elt: OpAtScope);
10229
10230 for (++i; i != e; ++i) {
10231 OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10232 NewOps.push_back(Elt: OpAtScope);
10233 }
10234
10235 return getWithOperands(S: V, NewOps);
10236 }
10237 }
10238 // If we got here, all operands are loop invariant.
10239 return V;
10240 }
10241 case scUnknown: {
10242 // If this instruction is evolved from a constant-evolving PHI, compute the
10243 // exit value from the loop without using SCEVs.
10244 const SCEVUnknown *SU = cast<SCEVUnknown>(Val: V);
10245 Instruction *I = dyn_cast<Instruction>(Val: SU->getValue());
10246 if (!I)
10247 return V; // This is some other type of SCEVUnknown, just return it.
10248
10249 if (PHINode *PN = dyn_cast<PHINode>(Val: I)) {
10250 const Loop *CurrLoop = this->LI[I->getParent()];
10251 // Looking for loop exit value.
10252 if (CurrLoop && CurrLoop->getParentLoop() == L &&
10253 PN->getParent() == CurrLoop->getHeader()) {
10254 // Okay, there is no closed form solution for the PHI node. Check
10255 // to see if the loop that contains it has a known backedge-taken
10256 // count. If so, we may be able to force computation of the exit
10257 // value.
10258 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: CurrLoop);
10259 // This trivial case can show up in some degenerate cases where
10260 // the incoming IR has not yet been fully simplified.
10261 if (BackedgeTakenCount->isZero()) {
10262 Value *InitValue = nullptr;
10263 bool MultipleInitValues = false;
10264 for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
10265 if (!CurrLoop->contains(BB: PN->getIncomingBlock(i))) {
10266 if (!InitValue)
10267 InitValue = PN->getIncomingValue(i);
10268 else if (InitValue != PN->getIncomingValue(i)) {
10269 MultipleInitValues = true;
10270 break;
10271 }
10272 }
10273 }
10274 if (!MultipleInitValues && InitValue)
10275 return getSCEV(V: InitValue);
10276 }
10277 // Do we have a loop invariant value flowing around the backedge
10278 // for a loop which must execute the backedge?
10279 if (!isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount) &&
10280 isKnownNonZero(S: BackedgeTakenCount) &&
10281 PN->getNumIncomingValues() == 2) {
10282
10283 unsigned InLoopPred =
10284 CurrLoop->contains(BB: PN->getIncomingBlock(i: 0)) ? 0 : 1;
10285 Value *BackedgeVal = PN->getIncomingValue(i: InLoopPred);
10286 if (CurrLoop->isLoopInvariant(V: BackedgeVal))
10287 return getSCEV(V: BackedgeVal);
10288 }
10289 if (auto *BTCC = dyn_cast<SCEVConstant>(Val: BackedgeTakenCount)) {
10290 // Okay, we know how many times the containing loop executes. If
10291 // this is a constant evolving PHI node, get the final value at
10292 // the specified iteration number.
10293 Constant *RV =
10294 getConstantEvolutionLoopExitValue(PN, BEs: BTCC->getAPInt(), L: CurrLoop);
10295 if (RV)
10296 return getSCEV(V: RV);
10297 }
10298 }
10299 }
10300
10301 // Okay, this is an expression that we cannot symbolically evaluate
10302 // into a SCEV. Check to see if it's possible to symbolically evaluate
10303 // the arguments into constants, and if so, try to constant propagate the
10304 // result. This is particularly useful for computing loop exit values.
10305 if (!CanConstantFold(I))
10306 return V; // This is some other type of SCEVUnknown, just return it.
10307
10308 SmallVector<Constant *, 4> Operands;
10309 Operands.reserve(N: I->getNumOperands());
10310 bool MadeImprovement = false;
10311 for (Value *Op : I->operands()) {
10312 if (Constant *C = dyn_cast<Constant>(Val: Op)) {
10313 Operands.push_back(Elt: C);
10314 continue;
10315 }
10316
10317 // If any of the operands is non-constant and if they are
10318 // non-integer and non-pointer, don't even try to analyze them
10319 // with scev techniques.
10320 if (!isSCEVable(Ty: Op->getType()))
10321 return V;
10322
10323 const SCEV *OrigV = getSCEV(V: Op);
10324 const SCEV *OpV = getSCEVAtScope(V: OrigV, L);
10325 MadeImprovement |= OrigV != OpV;
10326
10327 Constant *C = BuildConstantFromSCEV(V: OpV);
10328 if (!C)
10329 return V;
10330 assert(C->getType() == Op->getType() && "Type mismatch");
10331 Operands.push_back(Elt: C);
10332 }
10333
10334 // Check to see if getSCEVAtScope actually made an improvement.
10335 if (!MadeImprovement)
10336 return V; // This is some other type of SCEVUnknown, just return it.
10337
10338 Constant *C = nullptr;
10339 const DataLayout &DL = getDataLayout();
10340 C = ConstantFoldInstOperands(I, Ops: Operands, DL, TLI: &TLI,
10341 /*AllowNonDeterministic=*/false);
10342 if (!C)
10343 return V;
10344 return getSCEV(V: C);
10345 }
10346 case scCouldNotCompute:
10347 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10348 }
10349 llvm_unreachable("Unknown SCEV type!");
10350}
10351
10352const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
10353 return getSCEVAtScope(V: getSCEV(V), L);
10354}
10355
10356const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const {
10357 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: S))
10358 return stripInjectiveFunctions(S: ZExt->getOperand());
10359 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
10360 return stripInjectiveFunctions(S: SExt->getOperand());
10361 return S;
10362}
10363
10364/// Finds the minimum unsigned root of the following equation:
10365///
10366/// A * X = B (mod N)
10367///
10368/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
10369/// A and B isn't important.
10370///
10371/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
10372static const SCEV *
10373SolveLinEquationWithOverflow(const APInt &A, const SCEV *B,
10374 SmallVectorImpl<const SCEVPredicate *> *Predicates,
10375 ScalarEvolution &SE, const Loop *L) {
10376 uint32_t BW = A.getBitWidth();
10377 assert(BW == SE.getTypeSizeInBits(B->getType()));
10378 assert(A != 0 && "A must be non-zero.");
10379
10380 // 1. D = gcd(A, N)
10381 //
10382 // The gcd of A and N may have only one prime factor: 2. The number of
10383 // trailing zeros in A is its multiplicity
10384 uint32_t Mult2 = A.countr_zero();
10385 // D = 2^Mult2
10386
10387 // 2. Check if B is divisible by D.
10388 //
10389 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
10390 // is not less than multiplicity of this prime factor for D.
10391 unsigned MinTZ = SE.getMinTrailingZeros(S: B);
10392 // Try again with the terminator of the loop predecessor for context-specific
10393 // result, if MinTZ s too small.
10394 if (MinTZ < Mult2 && L->getLoopPredecessor())
10395 MinTZ = SE.getMinTrailingZeros(S: B, CtxI: L->getLoopPredecessor()->getTerminator());
10396 if (MinTZ < Mult2) {
10397 // Check if we can prove there's no remainder using URem.
10398 const SCEV *URem =
10399 SE.getURemExpr(LHS: B, RHS: SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2)));
10400 const SCEV *Zero = SE.getZero(Ty: B->getType());
10401 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: URem, RHS: Zero)) {
10402 // Try to add a predicate ensuring B is a multiple of 1 << Mult2.
10403 if (!Predicates)
10404 return SE.getCouldNotCompute();
10405
10406 // Avoid adding a predicate that is known to be false.
10407 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_NE, LHS: URem, RHS: Zero))
10408 return SE.getCouldNotCompute();
10409 Predicates->push_back(Elt: SE.getEqualPredicate(LHS: URem, RHS: Zero));
10410 }
10411 }
10412
10413 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
10414 // modulo (N / D).
10415 //
10416 // If D == 1, (N / D) == N == 2^BW, so we need one extra bit to represent
10417 // (N / D) in general. The inverse itself always fits into BW bits, though,
10418 // so we immediately truncate it.
10419 APInt AD = A.lshr(shiftAmt: Mult2).trunc(width: BW - Mult2); // AD = A / D
10420 APInt I = AD.multiplicativeInverse().zext(width: BW);
10421
10422 // 4. Compute the minimum unsigned root of the equation:
10423 // I * (B / D) mod (N / D)
10424 // To simplify the computation, we factor out the divide by D:
10425 // (I * B mod N) / D
10426 const SCEV *D = SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2));
10427 return SE.getUDivExactExpr(LHS: SE.getMulExpr(LHS: B, RHS: SE.getConstant(Val: I)), RHS: D);
10428}
10429
10430/// For a given quadratic addrec, generate coefficients of the corresponding
10431/// quadratic equation, multiplied by a common value to ensure that they are
10432/// integers.
10433/// The returned value is a tuple { A, B, C, M, BitWidth }, where
10434/// Ax^2 + Bx + C is the quadratic function, M is the value that A, B and C
10435/// were multiplied by, and BitWidth is the bit width of the original addrec
10436/// coefficients.
10437/// This function returns std::nullopt if the addrec coefficients are not
10438/// compile- time constants.
10439static std::optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
10440GetQuadraticEquation(const SCEVAddRecExpr *AddRec) {
10441 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
10442 const SCEVConstant *LC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 0));
10443 const SCEVConstant *MC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 1));
10444 const SCEVConstant *NC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 2));
10445 LLVM_DEBUG(dbgs() << __func__ << ": analyzing quadratic addrec: "
10446 << *AddRec << '\n');
10447
10448 // We currently can only solve this if the coefficients are constants.
10449 if (!LC || !MC || !NC) {
10450 LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n");
10451 return std::nullopt;
10452 }
10453
10454 APInt L = LC->getAPInt();
10455 APInt M = MC->getAPInt();
10456 APInt N = NC->getAPInt();
10457 assert(!N.isZero() && "This is not a quadratic addrec");
10458
10459 unsigned BitWidth = LC->getAPInt().getBitWidth();
10460 unsigned NewWidth = BitWidth + 1;
10461 LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "
10462 << BitWidth << '\n');
10463 // The sign-extension (as opposed to a zero-extension) here matches the
10464 // extension used in SolveQuadraticEquationWrap (with the same motivation).
10465 N = N.sext(width: NewWidth);
10466 M = M.sext(width: NewWidth);
10467 L = L.sext(width: NewWidth);
10468
10469 // The increments are M, M+N, M+2N, ..., so the accumulated values are
10470 // L+M, (L+M)+(M+N), (L+M)+(M+N)+(M+2N), ..., that is,
10471 // L+M, L+2M+N, L+3M+3N, ...
10472 // After n iterations the accumulated value Acc is L + nM + n(n-1)/2 N.
10473 //
10474 // The equation Acc = 0 is then
10475 // L + nM + n(n-1)/2 N = 0, or 2L + 2M n + n(n-1) N = 0.
10476 // In a quadratic form it becomes:
10477 // N n^2 + (2M-N) n + 2L = 0.
10478
10479 APInt A = N;
10480 APInt B = 2 * M - A;
10481 APInt C = 2 * L;
10482 APInt T = APInt(NewWidth, 2);
10483 LLVM_DEBUG(dbgs() << __func__ << ": equation " << A << "x^2 + " << B
10484 << "x + " << C << ", coeff bw: " << NewWidth
10485 << ", multiplied by " << T << '\n');
10486 return std::make_tuple(args&: A, args&: B, args&: C, args&: T, args&: BitWidth);
10487}
10488
10489/// Helper function to compare optional APInts:
10490/// (a) if X and Y both exist, return min(X, Y),
10491/// (b) if neither X nor Y exist, return std::nullopt,
10492/// (c) if exactly one of X and Y exists, return that value.
10493static std::optional<APInt> MinOptional(std::optional<APInt> X,
10494 std::optional<APInt> Y) {
10495 if (X && Y) {
10496 unsigned W = std::max(a: X->getBitWidth(), b: Y->getBitWidth());
10497 APInt XW = X->sext(width: W);
10498 APInt YW = Y->sext(width: W);
10499 return XW.slt(RHS: YW) ? *X : *Y;
10500 }
10501 if (!X && !Y)
10502 return std::nullopt;
10503 return X ? *X : *Y;
10504}
10505
10506/// Helper function to truncate an optional APInt to a given BitWidth.
10507/// When solving addrec-related equations, it is preferable to return a value
10508/// that has the same bit width as the original addrec's coefficients. If the
10509/// solution fits in the original bit width, truncate it (except for i1).
10510/// Returning a value of a different bit width may inhibit some optimizations.
10511///
10512/// In general, a solution to a quadratic equation generated from an addrec
10513/// may require BW+1 bits, where BW is the bit width of the addrec's
10514/// coefficients. The reason is that the coefficients of the quadratic
10515/// equation are BW+1 bits wide (to avoid truncation when converting from
10516/// the addrec to the equation).
10517static std::optional<APInt> TruncIfPossible(std::optional<APInt> X,
10518 unsigned BitWidth) {
10519 if (!X)
10520 return std::nullopt;
10521 unsigned W = X->getBitWidth();
10522 if (BitWidth > 1 && BitWidth < W && X->isIntN(N: BitWidth))
10523 return X->trunc(width: BitWidth);
10524 return X;
10525}
10526
10527/// Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n
10528/// iterations. The values L, M, N are assumed to be signed, and they
10529/// should all have the same bit widths.
10530/// Find the least n >= 0 such that c(n) = 0 in the arithmetic modulo 2^BW,
10531/// where BW is the bit width of the addrec's coefficients.
10532/// If the calculated value is a BW-bit integer (for BW > 1), it will be
10533/// returned as such, otherwise the bit width of the returned value may
10534/// be greater than BW.
10535///
10536/// This function returns std::nullopt if
10537/// (a) the addrec coefficients are not constant, or
10538/// (b) SolveQuadraticEquationWrap was unable to find a solution. For cases
10539/// like x^2 = 5, no integer solutions exist, in other cases an integer
10540/// solution may exist, but SolveQuadraticEquationWrap may fail to find it.
10541static std::optional<APInt>
10542SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
10543 APInt A, B, C, M;
10544 unsigned BitWidth;
10545 auto T = GetQuadraticEquation(AddRec);
10546 if (!T)
10547 return std::nullopt;
10548
10549 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10550 LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n");
10551 std::optional<APInt> X =
10552 APIntOps::SolveQuadraticEquationWrap(A, B, C, RangeWidth: BitWidth + 1);
10553 if (!X)
10554 return std::nullopt;
10555
10556 ConstantInt *CX = ConstantInt::get(Context&: SE.getContext(), V: *X);
10557 ConstantInt *V = EvaluateConstantChrecAtConstant(AddRec, C: CX, SE);
10558 if (!V->isZero())
10559 return std::nullopt;
10560
10561 return TruncIfPossible(X, BitWidth);
10562}
10563
10564/// Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n
10565/// iterations. The values M, N are assumed to be signed, and they
10566/// should all have the same bit widths.
10567/// Find the least n such that c(n) does not belong to the given range,
10568/// while c(n-1) does.
10569///
10570/// This function returns std::nullopt if
10571/// (a) the addrec coefficients are not constant, or
10572/// (b) SolveQuadraticEquationWrap was unable to find a solution for the
10573/// bounds of the range.
10574static std::optional<APInt>
10575SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec,
10576 const ConstantRange &Range, ScalarEvolution &SE) {
10577 assert(AddRec->getOperand(0)->isZero() &&
10578 "Starting value of addrec should be 0");
10579 LLVM_DEBUG(dbgs() << __func__ << ": solving boundary crossing for range "
10580 << Range << ", addrec " << *AddRec << '\n');
10581 // This case is handled in getNumIterationsInRange. Here we can assume that
10582 // we start in the range.
10583 assert(Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) &&
10584 "Addrec's initial value should be in range");
10585
10586 APInt A, B, C, M;
10587 unsigned BitWidth;
10588 auto T = GetQuadraticEquation(AddRec);
10589 if (!T)
10590 return std::nullopt;
10591
10592 // Be careful about the return value: there can be two reasons for not
10593 // returning an actual number. First, if no solutions to the equations
10594 // were found, and second, if the solutions don't leave the given range.
10595 // The first case means that the actual solution is "unknown", the second
10596 // means that it's known, but not valid. If the solution is unknown, we
10597 // cannot make any conclusions.
10598 // Return a pair: the optional solution and a flag indicating if the
10599 // solution was found.
10600 auto SolveForBoundary =
10601 [&](APInt Bound) -> std::pair<std::optional<APInt>, bool> {
10602 // Solve for signed overflow and unsigned overflow, pick the lower
10603 // solution.
10604 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: checking boundary "
10605 << Bound << " (before multiplying by " << M << ")\n");
10606 Bound *= M; // The quadratic equation multiplier.
10607
10608 std::optional<APInt> SO;
10609 if (BitWidth > 1) {
10610 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10611 "signed overflow\n");
10612 SO = APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth);
10613 }
10614 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10615 "unsigned overflow\n");
10616 std::optional<APInt> UO =
10617 APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth + 1);
10618
10619 auto LeavesRange = [&] (const APInt &X) {
10620 ConstantInt *C0 = ConstantInt::get(Context&: SE.getContext(), V: X);
10621 ConstantInt *V0 = EvaluateConstantChrecAtConstant(AddRec, C: C0, SE);
10622 if (Range.contains(Val: V0->getValue()))
10623 return false;
10624 // X should be at least 1, so X-1 is non-negative.
10625 ConstantInt *C1 = ConstantInt::get(Context&: SE.getContext(), V: X-1);
10626 ConstantInt *V1 = EvaluateConstantChrecAtConstant(AddRec, C: C1, SE);
10627 if (Range.contains(Val: V1->getValue()))
10628 return true;
10629 return false;
10630 };
10631
10632 // If SolveQuadraticEquationWrap returns std::nullopt, it means that there
10633 // can be a solution, but the function failed to find it. We cannot treat it
10634 // as "no solution".
10635 if (!SO || !UO)
10636 return {std::nullopt, false};
10637
10638 // Check the smaller value first to see if it leaves the range.
10639 // At this point, both SO and UO must have values.
10640 std::optional<APInt> Min = MinOptional(X: SO, Y: UO);
10641 if (LeavesRange(*Min))
10642 return { Min, true };
10643 std::optional<APInt> Max = Min == SO ? UO : SO;
10644 if (LeavesRange(*Max))
10645 return { Max, true };
10646
10647 // Solutions were found, but were eliminated, hence the "true".
10648 return {std::nullopt, true};
10649 };
10650
10651 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10652 // Lower bound is inclusive, subtract 1 to represent the exiting value.
10653 APInt Lower = Range.getLower().sext(width: A.getBitWidth()) - 1;
10654 APInt Upper = Range.getUpper().sext(width: A.getBitWidth());
10655 auto SL = SolveForBoundary(Lower);
10656 auto SU = SolveForBoundary(Upper);
10657 // If any of the solutions was unknown, no meaninigful conclusions can
10658 // be made.
10659 if (!SL.second || !SU.second)
10660 return std::nullopt;
10661
10662 // Claim: The correct solution is not some value between Min and Max.
10663 //
10664 // Justification: Assuming that Min and Max are different values, one of
10665 // them is when the first signed overflow happens, the other is when the
10666 // first unsigned overflow happens. Crossing the range boundary is only
10667 // possible via an overflow (treating 0 as a special case of it, modeling
10668 // an overflow as crossing k*2^W for some k).
10669 //
10670 // The interesting case here is when Min was eliminated as an invalid
10671 // solution, but Max was not. The argument is that if there was another
10672 // overflow between Min and Max, it would also have been eliminated if
10673 // it was considered.
10674 //
10675 // For a given boundary, it is possible to have two overflows of the same
10676 // type (signed/unsigned) without having the other type in between: this
10677 // can happen when the vertex of the parabola is between the iterations
10678 // corresponding to the overflows. This is only possible when the two
10679 // overflows cross k*2^W for the same k. In such case, if the second one
10680 // left the range (and was the first one to do so), the first overflow
10681 // would have to enter the range, which would mean that either we had left
10682 // the range before or that we started outside of it. Both of these cases
10683 // are contradictions.
10684 //
10685 // Claim: In the case where SolveForBoundary returns std::nullopt, the correct
10686 // solution is not some value between the Max for this boundary and the
10687 // Min of the other boundary.
10688 //
10689 // Justification: Assume that we had such Max_A and Min_B corresponding
10690 // to range boundaries A and B and such that Max_A < Min_B. If there was
10691 // a solution between Max_A and Min_B, it would have to be caused by an
10692 // overflow corresponding to either A or B. It cannot correspond to B,
10693 // since Min_B is the first occurrence of such an overflow. If it
10694 // corresponded to A, it would have to be either a signed or an unsigned
10695 // overflow that is larger than both eliminated overflows for A. But
10696 // between the eliminated overflows and this overflow, the values would
10697 // cover the entire value space, thus crossing the other boundary, which
10698 // is a contradiction.
10699
10700 return TruncIfPossible(X: MinOptional(X: SL.first, Y: SU.first), BitWidth);
10701}
10702
10703ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V,
10704 const Loop *L,
10705 bool ControlsOnlyExit,
10706 bool AllowPredicates) {
10707
10708 // This is only used for loops with a "x != y" exit test. The exit condition
10709 // is now expressed as a single expression, V = x-y. So the exit test is
10710 // effectively V != 0. We know and take advantage of the fact that this
10711 // expression only being used in a comparison by zero context.
10712
10713 SmallVector<const SCEVPredicate *> Predicates;
10714 // If the value is a constant
10715 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10716 // If the value is already zero, the branch will execute zero times.
10717 if (C->getValue()->isZero()) return C;
10718 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10719 }
10720
10721 const SCEVAddRecExpr *AddRec =
10722 dyn_cast<SCEVAddRecExpr>(Val: stripInjectiveFunctions(S: V));
10723
10724 if (!AddRec && AllowPredicates)
10725 // Try to make this an AddRec using runtime tests, in the first X
10726 // iterations of this loop, where X is the SCEV expression found by the
10727 // algorithm below.
10728 AddRec = convertSCEVToAddRecWithPredicates(S: V, L, Preds&: Predicates);
10729
10730 if (!AddRec || AddRec->getLoop() != L)
10731 return getCouldNotCompute();
10732
10733 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
10734 // the quadratic equation to solve it.
10735 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
10736 // We can only use this value if the chrec ends up with an exact zero
10737 // value at this index. When solving for "X*X != 5", for example, we
10738 // should not accept a root of 2.
10739 if (auto S = SolveQuadraticAddRecExact(AddRec, SE&: *this)) {
10740 const auto *R = cast<SCEVConstant>(Val: getConstant(Val: *S));
10741 return ExitLimit(R, R, R, false, Predicates);
10742 }
10743 return getCouldNotCompute();
10744 }
10745
10746 // Otherwise we can only handle this if it is affine.
10747 if (!AddRec->isAffine())
10748 return getCouldNotCompute();
10749
10750 // If this is an affine expression, the execution count of this branch is
10751 // the minimum unsigned root of the following equation:
10752 //
10753 // Start + Step*N = 0 (mod 2^BW)
10754 //
10755 // equivalent to:
10756 //
10757 // Step*N = -Start (mod 2^BW)
10758 //
10759 // where BW is the common bit width of Start and Step.
10760
10761 // Get the initial value for the loop.
10762 const SCEV *Start = getSCEVAtScope(V: AddRec->getStart(), L: L->getParentLoop());
10763 const SCEV *Step = getSCEVAtScope(V: AddRec->getOperand(i: 1), L: L->getParentLoop());
10764
10765 if (!isLoopInvariant(S: Step, L))
10766 return getCouldNotCompute();
10767
10768 LoopGuards Guards = LoopGuards::collect(L, SE&: *this);
10769 // Specialize step for this loop so we get context sensitive facts below.
10770 const SCEV *StepWLG = applyLoopGuards(Expr: Step, Guards);
10771
10772 // For positive steps (counting up until unsigned overflow):
10773 // N = -Start/Step (as unsigned)
10774 // For negative steps (counting down to zero):
10775 // N = Start/-Step
10776 // First compute the unsigned distance from zero in the direction of Step.
10777 bool CountDown = isKnownNegative(S: StepWLG);
10778 if (!CountDown && !isKnownNonNegative(S: StepWLG))
10779 return getCouldNotCompute();
10780
10781 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(V: Start);
10782 // Handle unitary steps, which cannot wraparound.
10783 // 1*N = -Start; -1*N = Start (mod 2^BW), so:
10784 // N = Distance (as unsigned)
10785
10786 if (match(S: Step, P: m_CombineOr(Ps: m_scev_One(), Ps: m_scev_AllOnes()))) {
10787 APInt MaxBECount = getUnsignedRangeMax(S: applyLoopGuards(Expr: Distance, Guards));
10788 MaxBECount = APIntOps::umin(A: MaxBECount, B: getUnsignedRangeMax(S: Distance));
10789
10790 // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated,
10791 // we end up with a loop whose backedge-taken count is n - 1. Detect this
10792 // case, and see if we can improve the bound.
10793 //
10794 // Explicitly handling this here is necessary because getUnsignedRange
10795 // isn't context-sensitive; it doesn't know that we only care about the
10796 // range inside the loop.
10797 const SCEV *Zero = getZero(Ty: Distance->getType());
10798 const SCEV *One = getOne(Ty: Distance->getType());
10799 const SCEV *DistancePlusOne = getAddExpr(LHS: Distance, RHS: One);
10800 if (isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: DistancePlusOne, RHS: Zero)) {
10801 // If Distance + 1 doesn't overflow, we can compute the maximum distance
10802 // as "unsigned_max(Distance + 1) - 1".
10803 ConstantRange CR = getUnsignedRange(S: DistancePlusOne);
10804 MaxBECount = APIntOps::umin(A: MaxBECount, B: CR.getUnsignedMax() - 1);
10805 }
10806 return ExitLimit(Distance, getConstant(Val: MaxBECount), Distance, false,
10807 Predicates);
10808 }
10809
10810 // If the condition controls loop exit (the loop exits only if the expression
10811 // is true) and the addition is no-wrap we can use unsigned divide to
10812 // compute the backedge count. In this case, the step may not divide the
10813 // distance, but we don't care because if the condition is "missed" the loop
10814 // will have undefined behavior due to wrapping.
10815 if (ControlsOnlyExit && AddRec->hasNoSelfWrap() &&
10816 loopHasNoAbnormalExits(L: AddRec->getLoop())) {
10817
10818 // If the stride is zero and the start is non-zero, the loop must be
10819 // infinite. In C++, most loops are finite by assumption, in which case the
10820 // step being zero implies UB must execute if the loop is entered.
10821 if (!(loopIsFiniteByAssumption(L) && isKnownNonZero(S: Start)) &&
10822 !isKnownNonZero(S: StepWLG))
10823 return getCouldNotCompute();
10824
10825 const SCEV *Exact =
10826 getUDivExpr(LHS: Distance, RHS: CountDown ? getNegativeSCEV(V: Step) : Step);
10827 const SCEV *ConstantMax = getCouldNotCompute();
10828 if (Exact != getCouldNotCompute()) {
10829 APInt MaxInt = getUnsignedRangeMax(S: applyLoopGuards(Expr: Exact, Guards));
10830 ConstantMax =
10831 getConstant(Val: APIntOps::umin(A: MaxInt, B: getUnsignedRangeMax(S: Exact)));
10832 }
10833 const SCEV *SymbolicMax =
10834 isa<SCEVCouldNotCompute>(Val: Exact) ? ConstantMax : Exact;
10835 return ExitLimit(Exact, ConstantMax, SymbolicMax, false, Predicates);
10836 }
10837
10838 // Solve the general equation.
10839 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Val: Step);
10840 if (!StepC || StepC->getValue()->isZero())
10841 return getCouldNotCompute();
10842 const SCEV *E = SolveLinEquationWithOverflow(
10843 A: StepC->getAPInt(), B: getNegativeSCEV(V: Start),
10844 Predicates: AllowPredicates ? &Predicates : nullptr, SE&: *this, L);
10845
10846 const SCEV *M = E;
10847 if (E != getCouldNotCompute()) {
10848 APInt MaxWithGuards = getUnsignedRangeMax(S: applyLoopGuards(Expr: E, Guards));
10849 M = getConstant(Val: APIntOps::umin(A: MaxWithGuards, B: getUnsignedRangeMax(S: E)));
10850 }
10851 auto *S = isa<SCEVCouldNotCompute>(Val: E) ? M : E;
10852 return ExitLimit(E, M, S, false, Predicates);
10853}
10854
10855ScalarEvolution::ExitLimit
10856ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) {
10857 // Loops that look like: while (X == 0) are very strange indeed. We don't
10858 // handle them yet except for the trivial case. This could be expanded in the
10859 // future as needed.
10860
10861 // If the value is a constant, check to see if it is known to be non-zero
10862 // already. If so, the backedge will execute zero times.
10863 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10864 if (!C->getValue()->isZero())
10865 return getZero(Ty: C->getType());
10866 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10867 }
10868
10869 // We could implement others, but I really doubt anyone writes loops like
10870 // this, and if they did, they would already be constant folded.
10871 return getCouldNotCompute();
10872}
10873
10874std::pair<const BasicBlock *, const BasicBlock *>
10875ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(const BasicBlock *BB)
10876 const {
10877 // If the block has a unique predecessor, then there is no path from the
10878 // predecessor to the block that does not go through the direct edge
10879 // from the predecessor to the block.
10880 if (const BasicBlock *Pred = BB->getSinglePredecessor())
10881 return {Pred, BB};
10882
10883 // A loop's header is defined to be a block that dominates the loop.
10884 // If the header has a unique predecessor outside the loop, it must be
10885 // a block that has exactly one successor that can reach the loop.
10886 if (const Loop *L = LI.getLoopFor(BB))
10887 return {L->getLoopPredecessor(), L->getHeader()};
10888
10889 return {nullptr, BB};
10890}
10891
10892/// SCEV structural equivalence is usually sufficient for testing whether two
10893/// expressions are equal, however for the purposes of looking for a condition
10894/// guarding a loop, it can be useful to be a little more general, since a
10895/// front-end may have replicated the controlling expression.
10896static bool HasSameValue(const SCEV *A, const SCEV *B) {
10897 // Quick check to see if they are the same SCEV.
10898 if (A == B) return true;
10899
10900 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) {
10901 // Not all instructions that are "identical" compute the same value. For
10902 // instance, two distinct alloca instructions allocating the same type are
10903 // identical and do not read memory; but compute distinct values.
10904 return A->isIdenticalTo(I: B) && (isa<BinaryOperator>(Val: A) || isa<GetElementPtrInst>(Val: A));
10905 };
10906
10907 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
10908 // two different instructions with the same value. Check for this case.
10909 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(Val: A))
10910 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(Val: B))
10911 if (const Instruction *AI = dyn_cast<Instruction>(Val: AU->getValue()))
10912 if (const Instruction *BI = dyn_cast<Instruction>(Val: BU->getValue()))
10913 if (ComputesEqualValues(AI, BI))
10914 return true;
10915
10916 // Otherwise assume they may have a different value.
10917 return false;
10918}
10919
10920static bool MatchBinarySub(const SCEV *S, SCEVUse &LHS, SCEVUse &RHS) {
10921 const SCEV *Op0, *Op1;
10922 if (!match(S, P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))))
10923 return false;
10924 if (match(S: Op0, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
10925 LHS = Op1;
10926 return true;
10927 }
10928 if (match(S: Op1, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
10929 LHS = Op0;
10930 return true;
10931 }
10932 return false;
10933}
10934
10935bool ScalarEvolution::SimplifyICmpOperands(CmpPredicate &Pred, SCEVUse &LHS,
10936 SCEVUse &RHS, unsigned Depth) {
10937 bool Changed = false;
10938 // Simplifies ICMP to trivial true or false by turning it into '0 == 0' or
10939 // '0 != 0'.
10940 auto TrivialCase = [&](bool TriviallyTrue) {
10941 LHS = RHS = getConstant(V: ConstantInt::getFalse(Context&: getContext()));
10942 Pred = TriviallyTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
10943 return true;
10944 };
10945 // If we hit the max recursion limit bail out.
10946 if (Depth >= 3)
10947 return false;
10948
10949 const SCEV *NewLHS, *NewRHS;
10950 if (match(U: LHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
10951 match(U: RHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale()))) {
10952 const SCEVMulExpr *LMul = cast<SCEVMulExpr>(Val&: LHS);
10953 const SCEVMulExpr *RMul = cast<SCEVMulExpr>(Val&: RHS);
10954
10955 // (X * vscale) pred (Y * vscale) ==> X pred Y
10956 // when both multiples are NSW.
10957 // (X * vscale) uicmp/eq/ne (Y * vscale) ==> X uicmp/eq/ne Y
10958 // when both multiples are NUW.
10959 if ((LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap()) ||
10960 (LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap() &&
10961 !ICmpInst::isSigned(Pred))) {
10962 LHS = NewLHS;
10963 RHS = NewRHS;
10964 Changed = true;
10965 }
10966 }
10967
10968 // Canonicalize a constant to the right side.
10969 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS)) {
10970 // Check for both operands constant.
10971 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
10972 if (!ICmpInst::compare(LHS: LHSC->getAPInt(), RHS: RHSC->getAPInt(), Pred))
10973 return TrivialCase(false);
10974 return TrivialCase(true);
10975 }
10976 // Otherwise swap the operands to put the constant on the right.
10977 std::swap(a&: LHS, b&: RHS);
10978 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
10979 Changed = true;
10980 }
10981
10982 // (K + A) pred (K + B) --> A pred B
10983 // For equality, no flags are needed.
10984 // For signed, both adds must be NSW. For unsigned, both must be NUW.
10985 {
10986 const SCEVConstant *C = nullptr;
10987 if (match(U: LHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: NewLHS))) &&
10988 match(U: RHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: NewRHS)))) {
10989 const auto *LAdd = cast<SCEVAddExpr>(Val&: LHS);
10990 const auto *RAdd = cast<SCEVAddExpr>(Val&: RHS);
10991 if (ICmpInst::isEquality(P: Pred) ||
10992 (ICmpInst::isSigned(Pred) && LAdd->hasNoSignedWrap() &&
10993 RAdd->hasNoSignedWrap()) ||
10994 (ICmpInst::isUnsigned(Pred) && LAdd->hasNoUnsignedWrap() &&
10995 RAdd->hasNoUnsignedWrap())) {
10996 LHS = NewLHS;
10997 RHS = NewRHS;
10998 Changed = true;
10999 }
11000 }
11001 }
11002
11003 // (C * A) pred (C * B) --> A pred B
11004 // For equality predicates, both muls must be NUW or both must be NSW
11005 // (either suffices to make multiplication by C injective; C == 0 is
11006 // impossible because SCEV folds 0 * X to 0).
11007 // For signed ordering, C must be positive and both muls must be NSW.
11008 // For unsigned ordering, both muls must be NUW.
11009 {
11010 const SCEVConstant *C = nullptr;
11011 if (match(U: LHS, P: m_scev_Mul(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: NewLHS))) &&
11012 match(U: RHS, P: m_scev_Mul(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: NewRHS)))) {
11013 const auto *LMul = cast<SCEVMulExpr>(Val&: LHS);
11014 const auto *RMul = cast<SCEVMulExpr>(Val&: RHS);
11015 bool BothNUW = LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap();
11016 bool BothNSW = LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap();
11017 if ((ICmpInst::isEquality(P: Pred) && (BothNUW || BothNSW)) ||
11018 (ICmpInst::isSigned(Pred) && BothNSW &&
11019 C->getAPInt().isStrictlyPositive()) ||
11020 (ICmpInst::isUnsigned(Pred) && BothNUW)) {
11021 LHS = NewLHS;
11022 RHS = NewRHS;
11023 Changed = true;
11024 }
11025 }
11026 }
11027
11028 // If we're comparing an addrec with a value which is loop-invariant in the
11029 // addrec's loop, put the addrec on the left. Also make a dominance check,
11030 // as both operands could be addrecs loop-invariant in each other's loop.
11031 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: RHS)) {
11032 const Loop *L = AR->getLoop();
11033 if (isLoopInvariant(S: LHS, L) && properlyDominates(S: LHS, BB: L->getHeader())) {
11034 std::swap(a&: LHS, b&: RHS);
11035 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11036 Changed = true;
11037 }
11038 }
11039
11040 // If there's a constant operand, canonicalize comparisons with boundary
11041 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
11042 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(Val&: RHS)) {
11043 const APInt &RA = RC->getAPInt();
11044
11045 bool SimplifiedByConstantRange = false;
11046
11047 if (!ICmpInst::isEquality(P: Pred)) {
11048 ConstantRange ExactCR = ConstantRange::makeExactICmpRegion(Pred, Other: RA);
11049 if (ExactCR.isFullSet())
11050 return TrivialCase(true);
11051 if (ExactCR.isEmptySet())
11052 return TrivialCase(false);
11053
11054 APInt NewRHS;
11055 CmpInst::Predicate NewPred;
11056 if (ExactCR.getEquivalentICmp(Pred&: NewPred, RHS&: NewRHS) &&
11057 ICmpInst::isEquality(P: NewPred)) {
11058 // We were able to convert an inequality to an equality.
11059 Pred = NewPred;
11060 RHS = getConstant(Val: NewRHS);
11061 Changed = SimplifiedByConstantRange = true;
11062 }
11063 }
11064
11065 if (!SimplifiedByConstantRange) {
11066 switch (Pred) {
11067 default:
11068 break;
11069 case ICmpInst::ICMP_EQ:
11070 case ICmpInst::ICMP_NE:
11071 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
11072 if (RA.isZero() && MatchBinarySub(S: LHS, LHS, RHS))
11073 Changed = true;
11074 break;
11075
11076 // The "Should have been caught earlier!" messages refer to the fact
11077 // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above
11078 // should have fired on the corresponding cases, and canonicalized the
11079 // check to trivial case.
11080
11081 case ICmpInst::ICMP_UGE:
11082 assert(!RA.isMinValue() && "Should have been caught earlier!");
11083 Pred = ICmpInst::ICMP_UGT;
11084 RHS = getConstant(Val: RA - 1);
11085 Changed = true;
11086 break;
11087 case ICmpInst::ICMP_ULE:
11088 assert(!RA.isMaxValue() && "Should have been caught earlier!");
11089 Pred = ICmpInst::ICMP_ULT;
11090 RHS = getConstant(Val: RA + 1);
11091 Changed = true;
11092 break;
11093 case ICmpInst::ICMP_SGE:
11094 assert(!RA.isMinSignedValue() && "Should have been caught earlier!");
11095 Pred = ICmpInst::ICMP_SGT;
11096 RHS = getConstant(Val: RA - 1);
11097 Changed = true;
11098 break;
11099 case ICmpInst::ICMP_SLE:
11100 assert(!RA.isMaxSignedValue() && "Should have been caught earlier!");
11101 Pred = ICmpInst::ICMP_SLT;
11102 RHS = getConstant(Val: RA + 1);
11103 Changed = true;
11104 break;
11105 }
11106 }
11107 }
11108
11109 // a /u b == 0 => a < b
11110 // a /u b != 0 => a >= b
11111 if (ICmpInst::isEquality(P: Pred) && RHS->isZero() &&
11112 match(U: LHS, P: m_scev_UDiv(Op0: m_SCEV(V&: LHS), Op1: m_SCEV(V&: RHS)))) {
11113 Pred = Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE;
11114 Changed = true;
11115 }
11116
11117 // Check for obvious equality.
11118 if (HasSameValue(A: LHS, B: RHS)) {
11119 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
11120 return TrivialCase(true);
11121 if (ICmpInst::isFalseWhenEqual(predicate: Pred))
11122 return TrivialCase(false);
11123 }
11124
11125 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
11126 // adding or subtracting 1 from one of the operands.
11127 switch (Pred) {
11128 case ICmpInst::ICMP_SLE:
11129 if (!getSignedRangeMax(S: RHS).isMaxSignedValue()) {
11130 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11131 Flags: SCEV::FlagNSW);
11132 Pred = ICmpInst::ICMP_SLT;
11133 Changed = true;
11134 } else if (!getSignedRangeMin(S: LHS).isMinSignedValue()) {
11135 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS,
11136 Flags: SCEV::FlagNSW);
11137 Pred = ICmpInst::ICMP_SLT;
11138 Changed = true;
11139 }
11140 break;
11141 case ICmpInst::ICMP_SGE:
11142 if (!getSignedRangeMin(S: RHS).isMinSignedValue()) {
11143 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS,
11144 Flags: SCEV::FlagNSW);
11145 Pred = ICmpInst::ICMP_SGT;
11146 Changed = true;
11147 } else if (!getSignedRangeMax(S: LHS).isMaxSignedValue()) {
11148 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11149 Flags: SCEV::FlagNSW);
11150 Pred = ICmpInst::ICMP_SGT;
11151 Changed = true;
11152 }
11153 break;
11154 case ICmpInst::ICMP_ULE:
11155 if (!getUnsignedRangeMax(S: RHS).isMaxValue()) {
11156 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11157 Flags: SCEV::FlagNUW);
11158 Pred = ICmpInst::ICMP_ULT;
11159 Changed = true;
11160 } else if (!getUnsignedRangeMin(S: LHS).isMinValue()) {
11161 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS);
11162 Pred = ICmpInst::ICMP_ULT;
11163 Changed = true;
11164 }
11165 break;
11166 case ICmpInst::ICMP_UGE:
11167 // If RHS is an op we can fold the -1, try that first.
11168 // Otherwise prefer LHS to preserve the nuw flag.
11169 if ((isa<SCEVConstant>(Val: RHS) ||
11170 (isa<SCEVAddExpr, SCEVAddRecExpr>(Val: RHS) &&
11171 isa<SCEVConstant>(Val: cast<SCEVNAryExpr>(Val&: RHS)->getOperand(i: 0)))) &&
11172 !getUnsignedRangeMin(S: RHS).isMinValue()) {
11173 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11174 Pred = ICmpInst::ICMP_UGT;
11175 Changed = true;
11176 } else if (!getUnsignedRangeMax(S: LHS).isMaxValue()) {
11177 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11178 Flags: SCEV::FlagNUW);
11179 Pred = ICmpInst::ICMP_UGT;
11180 Changed = true;
11181 } else if (!getUnsignedRangeMin(S: RHS).isMinValue()) {
11182 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11183 Pred = ICmpInst::ICMP_UGT;
11184 Changed = true;
11185 }
11186 break;
11187 default:
11188 break;
11189 }
11190
11191 // TODO: More simplifications are possible here.
11192
11193 // Recursively simplify until we either hit a recursion limit or nothing
11194 // changes.
11195 if (Changed)
11196 (void)SimplifyICmpOperands(Pred, LHS, RHS, Depth: Depth + 1);
11197
11198 return Changed;
11199}
11200
11201bool ScalarEvolution::isKnownNegative(const SCEV *S) {
11202 return getSignedRangeMax(S).isNegative();
11203}
11204
11205bool ScalarEvolution::isKnownPositive(const SCEV *S) {
11206 return getSignedRangeMin(S).isStrictlyPositive();
11207}
11208
11209bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
11210 return !getSignedRangeMin(S).isNegative();
11211}
11212
11213bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
11214 return !getSignedRangeMax(S).isStrictlyPositive();
11215}
11216
11217bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
11218 // Query push down for cases where the unsigned range is
11219 // less than sufficient.
11220 if (const auto *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
11221 return isKnownNonZero(S: SExt->getOperand(i: 0));
11222 return getUnsignedRangeMin(S) != 0;
11223}
11224
11225bool ScalarEvolution::isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero,
11226 bool OrNegative) {
11227 auto NonRecursive = [OrNegative](const SCEV *S) {
11228 if (auto *C = dyn_cast<SCEVConstant>(Val: S))
11229 return C->getAPInt().isPowerOf2() ||
11230 (OrNegative && C->getAPInt().isNegatedPowerOf2());
11231
11232 // vscale is a power-of-two.
11233 return isa<SCEVVScale>(Val: S);
11234 };
11235
11236 if (NonRecursive(S))
11237 return true;
11238
11239 auto *Mul = dyn_cast<SCEVMulExpr>(Val: S);
11240 if (!Mul)
11241 return false;
11242 return all_of(Range: Mul->operands(), P: NonRecursive) && (OrZero || isKnownNonZero(S));
11243}
11244
11245bool ScalarEvolution::isKnownMultipleOf(
11246 const SCEV *S, uint64_t M,
11247 SmallVectorImpl<const SCEVPredicate *> &Assumptions) {
11248 if (M == 0)
11249 return false;
11250 if (M == 1)
11251 return true;
11252
11253 // Recursively check AddRec operands. An AddRecExpr S is a multiple of M if S
11254 // starts with a multiple of M and at every iteration step S only adds
11255 // multiples of M.
11256 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
11257 return isKnownMultipleOf(S: AddRec->getStart(), M, Assumptions) &&
11258 isKnownMultipleOf(S: AddRec->getStepRecurrence(SE&: *this), M, Assumptions);
11259
11260 // For a constant, check that "S % M == 0".
11261 if (auto *Cst = dyn_cast<SCEVConstant>(Val: S)) {
11262 APInt C = Cst->getAPInt();
11263 return C.urem(RHS: M) == 0;
11264 }
11265
11266 // TODO: Also check other SCEV expressions, i.e., SCEVAddRecExpr, etc.
11267
11268 // Basic tests have failed.
11269 // Check "S % M == 0" at compile time and record runtime Assumptions.
11270 auto *STy = dyn_cast<IntegerType>(Val: S->getType());
11271 const SCEV *SmodM =
11272 getURemExpr(LHS: S, RHS: getConstant(V: ConstantInt::get(Ty: STy, V: M, IsSigned: false)));
11273 const SCEV *Zero = getZero(Ty: STy);
11274
11275 // Check whether "S % M == 0" is known at compile time.
11276 if (isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero))
11277 return true;
11278
11279 // Check whether "S % M != 0" is known at compile time.
11280 if (isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: SmodM, RHS: Zero))
11281 return false;
11282
11283 const SCEVPredicate *P = getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero);
11284
11285 // Detect redundant predicates.
11286 for (auto *A : Assumptions)
11287 if (A->implies(N: P, SE&: *this))
11288 return true;
11289
11290 // Only record non-redundant predicates.
11291 Assumptions.push_back(Elt: P);
11292 return true;
11293}
11294
11295bool ScalarEvolution::haveSameSign(const SCEV *S1, const SCEV *S2) {
11296 return ((isKnownNonNegative(S: S1) && isKnownNonNegative(S: S2)) ||
11297 (isKnownNegative(S: S1) && isKnownNegative(S: S2)));
11298}
11299
11300std::pair<const SCEV *, const SCEV *>
11301ScalarEvolution::SplitIntoInitAndPostInc(const Loop *L, const SCEV *S) {
11302 // Compute SCEV on entry of loop L.
11303 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, SE&: *this);
11304 if (Start == getCouldNotCompute())
11305 return { Start, Start };
11306 // Compute post increment SCEV for loop L.
11307 const SCEV *PostInc = SCEVPostIncRewriter::rewrite(S, L, SE&: *this);
11308 assert(PostInc != getCouldNotCompute() && "Unexpected could not compute");
11309 return { Start, PostInc };
11310}
11311
11312bool ScalarEvolution::isKnownViaInduction(CmpPredicate Pred, SCEVUse LHS,
11313 SCEVUse RHS) {
11314 // First collect all loops.
11315 SmallPtrSet<const Loop *, 8> LoopsUsed;
11316 getUsedLoops(S: LHS, LoopsUsed);
11317 getUsedLoops(S: RHS, LoopsUsed);
11318
11319 if (LoopsUsed.empty())
11320 return false;
11321
11322 // Domination relationship must be a linear order on collected loops.
11323#ifndef NDEBUG
11324 for (const auto *L1 : LoopsUsed)
11325 for (const auto *L2 : LoopsUsed)
11326 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||
11327 DT.dominates(L2->getHeader(), L1->getHeader())) &&
11328 "Domination relationship is not a linear order");
11329#endif
11330
11331 const Loop *MDL =
11332 *llvm::max_element(Range&: LoopsUsed, C: [&](const Loop *L1, const Loop *L2) {
11333 return DT.properlyDominates(A: L1->getHeader(), B: L2->getHeader());
11334 });
11335
11336 // Get init and post increment value for LHS.
11337 auto SplitLHS = SplitIntoInitAndPostInc(L: MDL, S: LHS);
11338 // if LHS contains unknown non-invariant SCEV then bail out.
11339 if (SplitLHS.first == getCouldNotCompute())
11340 return false;
11341 assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
11342 // Get init and post increment value for RHS.
11343 auto SplitRHS = SplitIntoInitAndPostInc(L: MDL, S: RHS);
11344 // if RHS contains unknown non-invariant SCEV then bail out.
11345 if (SplitRHS.first == getCouldNotCompute())
11346 return false;
11347 assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
11348 // It is possible that init SCEV contains an invariant load but it does
11349 // not dominate MDL and is not available at MDL loop entry, so we should
11350 // check it here.
11351 if (!isAvailableAtLoopEntry(S: SplitLHS.first, L: MDL) ||
11352 !isAvailableAtLoopEntry(S: SplitRHS.first, L: MDL))
11353 return false;
11354
11355 // It seems backedge guard check is faster than entry one so in some cases
11356 // it can speed up whole estimation by short circuit
11357 return isLoopBackedgeGuardedByCond(L: MDL, Pred, LHS: SplitLHS.second,
11358 RHS: SplitRHS.second) &&
11359 isLoopEntryGuardedByCond(L: MDL, Pred, LHS: SplitLHS.first, RHS: SplitRHS.first);
11360}
11361
11362bool ScalarEvolution::isKnownPredicate(CmpPredicate Pred, SCEVUse LHS,
11363 SCEVUse RHS) {
11364 // Canonicalize the inputs first.
11365 (void)SimplifyICmpOperands(Pred, LHS, RHS);
11366
11367 if (isKnownViaInduction(Pred, LHS, RHS))
11368 return true;
11369
11370 if (isKnownPredicateViaSplitting(Pred, LHS, RHS))
11371 return true;
11372
11373 // Otherwise see what can be done with some simple reasoning.
11374 return isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
11375}
11376
11377std::optional<bool> ScalarEvolution::evaluatePredicate(CmpPredicate Pred,
11378 const SCEV *LHS,
11379 const SCEV *RHS) {
11380 if (isKnownPredicate(Pred, LHS, RHS))
11381 return true;
11382 if (isKnownPredicate(Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11383 return false;
11384 return std::nullopt;
11385}
11386
11387bool ScalarEvolution::isKnownPredicateAt(CmpPredicate Pred, const SCEV *LHS,
11388 const SCEV *RHS,
11389 const Instruction *CtxI) {
11390 // TODO: Analyze guards and assumes from Context's block.
11391 return isKnownPredicate(Pred, LHS, RHS) ||
11392 isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS);
11393}
11394
11395std::optional<bool>
11396ScalarEvolution::evaluatePredicateAt(CmpPredicate Pred, const SCEV *LHS,
11397 const SCEV *RHS, const Instruction *CtxI) {
11398 std::optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
11399 if (KnownWithoutContext)
11400 return KnownWithoutContext;
11401
11402 if (isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS))
11403 return true;
11404 if (isBasicBlockEntryGuardedByCond(
11405 BB: CtxI->getParent(), Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11406 return false;
11407 return std::nullopt;
11408}
11409
11410bool ScalarEvolution::isKnownOnEveryIteration(CmpPredicate Pred,
11411 const SCEVAddRecExpr *LHS,
11412 const SCEV *RHS) {
11413 const Loop *L = LHS->getLoop();
11414 return isLoopEntryGuardedByCond(L, Pred, LHS: LHS->getStart(), RHS) &&
11415 isLoopBackedgeGuardedByCond(L, Pred, LHS: LHS->getPostIncExpr(SE&: *this), RHS);
11416}
11417
11418std::optional<ScalarEvolution::MonotonicPredicateType>
11419ScalarEvolution::getMonotonicPredicateType(const SCEVAddRecExpr *LHS,
11420 ICmpInst::Predicate Pred) {
11421 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
11422
11423#ifndef NDEBUG
11424 // Verify an invariant: inverting the predicate should turn a monotonically
11425 // increasing change to a monotonically decreasing one, and vice versa.
11426 if (Result) {
11427 auto ResultSwapped =
11428 getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred));
11429
11430 assert(*ResultSwapped != *Result &&
11431 "monotonicity should flip as we flip the predicate");
11432 }
11433#endif
11434
11435 return Result;
11436}
11437
11438std::optional<ScalarEvolution::MonotonicPredicateType>
11439ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
11440 ICmpInst::Predicate Pred) {
11441 // A zero step value for LHS means the induction variable is essentially a
11442 // loop invariant value. We don't really depend on the predicate actually
11443 // flipping from false to true (for increasing predicates, and the other way
11444 // around for decreasing predicates), all we care about is that *if* the
11445 // predicate changes then it only changes from false to true.
11446 //
11447 // A zero step value in itself is not very useful, but there may be places
11448 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
11449 // as general as possible.
11450
11451 // Only handle LE/LT/GE/GT predicates.
11452 if (!ICmpInst::isRelational(P: Pred))
11453 return std::nullopt;
11454
11455 bool IsGreater = ICmpInst::isGE(P: Pred) || ICmpInst::isGT(P: Pred);
11456 assert((IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) &&
11457 "Should be greater or less!");
11458
11459 // Check that AR does not wrap.
11460 if (ICmpInst::isUnsigned(Pred)) {
11461 if (!LHS->hasNoUnsignedWrap())
11462 return std::nullopt;
11463 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11464 }
11465 assert(ICmpInst::isSigned(Pred) &&
11466 "Relational predicate is either signed or unsigned!");
11467 if (!LHS->hasNoSignedWrap())
11468 return std::nullopt;
11469
11470 const SCEV *Step = LHS->getStepRecurrence(SE&: *this);
11471
11472 if (isKnownNonNegative(S: Step))
11473 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11474
11475 if (isKnownNonPositive(S: Step))
11476 return !IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11477
11478 return std::nullopt;
11479}
11480
11481std::optional<ScalarEvolution::LoopInvariantPredicate>
11482ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
11483 const SCEV *RHS, const Loop *L,
11484 const Instruction *CtxI) {
11485 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11486 if (!isLoopInvariant(S: RHS, L)) {
11487 if (!isLoopInvariant(S: LHS, L))
11488 return std::nullopt;
11489
11490 std::swap(a&: LHS, b&: RHS);
11491 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11492 }
11493
11494 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11495 if (!ArLHS || ArLHS->getLoop() != L)
11496 return std::nullopt;
11497
11498 auto MonotonicType = getMonotonicPredicateType(LHS: ArLHS, Pred);
11499 if (!MonotonicType)
11500 return std::nullopt;
11501 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
11502 // true as the loop iterates, and the backedge is control dependent on
11503 // "ArLHS `Pred` RHS" == true then we can reason as follows:
11504 //
11505 // * if the predicate was false in the first iteration then the predicate
11506 // is never evaluated again, since the loop exits without taking the
11507 // backedge.
11508 // * if the predicate was true in the first iteration then it will
11509 // continue to be true for all future iterations since it is
11510 // monotonically increasing.
11511 //
11512 // For both the above possibilities, we can replace the loop varying
11513 // predicate with its value on the first iteration of the loop (which is
11514 // loop invariant).
11515 //
11516 // A similar reasoning applies for a monotonically decreasing predicate, by
11517 // replacing true with false and false with true in the above two bullets.
11518 bool Increasing = *MonotonicType == ScalarEvolution::MonotonicallyIncreasing;
11519 auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
11520
11521 if (isLoopBackedgeGuardedByCond(L, Pred: P, LHS, RHS))
11522 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11523 RHS);
11524
11525 if (!CtxI)
11526 return std::nullopt;
11527 // Try to prove via context.
11528 // TODO: Support other cases.
11529 switch (Pred) {
11530 default:
11531 break;
11532 case ICmpInst::ICMP_ULE:
11533 case ICmpInst::ICMP_ULT: {
11534 assert(ArLHS->hasNoUnsignedWrap() && "Is a requirement of monotonicity!");
11535 // Given preconditions
11536 // (1) ArLHS does not cross the border of positive and negative parts of
11537 // range because of:
11538 // - Positive step; (TODO: lift this limitation)
11539 // - nuw - does not cross zero boundary;
11540 // - nsw - does not cross SINT_MAX boundary;
11541 // (2) ArLHS <s RHS
11542 // (3) RHS >=s 0
11543 // we can replace the loop variant ArLHS <u RHS condition with loop
11544 // invariant Start(ArLHS) <u RHS.
11545 //
11546 // Because of (1) there are two options:
11547 // - ArLHS is always negative. It means that ArLHS <u RHS is always false;
11548 // - ArLHS is always non-negative. Because of (3) RHS is also non-negative.
11549 // It means that ArLHS <s RHS <=> ArLHS <u RHS.
11550 // Because of (2) ArLHS <u RHS is trivially true.
11551 // All together it means that ArLHS <u RHS <=> Start(ArLHS) >=s 0.
11552 // We can strengthen this to Start(ArLHS) <u RHS.
11553 auto SignFlippedPred = ICmpInst::getFlippedSignednessPredicate(Pred);
11554 if (ArLHS->hasNoSignedWrap() && ArLHS->isAffine() &&
11555 isKnownPositive(S: ArLHS->getStepRecurrence(SE&: *this)) &&
11556 isKnownNonNegative(S: RHS) &&
11557 isKnownPredicateAt(Pred: SignFlippedPred, LHS: ArLHS, RHS, CtxI))
11558 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11559 RHS);
11560 }
11561 }
11562
11563 return std::nullopt;
11564}
11565
11566std::optional<ScalarEvolution::LoopInvariantPredicate>
11567ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations(
11568 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11569 const Instruction *CtxI, const SCEV *MaxIter) {
11570 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11571 Pred, LHS, RHS, L, CtxI, MaxIter))
11572 return LIP;
11573 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: MaxIter))
11574 // Number of iterations expressed as UMIN isn't always great for expressing
11575 // the value on the last iteration. If the straightforward approach didn't
11576 // work, try the following trick: if the a predicate is invariant for X, it
11577 // is also invariant for umin(X, ...). So try to find something that works
11578 // among subexpressions of MaxIter expressed as umin.
11579 for (SCEVUse Op : UMin->operands())
11580 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11581 Pred, LHS, RHS, L, CtxI, MaxIter: Op))
11582 return LIP;
11583 return std::nullopt;
11584}
11585
11586std::optional<ScalarEvolution::LoopInvariantPredicate>
11587ScalarEvolution::getLoopInvariantExitCondDuringFirstIterationsImpl(
11588 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11589 const Instruction *CtxI, const SCEV *MaxIter) {
11590 // Try to prove the following set of facts:
11591 // - The predicate is monotonic in the iteration space.
11592 // - If the check does not fail on the 1st iteration:
11593 // - No overflow will happen during first MaxIter iterations;
11594 // - It will not fail on the MaxIter'th iteration.
11595 // If the check does fail on the 1st iteration, we leave the loop and no
11596 // other checks matter.
11597
11598 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11599 if (!isLoopInvariant(S: RHS, L)) {
11600 if (!isLoopInvariant(S: LHS, L))
11601 return std::nullopt;
11602
11603 std::swap(a&: LHS, b&: RHS);
11604 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11605 }
11606
11607 auto *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11608 if (!AR || AR->getLoop() != L)
11609 return std::nullopt;
11610
11611 // Even if both are valid, we need to consistently chose the unsigned or the
11612 // signed predicate below, not mixtures of both. For now, prefer the unsigned
11613 // predicate.
11614 Pred = Pred.dropSameSign();
11615
11616 // The predicate must be relational (i.e. <, <=, >=, >).
11617 if (!ICmpInst::isRelational(P: Pred))
11618 return std::nullopt;
11619
11620 // TODO: Support steps other than +/- 1.
11621 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
11622 auto *One = getOne(Ty: Step->getType());
11623 auto *MinusOne = getNegativeSCEV(V: One);
11624 if (Step != One && Step != MinusOne)
11625 return std::nullopt;
11626
11627 // Type mismatch here means that MaxIter is potentially larger than max
11628 // unsigned value in start type, which mean we cannot prove no wrap for the
11629 // indvar.
11630 if (AR->getType() != MaxIter->getType())
11631 return std::nullopt;
11632
11633 // Value of IV on suggested last iteration.
11634 const SCEV *Last = AR->evaluateAtIteration(It: MaxIter, SE&: *this);
11635 // Does it still meet the requirement?
11636 if (!isLoopBackedgeGuardedByCond(L, Pred, LHS: Last, RHS))
11637 return std::nullopt;
11638 // Because step is +/- 1 and MaxIter has same type as Start (i.e. it does
11639 // not exceed max unsigned value of this type), this effectively proves
11640 // that there is no wrap during the iteration. To prove that there is no
11641 // signed/unsigned wrap, we need to check that
11642 // Start <= Last for step = 1 or Start >= Last for step = -1.
11643 ICmpInst::Predicate NoOverflowPred =
11644 CmpInst::isSigned(Pred) ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
11645 if (Step == MinusOne)
11646 NoOverflowPred = ICmpInst::getSwappedPredicate(pred: NoOverflowPred);
11647 const SCEV *Start = AR->getStart();
11648 if (!isKnownPredicateAt(Pred: NoOverflowPred, LHS: Start, RHS: Last, CtxI))
11649 return std::nullopt;
11650
11651 // Everything is fine.
11652 return ScalarEvolution::LoopInvariantPredicate(Pred, Start, RHS);
11653}
11654
11655bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
11656 SCEVUse LHS,
11657 SCEVUse RHS) {
11658 if (HasSameValue(A: LHS, B: RHS))
11659 return ICmpInst::isTrueWhenEqual(predicate: Pred);
11660
11661 auto CheckRange = [&](bool IsSigned) {
11662 auto RangeLHS = IsSigned ? getSignedRange(S: LHS) : getUnsignedRange(S: LHS);
11663 auto RangeRHS = IsSigned ? getSignedRange(S: RHS) : getUnsignedRange(S: RHS);
11664 return RangeLHS.icmp(Pred, Other: RangeRHS);
11665 };
11666
11667 // The check at the top of the function catches the case where the values are
11668 // known to be equal.
11669 if (Pred == CmpInst::ICMP_EQ)
11670 return false;
11671
11672 if (Pred == CmpInst::ICMP_NE) {
11673 if (CheckRange(true) || CheckRange(false))
11674 return true;
11675 auto *Diff = getMinusSCEV(LHS, RHS);
11676 return !isa<SCEVCouldNotCompute>(Val: Diff) && isKnownNonZero(S: Diff);
11677 }
11678
11679 return CheckRange(CmpInst::isSigned(Pred));
11680}
11681
11682bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
11683 SCEVUse LHS, SCEVUse RHS) {
11684 // Match X to (A + C1)<ExpectedFlags> and Y to (A + C2)<ExpectedFlags>, where
11685 // C1 and C2 are constant integers. If either X or Y are not add expressions,
11686 // consider them as X + 0 and Y + 0 respectively. C1 and C2 are returned via
11687 // OutC1 and OutC2.
11688 auto MatchBinaryAddToConst = [this](SCEVUse X, SCEVUse Y, APInt &OutC1,
11689 APInt &OutC2,
11690 SCEV::NoWrapFlags ExpectedFlags) {
11691 SCEVUse XNonConstOp, XConstOp;
11692 SCEVUse YNonConstOp, YConstOp;
11693 SCEV::NoWrapFlags XFlagsPresent;
11694 SCEV::NoWrapFlags YFlagsPresent;
11695
11696 if (!splitBinaryAdd(Expr: X, L&: XConstOp, R&: XNonConstOp, Flags&: XFlagsPresent)) {
11697 XConstOp = getZero(Ty: X->getType());
11698 XNonConstOp = X;
11699 XFlagsPresent = ExpectedFlags;
11700 }
11701 if (!isa<SCEVConstant>(Val: XConstOp))
11702 return false;
11703
11704 if (!splitBinaryAdd(Expr: Y, L&: YConstOp, R&: YNonConstOp, Flags&: YFlagsPresent)) {
11705 YConstOp = getZero(Ty: Y->getType());
11706 YNonConstOp = Y;
11707 YFlagsPresent = ExpectedFlags;
11708 }
11709
11710 if (YNonConstOp != XNonConstOp)
11711 return false;
11712
11713 if (!isa<SCEVConstant>(Val: YConstOp))
11714 return false;
11715
11716 // When matching ADDs with NUW flags (and unsigned predicates), only the
11717 // second ADD (with the larger constant) requires NUW.
11718 if ((YFlagsPresent & ExpectedFlags) != ExpectedFlags)
11719 return false;
11720 if (ExpectedFlags != SCEV::FlagNUW &&
11721 (XFlagsPresent & ExpectedFlags) != ExpectedFlags) {
11722 return false;
11723 }
11724
11725 OutC1 = cast<SCEVConstant>(Val&: XConstOp)->getAPInt();
11726 OutC2 = cast<SCEVConstant>(Val&: YConstOp)->getAPInt();
11727
11728 return true;
11729 };
11730
11731 APInt C1;
11732 APInt C2;
11733
11734 switch (Pred) {
11735 default:
11736 break;
11737
11738 case ICmpInst::ICMP_SGE:
11739 std::swap(a&: LHS, b&: RHS);
11740 [[fallthrough]];
11741 case ICmpInst::ICMP_SLE:
11742 // (X + C1)<nsw> s<= (X + C2)<nsw> if C1 s<= C2.
11743 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.sle(RHS: C2))
11744 return true;
11745
11746 break;
11747
11748 case ICmpInst::ICMP_SGT:
11749 std::swap(a&: LHS, b&: RHS);
11750 [[fallthrough]];
11751 case ICmpInst::ICMP_SLT:
11752 // (X + C1)<nsw> s< (X + C2)<nsw> if C1 s< C2.
11753 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.slt(RHS: C2))
11754 return true;
11755
11756 break;
11757
11758 case ICmpInst::ICMP_UGE:
11759 std::swap(a&: LHS, b&: RHS);
11760 [[fallthrough]];
11761 case ICmpInst::ICMP_ULE:
11762 // (X + C1) u<= (X + C2)<nuw> for C1 u<= C2.
11763 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(RHS: C2))
11764 return true;
11765
11766 break;
11767
11768 case ICmpInst::ICMP_UGT:
11769 std::swap(a&: LHS, b&: RHS);
11770 [[fallthrough]];
11771 case ICmpInst::ICMP_ULT:
11772 // (X + C1) u< (X + C2)<nuw> if C1 u< C2.
11773 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(RHS: C2))
11774 return true;
11775 break;
11776 }
11777
11778 return false;
11779}
11780
11781bool ScalarEvolution::isKnownPredicateViaSplitting(CmpPredicate Pred,
11782 SCEVUse LHS, SCEVUse RHS) {
11783 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate)
11784 return false;
11785
11786 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on
11787 // the stack can result in exponential time complexity.
11788 SaveAndRestore Restore(ProvingSplitPredicate, true);
11789
11790 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L
11791 //
11792 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use
11793 // isKnownPredicate. isKnownPredicate is more powerful, but also more
11794 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the
11795 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to
11796 // use isKnownPredicate later if needed.
11797 return isKnownNonNegative(S: RHS) &&
11798 isKnownPredicate(Pred: CmpInst::ICMP_SGE, LHS, RHS: getZero(Ty: LHS->getType())) &&
11799 isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS, RHS);
11800}
11801
11802bool ScalarEvolution::isImpliedViaGuard(const BasicBlock *BB, CmpPredicate Pred,
11803 const SCEV *LHS, const SCEV *RHS) {
11804 // No need to even try if we know the module has no guards.
11805 if (!HasGuards)
11806 return false;
11807
11808 return any_of(Range: *BB, P: [&](const Instruction &I) {
11809 using namespace llvm::PatternMatch;
11810
11811 Value *Condition;
11812 return match(V: &I, P: m_Intrinsic<Intrinsic::experimental_guard>(
11813 Ops: m_Value(V&: Condition))) &&
11814 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse: false);
11815 });
11816}
11817
11818/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
11819/// protected by a conditional between LHS and RHS. This is used to
11820/// to eliminate casts.
11821bool ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
11822 CmpPredicate Pred,
11823 const SCEV *LHS,
11824 const SCEV *RHS) {
11825 // Interpret a null as meaning no loop, where there is obviously no guard
11826 // (interprocedural conditions notwithstanding). Do not bother about
11827 // unreachable loops.
11828 if (!L || !DT.isReachableFromEntry(A: L->getHeader()))
11829 return true;
11830
11831 if (VerifyIR)
11832 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) &&
11833 "This cannot be done on broken IR!");
11834
11835
11836 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11837 return true;
11838
11839 BasicBlock *Latch = L->getLoopLatch();
11840 if (!Latch)
11841 return false;
11842
11843 CondBrInst *LoopContinuePredicate =
11844 dyn_cast<CondBrInst>(Val: Latch->getTerminator());
11845 if (LoopContinuePredicate &&
11846 isImpliedCond(Pred, LHS, RHS, FoundCondValue: LoopContinuePredicate->getCondition(),
11847 Inverse: LoopContinuePredicate->getSuccessor(i: 0) != L->getHeader()))
11848 return true;
11849
11850 // We don't want more than one activation of the following loops on the stack
11851 // -- that can lead to O(n!) time complexity.
11852 if (WalkingBEDominatingConds)
11853 return false;
11854
11855 SaveAndRestore ClearOnExit(WalkingBEDominatingConds, true);
11856
11857 // See if we can exploit a trip count to prove the predicate.
11858 const auto &BETakenInfo = getBackedgeTakenInfo(L);
11859 const SCEV *LatchBECount = BETakenInfo.getExact(ExitingBlock: Latch, SE: this);
11860 if (LatchBECount != getCouldNotCompute()) {
11861 // We know that Latch branches back to the loop header exactly
11862 // LatchBECount times. This means the backdege condition at Latch is
11863 // equivalent to "{0,+,1} u< LatchBECount".
11864 Type *Ty = LatchBECount->getType();
11865 auto NoWrapFlags = SCEV::NoWrapFlags(SCEV::FlagNUW | SCEV::FlagNW);
11866 const SCEV *LoopCounter =
11867 getAddRecExpr(Start: getZero(Ty), Step: getOne(Ty), L, Flags: NoWrapFlags);
11868 if (isImpliedCond(Pred, LHS, RHS, FoundPred: ICmpInst::ICMP_ULT, FoundLHS: LoopCounter,
11869 FoundRHS: LatchBECount))
11870 return true;
11871 }
11872
11873 // Check conditions due to any @llvm.assume intrinsics.
11874 for (auto &AssumeVH : AC.assumptions()) {
11875 if (!AssumeVH)
11876 continue;
11877 auto *CI = cast<CallInst>(Val&: AssumeVH);
11878 if (!DT.dominates(Def: CI, User: Latch->getTerminator()))
11879 continue;
11880
11881 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: CI->getArgOperand(i: 0), Inverse: false))
11882 return true;
11883 }
11884
11885 if (isImpliedViaGuard(BB: Latch, Pred, LHS, RHS))
11886 return true;
11887
11888 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
11889 DTN != HeaderDTN; DTN = DTN->getIDom()) {
11890 assert(DTN && "should reach the loop header before reaching the root!");
11891
11892 BasicBlock *BB = DTN->getBlock();
11893 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
11894 return true;
11895
11896 BasicBlock *PBB = BB->getSinglePredecessor();
11897 if (!PBB)
11898 continue;
11899
11900 CondBrInst *ContBr = dyn_cast<CondBrInst>(Val: PBB->getTerminator());
11901 if (!ContBr || ContBr->getSuccessor(i: 0) == ContBr->getSuccessor(i: 1))
11902 continue;
11903
11904 // If we have an edge `E` within the loop body that dominates the only
11905 // latch, the condition guarding `E` also guards the backedge. This
11906 // reasoning works only for loops with a single latch.
11907 // We're constructively (and conservatively) enumerating edges within the
11908 // loop body that dominate the latch. The dominator tree better agree
11909 // with us on this:
11910 assert(DT.dominates(BasicBlockEdge(PBB, BB), Latch) && "should be!");
11911 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: ContBr->getCondition(),
11912 Inverse: BB != ContBr->getSuccessor(i: 0)))
11913 return true;
11914 }
11915
11916 return false;
11917}
11918
11919bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
11920 CmpPredicate Pred,
11921 const SCEV *LHS,
11922 const SCEV *RHS) {
11923 // Do not bother proving facts for unreachable code.
11924 if (!DT.isReachableFromEntry(A: BB))
11925 return true;
11926 if (VerifyIR)
11927 assert(!verifyFunction(*BB->getParent(), &dbgs()) &&
11928 "This cannot be done on broken IR!");
11929
11930 // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
11931 // the facts (a >= b && a != b) separately. A typical situation is when the
11932 // non-strict comparison is known from ranges and non-equality is known from
11933 // dominating predicates. If we are proving strict comparison, we always try
11934 // to prove non-equality and non-strict comparison separately.
11935 CmpPredicate NonStrictPredicate = ICmpInst::getNonStrictCmpPredicate(Pred);
11936 const bool ProvingStrictComparison =
11937 Pred != NonStrictPredicate.dropSameSign();
11938 bool ProvedNonStrictComparison = false;
11939 bool ProvedNonEquality = false;
11940
11941 auto SplitAndProve = [&](std::function<bool(CmpPredicate)> Fn) -> bool {
11942 if (!ProvedNonStrictComparison)
11943 ProvedNonStrictComparison = Fn(NonStrictPredicate);
11944 if (!ProvedNonEquality)
11945 ProvedNonEquality = Fn(ICmpInst::ICMP_NE);
11946 if (ProvedNonStrictComparison && ProvedNonEquality)
11947 return true;
11948 return false;
11949 };
11950
11951 if (ProvingStrictComparison) {
11952 auto ProofFn = [&](CmpPredicate P) {
11953 return isKnownViaNonRecursiveReasoning(Pred: P, LHS, RHS);
11954 };
11955 if (SplitAndProve(ProofFn))
11956 return true;
11957 }
11958
11959 // Try to prove (Pred, LHS, RHS) using isImpliedCond.
11960 auto ProveViaCond = [&](const Value *Condition, bool Inverse) {
11961 const Instruction *CtxI = &BB->front();
11962 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI))
11963 return true;
11964 if (ProvingStrictComparison) {
11965 auto ProofFn = [&](CmpPredicate P) {
11966 return isImpliedCond(Pred: P, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI);
11967 };
11968 if (SplitAndProve(ProofFn))
11969 return true;
11970 }
11971 return false;
11972 };
11973
11974 // Starting at the block's predecessor, climb up the predecessor chain, as long
11975 // as there are predecessors that can be found that have unique successors
11976 // leading to the original block.
11977 const Loop *ContainingLoop = LI.getLoopFor(BB);
11978 const BasicBlock *PredBB;
11979 if (ContainingLoop && ContainingLoop->getHeader() == BB)
11980 PredBB = ContainingLoop->getLoopPredecessor();
11981 else
11982 PredBB = BB->getSinglePredecessor();
11983 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
11984 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
11985 const CondBrInst *BlockEntryPredicate =
11986 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
11987 if (!BlockEntryPredicate)
11988 continue;
11989
11990 if (ProveViaCond(BlockEntryPredicate->getCondition(),
11991 BlockEntryPredicate->getSuccessor(i: 0) != Pair.second))
11992 return true;
11993 }
11994
11995 // Check conditions due to any @llvm.assume intrinsics.
11996 for (auto &AssumeVH : AC.assumptions()) {
11997 if (!AssumeVH)
11998 continue;
11999 auto *CI = cast<CallInst>(Val&: AssumeVH);
12000 if (!DT.dominates(Def: CI, BB))
12001 continue;
12002
12003 if (ProveViaCond(CI->getArgOperand(i: 0), false))
12004 return true;
12005 }
12006
12007 // Check conditions due to any @llvm.experimental.guard intrinsics.
12008 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
12009 M: F.getParent(), id: Intrinsic::experimental_guard);
12010 if (GuardDecl)
12011 for (const auto *GU : GuardDecl->users())
12012 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
12013 if (Guard->getFunction() == BB->getParent() && DT.dominates(Def: Guard, BB))
12014 if (ProveViaCond(Guard->getArgOperand(i: 0), false))
12015 return true;
12016 return false;
12017}
12018
12019bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred,
12020 const SCEV *LHS,
12021 const SCEV *RHS) {
12022 // Interpret a null as meaning no loop, where there is obviously no guard
12023 // (interprocedural conditions notwithstanding).
12024 if (!L)
12025 return false;
12026
12027 // Both LHS and RHS must be available at loop entry.
12028 assert(isAvailableAtLoopEntry(LHS, L) &&
12029 "LHS is not available at Loop Entry");
12030 assert(isAvailableAtLoopEntry(RHS, L) &&
12031 "RHS is not available at Loop Entry");
12032
12033 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
12034 return true;
12035
12036 return isBasicBlockEntryGuardedByCond(BB: L->getHeader(), Pred, LHS, RHS);
12037}
12038
12039bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12040 const SCEV *RHS,
12041 const Value *FoundCondValue, bool Inverse,
12042 const Instruction *CtxI) {
12043 // False conditions implies anything. Do not bother analyzing it further.
12044 if (FoundCondValue ==
12045 ConstantInt::getBool(Context&: FoundCondValue->getContext(), V: Inverse))
12046 return true;
12047
12048 if (!PendingLoopPredicates.insert(Ptr: FoundCondValue).second)
12049 return false;
12050
12051 llvm::scope_exit ClearOnExit(
12052 [&]() { PendingLoopPredicates.erase(Ptr: FoundCondValue); });
12053
12054 // Recursively handle And and Or conditions.
12055 const Value *Op0, *Op1;
12056 if (match(V: FoundCondValue, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12057 if (!Inverse)
12058 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12059 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12060 } else if (match(V: FoundCondValue, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12061 if (Inverse)
12062 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12063 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12064 }
12065
12066 const ICmpInst *ICI = dyn_cast<ICmpInst>(Val: FoundCondValue);
12067 if (!ICI) return false;
12068
12069 // Now that we found a conditional branch that dominates the loop or controls
12070 // the loop latch. Check to see if it is the comparison we are looking for.
12071 CmpPredicate FoundPred;
12072 if (Inverse)
12073 FoundPred = ICI->getInverseCmpPredicate();
12074 else
12075 FoundPred = ICI->getCmpPredicate();
12076
12077 const SCEV *FoundLHS = getSCEV(V: ICI->getOperand(i_nocapture: 0));
12078 const SCEV *FoundRHS = getSCEV(V: ICI->getOperand(i_nocapture: 1));
12079
12080 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, Context: CtxI);
12081}
12082
12083bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12084 const SCEV *RHS, CmpPredicate FoundPred,
12085 const SCEV *FoundLHS, const SCEV *FoundRHS,
12086 const Instruction *CtxI) {
12087 // Balance the types.
12088 if (getTypeSizeInBits(Ty: LHS->getType()) <
12089 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12090 // For unsigned and equality predicates, try to prove that both found
12091 // operands fit into narrow unsigned range. If so, try to prove facts in
12092 // narrow types.
12093 if (!CmpInst::isSigned(Pred: FoundPred) && !FoundLHS->getType()->isPointerTy() &&
12094 !FoundRHS->getType()->isPointerTy()) {
12095 auto *NarrowType = LHS->getType();
12096 auto *WideType = FoundLHS->getType();
12097 auto BitWidth = getTypeSizeInBits(Ty: NarrowType);
12098 const SCEV *MaxValue = getZeroExtendExpr(
12099 Op: getConstant(Val: APInt::getMaxValue(numBits: BitWidth)), Ty: WideType);
12100 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundLHS,
12101 RHS: MaxValue) &&
12102 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundRHS,
12103 RHS: MaxValue)) {
12104 const SCEV *TruncFoundLHS = getTruncateExpr(Op: FoundLHS, Ty: NarrowType);
12105 const SCEV *TruncFoundRHS = getTruncateExpr(Op: FoundRHS, Ty: NarrowType);
12106 // We cannot preserve samesign after truncation.
12107 if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred: FoundPred.dropSameSign(),
12108 FoundLHS: TruncFoundLHS, FoundRHS: TruncFoundRHS, CtxI))
12109 return true;
12110 }
12111 }
12112
12113 if (LHS->getType()->isPointerTy() || RHS->getType()->isPointerTy())
12114 return false;
12115 if (CmpInst::isSigned(Pred)) {
12116 LHS = getSignExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12117 RHS = getSignExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12118 } else {
12119 LHS = getZeroExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12120 RHS = getZeroExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12121 }
12122 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
12123 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12124 if (FoundLHS->getType()->isPointerTy() || FoundRHS->getType()->isPointerTy())
12125 return false;
12126 if (CmpInst::isSigned(Pred: FoundPred)) {
12127 FoundLHS = getSignExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12128 FoundRHS = getSignExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12129 } else {
12130 FoundLHS = getZeroExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12131 FoundRHS = getZeroExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12132 }
12133 }
12134 return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS,
12135 FoundRHS, CtxI);
12136}
12137
12138bool ScalarEvolution::isImpliedCondBalancedTypes(
12139 CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS, CmpPredicate FoundPred,
12140 SCEVUse FoundLHS, SCEVUse FoundRHS, const Instruction *CtxI) {
12141 assert(getTypeSizeInBits(LHS->getType()) ==
12142 getTypeSizeInBits(FoundLHS->getType()) &&
12143 "Types should be balanced!");
12144 // Canonicalize the query to match the way instcombine will have
12145 // canonicalized the comparison.
12146 if (SimplifyICmpOperands(Pred, LHS, RHS))
12147 if (LHS == RHS)
12148 return CmpInst::isTrueWhenEqual(predicate: Pred);
12149 if (SimplifyICmpOperands(Pred&: FoundPred, LHS&: FoundLHS, RHS&: FoundRHS))
12150 if (FoundLHS == FoundRHS)
12151 return CmpInst::isFalseWhenEqual(predicate: FoundPred);
12152
12153 // Check to see if we can make the LHS or RHS match.
12154 if (LHS == FoundRHS || RHS == FoundLHS) {
12155 if (isa<SCEVConstant>(Val: RHS)) {
12156 std::swap(a&: FoundLHS, b&: FoundRHS);
12157 FoundPred = ICmpInst::getSwappedCmpPredicate(Pred: FoundPred);
12158 } else {
12159 std::swap(a&: LHS, b&: RHS);
12160 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12161 }
12162 }
12163
12164 // Check whether the found predicate is the same as the desired predicate.
12165 if (auto P = CmpPredicate::getMatching(A: FoundPred, B: Pred))
12166 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12167
12168 // Check whether swapping the found predicate makes it the same as the
12169 // desired predicate.
12170 if (auto P = CmpPredicate::getMatching(
12171 A: ICmpInst::getSwappedCmpPredicate(Pred: FoundPred), B: Pred)) {
12172 // We can write the implication
12173 // 0. LHS Pred RHS <- FoundLHS SwapPred FoundRHS
12174 // using one of the following ways:
12175 // 1. LHS Pred RHS <- FoundRHS Pred FoundLHS
12176 // 2. RHS SwapPred LHS <- FoundLHS SwapPred FoundRHS
12177 // 3. LHS Pred RHS <- ~FoundLHS Pred ~FoundRHS
12178 // 4. ~LHS SwapPred ~RHS <- FoundLHS SwapPred FoundRHS
12179 // Forms 1. and 2. require swapping the operands of one condition. Don't
12180 // do this if it would break canonical constant/addrec ordering.
12181 if (!isa<SCEVConstant>(Val: RHS) && !isa<SCEVAddRecExpr>(Val: LHS))
12182 return isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P), LHS: RHS,
12183 RHS: LHS, FoundLHS, FoundRHS, Context: CtxI);
12184 if (!isa<SCEVConstant>(Val: FoundRHS) && !isa<SCEVAddRecExpr>(Val: FoundLHS))
12185 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: FoundRHS, FoundRHS: FoundLHS, Context: CtxI);
12186
12187 // There's no clear preference between forms 3. and 4., try both. Avoid
12188 // forming getNotSCEV of pointer values as the resulting subtract is
12189 // not legal.
12190 if (!LHS->getType()->isPointerTy() && !RHS->getType()->isPointerTy() &&
12191 isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P),
12192 LHS: getNotSCEV(V: LHS), RHS: getNotSCEV(V: RHS), FoundLHS,
12193 FoundRHS, Context: CtxI))
12194 return true;
12195
12196 if (!FoundLHS->getType()->isPointerTy() &&
12197 !FoundRHS->getType()->isPointerTy() &&
12198 isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: getNotSCEV(V: FoundLHS),
12199 FoundRHS: getNotSCEV(V: FoundRHS), Context: CtxI))
12200 return true;
12201
12202 return false;
12203 }
12204
12205 auto IsSignFlippedPredicate = [](CmpInst::Predicate P1,
12206 CmpInst::Predicate P2) {
12207 assert(P1 != P2 && "Handled earlier!");
12208 return CmpInst::isRelational(P: P2) &&
12209 P1 == ICmpInst::getFlippedSignednessPredicate(Pred: P2);
12210 };
12211 if (IsSignFlippedPredicate(Pred, FoundPred)) {
12212 // Unsigned comparison is the same as signed comparison when both the
12213 // operands are non-negative or negative.
12214 if (haveSameSign(S1: FoundLHS, S2: FoundRHS))
12215 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12216 // Create local copies that we can freely swap and canonicalize our
12217 // conditions to "le/lt".
12218 CmpPredicate CanonicalPred = Pred, CanonicalFoundPred = FoundPred;
12219 const SCEV *CanonicalLHS = LHS, *CanonicalRHS = RHS,
12220 *CanonicalFoundLHS = FoundLHS, *CanonicalFoundRHS = FoundRHS;
12221 if (ICmpInst::isGT(P: CanonicalPred) || ICmpInst::isGE(P: CanonicalPred)) {
12222 CanonicalPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalPred);
12223 CanonicalFoundPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalFoundPred);
12224 std::swap(a&: CanonicalLHS, b&: CanonicalRHS);
12225 std::swap(a&: CanonicalFoundLHS, b&: CanonicalFoundRHS);
12226 }
12227 assert((ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) &&
12228 "Must be!");
12229 assert((ICmpInst::isLT(CanonicalFoundPred) ||
12230 ICmpInst::isLE(CanonicalFoundPred)) &&
12231 "Must be!");
12232 if (ICmpInst::isSigned(Pred: CanonicalPred) && isKnownNonNegative(S: CanonicalRHS))
12233 // Use implication:
12234 // x <u y && y >=s 0 --> x <s y.
12235 // If we can prove the left part, the right part is also proven.
12236 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
12237 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
12238 FoundRHS: CanonicalFoundRHS);
12239 if (ICmpInst::isUnsigned(Pred: CanonicalPred) && isKnownNegative(S: CanonicalRHS))
12240 // Use implication:
12241 // x <s y && y <s 0 --> x <u y.
12242 // If we can prove the left part, the right part is also proven.
12243 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
12244 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
12245 FoundRHS: CanonicalFoundRHS);
12246 }
12247
12248 // Check if we can make progress by sharpening ranges.
12249 if (FoundPred == ICmpInst::ICMP_NE &&
12250 (isa<SCEVConstant>(Val: FoundLHS) || isa<SCEVConstant>(Val: FoundRHS))) {
12251
12252 const SCEVConstant *C = nullptr;
12253 const SCEV *V = nullptr;
12254
12255 if (isa<SCEVConstant>(Val: FoundLHS)) {
12256 C = cast<SCEVConstant>(Val&: FoundLHS);
12257 V = FoundRHS;
12258 } else {
12259 C = cast<SCEVConstant>(Val&: FoundRHS);
12260 V = FoundLHS;
12261 }
12262
12263 // The guarding predicate tells us that C != V. If the known range
12264 // of V is [C, t), we can sharpen the range to [C + 1, t). The
12265 // range we consider has to correspond to same signedness as the
12266 // predicate we're interested in folding.
12267
12268 APInt Min = ICmpInst::isSigned(Pred) ?
12269 getSignedRangeMin(S: V) : getUnsignedRangeMin(S: V);
12270
12271 if (Min == C->getAPInt()) {
12272 // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
12273 // This is true even if (Min + 1) wraps around -- in case of
12274 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
12275
12276 APInt SharperMin = Min + 1;
12277
12278 switch (Pred) {
12279 case ICmpInst::ICMP_SGE:
12280 case ICmpInst::ICMP_UGE:
12281 // We know V `Pred` SharperMin. If this implies LHS `Pred`
12282 // RHS, we're done.
12283 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin),
12284 Context: CtxI))
12285 return true;
12286 [[fallthrough]];
12287
12288 case ICmpInst::ICMP_SGT:
12289 case ICmpInst::ICMP_UGT:
12290 // We know from the range information that (V `Pred` Min ||
12291 // V == Min). We know from the guarding condition that !(V
12292 // == Min). This gives us
12293 //
12294 // V `Pred` Min || V == Min && !(V == Min)
12295 // => V `Pred` Min
12296 //
12297 // If V `Pred` Min implies LHS `Pred` RHS, we're done.
12298
12299 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12300 return true;
12301 break;
12302
12303 // `LHS < RHS` and `LHS <= RHS` are handled in the same way as `RHS > LHS` and `RHS >= LHS` respectively.
12304 case ICmpInst::ICMP_SLE:
12305 case ICmpInst::ICMP_ULE:
12306 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12307 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin), Context: CtxI))
12308 return true;
12309 [[fallthrough]];
12310
12311 case ICmpInst::ICMP_SLT:
12312 case ICmpInst::ICMP_ULT:
12313 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12314 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12315 return true;
12316 break;
12317
12318 default:
12319 // No change
12320 break;
12321 }
12322 }
12323 }
12324
12325 // Check whether the actual condition is beyond sufficient.
12326 if (FoundPred == ICmpInst::ICMP_EQ)
12327 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
12328 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12329 return true;
12330 if (Pred == ICmpInst::ICMP_NE)
12331 if (!ICmpInst::isTrueWhenEqual(predicate: FoundPred))
12332 if (isImpliedCondOperands(Pred: FoundPred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12333 return true;
12334
12335 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS))
12336 return true;
12337
12338 // Otherwise assume the worst.
12339 return false;
12340}
12341
12342bool ScalarEvolution::splitBinaryAdd(SCEVUse Expr, SCEVUse &L, SCEVUse &R,
12343 SCEV::NoWrapFlags &Flags) {
12344 if (!match(U: Expr, P: m_scev_Add(Op0: m_SCEV(V&: L), Op1: m_SCEV(V&: R))))
12345 return false;
12346
12347 Flags = cast<SCEVAddExpr>(Val&: Expr)->getNoWrapFlags();
12348 return true;
12349}
12350
12351std::optional<APInt>
12352ScalarEvolution::computeConstantDifference(const SCEV *More, const SCEV *Less) {
12353 // We avoid subtracting expressions here because this function is usually
12354 // fairly deep in the call stack (i.e. is called many times).
12355
12356 unsigned BW = getTypeSizeInBits(Ty: More->getType());
12357 APInt Diff(BW, 0);
12358 APInt DiffMul(BW, 1);
12359 // Try various simplifications to reduce the difference to a constant. Limit
12360 // the number of allowed simplifications to keep compile-time low.
12361 for (unsigned I = 0; I < 8; ++I) {
12362 if (More == Less)
12363 return Diff;
12364
12365 // Reduce addrecs with identical steps to their start value.
12366 if (isa<SCEVAddRecExpr>(Val: Less) && isa<SCEVAddRecExpr>(Val: More)) {
12367 const auto *LAR = cast<SCEVAddRecExpr>(Val: Less);
12368 const auto *MAR = cast<SCEVAddRecExpr>(Val: More);
12369
12370 if (LAR->getLoop() != MAR->getLoop())
12371 return std::nullopt;
12372
12373 // We look at affine expressions only; not for correctness but to keep
12374 // getStepRecurrence cheap.
12375 if (!LAR->isAffine() || !MAR->isAffine())
12376 return std::nullopt;
12377
12378 if (LAR->getStepRecurrence(SE&: *this) != MAR->getStepRecurrence(SE&: *this))
12379 return std::nullopt;
12380
12381 Less = LAR->getStart();
12382 More = MAR->getStart();
12383 continue;
12384 }
12385
12386 // Try to match a common constant multiply.
12387 auto MatchConstMul =
12388 [](const SCEV *S) -> std::optional<std::pair<const SCEV *, APInt>> {
12389 const APInt *C;
12390 const SCEV *Op;
12391 if (match(S, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEV(V&: Op))))
12392 return {{Op, *C}};
12393 return std::nullopt;
12394 };
12395 if (auto MatchedMore = MatchConstMul(More)) {
12396 if (auto MatchedLess = MatchConstMul(Less)) {
12397 if (MatchedMore->second == MatchedLess->second) {
12398 More = MatchedMore->first;
12399 Less = MatchedLess->first;
12400 DiffMul *= MatchedMore->second;
12401 continue;
12402 }
12403 }
12404 }
12405
12406 // Try to cancel out common factors in two add expressions.
12407 SmallDenseMap<const SCEV *, int, 8> Multiplicity;
12408 auto Add = [&](const SCEV *S, int Mul) {
12409 if (auto *C = dyn_cast<SCEVConstant>(Val: S)) {
12410 if (Mul == 1) {
12411 Diff += C->getAPInt() * DiffMul;
12412 } else {
12413 assert(Mul == -1);
12414 Diff -= C->getAPInt() * DiffMul;
12415 }
12416 } else
12417 Multiplicity[S] += Mul;
12418 };
12419 auto Decompose = [&](const SCEV *S, int Mul) {
12420 if (isa<SCEVAddExpr>(Val: S)) {
12421 for (const SCEV *Op : S->operands())
12422 Add(Op, Mul);
12423 } else
12424 Add(S, Mul);
12425 };
12426 Decompose(More, 1);
12427 Decompose(Less, -1);
12428
12429 // Check whether all the non-constants cancel out, or reduce to new
12430 // More/Less values.
12431 const SCEV *NewMore = nullptr, *NewLess = nullptr;
12432 for (const auto &[S, Mul] : Multiplicity) {
12433 if (Mul == 0)
12434 continue;
12435 if (Mul == 1) {
12436 if (NewMore)
12437 return std::nullopt;
12438 NewMore = S;
12439 } else if (Mul == -1) {
12440 if (NewLess)
12441 return std::nullopt;
12442 NewLess = S;
12443 } else
12444 return std::nullopt;
12445 }
12446
12447 // Values stayed the same, no point in trying further.
12448 if (NewMore == More || NewLess == Less)
12449 return std::nullopt;
12450
12451 More = NewMore;
12452 Less = NewLess;
12453
12454 // Reduced to constant.
12455 if (!More && !Less)
12456 return Diff;
12457
12458 // Left with variable on only one side, bail out.
12459 if (!More || !Less)
12460 return std::nullopt;
12461 }
12462
12463 // Did not reduce to constant.
12464 return std::nullopt;
12465}
12466
12467bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
12468 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12469 const SCEV *FoundRHS, const Instruction *CtxI) {
12470 // Try to recognize the following pattern:
12471 //
12472 // FoundRHS = ...
12473 // ...
12474 // loop:
12475 // FoundLHS = {Start,+,W}
12476 // context_bb: // Basic block from the same loop
12477 // known(Pred, FoundLHS, FoundRHS)
12478 //
12479 // If some predicate is known in the context of a loop, it is also known on
12480 // each iteration of this loop, including the first iteration. Therefore, in
12481 // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to
12482 // prove the original pred using this fact.
12483 if (!CtxI)
12484 return false;
12485 const BasicBlock *ContextBB = CtxI->getParent();
12486 // Make sure AR varies in the context block.
12487 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS)) {
12488 const Loop *L = AR->getLoop();
12489 const auto *Latch = L->getLoopLatch();
12490 // Make sure that context belongs to the loop and executes on 1st iteration
12491 // (if it ever executes at all).
12492 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12493 return false;
12494 if (!isAvailableAtLoopEntry(S: FoundRHS, L: AR->getLoop()))
12495 return false;
12496 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: AR->getStart(), FoundRHS);
12497 }
12498
12499 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundRHS)) {
12500 const Loop *L = AR->getLoop();
12501 const auto *Latch = L->getLoopLatch();
12502 // Make sure that context belongs to the loop and executes on 1st iteration
12503 // (if it ever executes at all).
12504 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12505 return false;
12506 if (!isAvailableAtLoopEntry(S: FoundLHS, L: AR->getLoop()))
12507 return false;
12508 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS: AR->getStart());
12509 }
12510
12511 return false;
12512}
12513
12514bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(CmpPredicate Pred,
12515 const SCEV *LHS,
12516 const SCEV *RHS,
12517 const SCEV *FoundLHS,
12518 const SCEV *FoundRHS) {
12519 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT)
12520 return false;
12521
12522 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12523 if (!AddRecLHS)
12524 return false;
12525
12526 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12527 if (!AddRecFoundLHS)
12528 return false;
12529
12530 // We'd like to let SCEV reason about control dependencies, so we constrain
12531 // both the inequalities to be about add recurrences on the same loop. This
12532 // way we can use isLoopEntryGuardedByCond later.
12533
12534 const Loop *L = AddRecFoundLHS->getLoop();
12535 if (L != AddRecLHS->getLoop())
12536 return false;
12537
12538 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1)
12539 //
12540 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C)
12541 // ... (2)
12542 //
12543 // Informal proof for (2), assuming (1) [*]:
12544 //
12545 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**]
12546 //
12547 // Then
12548 //
12549 // FoundLHS s< FoundRHS s< INT_MIN - C
12550 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ]
12551 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ]
12552 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s<
12553 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ]
12554 // <=> FoundLHS + C s< FoundRHS + C
12555 //
12556 // [*]: (1) can be proved by ruling out overflow.
12557 //
12558 // [**]: This can be proved by analyzing all the four possibilities:
12559 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and
12560 // (A s>= 0, B s>= 0).
12561 //
12562 // Note:
12563 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C"
12564 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS
12565 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS
12566 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is
12567 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS +
12568 // C)".
12569
12570 std::optional<APInt> LDiff = computeConstantDifference(More: LHS, Less: FoundLHS);
12571 if (!LDiff)
12572 return false;
12573 std::optional<APInt> RDiff = computeConstantDifference(More: RHS, Less: FoundRHS);
12574 if (!RDiff || *LDiff != *RDiff)
12575 return false;
12576
12577 if (LDiff->isMinValue())
12578 return true;
12579
12580 APInt FoundRHSLimit;
12581
12582 if (Pred == CmpInst::ICMP_ULT) {
12583 FoundRHSLimit = -(*RDiff);
12584 } else {
12585 assert(Pred == CmpInst::ICMP_SLT && "Checked above!");
12586 FoundRHSLimit = APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: RHS->getType())) - *RDiff;
12587 }
12588
12589 // Try to prove (1) or (2), as needed.
12590 return isAvailableAtLoopEntry(S: FoundRHS, L) &&
12591 isLoopEntryGuardedByCond(L, Pred, LHS: FoundRHS,
12592 RHS: getConstant(Val: FoundRHSLimit));
12593}
12594
12595bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred, const SCEV *LHS,
12596 const SCEV *RHS, const SCEV *FoundLHS,
12597 const SCEV *FoundRHS, unsigned Depth) {
12598 const PHINode *LPhi = nullptr, *RPhi = nullptr;
12599
12600 llvm::scope_exit ClearOnExit([&]() {
12601 if (LPhi) {
12602 bool Erased = PendingMerges.erase(Ptr: LPhi);
12603 assert(Erased && "Failed to erase LPhi!");
12604 (void)Erased;
12605 }
12606 if (RPhi) {
12607 bool Erased = PendingMerges.erase(Ptr: RPhi);
12608 assert(Erased && "Failed to erase RPhi!");
12609 (void)Erased;
12610 }
12611 });
12612
12613 // Find respective Phis and check that they are not being pending.
12614 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(Val: LHS))
12615 if (auto *Phi = dyn_cast<PHINode>(Val: LU->getValue())) {
12616 if (!PendingMerges.insert(Ptr: Phi).second)
12617 return false;
12618 LPhi = Phi;
12619 }
12620 if (const SCEVUnknown *RU = dyn_cast<SCEVUnknown>(Val: RHS))
12621 if (auto *Phi = dyn_cast<PHINode>(Val: RU->getValue())) {
12622 // If we detect a loop of Phi nodes being processed by this method, for
12623 // example:
12624 //
12625 // %a = phi i32 [ %some1, %preheader ], [ %b, %latch ]
12626 // %b = phi i32 [ %some2, %preheader ], [ %a, %latch ]
12627 //
12628 // we don't want to deal with a case that complex, so return conservative
12629 // answer false.
12630 if (!PendingMerges.insert(Ptr: Phi).second)
12631 return false;
12632 RPhi = Phi;
12633 }
12634
12635 // If none of LHS, RHS is a Phi, nothing to do here.
12636 if (!LPhi && !RPhi)
12637 return false;
12638
12639 // If there is a SCEVUnknown Phi we are interested in, make it left.
12640 if (!LPhi) {
12641 std::swap(a&: LHS, b&: RHS);
12642 std::swap(a&: FoundLHS, b&: FoundRHS);
12643 std::swap(a&: LPhi, b&: RPhi);
12644 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12645 }
12646
12647 assert(LPhi && "LPhi should definitely be a SCEVUnknown Phi!");
12648 const BasicBlock *LBB = LPhi->getParent();
12649 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(Val: RHS);
12650
12651 auto ProvedEasily = [&](const SCEV *S1, const SCEV *S2) {
12652 return isKnownViaNonRecursiveReasoning(Pred, LHS: S1, RHS: S2) ||
12653 isImpliedCondOperandsViaRanges(Pred, LHS: S1, RHS: S2, FoundPred: Pred, FoundLHS, FoundRHS) ||
12654 isImpliedViaOperations(Pred, LHS: S1, RHS: S2, FoundLHS, FoundRHS, Depth);
12655 };
12656
12657 if (RPhi && RPhi->getParent() == LBB) {
12658 // Case one: RHS is also a SCEVUnknown Phi from the same basic block.
12659 // If we compare two Phis from the same block, and for each entry block
12660 // the predicate is true for incoming values from this block, then the
12661 // predicate is also true for the Phis.
12662 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12663 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12664 const SCEV *R = getSCEV(V: RPhi->getIncomingValueForBlock(BB: IncBB));
12665 if (!ProvedEasily(L, R))
12666 return false;
12667 }
12668 } else if (RAR && RAR->getLoop()->getHeader() == LBB) {
12669 // Case two: RHS is also a Phi from the same basic block, and it is an
12670 // AddRec. It means that there is a loop which has both AddRec and Unknown
12671 // PHIs, for it we can compare incoming values of AddRec from above the loop
12672 // and latch with their respective incoming values of LPhi.
12673 // TODO: Generalize to handle loops with many inputs in a header.
12674 if (LPhi->getNumIncomingValues() != 2) return false;
12675
12676 auto *RLoop = RAR->getLoop();
12677 auto *Predecessor = RLoop->getLoopPredecessor();
12678 assert(Predecessor && "Loop with AddRec with no predecessor?");
12679 const SCEV *L1 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Predecessor));
12680 if (!ProvedEasily(L1, RAR->getStart()))
12681 return false;
12682 auto *Latch = RLoop->getLoopLatch();
12683 assert(Latch && "Loop with AddRec with no latch?");
12684 const SCEV *L2 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Latch));
12685 if (!ProvedEasily(L2, RAR->getPostIncExpr(SE&: *this)))
12686 return false;
12687 } else {
12688 // In all other cases go over inputs of LHS and compare each of them to RHS,
12689 // the predicate is true for (LHS, RHS) if it is true for all such pairs.
12690 // At this point RHS is either a non-Phi, or it is a Phi from some block
12691 // different from LBB.
12692 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12693 // Check that RHS is available in this block.
12694 if (!dominates(S: RHS, BB: IncBB))
12695 return false;
12696 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12697 // Make sure L does not refer to a value from a potentially previous
12698 // iteration of a loop.
12699 if (!properlyDominates(S: L, BB: LBB))
12700 return false;
12701 // Addrecs are considered to properly dominate their loop, so are missed
12702 // by the previous check. Discard any values that have computable
12703 // evolution in this loop.
12704 if (auto *Loop = LI.getLoopFor(BB: LBB))
12705 if (hasComputableLoopEvolution(S: L, L: Loop))
12706 return false;
12707 if (!ProvedEasily(L, RHS))
12708 return false;
12709 }
12710 }
12711 return true;
12712}
12713
12714bool ScalarEvolution::isImpliedCondOperandsViaShift(CmpPredicate Pred,
12715 const SCEV *LHS,
12716 const SCEV *RHS,
12717 const SCEV *FoundLHS,
12718 const SCEV *FoundRHS) {
12719 // We want to imply LHS < RHS from LHS < (RHS >> shiftvalue). First, make
12720 // sure that we are dealing with same LHS.
12721 if (RHS == FoundRHS) {
12722 std::swap(a&: LHS, b&: RHS);
12723 std::swap(a&: FoundLHS, b&: FoundRHS);
12724 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12725 }
12726 if (LHS != FoundLHS)
12727 return false;
12728
12729 auto *SUFoundRHS = dyn_cast<SCEVUnknown>(Val: FoundRHS);
12730 if (!SUFoundRHS)
12731 return false;
12732
12733 Value *Shiftee, *ShiftValue;
12734
12735 using namespace PatternMatch;
12736 if (match(V: SUFoundRHS->getValue(),
12737 P: m_LShr(L: m_Value(V&: Shiftee), R: m_Value(V&: ShiftValue)))) {
12738 auto *ShifteeS = getSCEV(V: Shiftee);
12739 // Prove one of the following:
12740 // LHS <u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <u RHS
12741 // LHS <=u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <=u RHS
12742 // LHS <s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12743 // ---> LHS <s RHS
12744 // LHS <=s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12745 // ---> LHS <=s RHS
12746 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
12747 return isKnownPredicate(Pred: ICmpInst::ICMP_ULE, LHS: ShifteeS, RHS);
12748 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
12749 if (isKnownNonNegative(S: ShifteeS))
12750 return isKnownPredicate(Pred: ICmpInst::ICMP_SLE, LHS: ShifteeS, RHS);
12751 }
12752
12753 return false;
12754}
12755
12756bool ScalarEvolution::isImpliedCondOperandsViaMatchingDiff(
12757 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12758 const SCEV *FoundRHS) {
12759 // Only valid for equality predicates: (A == B) implies (C == D) when
12760 // the SCEV difference A - B equals C - D (they check the same
12761 // underlying relationship at every iteration).
12762 if (!ICmpInst::isEquality(P: Pred))
12763 return false;
12764
12765 // Restrict to cases involving loop recurrences - that's where this
12766 // pattern arises (correlated IV comparisons). This avoids calling
12767 // getMinusSCEV on arbitrary non-loop expressions.
12768 if ((!isa<SCEVAddRecExpr>(Val: LHS) && !isa<SCEVAddRecExpr>(Val: RHS)) ||
12769 (!isa<SCEVAddRecExpr>(Val: FoundLHS) && !isa<SCEVAddRecExpr>(Val: FoundRHS)))
12770 return false;
12771
12772 // AddRecs from different loops can never produce matching differences.
12773 const SCEVAddRecExpr *QueryAddRec = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12774 if (!QueryAddRec)
12775 QueryAddRec = cast<SCEVAddRecExpr>(Val: RHS);
12776 const SCEVAddRecExpr *FoundAddRec = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12777 if (!FoundAddRec)
12778 FoundAddRec = cast<SCEVAddRecExpr>(Val: FoundRHS);
12779 if (QueryAddRec->getLoop() != FoundAddRec->getLoop())
12780 return false;
12781
12782 // If the strides differ, the differences can never match.
12783 if (QueryAddRec->getStepRecurrence(SE&: *this) !=
12784 FoundAddRec->getStepRecurrence(SE&: *this))
12785 return false;
12786
12787 // Compute differences. For pointer-typed operands sharing the same base,
12788 // getMinusSCEV strips the common base and returns an integer SCEV.
12789 // For example, {base,+,8} - (base+8*n) = {-8n,+,8}
12790 const SCEV *FoundDiff = getMinusSCEV(LHS: FoundLHS, RHS: FoundRHS);
12791 if (isa<SCEVCouldNotCompute>(Val: FoundDiff))
12792 return false;
12793
12794 const SCEV *Diff = getMinusSCEV(LHS, RHS);
12795 if (isa<SCEVCouldNotCompute>(Val: Diff))
12796 return false;
12797
12798 return Diff == FoundDiff;
12799}
12800
12801bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred, const SCEV *LHS,
12802 const SCEV *RHS,
12803 const SCEV *FoundLHS,
12804 const SCEV *FoundRHS,
12805 const Instruction *CtxI) {
12806 return isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred: Pred, FoundLHS,
12807 FoundRHS) ||
12808 isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS,
12809 FoundRHS) ||
12810 isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS) ||
12811 isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
12812 CtxI) ||
12813 isImpliedCondOperandsViaMatchingDiff(Pred, LHS, RHS, FoundLHS,
12814 FoundRHS) ||
12815 isImpliedCondOperandsHelper(Pred, LHS, RHS, FoundLHS, FoundRHS);
12816}
12817
12818/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
12819template <typename MinMaxExprType>
12820static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
12821 const SCEV *Candidate) {
12822 const MinMaxExprType *MinMaxExpr = dyn_cast<MinMaxExprType>(MaybeMinMaxExpr);
12823 if (!MinMaxExpr)
12824 return false;
12825
12826 return is_contained(MinMaxExpr->operands(), Candidate);
12827}
12828
12829static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE,
12830 CmpPredicate Pred, const SCEV *LHS,
12831 const SCEV *RHS) {
12832 // If both sides are affine addrecs for the same loop, with equal
12833 // steps, and we know the recurrences don't wrap, then we only
12834 // need to check the predicate on the starting values.
12835
12836 if (!ICmpInst::isRelational(P: Pred))
12837 return false;
12838
12839 const SCEV *LStart, *RStart, *Step;
12840 const Loop *L;
12841 if (!match(S: LHS,
12842 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: LStart), Op1: m_SCEV(V&: Step), L: m_Loop(L))) ||
12843 !match(S: RHS, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: RStart), Op1: m_scev_Specific(S: Step),
12844 L: m_SpecificLoop(L))))
12845 return false;
12846 const SCEVAddRecExpr *LAR = cast<SCEVAddRecExpr>(Val: LHS);
12847 const SCEVAddRecExpr *RAR = cast<SCEVAddRecExpr>(Val: RHS);
12848 SCEV::NoWrapFlags NW = ICmpInst::isSigned(Pred) ?
12849 SCEV::FlagNSW : SCEV::FlagNUW;
12850 if (!LAR->getNoWrapFlags(Mask: NW) || !RAR->getNoWrapFlags(Mask: NW))
12851 return false;
12852
12853 return SE.isKnownPredicate(Pred, LHS: LStart, RHS: RStart);
12854}
12855
12856/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
12857/// expression?
12858static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, CmpPredicate Pred,
12859 const SCEV *LHS, const SCEV *RHS) {
12860 switch (Pred) {
12861 default:
12862 return false;
12863
12864 case ICmpInst::ICMP_SGE:
12865 std::swap(a&: LHS, b&: RHS);
12866 [[fallthrough]];
12867 case ICmpInst::ICMP_SLE:
12868 return
12869 // min(A, ...) <= A
12870 IsMinMaxConsistingOf<SCEVSMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12871 // A <= max(A, ...)
12872 IsMinMaxConsistingOf<SCEVSMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12873
12874 case ICmpInst::ICMP_UGE:
12875 std::swap(a&: LHS, b&: RHS);
12876 [[fallthrough]];
12877 case ICmpInst::ICMP_ULE:
12878 return
12879 // min(A, ...) <= A
12880 // FIXME: what about umin_seq?
12881 IsMinMaxConsistingOf<SCEVUMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12882 // A <= max(A, ...)
12883 IsMinMaxConsistingOf<SCEVUMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12884
12885 case ICmpInst::ICMP_UGT:
12886 std::swap(a&: LHS, b&: RHS);
12887 [[fallthrough]];
12888 case ICmpInst::ICMP_ULT:
12889 // umin(Ops) u<= each Op, so proving Op u< RHS for any Op proves
12890 // umin(Ops) u< RHS.
12891 //
12892 // Use computeConstantDifference instead of the more powerful
12893 // isKnownPredicate to keep this check cheap: isKnownPredicateViaMinOrMax
12894 // is called from isKnownViaNonRecursiveReasoning, so recursing into
12895 // the full predicate prover would be expensive.
12896 if (const auto *Min = dyn_cast<SCEVUMinExpr>(Val: LHS)) {
12897 for (SCEVUse Op : Min->operands()) {
12898 std::optional<APInt> Diff = SE.computeConstantDifference(More: RHS, Less: Op);
12899 // When Op and RHS share a common base differing by a
12900 // constant offset D (RHS - Op = D), Op u< RHS holds iff D != 0 and
12901 // RHS >= D (unsigned), i.e. the subtraction doesn't underflow.
12902 if (Diff && !Diff->isZero() && SE.getUnsignedRangeMin(S: RHS).uge(RHS: *Diff))
12903 return true;
12904 }
12905 }
12906 return false;
12907 }
12908
12909 llvm_unreachable("covered switch fell through?!");
12910}
12911
12912bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
12913 const SCEV *RHS,
12914 const SCEV *FoundLHS,
12915 const SCEV *FoundRHS,
12916 unsigned Depth) {
12917 assert(getTypeSizeInBits(LHS->getType()) ==
12918 getTypeSizeInBits(RHS->getType()) &&
12919 "LHS and RHS have different sizes?");
12920 assert(getTypeSizeInBits(FoundLHS->getType()) ==
12921 getTypeSizeInBits(FoundRHS->getType()) &&
12922 "FoundLHS and FoundRHS have different sizes?");
12923 // We want to avoid hurting the compile time with analysis of too big trees.
12924 if (Depth > MaxSCEVOperationsImplicationDepth)
12925 return false;
12926
12927 // We only want to work with GT comparison so far.
12928 if (ICmpInst::isLT(P: Pred)) {
12929 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12930 std::swap(a&: LHS, b&: RHS);
12931 std::swap(a&: FoundLHS, b&: FoundRHS);
12932 }
12933
12934 CmpInst::Predicate P = Pred.getPreferredSignedPredicate();
12935
12936 // For unsigned, try to reduce it to corresponding signed comparison.
12937 if (P == ICmpInst::ICMP_UGT)
12938 // We can replace unsigned predicate with its signed counterpart if all
12939 // involved values are non-negative.
12940 // TODO: We could have better support for unsigned.
12941 if (isKnownNonNegative(S: FoundLHS) && isKnownNonNegative(S: FoundRHS)) {
12942 // Knowing that both FoundLHS and FoundRHS are non-negative, and knowing
12943 // FoundLHS >u FoundRHS, we also know that FoundLHS >s FoundRHS. Let us
12944 // use this fact to prove that LHS and RHS are non-negative.
12945 const SCEV *MinusOne = getMinusOne(Ty: LHS->getType());
12946 if (isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS, RHS: MinusOne, FoundLHS,
12947 FoundRHS) &&
12948 isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS: RHS, RHS: MinusOne, FoundLHS,
12949 FoundRHS))
12950 P = ICmpInst::ICMP_SGT;
12951 }
12952
12953 if (P != ICmpInst::ICMP_SGT)
12954 return false;
12955
12956 auto GetOpFromSExt = [&](const SCEV *S) -> const SCEV * {
12957 if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(Val: S))
12958 return Ext->getOperand();
12959 // TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
12960 // the constant in some cases.
12961 return S;
12962 };
12963
12964 // Acquire values from extensions.
12965 auto *OrigLHS = LHS;
12966 auto *OrigFoundLHS = FoundLHS;
12967 LHS = GetOpFromSExt(LHS);
12968 FoundLHS = GetOpFromSExt(FoundLHS);
12969
12970 // Is the SGT predicate can be proved trivially or using the found context.
12971 auto IsSGTViaContext = [&](const SCEV *S1, const SCEV *S2) {
12972 return isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2) ||
12973 isImpliedViaOperations(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2, FoundLHS: OrigFoundLHS,
12974 FoundRHS, Depth: Depth + 1);
12975 };
12976
12977 if (auto *LHSAddExpr = dyn_cast<SCEVAddExpr>(Val: LHS)) {
12978 // We want to avoid creation of any new non-constant SCEV. Since we are
12979 // going to compare the operands to RHS, we should be certain that we don't
12980 // need any size extensions for this. So let's decline all cases when the
12981 // sizes of types of LHS and RHS do not match.
12982 // TODO: Maybe try to get RHS from sext to catch more cases?
12983 if (getTypeSizeInBits(Ty: LHS->getType()) != getTypeSizeInBits(Ty: RHS->getType()))
12984 return false;
12985
12986 // Should not overflow.
12987 if (!LHSAddExpr->hasNoSignedWrap())
12988 return false;
12989
12990 SCEVUse LL = LHSAddExpr->getOperand(i: 0);
12991 SCEVUse LR = LHSAddExpr->getOperand(i: 1);
12992 auto *MinusOne = getMinusOne(Ty: RHS->getType());
12993
12994 // Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
12995 auto IsSumGreaterThanRHS = [&](const SCEV *S1, const SCEV *S2) {
12996 return IsSGTViaContext(S1, MinusOne) && IsSGTViaContext(S2, RHS);
12997 };
12998 // Try to prove the following rule:
12999 // (LHS = LL + LR) && (LL >= 0) && (LR > RHS) => (LHS > RHS).
13000 // (LHS = LL + LR) && (LR >= 0) && (LL > RHS) => (LHS > RHS).
13001 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
13002 return true;
13003 } else if (auto *LHSUnknownExpr = dyn_cast<SCEVUnknown>(Val: LHS)) {
13004 Value *LL, *LR;
13005 // FIXME: Once we have SDiv implemented, we can get rid of this matching.
13006
13007 using namespace llvm::PatternMatch;
13008
13009 if (match(V: LHSUnknownExpr->getValue(), P: m_SDiv(L: m_Value(V&: LL), R: m_Value(V&: LR)))) {
13010 // Rules for division.
13011 // We are going to perform some comparisons with Denominator and its
13012 // derivative expressions. In general case, creating a SCEV for it may
13013 // lead to a complex analysis of the entire graph, and in particular it
13014 // can request trip count recalculation for the same loop. This would
13015 // cache as SCEVCouldNotCompute to avoid the infinite recursion. To avoid
13016 // this, we only want to create SCEVs that are constants in this section.
13017 // So we bail if Denominator is not a constant.
13018 if (!isa<ConstantInt>(Val: LR))
13019 return false;
13020
13021 auto *Denominator = cast<SCEVConstant>(Val: getSCEV(V: LR));
13022
13023 // We want to make sure that LHS = FoundLHS / Denominator. If it is so,
13024 // then a SCEV for the numerator already exists and matches with FoundLHS.
13025 auto *Numerator = getExistingSCEV(V: LL);
13026 if (!Numerator || Numerator->getType() != FoundLHS->getType())
13027 return false;
13028
13029 // Make sure that the numerator matches with FoundLHS and the denominator
13030 // is positive.
13031 if (!HasSameValue(A: Numerator, B: FoundLHS) || !isKnownPositive(S: Denominator))
13032 return false;
13033
13034 auto *DTy = Denominator->getType();
13035 auto *FRHSTy = FoundRHS->getType();
13036 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
13037 // One of types is a pointer and another one is not. We cannot extend
13038 // them properly to a wider type, so let us just reject this case.
13039 // TODO: Usage of getEffectiveSCEVType for DTy, FRHSTy etc should help
13040 // to avoid this check.
13041 return false;
13042
13043 // Given that:
13044 // FoundLHS > FoundRHS, LHS = FoundLHS / Denominator, Denominator > 0.
13045 auto *WTy = getWiderType(T1: DTy, T2: FRHSTy);
13046 auto *DenominatorExt = getNoopOrSignExtend(V: Denominator, Ty: WTy);
13047 auto *FoundRHSExt = getNoopOrSignExtend(V: FoundRHS, Ty: WTy);
13048
13049 // Try to prove the following rule:
13050 // (FoundRHS > Denominator - 2) && (RHS <= 0) => (LHS > RHS).
13051 // For example, given that FoundLHS > 2. It means that FoundLHS is at
13052 // least 3. If we divide it by Denominator < 4, we will have at least 1.
13053 auto *DenomMinusTwo = getMinusSCEV(LHS: DenominatorExt, RHS: getConstant(Ty: WTy, V: 2));
13054 if (isKnownNonPositive(S: RHS) &&
13055 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
13056 return true;
13057
13058 // Try to prove the following rule:
13059 // (FoundRHS > -1 - Denominator) && (RHS < 0) => (LHS > RHS).
13060 // For example, given that FoundLHS > -3. Then FoundLHS is at least -2.
13061 // If we divide it by Denominator > 2, then:
13062 // 1. If FoundLHS is negative, then the result is 0.
13063 // 2. If FoundLHS is non-negative, then the result is non-negative.
13064 // Anyways, the result is non-negative.
13065 auto *MinusOne = getMinusOne(Ty: WTy);
13066 auto *NegDenomMinusOne = getMinusSCEV(LHS: MinusOne, RHS: DenominatorExt);
13067 if (isKnownNegative(S: RHS) &&
13068 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
13069 return true;
13070 }
13071 }
13072
13073 // If our expression contained SCEVUnknown Phis, and we split it down and now
13074 // need to prove something for them, try to prove the predicate for every
13075 // possible incoming values of those Phis.
13076 if (isImpliedViaMerge(Pred, LHS: OrigLHS, RHS, FoundLHS: OrigFoundLHS, FoundRHS, Depth: Depth + 1))
13077 return true;
13078
13079 return false;
13080}
13081
13082static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS,
13083 const SCEV *RHS) {
13084 // zext x u<= sext x, sext x s<= zext x
13085 const SCEV *Op;
13086 switch (Pred) {
13087 case ICmpInst::ICMP_SGE:
13088 std::swap(a&: LHS, b&: RHS);
13089 [[fallthrough]];
13090 case ICmpInst::ICMP_SLE: {
13091 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
13092 return match(S: LHS, P: m_scev_SExt(Op0: m_SCEV(V&: Op))) &&
13093 match(S: RHS, P: m_scev_ZExt(Op0: m_scev_Specific(S: Op)));
13094 }
13095 case ICmpInst::ICMP_UGE:
13096 std::swap(a&: LHS, b&: RHS);
13097 [[fallthrough]];
13098 case ICmpInst::ICMP_ULE: {
13099 // If operand >=u 0 then ZExt == SExt. If operand <u 0 then ZExt <u SExt.
13100 return match(S: LHS, P: m_scev_ZExt(Op0: m_SCEV(V&: Op))) &&
13101 match(S: RHS, P: m_scev_SExt(Op0: m_scev_Specific(S: Op)));
13102 }
13103 default:
13104 return false;
13105 };
13106 llvm_unreachable("unhandled case");
13107}
13108
13109bool ScalarEvolution::isKnownViaNonRecursiveReasoning(CmpPredicate Pred,
13110 SCEVUse LHS,
13111 SCEVUse RHS) {
13112 return isKnownPredicateExtendIdiom(Pred, LHS, RHS) ||
13113 isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
13114 IsKnownPredicateViaMinOrMax(SE&: *this, Pred, LHS, RHS) ||
13115 IsKnownPredicateViaAddRecStart(SE&: *this, Pred, LHS, RHS) ||
13116 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
13117}
13118
13119bool ScalarEvolution::isImpliedCondOperandsHelper(CmpPredicate Pred,
13120 const SCEV *LHS,
13121 const SCEV *RHS,
13122 const SCEV *FoundLHS,
13123 const SCEV *FoundRHS) {
13124 switch (Pred) {
13125 default:
13126 llvm_unreachable("Unexpected CmpPredicate value!");
13127 case ICmpInst::ICMP_EQ:
13128 case ICmpInst::ICMP_NE:
13129 if (HasSameValue(A: LHS, B: FoundLHS) && HasSameValue(A: RHS, B: FoundRHS))
13130 return true;
13131 break;
13132 case ICmpInst::ICMP_SLT:
13133 case ICmpInst::ICMP_SLE:
13134 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS, RHS: FoundLHS) &&
13135 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS: RHS, RHS: FoundRHS))
13136 return true;
13137 break;
13138 case ICmpInst::ICMP_SGT:
13139 case ICmpInst::ICMP_SGE:
13140 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS, RHS: FoundLHS) &&
13141 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS: RHS, RHS: FoundRHS))
13142 return true;
13143 break;
13144 case ICmpInst::ICMP_ULT:
13145 case ICmpInst::ICMP_ULE:
13146 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS, RHS: FoundLHS) &&
13147 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS: RHS, RHS: FoundRHS))
13148 return true;
13149 break;
13150 case ICmpInst::ICMP_UGT:
13151 case ICmpInst::ICMP_UGE:
13152 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS, RHS: FoundLHS) &&
13153 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: RHS, RHS: FoundRHS))
13154 return true;
13155 break;
13156 }
13157
13158 // Maybe it can be proved via operations?
13159 if (isImpliedViaOperations(Pred, LHS, RHS, FoundLHS, FoundRHS))
13160 return true;
13161
13162 return false;
13163}
13164
13165bool ScalarEvolution::isImpliedCondOperandsViaRanges(
13166 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
13167 const SCEV *FoundLHS, const SCEV *FoundRHS) {
13168 if (!isa<SCEVConstant>(Val: RHS) || !isa<SCEVConstant>(Val: FoundRHS))
13169 // The restriction on `FoundRHS` be lifted easily -- it exists only to
13170 // reduce the compile time impact of this optimization.
13171 return false;
13172
13173 std::optional<APInt> Addend = computeConstantDifference(More: LHS, Less: FoundLHS);
13174 if (!Addend)
13175 return false;
13176
13177 const APInt &ConstFoundRHS = cast<SCEVConstant>(Val: FoundRHS)->getAPInt();
13178
13179 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the
13180 // antecedent "`FoundLHS` `FoundPred` `FoundRHS`".
13181 ConstantRange FoundLHSRange =
13182 ConstantRange::makeExactICmpRegion(Pred: FoundPred, Other: ConstFoundRHS);
13183
13184 // Since `LHS` is `FoundLHS` + `Addend`, we can compute a range for `LHS`:
13185 ConstantRange LHSRange = FoundLHSRange.add(Other: ConstantRange(*Addend));
13186
13187 // We can also compute the range of values for `LHS` that satisfy the
13188 // consequent, "`LHS` `Pred` `RHS`":
13189 const APInt &ConstRHS = cast<SCEVConstant>(Val: RHS)->getAPInt();
13190 // The antecedent implies the consequent if every value of `LHS` that
13191 // satisfies the antecedent also satisfies the consequent.
13192 return LHSRange.icmp(Pred, Other: ConstRHS);
13193}
13194
13195bool ScalarEvolution::canIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
13196 bool IsSigned) {
13197 assert(isKnownPositive(Stride) && "Positive stride expected!");
13198
13199 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
13200 const SCEV *One = getOne(Ty: Stride->getType());
13201
13202 if (IsSigned) {
13203 APInt MaxRHS = getSignedRangeMax(S: RHS);
13204 APInt MaxValue = APInt::getSignedMaxValue(numBits: BitWidth);
13205 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13206
13207 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
13208 return (std::move(MaxValue) - MaxStrideMinusOne).slt(RHS: MaxRHS);
13209 }
13210
13211 APInt MaxRHS = getUnsignedRangeMax(S: RHS);
13212 APInt MaxValue = APInt::getMaxValue(numBits: BitWidth);
13213 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13214
13215 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
13216 return (std::move(MaxValue) - MaxStrideMinusOne).ult(RHS: MaxRHS);
13217}
13218
13219bool ScalarEvolution::canIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride,
13220 bool IsSigned) {
13221
13222 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
13223 const SCEV *One = getOne(Ty: Stride->getType());
13224
13225 if (IsSigned) {
13226 APInt MinRHS = getSignedRangeMin(S: RHS);
13227 APInt MinValue = APInt::getSignedMinValue(numBits: BitWidth);
13228 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13229
13230 // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow!
13231 return (std::move(MinValue) + MaxStrideMinusOne).sgt(RHS: MinRHS);
13232 }
13233
13234 APInt MinRHS = getUnsignedRangeMin(S: RHS);
13235 APInt MinValue = APInt::getMinValue(numBits: BitWidth);
13236 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13237
13238 // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow!
13239 return (std::move(MinValue) + MaxStrideMinusOne).ugt(RHS: MinRHS);
13240}
13241
13242const SCEV *ScalarEvolution::getUDivCeilSCEV(const SCEV *N, const SCEV *D) {
13243 // umin(N, 1) + floor((N - umin(N, 1)) / D)
13244 // This is equivalent to "1 + floor((N - 1) / D)" for N != 0. The umin
13245 // expression fixes the case of N=0.
13246 const SCEV *MinNOne = getUMinExpr(LHS: N, RHS: getOne(Ty: N->getType()));
13247 const SCEV *NMinusOne = getMinusSCEV(LHS: N, RHS: MinNOne);
13248 return getAddExpr(LHS: MinNOne, RHS: getUDivExpr(LHS: NMinusOne, RHS: D));
13249}
13250
13251const SCEV *ScalarEvolution::computeMaxBECountForLT(const SCEV *Start,
13252 const SCEV *Stride,
13253 const SCEV *End,
13254 unsigned BitWidth,
13255 bool IsSigned) {
13256 // The logic in this function assumes we can represent a positive stride.
13257 // If we can't, the backedge-taken count must be zero.
13258 if (IsSigned && BitWidth == 1)
13259 return getZero(Ty: Stride->getType());
13260
13261 // This code below only been closely audited for negative strides in the
13262 // unsigned comparison case, it may be correct for signed comparison, but
13263 // that needs to be established.
13264 if (IsSigned && isKnownNegative(S: Stride))
13265 return getCouldNotCompute();
13266
13267 // Calculate the maximum backedge count based on the range of values
13268 // permitted by Start, End, and Stride.
13269 APInt MinStart =
13270 IsSigned ? getSignedRangeMin(S: Start) : getUnsignedRangeMin(S: Start);
13271
13272 APInt MinStride =
13273 IsSigned ? getSignedRangeMin(S: Stride) : getUnsignedRangeMin(S: Stride);
13274
13275 // We assume either the stride is positive, or the backedge-taken count
13276 // is zero. So force StrideForMaxBECount to be at least one.
13277 APInt One(BitWidth, 1);
13278 APInt StrideForMaxBECount = IsSigned ? APIntOps::smax(A: One, B: MinStride)
13279 : APIntOps::umax(A: One, B: MinStride);
13280
13281 APInt MaxValue = IsSigned ? APInt::getSignedMaxValue(numBits: BitWidth)
13282 : APInt::getMaxValue(numBits: BitWidth);
13283 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
13284
13285 // Although End can be a MAX expression we estimate MaxEnd considering only
13286 // the case End = RHS of the loop termination condition. This is safe because
13287 // in the other case (End - Start) is zero, leading to a zero maximum backedge
13288 // taken count.
13289 APInt MaxEnd = IsSigned ? APIntOps::smin(A: getSignedRangeMax(S: End), B: Limit)
13290 : APIntOps::umin(A: getUnsignedRangeMax(S: End), B: Limit);
13291
13292 // MaxBECount = ceil((max(MaxEnd, MinStart) - MinStart) / Stride)
13293 MaxEnd = IsSigned ? APIntOps::smax(A: MaxEnd, B: MinStart)
13294 : APIntOps::umax(A: MaxEnd, B: MinStart);
13295
13296 return getUDivCeilSCEV(N: getConstant(Val: MaxEnd - MinStart) /* Delta */,
13297 D: getConstant(Val: StrideForMaxBECount) /* Step */);
13298}
13299
13300ScalarEvolution::ExitLimit
13301ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
13302 const Loop *L, bool IsSigned,
13303 bool ControlsOnlyExit, bool AllowPredicates) {
13304 SmallVector<const SCEVPredicate *> Predicates;
13305
13306 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13307 bool PredicatedIV = false;
13308 if (!IV) {
13309 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: LHS)) {
13310 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: ZExt->getOperand());
13311 if (AR && AR->getLoop() == L && AR->isAffine()) {
13312 auto canProveNUW = [&]() {
13313 // We can use the comparison to infer no-wrap flags only if it fully
13314 // controls the loop exit.
13315 if (!ControlsOnlyExit)
13316 return false;
13317
13318 if (!isLoopInvariant(S: RHS, L))
13319 return false;
13320
13321 if (!isKnownNonZero(S: AR->getStepRecurrence(SE&: *this)))
13322 // We need the sequence defined by AR to strictly increase in the
13323 // unsigned integer domain for the logic below to hold.
13324 return false;
13325
13326 const unsigned InnerBitWidth = getTypeSizeInBits(Ty: AR->getType());
13327 const unsigned OuterBitWidth = getTypeSizeInBits(Ty: RHS->getType());
13328 // If RHS <=u Limit, then there must exist a value V in the sequence
13329 // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and
13330 // V <=u UINT_MAX. Thus, we must exit the loop before unsigned
13331 // overflow occurs. This limit also implies that a signed comparison
13332 // (in the wide bitwidth) is equivalent to an unsigned comparison as
13333 // the high bits on both sides must be zero.
13334 APInt StrideMax = getUnsignedRangeMax(S: AR->getStepRecurrence(SE&: *this));
13335 APInt Limit = APInt::getMaxValue(numBits: InnerBitWidth) - (StrideMax - 1);
13336 Limit = Limit.zext(width: OuterBitWidth);
13337 return getUnsignedRangeMax(S: applyLoopGuards(Expr: RHS, L)).ule(RHS: Limit);
13338 };
13339 auto Flags = AR->getNoWrapFlags();
13340 if (!hasFlags(Flags, TestFlags: SCEV::FlagNUW) && canProveNUW())
13341 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
13342
13343 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
13344 if (AR->hasNoUnsignedWrap()) {
13345 // Emulate what getZeroExtendExpr would have done during construction
13346 // if we'd been able to infer the fact just above at that time.
13347 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
13348 Type *Ty = ZExt->getType();
13349 auto *S = getAddRecExpr(
13350 Start: getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: 0),
13351 Step: getZeroExtendExpr(Op: Step, Ty, Depth: 0), L, Flags: AR->getNoWrapFlags());
13352 IV = dyn_cast<SCEVAddRecExpr>(Val: S);
13353 }
13354 }
13355 }
13356 }
13357
13358
13359 if (!IV && AllowPredicates) {
13360 // Try to make this an AddRec using runtime tests, in the first X
13361 // iterations of this loop, where X is the SCEV expression found by the
13362 // algorithm below.
13363 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13364 PredicatedIV = true;
13365 }
13366
13367 // Avoid weird loops
13368 if (!IV || IV->getLoop() != L || !IV->isAffine())
13369 return getCouldNotCompute();
13370
13371 // A precondition of this method is that the condition being analyzed
13372 // reaches an exiting branch which dominates the latch. Given that, we can
13373 // assume that an increment which violates the nowrap specification and
13374 // produces poison must cause undefined behavior when the resulting poison
13375 // value is branched upon and thus we can conclude that the backedge is
13376 // taken no more often than would be required to produce that poison value.
13377 // Note that a well defined loop can exit on the iteration which violates
13378 // the nowrap specification if there is another exit (either explicit or
13379 // implicit/exceptional) which causes the loop to execute before the
13380 // exiting instruction we're analyzing would trigger UB.
13381 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13382 bool NoWrap = ControlsOnlyExit && any(Val: IV->getNoWrapFlags(Mask: WrapType));
13383 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13384
13385 const SCEV *Stride = IV->getStepRecurrence(SE&: *this);
13386
13387 bool PositiveStride = isKnownPositive(S: Stride);
13388
13389 // Whether the IV may reach the maximum value before the exit is taken.
13390 bool IVMayOverflow = true;
13391
13392 // Avoid negative or zero stride values.
13393 if (!PositiveStride) {
13394 // We can compute the correct backedge taken count for loops with unknown
13395 // strides if we can prove that the loop is not an infinite loop with side
13396 // effects. Here's the loop structure we are trying to handle -
13397 //
13398 // i = start
13399 // do {
13400 // A[i] = i;
13401 // i += s;
13402 // } while (i < end);
13403 //
13404 // The backedge taken count for such loops is evaluated as -
13405 // (max(end, start + stride) - start - 1) /u stride
13406 //
13407 // The additional preconditions that we need to check to prove correctness
13408 // of the above formula is as follows -
13409 //
13410 // a) IV is either nuw or nsw depending upon signedness (indicated by the
13411 // NoWrap flag).
13412 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13413 // no side effects within the loop)
13414 // c) loop has a single static exit (with no abnormal exits)
13415 //
13416 // Precondition a) implies that if the stride is negative, this is a single
13417 // trip loop. The backedge taken count formula reduces to zero in this case.
13418 //
13419 // Precondition b) and c) combine to imply that if rhs is invariant in L,
13420 // then a zero stride means the backedge can't be taken without executing
13421 // undefined behavior.
13422 //
13423 // The positive stride case is the same as isKnownPositive(Stride) returning
13424 // true (original behavior of the function).
13425 //
13426 if (PredicatedIV || !NoWrap || !loopIsFiniteByAssumption(L) ||
13427 !loopHasNoAbnormalExits(L))
13428 return getCouldNotCompute();
13429
13430 if (!isKnownNonZero(S: Stride)) {
13431 // If we have a step of zero, and RHS isn't invariant in L, we don't know
13432 // if it might eventually be greater than start and if so, on which
13433 // iteration. We can't even produce a useful upper bound.
13434 if (!isLoopInvariant(S: RHS, L))
13435 return getCouldNotCompute();
13436
13437 // We allow a potentially zero stride, but we need to divide by stride
13438 // below. Since the loop can't be infinite and this check must control
13439 // the sole exit, we can infer the exit must be taken on the first
13440 // iteration (e.g. backedge count = 0) if the stride is zero. Given that,
13441 // we know the numerator in the divides below must be zero, so we can
13442 // pick an arbitrary non-zero value for the denominator (e.g. stride)
13443 // and produce the right result.
13444 // FIXME: Handle the case where Stride is poison?
13445 auto wouldZeroStrideBeUB = [&]() {
13446 // Proof by contradiction. Suppose the stride were zero. If we can
13447 // prove that the backedge *is* taken on the first iteration, then since
13448 // we know this condition controls the sole exit, we must have an
13449 // infinite loop. We can't have a (well defined) infinite loop per
13450 // check just above.
13451 // Note: The (Start - Stride) term is used to get the start' term from
13452 // (start' + stride,+,stride). Remember that we only care about the
13453 // result of this expression when stride == 0 at runtime.
13454 auto *StartIfZero = getMinusSCEV(LHS: IV->getStart(), RHS: Stride);
13455 return isLoopEntryGuardedByCond(L, Pred: Cond, LHS: StartIfZero, RHS);
13456 };
13457 if (!wouldZeroStrideBeUB()) {
13458 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13459 }
13460 }
13461 } else {
13462 // Avoid proven overflow cases: this will ensure that the backedge taken
13463 // count will not generate any unsigned overflow.
13464 IVMayOverflow = canIVOverflowOnLT(RHS, Stride, IsSigned);
13465 if (IVMayOverflow && !NoWrap)
13466 return getCouldNotCompute();
13467 }
13468
13469 // On all paths just preceeding, we established the following invariant:
13470 // IV can be assumed not to overflow up to and including the exiting
13471 // iteration. We proved this in one of two ways:
13472 // 1) We can show overflow doesn't occur before the exiting iteration
13473 // 1a) canIVOverflowOnLT, and b) step of one
13474 // 2) We can show that if overflow occurs, the loop must execute UB
13475 // before any possible exit.
13476 // Note that we have not yet proved RHS invariant (in general).
13477
13478 const SCEV *Start = IV->getStart();
13479
13480 // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond.
13481 // If we convert to integers, isLoopEntryGuardedByCond will miss some cases.
13482 // Use integer-typed versions for actual computation; we can't subtract
13483 // pointers in general.
13484 const SCEV *OrigStart = Start;
13485 const SCEV *OrigRHS = RHS;
13486 if (Start->getType()->isPointerTy()) {
13487 Start = getPtrToAddrExpr(Op: Start);
13488 if (isa<SCEVCouldNotCompute>(Val: Start))
13489 return Start;
13490 }
13491 if (RHS->getType()->isPointerTy()) {
13492 RHS = getPtrToAddrExpr(Op: RHS);
13493 if (isa<SCEVCouldNotCompute>(Val: RHS))
13494 return RHS;
13495 }
13496
13497 const SCEV *End = nullptr, *BECount = nullptr,
13498 *BECountIfBackedgeTaken = nullptr;
13499 if (!isLoopInvariant(S: RHS, L)) {
13500 const auto *RHSAddRec = dyn_cast<SCEVAddRecExpr>(Val: RHS);
13501 if (PositiveStride && RHSAddRec != nullptr && RHSAddRec->getLoop() == L &&
13502 any(Val: RHSAddRec->getNoWrapFlags())) {
13503 // The structure of loop we are trying to calculate backedge count of:
13504 //
13505 // left = left_start
13506 // right = right_start
13507 //
13508 // while(left < right){
13509 // ... do something here ...
13510 // left += s1; // stride of left is s1 (s1 > 0)
13511 // right += s2; // stride of right is s2 (s2 < 0)
13512 // }
13513 //
13514
13515 const SCEV *RHSStart = RHSAddRec->getStart();
13516 const SCEV *RHSStride = RHSAddRec->getStepRecurrence(SE&: *this);
13517
13518 // If Stride - RHSStride is positive and does not overflow, we can write
13519 // backedge count as ->
13520 // ceil((End - Start) /u (Stride - RHSStride))
13521 // Where, End = max(RHSStart, Start)
13522
13523 // Check if RHSStride < 0 and Stride - RHSStride will not overflow.
13524 if (isKnownNegative(S: RHSStride) &&
13525 willNotOverflow(BinOp: Instruction::Sub, /*Signed=*/true, LHS: Stride,
13526 RHS: RHSStride)) {
13527
13528 const SCEV *Denominator = getMinusSCEV(LHS: Stride, RHS: RHSStride);
13529 if (isKnownPositive(S: Denominator)) {
13530 End = IsSigned ? getSMaxExpr(LHS: RHSStart, RHS: Start)
13531 : getUMaxExpr(LHS: RHSStart, RHS: Start);
13532
13533 // We can do this because End >= Start, as End = max(RHSStart, Start)
13534 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13535
13536 BECount = getUDivCeilSCEV(N: Delta, D: Denominator);
13537 BECountIfBackedgeTaken =
13538 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHSStart, RHS: Start), D: Denominator);
13539 }
13540 }
13541 }
13542 if (BECount == nullptr) {
13543 // If we cannot calculate ExactBECount, we can calculate the MaxBECount,
13544 // given the start, stride and max value for the end bound of the
13545 // loop (RHS), and the fact that IV does not overflow (which is
13546 // checked above).
13547 const SCEV *MaxBECount = computeMaxBECountForLT(
13548 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned);
13549 return ExitLimit(getCouldNotCompute() /* ExactNotTaken */, MaxBECount,
13550 MaxBECount, false /*MaxOrZero*/, Predicates);
13551 }
13552 } else {
13553 // Let End = max(RHS,Start). We use the expression (End-Start)/Stride to
13554 // describe the backedge count: if the backedge is taken at least once then
13555 // End is RHS, and if not End is Start so we get a backedge count of zero.
13556 auto *OrigStartMinusStride = getMinusSCEV(LHS: OrigStart, RHS: Stride);
13557 assert(isAvailableAtLoopEntry(OrigStartMinusStride, L) && "Must be!");
13558 assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!");
13559 assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!");
13560 // Can we prove Start - Stride < RHS, and either Start - Stride < Start or
13561 // (via !IVMayOverflow) that RHS + Stride - 1 does not overflow?
13562 if ((!IVMayOverflow ||
13563 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigStartMinusStride, RHS: OrigStart)) &&
13564 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigStartMinusStride, RHS: OrigRHS)) {
13565 // In this case, we can use a refined formula for computing backedge
13566 // taken count. The general formula remains:
13567 // "End-Start /uceiling Stride"
13568 // We want to use the alternate formula:
13569 // "((RHS - 1) - (Start - Stride)) /u Stride"
13570 // Let's do a quick case analysis to show these are equivalent under
13571 // our preconditions.
13572 // * For RHS <= Start (End is Start), the backedge-taken count must be
13573 // zero. Together with the precondition "Start - Stride < RHS", we have
13574 // "Start - Stride < RHS <= Start". Subtracting Start - Stride from
13575 // all sides we get "0 < RHS - (Start - Stride) <= Stride".
13576 // Subtracting 1 we get "0 <= (RHS - 1) - (Start - Stride) < Stride".
13577 // So dividing that by Stride gives zero.
13578 //
13579 // * For RHS > Start (End is RHS), the backedge count must be
13580 // "RHS-Start /uceil Stride", so it is sufficient to show that the
13581 // numerator "((RHS - 1) - (Start - Stride))" does not overflow.
13582 //
13583 // If "Start - Stride < Start" holds, we have
13584 // "RHS > Start > Start - Stride". As such
13585 // "RHS - (Start - Stride) - 1" does not overflow, which is the
13586 // reassociated numerator.
13587 //
13588 // Otherwise !IVMayOverflow guarantees "RHS + (Stride - 1) <= MaxV",
13589 // where MaxV is the maximum signed/unsigned value. Let MinV be the
13590 // matching minimum value. "Start >= MinV" gives
13591 // "RHS + (Stride - 1) - Start <= MaxV - MinV", and as "MaxV - MinV" is
13592 // the largest unsigned value, the reassociated numerator does not
13593 // overflow.
13594 const SCEV *MinusOne = getMinusOne(Ty: Stride->getType());
13595 const SCEV *Numerator =
13596 getMinusSCEV(LHS: getAddExpr(LHS: RHS, RHS: MinusOne), RHS: getMinusSCEV(LHS: Start, RHS: Stride));
13597 BECount = getUDivExpr(LHS: Numerator, RHS: Stride);
13598 }
13599
13600 if (!BECount) {
13601 auto canProveRHSGreaterThanEqualStart = [&]() {
13602 auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13603 const SCEV *GuardedRHS = applyLoopGuards(Expr: OrigRHS, L);
13604 const SCEV *GuardedStart = applyLoopGuards(Expr: OrigStart, L);
13605
13606 if (isLoopEntryGuardedByCond(L, Pred: CondGE, LHS: OrigRHS, RHS: OrigStart) ||
13607 isKnownPredicate(Pred: CondGE, LHS: GuardedRHS, RHS: GuardedStart))
13608 return true;
13609
13610 // (RHS > Start - 1) implies RHS >= Start.
13611 // * "RHS >= Start" is trivially equivalent to "RHS > Start - 1" if
13612 // "Start - 1" doesn't overflow.
13613 // * For signed comparison, if Start - 1 does overflow, it's equal
13614 // to INT_MAX, and "RHS >s INT_MAX" is trivially false.
13615 // * For unsigned comparison, if Start - 1 does overflow, it's equal
13616 // to UINT_MAX, and "RHS >u UINT_MAX" is trivially false.
13617 //
13618 // FIXME: Should isLoopEntryGuardedByCond do this for us?
13619 auto CondGT = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13620 auto *StartMinusOne =
13621 getAddExpr(LHS: OrigStart, RHS: getMinusOne(Ty: OrigStart->getType()));
13622 return isLoopEntryGuardedByCond(L, Pred: CondGT, LHS: OrigRHS, RHS: StartMinusOne);
13623 };
13624
13625 // If we know that RHS >= Start in the context of loop, then we know
13626 // that max(RHS, Start) = RHS at this point.
13627 if (canProveRHSGreaterThanEqualStart()) {
13628 End = RHS;
13629 } else {
13630 // If RHS < Start, the backedge will be taken zero times. So in
13631 // general, we can write the backedge-taken count as:
13632 //
13633 // RHS >= Start ? ceil(RHS - Start) / Stride : 0
13634 //
13635 // We convert it to the following to make it more convenient for SCEV:
13636 //
13637 // ceil(max(RHS, Start) - Start) / Stride
13638 End = IsSigned ? getSMaxExpr(LHS: RHS, RHS: Start) : getUMaxExpr(LHS: RHS, RHS: Start);
13639
13640 // See what would happen if we assume the backedge is taken. This is
13641 // used to compute MaxBECount.
13642 BECountIfBackedgeTaken =
13643 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHS, RHS: Start), D: Stride);
13644 }
13645
13646 // At this point, we know:
13647 //
13648 // 1. If IsSigned, Start <=s End; otherwise, Start <=u End
13649 // 2. The index variable doesn't overflow.
13650 //
13651 // Therefore, we know N exists such that
13652 // (Start + Stride * N) >= End, and computing "(Start + Stride * N)"
13653 // doesn't overflow.
13654 //
13655 // Using this information, try to prove whether the addition in
13656 // "(Start - End) + (Stride - 1)" has unsigned overflow.
13657 //
13658 // If the IV cannot overflow, RHS is at least Stride - 1 below the maximum
13659 // value, so the distance End - Start is at most UMAX - (Stride - 1) and
13660 // the (Stride - 1) addition below cannot overflow.
13661 const SCEV *One = getOne(Ty: Stride->getType());
13662 bool MayAddOverflow = IVMayOverflow && [&] {
13663 if (isKnownToBeAPowerOfTwo(S: Stride)) {
13664 // Suppose Stride is a power of two, and Start/End are unsigned
13665 // integers. Let UMAX be the largest representable unsigned
13666 // integer.
13667 //
13668 // By the preconditions of this function, we know
13669 // "(Start + Stride * N) >= End", and this doesn't overflow.
13670 // As a formula:
13671 //
13672 // End <= (Start + Stride * N) <= UMAX
13673 //
13674 // Subtracting Start from all the terms:
13675 //
13676 // End - Start <= Stride * N <= UMAX - Start
13677 //
13678 // Since Start is unsigned, UMAX - Start <= UMAX. Therefore:
13679 //
13680 // End - Start <= Stride * N <= UMAX
13681 //
13682 // Stride * N is a multiple of Stride. Therefore,
13683 //
13684 // End - Start <= Stride * N <= UMAX - (UMAX mod Stride)
13685 //
13686 // Since Stride is a power of two, UMAX + 1 is divisible by
13687 // Stride. Therefore, UMAX mod Stride == Stride - 1. So we can
13688 // write:
13689 //
13690 // End - Start <= Stride * N <= UMAX - Stride - 1
13691 //
13692 // Dropping the middle term:
13693 //
13694 // End - Start <= UMAX - Stride - 1
13695 //
13696 // Adding Stride - 1 to both sides:
13697 //
13698 // (End - Start) + (Stride - 1) <= UMAX
13699 //
13700 // In other words, the addition doesn't have unsigned overflow.
13701 //
13702 // A similar proof works if we treat Start/End as signed values.
13703 // Just rewrite steps before "End - Start <= Stride * N <= UMAX"
13704 // to use signed max instead of unsigned max. Note that we're
13705 // trying to prove a lack of unsigned overflow in either case.
13706 return false;
13707 }
13708 if (Start == Stride || Start == getMinusSCEV(LHS: Stride, RHS: One)) {
13709 // If Start is equal to Stride, (End - Start) + (Stride - 1) == End
13710 // - 1. If !IsSigned, 0 <u Stride == Start <=u End; so 0 <u End - 1
13711 // <u End. If IsSigned, 0 <s Stride == Start <=s End; so 0 <s End -
13712 // 1 <s End.
13713 //
13714 // If Start is equal to Stride - 1, (End - Start) + Stride - 1 ==
13715 // End.
13716 return false;
13717 }
13718 return true;
13719 }();
13720
13721 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13722 if (!MayAddOverflow) {
13723 // floor((D + (S - 1)) / S)
13724 // We prefer this formulation if it's legal because it's fewer
13725 // operations.
13726 BECount =
13727 getUDivExpr(LHS: getAddExpr(LHS: Delta, RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13728 } else {
13729 BECount = getUDivCeilSCEV(N: Delta, D: Stride);
13730 }
13731 }
13732 }
13733
13734 const SCEV *ConstantMaxBECount;
13735 bool MaxOrZero = false;
13736 if (isa<SCEVConstant>(Val: BECount)) {
13737 ConstantMaxBECount = BECount;
13738 } else if (BECountIfBackedgeTaken &&
13739 isa<SCEVConstant>(Val: BECountIfBackedgeTaken)) {
13740 // If we know exactly how many times the backedge will be taken if it's
13741 // taken at least once, then the backedge count will either be that or
13742 // zero.
13743 ConstantMaxBECount = BECountIfBackedgeTaken;
13744 MaxOrZero = true;
13745 } else {
13746 ConstantMaxBECount = computeMaxBECountForLT(
13747 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned);
13748 }
13749
13750 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
13751 !isa<SCEVCouldNotCompute>(Val: BECount))
13752 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
13753
13754 const SCEV *SymbolicMaxBECount =
13755 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13756 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, MaxOrZero,
13757 Predicates);
13758}
13759
13760ScalarEvolution::ExitLimit ScalarEvolution::howManyGreaterThans(
13761 const SCEV *LHS, const SCEV *RHS, const Loop *L, bool IsSigned,
13762 bool ControlsOnlyExit, bool AllowPredicates) {
13763 SmallVector<const SCEVPredicate *> Predicates;
13764 // We handle only IV > Invariant
13765 if (!isLoopInvariant(S: RHS, L))
13766 return getCouldNotCompute();
13767
13768 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13769 if (!IV && AllowPredicates)
13770 // Try to make this an AddRec using runtime tests, in the first X
13771 // iterations of this loop, where X is the SCEV expression found by the
13772 // algorithm below.
13773 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13774
13775 // Avoid weird loops
13776 if (!IV || IV->getLoop() != L || !IV->isAffine())
13777 return getCouldNotCompute();
13778
13779 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13780 bool NoWrap = ControlsOnlyExit && any(Val: IV->getNoWrapFlags(Mask: WrapType));
13781 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13782
13783 const SCEV *Stride = getNegativeSCEV(V: IV->getStepRecurrence(SE&: *this));
13784
13785 // Avoid negative or zero stride values
13786 if (!isKnownPositive(S: Stride))
13787 return getCouldNotCompute();
13788
13789 // Avoid proven overflow cases: this will ensure that the backedge taken count
13790 // will not generate any unsigned overflow. Relaxed no-overflow conditions
13791 // exploit NoWrapFlags, allowing to optimize in presence of undefined
13792 // behaviors like the case of C language.
13793 bool MayAddOverflow = false;
13794 const SCEV *Start = IV->getStart();
13795 const SCEV *End = RHS;
13796 if (!Stride->isOne() && canIVOverflowOnGT(RHS, Stride, IsSigned)) {
13797 if (!NoWrap)
13798 return getCouldNotCompute();
13799 MayAddOverflow = true;
13800 }
13801
13802 if (!isLoopEntryGuardedByCond(L, Pred: Cond, LHS: getAddExpr(LHS: Start, RHS: Stride), RHS)) {
13803 // If we know that Start >= RHS in the context of loop, then we know that
13804 // min(RHS, Start) = RHS at this point.
13805 if (isLoopEntryGuardedByCond(
13806 L, Pred: IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, LHS: Start, RHS))
13807 End = RHS;
13808 else
13809 End = IsSigned ? getSMinExpr(LHS: RHS, RHS: Start) : getUMinExpr(LHS: RHS, RHS: Start);
13810 }
13811
13812 if (Start->getType()->isPointerTy()) {
13813 Start = getPtrToAddrExpr(Op: Start);
13814 if (isa<SCEVCouldNotCompute>(Val: Start))
13815 return Start;
13816 }
13817 if (End->getType()->isPointerTy()) {
13818 End = getPtrToAddrExpr(Op: End);
13819 if (isa<SCEVCouldNotCompute>(Val: End))
13820 return End;
13821 }
13822
13823 const SCEV *Delta = getMinusSCEV(LHS: Start, RHS: End);
13824 const SCEV *BECount;
13825 if (MayAddOverflow) {
13826 // The ceiling division instead needs Start >= End, so that (Start - End) is
13827 // the exact unsigned distance between them.
13828 if (!isLoopEntryGuardedByCond(
13829 L, Pred: IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, LHS: Start, RHS: End))
13830 return getCouldNotCompute();
13831 BECount = getUDivCeilSCEV(N: Delta, D: Stride);
13832 } else {
13833 // Compute ((Start - End) + (Stride - 1)) / Stride, if the IV cannot
13834 // overflow as it requires fewer operations.
13835 const SCEV *One = getOne(Ty: Stride->getType());
13836 BECount = getUDivExpr(LHS: getAddExpr(LHS: Delta, RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13837 }
13838
13839 APInt MaxStart = IsSigned ? getSignedRangeMax(S: Start)
13840 : getUnsignedRangeMax(S: Start);
13841
13842 APInt MinStride = IsSigned ? getSignedRangeMin(S: Stride)
13843 : getUnsignedRangeMin(S: Stride);
13844
13845 unsigned BitWidth = getTypeSizeInBits(Ty: LHS->getType());
13846 APInt Limit = IsSigned ? APInt::getSignedMinValue(numBits: BitWidth) + (MinStride - 1)
13847 : APInt::getMinValue(numBits: BitWidth) + (MinStride - 1);
13848
13849 // Although End can be a MIN expression we estimate MinEnd considering only
13850 // the case End = RHS. This is safe because in the other case (Start - End)
13851 // is zero, leading to a zero maximum backedge taken count.
13852 APInt MinEnd =
13853 IsSigned ? APIntOps::smax(A: getSignedRangeMin(S: RHS), B: Limit)
13854 : APIntOps::umax(A: getUnsignedRangeMin(S: RHS), B: Limit);
13855
13856 const SCEV *ConstantMaxBECount =
13857 isa<SCEVConstant>(Val: BECount)
13858 ? BECount
13859 : getUDivCeilSCEV(N: getConstant(Val: MaxStart - MinEnd),
13860 D: getConstant(Val: MinStride));
13861
13862 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount))
13863 ConstantMaxBECount = BECount;
13864 const SCEV *SymbolicMaxBECount =
13865 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13866
13867 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
13868 Predicates);
13869}
13870
13871const SCEV *SCEVAddRecExpr::getNumIterationsInRange(const ConstantRange &Range,
13872 ScalarEvolution &SE) const {
13873 if (Range.isFullSet()) // Infinite loop.
13874 return SE.getCouldNotCompute();
13875
13876 // If the start is a non-zero constant, shift the range to simplify things.
13877 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: getStart()))
13878 if (!SC->getValue()->isZero()) {
13879 SmallVector<SCEVUse, 4> Operands(operands());
13880 Operands[0] = SE.getZero(Ty: SC->getType());
13881 const SCEV *Shifted = SE.getAddRecExpr(Operands, L: getLoop(),
13882 Flags: getNoWrapFlags(Mask: FlagNW));
13883 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Val: Shifted))
13884 return ShiftedAddRec->getNumIterationsInRange(
13885 Range: Range.subtract(CI: SC->getAPInt()), SE);
13886 // This is strange and shouldn't happen.
13887 return SE.getCouldNotCompute();
13888 }
13889
13890 // The only time we can solve this is when we have all constant indices.
13891 // Otherwise, we cannot determine the overflow conditions.
13892 if (any_of(Range: operands(), P: [](const SCEV *Op) { return !isa<SCEVConstant>(Val: Op); }))
13893 return SE.getCouldNotCompute();
13894
13895 // Okay at this point we know that all elements of the chrec are constants and
13896 // that the start element is zero.
13897
13898 // First check to see if the range contains zero. If not, the first
13899 // iteration exits.
13900 unsigned BitWidth = SE.getTypeSizeInBits(Ty: getType());
13901 if (!Range.contains(Val: APInt(BitWidth, 0)))
13902 return SE.getZero(Ty: getType());
13903
13904 if (isAffine()) {
13905 // If this is an affine expression then we have this situation:
13906 // Solve {0,+,A} in Range === Ax in Range
13907
13908 // We know that zero is in the range. If A is positive then we know that
13909 // the upper value of the range must be the first possible exit value.
13910 // If A is negative then the lower of the range is the last possible loop
13911 // value. Also note that we already checked for a full range.
13912 APInt A = cast<SCEVConstant>(Val: getOperand(i: 1))->getAPInt();
13913 APInt End = A.sge(RHS: 1) ? (Range.getUpper() - 1) : Range.getLower();
13914
13915 // The exit value should be (End+A)/A.
13916 APInt ExitVal = (End + A).udiv(RHS: A);
13917 ConstantInt *ExitValue = ConstantInt::get(Context&: SE.getContext(), V: ExitVal);
13918
13919 // Evaluate at the exit value. If we really did fall out of the valid
13920 // range, then we computed our trip count, otherwise wrap around or other
13921 // things must have happened.
13922 ConstantInt *Val = EvaluateConstantChrecAtConstant(AddRec: this, C: ExitValue, SE);
13923 if (Range.contains(Val: Val->getValue()))
13924 return SE.getCouldNotCompute(); // Something strange happened
13925
13926 // Ensure that the previous value is in the range.
13927 assert(Range.contains(
13928 EvaluateConstantChrecAtConstant(this,
13929 ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) &&
13930 "Linear scev computation is off in a bad way!");
13931 return SE.getConstant(V: ExitValue);
13932 }
13933
13934 if (isQuadratic()) {
13935 if (auto S = SolveQuadraticAddRecRange(AddRec: this, Range, SE))
13936 return SE.getConstant(Val: *S);
13937 }
13938
13939 return SE.getCouldNotCompute();
13940}
13941
13942const SCEVAddRecExpr *
13943SCEVAddRecExpr::getPostIncExpr(ScalarEvolution &SE) const {
13944 assert(getNumOperands() > 1 && "AddRec with zero step?");
13945 // There is a temptation to just call getAddExpr(this, getStepRecurrence(SE)),
13946 // but in this case we cannot guarantee that the value returned will be an
13947 // AddRec because SCEV does not have a fixed point where it stops
13948 // simplification: it is legal to return ({rec1} + {rec2}). For example, it
13949 // may happen if we reach arithmetic depth limit while simplifying. So we
13950 // construct the returned value explicitly.
13951 SmallVector<SCEVUse, 3> Ops;
13952 // If this is {A,+,B,+,C,...,+,N}, then its step is {B,+,C,+,...,+,N}, and
13953 // (this + Step) is {A+B,+,B+C,+...,+,N}.
13954 for (unsigned i = 0, e = getNumOperands() - 1; i < e; ++i)
13955 Ops.push_back(Elt: SE.getAddExpr(LHS: getOperand(i), RHS: getOperand(i: i + 1)));
13956 // We know that the last operand is not a constant zero (otherwise it would
13957 // have been popped out earlier). This guarantees us that if the result has
13958 // the same last operand, then it will also not be popped out, meaning that
13959 // the returned value will be an AddRec.
13960 const SCEV *Last = getOperand(i: getNumOperands() - 1);
13961 assert(!Last->isZero() && "Recurrency with zero step?");
13962 Ops.push_back(Elt: Last);
13963 return cast<SCEVAddRecExpr>(Val: SE.getAddRecExpr(Operands&: Ops, L: getLoop(),
13964 Flags: SCEV::FlagAnyWrap));
13965}
13966
13967// Return true when S contains at least an undef value.
13968bool ScalarEvolution::containsUndefs(const SCEV *S) const {
13969 return SCEVExprContains(
13970 Root: S, Pred: [](const SCEV *S) { return match(S, P: m_scev_UndefOrPoison()); });
13971}
13972
13973// Return true when S contains a value that is a nullptr.
13974bool ScalarEvolution::containsErasedValue(const SCEV *S) const {
13975 return SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
13976 if (const auto *SU = dyn_cast<SCEVUnknown>(Val: S))
13977 return SU->getValue() == nullptr;
13978 return false;
13979 });
13980}
13981
13982/// Return the size of an element read or written by Inst.
13983const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) {
13984 Type *Ty;
13985 Type *PtrTy;
13986 if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst)) {
13987 Ty = Store->getValueOperand()->getType();
13988 PtrTy = Store->getPointerOperandType();
13989 } else if (LoadInst *Load = dyn_cast<LoadInst>(Val: Inst)) {
13990 Ty = Load->getType();
13991 PtrTy = Load->getPointerOperandType();
13992 } else {
13993 return nullptr;
13994 }
13995
13996 Type *ETy = getEffectiveSCEVType(Ty: PtrTy);
13997 return getSizeOfExpr(IntTy: ETy, AllocTy: Ty);
13998}
13999
14000//===----------------------------------------------------------------------===//
14001// SCEVCallbackVH Class Implementation
14002//===----------------------------------------------------------------------===//
14003
14004void ScalarEvolution::SCEVCallbackVH::deleted() {
14005 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
14006 if (PHINode *PN = dyn_cast<PHINode>(Val: getValPtr()))
14007 SE->ConstantEvolutionLoopExitValue.erase(Val: PN);
14008 SE->eraseValueFromMap(V: getValPtr());
14009 // this now dangles!
14010}
14011
14012void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
14013 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
14014
14015 // Forget all the expressions associated with users of the old value,
14016 // so that future queries will recompute the expressions using the new
14017 // value.
14018 SE->forgetValue(V: getValPtr());
14019 // this now dangles!
14020}
14021
14022ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
14023 : CallbackVH(V), SE(se) {}
14024
14025//===----------------------------------------------------------------------===//
14026// ScalarEvolution Class Implementation
14027//===----------------------------------------------------------------------===//
14028
14029ScalarEvolution::ScalarEvolution(Function &F, TargetLibraryInfo &TLI,
14030 AssumptionCache &AC, DominatorTree &DT,
14031 LoopInfo &LI)
14032 : F(F), DL(F.getDataLayout()), TLI(TLI), AC(AC), DT(DT), LI(LI),
14033 CouldNotCompute(new SCEVCouldNotCompute()), ValuesAtScopes(64),
14034 LoopDispositions(64), BlockDispositions(64) {
14035 // To use guards for proving predicates, we need to scan every instruction in
14036 // relevant basic blocks, and not just terminators. Doing this is a waste of
14037 // time if the IR does not actually contain any calls to
14038 // @llvm.experimental.guard, so do a quick check and remember this beforehand.
14039 //
14040 // This pessimizes the case where a pass that preserves ScalarEvolution wants
14041 // to _add_ guards to the module when there weren't any before, and wants
14042 // ScalarEvolution to optimize based on those guards. For now we prefer to be
14043 // efficient in lieu of being smart in that rather obscure case.
14044
14045 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
14046 M: F.getParent(), id: Intrinsic::experimental_guard);
14047 HasGuards = GuardDecl && !GuardDecl->use_empty();
14048}
14049
14050ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg)
14051 : F(Arg.F), DL(Arg.DL), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC),
14052 DT(Arg.DT), LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)),
14053 ValueExprMap(std::move(Arg.ValueExprMap)),
14054 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
14055 PendingMerges(std::move(Arg.PendingMerges)),
14056 ConstantMultipleCache(std::move(Arg.ConstantMultipleCache)),
14057 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
14058 PredicatedBackedgeTakenCounts(
14059 std::move(Arg.PredicatedBackedgeTakenCounts)),
14060 BECountUsers(std::move(Arg.BECountUsers)),
14061 ConstantEvolutionLoopExitValue(
14062 std::move(Arg.ConstantEvolutionLoopExitValue)),
14063 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
14064 ValuesAtScopesUsers(std::move(Arg.ValuesAtScopesUsers)),
14065 LoopDispositions(std::move(Arg.LoopDispositions)),
14066 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
14067 BlockDispositions(std::move(Arg.BlockDispositions)),
14068 SCEVUsers(std::move(Arg.SCEVUsers)),
14069 UnsignedRanges(std::move(Arg.UnsignedRanges)),
14070 SignedRanges(std::move(Arg.SignedRanges)),
14071 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
14072 UniquePreds(std::move(Arg.UniquePreds)),
14073 SCEVAllocator(std::move(Arg.SCEVAllocator)),
14074 ConstantSCEVs(std::move(Arg.ConstantSCEVs)),
14075 LoopUsers(std::move(Arg.LoopUsers)),
14076 PredicatedSCEVRewrites(std::move(Arg.PredicatedSCEVRewrites)),
14077 FirstUnknown(Arg.FirstUnknown) {
14078 Arg.FirstUnknown = nullptr;
14079}
14080
14081ScalarEvolution::~ScalarEvolution() {
14082 // Iterate through all the SCEVUnknown instances and call their
14083 // destructors, so that they release their references to their values.
14084 for (SCEVUnknown *U = FirstUnknown; U;) {
14085 SCEVUnknown *Tmp = U;
14086 U = U->Next;
14087 Tmp->~SCEVUnknown();
14088 }
14089 FirstUnknown = nullptr;
14090
14091 ExprValueMap.clear();
14092 ValueExprMap.clear();
14093 HasRecMap.clear();
14094 BackedgeTakenCounts.clear();
14095 PredicatedBackedgeTakenCounts.clear();
14096
14097 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage");
14098 assert(PendingMerges.empty() && "isImpliedViaMerge garbage");
14099 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!");
14100 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!");
14101}
14102
14103bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) {
14104 return !isa<SCEVCouldNotCompute>(Val: getBackedgeTakenCount(L));
14105}
14106
14107/// When printing a top-level SCEV for trip counts, it's helpful to include
14108/// a type for constants which are otherwise hard to disambiguate.
14109static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV* S) {
14110 if (isa<SCEVConstant>(Val: S))
14111 OS << *S->getType() << " ";
14112 OS << *S;
14113}
14114
14115static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE,
14116 const Loop *L) {
14117 // Print all inner loops first
14118 for (Loop *I : *L)
14119 PrintLoopInfo(OS, SE, L: I);
14120
14121 OS << "Loop ";
14122 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14123 OS << ": ";
14124
14125 SmallVector<BasicBlock *, 8> ExitingBlocks;
14126 L->getExitingBlocks(ExitingBlocks);
14127 if (ExitingBlocks.size() != 1)
14128 OS << "<multiple exits> ";
14129
14130 auto *BTC = SE->getBackedgeTakenCount(L);
14131 if (!isa<SCEVCouldNotCompute>(Val: BTC)) {
14132 OS << "backedge-taken count is ";
14133 PrintSCEVWithTypeHint(OS, S: BTC);
14134 } else
14135 OS << "Unpredictable backedge-taken count.";
14136 OS << "\n";
14137
14138 if (ExitingBlocks.size() > 1)
14139 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14140 OS << " exit count for " << ExitingBlock->getName() << ": ";
14141 const SCEV *EC = SE->getExitCount(L, ExitingBlock);
14142 PrintSCEVWithTypeHint(OS, S: EC);
14143 if (isa<SCEVCouldNotCompute>(Val: EC)) {
14144 // Retry with predicates.
14145 SmallVector<const SCEVPredicate *> Predicates;
14146 EC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates);
14147 if (!isa<SCEVCouldNotCompute>(Val: EC)) {
14148 OS << "\n predicated exit count for " << ExitingBlock->getName()
14149 << ": ";
14150 PrintSCEVWithTypeHint(OS, S: EC);
14151 OS << "\n Predicates:\n";
14152 for (const auto *P : Predicates)
14153 P->print(OS, Depth: 4);
14154 }
14155 }
14156 OS << "\n";
14157 }
14158
14159 OS << "Loop ";
14160 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14161 OS << ": ";
14162
14163 auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L);
14164 if (!isa<SCEVCouldNotCompute>(Val: ConstantBTC)) {
14165 OS << "constant max backedge-taken count is ";
14166 PrintSCEVWithTypeHint(OS, S: ConstantBTC);
14167 if (SE->isBackedgeTakenCountMaxOrZero(L))
14168 OS << ", actual taken count either this or zero.";
14169 } else {
14170 OS << "Unpredictable constant max backedge-taken count. ";
14171 }
14172
14173 OS << "\n"
14174 "Loop ";
14175 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14176 OS << ": ";
14177
14178 auto *SymbolicBTC = SE->getSymbolicMaxBackedgeTakenCount(L);
14179 if (!isa<SCEVCouldNotCompute>(Val: SymbolicBTC)) {
14180 OS << "symbolic max backedge-taken count is ";
14181 PrintSCEVWithTypeHint(OS, S: SymbolicBTC);
14182 if (SE->isBackedgeTakenCountMaxOrZero(L))
14183 OS << ", actual taken count either this or zero.";
14184 } else {
14185 OS << "Unpredictable symbolic max backedge-taken count. ";
14186 }
14187 OS << "\n";
14188
14189 if (ExitingBlocks.size() > 1)
14190 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14191 OS << " symbolic max exit count for " << ExitingBlock->getName() << ": ";
14192 auto *ExitBTC = SE->getExitCount(L, ExitingBlock,
14193 Kind: ScalarEvolution::SymbolicMaximum);
14194 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14195 if (isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14196 // Retry with predicates.
14197 SmallVector<const SCEVPredicate *> Predicates;
14198 ExitBTC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates,
14199 Kind: ScalarEvolution::SymbolicMaximum);
14200 if (!isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14201 OS << "\n predicated symbolic max exit count for "
14202 << ExitingBlock->getName() << ": ";
14203 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14204 OS << "\n Predicates:\n";
14205 for (const auto *P : Predicates)
14206 P->print(OS, Depth: 4);
14207 }
14208 }
14209 OS << "\n";
14210 }
14211
14212 SmallVector<const SCEVPredicate *, 4> Preds;
14213 auto *PBT = SE->getPredicatedBackedgeTakenCount(L, Preds);
14214 if (PBT != BTC) {
14215 OS << "Loop ";
14216 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14217 OS << ": ";
14218 if (!isa<SCEVCouldNotCompute>(Val: PBT)) {
14219 OS << "Predicated backedge-taken count is ";
14220 PrintSCEVWithTypeHint(OS, S: PBT);
14221 } else
14222 OS << "Unpredictable predicated backedge-taken count.";
14223 OS << "\n";
14224 OS << " Predicates:\n";
14225 for (const auto *P : Preds)
14226 P->print(OS, Depth: 4);
14227 }
14228 Preds.clear();
14229
14230 auto *PredConstantMax =
14231 SE->getPredicatedConstantMaxBackedgeTakenCount(L, Preds);
14232 if (PredConstantMax != ConstantBTC) {
14233 OS << "Loop ";
14234 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14235 OS << ": ";
14236 if (!isa<SCEVCouldNotCompute>(Val: PredConstantMax)) {
14237 OS << "Predicated constant max backedge-taken count is ";
14238 PrintSCEVWithTypeHint(OS, S: PredConstantMax);
14239 } else
14240 OS << "Unpredictable predicated constant max backedge-taken count.";
14241 OS << "\n";
14242 OS << " Predicates:\n";
14243 for (const auto *P : Preds)
14244 P->print(OS, Depth: 4);
14245 }
14246 Preds.clear();
14247
14248 auto *PredSymbolicMax =
14249 SE->getPredicatedSymbolicMaxBackedgeTakenCount(L, Preds);
14250 if (SymbolicBTC != PredSymbolicMax) {
14251 OS << "Loop ";
14252 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14253 OS << ": ";
14254 if (!isa<SCEVCouldNotCompute>(Val: PredSymbolicMax)) {
14255 OS << "Predicated symbolic max backedge-taken count is ";
14256 PrintSCEVWithTypeHint(OS, S: PredSymbolicMax);
14257 } else
14258 OS << "Unpredictable predicated symbolic max backedge-taken count.";
14259 OS << "\n";
14260 OS << " Predicates:\n";
14261 for (const auto *P : Preds)
14262 P->print(OS, Depth: 4);
14263 }
14264
14265 if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
14266 OS << "Loop ";
14267 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14268 OS << ": ";
14269 OS << "Trip multiple is " << SE->getSmallConstantTripMultiple(L) << "\n";
14270 }
14271}
14272
14273namespace llvm {
14274// Note: these overloaded operators need to be in the llvm namespace for them
14275// to be resolved correctly. If we put them outside the llvm namespace, the
14276//
14277// OS << ": " << SE.getLoopDisposition(SV, InnerL);
14278//
14279// code below "breaks" and start printing raw enum values as opposed to the
14280// string values.
14281static raw_ostream &operator<<(raw_ostream &OS,
14282 ScalarEvolution::LoopDisposition LD) {
14283 switch (LD) {
14284 case ScalarEvolution::LoopVariant:
14285 OS << "Variant";
14286 break;
14287 case ScalarEvolution::LoopInvariant:
14288 OS << "Invariant";
14289 break;
14290 case ScalarEvolution::LoopUniform:
14291 OS << "Uniform";
14292 break;
14293 case ScalarEvolution::LoopComputable:
14294 OS << "Computable";
14295 break;
14296 }
14297 return OS;
14298}
14299
14300static raw_ostream &operator<<(raw_ostream &OS,
14301 llvm::ScalarEvolution::BlockDisposition BD) {
14302 switch (BD) {
14303 case ScalarEvolution::DoesNotDominateBlock:
14304 OS << "DoesNotDominate";
14305 break;
14306 case ScalarEvolution::DominatesBlock:
14307 OS << "Dominates";
14308 break;
14309 case ScalarEvolution::ProperlyDominatesBlock:
14310 OS << "ProperlyDominates";
14311 break;
14312 }
14313 return OS;
14314}
14315} // namespace llvm
14316
14317void ScalarEvolution::print(raw_ostream &OS) const {
14318 // ScalarEvolution's implementation of the print method is to print
14319 // out SCEV values of all instructions that are interesting. Doing
14320 // this potentially causes it to create new SCEV objects though,
14321 // which technically conflicts with the const qualifier. This isn't
14322 // observable from outside the class though, so casting away the
14323 // const isn't dangerous.
14324 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14325
14326 if (ClassifyExpressions) {
14327 OS << "Classifying expressions for: ";
14328 F.printAsOperand(O&: OS, /*PrintType=*/false);
14329 OS << "\n";
14330 for (Instruction &I : instructions(F))
14331 if (isSCEVable(Ty: I.getType()) && !isa<CmpInst>(Val: I)) {
14332 OS << I << '\n';
14333 OS << " --> ";
14334 const SCEV *SV = SE.getSCEV(V: &I);
14335 SV->print(OS);
14336 if (!isa<SCEVCouldNotCompute>(Val: SV)) {
14337 OS << " U: ";
14338 SE.getUnsignedRange(S: SV).print(OS);
14339 OS << " S: ";
14340 SE.getSignedRange(S: SV).print(OS);
14341 }
14342
14343 const Loop *L = LI.getLoopFor(BB: I.getParent());
14344
14345 SCEVUse AtUse = SE.getSCEVAtScope(V: SV, L);
14346 if (AtUse != SV) {
14347 OS << " --> ";
14348 OS << AtUse;
14349 if (!isa<SCEVCouldNotCompute>(Val: AtUse)) {
14350 OS << " U: ";
14351 SE.getUnsignedRange(S: AtUse).print(OS);
14352 OS << " S: ";
14353 SE.getSignedRange(S: AtUse).print(OS);
14354 }
14355 }
14356
14357 if (L) {
14358 OS << "\t\t" "Exits: ";
14359 SCEVUse ExitValue = SE.getSCEVAtScope(V: SV, L: L->getParentLoop());
14360 if (!SE.isLoopInvariant(S: ExitValue, L)) {
14361 OS << "<<Unknown>>";
14362 } else {
14363 OS << ExitValue;
14364 }
14365
14366 ListSeparator LS(", ", "\t\tLoopDispositions: { ");
14367 for (const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
14368 OS << LS;
14369 Iter->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14370 OS << ": " << SE.getLoopDisposition(S: SV, L: Iter);
14371 }
14372
14373 for (const auto *InnerL : depth_first(G: L)) {
14374 if (InnerL == L)
14375 continue;
14376 OS << LS;
14377 InnerL->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14378 OS << ": " << SE.getLoopDisposition(S: SV, L: InnerL);
14379 }
14380
14381 OS << " }";
14382 }
14383
14384 OS << "\n";
14385 }
14386 }
14387
14388 OS << "Determining loop execution counts for: ";
14389 F.printAsOperand(O&: OS, /*PrintType=*/false);
14390 OS << "\n";
14391 for (Loop *I : LI)
14392 PrintLoopInfo(OS, SE: &SE, L: I);
14393}
14394
14395ScalarEvolution::LoopDisposition
14396ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) {
14397 auto &Values = LoopDispositions[S];
14398 for (auto &V : Values) {
14399 if (V.getPointer() == L)
14400 return V.getInt();
14401 }
14402 Values.emplace_back(Args&: L, Args: LoopVariant);
14403 LoopDisposition D = computeLoopDisposition(S, L);
14404 auto &Values2 = LoopDispositions[S];
14405 for (auto &V : llvm::reverse(C&: Values2)) {
14406 if (V.getPointer() == L) {
14407 V.setInt(D);
14408 break;
14409 }
14410 }
14411 return D;
14412}
14413
14414ScalarEvolution::LoopDisposition
14415ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
14416 switch (S->getSCEVType()) {
14417 case scConstant:
14418 case scVScale:
14419 return LoopInvariant;
14420 case scAddRecExpr: {
14421 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14422
14423 // If L is the addrec's loop, it's computable.
14424 if (AR->getLoop() == L)
14425 return LoopComputable;
14426
14427 // Add recurrences are never invariant in the function-body (null loop).
14428 if (!L)
14429 return LoopVariant;
14430
14431 // Everything that is not defined at loop entry is variant.
14432 if (DT.dominates(A: L->getHeader(), B: AR->getLoop()->getHeader())) {
14433 if (L->contains(L: AR->getLoop()) &&
14434 llvm::all_of(Range: AR->operands(),
14435 P: [&](const SCEV *Op) { return isLoopUniform(S: Op, L); }))
14436 return LoopUniform;
14437
14438 return LoopVariant;
14439 }
14440 assert(!L->contains(AR->getLoop()) && "Containing loop's header does not"
14441 " dominate the contained loop's header?");
14442
14443 // This recurrence is invariant w.r.t. L if AR's loop contains L.
14444 if (AR->getLoop()->contains(L))
14445 return LoopInvariant;
14446
14447 // This recurrence is variant w.r.t. L if any of its operands
14448 // are variant.
14449 for (SCEVUse Op : AR->operands())
14450 if (!isLoopInvariant(S: Op, L))
14451 return LoopVariant;
14452
14453 // Otherwise it's loop-invariant.
14454 return LoopInvariant;
14455 }
14456 case scTruncate:
14457 case scZeroExtend:
14458 case scSignExtend:
14459 case scPtrToAddr:
14460 case scAddExpr:
14461 case scMulExpr:
14462 case scUDivExpr:
14463 case scUMaxExpr:
14464 case scSMaxExpr:
14465 case scUMinExpr:
14466 case scSMinExpr:
14467 case scSequentialUMinExpr: {
14468 bool HasVarying = false;
14469 bool HasUniform = false;
14470 for (SCEVUse Op : S->operands()) {
14471 LoopDisposition D = getLoopDisposition(S: Op, L);
14472 if (D == LoopVariant)
14473 return LoopVariant;
14474 if (D == LoopComputable)
14475 HasVarying = true;
14476 if (D == LoopUniform)
14477 HasUniform = true;
14478 }
14479 return HasVarying ? (HasUniform ? LoopVariant : LoopComputable)
14480 : (HasUniform ? LoopUniform : LoopInvariant);
14481 }
14482 case scUnknown:
14483 // All non-instruction values are loop invariant. All instructions are loop
14484 // invariant if they are not contained in the specified loop.
14485 // Instructions are never considered invariant in the function body
14486 // (null loop) because they are defined within the "loop".
14487 if (auto *I = dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue()))
14488 return (L && !L->contains(Inst: I)) ? LoopInvariant : LoopVariant;
14489 return LoopInvariant;
14490 case scCouldNotCompute:
14491 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14492 }
14493 llvm_unreachable("Unknown SCEV kind!");
14494}
14495
14496bool ScalarEvolution::isLoopUniform(const SCEV *S, const Loop *L) {
14497 LoopDisposition D = getLoopDisposition(S, L);
14498 return D == LoopUniform || D == LoopInvariant;
14499}
14500
14501bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) {
14502 return getLoopDisposition(S, L) == LoopInvariant;
14503}
14504
14505bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) {
14506 return getLoopDisposition(S, L) == LoopComputable;
14507}
14508
14509ScalarEvolution::BlockDisposition
14510ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14511 auto &Values = BlockDispositions[S];
14512 for (auto &V : Values) {
14513 if (V.getPointer() == BB)
14514 return V.getInt();
14515 }
14516 Values.emplace_back(Args&: BB, Args: DoesNotDominateBlock);
14517 BlockDisposition D = computeBlockDisposition(S, BB);
14518 auto &Values2 = BlockDispositions[S];
14519 for (auto &V : llvm::reverse(C&: Values2)) {
14520 if (V.getPointer() == BB) {
14521 V.setInt(D);
14522 break;
14523 }
14524 }
14525 return D;
14526}
14527
14528ScalarEvolution::BlockDisposition
14529ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14530 switch (S->getSCEVType()) {
14531 case scConstant:
14532 case scVScale:
14533 return ProperlyDominatesBlock;
14534 case scAddRecExpr: {
14535 // This uses a "dominates" query instead of "properly dominates" query
14536 // to test for proper dominance too, because the instruction which
14537 // produces the addrec's value is a PHI, and a PHI effectively properly
14538 // dominates its entire containing block.
14539 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14540 if (!DT.dominates(A: AR->getLoop()->getHeader(), B: BB))
14541 return DoesNotDominateBlock;
14542
14543 // Fall through into SCEVNAryExpr handling.
14544 [[fallthrough]];
14545 }
14546 case scTruncate:
14547 case scZeroExtend:
14548 case scSignExtend:
14549 case scPtrToAddr:
14550 case scAddExpr:
14551 case scMulExpr:
14552 case scUDivExpr:
14553 case scUMaxExpr:
14554 case scSMaxExpr:
14555 case scUMinExpr:
14556 case scSMinExpr:
14557 case scSequentialUMinExpr: {
14558 bool Proper = true;
14559 for (const SCEV *NAryOp : S->operands()) {
14560 BlockDisposition D = getBlockDisposition(S: NAryOp, BB);
14561 if (D == DoesNotDominateBlock)
14562 return DoesNotDominateBlock;
14563 if (D == DominatesBlock)
14564 Proper = false;
14565 }
14566 return Proper ? ProperlyDominatesBlock : DominatesBlock;
14567 }
14568 case scUnknown:
14569 if (Instruction *I =
14570 dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue())) {
14571 if (I->getParent() == BB)
14572 return DominatesBlock;
14573 if (DT.properlyDominates(A: I->getParent(), B: BB))
14574 return ProperlyDominatesBlock;
14575 return DoesNotDominateBlock;
14576 }
14577 return ProperlyDominatesBlock;
14578 case scCouldNotCompute:
14579 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14580 }
14581 llvm_unreachable("Unknown SCEV kind!");
14582}
14583
14584bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
14585 return getBlockDisposition(S, BB) >= DominatesBlock;
14586}
14587
14588bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) {
14589 return getBlockDisposition(S, BB) == ProperlyDominatesBlock;
14590}
14591
14592bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const {
14593 return SCEVExprContains(Root: S, Pred: [&](const SCEV *Expr) { return Expr == Op; });
14594}
14595
14596void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
14597 bool Predicated) {
14598 auto &BECounts =
14599 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14600 auto It = BECounts.find(Val: L);
14601 if (It != BECounts.end()) {
14602 for (const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
14603 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14604 if (!isa<SCEVConstant>(Val: S)) {
14605 auto UserIt = BECountUsers.find(Val: S);
14606 assert(UserIt != BECountUsers.end());
14607 UserIt->second.erase(Ptr: {L, Predicated});
14608 }
14609 }
14610 }
14611 BECounts.erase(I: It);
14612 }
14613}
14614
14615void ScalarEvolution::forgetMemoizedResults(ArrayRef<SCEVUse> SCEVs) {
14616 SmallPtrSet<const SCEV *, 8> ToForget(llvm::from_range, SCEVs);
14617 SmallVector<SCEVUse, 8> Worklist(ToForget.begin(), ToForget.end());
14618
14619 while (!Worklist.empty()) {
14620 const SCEV *Curr = Worklist.pop_back_val();
14621 auto Users = SCEVUsers.find(Val: Curr);
14622 if (Users != SCEVUsers.end())
14623 for (const auto *User : Users->second)
14624 if (ToForget.insert(Ptr: User).second)
14625 Worklist.push_back(Elt: User);
14626 }
14627
14628 for (const auto *S : ToForget)
14629 forgetMemoizedResultsImpl(S);
14630
14631 PredicatedSCEVRewrites.remove_if(
14632 Pred: [&](const auto &Entry) { return ToForget.count(Ptr: Entry.first.first); });
14633}
14634
14635void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) {
14636 LoopDispositions.erase(Val: S);
14637 BlockDispositions.erase(Val: S);
14638 UnsignedRanges.erase(Val: S);
14639 SignedRanges.erase(Val: S);
14640 HasRecMap.erase(Val: S);
14641 ConstantMultipleCache.erase(Val: S);
14642
14643 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
14644 UnsignedWrapViaInductionTried.erase(Ptr: AR);
14645 SignedWrapViaInductionTried.erase(Ptr: AR);
14646 }
14647
14648 auto ExprIt = ExprValueMap.find(Val: S);
14649 if (ExprIt != ExprValueMap.end()) {
14650 for (Value *V : ExprIt->second) {
14651 auto ValueIt = ValueExprMap.find_as(Val: V);
14652 if (ValueIt != ValueExprMap.end())
14653 ValueExprMap.erase(I: ValueIt);
14654 }
14655 ExprValueMap.erase(I: ExprIt);
14656 }
14657
14658 auto ScopeIt = ValuesAtScopes.find(Val: S);
14659 if (ScopeIt != ValuesAtScopes.end()) {
14660 for (const auto &Pair : ScopeIt->second)
14661 if (!isa_and_nonnull<SCEVConstant>(Val: Pair.second))
14662 llvm::erase(C&: ValuesAtScopesUsers[Pair.second],
14663 V: std::make_pair(x: Pair.first, y&: S));
14664 ValuesAtScopes.erase(I: ScopeIt);
14665 }
14666
14667 auto ScopeUserIt = ValuesAtScopesUsers.find(Val: S);
14668 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
14669 for (const auto &Pair : ScopeUserIt->second)
14670 llvm::erase(C&: ValuesAtScopes[Pair.second], V: std::make_pair(x: Pair.first, y&: S));
14671 ValuesAtScopesUsers.erase(I: ScopeUserIt);
14672 }
14673
14674 auto BEUsersIt = BECountUsers.find(Val: S);
14675 if (BEUsersIt != BECountUsers.end()) {
14676 // Work on a copy, as forgetBackedgeTakenCounts() will modify the original.
14677 auto Copy = BEUsersIt->second;
14678 for (const auto &Pair : Copy)
14679 forgetBackedgeTakenCounts(L: Pair.getPointer(), Predicated: Pair.getInt());
14680 BECountUsers.erase(I: BEUsersIt);
14681 }
14682
14683 auto FoldUser = FoldCacheUser.find(Val: S);
14684 if (FoldUser != FoldCacheUser.end())
14685 for (auto &KV : FoldUser->second)
14686 FoldCache.erase(Val: KV);
14687 FoldCacheUser.erase(Val: S);
14688}
14689
14690void
14691ScalarEvolution::getUsedLoops(const SCEV *S,
14692 SmallPtrSetImpl<const Loop *> &LoopsUsed) {
14693 struct FindUsedLoops {
14694 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
14695 : LoopsUsed(LoopsUsed) {}
14696 SmallPtrSetImpl<const Loop *> &LoopsUsed;
14697 bool follow(const SCEV *S) {
14698 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S))
14699 LoopsUsed.insert(Ptr: AR->getLoop());
14700 return true;
14701 }
14702
14703 bool isDone() const { return false; }
14704 };
14705
14706 FindUsedLoops F(LoopsUsed);
14707 SCEVTraversal<FindUsedLoops>(F).visitAll(Root: S);
14708}
14709
14710void ScalarEvolution::getReachableBlocks(
14711 SmallPtrSetImpl<BasicBlock *> &Reachable, Function &F) {
14712 SmallVector<BasicBlock *> Worklist;
14713 Worklist.push_back(Elt: &F.getEntryBlock());
14714 while (!Worklist.empty()) {
14715 BasicBlock *BB = Worklist.pop_back_val();
14716 if (!Reachable.insert(Ptr: BB).second)
14717 continue;
14718
14719 Value *Cond;
14720 BasicBlock *TrueBB, *FalseBB;
14721 if (match(V: BB->getTerminator(), P: m_Br(C: m_Value(V&: Cond), T: m_BasicBlock(V&: TrueBB),
14722 F: m_BasicBlock(V&: FalseBB)))) {
14723 if (auto *C = dyn_cast<ConstantInt>(Val: Cond)) {
14724 Worklist.push_back(Elt: C->isOne() ? TrueBB : FalseBB);
14725 continue;
14726 }
14727
14728 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
14729 const SCEV *L = getSCEV(V: Cmp->getOperand(i_nocapture: 0));
14730 const SCEV *R = getSCEV(V: Cmp->getOperand(i_nocapture: 1));
14731 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getCmpPredicate(), LHS: L, RHS: R)) {
14732 Worklist.push_back(Elt: TrueBB);
14733 continue;
14734 }
14735 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getInverseCmpPredicate(), LHS: L,
14736 RHS: R)) {
14737 Worklist.push_back(Elt: FalseBB);
14738 continue;
14739 }
14740 }
14741 }
14742
14743 append_range(C&: Worklist, R: successors(BB));
14744 }
14745}
14746
14747void ScalarEvolution::verify() const {
14748 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14749 ScalarEvolution SE2(F, TLI, AC, DT, LI);
14750
14751 SmallVector<Loop *, 8> LoopStack(LI.begin(), LI.end());
14752
14753 // Map's SCEV expressions from one ScalarEvolution "universe" to another.
14754 struct SCEVMapper : public SCEVRewriteVisitor<SCEVMapper> {
14755 SCEVMapper(ScalarEvolution &SE) : SCEVRewriteVisitor<SCEVMapper>(SE) {}
14756
14757 const SCEV *visitConstant(const SCEVConstant *Constant) {
14758 return SE.getConstant(Val: Constant->getAPInt());
14759 }
14760
14761 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14762 return SE.getUnknown(V: Expr->getValue());
14763 }
14764
14765 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
14766 return SE.getCouldNotCompute();
14767 }
14768 };
14769
14770 SCEVMapper SCM(SE2);
14771 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
14772 SE2.getReachableBlocks(Reachable&: ReachableBlocks, F);
14773
14774 auto GetDelta = [&](const SCEV *Old, const SCEV *New) -> const SCEV * {
14775 if (containsUndefs(S: Old) || containsUndefs(S: New)) {
14776 // SCEV treats "undef" as an unknown but consistent value (i.e. it does
14777 // not propagate undef aggressively). This means we can (and do) fail
14778 // verification in cases where a transform makes a value go from "undef"
14779 // to "undef+1" (say). The transform is fine, since in both cases the
14780 // result is "undef", but SCEV thinks the value increased by 1.
14781 return nullptr;
14782 }
14783
14784 // Unless VerifySCEVStrict is set, we only compare constant deltas.
14785 const SCEV *Delta = SE2.getMinusSCEV(LHS: Old, RHS: New);
14786 if (!VerifySCEVStrict && !isa<SCEVConstant>(Val: Delta))
14787 return nullptr;
14788
14789 return Delta;
14790 };
14791
14792 while (!LoopStack.empty()) {
14793 auto *L = LoopStack.pop_back_val();
14794 llvm::append_range(C&: LoopStack, R&: *L);
14795
14796 // Only verify BECounts in reachable loops. For an unreachable loop,
14797 // any BECount is legal.
14798 if (!ReachableBlocks.contains(Ptr: L->getHeader()))
14799 continue;
14800
14801 // Only verify cached BECounts. Computing new BECounts may change the
14802 // results of subsequent SCEV uses.
14803 auto It = BackedgeTakenCounts.find(Val: L);
14804 if (It == BackedgeTakenCounts.end())
14805 continue;
14806
14807 auto *CurBECount =
14808 SCM.visit(S: It->second.getExact(L, SE: const_cast<ScalarEvolution *>(this)));
14809 auto *NewBECount = SE2.getBackedgeTakenCount(L);
14810
14811 if (CurBECount == SE2.getCouldNotCompute() ||
14812 NewBECount == SE2.getCouldNotCompute()) {
14813 // NB! This situation is legal, but is very suspicious -- whatever pass
14814 // change the loop to make a trip count go from could not compute to
14815 // computable or vice-versa *should have* invalidated SCEV. However, we
14816 // choose not to assert here (for now) since we don't want false
14817 // positives.
14818 continue;
14819 }
14820
14821 if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) >
14822 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14823 NewBECount = SE2.getZeroExtendExpr(Op: NewBECount, Ty: CurBECount->getType());
14824 else if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) <
14825 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14826 CurBECount = SE2.getZeroExtendExpr(Op: CurBECount, Ty: NewBECount->getType());
14827
14828 const SCEV *Delta = GetDelta(CurBECount, NewBECount);
14829 if (Delta && !Delta->isZero()) {
14830 dbgs() << "Trip Count for " << *L << " Changed!\n";
14831 dbgs() << "Old: " << *CurBECount << "\n";
14832 dbgs() << "New: " << *NewBECount << "\n";
14833 dbgs() << "Delta: " << *Delta << "\n";
14834 std::abort();
14835 }
14836 }
14837
14838 // Collect all valid loops currently in LoopInfo.
14839 SmallPtrSet<Loop *, 32> ValidLoops;
14840 SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end());
14841 while (!Worklist.empty()) {
14842 Loop *L = Worklist.pop_back_val();
14843 if (ValidLoops.insert(Ptr: L).second)
14844 Worklist.append(in_start: L->begin(), in_end: L->end());
14845 }
14846 for (const auto &KV : ValueExprMap) {
14847#ifndef NDEBUG
14848 // Check for SCEV expressions referencing invalid/deleted loops.
14849 if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
14850 assert(ValidLoops.contains(AR->getLoop()) &&
14851 "AddRec references invalid loop");
14852 }
14853#endif
14854
14855 // Check that the value is also part of the reverse map.
14856 auto It = ExprValueMap.find(Val: KV.second);
14857 if (It == ExprValueMap.end() || !It->second.contains(key: KV.first)) {
14858 dbgs() << "Value " << *KV.first
14859 << " is in ValueExprMap but not in ExprValueMap\n";
14860 std::abort();
14861 }
14862
14863 if (auto *I = dyn_cast<Instruction>(Val: &*KV.first)) {
14864 if (!ReachableBlocks.contains(Ptr: I->getParent()))
14865 continue;
14866 const SCEV *OldSCEV = SCM.visit(S: KV.second);
14867 const SCEV *NewSCEV = SE2.getSCEV(V: I);
14868 const SCEV *Delta = GetDelta(OldSCEV, NewSCEV);
14869 if (Delta && !Delta->isZero()) {
14870 dbgs() << "SCEV for value " << *I << " changed!\n"
14871 << "Old: " << *OldSCEV << "\n"
14872 << "New: " << *NewSCEV << "\n"
14873 << "Delta: " << *Delta << "\n";
14874 std::abort();
14875 }
14876 }
14877 }
14878
14879 for (const auto &KV : ExprValueMap) {
14880 for (Value *V : KV.second) {
14881 const SCEV *S = ValueExprMap.lookup(Val: V);
14882 if (!S) {
14883 dbgs() << "Value " << *V
14884 << " is in ExprValueMap but not in ValueExprMap\n";
14885 std::abort();
14886 }
14887 if (S != KV.first) {
14888 dbgs() << "Value " << *V << " mapped to " << *S << " rather than "
14889 << *KV.first << "\n";
14890 std::abort();
14891 }
14892 }
14893 }
14894
14895 // Verify integrity of SCEV users.
14896 for (const auto &S : UniqueSCEVs) {
14897 for (SCEVUse Op : S.operands()) {
14898 // We do not store dependencies of constants.
14899 if (isa<SCEVConstant>(Val: Op))
14900 continue;
14901 auto It = SCEVUsers.find(Val: Op);
14902 if (It != SCEVUsers.end() && It->second.count(Ptr: &S))
14903 continue;
14904 dbgs() << "Use of operand " << *Op << " by user " << S
14905 << " is not being tracked!\n";
14906 std::abort();
14907 }
14908 }
14909
14910 // Verify integrity of ValuesAtScopes users.
14911 for (const auto &ValueAndVec : ValuesAtScopes) {
14912 const SCEV *Value = ValueAndVec.first;
14913 for (const auto &LoopAndValueAtScope : ValueAndVec.second) {
14914 const Loop *L = LoopAndValueAtScope.first;
14915 const SCEV *ValueAtScope = LoopAndValueAtScope.second;
14916 if (!isa<SCEVConstant>(Val: ValueAtScope)) {
14917 auto It = ValuesAtScopesUsers.find(Val: ValueAtScope);
14918 if (It != ValuesAtScopesUsers.end() &&
14919 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: Value)))
14920 continue;
14921 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14922 << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
14923 std::abort();
14924 }
14925 }
14926 }
14927
14928 for (const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
14929 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
14930 for (const auto &LoopAndValue : ValueAtScopeAndVec.second) {
14931 const Loop *L = LoopAndValue.first;
14932 const SCEV *Value = LoopAndValue.second;
14933 assert(!isa<SCEVConstant>(Value));
14934 auto It = ValuesAtScopes.find(Val: Value);
14935 if (It != ValuesAtScopes.end() &&
14936 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: ValueAtScope)))
14937 continue;
14938 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14939 << *ValueAtScope << " missing in ValuesAtScopes\n";
14940 std::abort();
14941 }
14942 }
14943
14944 // Verify integrity of BECountUsers.
14945 auto VerifyBECountUsers = [&](bool Predicated) {
14946 auto &BECounts =
14947 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14948 for (const auto &LoopAndBEInfo : BECounts) {
14949 for (const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
14950 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14951 if (!isa<SCEVConstant>(Val: S)) {
14952 auto UserIt = BECountUsers.find(Val: S);
14953 if (UserIt != BECountUsers.end() &&
14954 UserIt->second.contains(Ptr: { LoopAndBEInfo.first, Predicated }))
14955 continue;
14956 dbgs() << "Value " << *S << " for loop " << *LoopAndBEInfo.first
14957 << " missing from BECountUsers\n";
14958 std::abort();
14959 }
14960 }
14961 }
14962 }
14963 };
14964 VerifyBECountUsers(/* Predicated */ false);
14965 VerifyBECountUsers(/* Predicated */ true);
14966
14967 // Verify intergity of loop disposition cache.
14968 for (auto &[S, Values] : LoopDispositions) {
14969 for (auto [Loop, CachedDisposition] : Values) {
14970 const auto RecomputedDisposition = SE2.getLoopDisposition(S, L: Loop);
14971 if (CachedDisposition != RecomputedDisposition) {
14972 dbgs() << "Cached disposition of " << *S << " for loop " << *Loop
14973 << " is incorrect: cached " << CachedDisposition << ", actual "
14974 << RecomputedDisposition << "\n";
14975 std::abort();
14976 }
14977 }
14978 }
14979
14980 // Verify integrity of the block disposition cache.
14981 for (auto &[S, Values] : BlockDispositions) {
14982 for (auto [BB, CachedDisposition] : Values) {
14983 const auto RecomputedDisposition = SE2.getBlockDisposition(S, BB);
14984 if (CachedDisposition != RecomputedDisposition) {
14985 dbgs() << "Cached disposition of " << *S << " for block %"
14986 << BB->getName() << " is incorrect: cached " << CachedDisposition
14987 << ", actual " << RecomputedDisposition << "\n";
14988 std::abort();
14989 }
14990 }
14991 }
14992
14993 // Verify FoldCache/FoldCacheUser caches.
14994 for (auto [FoldID, Expr] : FoldCache) {
14995 auto I = FoldCacheUser.find(Val: Expr);
14996 if (I == FoldCacheUser.end()) {
14997 dbgs() << "Missing entry in FoldCacheUser for cached expression " << *Expr
14998 << "!\n";
14999 std::abort();
15000 }
15001 if (!is_contained(Range: I->second, Element: FoldID)) {
15002 dbgs() << "Missing FoldID in cached users of " << *Expr << "!\n";
15003 std::abort();
15004 }
15005 }
15006 for (auto [Expr, IDs] : FoldCacheUser) {
15007 for (auto &FoldID : IDs) {
15008 const SCEV *S = FoldCache.lookup(Val: FoldID);
15009 if (!S) {
15010 dbgs() << "Missing entry in FoldCache for expression " << *Expr
15011 << "!\n";
15012 std::abort();
15013 }
15014 if (S != Expr) {
15015 dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: " << *S
15016 << " != " << *Expr << "!\n";
15017 std::abort();
15018 }
15019 }
15020 }
15021
15022 // Verify that ConstantMultipleCache computations are correct. We check that
15023 // cached multiples and recomputed multiples are multiples of each other to
15024 // verify correctness. It is possible that a recomputed multiple is different
15025 // from the cached multiple due to strengthened no wrap flags or changes in
15026 // KnownBits computations.
15027 for (auto [S, Multiple] : ConstantMultipleCache) {
15028 APInt RecomputedMultiple = SE2.getConstantMultiple(S);
15029 if ((Multiple != 0 && RecomputedMultiple != 0 &&
15030 Multiple.urem(RHS: RecomputedMultiple) != 0 &&
15031 RecomputedMultiple.urem(RHS: Multiple) != 0)) {
15032 dbgs() << "Incorrect cached computation in ConstantMultipleCache for "
15033 << *S << " : Computed " << RecomputedMultiple
15034 << " but cache contains " << Multiple << "!\n";
15035 std::abort();
15036 }
15037 }
15038}
15039
15040bool ScalarEvolution::invalidate(
15041 Function &F, const PreservedAnalyses &PA,
15042 FunctionAnalysisManager::Invalidator &Inv) {
15043 // Invalidate the ScalarEvolution object whenever it isn't preserved or one
15044 // of its dependencies is invalidated.
15045 auto PAC = PA.getChecker<ScalarEvolutionAnalysis>();
15046 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) ||
15047 Inv.invalidate<AssumptionAnalysis>(IR&: F, PA) ||
15048 Inv.invalidate<DominatorTreeAnalysis>(IR&: F, PA) ||
15049 Inv.invalidate<LoopAnalysis>(IR&: F, PA);
15050}
15051
15052AnalysisKey ScalarEvolutionAnalysis::Key;
15053
15054ScalarEvolution ScalarEvolutionAnalysis::run(Function &F,
15055 FunctionAnalysisManager &AM) {
15056 auto &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
15057 auto &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
15058 auto &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
15059 auto &LI = AM.getResult<LoopAnalysis>(IR&: F);
15060 return ScalarEvolution(F, TLI, AC, DT, LI);
15061}
15062
15063PreservedAnalyses
15064ScalarEvolutionVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
15065 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).verify();
15066 return PreservedAnalyses::all();
15067}
15068
15069PreservedAnalyses
15070ScalarEvolutionPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
15071 // For compatibility with opt's -analyze feature under legacy pass manager
15072 // which was not ported to NPM. This keeps tests using
15073 // update_analyze_test_checks.py working.
15074 OS << "Printing analysis 'Scalar Evolution Analysis' for function '"
15075 << F.getName() << "':\n";
15076 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).print(OS);
15077 return PreservedAnalyses::all();
15078}
15079
15080INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass, "scalar-evolution",
15081 "Scalar Evolution Analysis", false, true)
15082INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
15083INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
15084INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
15085INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
15086INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution",
15087 "Scalar Evolution Analysis", false, true)
15088
15089char ScalarEvolutionWrapperPass::ID = 0;
15090
15091ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) {}
15092
15093bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) {
15094 SE.reset(p: new ScalarEvolution(
15095 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F),
15096 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
15097 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
15098 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
15099 return false;
15100}
15101
15102void ScalarEvolutionWrapperPass::releaseMemory() { SE.reset(); }
15103
15104void ScalarEvolutionWrapperPass::print(raw_ostream &OS, const Module *) const {
15105 SE->print(OS);
15106}
15107
15108void ScalarEvolutionWrapperPass::verifyAnalysis() const {
15109 if (!VerifySCEV)
15110 return;
15111
15112 SE->verify();
15113}
15114
15115void ScalarEvolutionWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
15116 AU.setPreservesAll();
15117 AU.addRequiredTransitive<AssumptionCacheTracker>();
15118 AU.addRequiredTransitive<LoopInfoWrapperPass>();
15119 AU.addRequiredTransitive<DominatorTreeWrapperPass>();
15120 AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>();
15121}
15122
15123const SCEVPredicate *ScalarEvolution::getEqualPredicate(const SCEV *LHS,
15124 const SCEV *RHS) {
15125 return getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS, RHS);
15126}
15127
15128const SCEVPredicate *
15129ScalarEvolution::getComparePredicate(const ICmpInst::Predicate Pred,
15130 const SCEV *LHS, const SCEV *RHS) {
15131 FoldingSetNodeID ID;
15132 assert(LHS->getType() == RHS->getType() &&
15133 "Type mismatch between LHS and RHS");
15134 // Unique this node based on the arguments
15135 ID.AddInteger(I: SCEVPredicate::P_Compare);
15136 ID.AddInteger(I: Pred);
15137 ID.AddPointer(Ptr: LHS);
15138 ID.AddPointer(Ptr: RHS);
15139 FoldingSetInsertToken Token;
15140 if (const auto *S = UniquePreds.lookup(ID, Token))
15141 return S;
15142 SCEVComparePredicate *Eq = new (SCEVAllocator)
15143 SCEVComparePredicate(ID.Intern(Allocator&: SCEVAllocator), Pred, LHS, RHS);
15144 UniquePreds.insert(N: Eq, Token);
15145 return Eq;
15146}
15147
15148const SCEVPredicate *ScalarEvolution::getWrapPredicate(
15149 const SCEVAddRecExpr *AR,
15150 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15151 FoldingSetNodeID ID;
15152 // Unique this node based on the arguments
15153 ID.AddInteger(I: SCEVPredicate::P_Wrap);
15154 ID.AddPointer(Ptr: AR);
15155 ID.AddInteger(I: AddedFlags);
15156 FoldingSetInsertToken Token;
15157 if (const auto *S = UniquePreds.lookup(ID, Token))
15158 return S;
15159 auto *OF = new (SCEVAllocator)
15160 SCEVWrapPredicate(ID.Intern(Allocator&: SCEVAllocator), AR, AddedFlags);
15161 UniquePreds.insert(N: OF, Token);
15162 return OF;
15163}
15164
15165namespace {
15166
15167class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
15168public:
15169
15170 /// Rewrites \p S in the context of a loop L and the SCEV predication
15171 /// infrastructure.
15172 ///
15173 /// If \p Pred is non-null, the SCEV expression is rewritten to respect the
15174 /// equivalences present in \p Pred.
15175 ///
15176 /// If \p NewPreds is non-null, rewrite is free to add further predicates to
15177 /// \p NewPreds such that the result will be an AddRecExpr.
15178 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
15179 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15180 const SCEVPredicate *Pred) {
15181 SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred);
15182 return Rewriter.visit(S);
15183 }
15184
15185 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
15186 if (Pred) {
15187 if (auto *U = dyn_cast<SCEVUnionPredicate>(Val: Pred)) {
15188 for (const auto *Pred : U->getPredicates())
15189 if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred))
15190 if (IPred->getLHS() == Expr &&
15191 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15192 return IPred->getRHS();
15193 } else if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred)) {
15194 if (IPred->getLHS() == Expr &&
15195 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15196 return IPred->getRHS();
15197 }
15198 }
15199 return convertToAddRecWithPreds(Expr);
15200 }
15201
15202 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
15203 const SCEV *Operand = visit(S: Expr->getOperand());
15204 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15205 if (AR && AR->getLoop() == L && AR->isAffine()) {
15206 // This couldn't be folded because the operand didn't have the nuw
15207 // flag. Add the nusw flag as an assumption that we could make.
15208 const SCEV *Step = AR->getStepRecurrence(SE);
15209 Type *Ty = Expr->getType();
15210 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNUSW))
15211 return SE.getAddRecExpr(Start: SE.getZeroExtendExpr(Op: AR->getStart(), Ty),
15212 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15213 Flags: AR->getNoWrapFlags());
15214 }
15215 return SE.getZeroExtendExpr(Op: Operand, Ty: Expr->getType());
15216 }
15217
15218 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
15219 const SCEV *Operand = visit(S: Expr->getOperand());
15220 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15221 if (AR && AR->getLoop() == L && AR->isAffine()) {
15222 // This couldn't be folded because the operand didn't have the nsw
15223 // flag. Add the nssw flag as an assumption that we could make.
15224 const SCEV *Step = AR->getStepRecurrence(SE);
15225 Type *Ty = Expr->getType();
15226 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNSSW))
15227 return SE.getAddRecExpr(Start: SE.getSignExtendExpr(Op: AR->getStart(), Ty),
15228 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15229 Flags: AR->getNoWrapFlags());
15230 }
15231 return SE.getSignExtendExpr(Op: Operand, Ty: Expr->getType());
15232 }
15233
15234private:
15235 explicit SCEVPredicateRewriter(
15236 const Loop *L, ScalarEvolution &SE,
15237 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15238 const SCEVPredicate *Pred)
15239 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {}
15240
15241 bool addOverflowAssumption(const SCEVPredicate *P) {
15242 if (!NewPreds) {
15243 // Check if we've already made this assumption.
15244 return Pred && Pred->implies(N: P, SE);
15245 }
15246 NewPreds->push_back(Elt: P);
15247 return true;
15248 }
15249
15250 bool addOverflowAssumption(const SCEVAddRecExpr *AR,
15251 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15252 auto *A = SE.getWrapPredicate(AR, AddedFlags);
15253 return addOverflowAssumption(P: A);
15254 }
15255
15256 // If \p Expr represents a PHINode, we try to see if it can be represented
15257 // as an AddRec, possibly under a predicate (PHISCEVPred). If it is possible
15258 // to add this predicate as a runtime overflow check, we return the AddRec.
15259 // If \p Expr does not meet these conditions (is not a PHI node, or we
15260 // couldn't create an AddRec for it, or couldn't add the predicate), we just
15261 // return \p Expr.
15262 const SCEV *convertToAddRecWithPreds(const SCEVUnknown *Expr) {
15263 if (!isa<PHINode>(Val: Expr->getValue()))
15264 return Expr;
15265 std::optional<
15266 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
15267 PredicatedRewrite = SE.createAddRecFromPHIWithCasts(SymbolicPHI: Expr);
15268 if (!PredicatedRewrite)
15269 return Expr;
15270 for (const auto *P : PredicatedRewrite->second){
15271 // Wrap predicates from outer loops are not supported.
15272 if (auto *WP = dyn_cast<const SCEVWrapPredicate>(Val: P)) {
15273 if (L != WP->getExpr()->getLoop())
15274 return Expr;
15275 }
15276 if (!addOverflowAssumption(P))
15277 return Expr;
15278 }
15279 return PredicatedRewrite->first;
15280 }
15281
15282 SmallVectorImpl<const SCEVPredicate *> *NewPreds;
15283 const SCEVPredicate *Pred;
15284 const Loop *L;
15285};
15286
15287} // end anonymous namespace
15288
15289const SCEV *
15290ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L,
15291 const SCEVPredicate &Preds) {
15292 return SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: nullptr, Pred: &Preds);
15293}
15294
15295const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
15296 const SCEV *S, const Loop *L,
15297 SmallVectorImpl<const SCEVPredicate *> &Preds) {
15298 SmallVector<const SCEVPredicate *> TransformPreds;
15299 S = SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: &TransformPreds, Pred: nullptr);
15300 auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S);
15301
15302 if (!AddRec)
15303 return nullptr;
15304
15305 // Check if any of the transformed predicates is known to be false. In that
15306 // case, it doesn't make sense to convert to a predicated AddRec, as the
15307 // versioned loop will never execute.
15308 for (const SCEVPredicate *Pred : TransformPreds) {
15309 auto *WrapPred = dyn_cast<SCEVWrapPredicate>(Val: Pred);
15310 if (!WrapPred || WrapPred->getFlags() != SCEVWrapPredicate::IncrementNSSW)
15311 continue;
15312
15313 const SCEVAddRecExpr *AddRecToCheck = WrapPred->getExpr();
15314 const SCEV *ExitCount = getBackedgeTakenCount(L: AddRecToCheck->getLoop());
15315 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
15316 continue;
15317
15318 const SCEV *Step = AddRecToCheck->getStepRecurrence(SE&: *this);
15319 if (!Step->isOne())
15320 continue;
15321
15322 ExitCount = getTruncateOrSignExtend(V: ExitCount, Ty: Step->getType());
15323 const SCEV *Add = getAddExpr(LHS: AddRecToCheck->getStart(), RHS: ExitCount);
15324 if (isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS: Add, RHS: AddRecToCheck->getStart()))
15325 return nullptr;
15326 }
15327
15328 // Since the transformation was successful, we can now transfer the SCEV
15329 // predicates.
15330 Preds.append(in_start: TransformPreds.begin(), in_end: TransformPreds.end());
15331
15332 return AddRec;
15333}
15334
15335/// SCEV predicates
15336SCEVPredicate::SCEVPredicate(const FoldingSetNodeIDRef ID,
15337 SCEVPredicateKind Kind)
15338 : FastID(ID), Kind(Kind) {}
15339
15340SCEVComparePredicate::SCEVComparePredicate(const FoldingSetNodeIDRef ID,
15341 const ICmpInst::Predicate Pred,
15342 const SCEV *LHS, const SCEV *RHS)
15343 : SCEVPredicate(ID, P_Compare), Pred(Pred), LHS(LHS), RHS(RHS) {
15344 assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match");
15345 assert(LHS != RHS && "LHS and RHS are the same SCEV");
15346}
15347
15348bool SCEVComparePredicate::implies(const SCEVPredicate *N,
15349 ScalarEvolution &SE) const {
15350 const auto *Op = dyn_cast<SCEVComparePredicate>(Val: N);
15351
15352 if (!Op)
15353 return false;
15354
15355 if (Pred != ICmpInst::ICMP_EQ)
15356 return false;
15357
15358 return Op->LHS == LHS && Op->RHS == RHS;
15359}
15360
15361bool SCEVComparePredicate::isAlwaysTrue() const { return false; }
15362
15363void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
15364 if (Pred == ICmpInst::ICMP_EQ)
15365 OS.indent(NumSpaces: Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n";
15366 else
15367 OS.indent(NumSpaces: Depth) << "Compare predicate: " << *LHS << " " << Pred << ") "
15368 << *RHS << "\n";
15369
15370}
15371
15372SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
15373 const SCEVAddRecExpr *AR,
15374 IncrementWrapFlags Flags)
15375 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {}
15376
15377const SCEVAddRecExpr *SCEVWrapPredicate::getExpr() const { return AR; }
15378
15379bool SCEVWrapPredicate::implies(const SCEVPredicate *N,
15380 ScalarEvolution &SE) const {
15381 const auto *Op = dyn_cast<SCEVWrapPredicate>(Val: N);
15382 if (!Op || setFlags(Flags, OnFlags: Op->Flags) != Flags)
15383 return false;
15384
15385 if (Op->AR == AR)
15386 return true;
15387
15388 if (Flags != SCEVWrapPredicate::IncrementNSSW &&
15389 Flags != SCEVWrapPredicate::IncrementNUSW)
15390 return false;
15391
15392 const SCEV *Start = AR->getStart();
15393 const SCEV *OpStart = Op->AR->getStart();
15394 if (Start->getType()->isPointerTy() != OpStart->getType()->isPointerTy())
15395 return false;
15396
15397 // Reject pointers to different address spaces.
15398 if (Start->getType()->isPointerTy() && Start->getType() != OpStart->getType())
15399 return false;
15400
15401 // NUSW/NSSW on a wider-type AddRec does not imply the same on a
15402 // narrower-type AddRec.
15403 if (SE.getTypeSizeInBits(Ty: AR->getType()) >
15404 SE.getTypeSizeInBits(Ty: Op->AR->getType()))
15405 return false;
15406
15407 const SCEV *Step = AR->getStepRecurrence(SE);
15408 const SCEV *OpStep = Op->AR->getStepRecurrence(SE);
15409 if (!SE.isKnownPositive(S: Step) || !SE.isKnownPositive(S: OpStep))
15410 return false;
15411
15412 // If both steps are positive, this implies N, if N's start and step are
15413 // ULE/SLE (for NSUW/NSSW) than this'.
15414 Type *WiderTy = SE.getWiderType(T1: Step->getType(), T2: OpStep->getType());
15415 Step = SE.getNoopOrZeroExtend(V: Step, Ty: WiderTy);
15416 OpStep = SE.getNoopOrZeroExtend(V: OpStep, Ty: WiderTy);
15417
15418 bool IsNUW = Flags == SCEVWrapPredicate::IncrementNUSW;
15419 OpStart = IsNUW ? SE.getNoopOrZeroExtend(V: OpStart, Ty: WiderTy)
15420 : SE.getNoopOrSignExtend(V: OpStart, Ty: WiderTy);
15421 Start = IsNUW ? SE.getNoopOrZeroExtend(V: Start, Ty: WiderTy)
15422 : SE.getNoopOrSignExtend(V: Start, Ty: WiderTy);
15423 CmpInst::Predicate Pred = IsNUW ? CmpInst::ICMP_ULE : CmpInst::ICMP_SLE;
15424 return SE.isKnownPredicate(Pred, LHS: OpStep, RHS: Step) &&
15425 SE.isKnownPredicate(Pred, LHS: OpStart, RHS: Start);
15426}
15427
15428bool SCEVWrapPredicate::isAlwaysTrue() const {
15429 SCEV::NoWrapFlags ScevFlags = AR->getNoWrapFlags();
15430 IncrementWrapFlags IFlags = Flags;
15431
15432 if (ScalarEvolution::setFlags(Flags: ScevFlags, OnFlags: SCEV::FlagNSW) == ScevFlags)
15433 IFlags = clearFlags(Flags: IFlags, OffFlags: IncrementNSSW);
15434
15435 return IFlags == IncrementAnyWrap;
15436}
15437
15438void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const {
15439 OS.indent(NumSpaces: Depth) << *getExpr() << " Added Flags: ";
15440 if (SCEVWrapPredicate::IncrementNUSW & getFlags())
15441 OS << "<nusw>";
15442 if (SCEVWrapPredicate::IncrementNSSW & getFlags())
15443 OS << "<nssw>";
15444 OS << "\n";
15445}
15446
15447SCEVWrapPredicate::IncrementWrapFlags
15448SCEVWrapPredicate::getImpliedFlags(const SCEVAddRecExpr *AR,
15449 ScalarEvolution &SE) {
15450 IncrementWrapFlags ImpliedFlags = IncrementAnyWrap;
15451 SCEV::NoWrapFlags StaticFlags = AR->getNoWrapFlags();
15452
15453 // We can safely transfer the NSW flag as NSSW.
15454 if (ScalarEvolution::setFlags(Flags: StaticFlags, OnFlags: SCEV::FlagNSW) == StaticFlags)
15455 ImpliedFlags = IncrementNSSW;
15456
15457 if (ScalarEvolution::setFlags(Flags: StaticFlags, OnFlags: SCEV::FlagNUW) == StaticFlags) {
15458 // If the increment is positive, the SCEV NUW flag will also imply the
15459 // WrapPredicate NUSW flag.
15460 if (const auto *Step = dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE)))
15461 if (Step->getValue()->getValue().isNonNegative())
15462 ImpliedFlags = setFlags(Flags: ImpliedFlags, OnFlags: IncrementNUSW);
15463 }
15464
15465 return ImpliedFlags;
15466}
15467
15468/// Union predicates don't get cached so create a dummy set ID for it.
15469SCEVUnionPredicate::SCEVUnionPredicate(ArrayRef<const SCEVPredicate *> Preds,
15470 ScalarEvolution &SE)
15471 : SCEVPredicate(FoldingSetNodeIDRef(nullptr, 0), P_Union) {
15472 for (const auto *P : Preds)
15473 add(N: P, SE);
15474}
15475
15476bool SCEVUnionPredicate::isAlwaysTrue() const {
15477 return all_of(Range: Preds,
15478 P: [](const SCEVPredicate *I) { return I->isAlwaysTrue(); });
15479}
15480
15481bool SCEVUnionPredicate::implies(const SCEVPredicate *N,
15482 ScalarEvolution &SE) const {
15483 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N))
15484 return all_of(Range: Set->Preds, P: [this, &SE](const SCEVPredicate *I) {
15485 return this->implies(N: I, SE);
15486 });
15487
15488 if (any_of(Range: Preds,
15489 P: [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); }))
15490 return true;
15491
15492 // A wrap predicate may be implied by a wrap predicate in Preds after applying
15493 // equal predicates.
15494 const auto *NWrap = dyn_cast<SCEVWrapPredicate>(Val: N);
15495 if (!NWrap)
15496 return false;
15497 const Loop *L = NWrap->getExpr()->getLoop();
15498 return any_of(Range: Preds, P: [&](const SCEVPredicate *I) {
15499 const auto *IWrap = dyn_cast<SCEVWrapPredicate>(Val: I);
15500 if (!IWrap)
15501 return false;
15502 const auto *RewrittenAR = dyn_cast<SCEVAddRecExpr>(
15503 Val: SE.rewriteUsingPredicate(S: IWrap->getExpr(), L, Preds: *this));
15504 return RewrittenAR &&
15505 SE.getWrapPredicate(AR: RewrittenAR, AddedFlags: IWrap->getFlags())->implies(N, SE);
15506 });
15507}
15508
15509void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const {
15510 for (const auto *Pred : Preds)
15511 Pred->print(OS, Depth);
15512}
15513
15514void SCEVUnionPredicate::add(const SCEVPredicate *N, ScalarEvolution &SE) {
15515 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N)) {
15516 for (const auto *Pred : Set->Preds)
15517 add(N: Pred, SE);
15518 return;
15519 }
15520
15521 // Implication checks are quadratic in the number of predicates. Stop doing
15522 // them if there are many predicates, as they should be too expensive to use
15523 // anyway at that point.
15524 bool CheckImplies = Preds.size() < 16;
15525
15526 // Only add predicate if it is not already implied by this union predicate.
15527 if (CheckImplies && implies(N, SE))
15528 return;
15529
15530 // Build a new vector containing the current predicates, except the ones that
15531 // are implied by the new predicate N.
15532 SmallVector<const SCEVPredicate *> PrunedPreds;
15533 for (auto *P : Preds) {
15534 if (CheckImplies && N->implies(N: P, SE))
15535 continue;
15536 PrunedPreds.push_back(Elt: P);
15537 }
15538 Preds = std::move(PrunedPreds);
15539 Preds.push_back(Elt: N);
15540}
15541
15542PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE,
15543 Loop &L)
15544 : SE(SE), L(L) {
15545 SmallVector<const SCEVPredicate*, 4> Empty;
15546 Preds = std::make_unique<SCEVUnionPredicate>(args&: Empty, args&: SE);
15547}
15548
15549void ScalarEvolution::registerUser(const SCEV *User,
15550 ArrayRef<const SCEV *> Ops) {
15551 for (const auto *Op : Ops)
15552 // We do not expect that forgetting cached data for SCEVConstants will ever
15553 // open any prospects for sharpening or introduce any correctness issues,
15554 // so we don't bother storing their dependencies.
15555 if (!isa<SCEVConstant>(Val: Op))
15556 SCEVUsers[Op].insert(Ptr: User);
15557}
15558
15559void ScalarEvolution::registerUser(const SCEV *User, ArrayRef<SCEVUse> Ops) {
15560 for (const SCEV *Op : Ops)
15561 // We do not expect that forgetting cached data for SCEVConstants will ever
15562 // open any prospects for sharpening or introduce any correctness issues,
15563 // so we don't bother storing their dependencies.
15564 if (!isa<SCEVConstant>(Val: Op))
15565 SCEVUsers[Op].insert(Ptr: User);
15566}
15567
15568const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) {
15569 const SCEV *Expr = SE.getSCEV(V);
15570 return getPredicatedSCEV(Expr);
15571}
15572
15573const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
15574 RewriteEntry &Entry = RewriteMap[Expr];
15575
15576 // If we already have an entry and the version matches, return it.
15577 if (Entry.second && Generation == Entry.first)
15578 return Entry.second;
15579
15580 // We found an entry but it's stale. Rewrite the stale entry
15581 // according to the current predicate.
15582 if (Entry.second)
15583 Expr = Entry.second;
15584
15585 const SCEV *NewSCEV = SE.rewriteUsingPredicate(S: Expr, L: &L, Preds: *Preds);
15586 Entry = {Generation, NewSCEV};
15587
15588 return NewSCEV;
15589}
15590
15591const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
15592 if (!BackedgeCount) {
15593 SmallVector<const SCEVPredicate *, 4> Preds;
15594 BackedgeCount = SE.getPredicatedBackedgeTakenCount(L: &L, Preds);
15595 for (const auto *P : Preds)
15596 addPredicate(Pred: *P);
15597 }
15598 return BackedgeCount;
15599}
15600
15601const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
15602 if (!SymbolicMaxBackedgeCount) {
15603 SmallVector<const SCEVPredicate *, 4> Preds;
15604 SymbolicMaxBackedgeCount =
15605 SE.getPredicatedSymbolicMaxBackedgeTakenCount(L: &L, Preds);
15606 for (const auto *P : Preds)
15607 addPredicate(Pred: *P);
15608 }
15609 return SymbolicMaxBackedgeCount;
15610}
15611
15612unsigned PredicatedScalarEvolution::getSmallConstantMaxTripCount() {
15613 if (!SmallConstantMaxTripCount) {
15614 SmallVector<const SCEVPredicate *, 4> Preds;
15615 SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(L: &L, Predicates: &Preds);
15616 for (const auto *P : Preds)
15617 addPredicate(Pred: *P);
15618 }
15619 return *SmallConstantMaxTripCount;
15620}
15621
15622void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
15623 if (Preds->implies(N: &Pred, SE))
15624 return;
15625
15626 SmallVector<const SCEVPredicate *, 4> NewPreds(Preds->getPredicates());
15627 NewPreds.push_back(Elt: &Pred);
15628 Preds = std::make_unique<SCEVUnionPredicate>(args&: NewPreds, args&: SE);
15629 updateGeneration();
15630}
15631
15632void PredicatedScalarEvolution::addPredicates(
15633 ArrayRef<const SCEVPredicate *> Preds) {
15634 for (const SCEVPredicate *P : Preds)
15635 addPredicate(Pred: *P);
15636}
15637
15638const SCEVPredicate &PredicatedScalarEvolution::getPredicate() const {
15639 return *Preds;
15640}
15641
15642void PredicatedScalarEvolution::updateGeneration() {
15643 // If the generation number wrapped recompute everything.
15644 if (++Generation == 0) {
15645 for (auto &II : RewriteMap) {
15646 const SCEV *Rewritten = II.second.second;
15647 II.second = {Generation, SE.rewriteUsingPredicate(S: Rewritten, L: &L, Preds: *Preds)};
15648 }
15649 }
15650}
15651
15652bool PredicatedScalarEvolution::hasNoOverflow(
15653 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) {
15654 const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: getSCEV(V));
15655 if (!AR)
15656 return false;
15657
15658 Flags = SCEVWrapPredicate::clearFlags(
15659 Flags, OffFlags: SCEVWrapPredicate::getImpliedFlags(AR, SE));
15660
15661 return Flags == SCEVWrapPredicate::IncrementAnyWrap;
15662}
15663
15664const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(
15665 Value *V, SmallVectorImpl<const SCEVPredicate *> *ExtraPreds) {
15666 const SCEV *Expr = this->getSCEV(V);
15667 SmallVector<const SCEVPredicate *, 4> NewPreds;
15668 auto *New = SE.convertSCEVToAddRecWithPredicates(S: Expr, L: &L, Preds&: NewPreds);
15669
15670 if (!New)
15671 return nullptr;
15672
15673 if (ExtraPreds) {
15674 ExtraPreds->append(RHS: NewPreds);
15675 return New;
15676 }
15677
15678 addPredicates(Preds: NewPreds);
15679
15680 RewriteMap[SE.getSCEV(V)] = {Generation, New};
15681 return New;
15682}
15683
15684PredicatedScalarEvolution::PredicatedScalarEvolution(
15685 const PredicatedScalarEvolution &Init)
15686 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L),
15687 Preds(std::make_unique<SCEVUnionPredicate>(args: Init.Preds->getPredicates(),
15688 args&: SE)),
15689 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {}
15690
15691void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const {
15692 // For each block.
15693 for (auto *BB : L.getBlocks())
15694 for (auto &I : *BB) {
15695 if (!SE.isSCEVable(Ty: I.getType()))
15696 continue;
15697
15698 auto *Expr = SE.getSCEV(V: &I);
15699 auto II = RewriteMap.find(Val: Expr);
15700
15701 if (II == RewriteMap.end())
15702 continue;
15703
15704 // Don't print things that are not interesting.
15705 if (II->second.second == Expr)
15706 continue;
15707
15708 OS.indent(NumSpaces: Depth) << "[PSE]" << I << ":\n";
15709 OS.indent(NumSpaces: Depth + 2) << *Expr << "\n";
15710 OS.indent(NumSpaces: Depth + 2) << "--> " << *II->second.second << "\n";
15711 }
15712}
15713
15714ScalarEvolution::LoopGuards
15715ScalarEvolution::LoopGuards::collect(const Loop *L, ScalarEvolution &SE) {
15716 BasicBlock *Header = L->getHeader();
15717 BasicBlock *Pred = L->getLoopPredecessor();
15718 LoopGuards Guards(SE);
15719 if (!Pred)
15720 return Guards;
15721 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
15722 collectFromBlock(SE, Guards, Block: Header, Pred, VisitedBlocks);
15723 return Guards;
15724}
15725
15726void ScalarEvolution::LoopGuards::collectFromPHI(
15727 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15728 const PHINode &Phi, SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks,
15729 SmallDenseMap<const BasicBlock *, LoopGuards> &IncomingGuards,
15730 unsigned Depth) {
15731 if (!SE.isSCEVable(Ty: Phi.getType()))
15732 return;
15733
15734 using MinMaxPattern = std::pair<const SCEVConstant *, SCEVTypes>;
15735 auto GetMinMaxConst = [&](unsigned IncomingIdx) -> MinMaxPattern {
15736 const BasicBlock *InBlock = Phi.getIncomingBlock(i: IncomingIdx);
15737 if (!VisitedBlocks.insert(Ptr: InBlock).second)
15738 return {nullptr, scCouldNotCompute};
15739
15740 // Avoid analyzing unreachable blocks so that we don't get trapped
15741 // traversing cycles with ill-formed dominance or infinite cycles
15742 if (!SE.DT.isReachableFromEntry(A: InBlock))
15743 return {nullptr, scCouldNotCompute};
15744
15745 auto [G, Inserted] = IncomingGuards.try_emplace(Key: InBlock, Args: LoopGuards(SE));
15746 if (Inserted)
15747 collectFromBlock(SE, Guards&: G->second, Block: Phi.getParent(), Pred: InBlock, VisitedBlocks,
15748 Depth: Depth + 1);
15749 auto &RewriteMap = G->second.RewriteMap;
15750 if (RewriteMap.empty())
15751 return {nullptr, scCouldNotCompute};
15752 auto S = RewriteMap.find(Val: SE.getSCEV(V: Phi.getIncomingValue(i: IncomingIdx)));
15753 if (S == RewriteMap.end())
15754 return {nullptr, scCouldNotCompute};
15755 auto *SM = dyn_cast_if_present<SCEVMinMaxExpr>(Val: S->second);
15756 if (!SM)
15757 return {nullptr, scCouldNotCompute};
15758 if (const SCEVConstant *C0 = dyn_cast<SCEVConstant>(Val: SM->getOperand(i: 0)))
15759 return {C0, SM->getSCEVType()};
15760 return {nullptr, scCouldNotCompute};
15761 };
15762 auto MergeMinMaxConst = [](MinMaxPattern P1,
15763 MinMaxPattern P2) -> MinMaxPattern {
15764 auto [C1, T1] = P1;
15765 auto [C2, T2] = P2;
15766 if (!C1 || !C2 || T1 != T2)
15767 return {nullptr, scCouldNotCompute};
15768 switch (T1) {
15769 case scUMaxExpr:
15770 return {C1->getAPInt().ult(RHS: C2->getAPInt()) ? C1 : C2, T1};
15771 case scSMaxExpr:
15772 return {C1->getAPInt().slt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15773 case scUMinExpr:
15774 return {C1->getAPInt().ugt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15775 case scSMinExpr:
15776 return {C1->getAPInt().sgt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15777 default:
15778 llvm_unreachable("Trying to merge non-MinMaxExpr SCEVs.");
15779 }
15780 };
15781 auto P = GetMinMaxConst(0);
15782 for (unsigned int In = 1; In < Phi.getNumIncomingValues(); In++) {
15783 if (!P.first)
15784 break;
15785 P = MergeMinMaxConst(P, GetMinMaxConst(In));
15786 }
15787 if (P.first) {
15788 const SCEV *LHS = SE.getSCEV(V: const_cast<PHINode *>(&Phi));
15789 SmallVector<SCEVUse, 2> Ops({P.first, LHS});
15790 const SCEV *RHS = SE.getMinMaxExpr(Kind: P.second, Ops);
15791 Guards.RewriteMap.insert(KV: {LHS, RHS});
15792 }
15793}
15794
15795// Return a new SCEV that modifies \p Expr to the closest number divides by
15796// \p Divisor and less or equal than Expr. For now, only handle constant
15797// Expr.
15798static const SCEV *getPreviousSCEVDivisibleByDivisor(const SCEV *Expr,
15799 const APInt &DivisorVal,
15800 ScalarEvolution &SE) {
15801 const APInt *ExprVal;
15802 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15803 DivisorVal.isNonPositive())
15804 return Expr;
15805 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15806 // return the SCEV: Expr - Expr % Divisor
15807 return SE.getConstant(Val: *ExprVal - Rem);
15808}
15809
15810// Return a new SCEV that modifies \p Expr to the closest number divides by
15811// \p Divisor and greater or equal than Expr. For now, only handle constant
15812// Expr.
15813static const SCEV *getNextSCEVDivisibleByDivisor(const SCEV *Expr,
15814 const APInt &DivisorVal,
15815 ScalarEvolution &SE) {
15816 const APInt *ExprVal;
15817 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15818 DivisorVal.isNonPositive())
15819 return Expr;
15820 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15821 if (Rem.isZero())
15822 return Expr;
15823 // return the SCEV: Expr + Divisor - Expr % Divisor
15824 return SE.getConstant(Val: *ExprVal + DivisorVal - Rem);
15825}
15826
15827static bool collectDivisibilityInformation(
15828 ICmpInst::Predicate Predicate, const SCEV *LHS, const SCEV *RHS,
15829 DenseMap<const SCEV *, const SCEV *> &DivInfo,
15830 DenseMap<const SCEV *, APInt> &Multiples, ScalarEvolution &SE) {
15831 // If we have LHS == 0, check if LHS is computing a property of some unknown
15832 // SCEV %v which we can rewrite %v to express explicitly.
15833 if (Predicate != CmpInst::ICMP_EQ || !match(S: RHS, P: m_scev_Zero()))
15834 return false;
15835 // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to
15836 // explicitly express that.
15837 const SCEVUnknown *URemLHS = nullptr;
15838 const SCEV *URemRHS = nullptr;
15839 if (!match(S: LHS, P: m_scev_URem(LHS: m_SCEVUnknown(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE)))
15840 return false;
15841
15842 const SCEV *Multiple =
15843 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: URemLHS, RHS: URemRHS), RHS: URemRHS);
15844 DivInfo[URemLHS] = Multiple;
15845 if (auto *C = dyn_cast<SCEVConstant>(Val: URemRHS))
15846 Multiples[URemLHS] = C->getAPInt();
15847 return true;
15848}
15849
15850// Check if the condition is a divisibility guard (A % B == 0).
15851static bool isDivisibilityGuard(const SCEV *LHS, const SCEV *RHS,
15852 ScalarEvolution &SE) {
15853 const SCEV *X, *Y;
15854 return match(S: LHS, P: m_scev_URem(LHS: m_SCEV(V&: X), RHS: m_SCEV(V&: Y), SE)) && RHS->isZero();
15855}
15856
15857// Apply divisibility by \p Divisor on MinMaxExpr with constant values,
15858// recursively. This is done by aligning up/down the constant value to the
15859// Divisor.
15860static const SCEV *applyDivisibilityOnMinMaxExpr(const SCEV *MinMaxExpr,
15861 APInt Divisor,
15862 ScalarEvolution &SE) {
15863 // Return true if \p Expr is a MinMax SCEV expression with a non-negative
15864 // constant operand. If so, return in \p SCTy the SCEV type and in \p RHS
15865 // the non-constant operand and in \p LHS the constant operand.
15866 auto IsMinMaxSCEVWithNonNegativeConstant =
15867 [&](const SCEV *Expr, SCEVTypes &SCTy, const SCEV *&LHS,
15868 const SCEV *&RHS) {
15869 if (auto *MinMax = dyn_cast<SCEVMinMaxExpr>(Val: Expr)) {
15870 if (MinMax->getNumOperands() != 2)
15871 return false;
15872 if (auto *C = dyn_cast<SCEVConstant>(Val: MinMax->getOperand(i: 0))) {
15873 if (C->getAPInt().isNegative())
15874 return false;
15875 SCTy = MinMax->getSCEVType();
15876 LHS = MinMax->getOperand(i: 0);
15877 RHS = MinMax->getOperand(i: 1);
15878 return true;
15879 }
15880 }
15881 return false;
15882 };
15883
15884 const SCEV *MinMaxLHS = nullptr, *MinMaxRHS = nullptr;
15885 SCEVTypes SCTy;
15886 if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS,
15887 MinMaxRHS))
15888 return MinMaxExpr;
15889 auto IsMin = isa<SCEVSMinExpr>(Val: MinMaxExpr) || isa<SCEVUMinExpr>(Val: MinMaxExpr);
15890 assert(SE.isKnownNonNegative(MinMaxLHS) && "Expected non-negative operand!");
15891 auto *DivisibleExpr =
15892 IsMin ? getPreviousSCEVDivisibleByDivisor(Expr: MinMaxLHS, DivisorVal: Divisor, SE)
15893 : getNextSCEVDivisibleByDivisor(Expr: MinMaxLHS, DivisorVal: Divisor, SE);
15894 SmallVector<SCEVUse> Ops = {
15895 applyDivisibilityOnMinMaxExpr(MinMaxExpr: MinMaxRHS, Divisor, SE), DivisibleExpr};
15896 return SE.getMinMaxExpr(Kind: SCTy, Ops);
15897}
15898
15899void ScalarEvolution::LoopGuards::collectFromBlock(
15900 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15901 const BasicBlock *Block, const BasicBlock *Pred,
15902 SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks, unsigned Depth) {
15903
15904 assert(SE.DT.isReachableFromEntry(Block) && SE.DT.isReachableFromEntry(Pred));
15905
15906 SmallVector<SCEVUse> ExprsToRewrite;
15907 auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS,
15908 const SCEV *RHS,
15909 DenseMap<const SCEV *, const SCEV *> &RewriteMap,
15910 const LoopGuards &DivGuards) {
15911 // WARNING: It is generally unsound to apply any wrap flags to the proposed
15912 // replacement SCEV which isn't directly implied by the structure of that
15913 // SCEV. In particular, using contextual facts to imply flags is *NOT*
15914 // legal. See the scoping rules for flags in the header to understand why.
15915
15916 // Puts rewrite rule \p From -> \p To into the rewrite map. Also if \p From
15917 // and \p FromRewritten are the same (i.e. there has been no rewrite
15918 // registered for \p From), then puts this value in the list of rewritten
15919 // expressions.
15920 auto AddRewrite = [&](const SCEV *From, const SCEV *FromRewritten,
15921 const SCEV *To) {
15922 if (From == FromRewritten)
15923 ExprsToRewrite.push_back(Elt: From);
15924 RewriteMap[From] = To;
15925 };
15926
15927 // Checks whether \p S has already been rewritten. In that case returns the
15928 // existing rewrite because we want to chain further rewrites onto the
15929 // already rewritten value. Otherwise returns \p S.
15930 auto GetMaybeRewritten = [&](const SCEV *S) {
15931 return RewriteMap.lookup_or(Val: S, Default&: S);
15932 };
15933
15934 // Check for a condition of the form (-C1 + X < C2). InstCombine will
15935 // create this form when combining two checks of the form (X u< C2 + C1) and
15936 // (X >=u C1).
15937 auto MatchRangeCheckIdiom = [&](ICmpInst::Predicate Pred,
15938 const SCEV *MatchLHS,
15939 const SCEV *MatchRHS) {
15940 const SCEVConstant *C1;
15941 const SCEVUnknown *LHSUnknown;
15942 auto *C2 = dyn_cast<SCEVConstant>(Val: MatchRHS);
15943 if (!match(S: MatchLHS,
15944 P: m_scev_Add(Op0: m_SCEVConstant(V&: C1), Op1: m_SCEVUnknown(V&: LHSUnknown))) ||
15945 !C2)
15946 return false;
15947
15948 auto ExactRegion =
15949 ConstantRange::makeExactICmpRegion(Pred, Other: C2->getAPInt())
15950 .sub(Other: C1->getAPInt());
15951
15952 // Tighten the raw range with what we already know about LHSUnknown
15953 // from prior guards recorded in RewriteMap, or from SCEV's own range
15954 // analysis.
15955 const SCEV *RewrittenLHS = GetMaybeRewritten(LHSUnknown);
15956 ExactRegion = ExactRegion.intersectWith(CR: SE.getUnsignedRange(S: RewrittenLHS),
15957 Type: ConstantRange::Unsigned);
15958
15959 // Bail if the guard is inconsistent with prior facts, or if the range
15960 // is still not a monotonic non-wrapping interval after tightening.
15961 if (ExactRegion.isEmptySet() || ExactRegion.isWrappedSet() ||
15962 ExactRegion.isFullSet())
15963 return false;
15964
15965 const SCEV *RegionMin = SE.getConstant(Val: ExactRegion.getUnsignedMin());
15966 const SCEV *RegionMax = SE.getConstant(Val: ExactRegion.getUnsignedMax());
15967 const SCEV *ClampedLHS =
15968 SE.getUMaxExpr(LHS: RegionMin, RHS: SE.getUMinExpr(LHS: RewrittenLHS, RHS: RegionMax));
15969 AddRewrite(LHSUnknown, RewrittenLHS, ClampedLHS);
15970 return true;
15971 };
15972 if (MatchRangeCheckIdiom(Predicate, LHS, RHS))
15973 return;
15974
15975 // Do not apply information for constants or if RHS contains an AddRec.
15976 if (isa<SCEVConstant>(Val: LHS) || SE.containsAddRecurrence(S: RHS))
15977 return;
15978
15979 // If RHS is SCEVUnknown, make sure the information is applied to it.
15980 if (!isa<SCEVUnknown>(Val: LHS) && isa<SCEVUnknown>(Val: RHS)) {
15981 std::swap(a&: LHS, b&: RHS);
15982 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
15983 }
15984
15985 const SCEV *RewrittenLHS = GetMaybeRewritten(LHS);
15986 // Apply divisibility information when computing the constant multiple.
15987 const APInt &DividesBy =
15988 SE.getConstantMultiple(S: DivGuards.rewrite(Expr: RewrittenLHS));
15989
15990 // Collect rewrites for LHS and its transitive operands based on the
15991 // condition.
15992 // For min/max expressions, also apply the guard to its operands:
15993 // 'min(a, b) >= c' -> '(a >= c) and (b >= c)',
15994 // 'min(a, b) > c' -> '(a > c) and (b > c)',
15995 // 'max(a, b) <= c' -> '(a <= c) and (b <= c)',
15996 // 'max(a, b) < c' -> '(a < c) and (b < c)'.
15997
15998 // We cannot express strict predicates in SCEV, so instead we replace them
15999 // with non-strict ones against plus or minus one of RHS depending on the
16000 // predicate.
16001 const SCEV *One = SE.getOne(Ty: RHS->getType());
16002 switch (Predicate) {
16003 case CmpInst::ICMP_ULT:
16004 if (RHS->getType()->isPointerTy())
16005 return;
16006 RHS = SE.getUMaxExpr(LHS: RHS, RHS: One);
16007 [[fallthrough]];
16008 case CmpInst::ICMP_SLT: {
16009 RHS = SE.getMinusSCEV(LHS: RHS, RHS: One);
16010 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16011 break;
16012 }
16013 case CmpInst::ICMP_UGT:
16014 case CmpInst::ICMP_SGT:
16015 RHS = SE.getAddExpr(LHS: RHS, RHS: One);
16016 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16017 break;
16018 case CmpInst::ICMP_ULE:
16019 case CmpInst::ICMP_SLE:
16020 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16021 break;
16022 case CmpInst::ICMP_UGE:
16023 case CmpInst::ICMP_SGE:
16024 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16025 break;
16026 default:
16027 break;
16028 }
16029
16030 SmallVector<SCEVUse, 16> Worklist(1, LHS);
16031 SmallPtrSet<const SCEV *, 16> Visited;
16032
16033 auto EnqueueOperands = [&Worklist](const SCEVNAryExpr *S) {
16034 append_range(C&: Worklist, R: S->operands());
16035 };
16036
16037 while (!Worklist.empty()) {
16038 const SCEV *From = Worklist.pop_back_val();
16039 if (isa<SCEVConstant>(Val: From))
16040 continue;
16041 if (!Visited.insert(Ptr: From).second)
16042 continue;
16043 const SCEV *FromRewritten = GetMaybeRewritten(From);
16044 const SCEV *To = nullptr;
16045
16046 switch (Predicate) {
16047 case CmpInst::ICMP_ULT:
16048 case CmpInst::ICMP_ULE:
16049 To = SE.getUMinExpr(LHS: FromRewritten, RHS);
16050 if (auto *UMax = dyn_cast<SCEVUMaxExpr>(Val: FromRewritten))
16051 EnqueueOperands(UMax);
16052 break;
16053 case CmpInst::ICMP_SLT:
16054 case CmpInst::ICMP_SLE:
16055 To = SE.getSMinExpr(LHS: FromRewritten, RHS);
16056 if (auto *SMax = dyn_cast<SCEVSMaxExpr>(Val: FromRewritten))
16057 EnqueueOperands(SMax);
16058 break;
16059 case CmpInst::ICMP_UGT:
16060 case CmpInst::ICMP_UGE:
16061 To = SE.getUMaxExpr(LHS: FromRewritten, RHS);
16062 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: FromRewritten))
16063 EnqueueOperands(UMin);
16064 break;
16065 case CmpInst::ICMP_SGT:
16066 case CmpInst::ICMP_SGE:
16067 To = SE.getSMaxExpr(LHS: FromRewritten, RHS);
16068 if (auto *SMin = dyn_cast<SCEVSMinExpr>(Val: FromRewritten))
16069 EnqueueOperands(SMin);
16070 break;
16071 case CmpInst::ICMP_EQ:
16072 if (isa<SCEVConstant>(Val: RHS))
16073 To = RHS;
16074 break;
16075 case CmpInst::ICMP_NE:
16076 if (match(S: RHS, P: m_scev_Zero())) {
16077 const SCEV *OneAlignedUp =
16078 getNextSCEVDivisibleByDivisor(Expr: One, DivisorVal: DividesBy, SE);
16079 To = SE.getUMaxExpr(LHS: FromRewritten, RHS: OneAlignedUp);
16080 } else {
16081 // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS),
16082 // but creating the subtraction eagerly is expensive. Track the
16083 // inequalities in a separate map, and materialize the rewrite lazily
16084 // when encountering a suitable subtraction while re-writing.
16085 if (LHS->getType()->isPointerTy()) {
16086 LHS = SE.getPtrToAddrExpr(Op: LHS);
16087 RHS = SE.getPtrToAddrExpr(Op: RHS);
16088 if (isa<SCEVCouldNotCompute>(Val: LHS) || isa<SCEVCouldNotCompute>(Val: RHS))
16089 break;
16090 }
16091 const SCEVConstant *C;
16092 const SCEV *A, *B;
16093 if (match(S: RHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: A))) &&
16094 match(S: LHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: B)))) {
16095 RHS = A;
16096 LHS = B;
16097 }
16098 if (LHS > RHS)
16099 std::swap(a&: LHS, b&: RHS);
16100 Guards.NotEqual.insert(V: {LHS, RHS});
16101 continue;
16102 }
16103 break;
16104 default:
16105 break;
16106 }
16107
16108 if (To)
16109 AddRewrite(From, FromRewritten, To);
16110 }
16111 };
16112
16113 SmallVector<PointerIntPair<Value *, 1, bool>> Terms;
16114 // First, collect information from assumptions dominating the loop.
16115 for (auto &AssumeVH : SE.AC.assumptions()) {
16116 if (!AssumeVH)
16117 continue;
16118 auto *AssumeI = cast<CallInst>(Val&: AssumeVH);
16119 if (!SE.DT.dominates(Def: AssumeI, BB: Block))
16120 continue;
16121 Terms.emplace_back(Args: AssumeI->getOperand(i_nocapture: 0), Args: true);
16122 }
16123
16124 // Second, collect information from llvm.experimental.guards dominating the loop.
16125 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
16126 M: SE.F.getParent(), id: Intrinsic::experimental_guard);
16127 if (GuardDecl)
16128 for (const auto *GU : GuardDecl->users())
16129 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
16130 if (Guard->getFunction() == Block->getParent() &&
16131 SE.DT.dominates(Def: Guard, BB: Block))
16132 Terms.emplace_back(Args: Guard->getArgOperand(i: 0), Args: true);
16133
16134 // Third, collect conditions from dominating branches. Starting at the loop
16135 // predecessor, climb up the predecessor chain, as long as there are
16136 // predecessors that can be found that have unique successors leading to the
16137 // original header.
16138 // TODO: share this logic with isLoopEntryGuardedByCond.
16139 unsigned NumCollectedConditions = 0;
16140 VisitedBlocks.insert(Ptr: Block);
16141 std::pair<const BasicBlock *, const BasicBlock *> Pair(Pred, Block);
16142 for (; Pair.first;
16143 Pair = SE.getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
16144 VisitedBlocks.insert(Ptr: Pair.second);
16145 const CondBrInst *LoopEntryPredicate =
16146 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
16147 if (!LoopEntryPredicate)
16148 continue;
16149
16150 Terms.emplace_back(Args: LoopEntryPredicate->getCondition(),
16151 Args: LoopEntryPredicate->getSuccessor(i: 0) == Pair.second);
16152 NumCollectedConditions++;
16153
16154 // If we are recursively collecting guards stop after 2
16155 // conditions to limit compile-time impact for now.
16156 if (Depth > 0 && NumCollectedConditions == 2)
16157 break;
16158 }
16159 // Finally, if we stopped climbing the predecessor chain because
16160 // there wasn't a unique one to continue, try to collect conditions
16161 // for PHINodes by recursively following all of their incoming
16162 // blocks and try to merge the found conditions to build a new one
16163 // for the Phi.
16164 if (Pair.second->hasNPredecessorsOrMore(N: 2) &&
16165 Depth < MaxLoopGuardCollectionDepth) {
16166 SmallDenseMap<const BasicBlock *, LoopGuards> IncomingGuards;
16167 for (auto &Phi : Pair.second->phis())
16168 collectFromPHI(SE, Guards, Phi, VisitedBlocks, IncomingGuards, Depth);
16169 }
16170
16171 // Now apply the information from the collected conditions to
16172 // Guards.RewriteMap. Conditions are processed in reverse order, so the
16173 // earliest conditions is processed first, except guards with divisibility
16174 // information, which are moved to the back. This ensures the SCEVs with the
16175 // shortest dependency chains are constructed first.
16176 SmallVector<std::tuple<CmpInst::Predicate, const SCEV *, const SCEV *>>
16177 GuardsToProcess;
16178 for (auto [Term, EnterIfTrue] : reverse(C&: Terms)) {
16179 SmallVector<Value *, 8> Worklist;
16180 SmallPtrSet<Value *, 8> Visited;
16181 Worklist.push_back(Elt: Term);
16182 while (!Worklist.empty()) {
16183 Value *Cond = Worklist.pop_back_val();
16184 if (!Visited.insert(Ptr: Cond).second)
16185 continue;
16186
16187 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
16188 auto Predicate =
16189 EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate();
16190 const auto *LHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 0));
16191 const auto *RHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 1));
16192 // If LHS is a constant, apply information to the other expression.
16193 // TODO: If LHS is not a constant, check if using CompareSCEVComplexity
16194 // can improve results.
16195 if (isa<SCEVConstant>(Val: LHS)) {
16196 std::swap(a&: LHS, b&: RHS);
16197 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
16198 }
16199 GuardsToProcess.emplace_back(Args&: Predicate, Args&: LHS, Args&: RHS);
16200 continue;
16201 }
16202
16203 Value *L, *R;
16204 if (EnterIfTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: L), R: m_Value(V&: R)))
16205 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: L), R: m_Value(V&: R)))) {
16206 Worklist.push_back(Elt: L);
16207 Worklist.push_back(Elt: R);
16208 }
16209 }
16210 }
16211
16212 // Process divisibility guards in reverse order to populate DivGuards early.
16213 DenseMap<const SCEV *, APInt> Multiples;
16214 LoopGuards DivGuards(SE);
16215 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess) {
16216 if (!isDivisibilityGuard(LHS, RHS, SE))
16217 continue;
16218 collectDivisibilityInformation(Predicate, LHS, RHS, DivInfo&: DivGuards.RewriteMap,
16219 Multiples, SE);
16220 }
16221
16222 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess)
16223 CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap, DivGuards);
16224
16225 // Apply divisibility information last. This ensures it is applied to the
16226 // outermost expression after other rewrites for the given value.
16227 for (const auto &[K, Divisor] : Multiples) {
16228 const SCEV *DivisorSCEV = SE.getConstant(Val: Divisor);
16229 Guards.RewriteMap[K] =
16230 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: applyDivisibilityOnMinMaxExpr(
16231 MinMaxExpr: Guards.rewrite(Expr: K), Divisor, SE),
16232 RHS: DivisorSCEV),
16233 RHS: DivisorSCEV);
16234 ExprsToRewrite.push_back(Elt: K);
16235 }
16236
16237 // Let the rewriter preserve NUW/NSW flags if the unsigned/signed ranges of
16238 // the replacement expressions are contained in the ranges of the replaced
16239 // expressions.
16240 Guards.PreserveNUW = true;
16241 Guards.PreserveNSW = true;
16242 for (const SCEV *Expr : ExprsToRewrite) {
16243 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16244 Guards.PreserveNUW &=
16245 SE.getUnsignedRange(S: Expr).contains(CR: SE.getUnsignedRange(S: RewriteTo));
16246 Guards.PreserveNSW &=
16247 SE.getSignedRange(S: Expr).contains(CR: SE.getSignedRange(S: RewriteTo));
16248 }
16249
16250 // Now that all rewrite information is collect, rewrite the collected
16251 // expressions with the information in the map. This applies information to
16252 // sub-expressions.
16253 if (ExprsToRewrite.size() > 1) {
16254 for (const SCEV *Expr : ExprsToRewrite) {
16255 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16256 Guards.RewriteMap.erase(Val: Expr);
16257 Guards.RewriteMap.insert(KV: {Expr, Guards.rewrite(Expr: RewriteTo)});
16258 }
16259 }
16260}
16261
16262const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
16263 /// A rewriter to replace SCEV expressions in Map with the corresponding entry
16264 /// in the map. It skips AddRecExpr because we cannot guarantee that the
16265 /// replacement is loop invariant in the loop of the AddRec.
16266 class SCEVLoopGuardRewriter
16267 : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> {
16268 const DenseMap<const SCEV *, const SCEV *> &Map;
16269 const SmallDenseSet<std::pair<const SCEV *, const SCEV *>> &NotEqual;
16270
16271 SCEV::NoWrapFlags FlagMask = SCEV::FlagAnyWrap;
16272
16273 public:
16274 SCEVLoopGuardRewriter(ScalarEvolution &SE,
16275 const ScalarEvolution::LoopGuards &Guards)
16276 : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap),
16277 NotEqual(Guards.NotEqual) {
16278 if (Guards.PreserveNUW)
16279 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNUW);
16280 if (Guards.PreserveNSW)
16281 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNSW);
16282 }
16283
16284 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
16285
16286 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
16287 return Map.lookup_or(Val: Expr, Default&: Expr);
16288 }
16289
16290 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
16291 if (const SCEV *S = Map.lookup(Val: Expr))
16292 return S;
16293
16294 // If we didn't find the extact ZExt expr in the map, check if there's
16295 // an entry for a smaller ZExt we can use instead.
16296 Type *Ty = Expr->getType();
16297 const SCEV *Op = Expr->getOperand(i: 0);
16298 unsigned Bitwidth = Ty->getScalarSizeInBits() / 2;
16299 while (Bitwidth % 8 == 0 && Bitwidth >= 8 &&
16300 Bitwidth > Op->getType()->getScalarSizeInBits()) {
16301 Type *NarrowTy = IntegerType::get(C&: SE.getContext(), NumBits: Bitwidth);
16302 auto *NarrowExt = SE.getZeroExtendExpr(Op, Ty: NarrowTy);
16303 if (const SCEV *S = Map.lookup(Val: NarrowExt))
16304 return SE.getZeroExtendExpr(Op: S, Ty);
16305 Bitwidth = Bitwidth / 2;
16306 }
16307
16308 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitZeroExtendExpr(
16309 Expr);
16310 }
16311
16312 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
16313 if (const SCEV *S = Map.lookup(Val: Expr))
16314 return S;
16315 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSignExtendExpr(
16316 Expr);
16317 }
16318
16319 const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) {
16320 if (const SCEV *S = Map.lookup(Val: Expr))
16321 return S;
16322 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitUMinExpr(Expr);
16323 }
16324
16325 const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) {
16326 if (const SCEV *S = Map.lookup(Val: Expr))
16327 return S;
16328 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSMinExpr(Expr);
16329 }
16330
16331 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
16332 if (const SCEV *S = Map.lookup(Val: Expr))
16333 return S;
16334
16335 // Helper to check if S is a subtraction (A - B) where A != B, and if so,
16336 // return UMax(S, 1).
16337 auto RewriteSubtraction = [&](const SCEV *S) -> const SCEV * {
16338 SCEVUse LHS, RHS;
16339 if (MatchBinarySub(S, LHS, RHS)) {
16340 if (LHS > RHS)
16341 std::swap(a&: LHS, b&: RHS);
16342 if (NotEqual.contains(V: {LHS, RHS})) {
16343 const SCEV *OneAlignedUp = getNextSCEVDivisibleByDivisor(
16344 Expr: SE.getOne(Ty: S->getType()), DivisorVal: SE.getConstantMultiple(S), SE);
16345 return SE.getUMaxExpr(LHS: OneAlignedUp, RHS: S);
16346 }
16347 }
16348 return nullptr;
16349 };
16350
16351 // Check if Expr itself is a subtraction pattern with guard info.
16352 if (const SCEV *Rewritten = RewriteSubtraction(Expr))
16353 return Rewritten;
16354
16355 // Trip count expressions sometimes consist of adding 3 operands, i.e.
16356 // (Const + A + B). There may be guard info for A + B, and if so, apply
16357 // it.
16358 // TODO: Could more generally apply guards to Add sub-expressions.
16359 if (isa<SCEVConstant>(Val: Expr->getOperand(i: 0))) {
16360 if (Expr->getNumOperands() == 3) {
16361 const SCEV *Add =
16362 SE.getAddExpr(LHS: Expr->getOperand(i: 1), RHS: Expr->getOperand(i: 2));
16363 if (const SCEV *Rewritten = RewriteSubtraction(Add))
16364 return SE.getAddExpr(
16365 LHS: Expr->getOperand(i: 0), RHS: Rewritten,
16366 Flags: ScalarEvolution::maskFlags(Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16367 if (const SCEV *S = Map.lookup(Val: Add))
16368 return SE.getAddExpr(LHS: Expr->getOperand(i: 0), RHS: S);
16369 }
16370
16371 // For expressions of the form (Const + A), check if we have guard info
16372 // for (Const + 1 + A), and rewrite to ((Const + 1 + A) - 1). This makes
16373 // sure we don't lose information when rewriting expressions based on
16374 // back-edge taken counts in some cases.
16375 if (Expr->getNumOperands() == 2) {
16376 const SCEV *S = nullptr;
16377 // Handle (-1 + 1 + A) without constructing SCEVs.
16378 if (match(U: Expr->getOperand(i: 0), P: m_scev_AllOnes())) {
16379 S = Map.lookup(Val: Expr->getOperand(i: 1));
16380 } else {
16381 const SCEV *NewC =
16382 SE.getAddExpr(LHS: Expr->getOperand(i: 0), RHS: SE.getOne(Ty: Expr->getType()));
16383 S = Map.lookup(Val: SE.getAddExpr(LHS: NewC, RHS: Expr->getOperand(i: 1)));
16384 }
16385 if (S)
16386 return SE.getAddExpr(LHS: S, RHS: SE.getMinusOne(Ty: Expr->getType()));
16387 }
16388 }
16389 SmallVector<SCEVUse, 2> Operands;
16390 bool Changed = false;
16391 for (SCEVUse Op : Expr->operands()) {
16392 Operands.push_back(
16393 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
16394 Changed |= Op != Operands.back();
16395 }
16396 // We are only replacing operands with equivalent values, so transfer the
16397 // flags from the original expression.
16398 return !Changed ? Expr
16399 : SE.getAddExpr(Ops&: Operands,
16400 OrigFlags: ScalarEvolution::maskFlags(
16401 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16402 }
16403
16404 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
16405 SmallVector<SCEVUse, 2> Operands;
16406 bool Changed = false;
16407 for (SCEVUse Op : Expr->operands()) {
16408 Operands.push_back(
16409 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
16410 Changed |= Op != Operands.back();
16411 }
16412 // We are only replacing operands with equivalent values, so transfer the
16413 // flags from the original expression.
16414 return !Changed ? Expr
16415 : SE.getMulExpr(Ops&: Operands,
16416 OrigFlags: ScalarEvolution::maskFlags(
16417 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16418 }
16419 };
16420
16421 if (RewriteMap.empty() && NotEqual.empty())
16422 return Expr;
16423
16424 SCEVLoopGuardRewriter Rewriter(SE, *this);
16425 return Rewriter.visit(S: Expr);
16426}
16427
16428const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
16429 return applyLoopGuards(Expr, Guards: LoopGuards::collect(L, SE&: *this));
16430}
16431
16432const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr,
16433 const LoopGuards &Guards) {
16434 return Guards.rewrite(Expr);
16435}
16436