1//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the scalar evolution analysis
10// engine, which is used primarily to analyze expressions involving induction
11// variables in loops.
12//
13// There are several aspects to this library. First is the representation of
14// scalar expressions, which are represented as subclasses of the SCEV class.
15// These classes are used to represent certain types of subexpressions that we
16// can handle. We only create one SCEV of a particular shape, so
17// pointer-comparisons for equality are legal.
18//
19// One important aspect of the SCEV objects is that they are never cyclic, even
20// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
21// the PHI node is one of the idioms that we can represent (e.g., a polynomial
22// recurrence) then we represent it directly as a recurrence node, otherwise we
23// represent it as a SCEVUnknown node.
24//
25// In addition to being able to represent expressions of various types, we also
26// have folders that are used to build the *canonical* representation for a
27// particular expression. These folders are capable of using a variety of
28// rewrite rules to simplify the expressions.
29//
30// Once the folders are defined, we can implement the more interesting
31// higher-level code, such as the code that recognizes PHI nodes of various
32// types, computes the execution count of a loop, etc.
33//
34// TODO: We should use these routines and value representations to implement
35// dependence analysis!
36//
37//===----------------------------------------------------------------------===//
38//
39// There are several good references for the techniques used in this analysis.
40//
41// Chains of recurrences -- a method to expedite the evaluation
42// of closed-form functions
43// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
44//
45// On computational properties of chains of recurrences
46// Eugene V. Zima
47//
48// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
49// Robert A. van Engelen
50//
51// Efficient Symbolic Analysis for Optimizing Compilers
52// Robert A. van Engelen
53//
54// Using the chains of recurrences algebra for data dependence testing and
55// induction variable substitution
56// MS Thesis, Johnie Birch
57//
58//===----------------------------------------------------------------------===//
59
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/ADT/APInt.h"
62#include "llvm/ADT/ArrayRef.h"
63#include "llvm/ADT/DenseMap.h"
64#include "llvm/ADT/DepthFirstIterator.h"
65#include "llvm/ADT/FoldingSet.h"
66#include "llvm/ADT/STLExtras.h"
67#include "llvm/ADT/ScopeExit.h"
68#include "llvm/ADT/Sequence.h"
69#include "llvm/ADT/SmallPtrSet.h"
70#include "llvm/ADT/SmallVector.h"
71#include "llvm/ADT/Statistic.h"
72#include "llvm/ADT/StringExtras.h"
73#include "llvm/ADT/StringRef.h"
74#include "llvm/Analysis/AssumptionCache.h"
75#include "llvm/Analysis/ConstantFolding.h"
76#include "llvm/Analysis/InstructionSimplify.h"
77#include "llvm/Analysis/LoopInfo.h"
78#include "llvm/Analysis/MemoryBuiltins.h"
79#include "llvm/Analysis/ScalarEvolutionExpressions.h"
80#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
81#include "llvm/Analysis/TargetLibraryInfo.h"
82#include "llvm/Analysis/ValueTracking.h"
83#include "llvm/Config/llvm-config.h"
84#include "llvm/IR/Argument.h"
85#include "llvm/IR/BasicBlock.h"
86#include "llvm/IR/CFG.h"
87#include "llvm/IR/Constant.h"
88#include "llvm/IR/ConstantRange.h"
89#include "llvm/IR/Constants.h"
90#include "llvm/IR/DataLayout.h"
91#include "llvm/IR/DerivedTypes.h"
92#include "llvm/IR/Dominators.h"
93#include "llvm/IR/Function.h"
94#include "llvm/IR/GlobalAlias.h"
95#include "llvm/IR/GlobalValue.h"
96#include "llvm/IR/InstIterator.h"
97#include "llvm/IR/InstrTypes.h"
98#include "llvm/IR/Instruction.h"
99#include "llvm/IR/Instructions.h"
100#include "llvm/IR/IntrinsicInst.h"
101#include "llvm/IR/Intrinsics.h"
102#include "llvm/IR/LLVMContext.h"
103#include "llvm/IR/Operator.h"
104#include "llvm/IR/PatternMatch.h"
105#include "llvm/IR/Type.h"
106#include "llvm/IR/Use.h"
107#include "llvm/IR/User.h"
108#include "llvm/IR/Value.h"
109#include "llvm/IR/Verifier.h"
110#include "llvm/InitializePasses.h"
111#include "llvm/Pass.h"
112#include "llvm/Support/Casting.h"
113#include "llvm/Support/CommandLine.h"
114#include "llvm/Support/Compiler.h"
115#include "llvm/Support/Debug.h"
116#include "llvm/Support/ErrorHandling.h"
117#include "llvm/Support/InterleavedRange.h"
118#include "llvm/Support/KnownBits.h"
119#include "llvm/Support/SaveAndRestore.h"
120#include "llvm/Support/raw_ostream.h"
121#include <algorithm>
122#include <cassert>
123#include <climits>
124#include <cstdint>
125#include <cstdlib>
126#include <map>
127#include <memory>
128#include <numeric>
129#include <optional>
130#include <tuple>
131#include <utility>
132#include <vector>
133
134using namespace llvm;
135using namespace PatternMatch;
136using namespace SCEVPatternMatch;
137
138#define DEBUG_TYPE "scalar-evolution"
139
140STATISTIC(NumExitCountsComputed,
141 "Number of loop exits with predictable exit counts");
142STATISTIC(NumExitCountsNotComputed,
143 "Number of loop exits without predictable exit counts");
144STATISTIC(NumBruteForceTripCountsComputed,
145 "Number of loops with trip counts computed by force");
146
147#ifdef EXPENSIVE_CHECKS
148bool llvm::VerifySCEV = true;
149#else
150bool llvm::VerifySCEV = false;
151#endif
152
153static cl::opt<unsigned>
154 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
155 cl::desc("Maximum number of iterations SCEV will "
156 "symbolically execute a constant "
157 "derived loop"),
158 cl::init(Val: 100));
159
160static cl::opt<bool, true> VerifySCEVOpt(
161 "verify-scev", cl::Hidden, cl::location(L&: VerifySCEV),
162 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
163static cl::opt<bool> VerifySCEVStrict(
164 "verify-scev-strict", cl::Hidden,
165 cl::desc("Enable stricter verification with -verify-scev is passed"));
166
167static cl::opt<bool> VerifyIR(
168 "scev-verify-ir", cl::Hidden,
169 cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"),
170 cl::init(Val: false));
171
172static cl::opt<unsigned> MulOpsInlineThreshold(
173 "scev-mulops-inline-threshold", cl::Hidden,
174 cl::desc("Threshold for inlining multiplication operands into a SCEV"),
175 cl::init(Val: 32));
176
177static cl::opt<unsigned> AddOpsInlineThreshold(
178 "scev-addops-inline-threshold", cl::Hidden,
179 cl::desc("Threshold for inlining addition operands into a SCEV"),
180 cl::init(Val: 500));
181
182static cl::opt<unsigned> MaxSCEVCompareDepth(
183 "scalar-evolution-max-scev-compare-depth", cl::Hidden,
184 cl::desc("Maximum depth of recursive SCEV complexity comparisons"),
185 cl::init(Val: 32));
186
187static cl::opt<unsigned> MaxSCEVOperationsImplicationDepth(
188 "scalar-evolution-max-scev-operations-implication-depth", cl::Hidden,
189 cl::desc("Maximum depth of recursive SCEV operations implication analysis"),
190 cl::init(Val: 2));
191
192static cl::opt<unsigned> MaxValueCompareDepth(
193 "scalar-evolution-max-value-compare-depth", cl::Hidden,
194 cl::desc("Maximum depth of recursive value complexity comparisons"),
195 cl::init(Val: 2));
196
197static cl::opt<unsigned>
198 MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden,
199 cl::desc("Maximum depth of recursive arithmetics"),
200 cl::init(Val: 32));
201
202static cl::opt<unsigned> MaxConstantEvolvingDepth(
203 "scalar-evolution-max-constant-evolving-depth", cl::Hidden,
204 cl::desc("Maximum depth of recursive constant evolving"), cl::init(Val: 32));
205
206static cl::opt<unsigned>
207 MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden,
208 cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"),
209 cl::init(Val: 8));
210
211static cl::opt<unsigned>
212 MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden,
213 cl::desc("Max coefficients in AddRec during evolving"),
214 cl::init(Val: 8));
215
216static cl::opt<unsigned>
217 HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden,
218 cl::desc("Size of the expression which is considered huge"),
219 cl::init(Val: 4096));
220
221static cl::opt<unsigned> RangeIterThreshold(
222 "scev-range-iter-threshold", cl::Hidden,
223 cl::desc("Threshold for switching to iteratively computing SCEV ranges"),
224 cl::init(Val: 32));
225
226static cl::opt<unsigned> MaxLoopGuardCollectionDepth(
227 "scalar-evolution-max-loop-guard-collection-depth", cl::Hidden,
228 cl::desc("Maximum depth for recursive loop guard collection"), cl::init(Val: 1));
229
230static cl::opt<bool>
231ClassifyExpressions("scalar-evolution-classify-expressions",
232 cl::Hidden, cl::init(Val: true),
233 cl::desc("When printing analysis, include information on every instruction"));
234
235static cl::opt<bool> UseExpensiveRangeSharpening(
236 "scalar-evolution-use-expensive-range-sharpening", cl::Hidden,
237 cl::init(Val: false),
238 cl::desc("Use more powerful methods of sharpening expression ranges. May "
239 "be costly in terms of compile time"));
240
241static cl::opt<bool>
242 EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden,
243 cl::desc("Handle <= and >= in finite loops"),
244 cl::init(Val: true));
245
246static cl::opt<bool> UseContextForNoWrapFlagInference(
247 "scalar-evolution-use-context-for-no-wrap-flag-strenghening", cl::Hidden,
248 cl::desc("Infer nuw/nsw flags using context where suitable"),
249 cl::init(Val: true));
250
251//===----------------------------------------------------------------------===//
252// SCEV class definitions
253//===----------------------------------------------------------------------===//
254
255void SCEV::computeAndSetCanonical(ScalarEvolution &SE) {
256 // Leaf nodes are always their own canonical.
257 switch (getSCEVType()) {
258 case scConstant:
259 case scVScale:
260 case scUnknown:
261 CanonicalSCEV = this;
262 return;
263 default:
264 break;
265 }
266
267 // For all other expressions, check whether any immediate operand has a
268 // different canonical. Since operands are always created before their parent,
269 // their canonical pointers are already set — no recursion needed.
270 bool Changed = false;
271 SmallVector<SCEVUse, 4> CanonOps;
272 for (SCEVUse Op : operands()) {
273 CanonOps.push_back(Elt: Op->getCanonical());
274 Changed |= CanonOps.back() != Op;
275 }
276
277 if (!Changed) {
278 CanonicalSCEV = this;
279 return;
280 }
281
282 // Rebuild the expression from the canonical operands, stripping use flags.
283 CanonicalSCEV = SE.getWithOperands(S: this, NewOps&: CanonOps);
284}
285
286//===----------------------------------------------------------------------===//
287// Implementation of the SCEV class.
288//
289
290#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
291LLVM_DUMP_METHOD void SCEV::dump() const {
292 print(dbgs());
293 dbgs() << '\n';
294}
295#endif
296
297void SCEV::print(raw_ostream &OS) const {
298 switch (getSCEVType()) {
299 case scConstant:
300 cast<SCEVConstant>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
301 return;
302 case scVScale:
303 OS << "vscale";
304 return;
305 case scPtrToAddr: {
306 const SCEVCastExpr *PtrCast = cast<SCEVCastExpr>(Val: this);
307 SCEVUse Op = PtrCast->getOperand();
308 OS << "(ptrtoaddr " << *Op->getType() << " " << Op << " to "
309 << *PtrCast->getType() << ")";
310 return;
311 }
312 case scTruncate: {
313 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: this);
314 SCEVUse Op = Trunc->getOperand();
315 OS << "(trunc " << *Op->getType() << " " << Op << " to "
316 << *Trunc->getType() << ")";
317 return;
318 }
319 case scZeroExtend: {
320 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: this);
321 SCEVUse Op = ZExt->getOperand();
322 OS << "(zext " << *Op->getType() << " " << Op << " to " << *ZExt->getType()
323 << ")";
324 return;
325 }
326 case scSignExtend: {
327 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: this);
328 SCEVUse Op = SExt->getOperand();
329 OS << "(sext " << *Op->getType() << " " << Op << " to " << *SExt->getType()
330 << ")";
331 return;
332 }
333 case scAddRecExpr: {
334 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: this);
335 OS << "{" << AR->getOperand(i: 0);
336 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
337 OS << ",+," << AR->getOperand(i);
338 OS << "}<";
339 if (AR->hasNoUnsignedWrap())
340 OS << "nuw><";
341 if (AR->hasNoSignedWrap())
342 OS << "nsw><";
343 if (AR->hasNoSelfWrap() && !AR->hasNoUnsignedWrap() &&
344 !AR->hasNoSignedWrap())
345 OS << "nw><";
346 AR->getLoop()->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
347 OS << ">";
348 return;
349 }
350 case scAddExpr:
351 case scMulExpr:
352 case scUMaxExpr:
353 case scSMaxExpr:
354 case scUMinExpr:
355 case scSMinExpr:
356 case scSequentialUMinExpr: {
357 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(Val: this);
358 const char *OpStr = nullptr;
359 switch (NAry->getSCEVType()) {
360 case scAddExpr: OpStr = " + "; break;
361 case scMulExpr: OpStr = " * "; break;
362 case scUMaxExpr: OpStr = " umax "; break;
363 case scSMaxExpr: OpStr = " smax "; break;
364 case scUMinExpr:
365 OpStr = " umin ";
366 break;
367 case scSMinExpr:
368 OpStr = " smin ";
369 break;
370 case scSequentialUMinExpr:
371 OpStr = " umin_seq ";
372 break;
373 default:
374 llvm_unreachable("There are no other nary expression types.");
375 }
376 OS << "(" << llvm::interleaved(R: NAry->operands(), Separator: OpStr) << ")";
377 switch (NAry->getSCEVType()) {
378 case scAddExpr:
379 case scMulExpr:
380 if (NAry->hasNoUnsignedWrap())
381 OS << "<nuw>";
382 if (NAry->hasNoSignedWrap())
383 OS << "<nsw>";
384 break;
385 default:
386 // Nothing to print for other nary expressions.
387 break;
388 }
389 return;
390 }
391 case scUDivExpr: {
392 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: this);
393 OS << "(" << UDiv->getLHS() << " /u " << UDiv->getRHS() << ")";
394 return;
395 }
396 case scUnknown:
397 cast<SCEVUnknown>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
398 return;
399 case scCouldNotCompute:
400 OS << "***COULDNOTCOMPUTE***";
401 return;
402 }
403 llvm_unreachable("Unknown SCEV kind!");
404}
405
406ArrayRef<SCEVUse> SCEV::operands() const {
407 switch (getSCEVType()) {
408 case scConstant:
409 case scVScale:
410 case scUnknown:
411 return {};
412 case scPtrToAddr:
413 case scTruncate:
414 case scZeroExtend:
415 case scSignExtend:
416 return cast<SCEVCastExpr>(Val: this)->operands();
417 case scAddRecExpr:
418 case scAddExpr:
419 case scMulExpr:
420 case scUMaxExpr:
421 case scSMaxExpr:
422 case scUMinExpr:
423 case scSMinExpr:
424 case scSequentialUMinExpr:
425 return cast<SCEVNAryExpr>(Val: this)->operands();
426 case scUDivExpr:
427 return cast<SCEVUDivExpr>(Val: this)->operands();
428 case scCouldNotCompute:
429 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
430 }
431 llvm_unreachable("Unknown SCEV kind!");
432}
433
434bool SCEV::isZero() const { return match(S: this, P: m_scev_Zero()); }
435
436bool SCEV::isOne() const { return match(S: this, P: m_scev_One()); }
437
438bool SCEV::isAllOnesValue() const { return match(S: this, P: m_scev_AllOnes()); }
439
440bool SCEV::isNonConstantNegative() const {
441 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: this);
442 if (!Mul) return false;
443
444 // If there is a constant factor, it will be first.
445 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: 0));
446 if (!SC) return false;
447
448 // Return true if the value is negative, this matches things like (-42 * V).
449 return SC->getAPInt().isNegative();
450}
451
452SCEVCouldNotCompute::SCEVCouldNotCompute()
453 : SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0, nullptr) {}
454
455bool SCEVCouldNotCompute::classof(const SCEV *S) {
456 return S->getSCEVType() == scCouldNotCompute;
457}
458
459const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
460 auto &Entry = ConstantSCEVs[V];
461 if (Entry)
462 return Entry;
463
464 FoldingSetNodeID ID;
465 ID.AddInteger(I: scConstant);
466 ID.AddPointer(Ptr: V);
467 FoldingSetInsertToken Token;
468 if (SCEVConstant *S =
469 static_cast<SCEVConstant *>(UniqueSCEVs.lookup(ID, Token)))
470 return Entry = S;
471 SCEVConstant *S =
472 new (SCEVAllocator) SCEVConstant(ID.Intern(Allocator&: SCEVAllocator), V);
473 UniqueSCEVs.insert(N: S, Token);
474 S->computeAndSetCanonical(SE&: *this);
475 return Entry = S;
476}
477
478const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
479 return getConstant(V: ConstantInt::get(Context&: getContext(), V: Val));
480}
481
482const SCEV *
483ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
484 IntegerType *ITy = cast<IntegerType>(Val: getEffectiveSCEVType(Ty));
485 // TODO: Avoid implicit trunc?
486 // See https://github.com/llvm/llvm-project/issues/112510.
487 return getConstant(
488 V: ConstantInt::get(Ty: ITy, V, IsSigned: isSigned, /*ImplicitTrunc=*/true));
489}
490
491const SCEV *ScalarEvolution::getVScale(Type *Ty) {
492 FoldingSetNodeID ID;
493 ID.AddInteger(I: scVScale);
494 ID.AddPointer(Ptr: Ty);
495 FoldingSetInsertToken Token;
496 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
497 return S;
498 SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(Allocator&: SCEVAllocator), Ty);
499 UniqueSCEVs.insert(N: S, Token);
500 S->computeAndSetCanonical(SE&: *this);
501 return S;
502}
503
504const SCEV *ScalarEvolution::getElementCount(Type *Ty, ElementCount EC,
505 SCEV::NoWrapFlags Flags) {
506 const SCEV *Res = getConstant(Ty, V: EC.getKnownMinValue());
507 if (EC.isScalable())
508 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty), Flags);
509 return Res;
510}
511
512SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
513 SCEVUse op, Type *ty)
514 : SCEV(ID, SCEVTy, computeExpressionSize(Args: op), ty), Op(op) {}
515
516SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID,
517 const SCEV *Op, Type *ITy)
518 : SCEVCastExpr(ID, scPtrToAddr, Op, ITy) {
519 assert(getOperand()->getType()->isPointerTy() && getType()->isIntegerTy() &&
520 "Must be a non-bit-width-changing pointer-to-integer cast!");
521}
522
523SCEVIntegralCastExpr::SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID,
524 SCEVTypes SCEVTy, SCEVUse op,
525 Type *ty)
526 : SCEVCastExpr(ID, SCEVTy, op, ty) {}
527
528SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
529 Type *ty)
530 : SCEVIntegralCastExpr(ID, scTruncate, op, ty) {
531 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
532 "Cannot truncate non-integer value!");
533}
534
535SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
536 Type *ty)
537 : SCEVIntegralCastExpr(ID, scZeroExtend, op, ty) {
538 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
539 "Cannot zero extend non-integer value!");
540}
541
542SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
543 Type *ty)
544 : SCEVIntegralCastExpr(ID, scSignExtend, op, ty) {
545 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
546 "Cannot sign extend non-integer value!");
547}
548
549void SCEVUnknown::deleted() {
550 // Clear this SCEVUnknown from various maps.
551 SE->forgetMemoizedResults(SCEVs: {this});
552
553 // Remove this SCEVUnknown from the uniquing map.
554 SE->UniqueSCEVs.erase(N: this);
555
556 // Release the value.
557 setValPtr(nullptr);
558}
559
560void SCEVUnknown::allUsesReplacedWith(Value *New) {
561 // Clear this SCEVUnknown from various maps.
562 SE->forgetMemoizedResults(SCEVs: {this});
563
564 // Remove this SCEVUnknown from the uniquing map.
565 SE->UniqueSCEVs.erase(N: this);
566
567 // Replace the value pointer in case someone is still using this SCEVUnknown.
568 setValPtr(New);
569}
570
571//===----------------------------------------------------------------------===//
572// SCEV Utilities
573//===----------------------------------------------------------------------===//
574
575/// Compare the two values \p LV and \p RV in terms of their "complexity" where
576/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
577/// operands in SCEV expressions.
578static int CompareValueComplexity(const LoopInfo *const LI, Value *LV,
579 Value *RV, unsigned Depth) {
580 if (Depth > MaxValueCompareDepth)
581 return 0;
582
583 // Order pointer values after integer values. This helps SCEVExpander form
584 // GEPs.
585 bool LIsPointer = LV->getType()->isPointerTy(),
586 RIsPointer = RV->getType()->isPointerTy();
587 if (LIsPointer != RIsPointer)
588 return (int)LIsPointer - (int)RIsPointer;
589
590 // Compare getValueID values.
591 unsigned LID = LV->getValueID(), RID = RV->getValueID();
592 if (LID != RID)
593 return (int)LID - (int)RID;
594
595 // Sort arguments by their position.
596 if (const auto *LA = dyn_cast<Argument>(Val: LV)) {
597 const auto *RA = cast<Argument>(Val: RV);
598 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
599 return (int)LArgNo - (int)RArgNo;
600 }
601
602 if (const auto *LGV = dyn_cast<GlobalValue>(Val: LV)) {
603 const auto *RGV = cast<GlobalValue>(Val: RV);
604
605 if (auto L = LGV->getLinkage() - RGV->getLinkage())
606 return L;
607
608 const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
609 auto LT = GV->getLinkage();
610 return !(GlobalValue::isPrivateLinkage(Linkage: LT) ||
611 GlobalValue::isInternalLinkage(Linkage: LT));
612 };
613
614 // Use the names to distinguish the two values, but only if the
615 // names are semantically important.
616 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
617 return LGV->getName().compare(RHS: RGV->getName());
618 }
619
620 // For instructions, compare their loop depth, and their operand count. This
621 // is pretty loose.
622 if (const auto *LInst = dyn_cast<Instruction>(Val: LV)) {
623 const auto *RInst = cast<Instruction>(Val: RV);
624
625 // Compare loop depths.
626 const BasicBlock *LParent = LInst->getParent(),
627 *RParent = RInst->getParent();
628 if (LParent != RParent) {
629 unsigned LDepth = LI->getLoopDepth(BB: LParent),
630 RDepth = LI->getLoopDepth(BB: RParent);
631 if (LDepth != RDepth)
632 return (int)LDepth - (int)RDepth;
633 }
634
635 // Compare the number of operands.
636 unsigned LNumOps = LInst->getNumOperands(),
637 RNumOps = RInst->getNumOperands();
638 if (LNumOps != RNumOps)
639 return (int)LNumOps - (int)RNumOps;
640
641 for (unsigned Idx : seq(Size: LNumOps)) {
642 int Result = CompareValueComplexity(LI, LV: LInst->getOperand(i: Idx),
643 RV: RInst->getOperand(i: Idx), Depth: Depth + 1);
644 if (Result != 0)
645 return Result;
646 }
647 }
648
649 return 0;
650}
651
652// Return negative, zero, or positive, if LHS is less than, equal to, or greater
653// than RHS, respectively. A three-way result allows recursive comparisons to be
654// more efficient.
655// If the max analysis depth was reached, return std::nullopt, assuming we do
656// not know if they are equivalent for sure.
657static std::optional<int>
658CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
659 const SCEV *RHS, DominatorTree &DT, unsigned Depth = 0) {
660 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
661 if (LHS == RHS)
662 return 0;
663
664 // Primarily, sort the SCEVs by their getSCEVType().
665 SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
666 if (LType != RType)
667 return (int)LType - (int)RType;
668
669 if (Depth > MaxSCEVCompareDepth)
670 return std::nullopt;
671
672 // Aside from the getSCEVType() ordering, the particular ordering
673 // isn't very important except that it's beneficial to be consistent,
674 // so that (a + b) and (b + a) don't end up as different expressions.
675 switch (LType) {
676 case scUnknown: {
677 const SCEVUnknown *LU = cast<SCEVUnknown>(Val: LHS);
678 const SCEVUnknown *RU = cast<SCEVUnknown>(Val: RHS);
679
680 int X =
681 CompareValueComplexity(LI, LV: LU->getValue(), RV: RU->getValue(), Depth: Depth + 1);
682 return X;
683 }
684
685 case scConstant: {
686 const SCEVConstant *LC = cast<SCEVConstant>(Val: LHS);
687 const SCEVConstant *RC = cast<SCEVConstant>(Val: RHS);
688
689 // Compare constant values.
690 const APInt &LA = LC->getAPInt();
691 const APInt &RA = RC->getAPInt();
692 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
693 if (LBitWidth != RBitWidth)
694 return (int)LBitWidth - (int)RBitWidth;
695 return LA.ult(RHS: RA) ? -1 : 1;
696 }
697
698 case scVScale: {
699 const auto *LTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: LHS)->getType());
700 const auto *RTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: RHS)->getType());
701 return LTy->getBitWidth() - RTy->getBitWidth();
702 }
703
704 case scAddRecExpr: {
705 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(Val: LHS);
706 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(Val: RHS);
707
708 // There is always a dominance between two recs that are used by one SCEV,
709 // so we can safely sort recs by loop header dominance. We require such
710 // order in getAddExpr.
711 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
712 if (LLoop != RLoop) {
713 const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
714 assert(LHead != RHead && "Two loops share the same header?");
715 if (DT.dominates(A: LHead, B: RHead))
716 return 1;
717 assert(DT.dominates(RHead, LHead) &&
718 "No dominance between recurrences used by one SCEV?");
719 return -1;
720 }
721
722 [[fallthrough]];
723 }
724
725 case scTruncate:
726 case scZeroExtend:
727 case scSignExtend:
728 case scPtrToAddr:
729 case scAddExpr:
730 case scMulExpr:
731 case scUDivExpr:
732 case scSMaxExpr:
733 case scUMaxExpr:
734 case scSMinExpr:
735 case scUMinExpr:
736 case scSequentialUMinExpr: {
737 ArrayRef<SCEVUse> LOps = LHS->operands();
738 ArrayRef<SCEVUse> ROps = RHS->operands();
739
740 // Lexicographically compare n-ary-like expressions.
741 unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
742 if (LNumOps != RNumOps)
743 return (int)LNumOps - (int)RNumOps;
744
745 for (unsigned i = 0; i != LNumOps; ++i) {
746 auto X = CompareSCEVComplexity(LI, LHS: LOps[i].getPointer(),
747 RHS: ROps[i].getPointer(), DT, Depth: Depth + 1);
748 if (X != 0)
749 return X;
750 }
751 return 0;
752 }
753
754 case scCouldNotCompute:
755 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
756 }
757 llvm_unreachable("Unknown SCEV kind!");
758}
759
760/// Given a list of SCEV objects, order them by their complexity, and group
761/// objects of the same complexity together by value. When this routine is
762/// finished, we know that any duplicates in the vector are consecutive and that
763/// complexity is monotonically increasing.
764///
765/// Note that we go take special precautions to ensure that we get deterministic
766/// results from this routine. In other words, we don't want the results of
767/// this to depend on where the addresses of various SCEV objects happened to
768/// land in memory.
769static void GroupByComplexity(SmallVectorImpl<SCEVUse> &Ops, LoopInfo *LI,
770 DominatorTree &DT) {
771 if (Ops.size() < 2) return; // Noop
772
773 // Whether LHS has provably less complexity than RHS.
774 auto IsLessComplex = [&](SCEVUse LHS, SCEVUse RHS) {
775 auto Complexity = CompareSCEVComplexity(LI, LHS, RHS, DT);
776 return Complexity && *Complexity < 0;
777 };
778 if (Ops.size() == 2) {
779 // This is the common case, which also happens to be trivially simple.
780 // Special case it.
781 SCEVUse &LHS = Ops[0], &RHS = Ops[1];
782 if (IsLessComplex(RHS, LHS))
783 std::swap(a&: LHS, b&: RHS);
784 return;
785 }
786
787 // Do the rough sort by complexity.
788 llvm::stable_sort(
789 Range&: Ops, C: [&](SCEVUse LHS, SCEVUse RHS) { return IsLessComplex(LHS, RHS); });
790
791 // Now that we are sorted by complexity, group elements of the same
792 // complexity. Note that this is, at worst, N^2, but the vector is likely to
793 // be extremely short in practice. Note that we take this approach because we
794 // do not want to depend on the addresses of the objects we are grouping.
795 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
796 const SCEV *S = Ops[i];
797 unsigned Complexity = S->getSCEVType();
798
799 // If there are any objects of the same complexity and same value as this
800 // one, group them.
801 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
802 if (Ops[j] == S) { // Found a duplicate.
803 // Move it to immediately after i'th element.
804 std::swap(a&: Ops[i+1], b&: Ops[j]);
805 ++i; // no need to rescan it.
806 if (i == e-2) return; // Done!
807 }
808 }
809 }
810}
811
812/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
813/// least HugeExprThreshold nodes).
814static bool hasHugeExpression(ArrayRef<SCEVUse> Ops) {
815 return any_of(Range&: Ops, P: [](const SCEV *S) {
816 return S->getExpressionSize() >= HugeExprThreshold;
817 });
818}
819
820/// Performs a number of common optimizations on the passed \p Ops. If the
821/// whole expression reduces down to a single operand, it will be returned.
822///
823/// The following optimizations are performed:
824/// * Fold constants using the \p Fold function.
825/// * Remove identity constants satisfying \p IsIdentity.
826/// * If a constant satisfies \p IsAbsorber, return it.
827/// * Sort operands by complexity.
828template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
829static const SCEV *
830constantFoldAndGroupOps(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT,
831 SmallVectorImpl<SCEVUse> &Ops, FoldT Fold,
832 IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
833 const SCEVConstant *Folded = nullptr;
834 for (unsigned Idx = 0; Idx < Ops.size();) {
835 const SCEV *Op = Ops[Idx];
836 if (const auto *C = dyn_cast<SCEVConstant>(Val: Op)) {
837 if (!Folded)
838 Folded = C;
839 else
840 Folded = cast<SCEVConstant>(
841 SE.getConstant(Fold(Folded->getAPInt(), C->getAPInt())));
842 Ops.erase(CI: Ops.begin() + Idx);
843 continue;
844 }
845 ++Idx;
846 }
847
848 if (Ops.empty()) {
849 assert(Folded && "Must have folded value");
850 return Folded;
851 }
852
853 if (Folded && IsAbsorber(Folded->getAPInt()))
854 return Folded;
855
856 GroupByComplexity(Ops, LI: &LI, DT);
857 if (Folded && !IsIdentity(Folded->getAPInt()))
858 Ops.insert(I: Ops.begin(), Elt: Folded);
859
860 return Ops.size() == 1 ? Ops[0] : nullptr;
861}
862
863//===----------------------------------------------------------------------===//
864// Simple SCEV method implementations
865//===----------------------------------------------------------------------===//
866
867/// Compute BC(It, K). The result has width W. Assume, K > 0.
868static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
869 ScalarEvolution &SE,
870 Type *ResultTy) {
871 // Handle the simplest case efficiently.
872 if (K == 1)
873 return SE.getTruncateOrZeroExtend(V: It, Ty: ResultTy);
874
875 // We are using the following formula for BC(It, K):
876 //
877 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
878 //
879 // Suppose, W is the bitwidth of the return value. We must be prepared for
880 // overflow. Hence, we must assure that the result of our computation is
881 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
882 // safe in modular arithmetic.
883 //
884 // However, this code doesn't use exactly that formula; the formula it uses
885 // is something like the following, where T is the number of factors of 2 in
886 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
887 // exponentiation:
888 //
889 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
890 //
891 // This formula is trivially equivalent to the previous formula. However,
892 // this formula can be implemented much more efficiently. The trick is that
893 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
894 // arithmetic. To do exact division in modular arithmetic, all we have
895 // to do is multiply by the inverse. Therefore, this step can be done at
896 // width W.
897 //
898 // The next issue is how to safely do the division by 2^T. The way this
899 // is done is by doing the multiplication step at a width of at least W + T
900 // bits. This way, the bottom W+T bits of the product are accurate. Then,
901 // when we perform the division by 2^T (which is equivalent to a right shift
902 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
903 // truncated out after the division by 2^T.
904 //
905 // In comparison to just directly using the first formula, this technique
906 // is much more efficient; using the first formula requires W * K bits,
907 // but this formula less than W + K bits. Also, the first formula requires
908 // a division step, whereas this formula only requires multiplies and shifts.
909 //
910 // It doesn't matter whether the subtraction step is done in the calculation
911 // width or the input iteration count's width; if the subtraction overflows,
912 // the result must be zero anyway. We prefer here to do it in the width of
913 // the induction variable because it helps a lot for certain cases; CodeGen
914 // isn't smart enough to ignore the overflow, which leads to much less
915 // efficient code if the width of the subtraction is wider than the native
916 // register width.
917 //
918 // (It's possible to not widen at all by pulling out factors of 2 before
919 // the multiplication; for example, K=2 can be calculated as
920 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
921 // extra arithmetic, so it's not an obvious win, and it gets
922 // much more complicated for K > 3.)
923
924 // Protection from insane SCEVs; this bound is conservative,
925 // but it probably doesn't matter.
926 if (K > 1000)
927 return SE.getCouldNotCompute();
928
929 unsigned W = SE.getTypeSizeInBits(Ty: ResultTy);
930
931 // Calculate K! / 2^T and T; we divide out the factors of two before
932 // multiplying for calculating K! / 2^T to avoid overflow.
933 // Other overflow doesn't matter because we only care about the bottom
934 // W bits of the result.
935 APInt OddFactorial(W, 1);
936 unsigned T = 1;
937 for (unsigned i = 3; i <= K; ++i) {
938 unsigned TwoFactors = countr_zero(Val: i);
939 T += TwoFactors;
940 OddFactorial *= (i >> TwoFactors);
941 }
942
943 // We need at least W + T bits for the multiplication step
944 unsigned CalculationBits = W + T;
945
946 // Calculate 2^T, at width T+W.
947 APInt DivFactor = APInt::getOneBitSet(numBits: CalculationBits, BitNo: T);
948
949 // Calculate the multiplicative inverse of K! / 2^T;
950 // this multiplication factor will perform the exact division by
951 // K! / 2^T.
952 APInt MultiplyFactor = OddFactorial.multiplicativeInverse();
953
954 // Calculate the product, at width T+W
955 IntegerType *CalculationTy = IntegerType::get(C&: SE.getContext(),
956 NumBits: CalculationBits);
957 const SCEV *Dividend = SE.getTruncateOrZeroExtend(V: It, Ty: CalculationTy);
958 for (unsigned i = 1; i != K; ++i) {
959 const SCEV *S = SE.getMinusSCEV(LHS: It, RHS: SE.getConstant(Ty: It->getType(), V: i));
960 Dividend = SE.getMulExpr(LHS: Dividend,
961 RHS: SE.getTruncateOrZeroExtend(V: S, Ty: CalculationTy));
962 }
963
964 // Divide by 2^T
965 const SCEV *DivResult = SE.getUDivExpr(LHS: Dividend, RHS: SE.getConstant(Val: DivFactor));
966
967 // Truncate the result, and divide by K! / 2^T.
968
969 return SE.getMulExpr(LHS: SE.getConstant(Val: MultiplyFactor),
970 RHS: SE.getTruncateOrZeroExtend(V: DivResult, Ty: ResultTy));
971}
972
973/// Return the value of this chain of recurrences at the specified iteration
974/// number. We can evaluate this recurrence by multiplying each element in the
975/// chain by the binomial coefficient corresponding to it. In other words, we
976/// can evaluate {A,+,B,+,C,+,D} as:
977///
978/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
979///
980/// where BC(It, k) stands for binomial coefficient.
981const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
982 ScalarEvolution &SE) const {
983 return evaluateAtIteration(Operands: operands(), It, SE);
984}
985
986SCEVUse SCEVAddRecExpr::evaluateAtIteration(ArrayRef<SCEVUse> Operands,
987 const SCEV *It, ScalarEvolution &SE,
988 SCEV::NoWrapFlags UseFlags) {
989 assert(Operands.size() > 0);
990 assert((Operands.size() == 2 || UseFlags == SCEV::FlagAnyWrap) &&
991 "use-specific flags only supported for affine AddRecs");
992 SCEVUse Result = Operands[0].getPointer();
993 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
994 // The computation is correct in the face of overflow provided that the
995 // multiplication is performed _after_ the evaluation of the binomial
996 // coefficient.
997 const SCEV *Coeff = BinomialCoefficient(It, K: i, SE, ResultTy: Result->getType());
998 if (isa<SCEVCouldNotCompute>(Val: Coeff))
999 return Coeff;
1000
1001 const SCEV *Mul = SE.getMulExpr(LHS: Operands[i].getPointer(), RHS: Coeff);
1002 Result = SE.getAddExpr(LHS: Result, RHS: Mul, Flags: {SCEV::FlagAnyWrap, UseFlags});
1003 }
1004 return Result;
1005}
1006
1007SCEVUse SCEVAddRecExpr::getExitValue(ScalarEvolution &SE) const {
1008 const SCEV *BTC = SE.getBackedgeTakenCount(L: getLoop());
1009 if (isa<SCEVCouldNotCompute>(Val: BTC))
1010 return BTC;
1011 // The loop reaches iteration BTC, so the value this recurrence computes there
1012 // is the value it had, and that did not wrap.
1013 return evaluateAtIteration(Operands: operands(), It: BTC, SE,
1014 UseFlags: isAffine() ? getNoWrapFlags(Mask: SCEV::FlagNUW)
1015 : SCEV::FlagAnyWrap);
1016}
1017
1018//===----------------------------------------------------------------------===//
1019// SCEV Expression folder implementations
1020//===----------------------------------------------------------------------===//
1021
1022/// The SCEVCastSinkingRewriter takes a scalar evolution expression,
1023/// which computes a pointer-typed value, and rewrites the whole expression
1024/// tree so that *all* the computations are done on integers, and the only
1025/// pointer-typed operands in the expression are SCEVUnknown.
1026/// The CreatePtrCast callback is invoked to create the actual conversion
1027/// (ptrtoint or ptrtoaddr) at the SCEVUnknown leaves.
1028class SCEVCastSinkingRewriter
1029 : public SCEVRewriteVisitor<SCEVCastSinkingRewriter> {
1030 using Base = SCEVRewriteVisitor<SCEVCastSinkingRewriter>;
1031 using ConversionFn = function_ref<const SCEV *(const SCEVUnknown *)>;
1032 Type *TargetTy;
1033 ConversionFn CreatePtrCast;
1034
1035public:
1036 SCEVCastSinkingRewriter(ScalarEvolution &SE, Type *TargetTy,
1037 ConversionFn CreatePtrCast)
1038 : Base(SE), TargetTy(TargetTy), CreatePtrCast(std::move(CreatePtrCast)) {}
1039
1040 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE,
1041 Type *TargetTy, ConversionFn CreatePtrCast) {
1042 SCEVCastSinkingRewriter Rewriter(SE, TargetTy, std::move(CreatePtrCast));
1043 return Rewriter.visit(S: Scev);
1044 }
1045
1046 const SCEV *visit(const SCEV *S) {
1047 Type *STy = S->getType();
1048 // If the expression is not pointer-typed, just keep it as-is.
1049 if (!STy->isPointerTy())
1050 return S;
1051 // Else, recursively sink the cast down into it.
1052 return Base::visit(S);
1053 }
1054
1055 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
1056 // Preserve wrap flags on rewritten SCEVAddExpr, which the default
1057 // implementation drops.
1058 SmallVector<SCEVUse, 2> Operands;
1059 bool Changed = false;
1060 for (SCEVUse Op : Expr->operands()) {
1061 Operands.push_back(Elt: visit(S: Op.getPointer()));
1062 Changed |= Op.getPointer() != Operands.back();
1063 }
1064 return !Changed ? Expr : SE.getAddExpr(Ops&: Operands, Flags: Expr->getNoWrapFlags());
1065 }
1066
1067 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
1068 assert(Expr->getType()->isPointerTy() &&
1069 "Should only reach pointer-typed SCEVUnknown's.");
1070 // Perform some basic constant folding. If the operand of the cast is a
1071 // null pointer, don't create a cast SCEV expression (that will be left
1072 // as-is), but produce a zero constant.
1073 if (isa<ConstantPointerNull>(Val: Expr->getValue()))
1074 return SE.getZero(Ty: TargetTy);
1075 return CreatePtrCast(Expr);
1076 }
1077};
1078
1079const SCEV *ScalarEvolution::getPtrToAddrExpr(const SCEV *Op) {
1080 assert(Op->getType()->isPointerTy() && "Op must be a pointer");
1081
1082 // Treat pointers with unstable representation conservatively, since the
1083 // address bits may change.
1084 if (DL.hasUnstableRepresentation(Ty: Op->getType()))
1085 return getCouldNotCompute();
1086
1087 Type *Ty = DL.getAddressType(PtrTy: Op->getType());
1088
1089 // Use the rewriter to sink the cast down to SCEVUnknown leaves.
1090 // The rewriter handles null pointer constant folding.
1091 const SCEV *IntOp = SCEVCastSinkingRewriter::rewrite(
1092 Scev: Op, SE&: *this, TargetTy: Ty, CreatePtrCast: [this, Ty](const SCEVUnknown *U) {
1093 FoldingSetNodeID ID;
1094 ID.AddInteger(I: scPtrToAddr);
1095 ID.AddPointer(Ptr: U);
1096 ID.AddPointer(Ptr: Ty);
1097 FoldingSetInsertToken Token;
1098 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1099 return S;
1100 SCEV *S = new (SCEVAllocator)
1101 SCEVPtrToAddrExpr(ID.Intern(Allocator&: SCEVAllocator), U, Ty);
1102 UniqueSCEVs.insert(N: S, Token);
1103 S->computeAndSetCanonical(SE&: *this);
1104 registerUser(User: S, Ops: {U});
1105 return static_cast<const SCEV *>(S);
1106 });
1107 assert(IntOp->getType()->isIntegerTy() &&
1108 "We must have succeeded in sinking the cast, "
1109 "and ending up with an integer-typed expression!");
1110 return IntOp;
1111}
1112
1113const SCEV *ScalarEvolution::getTruncateExpr(SCEVUse Op, Type *Ty,
1114 unsigned Depth) {
1115 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
1116 "This is not a truncating conversion!");
1117 assert(isSCEVable(Ty) &&
1118 "This is not a conversion to a SCEVable type!");
1119 assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!");
1120 Ty = getEffectiveSCEVType(Ty);
1121
1122 FoldingSetNodeID ID;
1123 ID.AddInteger(I: scTruncate);
1124 ID.AddPointer(Ptr: Op.getOpaqueValue());
1125 ID.AddPointer(Ptr: Ty);
1126 FoldingSetInsertToken Token;
1127 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1128 return S;
1129
1130 // Fold if the operand is constant.
1131 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1132 return getConstant(
1133 V: cast<ConstantInt>(Val: ConstantExpr::getTrunc(C: SC->getValue(), Ty)));
1134
1135 // trunc(trunc(x)) --> trunc(x)
1136 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op))
1137 return getTruncateExpr(Op: ST->getOperand(), Ty, Depth: Depth + 1);
1138
1139 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
1140 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val&: Op))
1141 return getTruncateOrSignExtend(V: SS->getOperand(), Ty, Depth: Depth + 1);
1142
1143 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
1144 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1145 return getTruncateOrZeroExtend(V: SZ->getOperand(), Ty, Depth: Depth + 1);
1146
1147 if (Depth > MaxCastDepth) {
1148 SCEV *S =
1149 new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator), Op, Ty);
1150 UniqueSCEVs.insert(N: S, Token);
1151 S->computeAndSetCanonical(SE&: *this);
1152 registerUser(User: S, Ops: Op);
1153 return S;
1154 }
1155
1156 // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
1157 // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
1158 // if after transforming we have at most one truncate, not counting truncates
1159 // that replace other casts.
1160 if (isa<SCEVAddExpr>(Val: Op) || isa<SCEVMulExpr>(Val: Op)) {
1161 auto *CommOp = cast<SCEVCommutativeExpr>(Val&: Op);
1162 SmallVector<SCEVUse, 4> Operands;
1163 unsigned numTruncs = 0;
1164 for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1165 ++i) {
1166 const SCEV *S = getTruncateExpr(Op: CommOp->getOperand(i), Ty, Depth: Depth + 1);
1167 if (!isa<SCEVIntegralCastExpr>(Val: CommOp->getOperand(i)) &&
1168 isa<SCEVTruncateExpr>(Val: S))
1169 numTruncs++;
1170 Operands.push_back(Elt: S);
1171 }
1172 if (numTruncs < 2) {
1173 if (isa<SCEVAddExpr>(Val: Op))
1174 return getAddExpr(Ops&: Operands);
1175 if (isa<SCEVMulExpr>(Val: Op))
1176 return getMulExpr(Ops&: Operands);
1177 llvm_unreachable("Unexpected SCEV type for Op.");
1178 }
1179 // Although we checked in the beginning that ID is not in the cache, it is
1180 // possible that during recursion and different modification ID was inserted
1181 // into the cache. So if we find it, just return it.
1182 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1183 return S;
1184 }
1185
1186 // If the input value is a chrec scev, truncate the chrec's operands.
1187 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: Op)) {
1188 SmallVector<SCEVUse, 4> Operands;
1189 for (const SCEV *Op : AddRec->operands())
1190 Operands.push_back(Elt: getTruncateExpr(Op, Ty, Depth: Depth + 1));
1191 return getAddRecExpr(Operands, L: AddRec->getLoop(), Flags: SCEV::FlagAnyWrap);
1192 }
1193
1194 // Return zero if truncating to known zeros.
1195 uint32_t MinTrailingZeros = getMinTrailingZeros(S: Op);
1196 if (MinTrailingZeros >= getTypeSizeInBits(Ty))
1197 return getZero(Ty);
1198
1199 // The cast wasn't folded; create an explicit cast node. We can reuse
1200 // the existing insert position since if we get here, we won't have
1201 // made any changes which would invalidate it.
1202 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator),
1203 Op, Ty);
1204 UniqueSCEVs.insert(N: S, Token);
1205 S->computeAndSetCanonical(SE&: *this);
1206 registerUser(User: S, Ops: Op);
1207 return S;
1208}
1209
1210// Get the limit of a recurrence such that incrementing by Step cannot cause
1211// signed overflow as long as the value of the recurrence within the
1212// loop does not exceed this limit before incrementing.
1213static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
1214 ICmpInst::Predicate *Pred,
1215 ScalarEvolution *SE) {
1216 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1217 if (SE->isKnownPositive(S: Step)) {
1218 *Pred = ICmpInst::ICMP_SLT;
1219 return SE->getConstant(Val: APInt::getSignedMinValue(numBits: BitWidth) -
1220 SE->getSignedRangeMax(S: Step));
1221 }
1222 if (SE->isKnownNegative(S: Step)) {
1223 *Pred = ICmpInst::ICMP_SGT;
1224 return SE->getConstant(Val: APInt::getSignedMaxValue(numBits: BitWidth) -
1225 SE->getSignedRangeMin(S: Step));
1226 }
1227 return nullptr;
1228}
1229
1230// Get the limit of a recurrence such that incrementing by Step cannot cause
1231// unsigned overflow as long as the value of the recurrence within the loop does
1232// not exceed this limit before incrementing.
1233static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step,
1234 ICmpInst::Predicate *Pred,
1235 ScalarEvolution *SE) {
1236 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1237 *Pred = ICmpInst::ICMP_ULT;
1238
1239 return SE->getConstant(Val: APInt::getMinValue(numBits: BitWidth) -
1240 SE->getUnsignedRangeMax(S: Step));
1241}
1242
1243namespace {
1244
1245struct ExtendOpTraitsBase {
1246 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(SCEVUse, Type *,
1247 unsigned);
1248};
1249
1250// Used to make code generic over signed and unsigned overflow.
1251template <typename ExtendOp> struct ExtendOpTraits {
1252 // Members present:
1253 //
1254 // static const SCEV::NoWrapFlags WrapType;
1255 //
1256 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
1257 //
1258 // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1259 // ICmpInst::Predicate *Pred,
1260 // ScalarEvolution *SE);
1261};
1262
1263template <>
1264struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
1265 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW;
1266
1267 static const GetExtendExprTy GetExtendExpr;
1268
1269 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1270 ICmpInst::Predicate *Pred,
1271 ScalarEvolution *SE) {
1272 return getSignedOverflowLimitForStep(Step, Pred, SE);
1273 }
1274};
1275
1276const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1277 SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr;
1278
1279template <>
1280struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
1281 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW;
1282
1283 static const GetExtendExprTy GetExtendExpr;
1284
1285 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1286 ICmpInst::Predicate *Pred,
1287 ScalarEvolution *SE) {
1288 return getUnsignedOverflowLimitForStep(Step, Pred, SE);
1289 }
1290};
1291
1292const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1293 SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr;
1294
1295} // end anonymous namespace
1296
1297// The recurrence AR has been shown to have no signed/unsigned wrap or something
1298// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
1299// easily prove NSW/NUW for its preincrement or postincrement sibling. This
1300// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
1301// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
1302// expression "Step + sext/zext(PreIncAR)" is congruent with
1303// "sext/zext(PostIncAR)"
1304template <typename ExtendOpTy>
1305static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR,
1306 ScalarEvolution *SE, unsigned Depth) {
1307 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1308 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1309
1310 const Loop *L = AR->getLoop();
1311 const SCEV *Start = AR->getStart();
1312 const SCEV *Step = AR->getStepRecurrence(SE&: *SE);
1313
1314 // Check for a simple looking step prior to loop entry.
1315 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Val: Start);
1316 if (!SA)
1317 return nullptr;
1318
1319 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
1320 // subtraction is expensive. For this purpose, perform a quick and dirty
1321 // difference, by checking for Step in the operand list. Note, that
1322 // SA might have repeated ops, like %a + %a + ..., so only remove one.
1323 SmallVector<SCEVUse, 4> DiffOps(SA->operands());
1324 for (auto It = DiffOps.begin(); It != DiffOps.end(); ++It)
1325 if (*It == Step) {
1326 DiffOps.erase(CI: It);
1327 break;
1328 }
1329
1330 if (DiffOps.size() == SA->getNumOperands())
1331 return nullptr;
1332
1333 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
1334 // `Step`:
1335
1336 // 1. NSW/NUW flags on the step increment.
1337 auto PreStartFlags =
1338 ScalarEvolution::maskFlags(Flags: SA->getNoWrapFlags(), Mask: SCEV::FlagNUW);
1339 const SCEV *PreStart = SE->getAddExpr(Ops&: DiffOps, Flags: PreStartFlags);
1340 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
1341 Val: SE->getAddRecExpr(Start: PreStart, Step, L, Flags: SCEV::FlagAnyWrap));
1342
1343 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
1344 // "S+X does not sign/unsign-overflow".
1345 //
1346
1347 const SCEV *BECount = SE->getBackedgeTakenCount(L);
1348 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType)) &&
1349 !isa<SCEVCouldNotCompute>(Val: BECount) && SE->isKnownPositive(S: BECount))
1350 return PreStart;
1351
1352 // 2. Direct overflow check on the step operation's expression.
1353 unsigned BitWidth = SE->getTypeSizeInBits(Ty: AR->getType());
1354 Type *WideTy = IntegerType::get(C&: SE->getContext(), NumBits: BitWidth * 2);
1355 const SCEV *OperandExtendedStart =
1356 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
1357 (SE->*GetExtendExpr)(Step, WideTy, Depth));
1358 if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
1359 if (PreAR && any(AR->getNoWrapFlags(Mask: WrapType))) {
1360 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
1361 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
1362 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
1363 SE->setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(PreAR), Flags: WrapType);
1364 }
1365 return PreStart;
1366 }
1367
1368 // 3. Loop precondition.
1369 ICmpInst::Predicate Pred;
1370 const SCEV *OverflowLimit =
1371 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1372
1373 if (OverflowLimit &&
1374 SE->isLoopEntryGuardedByCond(L, Pred, LHS: PreStart, RHS: OverflowLimit))
1375 return PreStart;
1376
1377 return nullptr;
1378}
1379
1380// Get the normalized zero or sign extended expression for this AddRec's Start.
1381template <typename ExtendOpTy>
1382static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
1383 ScalarEvolution *SE,
1384 unsigned Depth) {
1385 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1386
1387 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, SE, Depth);
1388 if (!PreStart)
1389 return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
1390
1391 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(SE&: *SE), Ty,
1392 Depth),
1393 (SE->*GetExtendExpr)(PreStart, Ty, Depth));
1394}
1395
1396// Try to prove away overflow by looking at "nearby" add recurrences. A
1397// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
1398// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
1399//
1400// Formally:
1401//
1402// {S,+,X} == {S-T,+,X} + T
1403// => Ext({S,+,X}) == Ext({S-T,+,X} + T)
1404//
1405// If ({S-T,+,X} + T) does not overflow ... (1)
1406//
1407// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
1408//
1409// If {S-T,+,X} does not overflow ... (2)
1410//
1411// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
1412// == {Ext(S-T)+Ext(T),+,Ext(X)}
1413//
1414// If (S-T)+T does not overflow ... (3)
1415//
1416// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
1417// == {Ext(S),+,Ext(X)} == LHS
1418//
1419// Thus, if (1), (2) and (3) are true for some T, then
1420// Ext({S,+,X}) == {Ext(S),+,Ext(X)}
1421//
1422// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
1423// does not overflow" restricted to the 0th iteration. Therefore we only need
1424// to check for (1) and (2).
1425//
1426// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
1427// is `Delta` (defined below).
1428template <typename ExtendOpTy>
1429bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
1430 const SCEV *Step,
1431 const Loop *L) {
1432 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1433
1434 // We restrict `Start` to a constant to prevent SCEV from spending too much
1435 // time here. It is correct (but more expensive) to continue with a
1436 // non-constant `Start` and do a general SCEV subtraction to compute
1437 // `PreStart` below.
1438 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Val: Start);
1439 if (!StartC)
1440 return false;
1441
1442 APInt StartAI = StartC->getAPInt();
1443
1444 for (unsigned Delta : {-2, -1, 1, 2}) {
1445 const SCEV *PreStart = getConstant(Val: StartAI - Delta);
1446
1447 FoldingSetNodeID ID;
1448 ID.AddInteger(I: scAddRecExpr);
1449 ID.AddPointer(Ptr: PreStart);
1450 ID.AddPointer(Ptr: Step);
1451 ID.AddPointer(Ptr: L);
1452 FoldingSetInsertToken Token;
1453 const auto *PreAR =
1454 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.lookup(ID, Token));
1455
1456 // Give up if we don't already have the add recurrence we need because
1457 // actually constructing an add recurrence is relatively expensive.
1458 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType))) { // proves (2)
1459 const SCEV *DeltaS = getConstant(Ty: StartC->getType(), V: Delta);
1460 ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
1461 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1462 DeltaS, &Pred, this);
1463 if (Limit && isKnownPredicate(Pred, LHS: PreAR, RHS: Limit)) // proves (1)
1464 return true;
1465 }
1466 }
1467
1468 return false;
1469}
1470
1471// Finds an integer D for an expression (C + x + y + ...) such that the top
1472// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
1473// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
1474// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
1475// the (C + x + y + ...) expression is \p WholeAddExpr.
1476static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1477 const SCEVConstant *ConstantTerm,
1478 const SCEVAddExpr *WholeAddExpr) {
1479 const APInt &C = ConstantTerm->getAPInt();
1480 const unsigned BitWidth = C.getBitWidth();
1481 // Find number of trailing zeros of (x + y + ...) w/o the C first:
1482 uint32_t TZ = BitWidth;
1483 for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
1484 TZ = std::min(a: TZ, b: SE.getMinTrailingZeros(S: WholeAddExpr->getOperand(i: I)));
1485 if (TZ) {
1486 // Set D to be as many least significant bits of C as possible while still
1487 // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
1488 return TZ < BitWidth ? C.trunc(width: TZ).zext(width: BitWidth) : C;
1489 }
1490 return APInt(BitWidth, 0);
1491}
1492
1493// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
1494// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
1495// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
1496// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
1497static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1498 const APInt &ConstantStart,
1499 const SCEV *Step) {
1500 const unsigned BitWidth = ConstantStart.getBitWidth();
1501 const uint32_t TZ = SE.getMinTrailingZeros(S: Step);
1502 if (TZ)
1503 return TZ < BitWidth ? ConstantStart.trunc(width: TZ).zext(width: BitWidth)
1504 : ConstantStart;
1505 return APInt(BitWidth, 0);
1506}
1507
1508static void insertFoldCacheEntry(
1509 const ScalarEvolution::FoldID &ID, const SCEV *S,
1510 DenseMap<ScalarEvolution::FoldID, const SCEV *> &FoldCache,
1511 DenseMap<const SCEV *, SmallVector<ScalarEvolution::FoldID, 2>>
1512 &FoldCacheUser) {
1513 auto I = FoldCache.insert(KV: {ID, S});
1514 if (!I.second) {
1515 // Remove FoldCacheUser entry for ID when replacing an existing FoldCache
1516 // entry.
1517 auto &UserIDs = FoldCacheUser[I.first->second];
1518 assert(count(UserIDs, ID) == 1 && "unexpected duplicates in UserIDs");
1519 for (unsigned I = 0; I != UserIDs.size(); ++I)
1520 if (UserIDs[I] == ID) {
1521 std::swap(a&: UserIDs[I], b&: UserIDs.back());
1522 break;
1523 }
1524 UserIDs.pop_back();
1525 I.first->second = S;
1526 }
1527 FoldCacheUser[S].push_back(Elt: ID);
1528}
1529
1530const SCEV *ScalarEvolution::getZeroExtendExpr(SCEVUse Op, Type *Ty,
1531 unsigned Depth) {
1532 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1533 "This is not an extending conversion!");
1534 assert(isSCEVable(Ty) &&
1535 "This is not a conversion to a SCEVable type!");
1536 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1537 Ty = getEffectiveSCEVType(Ty);
1538
1539 FoldID ID(scZeroExtend, Op, Ty);
1540 if (const SCEV *S = FoldCache.lookup(Val: ID))
1541 return S;
1542
1543 const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
1544 if (!isa<SCEVZeroExtendExpr>(Val: S))
1545 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1546 return S;
1547}
1548
1549const SCEV *ScalarEvolution::getZeroExtendExprImpl(SCEVUse Op, Type *Ty,
1550 unsigned Depth) {
1551 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1552 "This is not an extending conversion!");
1553 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1554 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1555
1556 // Fold if the operand is constant.
1557 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1558 return getConstant(Val: SC->getAPInt().zext(width: getTypeSizeInBits(Ty)));
1559
1560 // zext(zext(x)) --> zext(x)
1561 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1562 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1563
1564 // If the operand is an affine AddRec with the no-unsigned-wrap flag, the
1565 // zero-extension distributes over the recurrence.
1566 const SCEV *Start, *Step;
1567 const Loop *L;
1568 if (Depth <= MaxCastDepth &&
1569 match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1570 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1571 if (AR->hasNoUnsignedWrap()) {
1572 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1573 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1574 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1575 }
1576 }
1577
1578 // Before doing any expensive analysis, check to see if we've already
1579 // computed a SCEV for this Op and Ty.
1580 FoldingSetNodeID ID;
1581 ID.AddInteger(I: scZeroExtend);
1582 ID.AddPointer(Ptr: Op.getOpaqueValue());
1583 ID.AddPointer(Ptr: Ty);
1584 FoldingSetInsertToken Token;
1585 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1586 return S;
1587 if (Depth > MaxCastDepth) {
1588 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1589 Op, Ty);
1590 UniqueSCEVs.insert(N: S, Token);
1591 S->computeAndSetCanonical(SE&: *this);
1592 registerUser(User: S, Ops: Op);
1593 return S;
1594 }
1595
1596 // zext(trunc(x)) --> zext(x) or x or trunc(x)
1597 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
1598 // It's possible the bits taken off by the truncate were all zero bits. If
1599 // so, we should be able to simplify this further.
1600 const SCEV *X = ST->getOperand();
1601 ConstantRange CR = getUnsignedRange(S: X);
1602 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1603 unsigned NewBits = getTypeSizeInBits(Ty);
1604 if (CR.truncate(BitWidth: TruncBits).zeroExtend(BitWidth: NewBits).contains(
1605 CR: CR.zextOrTrunc(BitWidth: NewBits)))
1606 return getTruncateOrZeroExtend(V: X, Ty, Depth);
1607 }
1608
1609 // If the input value is a chrec scev, and we can prove that the value
1610 // did not overflow the old, smaller, value, we can zero extend all of the
1611 // operands (often constants). This allows analysis of something like
1612 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
1613 if (match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1614 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1615 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1616
1617 // The no-unsigned-wrap case is handled before the uniquing lookup above.
1618
1619 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1620 // Note that this serves two purposes: It filters out loops that are
1621 // simply not analyzable, and it covers the case where this code is
1622 // being called from within backedge-taken count analysis, such that
1623 // attempting to ask for the backedge-taken count would likely result
1624 // in infinite recursion. In the later case, the analysis code will
1625 // cope with a conservative value, and it will take care to purge
1626 // that value once it has finished.
1627 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1628 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
1629 // Manually compute the final value for AR, checking for overflow.
1630
1631 // Check whether the backedge-taken count can be losslessly casted to
1632 // the addrec's type. The count is always unsigned.
1633 const SCEV *CastedMaxBECount =
1634 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
1635 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1636 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
1637 if (MaxBECount == RecastedMaxBECount) {
1638 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
1639 // Check whether Start+Step*MaxBECount has no unsigned overflow.
1640 const SCEV *ZMul =
1641 getMulExpr(LHS: CastedMaxBECount, RHS: Step, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1642 const SCEV *ZAdd = getZeroExtendExpr(
1643 Op: getAddExpr(LHS: Start, RHS: ZMul, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1), Ty: WideTy,
1644 Depth: Depth + 1);
1645 const SCEV *WideStart = getZeroExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
1646 const SCEV *WideMaxBECount =
1647 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
1648 const SCEV *OperandExtendedAdd =
1649 getAddExpr(LHS: WideStart,
1650 RHS: getMulExpr(LHS: WideMaxBECount,
1651 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1652 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
1653 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1654 if (ZAdd == OperandExtendedAdd) {
1655 // Cache knowledge of AR NUW, which is propagated to this AddRec.
1656 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1657 // Return the expression with the addrec on the outside.
1658 Start =
1659 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1660 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1661 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1662 }
1663 // Similar to above, only this time treat the step value as signed.
1664 // This covers loops that count down.
1665 OperandExtendedAdd =
1666 getAddExpr(LHS: WideStart,
1667 RHS: getMulExpr(LHS: WideMaxBECount,
1668 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1669 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
1670 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1671 if (ZAdd == OperandExtendedAdd) {
1672 // Cache knowledge of AR NW, which is propagated to this AddRec.
1673 // Negative step causes unsigned wrap, but it still can't self-wrap.
1674 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1675 // Return the expression with the addrec on the outside.
1676 Start =
1677 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1678 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1679 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1680 }
1681 }
1682 }
1683
1684 // Normally, in the cases we can prove no-overflow via a
1685 // backedge guarding condition, we can also compute a backedge
1686 // taken count for the loop. The exceptions are assumptions and
1687 // guards present in the loop -- SCEV is not great at exploiting
1688 // these to compute max backedge taken counts, but can still use
1689 // these to prove lack of overflow. Use this fact to avoid
1690 // doing extra work that may not pay off.
1691 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount) || HasGuards ||
1692 !AC.assumptions().empty()) {
1693
1694 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1695 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
1696 if (AR->hasNoUnsignedWrap()) {
1697 // Same as nuw case above - duplicated here to avoid a compile time
1698 // issue. It's not clear that the order of checks does matter, but
1699 // it's one of two issue possible causes for a change which was
1700 // reverted. Be conservative for the moment.
1701 Start =
1702 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1703 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1704 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1705 }
1706
1707 // For a negative step, we can extend the operands iff doing so only
1708 // traverses values in the range zext([0,UINT_MAX]).
1709 if (isKnownNegative(S: Step)) {
1710 const SCEV *N =
1711 getConstant(Val: APInt::getMaxValue(numBits: BitWidth) - getSignedRangeMin(S: Step));
1712 if (isLoopBackedgeGuardedByCond(L, Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N) ||
1713 isKnownOnEveryIteration(Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N)) {
1714 // Cache knowledge of AR NW, which is propagated to this
1715 // AddRec. Negative step causes unsigned wrap, but it
1716 // still can't self-wrap.
1717 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1718 // Return the expression with the addrec on the outside.
1719 Start =
1720 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1721 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1722 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1723 }
1724 }
1725 }
1726
1727 // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
1728 // if D + (C - D + Step * n) could be proven to not unsigned wrap
1729 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1730 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
1731 const APInt &C = SC->getAPInt();
1732 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
1733 if (D != 0) {
1734 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1735 const SCEV *SResidual =
1736 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
1737 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1738 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: SCEV::FlagNSW | SCEV::FlagNUW,
1739 Depth: Depth + 1);
1740 }
1741 }
1742
1743 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1744 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1745 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1746 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1747 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1748 }
1749 }
1750
1751 // zext(A % B) --> zext(A) % zext(B)
1752 {
1753 const SCEV *LHS;
1754 const SCEV *RHS;
1755 if (match(U: Op, P: m_scev_URem(LHS: m_SCEV(V&: LHS), RHS: m_SCEV(V&: RHS), SE&: *this)))
1756 return getURemExpr(LHS: getZeroExtendExpr(Op: LHS, Ty, Depth: Depth + 1),
1757 RHS: getZeroExtendExpr(Op: RHS, Ty, Depth: Depth + 1));
1758 }
1759
1760 // zext(A / B) --> zext(A) / zext(B).
1761 if (auto *Div = dyn_cast<SCEVUDivExpr>(Val&: Op))
1762 return getUDivExpr(LHS: getZeroExtendExpr(Op: Div->getLHS(), Ty, Depth: Depth + 1),
1763 RHS: getZeroExtendExpr(Op: Div->getRHS(), Ty, Depth: Depth + 1));
1764
1765 if (auto *SA = dyn_cast<SCEVAddExpr>(Val&: Op)) {
1766 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
1767 if (SA->hasNoUnsignedWrap()) {
1768 // If the addition does not unsign overflow then we can, by definition,
1769 // commute the zero extension with the addition operation.
1770 SmallVector<SCEVUse, 4> Ops;
1771 for (SCEVUse Op : SA->operands())
1772 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1773 return getAddExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1774 }
1775
1776 const APInt *C, *C2;
1777 // zext (C + A)<nsw> -> (sext(C) + sext(A))<nsw> if zext (C + A)<nsw> >=s 0.
1778 // Currently the non-negative check is done manually, as isKnownNonNegative
1779 // is too expensive.
1780 if (SA->hasNoSignedWrap() &&
1781 match(V: SA, P: m_scev_Add(Op0: m_scev_APInt(C),
1782 Op1: m_scev_SMax(Op0: m_scev_APInt(C&: C2), Op1: m_SCEV()))) &&
1783 C->isNegative() && !C->isMinSignedValue() && C2->sge(RHS: C->abs())) {
1784 assert(isKnownNonNegative(SA) && "incorrectly determined non-negative");
1785 return getAddExpr(LHS: getSignExtendExpr(Op: SA->getOperand(i: 0), Ty, Depth: Depth + 1),
1786 RHS: getSignExtendExpr(Op: SA->getOperand(i: 1), Ty, Depth: Depth + 1),
1787 Flags: SCEV::FlagNSW, Depth: Depth + 1);
1788 }
1789
1790 // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
1791 // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
1792 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1793 //
1794 // Often address arithmetics contain expressions like
1795 // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
1796 // This transformation is useful while proving that such expressions are
1797 // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
1798 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1799 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1800 if (D != 0) {
1801 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1802 const SCEV *SResidual =
1803 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagAnyWrap, Depth);
1804 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1805 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1806 Depth: Depth + 1);
1807 }
1808 }
1809 }
1810
1811 if (auto *SM = dyn_cast<SCEVMulExpr>(Val&: Op)) {
1812 // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
1813 if (SM->hasNoUnsignedWrap()) {
1814 // If the multiply does not unsign overflow then we can, by definition,
1815 // commute the zero extension with the multiply operation.
1816 SmallVector<SCEVUse, 4> Ops;
1817 for (SCEVUse Op : SM->operands())
1818 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1819 return getMulExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1820 }
1821
1822 // zext(2^K * (trunc X to iN)) to iM ->
1823 // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
1824 //
1825 // Proof:
1826 //
1827 // zext(2^K * (trunc X to iN)) to iM
1828 // = zext((trunc X to iN) << K) to iM
1829 // = zext((trunc X to i{N-K}) << K)<nuw> to iM
1830 // (because shl removes the top K bits)
1831 // = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
1832 // = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
1833 //
1834 const APInt *C;
1835 const SCEV *TruncRHS;
1836 if (match(V: SM,
1837 P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_scev_Trunc(Op0: m_SCEV(V&: TruncRHS)))) &&
1838 C->isPowerOf2()) {
1839 int NewTruncBits =
1840 getTypeSizeInBits(Ty: SM->getOperand(i: 1)->getType()) - C->logBase2();
1841 Type *NewTruncTy = IntegerType::get(C&: getContext(), NumBits: NewTruncBits);
1842 return getMulExpr(
1843 LHS: getZeroExtendExpr(Op: SM->getOperand(i: 0), Ty),
1844 RHS: getZeroExtendExpr(Op: getTruncateExpr(Op: TruncRHS, Ty: NewTruncTy), Ty),
1845 Flags: SCEV::FlagNUW, Depth: Depth + 1);
1846 }
1847 }
1848
1849 // zext(umin(x, y)) -> umin(zext(x), zext(y))
1850 // zext(umax(x, y)) -> umax(zext(x), zext(y))
1851 if (isa<SCEVUMinExpr>(Val: Op) || isa<SCEVUMaxExpr>(Val: Op)) {
1852 auto *MinMax = cast<SCEVMinMaxExpr>(Val&: Op);
1853 SmallVector<SCEVUse, 4> Operands;
1854 for (SCEVUse Operand : MinMax->operands())
1855 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1856 if (isa<SCEVUMinExpr>(Val: MinMax))
1857 return getUMinExpr(Operands);
1858 return getUMaxExpr(Operands);
1859 }
1860
1861 // zext(umin_seq(x, y)) -> umin_seq(zext(x), zext(y))
1862 if (auto *MinMax = dyn_cast<SCEVSequentialMinMaxExpr>(Val&: Op)) {
1863 assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
1864 SmallVector<SCEVUse, 4> Operands;
1865 for (SCEVUse Operand : MinMax->operands())
1866 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1867 return getUMinExpr(Operands, /*Sequential*/ true);
1868 }
1869
1870 // The cast wasn't folded; create an explicit cast node.
1871 // Recompute the insert position, as it may have been invalidated.
1872 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1873 return S;
1874 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1875 Op, Ty);
1876 UniqueSCEVs.insert(N: S, Token);
1877 S->computeAndSetCanonical(SE&: *this);
1878 registerUser(User: S, Ops: Op);
1879 return S;
1880}
1881
1882const SCEV *ScalarEvolution::getSignExtendExpr(SCEVUse Op, Type *Ty,
1883 unsigned Depth) {
1884 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1885 "This is not an extending conversion!");
1886 assert(isSCEVable(Ty) &&
1887 "This is not a conversion to a SCEVable type!");
1888 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1889 Ty = getEffectiveSCEVType(Ty);
1890
1891 FoldID ID(scSignExtend, Op, Ty);
1892 if (const SCEV *S = FoldCache.lookup(Val: ID))
1893 return S;
1894
1895 const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
1896 if (!isa<SCEVSignExtendExpr>(Val: S))
1897 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1898 return S;
1899}
1900
1901const SCEV *ScalarEvolution::getSignExtendExprImpl(SCEVUse Op, Type *Ty,
1902 unsigned Depth) {
1903 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1904 "This is not an extending conversion!");
1905 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1906 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1907 Ty = getEffectiveSCEVType(Ty);
1908
1909 // Fold if the operand is constant.
1910 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1911 return getConstant(Val: SC->getAPInt().sext(width: getTypeSizeInBits(Ty)));
1912
1913 // sext(sext(x)) --> sext(x)
1914 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val&: Op))
1915 return getSignExtendExpr(Op: SS->getOperand(), Ty, Depth: Depth + 1);
1916
1917 // sext(zext(x)) --> zext(x)
1918 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1919 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1920
1921 // If the operand is an affine AddRec with the no-signed-wrap flag, the
1922 // sign-extension distributes over the recurrence.
1923 const SCEV *Start, *Step;
1924 const Loop *L;
1925 if (Depth <= MaxCastDepth &&
1926 match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1927 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1928 if (AR->hasNoSignedWrap()) {
1929 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1930 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1931 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1932 }
1933 }
1934
1935 // Before doing any expensive analysis, check to see if we've already
1936 // computed a SCEV for this Op and Ty.
1937 FoldingSetNodeID ID;
1938 ID.AddInteger(I: scSignExtend);
1939 ID.AddPointer(Ptr: Op.getOpaqueValue());
1940 ID.AddPointer(Ptr: Ty);
1941 FoldingSetInsertToken Token;
1942 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1943 return S;
1944 // Limit recursion depth.
1945 if (Depth > MaxCastDepth) {
1946 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1947 Op, Ty);
1948 UniqueSCEVs.insert(N: S, Token);
1949 S->computeAndSetCanonical(SE&: *this);
1950 registerUser(User: S, Ops: Op);
1951 return S;
1952 }
1953
1954 // sext(trunc(x)) --> sext(x) or x or trunc(x)
1955 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
1956 // It's possible the bits taken off by the truncate were all sign bits. If
1957 // so, we should be able to simplify this further.
1958 const SCEV *X = ST->getOperand();
1959 ConstantRange CR = getSignedRange(S: X);
1960 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1961 unsigned NewBits = getTypeSizeInBits(Ty);
1962 if (CR.truncate(BitWidth: TruncBits).signExtend(BitWidth: NewBits).contains(
1963 CR: CR.sextOrTrunc(BitWidth: NewBits)))
1964 return getTruncateOrSignExtend(V: X, Ty, Depth);
1965 }
1966
1967 if (auto *SA = dyn_cast<SCEVAddExpr>(Val&: Op)) {
1968 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
1969 if (SA->hasNoSignedWrap()) {
1970 // If the addition does not sign overflow then we can, by definition,
1971 // commute the sign extension with the addition operation.
1972 SmallVector<SCEVUse, 4> Ops;
1973 for (SCEVUse Op : SA->operands())
1974 Ops.push_back(Elt: getSignExtendExpr(Op, Ty, Depth: Depth + 1));
1975 return getAddExpr(Ops, Flags: SCEV::FlagNSW, Depth: Depth + 1);
1976 }
1977
1978 // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
1979 // if D + (C - D + x + y + ...) could be proven to not signed wrap
1980 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1981 //
1982 // For instance, this will bring two seemingly different expressions:
1983 // 1 + sext(5 + 20 * %x + 24 * %y) and
1984 // sext(6 + 20 * %x + 24 * %y)
1985 // to the same form:
1986 // 2 + sext(4 + 20 * %x + 24 * %y)
1987 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1988 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1989 if (D != 0) {
1990 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1991 const SCEV *SResidual =
1992 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagAnyWrap, Depth);
1993 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1994 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1995 Depth: Depth + 1);
1996 }
1997 }
1998 }
1999 // If the input value is a chrec scev, and we can prove that the value
2000 // did not overflow the old, smaller, value, we can sign extend all of the
2001 // operands (often constants). This allows analysis of something like
2002 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
2003 if (match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
2004 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
2005 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
2006
2007 // The no-signed-wrap case is handled before the uniquing lookup above.
2008
2009 // Check whether the backedge-taken count is SCEVCouldNotCompute.
2010 // Note that this serves two purposes: It filters out loops that are
2011 // simply not analyzable, and it covers the case where this code is
2012 // being called from within backedge-taken count analysis, such that
2013 // attempting to ask for the backedge-taken count would likely result
2014 // in infinite recursion. In the later case, the analysis code will
2015 // cope with a conservative value, and it will take care to purge
2016 // that value once it has finished.
2017 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
2018 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
2019 // Manually compute the final value for AR, checking for
2020 // overflow.
2021
2022 // Check whether the backedge-taken count can be losslessly casted to
2023 // the addrec's type. The count is always unsigned.
2024 const SCEV *CastedMaxBECount =
2025 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
2026 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
2027 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
2028 if (MaxBECount == RecastedMaxBECount) {
2029 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
2030 // Check whether Start+Step*MaxBECount has no signed overflow.
2031 const SCEV *SMul =
2032 getMulExpr(LHS: CastedMaxBECount, RHS: Step, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2033 const SCEV *SAdd = getSignExtendExpr(
2034 Op: getAddExpr(LHS: Start, RHS: SMul, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1), Ty: WideTy,
2035 Depth: Depth + 1);
2036 const SCEV *WideStart = getSignExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
2037 const SCEV *WideMaxBECount =
2038 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
2039 const SCEV *OperandExtendedAdd =
2040 getAddExpr(LHS: WideStart,
2041 RHS: getMulExpr(LHS: WideMaxBECount,
2042 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2043 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2044 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2045 if (SAdd == OperandExtendedAdd) {
2046 // Cache knowledge of AR NSW, which is propagated to this AddRec.
2047 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2048 // Return the expression with the addrec on the outside.
2049 Start =
2050 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2051 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2052 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2053 }
2054 // Similar to above, only this time treat the step value as unsigned.
2055 // This covers loops that count up with an unsigned step.
2056 OperandExtendedAdd =
2057 getAddExpr(LHS: WideStart,
2058 RHS: getMulExpr(LHS: WideMaxBECount,
2059 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2060 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2061 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2062 if (SAdd == OperandExtendedAdd) {
2063 // If AR wraps around then
2064 //
2065 // abs(Step) * MaxBECount > unsigned-max(AR->getType())
2066 // => SAdd != OperandExtendedAdd
2067 //
2068 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
2069 // (SAdd == OperandExtendedAdd => AR is NW)
2070
2071 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
2072
2073 // Return the expression with the addrec on the outside.
2074 Start =
2075 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2076 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2077 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2078 }
2079 }
2080 }
2081
2082 auto NewFlags = proveNoSignedWrapViaInduction(AR);
2083 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
2084 if (AR->hasNoSignedWrap()) {
2085 // Same as nsw case above - duplicated here to avoid a compile time
2086 // issue. It's not clear that the order of checks does matter, but
2087 // it's one of two issue possible causes for a change which was
2088 // reverted. Be conservative for the moment.
2089 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2090 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2091 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2092 }
2093
2094 // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
2095 // if D + (C - D + Step * n) could be proven to not signed wrap
2096 // where D maximizes the number of trailing zeros of (C - D + Step * n)
2097 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
2098 const APInt &C = SC->getAPInt();
2099 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
2100 if (D != 0) {
2101 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
2102 const SCEV *SResidual =
2103 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
2104 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
2105 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
2106 Depth: Depth + 1);
2107 }
2108 }
2109
2110 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
2111 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2112 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2113 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2114 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2115 }
2116 }
2117
2118 // If the input value is provably positive and we could not simplify
2119 // away the sext build a zext instead.
2120 if (isKnownNonNegative(S: Op))
2121 return getZeroExtendExpr(Op, Ty, Depth: Depth + 1);
2122
2123 // sext(smin(x, y)) -> smin(sext(x), sext(y))
2124 // sext(smax(x, y)) -> smax(sext(x), sext(y))
2125 if (isa<SCEVSMinExpr>(Val: Op) || isa<SCEVSMaxExpr>(Val: Op)) {
2126 auto *MinMax = cast<SCEVMinMaxExpr>(Val&: Op);
2127 SmallVector<SCEVUse, 4> Operands;
2128 for (SCEVUse Operand : MinMax->operands())
2129 Operands.push_back(Elt: getSignExtendExpr(Op: Operand, Ty));
2130 if (isa<SCEVSMinExpr>(Val: MinMax))
2131 return getSMinExpr(Operands);
2132 return getSMaxExpr(Operands);
2133 }
2134
2135 // The cast wasn't folded; create an explicit cast node.
2136 // Recompute the insert position, as it may have been invalidated.
2137 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
2138 return S;
2139 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2140 Op, Ty);
2141 UniqueSCEVs.insert(N: S, Token);
2142 S->computeAndSetCanonical(SE&: *this);
2143 registerUser(User: S, Ops: Op);
2144 return S;
2145}
2146
2147const SCEV *ScalarEvolution::getCastExpr(SCEVTypes Kind, SCEVUse Op, Type *Ty) {
2148 switch (Kind) {
2149 case scTruncate:
2150 return getTruncateExpr(Op, Ty);
2151 case scZeroExtend:
2152 return getZeroExtendExpr(Op, Ty);
2153 case scSignExtend:
2154 return getSignExtendExpr(Op, Ty);
2155 case scPtrToAddr: {
2156 const SCEV *Expr = getPtrToAddrExpr(Op);
2157 assert(Expr->getType() == Ty && "requested type must match");
2158 return Expr;
2159 }
2160 default:
2161 llvm_unreachable("Not a SCEV cast expression!");
2162 }
2163}
2164
2165/// getAnyExtendExpr - Return a SCEV for the given operand extended with
2166/// unspecified bits out to the given type.
2167const SCEV *ScalarEvolution::getAnyExtendExpr(SCEVUse Op, Type *Ty) {
2168 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2169 "This is not an extending conversion!");
2170 assert(isSCEVable(Ty) &&
2171 "This is not a conversion to a SCEVable type!");
2172 Ty = getEffectiveSCEVType(Ty);
2173
2174 // Sign-extend negative constants.
2175 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
2176 if (SC->getAPInt().isNegative())
2177 return getSignExtendExpr(Op, Ty);
2178
2179 // Peel off a truncate cast.
2180 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
2181 const SCEV *NewOp = T->getOperand();
2182 if (getTypeSizeInBits(Ty: NewOp->getType()) < getTypeSizeInBits(Ty))
2183 return getAnyExtendExpr(Op: NewOp, Ty);
2184 return getTruncateOrNoop(V: NewOp, Ty);
2185 }
2186
2187 // Next try a zext cast. If the cast is folded, use it.
2188 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
2189 if (!isa<SCEVZeroExtendExpr>(Val: ZExt))
2190 return ZExt;
2191
2192 // Next try a sext cast. If the cast is folded, use it.
2193 const SCEV *SExt = getSignExtendExpr(Op, Ty);
2194 if (!isa<SCEVSignExtendExpr>(Val: SExt))
2195 return SExt;
2196
2197 // Force the cast to be folded into the operands of an addrec.
2198 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: Op)) {
2199 SmallVector<SCEVUse, 4> Ops;
2200 for (const SCEV *Op : AR->operands())
2201 Ops.push_back(Elt: getAnyExtendExpr(Op, Ty));
2202 return getAddRecExpr(Operands&: Ops, L: AR->getLoop(), Flags: SCEV::FlagNW);
2203 }
2204
2205 // If the expression is obviously signed, use the sext cast value.
2206 if (isa<SCEVSMaxExpr>(Val: Op))
2207 return SExt;
2208
2209 // Absent any other information, use the zext cast value.
2210 return ZExt;
2211}
2212
2213/// Process the given Ops list, which is a list of operands to be added under
2214/// the given scale, update the given map. This is a helper function for
2215/// getAddRecExpr. As an example of what it does, given a sequence of operands
2216/// that would form an add expression like this:
2217///
2218/// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
2219///
2220/// where A and B are constants, update the map with these values:
2221///
2222/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
2223///
2224/// and add 13 + A*B*29 to AccumulatedConstant.
2225/// This will allow getAddRecExpr to produce this:
2226///
2227/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
2228///
2229/// This form often exposes folding opportunities that are hidden in
2230/// the original operand list.
2231///
2232/// Return true iff it appears that any interesting folding opportunities
2233/// may be exposed. This helps getAddRecExpr short-circuit extra work in
2234/// the common case where no interesting opportunities are present, and
2235/// is also used as a check to avoid infinite recursion.
2236static bool CollectAddOperandsWithScales(SmallDenseMap<SCEVUse, APInt, 16> &M,
2237 SmallVectorImpl<SCEVUse> &NewOps,
2238 APInt &AccumulatedConstant,
2239 ArrayRef<SCEVUse> Ops,
2240 const APInt &Scale,
2241 ScalarEvolution &SE) {
2242 bool Interesting = false;
2243
2244 // Iterate over the add operands. They are sorted, with constants first.
2245 unsigned i = 0;
2246 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Ops[i])) {
2247 ++i;
2248 // Pull a buried constant out to the outside.
2249 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
2250 Interesting = true;
2251 AccumulatedConstant += Scale * C->getAPInt();
2252 }
2253
2254 // Next comes everything else. We're especially interested in multiplies
2255 // here, but they're in the middle, so just visit the rest with one loop.
2256 for (; i != Ops.size(); ++i) {
2257 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: Ops[i]);
2258 if (Mul && isa<SCEVConstant>(Val: Mul->getOperand(i: 0))) {
2259 APInt NewScale =
2260 Scale * cast<SCEVConstant>(Val: Mul->getOperand(i: 0))->getAPInt();
2261 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Val: Mul->getOperand(i: 1))) {
2262 // A multiplication of a constant with another add; recurse.
2263 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: Mul->getOperand(i: 1));
2264 Interesting |= CollectAddOperandsWithScales(
2265 M, NewOps, AccumulatedConstant, Ops: Add->operands(), Scale: NewScale, SE);
2266 } else {
2267 // A multiplication of a constant with some other value. Update
2268 // the map.
2269 SmallVector<SCEVUse, 4> MulOps(drop_begin(RangeOrContainer: Mul->operands()));
2270 const SCEV *Key = SE.getMulExpr(Ops&: MulOps);
2271 auto Pair = M.insert(KV: {Key, NewScale});
2272 if (Pair.second) {
2273 NewOps.push_back(Elt: Pair.first->first);
2274 } else {
2275 Pair.first->second += NewScale;
2276 // The map already had an entry for this value, which may indicate
2277 // a folding opportunity.
2278 Interesting = true;
2279 }
2280 }
2281 } else {
2282 // An ordinary operand. Update the map.
2283 auto Pair = M.insert(KV: {Ops[i], Scale});
2284 if (Pair.second) {
2285 NewOps.push_back(Elt: Pair.first->first);
2286 } else {
2287 Pair.first->second += Scale;
2288 // The map already had an entry for this value, which may indicate
2289 // a folding opportunity.
2290 Interesting = true;
2291 }
2292 }
2293 }
2294
2295 return Interesting;
2296}
2297
2298bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed,
2299 const SCEV *LHS, const SCEV *RHS,
2300 const Instruction *CtxI) {
2301 auto Operation = [this, BinOp](SCEVUse L, SCEVUse R) -> const SCEV * {
2302 switch (BinOp) {
2303 default:
2304 llvm_unreachable("Unsupported binary op");
2305 case Instruction::Add:
2306 return getAddExpr(LHS: L, RHS: R);
2307 case Instruction::Sub:
2308 return getMinusSCEV(LHS: L, RHS: R);
2309 case Instruction::Mul:
2310 return getMulExpr(LHS: L, RHS: R);
2311 }
2312 };
2313
2314 const SCEV *(ScalarEvolution::*Extension)(SCEVUse, Type *, unsigned) =
2315 Signed ? &ScalarEvolution::getSignExtendExpr
2316 : &ScalarEvolution::getZeroExtendExpr;
2317
2318 // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
2319 auto *NarrowTy = cast<IntegerType>(Val: LHS->getType());
2320 auto *WideTy =
2321 IntegerType::get(C&: NarrowTy->getContext(), NumBits: NarrowTy->getBitWidth() * 2);
2322
2323 const SCEV *A = (this->*Extension)(Operation(LHS, RHS), WideTy, 0);
2324 const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
2325 const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
2326 const SCEV *B = Operation(LHSB, RHSB);
2327 if (A == B)
2328 return true;
2329 // Can we use context to prove the fact we need?
2330 if (!CtxI)
2331 return false;
2332 // TODO: Support mul.
2333 if (BinOp == Instruction::Mul)
2334 return false;
2335 auto *RHSC = dyn_cast<SCEVConstant>(Val: RHS);
2336 // TODO: Lift this limitation.
2337 if (!RHSC)
2338 return false;
2339 APInt C = RHSC->getAPInt();
2340 unsigned NumBits = C.getBitWidth();
2341 bool IsSub = (BinOp == Instruction::Sub);
2342 bool IsNegativeConst = (Signed && C.isNegative());
2343 // Compute the direction and magnitude by which we need to check overflow.
2344 bool OverflowDown = IsSub ^ IsNegativeConst;
2345 APInt Magnitude = C;
2346 if (IsNegativeConst) {
2347 if (C == APInt::getSignedMinValue(numBits: NumBits))
2348 // TODO: SINT_MIN on inversion gives the same negative value, we don't
2349 // want to deal with that.
2350 return false;
2351 Magnitude = -C;
2352 }
2353
2354 ICmpInst::Predicate Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2355 if (OverflowDown) {
2356 // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
2357 APInt Min = Signed ? APInt::getSignedMinValue(numBits: NumBits)
2358 : APInt::getMinValue(numBits: NumBits);
2359 APInt Limit = Min + Magnitude;
2360 return isKnownPredicateAt(Pred, LHS: getConstant(Val: Limit), RHS: LHS, CtxI);
2361 } else {
2362 // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
2363 APInt Max = Signed ? APInt::getSignedMaxValue(numBits: NumBits)
2364 : APInt::getMaxValue(numBits: NumBits);
2365 APInt Limit = Max - Magnitude;
2366 return isKnownPredicateAt(Pred, LHS, RHS: getConstant(Val: Limit), CtxI);
2367 }
2368}
2369
2370std::optional<SCEV::NoWrapFlags>
2371ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp(
2372 const OverflowingBinaryOperator *OBO) {
2373 // It cannot be done any better.
2374 if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
2375 return std::nullopt;
2376
2377 SCEV::NoWrapFlags Flags = SCEV::NoWrapFlags::FlagAnyWrap;
2378
2379 if (OBO->hasNoUnsignedWrap())
2380 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2381 if (OBO->hasNoSignedWrap())
2382 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2383
2384 bool Deduced = false;
2385
2386 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)OBO->getOpcode();
2387 const SCEV *LHS = getSCEV(V: OBO->getOperand(i_nocapture: 0));
2388 const SCEV *RHS = getSCEV(V: OBO->getOperand(i_nocapture: 1));
2389
2390 bool CanUseNSW = true;
2391 const APInt *ShiftAmt;
2392 // Treat `shl %a, C` as `mul %a, 1 << C`.
2393 if (match(V: OBO, P: m_Shl(L: m_Value(), R: m_APInt(Res&: ShiftAmt)))) {
2394 unsigned BitWidth = ShiftAmt->getBitWidth();
2395 if (ShiftAmt->uge(RHS: BitWidth))
2396 return std::nullopt;
2397 // NSW only transfers if the shift amount is < BitWidth - 1, as INT_MIN * -1
2398 // overflows.
2399 CanUseNSW = ShiftAmt->ult(RHS: BitWidth - 1);
2400 Opcode = Instruction::Mul;
2401 RHS = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: ShiftAmt->getZExtValue()));
2402 } else if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
2403 Opcode != Instruction::Mul) {
2404 return std::nullopt;
2405 }
2406
2407 const Instruction *CtxI =
2408 UseContextForNoWrapFlagInference ? dyn_cast<Instruction>(Val: OBO) : nullptr;
2409 if (!OBO->hasNoUnsignedWrap() &&
2410 willNotOverflow(BinOp: Opcode, /* Signed */ false, LHS, RHS, CtxI)) {
2411 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2412 Deduced = true;
2413 }
2414
2415 if (CanUseNSW && !OBO->hasNoSignedWrap() &&
2416 willNotOverflow(BinOp: Opcode, /* Signed */ true, LHS, RHS, CtxI)) {
2417 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2418 Deduced = true;
2419 }
2420
2421 if (Deduced)
2422 return Flags;
2423 return std::nullopt;
2424}
2425
2426// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
2427// `OldFlags' as can't-wrap behavior. Infer a more aggressive set of
2428// can't-overflow flags for the operation if possible.
2429static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE,
2430 SCEVTypes Type,
2431 ArrayRef<SCEVUse> Ops,
2432 SCEV::NoWrapFlags Flags) {
2433 using namespace std::placeholders;
2434
2435 using OBO = OverflowingBinaryOperator;
2436
2437 bool CanAnalyze =
2438 Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr;
2439 (void)CanAnalyze;
2440 assert(CanAnalyze && "don't call from other places!");
2441
2442 SCEV::NoWrapFlags SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
2443 SCEV::NoWrapFlags SignOrUnsignWrap =
2444 ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2445
2446 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
2447 auto IsKnownNonNegative = [&](SCEVUse U) {
2448 return SE->isKnownNonNegative(S: U);
2449 };
2450
2451 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Range&: Ops, P: IsKnownNonNegative))
2452 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SignOrUnsignMask);
2453
2454 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2455
2456 if (SignOrUnsignWrap != SignOrUnsignMask &&
2457 (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
2458 isa<SCEVConstant>(Val: Ops[0])) {
2459
2460 auto Opcode = [&] {
2461 switch (Type) {
2462 case scAddExpr:
2463 return Instruction::Add;
2464 case scMulExpr:
2465 return Instruction::Mul;
2466 default:
2467 llvm_unreachable("Unexpected SCEV op.");
2468 }
2469 }();
2470
2471 const APInt &C = cast<SCEVConstant>(Val: Ops[0])->getAPInt();
2472
2473 // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
2474 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
2475 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2476 BinOp: Opcode, Other: C, NoWrapKind: OBO::NoSignedWrap);
2477 if (NSWRegion.contains(CR: SE->getSignedRange(S: Ops[1])))
2478 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2479 }
2480
2481 // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
2482 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
2483 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2484 BinOp: Opcode, Other: C, NoWrapKind: OBO::NoUnsignedWrap);
2485 if (NUWRegion.contains(CR: SE->getUnsignedRange(S: Ops[1])))
2486 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2487 }
2488 }
2489
2490 // <0,+,nonnegative><nw> is also nuw
2491 // TODO: Add corresponding nsw case
2492 if (Type == scAddRecExpr && ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNW) &&
2493 !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) && Ops.size() == 2 &&
2494 Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
2495 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2496
2497 // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
2498 if (Type == scMulExpr && !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) &&
2499 Ops.size() == 2) {
2500 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[0]))
2501 if (UDiv->getOperand(i: 1) == Ops[1])
2502 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2503 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[1]))
2504 if (UDiv->getOperand(i: 1) == Ops[0])
2505 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2506 }
2507
2508 return Flags;
2509}
2510
2511bool ScalarEvolution::isAvailableAtLoopEntry(const SCEV *S, const Loop *L) {
2512 return isLoopInvariant(S, L) && properlyDominates(S, BB: L->getHeader());
2513}
2514
2515/// Get a canonical add expression, or something simpler if possible.
2516SCEVUse ScalarEvolution::getAddExpr(SmallVectorImpl<SCEVUse> &Ops,
2517 SCEVFlags Flags, unsigned Depth) {
2518 SCEV::NoWrapFlags OrigFlags = Flags.ExprFlags;
2519 SCEV::NoWrapFlags UseFlags = Flags.UseFlags;
2520 assert(!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2521 "only nuw or nsw allowed");
2522 assert(!(UseFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2523 "only nuw or nsw allowed");
2524 assert(!Ops.empty() && "Cannot get empty add!");
2525 if (Ops.size() == 1) return Ops[0];
2526#ifndef NDEBUG
2527 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
2528 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
2529 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
2530 "SCEVAddExpr operand types don't match!");
2531 unsigned NumPtrs = count_if(
2532 Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
2533 assert(NumPtrs <= 1 && "add has at most one pointer operand");
2534#endif
2535
2536 const SCEV *Folded = constantFoldAndGroupOps(
2537 SE&: *this, LI, DT, Ops,
2538 Fold: [](const APInt &C1, const APInt &C2) { return C1 + C2; },
2539 IsIdentity: [](const APInt &C) { return C.isZero(); }, // identity
2540 IsAbsorber: [](const APInt &C) { return false; }); // absorber
2541 if (Folded)
2542 return Folded;
2543
2544#ifndef NDEBUG
2545 // Keep track of operands after constant folding, for verification when adding
2546 // use-specific flags.
2547 const SmallVector<SCEVUse, 8> OrigOps(Ops.begin(), Ops.end());
2548#endif
2549
2550 unsigned Idx = isa<SCEVConstant>(Val: Ops[0]) ? 1 : 0;
2551
2552 // Delay expensive flag strengthening until necessary.
2553 auto ComputeFlags = [this, OrigFlags](ArrayRef<SCEVUse> Ops) {
2554 return StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops, Flags: OrigFlags);
2555 };
2556
2557 // Limit recursion calls depth.
2558 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
2559 return {getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
2560
2561 if (SCEV *S = findExistingSCEVInCache(SCEVType: scAddExpr, Ops)) {
2562 // Don't strengthen flags if we have no new information.
2563 SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
2564 if (Add->getNoWrapFlags(Mask: OrigFlags) != OrigFlags)
2565 Add->setNoWrapFlags(ComputeFlags(Ops));
2566 return {S, UseFlags};
2567 }
2568
2569 // Okay, check to see if the same value occurs in the operand list more than
2570 // once. If so, merge them together into an multiply expression. Since we
2571 // sorted the list, these values are required to be adjacent.
2572 Type *Ty = Ops[0]->getType();
2573 bool FoundMatch = false;
2574 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
2575 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2
2576 // Scan ahead to count how many equal operands there are.
2577 unsigned Count = 2;
2578 while (i+Count != e && Ops[i+Count] == Ops[i])
2579 ++Count;
2580 // Merge the values into a multiply.
2581 SCEVUse Scale = getConstant(Ty, V: Count);
2582 const SCEV *Mul = getMulExpr(LHS: Scale, RHS: Ops[i], Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2583 if (Ops.size() == Count)
2584 return Mul;
2585 Ops[i] = Mul;
2586 Ops.erase(CS: Ops.begin()+i+1, CE: Ops.begin()+i+Count);
2587 --i; e -= Count - 1;
2588 FoundMatch = true;
2589 }
2590 if (FoundMatch)
2591 return getAddExpr(Ops, Flags: OrigFlags, Depth: Depth + 1);
2592
2593 // Check for truncates. If all the operands are truncated from the same
2594 // type, see if factoring out the truncate would permit the result to be
2595 // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
2596 // if the contents of the resulting outer trunc fold to something simple.
2597 auto FindTruncSrcType = [&]() -> Type * {
2598 // We're ultimately looking to fold an addrec of truncs and muls of only
2599 // constants and truncs, so if we find any other types of SCEV
2600 // as operands of the addrec then we bail and return nullptr here.
2601 // Otherwise, we return the type of the operand of a trunc that we find.
2602 if (auto *T = dyn_cast<SCEVTruncateExpr>(Val&: Ops[Idx]))
2603 return T->getOperand()->getType();
2604 if (const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
2605 SCEVUse LastOp = Mul->getOperand(i: Mul->getNumOperands() - 1);
2606 if (const auto *T = dyn_cast<SCEVTruncateExpr>(Val&: LastOp))
2607 return T->getOperand()->getType();
2608 }
2609 return nullptr;
2610 };
2611 if (auto *SrcType = FindTruncSrcType()) {
2612 SmallVector<SCEVUse, 8> LargeOps;
2613 bool Ok = true;
2614 // Check all the operands to see if they can be represented in the
2615 // source type of the truncate.
2616 for (const SCEV *Op : Ops) {
2617 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2618 if (T->getOperand()->getType() != SrcType) {
2619 Ok = false;
2620 break;
2621 }
2622 LargeOps.push_back(Elt: T->getOperand());
2623 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Op)) {
2624 LargeOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2625 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val: Op)) {
2626 SmallVector<SCEVUse, 8> LargeMulOps;
2627 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2628 if (const SCEVTruncateExpr *T =
2629 dyn_cast<SCEVTruncateExpr>(Val: M->getOperand(i: j))) {
2630 if (T->getOperand()->getType() != SrcType) {
2631 Ok = false;
2632 break;
2633 }
2634 LargeMulOps.push_back(Elt: T->getOperand());
2635 } else if (const auto *C = dyn_cast<SCEVConstant>(Val: M->getOperand(i: j))) {
2636 LargeMulOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2637 } else {
2638 Ok = false;
2639 break;
2640 }
2641 }
2642 if (Ok)
2643 LargeOps.push_back(Elt: getMulExpr(Ops&: LargeMulOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2644 } else {
2645 Ok = false;
2646 break;
2647 }
2648 }
2649 if (Ok) {
2650 // Evaluate the expression in the larger type.
2651 const SCEV *Fold = getAddExpr(Ops&: LargeOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2652 // If it folds to something simple, use it. Otherwise, don't.
2653 if (isa<SCEVConstant>(Val: Fold) || isa<SCEVUnknown>(Val: Fold))
2654 return getTruncateExpr(Op: Fold, Ty);
2655 }
2656 }
2657
2658 if (Ops.size() == 2) {
2659 // Check if we have an expression of the form ((X + C1) - C2), where C1 and
2660 // C2 can be folded in a way that allows retaining wrapping flags of (X +
2661 // C1).
2662 const SCEV *A = Ops[0];
2663 const SCEV *B = Ops[1];
2664 auto *AddExpr = dyn_cast<SCEVAddExpr>(Val: B);
2665 auto *C = dyn_cast<SCEVConstant>(Val: A);
2666 if (AddExpr && C && isa<SCEVConstant>(Val: AddExpr->getOperand(i: 0))) {
2667 auto C1 = cast<SCEVConstant>(Val: AddExpr->getOperand(i: 0))->getAPInt();
2668 auto C2 = C->getAPInt();
2669 SCEV::NoWrapFlags PreservedFlags = SCEV::FlagAnyWrap;
2670
2671 APInt ConstAdd = C1 + C2;
2672 auto AddFlags = AddExpr->getNoWrapFlags();
2673 // Adding a smaller constant is NUW if the original AddExpr was NUW.
2674 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNUW) &&
2675 ConstAdd.ule(RHS: C1)) {
2676 PreservedFlags =
2677 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNUW);
2678 }
2679
2680 // Adding a constant with the same sign and small magnitude is NSW, if the
2681 // original AddExpr was NSW.
2682 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNSW) &&
2683 C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
2684 ConstAdd.abs().ule(RHS: C1.abs())) {
2685 PreservedFlags =
2686 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNSW);
2687 }
2688
2689 if (PreservedFlags != SCEV::FlagAnyWrap) {
2690 SmallVector<SCEVUse, 4> NewOps(AddExpr->operands());
2691 NewOps[0] = getConstant(Val: ConstAdd);
2692 return getAddExpr(Ops&: NewOps, Flags: PreservedFlags);
2693 }
2694 }
2695
2696 // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext
2697 // (B), if trunc (A) + -A + B does not unsigned-wrap.
2698 const SCEVAddExpr *InnerAdd;
2699 if (match(S: B, P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
2700 const SCEV *NarrowA = getTruncateExpr(Op: A, Ty: InnerAdd->getType());
2701 if (NarrowA == getNegativeSCEV(V: InnerAdd->getOperand(i: 0)) &&
2702 getZeroExtendExpr(Op: NarrowA, Ty: B->getType()) == A &&
2703 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops: {NarrowA, InnerAdd},
2704 Flags: SCEV::FlagAnyWrap),
2705 TestFlags: SCEV::FlagNUW)) {
2706 return getZeroExtendExpr(Op: getAddExpr(LHS: NarrowA, RHS: InnerAdd), Ty: B->getType());
2707 }
2708 }
2709 }
2710
2711 // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
2712 const SCEV *Y;
2713 if (Ops.size() == 2 &&
2714 match(U: Ops[0],
2715 P: m_scev_Mul(Op0: m_scev_AllOnes(),
2716 Op1: m_scev_URem(LHS: m_scev_Specific(S: Ops[1]), RHS: m_SCEV(V&: Y), SE&: *this))))
2717 return getMulExpr(LHS: Y, RHS: getUDivExpr(LHS: Ops[1], RHS: Y));
2718
2719 // Skip past any other cast SCEVs.
2720 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
2721 ++Idx;
2722
2723 // If there are add operands they would be next.
2724 if (Idx < Ops.size()) {
2725 bool DeletedAdd = false;
2726 // If the original flags and all inlined SCEVAddExprs are NUW, use the
2727 // common NUW flag for expression after inlining. Other flags cannot be
2728 // preserved, because they may depend on the original order of operations.
2729 SCEV::NoWrapFlags CommonFlags = maskFlags(Flags: OrigFlags, Mask: SCEV::FlagNUW);
2730 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[Idx])) {
2731 if (Ops.size() > AddOpsInlineThreshold ||
2732 Add->getNumOperands() > AddOpsInlineThreshold)
2733 break;
2734 // If we have an add, expand the add operands onto the end of the operands
2735 // list.
2736 Ops.erase(CI: Ops.begin()+Idx);
2737 append_range(C&: Ops, R: Add->operands());
2738 DeletedAdd = true;
2739 CommonFlags = maskFlags(Flags: CommonFlags, Mask: Add->getNoWrapFlags());
2740 }
2741
2742 // If we deleted at least one add, we added operands to the end of the list,
2743 // and they are not necessarily sorted. Recurse to resort and resimplify
2744 // any operands we just acquired.
2745 if (DeletedAdd)
2746 return getAddExpr(Ops, Flags: CommonFlags, Depth: Depth + 1);
2747 }
2748
2749 // Skip over the add expression until we get to a multiply.
2750 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
2751 ++Idx;
2752
2753 // Check to see if there are any folding opportunities present with
2754 // operands multiplied by constant values.
2755 if (Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx])) {
2756 uint64_t BitWidth = getTypeSizeInBits(Ty);
2757 SmallDenseMap<SCEVUse, APInt, 16> M;
2758 SmallVector<SCEVUse, 8> NewOps;
2759 APInt AccumulatedConstant(BitWidth, 0);
2760 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2761 Ops, Scale: APInt(BitWidth, 1), SE&: *this)) {
2762 struct APIntCompare {
2763 bool operator()(const APInt &LHS, const APInt &RHS) const {
2764 return LHS.ult(RHS);
2765 }
2766 };
2767
2768 // Some interesting folding opportunity is present, so its worthwhile to
2769 // re-generate the operands list. Group the operands by constant scale,
2770 // to avoid multiplying by the same constant scale multiple times.
2771 std::map<APInt, SmallVector<SCEVUse, 4>, APIntCompare> MulOpLists;
2772 for (const SCEV *NewOp : NewOps)
2773 MulOpLists[M.find(Val: NewOp)->second].push_back(Elt: NewOp);
2774 // Re-generate the operands list.
2775 Ops.clear();
2776 if (AccumulatedConstant != 0)
2777 Ops.push_back(Elt: getConstant(Val: AccumulatedConstant));
2778 for (auto &MulOp : MulOpLists) {
2779 if (MulOp.first == 1) {
2780 Ops.push_back(Elt: getAddExpr(Ops&: MulOp.second, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2781 } else if (MulOp.first != 0) {
2782 Ops.push_back(Elt: getMulExpr(
2783 LHS: getConstant(Val: MulOp.first),
2784 RHS: getAddExpr(Ops&: MulOp.second, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2785 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2786 }
2787 }
2788 if (Ops.empty())
2789 return getZero(Ty);
2790 if (Ops.size() == 1)
2791 return Ops[0];
2792 return getAddExpr(Ops, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2793 }
2794 }
2795
2796 // Given a SCEVMulExpr and an operand index, return the product of all
2797 // operands except the one at OpIdx.
2798 auto StripFactor = [&](const SCEVMulExpr *M, unsigned OpIdx) -> SCEVUse {
2799 if (M->getNumOperands() == 2)
2800 return M->getOperand(i: OpIdx == 0);
2801 SmallVector<SCEVUse, 4> Remaining(M->operands().take_front(N: OpIdx));
2802 append_range(C&: Remaining, R: M->operands().drop_front(N: OpIdx + 1));
2803 return getMulExpr(Ops&: Remaining, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2804 };
2805
2806 // If we are adding something to a multiply expression, make sure the
2807 // something is not already an operand of the multiply. If so, merge it into
2808 // the multiply.
2809 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx]); ++Idx) {
2810 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val&: Ops[Idx]);
2811 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
2812 // Scan all terms to find every occurrence of common factor MulOpSCEV
2813 // and fold them in one shot:
2814 // A1*X + A2*X + ... + An*X --> X * (A1 + A2 + ... + An)
2815 const SCEV *MulOpSCEV = Mul->getOperand(i: MulOp);
2816 if (isa<SCEVConstant>(Val: MulOpSCEV))
2817 continue;
2818
2819 // Cofactors: 1 for bare addends matching MulOpSCEV, or the
2820 // remaining product for multiply terms containing MulOpSCEV.
2821 SmallVector<SCEVUse, 4> Cofactors;
2822 SmallVector<unsigned, 4> DeadIndices;
2823 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) {
2824 if (MulOpSCEV == Ops[AddOp]) {
2825 // W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
2826 Cofactors.push_back(Elt: getOne(Ty));
2827 DeadIndices.push_back(Elt: AddOp);
2828 continue;
2829 }
2830
2831 if (AddOp <= Idx || !isa<SCEVMulExpr>(Val: Ops[AddOp]))
2832 continue;
2833
2834 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Val&: Ops[AddOp]);
2835 for (unsigned OMulOp = 0, OE = OtherMul->getNumOperands(); OMulOp != OE;
2836 ++OMulOp) {
2837 if (OtherMul->getOperand(i: OMulOp) == MulOpSCEV) {
2838 // (A*B*C) + (A*D*E) --> A * (B*C + D*E)
2839 Cofactors.push_back(Elt: StripFactor(OtherMul, OMulOp));
2840 DeadIndices.push_back(Elt: AddOp);
2841 break;
2842 }
2843 }
2844 }
2845
2846 // Fold all collected cofactors with the anchor multiply's cofactor:
2847 // MulOpSCEV * (Cofactor_1 + ... + Cofactor_n + AnchorCofactor)
2848 if (!Cofactors.empty()) {
2849 Cofactors.push_back(Elt: StripFactor(Mul, MulOp));
2850
2851 SCEVUse InnerSum = getAddExpr(Ops&: Cofactors, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2852 SCEVUse OuterMul =
2853 getMulExpr(LHS: MulOpSCEV, RHS: InnerSum, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2854
2855 // DeadIndices does not include Idx (the anchor), hence +1.
2856 if (Ops.size() == DeadIndices.size() + 1)
2857 return OuterMul;
2858
2859 // Erase Ops[Idx] first, then erase DeadIndices in reverse order.
2860 // The -1 adjustment accounts for the shift from removing Idx;
2861 // reverse order means each erasure only shifts later positions,
2862 // which have already been processed.
2863 Ops.erase(CI: Ops.begin() + Idx);
2864 for (unsigned Dead : reverse(C&: DeadIndices))
2865 Ops.erase(CI: Ops.begin() + (Dead > Idx ? Dead - 1 : Dead));
2866
2867 Ops.push_back(Elt: OuterMul);
2868 return getAddExpr(Ops, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2869 }
2870 }
2871 }
2872
2873 // If there are any add recurrences in the operands list, see if any other
2874 // added values are loop invariant. If so, we can fold them into the
2875 // recurrence.
2876 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
2877 ++Idx;
2878
2879 // Scan over all recurrences, trying to fold loop invariants into them.
2880 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
2881 // Scan all of the other operands to this add and add them to the vector if
2882 // they are loop invariant w.r.t. the recurrence.
2883 SmallVector<SCEVUse, 8> LIOps;
2884 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
2885 const Loop *AddRecLoop = AddRec->getLoop();
2886 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2887 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRecLoop)) {
2888 LIOps.push_back(Elt: Ops[i]);
2889 Ops.erase(CI: Ops.begin()+i);
2890 --i; --e;
2891 }
2892
2893 // If we found some loop invariants, fold them into the recurrence.
2894 if (!LIOps.empty()) {
2895 // Compute nowrap flags for the addition of the loop-invariant ops and
2896 // the addrec. Temporarily push it as an operand for that purpose. These
2897 // flags are valid in the scope of the addrec only.
2898 LIOps.push_back(Elt: AddRec);
2899 SCEV::NoWrapFlags Flags = ComputeFlags(LIOps);
2900 LIOps.pop_back();
2901
2902 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
2903 LIOps.push_back(Elt: AddRec->getStart());
2904
2905 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
2906
2907 // It is not in general safe to propagate flags valid on an add within
2908 // the addrec scope to one outside it. We must prove that the inner
2909 // scope is guaranteed to execute if the outer one does to be able to
2910 // safely propagate. We know the program is undefined if poison is
2911 // produced on the inner scoped addrec. We also know that *for this use*
2912 // the outer scoped add can't overflow (because of the flags we just
2913 // computed for the inner scoped add) without the program being undefined.
2914 // Proving that entry to the outer scope neccesitates entry to the inner
2915 // scope, thus proves the program undefined if the flags would be violated
2916 // in the outer scope.
2917 SCEV::NoWrapFlags AddFlags = Flags;
2918 if (AddFlags != SCEV::FlagAnyWrap) {
2919 auto *DefI = getDefiningScopeBound(Ops: LIOps);
2920 auto *ReachI = &*AddRecLoop->getHeader()->begin();
2921 if (!isGuaranteedToTransferExecutionTo(A: DefI, B: ReachI))
2922 AddFlags = SCEV::FlagAnyWrap;
2923 }
2924 AddRecOps[0] = getAddExpr(Ops&: LIOps, Flags: AddFlags, Depth: Depth + 1);
2925
2926 // Build the new addrec. Propagate the NUW and NSW flags if both the
2927 // outer add and the inner addrec are guaranteed to have no overflow.
2928 // Always propagate NW.
2929 Flags = AddRec->getNoWrapFlags(Mask: setFlags(Flags, OnFlags: SCEV::FlagNW));
2930 const SCEV *NewRec = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags);
2931
2932 // If all of the other operands were loop invariant, we are done.
2933 if (Ops.size() == 1) return NewRec;
2934
2935 // Otherwise, add the folded AddRec by the non-invariant parts.
2936 for (unsigned i = 0;; ++i)
2937 if (Ops[i] == AddRec) {
2938 Ops[i] = NewRec;
2939 break;
2940 }
2941 return getAddExpr(Ops, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2942 }
2943
2944 // Okay, if there weren't any loop invariants to be folded, check to see if
2945 // there are multiple AddRec's with the same loop induction variable being
2946 // added together. If so, we can fold them.
2947 for (unsigned OtherIdx = Idx+1;
2948 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2949 ++OtherIdx) {
2950 // We expect the AddRecExpr's to be sorted in reverse dominance order,
2951 // so that the 1st found AddRecExpr is dominated by all others.
2952 assert(DT.dominates(
2953 cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),
2954 AddRec->getLoop()->getHeader()) &&
2955 "AddRecExprs are not sorted in reverse dominance order?");
2956 if (AddRecLoop == cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx])->getLoop()) {
2957 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L>
2958 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
2959 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2960 ++OtherIdx) {
2961 const auto *OtherAddRec = cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
2962 if (OtherAddRec->getLoop() == AddRecLoop) {
2963 for (unsigned i = 0, e = OtherAddRec->getNumOperands();
2964 i != e; ++i) {
2965 if (i >= AddRecOps.size()) {
2966 append_range(C&: AddRecOps, R: OtherAddRec->operands().drop_front(N: i));
2967 break;
2968 }
2969 AddRecOps[i] =
2970 getAddExpr(LHS: AddRecOps[i], RHS: OtherAddRec->getOperand(i),
2971 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2972 }
2973 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
2974 }
2975 }
2976 // Step size has changed, so we cannot guarantee no self-wraparound.
2977 Ops[Idx] = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags: SCEV::FlagAnyWrap);
2978 return getAddExpr(Ops, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2979 }
2980 }
2981
2982 // Otherwise couldn't fold anything into this recurrence. Move onto the
2983 // next one.
2984 }
2985
2986 // Okay, it looks like we really DO need an add expr. Check to see if we
2987 // already have one, otherwise create a new one.
2988 assert((UseFlags == SCEV::FlagAnyWrap || equal(OrigOps, Ops)) &&
2989 "Tried to add SCEVUse flags after operands changed");
2990 return {getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
2991}
2992
2993const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
2994 SCEV::NoWrapFlags Flags) {
2995 FoldingSetNodeID ID;
2996 ID.AddInteger(I: scAddExpr);
2997 for (SCEVUse Op : Ops)
2998 ID.AddPointer(Ptr: Op.getOpaqueValue());
2999 FoldingSetInsertToken Token;
3000 SCEVAddExpr *S = static_cast<SCEVAddExpr *>(UniqueSCEVs.lookup(ID, Token));
3001 if (!S) {
3002 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3003 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3004 S = new (SCEVAllocator)
3005 SCEVAddExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size());
3006 UniqueSCEVs.insert(N: S, Token);
3007 S->computeAndSetCanonical(SE&: *this);
3008 registerUser(User: S, Ops);
3009 }
3010 S->setNoWrapFlags(Flags);
3011 return S;
3012}
3013
3014const SCEV *ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<SCEVUse> Ops,
3015 const Loop *L,
3016 SCEV::NoWrapFlags Flags) {
3017 FoldingSetNodeID ID;
3018 ID.AddInteger(I: scAddRecExpr);
3019 for (SCEVUse Op : Ops)
3020 ID.AddPointer(Ptr: Op.getOpaqueValue());
3021 ID.AddPointer(Ptr: L);
3022 FoldingSetInsertToken Token;
3023 SCEVAddRecExpr *S =
3024 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.lookup(ID, Token));
3025 if (!S) {
3026 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3027 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3028 S = new (SCEVAllocator)
3029 SCEVAddRecExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size(), L);
3030 UniqueSCEVs.insert(N: S, Token);
3031 S->computeAndSetCanonical(SE&: *this);
3032 LoopUsers[L].push_back(Elt: S);
3033 registerUser(User: S, Ops);
3034 }
3035 setNoWrapFlags(AddRec: S, Flags);
3036 return S;
3037}
3038
3039const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
3040 SCEV::NoWrapFlags Flags) {
3041 FoldingSetNodeID ID;
3042 ID.AddInteger(I: scMulExpr);
3043 for (SCEVUse Op : Ops)
3044 ID.AddPointer(Ptr: Op.getOpaqueValue());
3045 FoldingSetInsertToken Token;
3046 SCEVMulExpr *S = static_cast<SCEVMulExpr *>(UniqueSCEVs.lookup(ID, Token));
3047 if (!S) {
3048 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3049 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3050 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(Allocator&: SCEVAllocator),
3051 O, Ops.size());
3052 UniqueSCEVs.insert(N: S, Token);
3053 S->computeAndSetCanonical(SE&: *this);
3054 registerUser(User: S, Ops);
3055 }
3056 S->setNoWrapFlags(Flags);
3057 return S;
3058}
3059
3060const SCEV *ScalarEvolution::getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS) {
3061 FoldingSetNodeID ID;
3062 ID.AddInteger(I: scUDivExpr);
3063 ID.AddPointer(Ptr: LHS.getOpaqueValue());
3064 ID.AddPointer(Ptr: RHS.getOpaqueValue());
3065 FoldingSetInsertToken Token;
3066 SCEV *S = UniqueSCEVs.lookup(ID, Token);
3067 if (!S) {
3068 S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(Allocator&: SCEVAllocator), LHS, RHS);
3069 UniqueSCEVs.insert(N: S, Token);
3070 S->computeAndSetCanonical(SE&: *this);
3071 registerUser(User: S, Ops: {LHS, RHS});
3072 }
3073 return S;
3074}
3075
3076static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
3077 uint64_t k = i*j;
3078 if (j > 1 && k / j != i) Overflow = true;
3079 return k;
3080}
3081
3082/// Compute the result of "n choose k", the binomial coefficient. If an
3083/// intermediate computation overflows, Overflow will be set and the return will
3084/// be garbage. Overflow is not cleared on absence of overflow.
3085static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
3086 // We use the multiplicative formula:
3087 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
3088 // At each iteration, we take the n-th term of the numeral and divide by the
3089 // (k-n)th term of the denominator. This division will always produce an
3090 // integral result, and helps reduce the chance of overflow in the
3091 // intermediate computations. However, we can still overflow even when the
3092 // final result would fit.
3093
3094 if (n == 0 || n == k) return 1;
3095 if (k > n) return 0;
3096
3097 if (k > n/2)
3098 k = n-k;
3099
3100 uint64_t r = 1;
3101 for (uint64_t i = 1; i <= k; ++i) {
3102 r = umul_ov(i: r, j: n-(i-1), Overflow);
3103 r /= i;
3104 }
3105 return r;
3106}
3107
3108/// Determine if any of the operands in this SCEV are a constant or if
3109/// any of the add or multiply expressions in this SCEV contain a constant.
3110static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
3111 struct FindConstantInAddMulChain {
3112 bool FoundConstant = false;
3113
3114 bool follow(const SCEV *S) {
3115 FoundConstant |= isa<SCEVConstant>(Val: S);
3116 return isa<SCEVAddExpr>(Val: S) || isa<SCEVMulExpr>(Val: S);
3117 }
3118
3119 bool isDone() const {
3120 return FoundConstant;
3121 }
3122 };
3123
3124 FindConstantInAddMulChain F;
3125 SCEVTraversal<FindConstantInAddMulChain> ST(F);
3126 ST.visitAll(Root: StartExpr);
3127 return F.FoundConstant;
3128}
3129
3130/// Get a canonical multiply expression, or something simpler if possible.
3131SCEVUse ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
3132 SCEVFlags Flags, unsigned Depth) {
3133 SCEV::NoWrapFlags OrigFlags = Flags.ExprFlags;
3134 SCEV::NoWrapFlags UseFlags = Flags.UseFlags;
3135 assert(OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3136 "only nuw or nsw allowed");
3137 assert(UseFlags == maskFlags(UseFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3138 "only nuw or nsw allowed");
3139 assert(!Ops.empty() && "Cannot get empty mul!");
3140 if (Ops.size() == 1) return Ops[0];
3141#ifndef NDEBUG
3142 Type *ETy = Ops[0]->getType();
3143 assert(!ETy->isPointerTy());
3144 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
3145 assert(Ops[i]->getType() == ETy &&
3146 "SCEVMulExpr operand types don't match!");
3147#endif
3148
3149 const SCEV *Folded = constantFoldAndGroupOps(
3150 SE&: *this, LI, DT, Ops,
3151 Fold: [](const APInt &C1, const APInt &C2) { return C1 * C2; },
3152 IsIdentity: [](const APInt &C) { return C.isOne(); }, // identity
3153 IsAbsorber: [](const APInt &C) { return C.isZero(); }); // absorber
3154 if (Folded)
3155 return Folded;
3156
3157#ifndef NDEBUG
3158 // Keep track of operands after constant folding, for verification when adding
3159 // use-specific flags.
3160 const SmallVector<SCEVUse, 8> OrigOps(Ops.begin(), Ops.end());
3161#endif
3162
3163 // Delay expensive flag strengthening until necessary.
3164 auto ComputeFlags = [this, OrigFlags](const ArrayRef<SCEVUse> Ops) {
3165 return StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops, Flags: OrigFlags);
3166 };
3167
3168 // Limit recursion calls depth.
3169 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
3170 return {getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
3171
3172 if (SCEV *S = findExistingSCEVInCache(SCEVType: scMulExpr, Ops)) {
3173 // Don't strengthen flags if we have no new information.
3174 SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
3175 if (Mul->getNoWrapFlags(Mask: OrigFlags) != OrigFlags)
3176 Mul->setNoWrapFlags(ComputeFlags(Ops));
3177 return {S, UseFlags};
3178 }
3179
3180 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: Ops[0])) {
3181 if (Ops.size() == 2) {
3182 // C1*(C2+V) -> C1*C2 + C1*V
3183 // If any of Add's ops are Adds or Muls with a constant, apply this
3184 // transformation as well.
3185 //
3186 // TODO: There are some cases where this transformation is not
3187 // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of
3188 // this transformation should be narrowed down.
3189 const SCEV *Op0, *Op1;
3190 if (match(U: Ops[1], P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))) &&
3191 containsConstantInAddMulChain(StartExpr: Ops[1])) {
3192 const SCEV *LHS = getMulExpr(LHS: LHSC, RHS: Op0, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3193 const SCEV *RHS = getMulExpr(LHS: LHSC, RHS: Op1, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3194 return getAddExpr(LHS, RHS, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3195 }
3196
3197 if (Ops[0]->isAllOnesValue()) {
3198 // If we have a mul by -1 of an add, try distributing the -1 among the
3199 // add operands.
3200 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[1])) {
3201 SmallVector<SCEVUse, 4> NewOps;
3202 bool AnyFolded = false;
3203 for (const SCEV *AddOp : Add->operands()) {
3204 const SCEV *Mul = getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddOp),
3205 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3206 if (!isa<SCEVMulExpr>(Val: Mul)) AnyFolded = true;
3207 NewOps.push_back(Elt: Mul);
3208 }
3209 if (AnyFolded)
3210 return getAddExpr(Ops&: NewOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3211 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: Ops[1])) {
3212 // Negation preserves a recurrence's no self-wrap property.
3213 SmallVector<SCEVUse, 4> Operands;
3214 for (const SCEV *AddRecOp : AddRec->operands())
3215 Operands.push_back(Elt: getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddRecOp),
3216 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3217 // Let M be the minimum representable signed value. AddRec with nsw
3218 // multiplied by -1 can have signed overflow if and only if it takes a
3219 // value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
3220 // maximum signed value. In all other cases signed overflow is
3221 // impossible.
3222 auto FlagsMask = SCEV::FlagNW;
3223 if (AddRec->hasNoSignedWrap()) {
3224 auto MinInt =
3225 APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: AddRec->getType()));
3226 if (getSignedRangeMin(S: AddRec) != MinInt)
3227 FlagsMask = setFlags(Flags: FlagsMask, OnFlags: SCEV::FlagNSW);
3228 }
3229 return getAddRecExpr(Operands, L: AddRec->getLoop(),
3230 Flags: AddRec->getNoWrapFlags(Mask: FlagsMask));
3231 }
3232 }
3233
3234 // Try to push the constant operand into a ZExt: C * zext (A + B) ->
3235 // zext (C*A + C*B) if trunc (C) * (A + B) does not unsigned-wrap.
3236 const SCEVAddExpr *InnerAdd;
3237 if (match(U: Ops[1], P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
3238 const SCEV *NarrowC = getTruncateExpr(Op: LHSC, Ty: InnerAdd->getType());
3239 if (isa<SCEVConstant>(Val: InnerAdd->getOperand(i: 0)) &&
3240 getZeroExtendExpr(Op: NarrowC, Ty: Ops[1]->getType()) == LHSC &&
3241 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops: {NarrowC, InnerAdd},
3242 Flags: SCEV::FlagAnyWrap),
3243 TestFlags: SCEV::FlagNUW)) {
3244 const SCEV *Res =
3245 getMulExpr(LHS: NarrowC, RHS: InnerAdd, Flags: SCEV::FlagNUW, Depth: Depth + 1);
3246 return getZeroExtendExpr(Op: Res, Ty: Ops[1]->getType(), Depth: Depth + 1);
3247 };
3248 }
3249
3250 // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2,
3251 // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple
3252 // of C1, fold to (D /u (C2 /u C1)).
3253 const SCEV *D;
3254 APInt C1V = LHSC->getAPInt();
3255 // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1
3256 // as -1 * 1, as it won't enable additional folds.
3257 if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes())
3258 C1V = C1V.abs();
3259 const SCEVConstant *C2;
3260 if (C1V.isPowerOf2() &&
3261 match(U: Ops[1], P: m_scev_UDiv(Op0: m_SCEV(V&: D), Op1: m_SCEVConstant(V&: C2))) &&
3262 C2->getAPInt().isPowerOf2() &&
3263 C1V.logBase2() <= getMinTrailingZeros(S: D)) {
3264 const SCEV *NewMul = nullptr;
3265 if (C1V.uge(RHS: C2->getAPInt())) {
3266 NewMul = getMulExpr(LHS: getUDivExpr(LHS: getConstant(Val: C1V), RHS: C2), RHS: D);
3267 } else if (C2->getAPInt().logBase2() <= getMinTrailingZeros(S: D)) {
3268 assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier");
3269 NewMul = getUDivExpr(LHS: D, RHS: getUDivExpr(LHS: C2, RHS: getConstant(Val: C1V)));
3270 }
3271 if (NewMul)
3272 return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(V: NewMul);
3273 }
3274 }
3275 }
3276
3277 // Skip over the add expression until we get to a multiply.
3278 unsigned Idx = 0;
3279 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
3280 ++Idx;
3281
3282 // If there are mul operands inline them all into this expression.
3283 if (Idx < Ops.size()) {
3284 bool DeletedMul = false;
3285 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
3286 if (Ops.size() > MulOpsInlineThreshold)
3287 break;
3288 // If we have an mul, expand the mul operands onto the end of the
3289 // operands list.
3290 Ops.erase(CI: Ops.begin()+Idx);
3291 append_range(C&: Ops, R: Mul->operands());
3292 DeletedMul = true;
3293 }
3294
3295 // If we deleted at least one mul, we added operands to the end of the
3296 // list, and they are not necessarily sorted. Recurse to resort and
3297 // resimplify any operands we just acquired.
3298 if (DeletedMul)
3299 return getMulExpr(Ops, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3300 }
3301
3302 // If there are any add recurrences in the operands list, see if any other
3303 // added values are loop invariant. If so, we can fold them into the
3304 // recurrence.
3305 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3306 ++Idx;
3307
3308 // Scan over all recurrences, trying to fold loop invariants into them.
3309 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
3310 // Scan all of the other operands to this mul and add them to the vector
3311 // if they are loop invariant w.r.t. the recurrence.
3312 SmallVector<SCEVUse, 8> LIOps;
3313 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
3314 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3315 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRec->getLoop())) {
3316 LIOps.push_back(Elt: Ops[i]);
3317 Ops.erase(CI: Ops.begin()+i);
3318 --i; --e;
3319 }
3320
3321 // If we found some loop invariants, fold them into the recurrence.
3322 if (!LIOps.empty()) {
3323 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
3324 SmallVector<SCEVUse, 4> NewOps;
3325 NewOps.reserve(N: AddRec->getNumOperands());
3326 const SCEV *Scale = getMulExpr(Ops&: LIOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3327
3328 // If both the mul and addrec are nuw, we can preserve nuw.
3329 // If both the mul and addrec are nsw, we can only preserve nsw if either
3330 // a) they are also nuw, or
3331 // b) all multiplications of addrec operands with scale are nsw.
3332 SCEV::NoWrapFlags Flags =
3333 AddRec->getNoWrapFlags(Mask: ComputeFlags({Scale, AddRec}));
3334
3335 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
3336 NewOps.push_back(Elt: getMulExpr(LHS: Scale, RHS: AddRec->getOperand(i),
3337 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3338
3339 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW) && !hasFlags(Flags, TestFlags: SCEV::FlagNUW)) {
3340 ConstantRange NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
3341 BinOp: Instruction::Mul, Other: getSignedRange(S: Scale),
3342 NoWrapKind: OverflowingBinaryOperator::NoSignedWrap);
3343 if (!NSWRegion.contains(CR: getSignedRange(S: AddRec->getOperand(i))))
3344 Flags = clearFlags(Flags, OffFlags: SCEV::FlagNSW);
3345 }
3346 }
3347
3348 const SCEV *NewRec = getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags);
3349
3350 // If all of the other operands were loop invariant, we are done.
3351 if (Ops.size() == 1) return NewRec;
3352
3353 // Otherwise, multiply the folded AddRec by the non-invariant parts.
3354 for (unsigned i = 0;; ++i)
3355 if (Ops[i] == AddRec) {
3356 Ops[i] = NewRec;
3357 break;
3358 }
3359 return getMulExpr(Ops, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3360 }
3361
3362 // Okay, if there weren't any loop invariants to be folded, check to see
3363 // if there are multiple AddRec's with the same loop induction variable
3364 // being multiplied together. If so, we can fold them.
3365
3366 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
3367 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
3368 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
3369 // ]]],+,...up to x=2n}.
3370 // Note that the arguments to choose() are always integers with values
3371 // known at compile time, never SCEV objects.
3372 //
3373 // The implementation avoids pointless extra computations when the two
3374 // addrec's are of different length (mathematically, it's equivalent to
3375 // an infinite stream of zeros on the right).
3376 bool OpsModified = false;
3377 for (unsigned OtherIdx = Idx+1;
3378 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3379 ++OtherIdx) {
3380 const SCEVAddRecExpr *OtherAddRec =
3381 dyn_cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
3382 if (!OtherAddRec || OtherAddRec->getLoop() != AddRec->getLoop())
3383 continue;
3384
3385 // Limit max number of arguments to avoid creation of unreasonably big
3386 // SCEVAddRecs with very complex operands.
3387 if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
3388 MaxAddRecSize || hasHugeExpression(Ops: {AddRec, OtherAddRec}))
3389 continue;
3390
3391 bool Overflow = false;
3392 Type *Ty = AddRec->getType();
3393 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
3394 SmallVector<SCEVUse, 7> AddRecOps;
3395 for (int x = 0, xe = AddRec->getNumOperands() +
3396 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
3397 SmallVector<SCEVUse, 7> SumOps;
3398 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3399 uint64_t Coeff1 = Choose(n: x, k: 2*x - y, Overflow);
3400 for (int z = std::max(a: y-x, b: y-(int)AddRec->getNumOperands()+1),
3401 ze = std::min(a: x+1, b: (int)OtherAddRec->getNumOperands());
3402 z < ze && !Overflow; ++z) {
3403 uint64_t Coeff2 = Choose(n: 2*x - y, k: x-z, Overflow);
3404 uint64_t Coeff;
3405 if (LargerThan64Bits)
3406 Coeff = umul_ov(i: Coeff1, j: Coeff2, Overflow);
3407 else
3408 Coeff = Coeff1*Coeff2;
3409 const SCEV *CoeffTerm = getConstant(Ty, V: Coeff);
3410 const SCEV *Term1 = AddRec->getOperand(i: y-z);
3411 const SCEV *Term2 = OtherAddRec->getOperand(i: z);
3412 SumOps.push_back(Elt: getMulExpr(Op0: CoeffTerm, Op1: Term1, Op2: Term2,
3413 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3414 }
3415 }
3416 if (SumOps.empty())
3417 SumOps.push_back(Elt: getZero(Ty));
3418 AddRecOps.push_back(Elt: getAddExpr(Ops&: SumOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3419 }
3420 if (!Overflow) {
3421 const SCEV *NewAddRec = getAddRecExpr(Operands&: AddRecOps, L: AddRec->getLoop(),
3422 Flags: SCEV::FlagAnyWrap);
3423 if (Ops.size() == 2) return NewAddRec;
3424 Ops[Idx] = NewAddRec;
3425 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
3426 OpsModified = true;
3427 AddRec = dyn_cast<SCEVAddRecExpr>(Val: NewAddRec);
3428 if (!AddRec)
3429 break;
3430 }
3431 }
3432 if (OpsModified)
3433 return getMulExpr(Ops, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3434
3435 // Otherwise couldn't fold anything into this recurrence. Move onto the
3436 // next one.
3437 }
3438
3439 // Okay, it looks like we really DO need an mul expr. Check to see if we
3440 // already have one, otherwise create a new one.
3441 assert((UseFlags == SCEV::FlagAnyWrap || equal(OrigOps, Ops)) &&
3442 "Tried to add SCEVUse flags after operands changed");
3443 return {getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
3444}
3445
3446/// Represents an unsigned remainder expression based on unsigned division.
3447const SCEV *ScalarEvolution::getURemExpr(SCEVUse LHS, SCEVUse RHS) {
3448 assert(getEffectiveSCEVType(LHS->getType()) ==
3449 getEffectiveSCEVType(RHS->getType()) &&
3450 "SCEVURemExpr operand types don't match!");
3451
3452 // Short-circuit easy cases
3453 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3454 // If constant is one, the result is trivial
3455 if (RHSC->getValue()->isOne())
3456 return getZero(Ty: LHS->getType()); // X urem 1 --> 0
3457
3458 // If constant is a power of two, fold into a zext(trunc(LHS)).
3459 if (RHSC->getAPInt().isPowerOf2()) {
3460 Type *FullTy = LHS->getType();
3461 Type *TruncTy =
3462 IntegerType::get(C&: getContext(), NumBits: RHSC->getAPInt().logBase2());
3463 return getZeroExtendExpr(Op: getTruncateExpr(Op: LHS, Ty: TruncTy), Ty: FullTy);
3464 }
3465 }
3466
3467 // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
3468 const SCEV *UDiv = getUDivExpr(LHS, RHS);
3469 const SCEV *Mult = getMulExpr(LHS: UDiv, RHS, Flags: SCEV::FlagNUW);
3470 return getMinusSCEV(LHS, RHS: Mult, Flags: SCEV::FlagNUW);
3471}
3472
3473/// Get a canonical unsigned division expression, or something simpler if
3474/// possible.
3475const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
3476 assert(!LHS->getType()->isPointerTy() &&
3477 "SCEVUDivExpr operand can't be pointer!");
3478 assert(LHS->getType() == RHS->getType() &&
3479 "SCEVUDivExpr operand types don't match!");
3480
3481 if (SCEV *S = findExistingSCEVInCache(SCEVType: scUDivExpr, Ops: {LHS, RHS}))
3482 return S;
3483
3484 // 0 udiv Y == 0
3485 if (match(U: LHS, P: m_scev_Zero()))
3486 return LHS;
3487
3488 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3489 if (RHSC->getValue()->isOne())
3490 return LHS; // X udiv 1 --> x
3491 // If the denominator is zero, the result of the udiv is undefined. Don't
3492 // try to analyze it, because the resolution chosen here may differ from
3493 // the resolution chosen in other parts of the compiler.
3494 if (!RHSC->getValue()->isZero()) {
3495 // Determine if the division can be folded into the operands of
3496 // its operands.
3497 // TODO: Generalize this to non-constants by using known-bits information.
3498 Type *Ty = LHS->getType();
3499 unsigned LZ = RHSC->getAPInt().countl_zero();
3500 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
3501 // For non-power-of-two values, effectively round the value up to the
3502 // nearest power of two.
3503 if (!RHSC->getAPInt().isPowerOf2())
3504 ++MaxShiftAmt;
3505 IntegerType *ExtTy =
3506 IntegerType::get(C&: getContext(), NumBits: getTypeSizeInBits(Ty) + MaxShiftAmt);
3507 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
3508 if (const SCEVConstant *Step =
3509 dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE&: *this))) {
3510 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
3511 const APInt &StepInt = Step->getAPInt();
3512 const APInt &DivInt = RHSC->getAPInt();
3513 if (!StepInt.urem(RHS: DivInt) &&
3514 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3515 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3516 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy),
3517 L: AR->getLoop(), Flags: SCEV::FlagAnyWrap)) {
3518 SmallVector<SCEVUse, 4> Operands;
3519 for (const SCEV *Op : AR->operands())
3520 Operands.push_back(Elt: getUDivExpr(LHS: Op, RHS));
3521 return getAddRecExpr(Operands, L: AR->getLoop(), Flags: SCEV::FlagNW);
3522 }
3523 /// Get a canonical UDivExpr for a recurrence.
3524 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
3525 const APInt *StartRem;
3526 if (!DivInt.urem(RHS: StepInt) && match(S: getURemExpr(LHS: AR->getStart(), RHS: Step),
3527 P: m_scev_APInt(C&: StartRem))) {
3528 bool NoWrap =
3529 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3530 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3531 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy), L: AR->getLoop(),
3532 Flags: SCEV::FlagAnyWrap);
3533
3534 // With N <= C and both N, C as powers-of-2, the transformation
3535 // {X,+,N}/C => {(X - X%N),+,N}/C preserves division results even
3536 // if wrapping occurs, as the division results remain equivalent for
3537 // all offsets in [[(X - X%N), X).
3538 bool CanFoldWithWrap = StepInt.ule(RHS: DivInt) && // N <= C
3539 StepInt.isPowerOf2() && DivInt.isPowerOf2();
3540 // Only fold if the subtraction can be folded in the start
3541 // expression.
3542 const SCEV *NewStart =
3543 getMinusSCEV(LHS: AR->getStart(), RHS: getConstant(Val: *StartRem));
3544 if (*StartRem != 0 && (NoWrap || CanFoldWithWrap) &&
3545 !isa<SCEVAddExpr>(Val: NewStart)) {
3546 const SCEV *NewLHS =
3547 getAddRecExpr(Start: NewStart, Step, L: AR->getLoop(),
3548 Flags: NoWrap ? SCEV::FlagNW : SCEV::FlagAnyWrap);
3549 if (LHS != NewLHS)
3550 return getUDivExpr(LHS: NewLHS, RHS);
3551 }
3552 }
3553 }
3554 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
3555 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val&: LHS)) {
3556 if (M->hasNoUnsignedWrap()) {
3557 // Find an operand that's safely divisible.
3558 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3559 const SCEV *Op = M->getOperand(i);
3560 const SCEV *Div = getUDivExpr(LHS: Op, RHS: RHSC);
3561 if (!isa<SCEVUDivExpr>(Val: Div) && getMulExpr(LHS: Div, RHS: RHSC) == Op) {
3562 SmallVector<SCEVUse, 4> Operands(M->operands());
3563 Operands[i] = Div;
3564 return getMulExpr(Ops&: Operands);
3565 }
3566 }
3567
3568 // Even if it's not divisible, try to remove a common factor.
3569 if (const auto *LHSC = dyn_cast<SCEVConstant>(Val: M->getOperand(i: 0))) {
3570 APInt Factor = APIntOps::GreatestCommonDivisor(A: LHSC->getAPInt(),
3571 B: RHSC->getAPInt());
3572 if (!Factor.isIntN(N: 1)) {
3573 SmallVector<SCEVUse, 2> NewOperands;
3574 NewOperands.push_back(Elt: getConstant(Val: LHSC->getAPInt().udiv(RHS: Factor)));
3575 append_range(C&: NewOperands, R: M->operands().drop_front());
3576 const SCEV *NewMul = getMulExpr(Ops&: NewOperands);
3577 return getUDivExpr(LHS: NewMul,
3578 RHS: getConstant(Val: RHSC->getAPInt().udiv(RHS: Factor)));
3579 }
3580 }
3581 }
3582 }
3583
3584 // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
3585 if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(Val&: LHS)) {
3586 if (auto *DivisorConstant =
3587 dyn_cast<SCEVConstant>(Val: OtherDiv->getRHS())) {
3588 bool Overflow = false;
3589 APInt NewRHS =
3590 DivisorConstant->getAPInt().umul_ov(RHS: RHSC->getAPInt(), Overflow);
3591 if (Overflow) {
3592 return getConstant(Ty: RHSC->getType(), V: 0, isSigned: false);
3593 }
3594 return getUDivExpr(LHS: OtherDiv->getLHS(), RHS: getConstant(Val: NewRHS));
3595 }
3596 }
3597
3598 // (A+B)/C --> (A/C + B/C) if the add does not unsigned wrap and A/C and
3599 // B/C can be folded.
3600 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(Val&: LHS)) {
3601 if (A->hasNoUnsignedWrap()) {
3602 SmallVector<SCEVUse, 4> Operands;
3603 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
3604 const SCEV *Op = getUDivExpr(LHS: A->getOperand(i), RHS);
3605 if (isa<SCEVUDivExpr>(Val: Op) ||
3606 getMulExpr(LHS: Op, RHS) != A->getOperand(i))
3607 break;
3608 Operands.push_back(Elt: Op);
3609 }
3610 if (Operands.size() == A->getNumOperands())
3611 return getAddExpr(Ops&: Operands);
3612 }
3613 }
3614
3615 // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
3616 // This is an idiom for rounding A up to the next multiple of N, where A
3617 // is aready known to be a multiple of M. In this case, instcombine can
3618 // see that some low bits of the added constant are unused, so can clear
3619 // them, but we want to canonicalise to set the low bits. This makes the
3620 // pattern easier to match, without needing to check for known bits in
3621 // A*M.
3622 const APInt &N = RHSC->getAPInt();
3623 const APInt *NMinusM, *M;
3624 const SCEV *A;
3625 if (match(U: LHS, P: m_scev_Add(Op0: m_scev_APInt(C&: NMinusM),
3626 Op1: m_scev_Mul(Op0: m_scev_APInt(C&: M), Op1: m_SCEV(V&: A))))) {
3627 if (N.isPowerOf2() && M->isPowerOf2() && M->ult(RHS: N) &&
3628 *NMinusM == N - *M) {
3629 return getUDivExpr(
3630 LHS: getAddExpr(LHS: getConstant(Val: N - 1), RHS: getMulExpr(LHS: getConstant(Val: *M), RHS: A)),
3631 RHS);
3632 }
3633 }
3634
3635 // Fold if both operands are constant.
3636 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS))
3637 return getConstant(Val: LHSC->getAPInt().udiv(RHS: RHSC->getAPInt()));
3638 }
3639 }
3640
3641 // ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
3642 const APInt *NegC, *C;
3643 if (match(U: LHS,
3644 P: m_scev_Add(Op0: m_scev_APInt(C&: NegC),
3645 Op1: m_scev_SMax(Op0: m_scev_APInt(C), Op1: m_scev_Specific(S: RHS)))) &&
3646 NegC->isNegative() && !NegC->isMinSignedValue() && *C == -*NegC)
3647 return getZero(Ty: LHS->getType());
3648
3649 // (%a * %b)<nuw> / %b -> %a
3650 const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: LHS);
3651 if (Mul && Mul->hasNoUnsignedWrap()) {
3652 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
3653 if (Mul->getOperand(i) == RHS) {
3654 SmallVector<SCEVUse, 2> Operands;
3655 append_range(C&: Operands, R: Mul->operands().take_front(N: i));
3656 append_range(C&: Operands, R: Mul->operands().drop_front(N: i + 1));
3657 return getMulExpr(Ops&: Operands);
3658 }
3659 }
3660 }
3661
3662 // TODO: Generalize to handle any common factors.
3663 // udiv (mul nuw a, vscale), (mul nuw b, vscale) --> udiv a, b
3664 const SCEV *NewLHS, *NewRHS;
3665 if (match(U: LHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
3666 match(U: RHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale())))
3667 return getUDivExpr(LHS: NewLHS, RHS: NewRHS);
3668
3669 return getOrCreateUDivExpr(LHS, RHS);
3670}
3671
3672/// Get a canonical unsigned division expression, or something simpler if
3673/// possible. There is no representation for an exact udiv in SCEV IR, but we
3674/// can attempt to optimize it prior to construction.
3675const SCEV *ScalarEvolution::getUDivExactExpr(SCEVUse LHS, SCEVUse RHS) {
3676 // Currently there is no exact specific logic.
3677
3678 return getUDivExpr(LHS, RHS);
3679}
3680
3681/// Get an add recurrence expression for the specified loop. Simplify the
3682/// expression as much as possible.
3683SCEVUse ScalarEvolution::getAddRecExpr(SCEVUse Start, SCEVUse Step,
3684 const Loop *L, SCEVFlags Flags) {
3685 SmallVector<SCEVUse, 4> Operands;
3686 Operands.push_back(Elt: Start);
3687 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Val&: Step))
3688 if (StepChrec->getLoop() == L) {
3689 append_range(C&: Operands, R: StepChrec->operands());
3690 // The use flags describe the two-operand recurrence, not the flattened
3691 // one built here, so drop them just like the expression's NUW/NSW.
3692 return getAddRecExpr(Operands, L,
3693 Flags: maskFlags(Flags: Flags.ExprFlags, Mask: SCEV::FlagNW));
3694 }
3695
3696 Operands.push_back(Elt: Step);
3697 return getAddRecExpr(Operands, L, Flags);
3698}
3699
3700/// Get an add recurrence expression for the specified loop. Simplify the
3701/// expression as much as possible.
3702SCEVUse ScalarEvolution::getAddRecExpr(SmallVectorImpl<SCEVUse> &Operands,
3703 const Loop *L, SCEVFlags NWFlags) {
3704 SCEV::NoWrapFlags Flags = NWFlags.ExprFlags;
3705 SCEV::NoWrapFlags UseFlags = NWFlags.UseFlags;
3706 assert(!(UseFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
3707 "only nuw or nsw allowed");
3708 if (Operands.size() == 1) return Operands[0];
3709#ifndef NDEBUG
3710 Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
3711 for (const SCEV *Op : llvm::drop_begin(Operands)) {
3712 assert(getEffectiveSCEVType(Op->getType()) == ETy &&
3713 "SCEVAddRecExpr operand types don't match!");
3714 assert(!Op->getType()->isPointerTy() && "Step must be integer");
3715 }
3716 for (const SCEV *Op : Operands)
3717 assert(isAvailableAtLoopEntry(Op, L) &&
3718 "SCEVAddRecExpr operand is not available at loop entry!");
3719
3720 // Keep track of the original operands, for verification when adding
3721 // use-specific flags.
3722 const SmallVector<SCEVUse, 4> OrigOperands(Operands.begin(), Operands.end());
3723#endif
3724
3725 if (Operands.back()->isZero()) {
3726 Operands.pop_back();
3727 return getAddRecExpr(Operands, L, NWFlags: SCEV::FlagAnyWrap); // {X,+,0} --> X
3728 }
3729
3730 // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
3731 // use that information to infer NUW and NSW flags. However, computing a
3732 // BE count requires calling getAddRecExpr, so we may not yet have a
3733 // meaningful BE count at this point (and if we don't, we'd be stuck
3734 // with a SCEVCouldNotCompute as the cached BE count).
3735
3736 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
3737
3738 // Canonicalize nested AddRecs in by nesting them in order of loop depth.
3739 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Val&: Operands[0])) {
3740 const Loop *NestedLoop = NestedAR->getLoop();
3741 if (L->contains(L: NestedLoop)
3742 ? (L->getLoopDepth() < NestedLoop->getLoopDepth())
3743 : (!NestedLoop->contains(L) &&
3744 DT.dominates(A: L->getHeader(), B: NestedLoop->getHeader()))) {
3745 SmallVector<SCEVUse, 4> NestedOperands(NestedAR->operands());
3746 Operands[0] = NestedAR->getStart();
3747 // AddRecs require their operands be loop-invariant with respect to their
3748 // loops. Don't perform this transformation if it would break this
3749 // requirement.
3750 bool AllInvariant = all_of(
3751 Range&: Operands, P: [&](const SCEV *Op) { return isLoopInvariant(S: Op, L); });
3752
3753 if (AllInvariant) {
3754 // Create a recurrence for the outer loop with the same step size.
3755 //
3756 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
3757 // inner recurrence has the same property.
3758 SCEV::NoWrapFlags OuterFlags =
3759 maskFlags(Flags, Mask: SCEV::FlagNW | NestedAR->getNoWrapFlags());
3760
3761 NestedOperands[0] = getAddRecExpr(Operands, L, NWFlags: OuterFlags);
3762 AllInvariant = all_of(Range&: NestedOperands, P: [&](const SCEV *Op) {
3763 return isLoopInvariant(S: Op, L: NestedLoop);
3764 });
3765
3766 if (AllInvariant) {
3767 // Ok, both add recurrences are valid after the transformation.
3768 //
3769 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
3770 // the outer recurrence has the same property.
3771 SCEV::NoWrapFlags InnerFlags =
3772 maskFlags(Flags: NestedAR->getNoWrapFlags(), Mask: SCEV::FlagNW | Flags);
3773 return getAddRecExpr(Operands&: NestedOperands, L: NestedLoop, NWFlags: InnerFlags);
3774 }
3775 }
3776 // Reset Operands to its original state.
3777 Operands[0] = NestedAR;
3778 }
3779 }
3780
3781 // Okay, it looks like we really DO need an addrec expr. Check to see if we
3782 // already have one, otherwise create a new one.
3783 assert((UseFlags == SCEV::FlagAnyWrap || equal(OrigOperands, Operands)) &&
3784 "Tried to add SCEVUse flags after operands changed");
3785 return {getOrCreateAddRecExpr(Ops: Operands, L, Flags), UseFlags};
3786}
3787
3788const SCEV *ScalarEvolution::getGEPExpr(GEPOperator *GEP,
3789 ArrayRef<SCEVUse> IndexExprs) {
3790 const SCEV *BaseExpr = getSCEV(V: GEP->getPointerOperand());
3791 // getSCEV(Base)->getType() has the same address space as Base->getType()
3792 // because SCEV::getType() preserves the address space.
3793 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
3794 if (NW != GEPNoWrapFlags::none()) {
3795 // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
3796 // but to do that, we have to ensure that said flag is valid in the entire
3797 // defined scope of the SCEV.
3798 // TODO: non-instructions have global scope. We might be able to prove
3799 // some global scope cases
3800 auto *GEPI = dyn_cast<Instruction>(Val: GEP);
3801 if (!GEPI || !isSCEVExprNeverPoison(I: GEPI))
3802 NW = GEPNoWrapFlags::none();
3803 }
3804
3805 return getGEPExpr(BaseExpr, IndexExprs, SrcElementTy: GEP->getSourceElementType(), NW);
3806}
3807
3808const SCEV *ScalarEvolution::getGEPExpr(SCEVUse BaseExpr,
3809 ArrayRef<SCEVUse> IndexExprs,
3810 Type *SrcElementTy, GEPNoWrapFlags NW) {
3811 SCEV::NoWrapFlags OffsetWrap = SCEV::FlagAnyWrap;
3812 if (NW.hasNoUnsignedSignedWrap())
3813 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNSW);
3814 if (NW.hasNoUnsignedWrap())
3815 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNUW);
3816
3817 Type *CurTy = BaseExpr->getType();
3818 Type *IntIdxTy = getEffectiveSCEVType(Ty: BaseExpr->getType());
3819 bool FirstIter = true;
3820 SmallVector<SCEVUse, 4> Offsets;
3821 for (SCEVUse IndexExpr : IndexExprs) {
3822 // Compute the (potentially symbolic) offset in bytes for this index.
3823 if (StructType *STy = dyn_cast<StructType>(Val: CurTy)) {
3824 // For a struct, add the member offset.
3825 ConstantInt *Index = cast<SCEVConstant>(Val&: IndexExpr)->getValue();
3826 unsigned FieldNo = Index->getZExtValue();
3827 const SCEV *FieldOffset = getOffsetOfExpr(IntTy: IntIdxTy, STy, FieldNo);
3828 Offsets.push_back(Elt: FieldOffset);
3829
3830 // Update CurTy to the type of the field at Index.
3831 CurTy = STy->getTypeAtIndex(V: Index);
3832 } else {
3833 // Update CurTy to its element type.
3834 if (FirstIter) {
3835 assert(isa<PointerType>(CurTy) &&
3836 "The first index of a GEP indexes a pointer");
3837 CurTy = SrcElementTy;
3838 FirstIter = false;
3839 } else {
3840 CurTy = GetElementPtrInst::getTypeAtIndex(Ty: CurTy, Idx: (uint64_t)0);
3841 }
3842 // For an array, add the element offset, explicitly scaled.
3843 const SCEV *ElementSize = getSizeOfExpr(IntTy: IntIdxTy, AllocTy: CurTy);
3844 // Getelementptr indices are signed.
3845 IndexExpr = getTruncateOrSignExtend(V: IndexExpr, Ty: IntIdxTy);
3846
3847 // Multiply the index by the element size to compute the element offset.
3848 const SCEV *LocalOffset = getMulExpr(LHS: IndexExpr, RHS: ElementSize, Flags: OffsetWrap);
3849 Offsets.push_back(Elt: LocalOffset);
3850 }
3851 }
3852
3853 // Handle degenerate case of GEP without offsets.
3854 if (Offsets.empty())
3855 return BaseExpr;
3856
3857 // Add the offsets together, assuming nsw if inbounds.
3858 const SCEV *Offset = getAddExpr(Ops&: Offsets, Flags: OffsetWrap);
3859 // Add the base address and the offset. We cannot use the nsw flag, as the
3860 // base address is unsigned. However, if we know that the offset is
3861 // non-negative, we can use nuw.
3862 bool NUW = NW.hasNoUnsignedWrap() ||
3863 (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Offset));
3864 SCEV::NoWrapFlags BaseWrap = NUW ? SCEV::FlagNUW : SCEV::FlagAnyWrap;
3865 const SCEV *GEPExpr = getAddExpr(LHS: BaseExpr, RHS: Offset, Flags: BaseWrap);
3866 assert(BaseExpr->getType() == GEPExpr->getType() &&
3867 "GEP should not change type mid-flight.");
3868 return GEPExpr;
3869}
3870
3871SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
3872 ArrayRef<SCEVUse> Ops) {
3873 FoldingSetNodeID ID;
3874 ID.AddInteger(I: SCEVType);
3875 for (SCEVUse Op : Ops)
3876 ID.AddPointer(Ptr: Op.getOpaqueValue());
3877 FoldingSetInsertToken Token;
3878 return UniqueSCEVs.lookup(ID, Token);
3879}
3880
3881const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
3882 SCEV::NoWrapFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
3883 return getSMaxExpr(LHS: Op, RHS: getNegativeSCEV(V: Op, Flags));
3884}
3885
3886const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
3887 SmallVectorImpl<SCEVUse> &Ops) {
3888 assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
3889 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
3890 if (Ops.size() == 1) return Ops[0];
3891#ifndef NDEBUG
3892 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
3893 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
3894 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
3895 "Operand types don't match!");
3896 assert(Ops[0]->getType()->isPointerTy() ==
3897 Ops[i]->getType()->isPointerTy() &&
3898 "min/max should be consistently pointerish");
3899 }
3900#endif
3901
3902 bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
3903 bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
3904
3905 const SCEV *Folded = constantFoldAndGroupOps(
3906 SE&: *this, LI, DT, Ops,
3907 Fold: [&](const APInt &C1, const APInt &C2) {
3908 switch (Kind) {
3909 case scSMaxExpr:
3910 return APIntOps::smax(A: C1, B: C2);
3911 case scSMinExpr:
3912 return APIntOps::smin(A: C1, B: C2);
3913 case scUMaxExpr:
3914 return APIntOps::umax(A: C1, B: C2);
3915 case scUMinExpr:
3916 return APIntOps::umin(A: C1, B: C2);
3917 default:
3918 llvm_unreachable("Unknown SCEV min/max opcode");
3919 }
3920 },
3921 IsIdentity: [&](const APInt &C) {
3922 // identity
3923 if (IsMax)
3924 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3925 else
3926 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3927 },
3928 IsAbsorber: [&](const APInt &C) {
3929 // absorber
3930 if (IsMax)
3931 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3932 else
3933 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3934 });
3935 if (Folded)
3936 return Folded;
3937
3938 // Check if we have created the same expression before.
3939 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops)) {
3940 return S;
3941 }
3942
3943 // Find the first operation of the same kind
3944 unsigned Idx = 0;
3945 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
3946 ++Idx;
3947
3948 // Check to see if one of the operands is of the same kind. If so, expand its
3949 // operands onto our operand list, and recurse to simplify.
3950 if (Idx < Ops.size()) {
3951 bool DeletedAny = false;
3952 while (Ops[Idx]->getSCEVType() == Kind) {
3953 const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Val&: Ops[Idx]);
3954 Ops.erase(CI: Ops.begin()+Idx);
3955 append_range(C&: Ops, R: SMME->operands());
3956 DeletedAny = true;
3957 }
3958
3959 if (DeletedAny)
3960 return getMinMaxExpr(Kind, Ops);
3961 }
3962
3963 // Okay, check to see if the same value occurs in the operand list twice. If
3964 // so, delete one. Since we sorted the list, these values are required to
3965 // be adjacent.
3966 llvm::CmpInst::Predicate GEPred =
3967 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
3968 llvm::CmpInst::Predicate LEPred =
3969 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
3970 llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
3971 llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
3972 for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
3973 if (Ops[i] == Ops[i + 1] ||
3974 isKnownViaNonRecursiveReasoning(Pred: FirstPred, LHS: Ops[i], RHS: Ops[i + 1])) {
3975 // X op Y op Y --> X op Y
3976 // X op Y --> X, if we know X, Y are ordered appropriately
3977 Ops.erase(CS: Ops.begin() + i + 1, CE: Ops.begin() + i + 2);
3978 --i;
3979 --e;
3980 } else if (isKnownViaNonRecursiveReasoning(Pred: SecondPred, LHS: Ops[i],
3981 RHS: Ops[i + 1])) {
3982 // X op Y --> Y, if we know X, Y are ordered appropriately
3983 Ops.erase(CS: Ops.begin() + i, CE: Ops.begin() + i + 1);
3984 --i;
3985 --e;
3986 }
3987 }
3988
3989 if (Ops.size() == 1) return Ops[0];
3990
3991 assert(!Ops.empty() && "Reduced smax down to nothing!");
3992
3993 // Okay, it looks like we really DO need an expr. Check to see if we
3994 // already have one, otherwise create a new one.
3995 FoldingSetNodeID ID;
3996 ID.AddInteger(I: Kind);
3997 for (SCEVUse Op : Ops)
3998 ID.AddPointer(Ptr: Op.getOpaqueValue());
3999 FoldingSetInsertToken Token;
4000 const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
4001 if (ExistingSCEV)
4002 return ExistingSCEV;
4003 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
4004 llvm::uninitialized_copy(Src&: Ops, Dst: O);
4005 SCEV *S = new (SCEVAllocator)
4006 SCEVMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
4007
4008 UniqueSCEVs.insert(N: S, Token);
4009 S->computeAndSetCanonical(SE&: *this);
4010 registerUser(User: S, Ops);
4011 return S;
4012}
4013
4014namespace {
4015
4016class SCEVSequentialMinMaxDeduplicatingVisitor final
4017 : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
4018 std::optional<const SCEV *>> {
4019 using RetVal = std::optional<const SCEV *>;
4020
4021 ScalarEvolution &SE;
4022 const SCEVTypes RootKind; // Must be a sequential min/max expression.
4023 const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
4024 SmallPtrSet<const SCEV *, 16> SeenOps;
4025
4026 bool canRecurseInto(SCEVTypes Kind) const {
4027 // We can only recurse into the SCEV expression of the same effective type
4028 // as the type of our root SCEV expression.
4029 return RootKind == Kind || NonSequentialRootKind == Kind;
4030 };
4031
4032 RetVal visit(const SCEV *S) {
4033 // Has the whole operand been seen already?
4034 if (!SeenOps.insert(Ptr: S).second)
4035 return std::nullopt;
4036 if (isa<SCEVMinMaxExpr, SCEVSequentialMinMaxExpr>(Val: S)) {
4037 SCEVTypes Kind = S->getSCEVType();
4038
4039 if (!canRecurseInto(Kind))
4040 return S;
4041
4042 auto *NAry = cast<SCEVNAryExpr>(Val: S);
4043 SmallVector<SCEVUse> NewOps;
4044 bool Changed = visit(Kind, OrigOps: NAry->operands(), NewOps);
4045
4046 if (!Changed)
4047 return S;
4048 if (NewOps.empty())
4049 return std::nullopt;
4050
4051 return isa<SCEVSequentialMinMaxExpr>(Val: S)
4052 ? SE.getSequentialMinMaxExpr(Kind, Operands&: NewOps)
4053 : SE.getMinMaxExpr(Kind, Ops&: NewOps);
4054 }
4055 return S;
4056 }
4057
4058public:
4059 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
4060 SCEVTypes RootKind)
4061 : SE(SE), RootKind(RootKind),
4062 NonSequentialRootKind(
4063 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
4064 Ty: RootKind)) {}
4065
4066 bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<SCEVUse> OrigOps,
4067 SmallVectorImpl<SCEVUse> &NewOps) {
4068 bool Changed = false;
4069 SmallVector<SCEVUse> Ops;
4070 Ops.reserve(N: OrigOps.size());
4071
4072 for (const SCEV *Op : OrigOps) {
4073 RetVal NewOp = visit(S: Op);
4074 if (NewOp != Op)
4075 Changed = true;
4076 if (NewOp)
4077 Ops.emplace_back(Args&: *NewOp);
4078 }
4079
4080 if (Changed)
4081 NewOps = std::move(Ops);
4082 return Changed;
4083 }
4084};
4085
4086} // namespace
4087
4088static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind) {
4089 switch (Kind) {
4090 case scConstant:
4091 case scVScale:
4092 case scTruncate:
4093 case scZeroExtend:
4094 case scSignExtend:
4095 case scPtrToAddr:
4096 case scAddExpr:
4097 case scMulExpr:
4098 case scUDivExpr:
4099 case scAddRecExpr:
4100 case scUMaxExpr:
4101 case scSMaxExpr:
4102 case scUMinExpr:
4103 case scSMinExpr:
4104 case scUnknown:
4105 // If any operand is poison, the whole expression is poison.
4106 return true;
4107 case scSequentialUMinExpr:
4108 // FIXME: if the *first* operand is poison, the whole expression is poison.
4109 return false; // Pessimistically, say that it does not propagate poison.
4110 case scCouldNotCompute:
4111 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
4112 }
4113 llvm_unreachable("Unknown SCEV kind!");
4114}
4115
4116namespace {
4117// The only way poison may be introduced in a SCEV expression is from a
4118// poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown,
4119// not SCEVConstant). Notably, nowrap flags in SCEV nodes can *not*
4120// introduce poison -- they encode guaranteed, non-speculated knowledge.
4121//
4122// Additionally, all SCEV nodes propagate poison from inputs to outputs,
4123// with the notable exception of umin_seq, where only poison from the first
4124// operand is (unconditionally) propagated.
4125struct SCEVPoisonCollector {
4126 bool LookThroughMaybePoisonBlocking;
4127 SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
4128 SCEVPoisonCollector(bool LookThroughMaybePoisonBlocking)
4129 : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
4130
4131 bool follow(const SCEV *S) {
4132 if (!LookThroughMaybePoisonBlocking &&
4133 !scevUnconditionallyPropagatesPoisonFromOperands(Kind: S->getSCEVType()))
4134 return false;
4135
4136 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
4137 if (!isGuaranteedNotToBePoison(V: SU->getValue()))
4138 MaybePoison.insert(Ptr: SU);
4139 }
4140 return true;
4141 }
4142 bool isDone() const { return false; }
4143};
4144} // namespace
4145
4146/// Return true if V is poison given that AssumedPoison is already poison.
4147static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) {
4148 // First collect all SCEVs that might result in AssumedPoison to be poison.
4149 // We need to look through potentially poison-blocking operations here,
4150 // because we want to find all SCEVs that *might* result in poison, not only
4151 // those that are *required* to.
4152 SCEVPoisonCollector PC1(/* LookThroughMaybePoisonBlocking */ true);
4153 visitAll(Root: AssumedPoison, Visitor&: PC1);
4154
4155 // AssumedPoison is never poison. As the assumption is false, the implication
4156 // is true. Don't bother walking the other SCEV in this case.
4157 if (PC1.MaybePoison.empty())
4158 return true;
4159
4160 // Collect all SCEVs in S that, if poison, *will* result in S being poison
4161 // as well. We cannot look through potentially poison-blocking operations
4162 // here, as their arguments only *may* make the result poison.
4163 SCEVPoisonCollector PC2(/* LookThroughMaybePoisonBlocking */ false);
4164 visitAll(Root: S, Visitor&: PC2);
4165
4166 // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison,
4167 // it will also make S poison by being part of PC2.MaybePoison.
4168 return llvm::set_is_subset(S1: PC1.MaybePoison, S2: PC2.MaybePoison);
4169}
4170
4171void ScalarEvolution::getPoisonGeneratingValues(
4172 SmallPtrSetImpl<const Value *> &Result, const SCEV *S) {
4173 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ false);
4174 visitAll(Root: S, Visitor&: PC);
4175 for (const SCEVUnknown *SU : PC.MaybePoison)
4176 Result.insert(Ptr: SU->getValue());
4177}
4178
4179bool ScalarEvolution::canReuseInstruction(
4180 const SCEV *S, Instruction *I,
4181 SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts) {
4182 // If the instruction cannot be poison, it's always safe to reuse.
4183 if (programUndefinedIfPoison(Inst: I))
4184 return true;
4185
4186 // Otherwise, it is possible that I is more poisonous that S. Collect the
4187 // poison-contributors of S, and then check whether I has any additional
4188 // poison-contributors. Poison that is contributed through poison-generating
4189 // flags is handled by dropping those flags instead.
4190 SmallPtrSet<const Value *, 8> PoisonVals;
4191 getPoisonGeneratingValues(Result&: PoisonVals, S);
4192
4193 SmallVector<Value *> Worklist;
4194 SmallPtrSet<Value *, 8> Visited;
4195 Worklist.push_back(Elt: I);
4196 while (!Worklist.empty()) {
4197 Value *V = Worklist.pop_back_val();
4198 if (!Visited.insert(Ptr: V).second)
4199 continue;
4200
4201 // Avoid walking large instruction graphs.
4202 if (Visited.size() > 16)
4203 return false;
4204
4205 // Either the value can't be poison, or the S would also be poison if it
4206 // is.
4207 if (PoisonVals.contains(Ptr: V) || ::isGuaranteedNotToBePoison(V))
4208 continue;
4209
4210 auto *I = dyn_cast<Instruction>(Val: V);
4211 if (!I)
4212 return false;
4213
4214 // Disjoint or instructions are interpreted as adds by SCEV. However, we
4215 // can't replace an arbitrary add with disjoint or, even if we drop the
4216 // flag. We would need to convert the or into an add.
4217 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: I))
4218 if (PDI->isDisjoint())
4219 return false;
4220
4221 // FIXME: Ignore vscale, even though it technically could be poison. Do this
4222 // because SCEV currently assumes it can't be poison. Remove this special
4223 // case once we proper model when vscale can be poison.
4224 if (auto *II = dyn_cast<IntrinsicInst>(Val: I);
4225 II && II->getIntrinsicID() == Intrinsic::vscale)
4226 continue;
4227
4228 if (canCreatePoison(Op: cast<Operator>(Val: I), /*ConsiderFlagsAndMetadata*/ false))
4229 return false;
4230
4231 // If the instruction can't create poison, we can recurse to its operands.
4232 if (I->hasPoisonGeneratingAnnotations())
4233 DropPoisonGeneratingInsts.push_back(Elt: I);
4234
4235 llvm::append_range(C&: Worklist, R: I->operands());
4236 }
4237 return true;
4238}
4239
4240const SCEV *
4241ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
4242 SmallVectorImpl<SCEVUse> &Ops) {
4243 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
4244 "Not a SCEVSequentialMinMaxExpr!");
4245 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4246 if (Ops.size() == 1)
4247 return Ops[0];
4248#ifndef NDEBUG
4249 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4250 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4251 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4252 "Operand types don't match!");
4253 assert(Ops[0]->getType()->isPointerTy() ==
4254 Ops[i]->getType()->isPointerTy() &&
4255 "min/max should be consistently pointerish");
4256 }
4257#endif
4258
4259 // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
4260 // so we can *NOT* do any kind of sorting of the expressions!
4261
4262 // Check if we have created the same expression before.
4263 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops))
4264 return S;
4265
4266 // FIXME: there are *some* simplifications that we can do here.
4267
4268 // Keep only the first instance of an operand.
4269 {
4270 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
4271 bool Changed = Deduplicator.visit(Kind, OrigOps: Ops, NewOps&: Ops);
4272 if (Changed)
4273 return getSequentialMinMaxExpr(Kind, Ops);
4274 }
4275
4276 // Check to see if one of the operands is of the same kind. If so, expand its
4277 // operands onto our operand list, and recurse to simplify.
4278 {
4279 unsigned Idx = 0;
4280 bool DeletedAny = false;
4281 while (Idx < Ops.size()) {
4282 if (Ops[Idx]->getSCEVType() != Kind) {
4283 ++Idx;
4284 continue;
4285 }
4286 const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Val&: Ops[Idx]);
4287 Ops.erase(CI: Ops.begin() + Idx);
4288 Ops.insert(I: Ops.begin() + Idx, From: SMME->operands().begin(),
4289 To: SMME->operands().end());
4290 DeletedAny = true;
4291 }
4292
4293 if (DeletedAny)
4294 return getSequentialMinMaxExpr(Kind, Ops);
4295 }
4296
4297 const SCEV *SaturationPoint;
4298 ICmpInst::Predicate Pred;
4299 switch (Kind) {
4300 case scSequentialUMinExpr:
4301 SaturationPoint = getZero(Ty: Ops[0]->getType());
4302 Pred = ICmpInst::ICMP_ULE;
4303 break;
4304 default:
4305 llvm_unreachable("Not a sequential min/max type.");
4306 }
4307
4308 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4309 if (!isGuaranteedNotToCauseUB(Op: Ops[i]))
4310 continue;
4311 // We can replace %x umin_seq %y with %x umin %y if either:
4312 // * %y being poison implies %x is also poison.
4313 // * %x cannot be the saturating value (e.g. zero for umin).
4314 if (::impliesPoison(AssumedPoison: Ops[i], S: Ops[i - 1]) ||
4315 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_NE, LHS: Ops[i - 1],
4316 RHS: SaturationPoint)) {
4317 SmallVector<SCEVUse, 2> SeqOps = {Ops[i - 1], Ops[i]};
4318 Ops[i - 1] = getMinMaxExpr(
4319 Kind: SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Ty: Kind),
4320 Ops&: SeqOps);
4321 Ops.erase(CI: Ops.begin() + i);
4322 return getSequentialMinMaxExpr(Kind, Ops);
4323 }
4324 // Fold %x umin_seq %y to %x if %x ule %y.
4325 // TODO: We might be able to prove the predicate for a later operand.
4326 if (isKnownViaNonRecursiveReasoning(Pred, LHS: Ops[i - 1], RHS: Ops[i])) {
4327 Ops.erase(CI: Ops.begin() + i);
4328 return getSequentialMinMaxExpr(Kind, Ops);
4329 }
4330 }
4331
4332 // Okay, it looks like we really DO need an expr. Check to see if we
4333 // already have one, otherwise create a new one.
4334 FoldingSetNodeID ID;
4335 ID.AddInteger(I: Kind);
4336 for (SCEVUse Op : Ops)
4337 ID.AddPointer(Ptr: Op.getOpaqueValue());
4338 FoldingSetInsertToken Token;
4339 const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
4340 if (ExistingSCEV)
4341 return ExistingSCEV;
4342
4343 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
4344 llvm::uninitialized_copy(Src&: Ops, Dst: O);
4345 SCEV *S = new (SCEVAllocator)
4346 SCEVSequentialMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
4347
4348 UniqueSCEVs.insert(N: S, Token);
4349 S->computeAndSetCanonical(SE&: *this);
4350 registerUser(User: S, Ops);
4351 return S;
4352}
4353
4354const SCEV *ScalarEvolution::getSMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4355 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4356 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4357}
4358
4359const SCEV *ScalarEvolution::getSMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4360 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4361}
4362
4363const SCEV *ScalarEvolution::getUMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4364 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4365 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4366}
4367
4368const SCEV *ScalarEvolution::getUMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4369 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4370}
4371
4372const SCEV *ScalarEvolution::getSMinExpr(SCEVUse LHS, SCEVUse RHS) {
4373 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4374 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4375}
4376
4377const SCEV *ScalarEvolution::getSMinExpr(SmallVectorImpl<SCEVUse> &Ops) {
4378 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4379}
4380
4381const SCEV *ScalarEvolution::getUMinExpr(SCEVUse LHS, SCEVUse RHS,
4382 bool Sequential) {
4383 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4384 return getUMinExpr(Operands&: Ops, Sequential);
4385}
4386
4387const SCEV *ScalarEvolution::getUMinExpr(SmallVectorImpl<SCEVUse> &Ops,
4388 bool Sequential) {
4389 return Sequential ? getSequentialMinMaxExpr(Kind: scSequentialUMinExpr, Ops)
4390 : getMinMaxExpr(Kind: scUMinExpr, Ops);
4391}
4392
4393const SCEV *
4394ScalarEvolution::getSizeOfExpr(Type *IntTy, TypeSize Size) {
4395 const SCEV *Res = getConstant(Ty: IntTy, V: Size.getKnownMinValue());
4396 if (Size.isScalable())
4397 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty: IntTy));
4398 return Res;
4399}
4400
4401const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
4402 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeAllocSize(Ty: AllocTy));
4403}
4404
4405const SCEV *ScalarEvolution::getStoreSizeOfExpr(Type *IntTy, Type *StoreTy) {
4406 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeStoreSize(Ty: StoreTy));
4407}
4408
4409const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
4410 StructType *STy,
4411 unsigned FieldNo) {
4412 // We can bypass creating a target-independent constant expression and then
4413 // folding it back into a ConstantInt. This is just a compile-time
4414 // optimization.
4415 const StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
4416 assert(!SL->getSizeInBits().isScalable() &&
4417 "Cannot get offset for structure containing scalable vector types");
4418 return getConstant(Ty: IntTy, V: SL->getElementOffset(Idx: FieldNo));
4419}
4420
4421const SCEV *ScalarEvolution::getUnknown(Value *V) {
4422 // Don't attempt to do anything other than create a SCEVUnknown object
4423 // here. createSCEV only calls getUnknown after checking for all other
4424 // interesting possibilities, and any other code that calls getUnknown
4425 // is doing so in order to hide a value from SCEV canonicalization.
4426
4427 FoldingSetNodeID ID;
4428 ID.AddInteger(I: scUnknown);
4429 ID.AddPointer(Ptr: V);
4430 FoldingSetInsertToken Token;
4431 if (SCEV *S = UniqueSCEVs.lookup(ID, Token)) {
4432 assert(cast<SCEVUnknown>(S)->getValue() == V &&
4433 "Stale SCEVUnknown in uniquing map!");
4434 return S;
4435 }
4436 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(Allocator&: SCEVAllocator), V, this,
4437 FirstUnknown);
4438 FirstUnknown = cast<SCEVUnknown>(Val: S);
4439 UniqueSCEVs.insert(N: S, Token);
4440 S->computeAndSetCanonical(SE&: *this);
4441 return S;
4442}
4443
4444//===----------------------------------------------------------------------===//
4445// Basic SCEV Analysis and PHI Idiom Recognition Code
4446//
4447
4448/// Test if values of the given type are analyzable within the SCEV
4449/// framework. This primarily includes integer types, and it can optionally
4450/// include pointer types if the ScalarEvolution class has access to
4451/// target-specific information.
4452bool ScalarEvolution::isSCEVable(Type *Ty) const {
4453 // Integers and pointers are always SCEVable.
4454 return Ty->isIntOrPtrTy();
4455}
4456
4457/// Return the size in bits of the specified type, for which isSCEVable must
4458/// return true.
4459uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
4460 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4461 if (Ty->isPointerTy())
4462 return getDataLayout().getIndexTypeSizeInBits(Ty);
4463 return getDataLayout().getTypeSizeInBits(Ty);
4464}
4465
4466/// Return a type with the same bitwidth as the given type and which represents
4467/// how SCEV will treat the given type, for which isSCEVable must return
4468/// true. For pointer types, this is the pointer index sized integer type.
4469Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
4470 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4471
4472 if (Ty->isIntegerTy())
4473 return Ty;
4474
4475 // The only other support type is pointer.
4476 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
4477 return getDataLayout().getIndexType(PtrTy: Ty);
4478}
4479
4480Type *ScalarEvolution::getWiderType(Type *T1, Type *T2) const {
4481 return getTypeSizeInBits(Ty: T1) >= getTypeSizeInBits(Ty: T2) ? T1 : T2;
4482}
4483
4484bool ScalarEvolution::instructionCouldExistWithOperands(const SCEV *A,
4485 const SCEV *B) {
4486 /// For a valid use point to exist, the defining scope of one operand
4487 /// must dominate the other.
4488 bool PreciseA, PreciseB;
4489 auto *ScopeA = getDefiningScopeBound(Ops: {A}, Precise&: PreciseA);
4490 auto *ScopeB = getDefiningScopeBound(Ops: {B}, Precise&: PreciseB);
4491 if (!PreciseA || !PreciseB)
4492 // Can't tell.
4493 return false;
4494 return (ScopeA == ScopeB) || DT.dominates(Def: ScopeA, User: ScopeB) ||
4495 DT.dominates(Def: ScopeB, User: ScopeA);
4496}
4497
4498const SCEV *ScalarEvolution::getCouldNotCompute() {
4499 return CouldNotCompute.get();
4500}
4501
4502bool ScalarEvolution::checkValidity(const SCEV *S) const {
4503 bool ContainsNulls = SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
4504 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
4505 return SU && SU->getValue() == nullptr;
4506 });
4507
4508 return !ContainsNulls;
4509}
4510
4511bool ScalarEvolution::containsAddRecurrence(const SCEV *S) {
4512 HasRecMapType::iterator I = HasRecMap.find(Val: S);
4513 if (I != HasRecMap.end())
4514 return I->second;
4515
4516 bool FoundAddRec =
4517 SCEVExprContains(Root: S, Pred: [](const SCEV *S) { return isa<SCEVAddRecExpr>(Val: S); });
4518 HasRecMap.insert(KV: {S, FoundAddRec});
4519 return FoundAddRec;
4520}
4521
4522/// Return the ValueOffsetPair set for \p S. \p S can be represented
4523/// by the value and offset from any ValueOffsetPair in the set.
4524ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) {
4525 ExprValueMapType::iterator SI = ExprValueMap.find_as(Val: S);
4526 if (SI == ExprValueMap.end())
4527 return {};
4528 return SI->second.getArrayRef();
4529}
4530
4531/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
4532/// cannot be used separately. eraseValueFromMap should be used to remove
4533/// V from ValueExprMap and ExprValueMap at the same time.
4534void ScalarEvolution::eraseValueFromMap(Value *V) {
4535 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4536 if (I != ValueExprMap.end()) {
4537 auto EVIt = ExprValueMap.find(Val: I->second);
4538 bool Removed = EVIt->second.remove(X: V);
4539 (void) Removed;
4540 assert(Removed && "Value not in ExprValueMap?");
4541 ValueExprMap.erase(I);
4542 }
4543}
4544
4545void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
4546 // A recursive query may have already computed the SCEV. It should be
4547 // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
4548 // inferred nowrap flags.
4549 auto It = ValueExprMap.find_as(Val: V);
4550 if (It == ValueExprMap.end()) {
4551 ValueExprMap.insert(KV: {SCEVCallbackVH(V, this), S});
4552 ExprValueMap[S].insert(X: V);
4553 }
4554}
4555
4556/// Return an existing SCEV if it exists, otherwise analyze the expression and
4557/// create a new one.
4558const SCEV *ScalarEvolution::getSCEV(Value *V) {
4559 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4560
4561 if (const SCEV *S = getExistingSCEV(V))
4562 return S;
4563 return createSCEVIter(V);
4564}
4565
4566const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
4567 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4568
4569 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4570 if (I != ValueExprMap.end()) {
4571 const SCEV *S = I->second;
4572 assert(checkValidity(S) &&
4573 "existing SCEV has not been properly invalidated");
4574 return S;
4575 }
4576 return nullptr;
4577}
4578
4579/// Return a SCEV corresponding to -V = -1*V
4580const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V,
4581 SCEV::NoWrapFlags Flags) {
4582 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4583 return getConstant(
4584 V: cast<ConstantInt>(Val: ConstantExpr::getNeg(C: VC->getValue())));
4585
4586 Type *Ty = V->getType();
4587 Ty = getEffectiveSCEVType(Ty);
4588 return getMulExpr(LHS: V, RHS: getMinusOne(Ty), Flags);
4589}
4590
4591/// If Expr computes ~A, return A else return nullptr
4592static const SCEV *MatchNotExpr(const SCEV *Expr) {
4593 const SCEV *MulOp;
4594 if (match(S: Expr, P: m_scev_Add(Op0: m_scev_AllOnes(),
4595 Op1: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: MulOp)))))
4596 return MulOp;
4597 return nullptr;
4598}
4599
4600/// Return a SCEV corresponding to ~V = -1-V
4601const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
4602 assert(!V->getType()->isPointerTy() && "Can't negate pointer");
4603
4604 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4605 return getConstant(
4606 V: cast<ConstantInt>(Val: ConstantExpr::getNot(C: VC->getValue())));
4607
4608 // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
4609 if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(Val: V)) {
4610 auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
4611 SmallVector<SCEVUse, 2> MatchedOperands;
4612 for (const SCEV *Operand : MME->operands()) {
4613 const SCEV *Matched = MatchNotExpr(Expr: Operand);
4614 if (!Matched)
4615 return (const SCEV *)nullptr;
4616 MatchedOperands.push_back(Elt: Matched);
4617 }
4618 return getMinMaxExpr(Kind: SCEVMinMaxExpr::negate(T: MME->getSCEVType()),
4619 Ops&: MatchedOperands);
4620 };
4621 if (const SCEV *Replaced = MatchMinMaxNegation(MME))
4622 return Replaced;
4623 }
4624
4625 Type *Ty = V->getType();
4626 Ty = getEffectiveSCEVType(Ty);
4627 return getMinusSCEV(LHS: getMinusOne(Ty), RHS: V);
4628}
4629
4630const SCEV *ScalarEvolution::removePointerBase(const SCEV *P) {
4631 assert(P->getType()->isPointerTy());
4632
4633 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: P)) {
4634 // The base of an AddRec is the first operand.
4635 SmallVector<SCEVUse> Ops{AddRec->operands()};
4636 Ops[0] = removePointerBase(P: Ops[0]);
4637 // Don't try to transfer nowrap flags for now. We could in some cases
4638 // (for example, if pointer operand of the AddRec is a SCEVUnknown).
4639 return getAddRecExpr(Operands&: Ops, L: AddRec->getLoop(), NWFlags: SCEV::FlagAnyWrap);
4640 }
4641 if (auto *Add = dyn_cast<SCEVAddExpr>(Val: P)) {
4642 // The base of an Add is the pointer operand.
4643 SmallVector<SCEVUse> Ops{Add->operands()};
4644 SCEVUse *PtrOp = nullptr;
4645 for (SCEVUse &AddOp : Ops) {
4646 if (AddOp->getType()->isPointerTy()) {
4647 assert(!PtrOp && "Cannot have multiple pointer ops");
4648 PtrOp = &AddOp;
4649 }
4650 }
4651 *PtrOp = removePointerBase(P: *PtrOp);
4652 // Don't try to transfer nowrap flags for now. We could in some cases
4653 // (for example, if the pointer operand of the Add is a SCEVUnknown).
4654 return getAddExpr(Ops);
4655 }
4656 // Any other expression must be a pointer base.
4657 return getZero(Ty: P->getType());
4658}
4659
4660const SCEV *ScalarEvolution::getMinusSCEV(SCEVUse LHS, SCEVUse RHS,
4661 SCEV::NoWrapFlags Flags,
4662 unsigned Depth) {
4663 // Fast path: X - X --> 0.
4664 if (LHS == RHS)
4665 return getZero(Ty: LHS->getType());
4666
4667 // If we subtract two pointers with different pointer bases, bail.
4668 // Eventually, we're going to add an assertion to getMulExpr that we
4669 // can't multiply by a pointer.
4670 if (RHS->getType()->isPointerTy()) {
4671 if (!LHS->getType()->isPointerTy() ||
4672 getPointerBase(V: LHS) != getPointerBase(V: RHS))
4673 return getCouldNotCompute();
4674 LHS = removePointerBase(P: LHS);
4675 RHS = removePointerBase(P: RHS);
4676 }
4677
4678 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
4679 // makes it so that we cannot make much use of NUW.
4680 auto AddFlags = SCEV::FlagAnyWrap;
4681 const bool RHSIsNotMinSigned =
4682 !getSignedRangeMin(S: RHS).isMinSignedValue();
4683 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW)) {
4684 // Let M be the minimum representable signed value. Then (-1)*RHS
4685 // signed-wraps if and only if RHS is M. That can happen even for
4686 // a NSW subtraction because e.g. (-1)*M signed-wraps even though
4687 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
4688 // (-1)*RHS, we need to prove that RHS != M.
4689 //
4690 // If LHS is non-negative and we know that LHS - RHS does not
4691 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
4692 // either by proving that RHS > M or that LHS >= 0.
4693 if (RHSIsNotMinSigned || isKnownNonNegative(S: LHS)) {
4694 AddFlags = SCEV::FlagNSW;
4695 }
4696 }
4697
4698 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
4699 // RHS is NSW and LHS >= 0.
4700 //
4701 // The difficulty here is that the NSW flag may have been proven
4702 // relative to a loop that is to be found in a recurrence in LHS and
4703 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
4704 // larger scope than intended.
4705 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
4706
4707 return getAddExpr(LHS, RHS: getNegativeSCEV(V: RHS, Flags: NegFlags), Flags: AddFlags, Depth);
4708}
4709
4710const SCEV *ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty,
4711 unsigned Depth) {
4712 Type *SrcTy = V->getType();
4713 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4714 "Cannot truncate or zero extend with non-integer arguments!");
4715 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4716 return V; // No conversion
4717 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4718 return getTruncateExpr(Op: V, Ty, Depth);
4719 return getZeroExtendExpr(Op: V, Ty, Depth);
4720}
4721
4722const SCEV *ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, Type *Ty,
4723 unsigned Depth) {
4724 Type *SrcTy = V->getType();
4725 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4726 "Cannot truncate or zero extend with non-integer arguments!");
4727 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4728 return V; // No conversion
4729 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4730 return getTruncateExpr(Op: V, Ty, Depth);
4731 return getSignExtendExpr(Op: V, Ty, Depth);
4732}
4733
4734const SCEV *ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
4735 Type *SrcTy = V->getType();
4736 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4737 "Cannot noop or zero extend with non-integer arguments!");
4738 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4739 "getNoopOrZeroExtend cannot truncate!");
4740 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4741 return V; // No conversion
4742 return getZeroExtendExpr(Op: V, Ty);
4743}
4744
4745const SCEV *ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
4746 Type *SrcTy = V->getType();
4747 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4748 "Cannot noop or sign extend with non-integer arguments!");
4749 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4750 "getNoopOrSignExtend cannot truncate!");
4751 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4752 return V; // No conversion
4753 return getSignExtendExpr(Op: V, Ty);
4754}
4755
4756const SCEV *ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
4757 Type *SrcTy = V->getType();
4758 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4759 "Cannot noop or any extend with non-integer arguments!");
4760 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4761 "getNoopOrAnyExtend cannot truncate!");
4762 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4763 return V; // No conversion
4764 return getAnyExtendExpr(Op: V, Ty);
4765}
4766
4767const SCEV *ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
4768 Type *SrcTy = V->getType();
4769 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4770 "Cannot truncate or noop with non-integer arguments!");
4771 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
4772 "getTruncateOrNoop cannot extend!");
4773 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4774 return V; // No conversion
4775 return getTruncateExpr(Op: V, Ty);
4776}
4777
4778const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
4779 const SCEV *RHS) {
4780 const SCEV *PromotedLHS = LHS;
4781 const SCEV *PromotedRHS = RHS;
4782
4783 if (getTypeSizeInBits(Ty: LHS->getType()) > getTypeSizeInBits(Ty: RHS->getType()))
4784 PromotedRHS = getZeroExtendExpr(Op: RHS, Ty: LHS->getType());
4785 else
4786 PromotedLHS = getNoopOrZeroExtend(V: LHS, Ty: RHS->getType());
4787
4788 return getUMaxExpr(LHS: PromotedLHS, RHS: PromotedRHS);
4789}
4790
4791const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
4792 const SCEV *RHS,
4793 bool Sequential) {
4794 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4795 return getUMinFromMismatchedTypes(Ops, Sequential);
4796}
4797
4798const SCEV *
4799ScalarEvolution::getUMinFromMismatchedTypes(SmallVectorImpl<SCEVUse> &Ops,
4800 bool Sequential) {
4801 assert(!Ops.empty() && "At least one operand must be!");
4802 // Trivial case.
4803 if (Ops.size() == 1)
4804 return Ops[0];
4805
4806 // Find the max type first.
4807 Type *MaxType = nullptr;
4808 for (SCEVUse S : Ops)
4809 if (MaxType)
4810 MaxType = getWiderType(T1: MaxType, T2: S->getType());
4811 else
4812 MaxType = S->getType();
4813 assert(MaxType && "Failed to find maximum type!");
4814
4815 // Extend all ops to max type.
4816 SmallVector<SCEVUse, 2> PromotedOps;
4817 for (SCEVUse S : Ops)
4818 PromotedOps.push_back(Elt: getNoopOrZeroExtend(V: S, Ty: MaxType));
4819
4820 // Generate umin.
4821 return getUMinExpr(Ops&: PromotedOps, Sequential);
4822}
4823
4824const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
4825 // A pointer operand may evaluate to a nonpointer expression, such as null.
4826 if (!V->getType()->isPointerTy())
4827 return V;
4828
4829 while (true) {
4830 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: V)) {
4831 V = AddRec->getStart();
4832 } else if (auto *Add = dyn_cast<SCEVAddExpr>(Val: V)) {
4833 const SCEV *PtrOp = nullptr;
4834 for (const SCEV *AddOp : Add->operands()) {
4835 if (AddOp->getType()->isPointerTy()) {
4836 assert(!PtrOp && "Cannot have multiple pointer ops");
4837 PtrOp = AddOp;
4838 }
4839 }
4840 assert(PtrOp && "Must have pointer op");
4841 V = PtrOp;
4842 } else // Not something we can look further into.
4843 return V;
4844 }
4845}
4846
4847/// Push users of the given Instruction onto the given Worklist.
4848static void PushDefUseChildren(Instruction *I,
4849 SmallVectorImpl<Instruction *> &Worklist,
4850 SmallPtrSetImpl<Instruction *> &Visited) {
4851 // Push the def-use children onto the Worklist stack.
4852 for (User *U : I->users()) {
4853 auto *UserInsn = cast<Instruction>(Val: U);
4854 if (Visited.insert(Ptr: UserInsn).second)
4855 Worklist.push_back(Elt: UserInsn);
4856 }
4857}
4858
4859namespace {
4860
4861/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
4862/// expression in case its Loop is L. If it is not L then
4863/// if IgnoreOtherLoops is true then use AddRec itself
4864/// otherwise rewrite cannot be done.
4865/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4866class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
4867public:
4868 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
4869 bool IgnoreOtherLoops = true) {
4870 SCEVInitRewriter Rewriter(L, SE);
4871 const SCEV *Result = Rewriter.visit(S);
4872 if (Rewriter.hasSeenLoopVariantSCEVUnknown())
4873 return SE.getCouldNotCompute();
4874 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
4875 ? SE.getCouldNotCompute()
4876 : Result;
4877 }
4878
4879 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4880 if (!SE.isLoopInvariant(S: Expr, L))
4881 SeenLoopVariantSCEVUnknown = true;
4882 return Expr;
4883 }
4884
4885 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4886 // Only re-write AddRecExprs for this loop.
4887 if (Expr->getLoop() == L)
4888 return Expr->getStart();
4889 SeenOtherLoops = true;
4890 return Expr;
4891 }
4892
4893 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4894
4895 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4896
4897private:
4898 explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
4899 : SCEVRewriteVisitor(SE), L(L) {}
4900
4901 const Loop *L;
4902 bool SeenLoopVariantSCEVUnknown = false;
4903 bool SeenOtherLoops = false;
4904};
4905
4906/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
4907/// increment expression in case its Loop is L. If it is not L then
4908/// use AddRec itself.
4909/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4910class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
4911public:
4912 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
4913 SCEVPostIncRewriter Rewriter(L, SE);
4914 const SCEV *Result = Rewriter.visit(S);
4915 return Rewriter.hasSeenLoopVariantSCEVUnknown()
4916 ? SE.getCouldNotCompute()
4917 : Result;
4918 }
4919
4920 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4921 if (!SE.isLoopInvariant(S: Expr, L))
4922 SeenLoopVariantSCEVUnknown = true;
4923 return Expr;
4924 }
4925
4926 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4927 // Only re-write AddRecExprs for this loop.
4928 if (Expr->getLoop() == L)
4929 return Expr->getPostIncExpr(SE);
4930 SeenOtherLoops = true;
4931 return Expr;
4932 }
4933
4934 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4935
4936 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4937
4938private:
4939 explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
4940 : SCEVRewriteVisitor(SE), L(L) {}
4941
4942 const Loop *L;
4943 bool SeenLoopVariantSCEVUnknown = false;
4944 bool SeenOtherLoops = false;
4945};
4946
4947/// This class evaluates the compare condition by matching it against the
4948/// condition of loop latch. If there is a match we assume a true value
4949/// for the condition while building SCEV nodes.
4950class SCEVBackedgeConditionFolder
4951 : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
4952public:
4953 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4954 ScalarEvolution &SE) {
4955 bool IsPosBECond = false;
4956 Value *BECond = nullptr;
4957 if (BasicBlock *Latch = L->getLoopLatch()) {
4958 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Latch->getTerminator())) {
4959 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
4960 "Both outgoing branches should not target same header!");
4961 BECond = BI->getCondition();
4962 IsPosBECond = BI->getSuccessor(i: 0) == L->getHeader();
4963 } else {
4964 return S;
4965 }
4966 }
4967 SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
4968 return Rewriter.visit(S);
4969 }
4970
4971 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4972 const SCEV *Result = Expr;
4973 bool InvariantF = SE.isLoopInvariant(S: Expr, L);
4974
4975 if (!InvariantF) {
4976 Instruction *I = cast<Instruction>(Val: Expr->getValue());
4977 switch (I->getOpcode()) {
4978 case Instruction::Select: {
4979 SelectInst *SI = cast<SelectInst>(Val: I);
4980 std::optional<const SCEV *> Res =
4981 compareWithBackedgeCondition(IC: SI->getCondition());
4982 if (Res) {
4983 bool IsOne = cast<SCEVConstant>(Val: *Res)->getValue()->isOne();
4984 Result = SE.getSCEV(V: IsOne ? SI->getTrueValue() : SI->getFalseValue());
4985 }
4986 break;
4987 }
4988 default: {
4989 std::optional<const SCEV *> Res = compareWithBackedgeCondition(IC: I);
4990 if (Res)
4991 Result = *Res;
4992 break;
4993 }
4994 }
4995 }
4996 return Result;
4997 }
4998
4999private:
5000 explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
5001 bool IsPosBECond, ScalarEvolution &SE)
5002 : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
5003 IsPositiveBECond(IsPosBECond) {}
5004
5005 std::optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
5006
5007 const Loop *L;
5008 /// Loop back condition.
5009 Value *BackedgeCond = nullptr;
5010 /// Set to true if loop back is on positive branch condition.
5011 bool IsPositiveBECond;
5012};
5013
5014std::optional<const SCEV *>
5015SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
5016
5017 // If value matches the backedge condition for loop latch,
5018 // then return a constant evolution node based on loopback
5019 // branch taken.
5020 if (BackedgeCond == IC)
5021 return IsPositiveBECond ? SE.getOne(Ty: Type::getInt1Ty(C&: SE.getContext()))
5022 : SE.getZero(Ty: Type::getInt1Ty(C&: SE.getContext()));
5023 return std::nullopt;
5024}
5025
5026class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
5027public:
5028 static const SCEV *rewrite(const SCEV *S, const Loop *L,
5029 ScalarEvolution &SE) {
5030 SCEVShiftRewriter Rewriter(L, SE);
5031 const SCEV *Result = Rewriter.visit(S);
5032 return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
5033 }
5034
5035 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5036 // Only allow AddRecExprs for this loop.
5037 if (!SE.isLoopInvariant(S: Expr, L))
5038 Valid = false;
5039 return Expr;
5040 }
5041
5042 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5043 if (Expr->getLoop() == L && Expr->isAffine())
5044 return SE.getMinusSCEV(LHS: Expr, RHS: Expr->getStepRecurrence(SE));
5045 Valid = false;
5046 return Expr;
5047 }
5048
5049 bool isValid() { return Valid; }
5050
5051private:
5052 explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
5053 : SCEVRewriteVisitor(SE), L(L) {}
5054
5055 const Loop *L;
5056 bool Valid = true;
5057};
5058
5059} // end anonymous namespace
5060
5061void ScalarEvolution::inferNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
5062 if (!AR->isAffine())
5063 return;
5064
5065 // Force computation of ranges, which will also perform range-based flag
5066 // inference.
5067 if (!AR->hasNoSignedWrap())
5068 (void)getSignedRange(S: AR);
5069
5070 if (!AR->hasNoUnsignedWrap())
5071 (void)getUnsignedRange(S: AR);
5072
5073 if (!AR->hasNoSelfWrap()) {
5074 const SCEV *BECount = getConstantMaxBackedgeTakenCount(L: AR->getLoop());
5075 if (const SCEVConstant *BECountMax = dyn_cast<SCEVConstant>(Val: BECount)) {
5076 ConstantRange StepCR = getSignedRange(S: AR->getStepRecurrence(SE&: *this));
5077 const APInt &BECountAP = BECountMax->getAPInt();
5078 unsigned NoOverflowBitWidth =
5079 BECountAP.getActiveBits() + StepCR.getMinSignedBits();
5080 if (NoOverflowBitWidth <= getTypeSizeInBits(Ty: AR->getType()))
5081 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
5082 }
5083 }
5084}
5085
5086SCEV::NoWrapFlags
5087ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5088 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
5089
5090 if (AR->hasNoSignedWrap())
5091 return Result;
5092
5093 if (!AR->isAffine())
5094 return Result;
5095
5096 // This function can be expensive, only try to prove NSW once per AddRec.
5097 if (!SignedWrapViaInductionTried.insert(Ptr: AR).second)
5098 return Result;
5099
5100 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5101 const Loop *L = AR->getLoop();
5102
5103 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5104 // Note that this serves two purposes: It filters out loops that are
5105 // simply not analyzable, and it covers the case where this code is
5106 // being called from within backedge-taken count analysis, such that
5107 // attempting to ask for the backedge-taken count would likely result
5108 // in infinite recursion. In the later case, the analysis code will
5109 // cope with a conservative value, and it will take care to purge
5110 // that value once it has finished.
5111 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5112
5113 // Normally, in the cases we can prove no-overflow via a
5114 // backedge guarding condition, we can also compute a backedge
5115 // taken count for the loop. The exceptions are assumptions and
5116 // guards present in the loop -- SCEV is not great at exploiting
5117 // these to compute max backedge taken counts, but can still use
5118 // these to prove lack of overflow. Use this fact to avoid
5119 // doing extra work that may not pay off.
5120
5121 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5122 AC.assumptions().empty())
5123 return Result;
5124
5125 // If the backedge is guarded by a comparison with the pre-inc value the
5126 // addrec is safe. Also, if the entry is guarded by a comparison with the
5127 // start value and the backedge is guarded by a comparison with the post-inc
5128 // value, the addrec is safe.
5129 ICmpInst::Predicate Pred;
5130 const SCEV *OverflowLimit =
5131 getSignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
5132 if (OverflowLimit &&
5133 (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
5134 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))) {
5135 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNSW);
5136 }
5137 return Result;
5138}
5139SCEV::NoWrapFlags
5140ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5141 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
5142
5143 if (AR->hasNoUnsignedWrap())
5144 return Result;
5145
5146 if (!AR->isAffine())
5147 return Result;
5148
5149 // This function can be expensive, only try to prove NUW once per AddRec.
5150 if (!UnsignedWrapViaInductionTried.insert(Ptr: AR).second)
5151 return Result;
5152
5153 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5154 const Loop *L = AR->getLoop();
5155
5156 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5157 // Note that this serves two purposes: It filters out loops that are
5158 // simply not analyzable, and it covers the case where this code is
5159 // being called from within backedge-taken count analysis, such that
5160 // attempting to ask for the backedge-taken count would likely result
5161 // in infinite recursion. In the later case, the analysis code will
5162 // cope with a conservative value, and it will take care to purge
5163 // that value once it has finished.
5164 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5165
5166 // Normally, in the cases we can prove no-overflow via a
5167 // backedge guarding condition, we can also compute a backedge
5168 // taken count for the loop. The exceptions are assumptions and
5169 // guards present in the loop -- SCEV is not great at exploiting
5170 // these to compute max backedge taken counts, but can still use
5171 // these to prove lack of overflow. Use this fact to avoid
5172 // doing extra work that may not pay off.
5173
5174 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5175 AC.assumptions().empty())
5176 return Result;
5177
5178 // If the backedge is guarded by a comparison with the pre-inc value the
5179 // addrec is safe. Also, if the entry is guarded by a comparison with the
5180 // start value and the backedge is guarded by a comparison with the post-inc
5181 // value, the addrec is safe.
5182 if (isKnownPositive(S: Step)) {
5183 ICmpInst::Predicate Pred;
5184 const SCEV *OverflowLimit =
5185 getUnsignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
5186 if (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
5187 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))
5188 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNUW);
5189 }
5190 return Result;
5191}
5192
5193namespace {
5194
5195/// Represents an abstract binary operation. This may exist as a
5196/// normal instruction or constant expression, or may have been
5197/// derived from an expression tree.
5198struct BinaryOp {
5199 unsigned Opcode;
5200 Value *LHS;
5201 Value *RHS;
5202 bool IsNSW = false;
5203 bool IsNUW = false;
5204
5205 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
5206 /// constant expression.
5207 Operator *Op = nullptr;
5208
5209 explicit BinaryOp(Operator *Op)
5210 : Opcode(Op->getOpcode()), LHS(Op->getOperand(i: 0)), RHS(Op->getOperand(i: 1)),
5211 Op(Op) {
5212 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: Op)) {
5213 IsNSW = OBO->hasNoSignedWrap();
5214 IsNUW = OBO->hasNoUnsignedWrap();
5215 }
5216 }
5217
5218 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
5219 bool IsNUW = false)
5220 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5221};
5222
5223} // end anonymous namespace
5224
5225/// Try to map \p V into a BinaryOp, and return \c std::nullopt on failure.
5226static std::optional<BinaryOp> MatchBinaryOp(Value *V, const DataLayout &DL,
5227 AssumptionCache &AC,
5228 const DominatorTree &DT,
5229 const Instruction *CxtI) {
5230 auto *Op = dyn_cast<Operator>(Val: V);
5231 if (!Op)
5232 return std::nullopt;
5233
5234 // Implementation detail: all the cleverness here should happen without
5235 // creating new SCEV expressions -- our caller knowns tricks to avoid creating
5236 // SCEV expressions when possible, and we should not break that.
5237
5238 switch (Op->getOpcode()) {
5239 case Instruction::Add:
5240 case Instruction::Sub:
5241 case Instruction::Mul:
5242 case Instruction::UDiv:
5243 case Instruction::URem:
5244 case Instruction::And:
5245 case Instruction::AShr:
5246 case Instruction::Shl:
5247 return BinaryOp(Op);
5248
5249 case Instruction::Or: {
5250 // Convert or disjoint into add nuw nsw.
5251 if (cast<PossiblyDisjointInst>(Val: Op)->isDisjoint()) {
5252 BinaryOp BinOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1),
5253 /*IsNSW=*/true, /*IsNUW=*/true);
5254 // Keep the reference to the original instruction so that we can later
5255 // check whether it can produce poison value or not.
5256 BinOp.Op = Op;
5257 return BinOp;
5258 }
5259 return BinaryOp(Op);
5260 }
5261
5262 case Instruction::Xor:
5263 if (auto *RHSC = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1)))
5264 // If the RHS of the xor is a signmask, then this is just an add.
5265 // Instcombine turns add of signmask into xor as a strength reduction step.
5266 if (RHSC->getValue().isSignMask())
5267 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5268 // Binary `xor` is a bit-wise `add`.
5269 if (V->getType()->isIntegerTy(BitWidth: 1))
5270 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5271 return BinaryOp(Op);
5272
5273 case Instruction::LShr:
5274 // Turn logical shift right of a constant into a unsigned divide.
5275 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1))) {
5276 uint32_t BitWidth = cast<IntegerType>(Val: Op->getType())->getBitWidth();
5277
5278 // If the shift count is not less than the bitwidth, the result of
5279 // the shift is undefined. Don't try to analyze it, because the
5280 // resolution chosen here may differ from the resolution chosen in
5281 // other parts of the compiler.
5282 if (SA->getValue().ult(RHS: BitWidth)) {
5283 Constant *X =
5284 ConstantInt::get(Context&: SA->getContext(),
5285 V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
5286 return BinaryOp(Instruction::UDiv, Op->getOperand(i: 0), X);
5287 }
5288 }
5289 return BinaryOp(Op);
5290
5291 case Instruction::ExtractValue: {
5292 auto *EVI = cast<ExtractValueInst>(Val: Op);
5293 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5294 break;
5295
5296 auto *WO = dyn_cast<WithOverflowInst>(Val: EVI->getAggregateOperand());
5297 if (!WO)
5298 break;
5299
5300 Instruction::BinaryOps BinOp = WO->getBinaryOp();
5301 bool Signed = WO->isSigned();
5302 // TODO: Should add nuw/nsw flags for mul as well.
5303 if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
5304 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5305
5306 // Now that we know that all uses of the arithmetic-result component of
5307 // CI are guarded by the overflow check, we can go ahead and pretend
5308 // that the arithmetic is non-overflowing.
5309 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5310 /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
5311 }
5312
5313 default:
5314 break;
5315 }
5316
5317 // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
5318 // semantics as a Sub, return a binary sub expression.
5319 if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
5320 if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5321 return BinaryOp(Instruction::Sub, II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1));
5322
5323 return std::nullopt;
5324}
5325
5326/// Helper function to createAddRecFromPHIWithCasts. We have a phi
5327/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
5328/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
5329/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
5330/// follows one of the following patterns:
5331/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5332/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5333/// If the SCEV expression of \p Op conforms with one of the expected patterns
5334/// we return the type of the truncation operation, and indicate whether the
5335/// truncated type should be treated as signed/unsigned by setting
5336/// \p Signed to true/false, respectively.
5337static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
5338 bool &Signed, ScalarEvolution &SE) {
5339 // The case where Op == SymbolicPHI (that is, with no type conversions on
5340 // the way) is handled by the regular add recurrence creating logic and
5341 // would have already been triggered in createAddRecForPHI. Reaching it here
5342 // means that createAddRecFromPHI had failed for this PHI before (e.g.,
5343 // because one of the other operands of the SCEVAddExpr updating this PHI is
5344 // not invariant).
5345 //
5346 // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
5347 // this case predicates that allow us to prove that Op == SymbolicPHI will
5348 // be added.
5349 if (Op == SymbolicPHI)
5350 return nullptr;
5351
5352 unsigned SourceBits = SE.getTypeSizeInBits(Ty: SymbolicPHI->getType());
5353 unsigned NewBits = SE.getTypeSizeInBits(Ty: Op->getType());
5354 if (SourceBits != NewBits)
5355 return nullptr;
5356
5357 if (match(S: Op, P: m_scev_SExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5358 Signed = true;
5359 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5360 }
5361 if (match(S: Op, P: m_scev_ZExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5362 Signed = false;
5363 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5364 }
5365 return nullptr;
5366}
5367
5368static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
5369 if (!PN->getType()->isIntegerTy())
5370 return nullptr;
5371 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5372 if (!L || L->getHeader() != PN->getParent())
5373 return nullptr;
5374 return L;
5375}
5376
5377// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
5378// computation that updates the phi follows the following pattern:
5379// (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
5380// which correspond to a phi->trunc->sext/zext->add->phi update chain.
5381// If so, try to see if it can be rewritten as an AddRecExpr under some
5382// Predicates. If successful, return them as a pair. Also cache the results
5383// of the analysis.
5384//
5385// Example usage scenario:
5386// Say the Rewriter is called for the following SCEV:
5387// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5388// where:
5389// %X = phi i64 (%Start, %BEValue)
5390// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
5391// and call this function with %SymbolicPHI = %X.
5392//
5393// The analysis will find that the value coming around the backedge has
5394// the following SCEV:
5395// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5396// Upon concluding that this matches the desired pattern, the function
5397// will return the pair {NewAddRec, SmallPredsVec} where:
5398// NewAddRec = {%Start,+,%Step}
5399// SmallPredsVec = {P1, P2, P3} as follows:
5400// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
5401// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
5402// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
5403// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
5404// under the predicates {P1,P2,P3}.
5405// This predicated rewrite will be cached in PredicatedSCEVRewrites:
5406// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
5407//
5408// TODO's:
5409//
5410// 1) Extend the Induction descriptor to also support inductions that involve
5411// casts: When needed (namely, when we are called in the context of the
5412// vectorizer induction analysis), a Set of cast instructions will be
5413// populated by this method, and provided back to isInductionPHI. This is
5414// needed to allow the vectorizer to properly record them to be ignored by
5415// the cost model and to avoid vectorizing them (otherwise these casts,
5416// which are redundant under the runtime overflow checks, will be
5417// vectorized, which can be costly).
5418//
5419// 2) Support additional induction/PHISCEV patterns: We also want to support
5420// inductions where the sext-trunc / zext-trunc operations (partly) occur
5421// after the induction update operation (the induction increment):
5422//
5423// (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
5424// which correspond to a phi->add->trunc->sext/zext->phi update chain.
5425//
5426// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
5427// which correspond to a phi->trunc->add->sext/zext->phi update chain.
5428//
5429// 3) Outline common code with createAddRecFromPHI to avoid duplication.
5430std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5431ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
5432 SmallVector<const SCEVPredicate *, 3> Predicates;
5433
5434 // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
5435 // return an AddRec expression under some predicate.
5436
5437 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5438 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5439 assert(L && "Expecting an integer loop header phi");
5440
5441 // The loop may have multiple entrances or multiple exits; we can analyze
5442 // this phi as an addrec if it has a unique entry value and a unique
5443 // backedge value.
5444 Value *BEValueV = nullptr, *StartValueV = nullptr;
5445 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5446 Value *V = PN->getIncomingValue(i);
5447 if (L->contains(BB: PN->getIncomingBlock(i))) {
5448 if (!BEValueV) {
5449 BEValueV = V;
5450 } else if (BEValueV != V) {
5451 BEValueV = nullptr;
5452 break;
5453 }
5454 } else if (!StartValueV) {
5455 StartValueV = V;
5456 } else if (StartValueV != V) {
5457 StartValueV = nullptr;
5458 break;
5459 }
5460 }
5461 if (!BEValueV || !StartValueV)
5462 return std::nullopt;
5463
5464 const SCEV *BEValue = getSCEV(V: BEValueV);
5465
5466 // If the value coming around the backedge is an add with the symbolic
5467 // value we just inserted, possibly with casts that we can ignore under
5468 // an appropriate runtime guard, then we found a simple induction variable!
5469 const auto *Add = dyn_cast<SCEVAddExpr>(Val: BEValue);
5470 if (!Add)
5471 return std::nullopt;
5472
5473 // If there is a single occurrence of the symbolic value, possibly
5474 // casted, replace it with a recurrence.
5475 unsigned FoundIndex = Add->getNumOperands();
5476 Type *TruncTy = nullptr;
5477 bool Signed;
5478 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5479 if ((TruncTy =
5480 isSimpleCastedPHI(Op: Add->getOperand(i), SymbolicPHI, Signed, SE&: *this)))
5481 if (FoundIndex == e) {
5482 FoundIndex = i;
5483 break;
5484 }
5485
5486 if (FoundIndex == Add->getNumOperands())
5487 return std::nullopt;
5488
5489 // Create an add with everything but the specified operand.
5490 SmallVector<SCEVUse, 8> Ops;
5491 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5492 if (i != FoundIndex)
5493 Ops.push_back(Elt: Add->getOperand(i));
5494 const SCEV *Accum = getAddExpr(Ops);
5495
5496 // The runtime checks will not be valid if the step amount is
5497 // varying inside the loop.
5498 if (!isLoopInvariant(S: Accum, L))
5499 return std::nullopt;
5500
5501 // *** Part2: Create the predicates
5502
5503 // Analysis was successful: we have a phi-with-cast pattern for which we
5504 // can return an AddRec expression under the following predicates:
5505 //
5506 // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
5507 // fits within the truncated type (does not overflow) for i = 0 to n-1.
5508 // P2: An Equal predicate that guarantees that
5509 // Start = (Ext ix (Trunc iy (Start) to ix) to iy)
5510 // P3: An Equal predicate that guarantees that
5511 // Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
5512 //
5513 // As we next prove, the above predicates guarantee that:
5514 // Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
5515 //
5516 //
5517 // More formally, we want to prove that:
5518 // Expr(i+1) = Start + (i+1) * Accum
5519 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5520 //
5521 // Given that:
5522 // 1) Expr(0) = Start
5523 // 2) Expr(1) = Start + Accum
5524 // = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
5525 // 3) Induction hypothesis (step i):
5526 // Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
5527 //
5528 // Proof:
5529 // Expr(i+1) =
5530 // = Start + (i+1)*Accum
5531 // = (Start + i*Accum) + Accum
5532 // = Expr(i) + Accum
5533 // = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
5534 // :: from step i
5535 //
5536 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
5537 //
5538 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
5539 // + (Ext ix (Trunc iy (Accum) to ix) to iy)
5540 // + Accum :: from P3
5541 //
5542 // = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
5543 // + Accum :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
5544 //
5545 // = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
5546 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5547 //
5548 // By induction, the same applies to all iterations 1<=i<n:
5549 //
5550
5551 // Create a truncated addrec for which we will add a no overflow check (P1).
5552 const SCEV *StartVal = getSCEV(V: StartValueV);
5553 const SCEV *PHISCEV =
5554 getAddRecExpr(Start: getTruncateExpr(Op: StartVal, Ty: TruncTy),
5555 Step: getTruncateExpr(Op: Accum, Ty: TruncTy), L, Flags: SCEV::FlagAnyWrap);
5556
5557 // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
5558 // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
5559 // will be constant.
5560 //
5561 // If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
5562 // add P1.
5563 if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV)) {
5564 SCEVWrapPredicate::IncrementWrapFlags AddedFlags =
5565 Signed ? SCEVWrapPredicate::IncrementNSSW
5566 : SCEVWrapPredicate::IncrementNUSW;
5567 const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
5568 Predicates.push_back(Elt: AddRecPred);
5569 }
5570
5571 // Create the Equal Predicates P2,P3:
5572
5573 // It is possible that the predicates P2 and/or P3 are computable at
5574 // compile time due to StartVal and/or Accum being constants.
5575 // If either one is, then we can check that now and escape if either P2
5576 // or P3 is false.
5577
5578 // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
5579 // for each of StartVal and Accum
5580 auto getExtendedExpr = [&](const SCEV *Expr,
5581 bool CreateSignExtend) -> const SCEV * {
5582 assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant");
5583 const SCEV *TruncatedExpr = getTruncateExpr(Op: Expr, Ty: TruncTy);
5584 const SCEV *ExtendedExpr =
5585 CreateSignExtend ? getSignExtendExpr(Op: TruncatedExpr, Ty: Expr->getType())
5586 : getZeroExtendExpr(Op: TruncatedExpr, Ty: Expr->getType());
5587 return ExtendedExpr;
5588 };
5589
5590 // Given:
5591 // ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
5592 // = getExtendedExpr(Expr)
5593 // Determine whether the predicate P: Expr == ExtendedExpr
5594 // is known to be false at compile time
5595 auto PredIsKnownFalse = [&](const SCEV *Expr,
5596 const SCEV *ExtendedExpr) -> bool {
5597 return Expr != ExtendedExpr &&
5598 isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: Expr, RHS: ExtendedExpr);
5599 };
5600
5601 const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
5602 if (PredIsKnownFalse(StartVal, StartExtended)) {
5603 LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";);
5604 return std::nullopt;
5605 }
5606
5607 // The Step is always Signed (because the overflow checks are either
5608 // NSSW or NUSW)
5609 const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
5610 if (PredIsKnownFalse(Accum, AccumExtended)) {
5611 LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";);
5612 return std::nullopt;
5613 }
5614
5615 auto AppendPredicate = [&](const SCEV *Expr,
5616 const SCEV *ExtendedExpr) -> void {
5617 if (Expr != ExtendedExpr &&
5618 !isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: Expr, RHS: ExtendedExpr)) {
5619 const SCEVPredicate *Pred = getEqualPredicate(LHS: Expr, RHS: ExtendedExpr);
5620 LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred);
5621 Predicates.push_back(Elt: Pred);
5622 }
5623 };
5624
5625 AppendPredicate(StartVal, StartExtended);
5626 AppendPredicate(Accum, AccumExtended);
5627
5628 // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
5629 // which the casts had been folded away. The caller can rewrite SymbolicPHI
5630 // into NewAR if it will also add the runtime overflow checks specified in
5631 // Predicates.
5632 const SCEV *NewAR = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags: SCEV::FlagAnyWrap);
5633
5634 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5635 std::make_pair(x&: NewAR, y&: Predicates);
5636 // Remember the result of the analysis for this SCEV at this locayyytion.
5637 PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
5638 return PredRewrite;
5639}
5640
5641std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5642ScalarEvolution::createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI) {
5643 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5644 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5645 if (!L)
5646 return std::nullopt;
5647
5648 // Check to see if we already analyzed this PHI.
5649 auto I = PredicatedSCEVRewrites.find(Val: {SymbolicPHI, L});
5650 if (I != PredicatedSCEVRewrites.end()) {
5651 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5652 I->second;
5653 // Analysis was done before and failed to create an AddRec:
5654 if (Rewrite.first == SymbolicPHI)
5655 return std::nullopt;
5656 // Analysis was done before and succeeded to create an AddRec under
5657 // a predicate:
5658 assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec");
5659 assert(!(Rewrite.second).empty() && "Expected to find Predicates");
5660 return Rewrite;
5661 }
5662
5663 std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5664 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5665
5666 // Record in the cache that the analysis failed
5667 if (!Rewrite) {
5668 SmallVector<const SCEVPredicate *, 3> Predicates;
5669 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5670 return std::nullopt;
5671 }
5672
5673 return Rewrite;
5674}
5675
5676// FIXME: This utility is currently required because the Rewriter currently
5677// does not rewrite this expression:
5678// {0, +, (sext ix (trunc iy to ix) to iy)}
5679// into {0, +, %step},
5680// even when the following Equal predicate exists:
5681// "%step == (sext ix (trunc iy to ix) to iy)".
5682bool PredicatedScalarEvolution::areAddRecsEqualWithPreds(
5683 const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2,
5684 ArrayRef<const SCEVPredicate *> NoWrapPreds) const {
5685 if (AR1 == AR2)
5686 return true;
5687
5688 SCEVUnionPredicate NoWrapUnionPred(NoWrapPreds, SE);
5689 SCEVUnionPredicate AllPreds = Preds->getUnionWith(N: &NoWrapUnionPred, SE);
5690 auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
5691 if (Expr1 != Expr2 &&
5692 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr1, RHS: Expr2), SE) &&
5693 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr2, RHS: Expr1), SE))
5694 return false;
5695 return true;
5696 };
5697
5698 if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
5699 !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
5700 return false;
5701 return true;
5702}
5703
5704static SCEV::NoWrapFlags
5705getNoWrapFlagsForGEP(GEPOperator *GEP, const SCEV *Accum, ScalarEvolution &SE) {
5706 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5707 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
5708 // If the increment has any nowrap flags, then we know the address
5709 // space cannot be wrapped around.
5710 if (NW != GEPNoWrapFlags::none())
5711 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNW);
5712 // If the GEP is nuw or nusw with non-negative offset, we know that
5713 // no unsigned wrap occurs. We cannot set the nsw flag as only the
5714 // offset is treated as signed, while the base is unsigned.
5715 if (NW.hasNoUnsignedWrap() ||
5716 (NW.hasNoUnsignedSignedWrap() && SE.isKnownNonNegative(S: Accum)))
5717 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
5718
5719 return Flags;
5720}
5721
5722/// A helper function for createAddRecFromPHI to handle simple cases.
5723///
5724/// This function tries to find an AddRec expression for the simplest (yet most
5725/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
5726/// If it fails, createAddRecFromPHI will use a more general, but slow,
5727/// technique for finding the AddRec expression.
5728const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
5729 Value *BEValueV,
5730 Value *StartValueV) {
5731 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5732 assert(L && L->getHeader() == PN->getParent());
5733 assert(BEValueV && StartValueV);
5734
5735 const SCEV *Accum = nullptr;
5736 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5737 if (auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CxtI: PN)) {
5738 if (BO->Opcode != Instruction::Add)
5739 return nullptr;
5740
5741 if (BO->LHS == PN && L->isLoopInvariant(V: BO->RHS))
5742 Accum = getSCEV(V: BO->RHS);
5743 else if (BO->RHS == PN && L->isLoopInvariant(V: BO->LHS))
5744 Accum = getSCEV(V: BO->LHS);
5745
5746 if (!Accum)
5747 return nullptr;
5748
5749 if (BO->IsNUW)
5750 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5751 if (BO->IsNSW)
5752 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5753 } else {
5754 // Handle pointer induction variable: PN = PHI(Start, gep PN,
5755 // LoopInvariant).
5756 auto *GEP = dyn_cast<GEPOperator>(Val: BEValueV);
5757 if (!GEP || GEP->getPointerOperand() != PN || GEP->getNumIndices() != 1)
5758 return nullptr;
5759 Value *Idx = *GEP->idx_begin();
5760 if (!L->isLoopInvariant(V: Idx))
5761 return nullptr;
5762
5763 Type *IntIdxTy = getEffectiveSCEVType(Ty: GEP->getType());
5764 Accum = getMulExpr(LHS: getTruncateOrSignExtend(V: getSCEV(V: Idx), Ty: IntIdxTy),
5765 RHS: getSizeOfExpr(IntTy: IntIdxTy, AllocTy: GEP->getSourceElementType()));
5766 Flags = getNoWrapFlagsForGEP(GEP, Accum, SE&: *this);
5767 }
5768
5769 const SCEV *StartVal = getSCEV(V: StartValueV);
5770 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5771 insertValueToMap(V: PN, S: PHISCEV);
5772
5773 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
5774 inferNoWrapViaConstantRanges(AR);
5775
5776 // We can add Flags to the post-inc expression only if we
5777 // know that it is *undefined behavior* for BEValueV to
5778 // overflow.
5779 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV)) {
5780 assert(isLoopInvariant(Accum, L) &&
5781 "Accum is defined outside L, but is not invariant?");
5782 if (isAddRecNeverPoison(I: BEInst, L))
5783 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5784 }
5785
5786 return PHISCEV;
5787}
5788
5789const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
5790 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5791 if (!L || L->getHeader() != PN->getParent())
5792 return nullptr;
5793
5794 // The loop may have multiple entrances or multiple exits; we can analyze
5795 // this phi as an addrec if it has a unique entry value and a unique
5796 // backedge value.
5797 Value *BEValueV = nullptr, *StartValueV = nullptr;
5798 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5799 Value *V = PN->getIncomingValue(i);
5800 if (L->contains(BB: PN->getIncomingBlock(i))) {
5801 if (!BEValueV) {
5802 BEValueV = V;
5803 } else if (BEValueV != V) {
5804 BEValueV = nullptr;
5805 break;
5806 }
5807 } else if (!StartValueV) {
5808 StartValueV = V;
5809 } else if (StartValueV != V) {
5810 StartValueV = nullptr;
5811 break;
5812 }
5813 }
5814 if (!BEValueV || !StartValueV)
5815 return nullptr;
5816
5817 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
5818 "PHI node already processed?");
5819
5820 // First, try to find AddRec expression without creating a fictituos symbolic
5821 // value for PN.
5822 if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5823 return S;
5824
5825 // Handle PHI node value symbolically.
5826 const SCEV *SymbolicName = getUnknown(V: PN);
5827 insertValueToMap(V: PN, S: SymbolicName);
5828
5829 // Using this symbolic name for the PHI, analyze the value coming around
5830 // the back-edge.
5831 const SCEV *BEValue = getSCEV(V: BEValueV);
5832
5833 // NOTE: If BEValue is loop invariant, we know that the PHI node just
5834 // has a special value for the first iteration of the loop.
5835
5836 // If the value coming around the backedge is an add with the symbolic
5837 // value we just inserted, then we found a simple induction variable!
5838 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: BEValue)) {
5839 // If there is a single occurrence of the symbolic value, replace it
5840 // with a recurrence.
5841 unsigned FoundIndex = Add->getNumOperands();
5842 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5843 if (Add->getOperand(i) == SymbolicName)
5844 if (FoundIndex == e) {
5845 FoundIndex = i;
5846 break;
5847 }
5848
5849 if (FoundIndex != Add->getNumOperands()) {
5850 // Create an add with everything but the specified operand.
5851 SmallVector<SCEVUse, 8> Ops;
5852 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5853 if (i != FoundIndex)
5854 Ops.push_back(Elt: SCEVBackedgeConditionFolder::rewrite(S: Add->getOperand(i),
5855 L, SE&: *this));
5856 const SCEV *Accum = getAddExpr(Ops);
5857
5858 // This is not a valid addrec if the step amount is varying each
5859 // loop iteration, but is not itself an addrec in this loop.
5860 if (isLoopInvariant(S: Accum, L) ||
5861 (isa<SCEVAddRecExpr>(Val: Accum) &&
5862 cast<SCEVAddRecExpr>(Val: Accum)->getLoop() == L)) {
5863 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5864
5865 if (auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CxtI: PN)) {
5866 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
5867 if (BO->IsNUW)
5868 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5869 if (BO->IsNSW)
5870 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5871 }
5872 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: BEValueV)) {
5873 if (GEP->getOperand(i_nocapture: 0) == PN)
5874 Flags = getNoWrapFlagsForGEP(GEP, Accum, SE&: *this);
5875
5876 // We cannot transfer nuw and nsw flags from subtraction
5877 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
5878 // for instance.
5879 }
5880
5881 const SCEV *StartVal = getSCEV(V: StartValueV);
5882 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5883
5884 // Okay, for the entire analysis of this edge we assumed the PHI
5885 // to be symbolic. We now need to go back and purge all of the
5886 // entries for the scalars that use the symbolic expression.
5887 forgetMemoizedResults(SCEVs: {SymbolicName});
5888 insertValueToMap(V: PN, S: PHISCEV);
5889
5890 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
5891 inferNoWrapViaConstantRanges(AR);
5892
5893 // We can add Flags to the post-inc expression only if we
5894 // know that it is *undefined behavior* for BEValueV to
5895 // overflow.
5896 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV))
5897 if (isLoopInvariant(S: Accum, L) && isAddRecNeverPoison(I: BEInst, L))
5898 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5899
5900 return PHISCEV;
5901 }
5902 }
5903 } else {
5904 // Otherwise, this could be a loop like this:
5905 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
5906 // In this case, j = {1,+,1} and BEValue is j.
5907 // Because the other in-value of i (0) fits the evolution of BEValue
5908 // i really is an addrec evolution.
5909 //
5910 // We can generalize this saying that i is the shifted value of BEValue
5911 // by one iteration:
5912 // PHI(f(0), f({1,+,1})) --> f({0,+,1})
5913
5914 // Do not allow refinement in rewriting of BEValue.
5915 const SCEV *Shifted = SCEVShiftRewriter::rewrite(S: BEValue, L, SE&: *this);
5916 const SCEV *Start = SCEVInitRewriter::rewrite(S: Shifted, L, SE&: *this, IgnoreOtherLoops: false);
5917 if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
5918 isGuaranteedNotToCauseUB(Op: Shifted) && ::impliesPoison(AssumedPoison: Shifted, S: Start)) {
5919 const SCEV *StartVal = getSCEV(V: StartValueV);
5920 if (Start == StartVal) {
5921 // Okay, for the entire analysis of this edge we assumed the PHI
5922 // to be symbolic. We now need to go back and purge all of the
5923 // entries for the scalars that use the symbolic expression.
5924 forgetMemoizedResults(SCEVs: {SymbolicName});
5925 insertValueToMap(V: PN, S: Shifted);
5926 return Shifted;
5927 }
5928 }
5929 }
5930
5931 // Remove the temporary PHI node SCEV that has been inserted while intending
5932 // to create an AddRecExpr for this PHI node. We can not keep this temporary
5933 // as it will prevent later (possibly simpler) SCEV expressions to be added
5934 // to the ValueExprMap.
5935 eraseValueFromMap(V: PN);
5936
5937 return nullptr;
5938}
5939
5940// Try to match a control flow sequence that branches out at BI and merges back
5941// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
5942// match.
5943static bool BrPHIToSelect(DominatorTree &DT, CondBrInst *BI, PHINode *Merge,
5944 Value *&C, Value *&LHS, Value *&RHS) {
5945 C = BI->getCondition();
5946
5947 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(i: 0));
5948 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(i: 1));
5949
5950 Use &LeftUse = Merge->getOperandUse(i: 0);
5951 Use &RightUse = Merge->getOperandUse(i: 1);
5952
5953 if (DT.dominates(BBE: LeftEdge, U: LeftUse) && DT.dominates(BBE: RightEdge, U: RightUse)) {
5954 LHS = LeftUse;
5955 RHS = RightUse;
5956 return true;
5957 }
5958
5959 if (DT.dominates(BBE: LeftEdge, U: RightUse) && DT.dominates(BBE: RightEdge, U: LeftUse)) {
5960 LHS = RightUse;
5961 RHS = LeftUse;
5962 return true;
5963 }
5964
5965 return false;
5966}
5967
5968static bool getOperandsForSelectLikePHI(DominatorTree &DT, PHINode *PN,
5969 Value *&Cond, Value *&LHS,
5970 Value *&RHS) {
5971 auto IsReachable =
5972 [&](BasicBlock *BB) { return DT.isReachableFromEntry(A: BB); };
5973 if (PN->getNumIncomingValues() == 2 && all_of(Range: PN->blocks(), P: IsReachable)) {
5974 // Try to match
5975 //
5976 // br %cond, label %left, label %right
5977 // left:
5978 // br label %merge
5979 // right:
5980 // br label %merge
5981 // merge:
5982 // V = phi [ %x, %left ], [ %y, %right ]
5983 //
5984 // as "select %cond, %x, %y"
5985
5986 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
5987 assert(IDom && "At least the entry block should dominate PN");
5988
5989 auto *BI = dyn_cast<CondBrInst>(Val: IDom->getTerminator());
5990 return BI && BrPHIToSelect(DT, BI, Merge: PN, C&: Cond, LHS, RHS);
5991 }
5992 return false;
5993}
5994
5995const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
5996 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
5997 if (getOperandsForSelectLikePHI(DT, PN, Cond, LHS, RHS) &&
5998 properlyDominates(S: getSCEV(V: LHS), BB: PN->getParent()) &&
5999 properlyDominates(S: getSCEV(V: RHS), BB: PN->getParent()))
6000 return createNodeForSelectOrPHI(V: PN, Cond, TrueVal: LHS, FalseVal: RHS);
6001
6002 return nullptr;
6003}
6004
6005static BinaryOperator *getCommonInstForPHI(PHINode *PN) {
6006 BinaryOperator *CommonInst = nullptr;
6007 // Check if instructions are identical.
6008 for (Value *Incoming : PN->incoming_values()) {
6009 auto *IncomingInst = dyn_cast<BinaryOperator>(Val: Incoming);
6010 if (!IncomingInst)
6011 return nullptr;
6012 if (CommonInst) {
6013 if (!CommonInst->isIdenticalToWhenDefined(I: IncomingInst))
6014 return nullptr; // Not identical, give up
6015 } else {
6016 // Remember binary operator
6017 CommonInst = IncomingInst;
6018 }
6019 }
6020 return CommonInst;
6021}
6022
6023/// Returns SCEV for the first operand of a phi if all phi operands have
6024/// identical opcodes and operands
6025/// eg.
6026/// a: %add = %a + %b
6027/// br %c
6028/// b: %add1 = %a + %b
6029/// br %c
6030/// c: %phi = phi [%add, a], [%add1, b]
6031/// scev(%phi) => scev(%add)
6032const SCEV *
6033ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
6034 BinaryOperator *CommonInst = getCommonInstForPHI(PN);
6035 if (!CommonInst)
6036 return nullptr;
6037
6038 // Check if SCEV exprs for instructions are identical.
6039 const SCEV *CommonSCEV = getSCEV(V: CommonInst);
6040 bool SCEVExprsIdentical =
6041 all_of(Range: drop_begin(RangeOrContainer: PN->incoming_values()),
6042 P: [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
6043 return SCEVExprsIdentical ? CommonSCEV : nullptr;
6044}
6045
6046const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
6047 if (const SCEV *S = createAddRecFromPHI(PN))
6048 return S;
6049
6050 // We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
6051 // phi node for X.
6052 if (Value *V = simplifyInstruction(
6053 I: PN, Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
6054 /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
6055 return getSCEV(V);
6056
6057 if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
6058 return S;
6059
6060 if (const SCEV *S = createNodeFromSelectLikePHI(PN))
6061 return S;
6062
6063 // If it's not a loop phi, we can't handle it yet.
6064 return getUnknown(V: PN);
6065}
6066
6067bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
6068 SCEVTypes RootKind) {
6069 struct FindClosure {
6070 const SCEV *OperandToFind;
6071 const SCEVTypes RootKind; // Must be a sequential min/max expression.
6072 const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind.
6073
6074 bool Found = false;
6075
6076 bool canRecurseInto(SCEVTypes Kind) const {
6077 // We can only recurse into the SCEV expression of the same effective type
6078 // as the type of our root SCEV expression, and into zero-extensions.
6079 return RootKind == Kind || NonSequentialRootKind == Kind ||
6080 scZeroExtend == Kind;
6081 };
6082
6083 FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind)
6084 : OperandToFind(OperandToFind), RootKind(RootKind),
6085 NonSequentialRootKind(
6086 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
6087 Ty: RootKind)) {}
6088
6089 bool follow(const SCEV *S) {
6090 Found = S == OperandToFind;
6091
6092 return !isDone() && canRecurseInto(Kind: S->getSCEVType());
6093 }
6094
6095 bool isDone() const { return Found; }
6096 };
6097
6098 FindClosure FC(OperandToFind, RootKind);
6099 visitAll(Root, Visitor&: FC);
6100 return FC.Found;
6101}
6102
6103std::optional<const SCEV *>
6104ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(Type *Ty,
6105 ICmpInst *Cond,
6106 Value *TrueVal,
6107 Value *FalseVal) {
6108 // Try to match some simple smax or umax patterns.
6109 auto *ICI = Cond;
6110
6111 Value *LHS = ICI->getOperand(i_nocapture: 0);
6112 Value *RHS = ICI->getOperand(i_nocapture: 1);
6113
6114 switch (ICI->getPredicate()) {
6115 case ICmpInst::ICMP_SLT:
6116 case ICmpInst::ICMP_SLE:
6117 case ICmpInst::ICMP_ULT:
6118 case ICmpInst::ICMP_ULE:
6119 std::swap(a&: LHS, b&: RHS);
6120 [[fallthrough]];
6121 case ICmpInst::ICMP_SGT:
6122 case ICmpInst::ICMP_SGE:
6123 case ICmpInst::ICMP_UGT:
6124 case ICmpInst::ICMP_UGE:
6125 // a > b ? a+x : b+x -> max(a, b)+x
6126 // a > b ? b+x : a+x -> min(a, b)+x
6127 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty)) {
6128 bool Signed = ICI->isSigned();
6129 const SCEV *LA = getSCEV(V: TrueVal);
6130 const SCEV *RA = getSCEV(V: FalseVal);
6131 const SCEV *LS = getSCEV(V: LHS);
6132 const SCEV *RS = getSCEV(V: RHS);
6133 if (LA->getType()->isPointerTy()) {
6134 // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
6135 // Need to make sure we can't produce weird expressions involving
6136 // negated pointers.
6137 if (LA == LS && RA == RS)
6138 return Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS);
6139 if (LA == RS && RA == LS)
6140 return Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS);
6141 }
6142 auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
6143 if (Op->getType()->isPointerTy()) {
6144 Op = getPtrToAddrExpr(Op);
6145 if (isa<SCEVCouldNotCompute>(Val: Op))
6146 return Op;
6147 }
6148 if (Signed)
6149 Op = getNoopOrSignExtend(V: Op, Ty);
6150 else
6151 Op = getNoopOrZeroExtend(V: Op, Ty);
6152 return Op;
6153 };
6154 LS = CoerceOperand(LS);
6155 RS = CoerceOperand(RS);
6156 if (isa<SCEVCouldNotCompute>(Val: LS) || isa<SCEVCouldNotCompute>(Val: RS))
6157 break;
6158 const SCEV *LDiff = getMinusSCEV(LHS: LA, RHS: LS);
6159 const SCEV *RDiff = getMinusSCEV(LHS: RA, RHS: RS);
6160 if (LDiff == RDiff)
6161 return getAddExpr(LHS: Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS),
6162 RHS: LDiff);
6163 LDiff = getMinusSCEV(LHS: LA, RHS: RS);
6164 RDiff = getMinusSCEV(LHS: RA, RHS: LS);
6165 if (LDiff == RDiff)
6166 return getAddExpr(LHS: Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS),
6167 RHS: LDiff);
6168 }
6169 break;
6170 case ICmpInst::ICMP_NE:
6171 // x != 0 ? x+y : C+y -> x == 0 ? C+y : x+y
6172 std::swap(a&: TrueVal, b&: FalseVal);
6173 [[fallthrough]];
6174 case ICmpInst::ICMP_EQ:
6175 // x == 0 ? C+y : x+y -> umax(x, C)+y iff C u<= 1
6176 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty) &&
6177 isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()) {
6178 const SCEV *X = getNoopOrZeroExtend(V: getSCEV(V: LHS), Ty);
6179 const SCEV *TrueValExpr = getSCEV(V: TrueVal); // C+y
6180 const SCEV *FalseValExpr = getSCEV(V: FalseVal); // x+y
6181 const SCEV *Y = getMinusSCEV(LHS: FalseValExpr, RHS: X); // y = (x+y)-x
6182 const SCEV *C = getMinusSCEV(LHS: TrueValExpr, RHS: Y); // C = (C+y)-y
6183 if (isa<SCEVConstant>(Val: C) && cast<SCEVConstant>(Val: C)->getAPInt().ule(RHS: 1))
6184 return getAddExpr(LHS: getUMaxExpr(LHS: X, RHS: C), RHS: Y);
6185 }
6186 // x == 0 ? 0 : umin (..., x, ...) -> umin_seq(x, umin (...))
6187 // x == 0 ? 0 : umin_seq(..., x, ...) -> umin_seq(x, umin_seq(...))
6188 // x == 0 ? 0 : umin (..., umin_seq(..., x, ...), ...)
6189 // -> umin_seq(x, umin (..., umin_seq(...), ...))
6190 if (isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero() &&
6191 isa<ConstantInt>(Val: TrueVal) && cast<ConstantInt>(Val: TrueVal)->isZero()) {
6192 const SCEV *X = getSCEV(V: LHS);
6193 while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: X))
6194 X = ZExt->getOperand();
6195 if (getTypeSizeInBits(Ty: X->getType()) <= getTypeSizeInBits(Ty)) {
6196 const SCEV *FalseValExpr = getSCEV(V: FalseVal);
6197 if (SCEVMinMaxExprContains(Root: FalseValExpr, OperandToFind: X, RootKind: scSequentialUMinExpr))
6198 return getUMinExpr(LHS: getNoopOrZeroExtend(V: X, Ty), RHS: FalseValExpr,
6199 /*Sequential=*/true);
6200 }
6201 }
6202 break;
6203 default:
6204 break;
6205 }
6206
6207 return std::nullopt;
6208}
6209
6210static std::optional<const SCEV *>
6211createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr,
6212 const SCEV *TrueExpr, const SCEV *FalseExpr) {
6213 assert(CondExpr->getType()->isIntegerTy(1) &&
6214 TrueExpr->getType() == FalseExpr->getType() &&
6215 TrueExpr->getType()->isIntegerTy(1) &&
6216 "Unexpected operands of a select.");
6217
6218 // i1 cond ? i1 x : i1 C --> C + (i1 cond ? (i1 x - i1 C) : i1 0)
6219 // --> C + (umin_seq cond, x - C)
6220 //
6221 // i1 cond ? i1 C : i1 x --> C + (i1 cond ? i1 0 : (i1 x - i1 C))
6222 // --> C + (i1 ~cond ? (i1 x - i1 C) : i1 0)
6223 // --> C + (umin_seq ~cond, x - C)
6224
6225 // FIXME: while we can't legally model the case where both of the hands
6226 // are fully variable, we only require that the *difference* is constant.
6227 if (!isa<SCEVConstant>(Val: TrueExpr) && !isa<SCEVConstant>(Val: FalseExpr))
6228 return std::nullopt;
6229
6230 const SCEV *X, *C;
6231 if (isa<SCEVConstant>(Val: TrueExpr)) {
6232 CondExpr = SE->getNotSCEV(V: CondExpr);
6233 X = FalseExpr;
6234 C = TrueExpr;
6235 } else {
6236 X = TrueExpr;
6237 C = FalseExpr;
6238 }
6239 return SE->getAddExpr(LHS: C, RHS: SE->getUMinExpr(LHS: CondExpr, RHS: SE->getMinusSCEV(LHS: X, RHS: C),
6240 /*Sequential=*/true));
6241}
6242
6243static std::optional<const SCEV *>
6244createNodeForSelectViaUMinSeq(ScalarEvolution *SE, Value *Cond, Value *TrueVal,
6245 Value *FalseVal) {
6246 if (!isa<ConstantInt>(Val: TrueVal) && !isa<ConstantInt>(Val: FalseVal))
6247 return std::nullopt;
6248
6249 const auto *SECond = SE->getSCEV(V: Cond);
6250 const auto *SETrue = SE->getSCEV(V: TrueVal);
6251 const auto *SEFalse = SE->getSCEV(V: FalseVal);
6252 return createNodeForSelectViaUMinSeq(SE, CondExpr: SECond, TrueExpr: SETrue, FalseExpr: SEFalse);
6253}
6254
6255const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
6256 Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) {
6257 assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?");
6258 assert(TrueVal->getType() == FalseVal->getType() &&
6259 V->getType() == TrueVal->getType() &&
6260 "Types of select hands and of the result must match.");
6261
6262 // For now, only deal with i1-typed `select`s.
6263 if (!V->getType()->isIntegerTy(BitWidth: 1))
6264 return getUnknown(V);
6265
6266 if (std::optional<const SCEV *> S =
6267 createNodeForSelectViaUMinSeq(SE: this, Cond, TrueVal, FalseVal))
6268 return *S;
6269
6270 return getUnknown(V);
6271}
6272
6273const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond,
6274 Value *TrueVal,
6275 Value *FalseVal) {
6276 // Handle "constant" branch or select. This can occur for instance when a
6277 // loop pass transforms an inner loop and moves on to process the outer loop.
6278 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond))
6279 return getSCEV(V: CI->isOne() ? TrueVal : FalseVal);
6280
6281 if (auto *I = dyn_cast<Instruction>(Val: V)) {
6282 if (auto *ICI = dyn_cast<ICmpInst>(Val: Cond)) {
6283 if (std::optional<const SCEV *> S =
6284 createNodeForSelectOrPHIInstWithICmpInstCond(Ty: I->getType(), Cond: ICI,
6285 TrueVal, FalseVal))
6286 return *S;
6287 }
6288 }
6289
6290 return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal);
6291}
6292
6293/// Expand GEP instructions into add and multiply operations. This allows them
6294/// to be analyzed by regular SCEV code.
6295const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
6296 assert(GEP->getSourceElementType()->isSized() &&
6297 "GEP source element type must be sized");
6298
6299 SmallVector<SCEVUse, 4> IndexExprs;
6300 for (Value *Index : GEP->indices())
6301 IndexExprs.push_back(Elt: getSCEV(V: Index));
6302 return getGEPExpr(GEP, IndexExprs);
6303}
6304
6305APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S,
6306 const Instruction *CtxI) {
6307 uint64_t BitWidth = getTypeSizeInBits(Ty: S->getType());
6308 auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
6309 return TrailingZeros >= BitWidth
6310 ? APInt::getZero(numBits: BitWidth)
6311 : APInt::getOneBitSet(numBits: BitWidth, BitNo: TrailingZeros);
6312 };
6313 auto GetGCDMultiple = [this, CtxI](const SCEVNAryExpr *N) {
6314 // The result is GCD of all operands results.
6315 APInt Res = getConstantMultiple(S: N->getOperand(i: 0), CtxI);
6316 for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
6317 Res = APIntOps::GreatestCommonDivisor(
6318 A: Res, B: getConstantMultiple(S: N->getOperand(i: I), CtxI));
6319 return Res;
6320 };
6321
6322 switch (S->getSCEVType()) {
6323 case scConstant:
6324 return cast<SCEVConstant>(Val: S)->getAPInt();
6325 case scPtrToAddr:
6326 return getConstantMultiple(S: cast<SCEVCastExpr>(Val: S)->getOperand());
6327 case scUDivExpr:
6328 case scVScale:
6329 return APInt(BitWidth, 1);
6330 case scTruncate: {
6331 // Only multiples that are a power of 2 will hold after truncation.
6332 const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(Val: S);
6333 uint32_t TZ = getMinTrailingZeros(S: T->getOperand(), CtxI);
6334 return GetShiftedByZeros(TZ);
6335 }
6336 case scZeroExtend: {
6337 const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(Val: S);
6338 return getConstantMultiple(S: Z->getOperand(), CtxI).zext(width: BitWidth);
6339 }
6340 case scSignExtend: {
6341 // Only multiples that are a power of 2 will hold after sext.
6342 const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(Val: S);
6343 uint32_t TZ = getMinTrailingZeros(S: E->getOperand(), CtxI);
6344 return GetShiftedByZeros(TZ);
6345 }
6346 case scMulExpr: {
6347 const SCEVMulExpr *M = cast<SCEVMulExpr>(Val: S);
6348 if (M->hasNoUnsignedWrap()) {
6349 // The result is the product of all operand results.
6350 APInt Res = getConstantMultiple(S: M->getOperand(i: 0), CtxI);
6351 for (const SCEV *Operand : M->operands().drop_front())
6352 Res = Res * getConstantMultiple(S: Operand, CtxI);
6353 return Res;
6354 }
6355
6356 // If there are no wrap guarentees, find the trailing zeros, which is the
6357 // sum of trailing zeros for all its operands.
6358 uint32_t TZ = 0;
6359 for (const SCEV *Operand : M->operands())
6360 TZ += getMinTrailingZeros(S: Operand, CtxI);
6361 return GetShiftedByZeros(TZ);
6362 }
6363 case scAddExpr:
6364 case scAddRecExpr: {
6365 const SCEVNAryExpr *N = cast<SCEVNAryExpr>(Val: S);
6366 if (N->hasNoUnsignedWrap())
6367 return GetGCDMultiple(N);
6368 // Find the trailing bits, which is the minimum of its operands.
6369 uint32_t TZ = getMinTrailingZeros(S: N->getOperand(i: 0), CtxI);
6370 for (const SCEV *Operand : N->operands().drop_front())
6371 TZ = std::min(a: TZ, b: getMinTrailingZeros(S: Operand, CtxI));
6372 return GetShiftedByZeros(TZ);
6373 }
6374 case scUMaxExpr:
6375 case scSMaxExpr:
6376 case scUMinExpr:
6377 case scSMinExpr:
6378 case scSequentialUMinExpr:
6379 return GetGCDMultiple(cast<SCEVNAryExpr>(Val: S));
6380 case scUnknown: {
6381 // Ask ValueTracking for known bits. SCEVUnknown only become available at
6382 // the point their underlying IR instruction has been defined. If CtxI was
6383 // not provided, use:
6384 // * the first instruction in the entry block if it is an argument
6385 // * the instruction itself otherwise.
6386 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6387 if (!CtxI) {
6388 if (isa<Argument>(Val: U->getValue()))
6389 CtxI = &*F.getEntryBlock().begin();
6390 else if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
6391 CtxI = I;
6392 }
6393 unsigned Known =
6394 computeKnownBits(V: U->getValue(),
6395 Q: SimplifyQuery(getDataLayout(), &DT, &AC, CtxI)
6396 .allowEphemerals(AllowEphemerals: true))
6397 .countMinTrailingZeros();
6398 return GetShiftedByZeros(Known);
6399 }
6400 case scCouldNotCompute:
6401 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6402 }
6403 llvm_unreachable("Unknown SCEV kind!");
6404}
6405
6406APInt ScalarEvolution::getConstantMultiple(const SCEV *S,
6407 const Instruction *CtxI) {
6408 // Skip looking up and updating the cache if there is a context instruction,
6409 // as the result will only be valid in the specified context.
6410 if (CtxI)
6411 return getConstantMultipleImpl(S, CtxI);
6412
6413 auto I = ConstantMultipleCache.find(Val: S);
6414 if (I != ConstantMultipleCache.end())
6415 return I->second;
6416
6417 APInt Result = getConstantMultipleImpl(S, CtxI);
6418 auto InsertPair = ConstantMultipleCache.insert(KV: {S, Result});
6419 assert(InsertPair.second && "Should insert a new key");
6420 return InsertPair.first->second;
6421}
6422
6423APInt ScalarEvolution::getNonZeroConstantMultiple(const SCEV *S) {
6424 APInt Multiple = getConstantMultiple(S);
6425 return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
6426}
6427
6428uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S,
6429 const Instruction *CtxI) {
6430 return std::min(a: getConstantMultiple(S, CtxI).countTrailingZeros(),
6431 b: (unsigned)getTypeSizeInBits(Ty: S->getType()));
6432}
6433
6434/// Helper method to assign a range to V from metadata present in the IR.
6435static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
6436 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
6437 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_range))
6438 return getConstantRangeFromMetadata(RangeMD: *MD);
6439 if (const auto *CB = dyn_cast<CallBase>(Val: V))
6440 if (std::optional<ConstantRange> Range = CB->getRange())
6441 return Range;
6442 }
6443 if (auto *A = dyn_cast<Argument>(Val: V))
6444 if (std::optional<ConstantRange> Range = A->getRange())
6445 return Range;
6446
6447 return std::nullopt;
6448}
6449
6450void ScalarEvolution::setNoWrapFlags(SCEVAddRecExpr *AddRec,
6451 SCEV::NoWrapFlags Flags) {
6452 if (AddRec->getNoWrapFlags(Mask: Flags) != Flags) {
6453 AddRec->setNoWrapFlags(Flags);
6454 UnsignedRanges.erase(Val: AddRec);
6455 SignedRanges.erase(Val: AddRec);
6456 ConstantMultipleCache.erase(Val: AddRec);
6457 }
6458}
6459
6460ConstantRange ScalarEvolution::
6461getRangeForUnknownRecurrence(const SCEVUnknown *U) {
6462 const DataLayout &DL = getDataLayout();
6463
6464 unsigned BitWidth = getTypeSizeInBits(Ty: U->getType());
6465 const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
6466
6467 // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
6468 // use information about the trip count to improve our available range. Note
6469 // that the trip count independent cases are already handled by known bits.
6470 // WARNING: The definition of recurrence used here is subtly different than
6471 // the one used by AddRec (and thus most of this file). Step is allowed to
6472 // be arbitrarily loop varying here, where AddRec allows only loop invariant
6473 // and other addrecs in the same loop (for non-affine addrecs). The code
6474 // below intentionally handles the case where step is not loop invariant.
6475 auto *P = dyn_cast<PHINode>(Val: U->getValue());
6476 if (!P)
6477 return FullSet;
6478
6479 // Make sure that no Phi input comes from an unreachable block. Otherwise,
6480 // even the values that are not available in these blocks may come from them,
6481 // and this leads to false-positive recurrence test.
6482 for (auto *Pred : predecessors(BB: P->getParent()))
6483 if (!DT.isReachableFromEntry(A: Pred))
6484 return FullSet;
6485
6486 BinaryOperator *BO;
6487 Value *Start, *Step;
6488 if (!matchSimpleRecurrence(P, BO, Start, Step))
6489 return FullSet;
6490
6491 // If we found a recurrence in reachable code, we must be in a loop. Note
6492 // that BO might be in some subloop of L, and that's completely okay.
6493 auto *L = LI.getLoopFor(BB: P->getParent());
6494 assert(L && L->getHeader() == P->getParent());
6495 if (!L->contains(BB: BO->getParent()))
6496 // NOTE: This bailout should be an assert instead. However, asserting
6497 // the condition here exposes a case where LoopFusion is querying SCEV
6498 // with malformed loop information during the midst of the transform.
6499 // There doesn't appear to be an obvious fix, so for the moment bailout
6500 // until the caller issue can be fixed. PR49566 tracks the bug.
6501 return FullSet;
6502
6503 // TODO: Extend to other opcodes such as mul, and div
6504 switch (BO->getOpcode()) {
6505 default:
6506 return FullSet;
6507 case Instruction::AShr:
6508 case Instruction::LShr:
6509 case Instruction::Shl:
6510 break;
6511 };
6512
6513 if (BO->getOperand(i_nocapture: 0) != P)
6514 // TODO: Handle the power function forms some day.
6515 return FullSet;
6516
6517 unsigned TC = getSmallConstantMaxTripCount(L);
6518 if (!TC || TC >= BitWidth)
6519 return FullSet;
6520
6521 auto KnownStart = computeKnownBits(V: Start, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6522 auto KnownStep = computeKnownBits(V: Step, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6523 assert(KnownStart.getBitWidth() == BitWidth &&
6524 KnownStep.getBitWidth() == BitWidth);
6525
6526 // Compute total shift amount, being careful of overflow and bitwidths.
6527 auto MaxShiftAmt = KnownStep.getMaxValue();
6528 APInt TCAP(BitWidth, TC-1);
6529 bool Overflow = false;
6530 auto TotalShift = MaxShiftAmt.umul_ov(RHS: TCAP, Overflow);
6531 if (Overflow)
6532 return FullSet;
6533
6534 switch (BO->getOpcode()) {
6535 default:
6536 llvm_unreachable("filtered out above");
6537 case Instruction::AShr: {
6538 // For each ashr, three cases:
6539 // shift = 0 => unchanged value
6540 // saturation => 0 or -1
6541 // other => a value closer to zero (of the same sign)
6542 // Thus, the end value is closer to zero than the start.
6543 auto KnownEnd = KnownBits::ashr(LHS: KnownStart,
6544 RHS: KnownBits::makeConstant(C: TotalShift));
6545 if (KnownStart.isNonNegative())
6546 // Analogous to lshr (simply not yet canonicalized)
6547 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6548 Upper: KnownStart.getMaxValue() + 1);
6549 if (KnownStart.isNegative())
6550 // End >=u Start && End <=s Start
6551 return ConstantRange::getNonEmpty(Lower: KnownStart.getMinValue(),
6552 Upper: KnownEnd.getMaxValue() + 1);
6553 break;
6554 }
6555 case Instruction::LShr: {
6556 // For each lshr, three cases:
6557 // shift = 0 => unchanged value
6558 // saturation => 0
6559 // other => a smaller positive number
6560 // Thus, the low end of the unsigned range is the last value produced.
6561 auto KnownEnd = KnownBits::lshr(LHS: KnownStart,
6562 RHS: KnownBits::makeConstant(C: TotalShift));
6563 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6564 Upper: KnownStart.getMaxValue() + 1);
6565 }
6566 case Instruction::Shl: {
6567 // Iff no bits are shifted out, value increases on every shift.
6568 auto KnownEnd = KnownBits::shl(LHS: KnownStart,
6569 RHS: KnownBits::makeConstant(C: TotalShift));
6570 if (TotalShift.ult(RHS: KnownStart.countMinLeadingZeros()))
6571 return ConstantRange(KnownStart.getMinValue(),
6572 KnownEnd.getMaxValue() + 1);
6573 break;
6574 }
6575 };
6576 return FullSet;
6577}
6578
6579// The goal of this function is to check if recursively visiting the operands
6580// of this PHI might lead to an infinite loop. If we do see such a loop,
6581// there's no good way to break it, so we avoid analyzing such cases.
6582//
6583// getRangeRef previously used a visited set to avoid infinite loops, but this
6584// caused other issues: the result was dependent on the order of getRangeRef
6585// calls, and the interaction with createSCEVIter could cause a stack overflow
6586// in some cases (see issue #148253).
6587//
6588// FIXME: The way this is implemented is overly conservative; this checks
6589// for a few obviously safe patterns, but anything that doesn't lead to
6590// recursion is fine.
6591static bool RangeRefPHIAllowedOperands(DominatorTree &DT, PHINode *PHI) {
6592 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
6593 if (getOperandsForSelectLikePHI(DT, PN: PHI, Cond, LHS, RHS))
6594 return true;
6595
6596 if (all_of(Range: PHI->operands(),
6597 P: [&](Value *Operand) { return DT.dominates(Def: Operand, User: PHI); }))
6598 return true;
6599
6600 return false;
6601}
6602
6603const ConstantRange &
6604ScalarEvolution::getRangeRefIter(const SCEV *S,
6605 ScalarEvolution::RangeSignHint SignHint) {
6606 DenseMap<const SCEV *, ConstantRange> &Cache =
6607 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6608 : SignedRanges;
6609 SmallVector<SCEVUse> WorkList;
6610 SmallPtrSet<const SCEV *, 8> Seen;
6611
6612 // Add Expr to the worklist, if Expr is either an N-ary expression or a
6613 // SCEVUnknown PHI node.
6614 auto AddToWorklist = [&WorkList, &Seen, &Cache](const SCEV *Expr) {
6615 if (!Seen.insert(Ptr: Expr).second)
6616 return;
6617 if (Cache.contains(Val: Expr))
6618 return;
6619 switch (Expr->getSCEVType()) {
6620 case scUnknown:
6621 if (!isa<PHINode>(Val: cast<SCEVUnknown>(Val: Expr)->getValue()))
6622 break;
6623 [[fallthrough]];
6624 case scConstant:
6625 case scVScale:
6626 case scTruncate:
6627 case scZeroExtend:
6628 case scSignExtend:
6629 case scPtrToAddr:
6630 case scAddExpr:
6631 case scMulExpr:
6632 case scUDivExpr:
6633 case scAddRecExpr:
6634 case scUMaxExpr:
6635 case scSMaxExpr:
6636 case scUMinExpr:
6637 case scSMinExpr:
6638 case scSequentialUMinExpr:
6639 WorkList.push_back(Elt: Expr);
6640 break;
6641 case scCouldNotCompute:
6642 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6643 }
6644 };
6645 AddToWorklist(S);
6646
6647 // Build worklist by queuing operands of N-ary expressions and phi nodes.
6648 for (unsigned I = 0; I != WorkList.size(); ++I) {
6649 const SCEV *P = WorkList[I];
6650 auto *UnknownS = dyn_cast<SCEVUnknown>(Val: P);
6651 // If it is not a `SCEVUnknown`, just recurse into operands.
6652 if (!UnknownS) {
6653 for (const SCEV *Op : P->operands())
6654 AddToWorklist(Op);
6655 continue;
6656 }
6657 // `SCEVUnknown`'s require special treatment.
6658 if (PHINode *P = dyn_cast<PHINode>(Val: UnknownS->getValue())) {
6659 if (!RangeRefPHIAllowedOperands(DT, PHI: P))
6660 continue;
6661 for (auto &Op : reverse(C: P->operands()))
6662 AddToWorklist(getSCEV(V: Op));
6663 }
6664 }
6665
6666 if (!WorkList.empty()) {
6667 // Use getRangeRef to compute ranges for items in the worklist in reverse
6668 // order. This will force ranges for earlier operands to be computed before
6669 // their users in most cases.
6670 for (const SCEV *P : reverse(C: drop_begin(RangeOrContainer&: WorkList))) {
6671 getRangeRef(S: P, Hint: SignHint);
6672 }
6673 }
6674
6675 return getRangeRef(S, Hint: SignHint, Depth: 0);
6676}
6677
6678const APInt *ScalarEvolution::getConstantAPIntOrNull(const SCEV *S) {
6679 if (const auto *C = dyn_cast<SCEVConstant>(Val: S))
6680 return &C->getAPInt();
6681 return nullptr;
6682}
6683
6684/// Determine the range for a particular SCEV. If SignHint is
6685/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
6686/// with a "cleaner" unsigned (resp. signed) representation.
6687const ConstantRange &ScalarEvolution::getRangeRef(
6688 const SCEV *S, ScalarEvolution::RangeSignHint SignHint, unsigned Depth) {
6689 DenseMap<const SCEV *, ConstantRange> &Cache =
6690 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6691 : SignedRanges;
6692 ConstantRange::PreferredRangeType RangeType =
6693 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? ConstantRange::Unsigned
6694 : ConstantRange::Signed;
6695
6696 // See if we've computed this range already.
6697 auto I = Cache.find(Val: S);
6698 if (I != Cache.end())
6699 return I->second;
6700
6701 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: S))
6702 return setRange(S: C, Hint: SignHint, CR: ConstantRange(C->getAPInt()));
6703
6704 // Switch to iteratively computing the range for S, if it is part of a deeply
6705 // nested expression.
6706 if (Depth > RangeIterThreshold)
6707 return getRangeRefIter(S, SignHint);
6708
6709 unsigned BitWidth = getTypeSizeInBits(Ty: S->getType());
6710 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
6711 using OBO = OverflowingBinaryOperator;
6712
6713 // If the value has known zeros, the maximum value will have those known zeros
6714 // as well.
6715 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
6716 APInt Multiple = getNonZeroConstantMultiple(S);
6717 APInt Remainder = APInt::getMaxValue(numBits: BitWidth).urem(RHS: Multiple);
6718 if (!Remainder.isZero())
6719 ConservativeResult =
6720 ConstantRange(APInt::getMinValue(numBits: BitWidth),
6721 APInt::getMaxValue(numBits: BitWidth) - Remainder + 1);
6722 }
6723 else {
6724 uint32_t TZ = getMinTrailingZeros(S);
6725 if (TZ != 0) {
6726 ConservativeResult = ConstantRange(
6727 APInt::getSignedMinValue(numBits: BitWidth),
6728 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: TZ).shl(shiftAmt: TZ) + 1);
6729 }
6730 }
6731
6732 switch (S->getSCEVType()) {
6733 case scConstant:
6734 llvm_unreachable("Already handled above.");
6735 case scVScale:
6736 return setRange(S, Hint: SignHint, CR: getVScaleRange(F: &F, BitWidth));
6737 case scTruncate: {
6738 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: S);
6739 ConstantRange X = getRangeRef(S: Trunc->getOperand(), SignHint, Depth: Depth + 1);
6740 return setRange(
6741 S: Trunc, Hint: SignHint,
6742 CR: ConservativeResult.intersectWith(CR: X.truncate(BitWidth), Type: RangeType));
6743 }
6744 case scZeroExtend: {
6745 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: S);
6746 ConstantRange X = getRangeRef(S: ZExt->getOperand(), SignHint, Depth: Depth + 1);
6747 return setRange(
6748 S: ZExt, Hint: SignHint,
6749 CR: ConservativeResult.intersectWith(CR: X.zeroExtend(BitWidth), Type: RangeType));
6750 }
6751 case scSignExtend: {
6752 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: S);
6753 ConstantRange X = getRangeRef(S: SExt->getOperand(), SignHint, Depth: Depth + 1);
6754 return setRange(
6755 S: SExt, Hint: SignHint,
6756 CR: ConservativeResult.intersectWith(CR: X.signExtend(BitWidth), Type: RangeType));
6757 }
6758 case scPtrToAddr: {
6759 const SCEVCastExpr *Cast = cast<SCEVCastExpr>(Val: S);
6760 ConstantRange X = getRangeRef(S: Cast->getOperand(), SignHint, Depth: Depth + 1);
6761 return setRange(S: Cast, Hint: SignHint, CR: X);
6762 }
6763 case scAddExpr: {
6764 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: S);
6765 // Check if this is a URem pattern: A - (A / B) * B, which is always < B.
6766 const SCEV *URemLHS = nullptr, *URemRHS = nullptr;
6767 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED &&
6768 match(S, P: m_scev_URem(LHS: m_SCEV(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE&: *this))) {
6769 ConstantRange LHSRange = getRangeRef(S: URemLHS, SignHint, Depth: Depth + 1);
6770 ConstantRange RHSRange = getRangeRef(S: URemRHS, SignHint, Depth: Depth + 1);
6771 ConservativeResult =
6772 ConservativeResult.intersectWith(CR: LHSRange.urem(Other: RHSRange), Type: RangeType);
6773 }
6774 ConstantRange X = getRangeRef(S: Add->getOperand(i: 0), SignHint, Depth: Depth + 1);
6775 unsigned WrapType = OBO::AnyWrap;
6776 if (Add->hasNoSignedWrap())
6777 WrapType |= OBO::NoSignedWrap;
6778 if (Add->hasNoUnsignedWrap())
6779 WrapType |= OBO::NoUnsignedWrap;
6780 for (const SCEV *Op : drop_begin(RangeOrContainer: Add->operands()))
6781 X = X.addWithNoWrap(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1), NoWrapKind: WrapType,
6782 RangeType);
6783 return setRange(S: Add, Hint: SignHint,
6784 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6785 }
6786 case scMulExpr: {
6787 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val: S);
6788 ConstantRange X = getRangeRef(S: Mul->getOperand(i: 0), SignHint, Depth: Depth + 1);
6789 for (const SCEV *Op : drop_begin(RangeOrContainer: Mul->operands()))
6790 X = X.multiply(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1));
6791 return setRange(S: Mul, Hint: SignHint,
6792 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6793 }
6794 case scUDivExpr: {
6795 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: S);
6796 ConstantRange X = getRangeRef(S: UDiv->getLHS(), SignHint, Depth: Depth + 1);
6797 ConstantRange Y = getRangeRef(S: UDiv->getRHS(), SignHint, Depth: Depth + 1);
6798 return setRange(S: UDiv, Hint: SignHint,
6799 CR: ConservativeResult.intersectWith(CR: X.udiv(Other: Y), Type: RangeType));
6800 }
6801 case scAddRecExpr: {
6802 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: S);
6803 // If there's no unsigned wrap, the value will never be less than its
6804 // initial value.
6805 if (AddRec->hasNoUnsignedWrap()) {
6806 APInt UnsignedMinValue = getUnsignedRangeMin(S: AddRec->getStart());
6807 if (!UnsignedMinValue.isZero())
6808 ConservativeResult = ConservativeResult.intersectWith(
6809 CR: ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), Type: RangeType);
6810 }
6811
6812 // If there's no signed wrap, and all the operands except initial value have
6813 // the same sign or zero, the value won't ever be:
6814 // 1: smaller than initial value if operands are non negative,
6815 // 2: bigger than initial value if operands are non positive.
6816 // For both cases, value can not cross signed min/max boundary.
6817 if (AddRec->hasNoSignedWrap()) {
6818 bool AllNonNeg = true;
6819 bool AllNonPos = true;
6820 for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
6821 if (!isKnownNonNegative(S: AddRec->getOperand(i)))
6822 AllNonNeg = false;
6823 if (!isKnownNonPositive(S: AddRec->getOperand(i)))
6824 AllNonPos = false;
6825 }
6826 if (AllNonNeg)
6827 ConservativeResult = ConservativeResult.intersectWith(
6828 CR: ConstantRange::getNonEmpty(Lower: getSignedRangeMin(S: AddRec->getStart()),
6829 Upper: APInt::getSignedMinValue(numBits: BitWidth)),
6830 Type: RangeType);
6831 else if (AllNonPos)
6832 ConservativeResult = ConservativeResult.intersectWith(
6833 CR: ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
6834 Upper: getSignedRangeMax(S: AddRec->getStart()) +
6835 1),
6836 Type: RangeType);
6837 }
6838
6839 // TODO: non-affine addrec
6840 if (AddRec->isAffine()) {
6841 const SCEV *MaxBEScev =
6842 getConstantMaxBackedgeTakenCount(L: AddRec->getLoop());
6843 if (!isa<SCEVCouldNotCompute>(Val: MaxBEScev)) {
6844 APInt MaxBECount = cast<SCEVConstant>(Val: MaxBEScev)->getAPInt();
6845
6846 // Adjust MaxBECount to the same bitwidth as AddRec. We can truncate if
6847 // MaxBECount's active bits are all <= AddRec's bit width.
6848 if (MaxBECount.getBitWidth() > BitWidth &&
6849 MaxBECount.getActiveBits() <= BitWidth)
6850 MaxBECount = MaxBECount.trunc(width: BitWidth);
6851 else if (MaxBECount.getBitWidth() < BitWidth)
6852 MaxBECount = MaxBECount.zext(width: BitWidth);
6853
6854 if (MaxBECount.getBitWidth() == BitWidth) {
6855 auto [RangeFromAffine, Flags] = getRangeForAffineAR(
6856 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
6857 ConservativeResult =
6858 ConservativeResult.intersectWith(CR: RangeFromAffine, Type: RangeType);
6859 const_cast<SCEVAddRecExpr *>(AddRec)->setNoWrapFlags(Flags);
6860
6861 auto RangeFromFactoring = getRangeViaFactoring(
6862 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
6863 ConservativeResult =
6864 ConservativeResult.intersectWith(CR: RangeFromFactoring, Type: RangeType);
6865 }
6866 }
6867
6868 // Now try symbolic BE count and more powerful methods.
6869 if (UseExpensiveRangeSharpening) {
6870 const SCEV *SymbolicMaxBECount =
6871 getSymbolicMaxBackedgeTakenCount(L: AddRec->getLoop());
6872 if (!isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount) &&
6873 getTypeSizeInBits(Ty: MaxBEScev->getType()) <= BitWidth &&
6874 AddRec->hasNoSelfWrap()) {
6875 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
6876 AddRec, MaxBECount: SymbolicMaxBECount, BitWidth, SignHint);
6877 ConservativeResult =
6878 ConservativeResult.intersectWith(CR: RangeFromAffineNew, Type: RangeType);
6879 }
6880 }
6881 }
6882
6883 return setRange(S: AddRec, Hint: SignHint, CR: std::move(ConservativeResult));
6884 }
6885 case scUMaxExpr:
6886 case scSMaxExpr:
6887 case scUMinExpr:
6888 case scSMinExpr:
6889 case scSequentialUMinExpr: {
6890 Intrinsic::ID ID;
6891 switch (S->getSCEVType()) {
6892 case scUMaxExpr:
6893 ID = Intrinsic::umax;
6894 break;
6895 case scSMaxExpr:
6896 ID = Intrinsic::smax;
6897 break;
6898 case scUMinExpr:
6899 case scSequentialUMinExpr:
6900 ID = Intrinsic::umin;
6901 break;
6902 case scSMinExpr:
6903 ID = Intrinsic::smin;
6904 break;
6905 default:
6906 llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.");
6907 }
6908
6909 const auto *NAry = cast<SCEVNAryExpr>(Val: S);
6910 ConstantRange X = getRangeRef(S: NAry->getOperand(i: 0), SignHint, Depth: Depth + 1);
6911 for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
6912 X = X.intrinsic(
6913 IntrinsicID: ID, Ops: {X, getRangeRef(S: NAry->getOperand(i), SignHint, Depth: Depth + 1)});
6914 return setRange(S, Hint: SignHint,
6915 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6916 }
6917 case scUnknown: {
6918 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6919 Value *V = U->getValue();
6920
6921 // Check if the IR explicitly contains !range metadata.
6922 std::optional<ConstantRange> MDRange = GetRangeFromMetadata(V);
6923 if (MDRange)
6924 ConservativeResult =
6925 ConservativeResult.intersectWith(CR: *MDRange, Type: RangeType);
6926
6927 // Use facts about recurrences in the underlying IR. Note that add
6928 // recurrences are AddRecExprs and thus don't hit this path. This
6929 // primarily handles shift recurrences.
6930 auto CR = getRangeForUnknownRecurrence(U);
6931 ConservativeResult = ConservativeResult.intersectWith(CR);
6932
6933 // See if ValueTracking can give us a useful range.
6934 const DataLayout &DL = getDataLayout();
6935 KnownBits Known = computeKnownBits(V, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6936 if (Known.getBitWidth() != BitWidth)
6937 Known = Known.zextOrTrunc(BitWidth);
6938
6939 // ValueTracking may be able to compute a tighter result for the number of
6940 // sign bits than for the value of those sign bits.
6941 unsigned NS = ComputeNumSignBits(Op: V, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6942 if (U->getType()->isPointerTy()) {
6943 // If the pointer size is larger than the index size type, this can cause
6944 // NS to be larger than BitWidth. So compensate for this.
6945 unsigned ptrSize = DL.getPointerTypeSizeInBits(U->getType());
6946 int ptrIdxDiff = ptrSize - BitWidth;
6947 if (ptrIdxDiff > 0 && ptrSize > BitWidth && NS > (unsigned)ptrIdxDiff)
6948 NS -= ptrIdxDiff;
6949 }
6950
6951 if (NS > 1) {
6952 // If we know any of the sign bits, we know all of the sign bits.
6953 if (!Known.Zero.getHiBits(numBits: NS).isZero())
6954 Known.Zero.setHighBits(NS);
6955 if (!Known.One.getHiBits(numBits: NS).isZero())
6956 Known.One.setHighBits(NS);
6957 }
6958
6959 if (Known.getMinValue() != Known.getMaxValue() + 1)
6960 ConservativeResult = ConservativeResult.intersectWith(
6961 CR: ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
6962 Type: RangeType);
6963 if (NS > 1)
6964 ConservativeResult = ConservativeResult.intersectWith(
6965 CR: ConstantRange(APInt::getSignedMinValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1),
6966 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1) + 1),
6967 Type: RangeType);
6968
6969 if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
6970 // Strengthen the range if the underlying IR value is a
6971 // global/alloca/heap allocation using the size of the object.
6972 bool CanBeNull;
6973 uint64_t DerefBytes = V->getPointerDereferenceableBytes(
6974 DL, CanBeNull, /*CanBeFreed=*/nullptr);
6975 if (DerefBytes > 1 && isUIntN(N: BitWidth, x: DerefBytes)) {
6976 // The highest address the object can start is DerefBytes bytes before
6977 // the end (unsigned max value). If this value is not a multiple of the
6978 // alignment, the last possible start value is the next lowest multiple
6979 // of the alignment. Note: The computations below cannot overflow,
6980 // because if they would there's no possible start address for the
6981 // object.
6982 APInt MaxVal =
6983 APInt::getMaxValue(numBits: BitWidth) - APInt(BitWidth, DerefBytes);
6984 uint64_t Align = U->getValue()->getPointerAlignment(DL).value();
6985 uint64_t Rem = MaxVal.urem(RHS: Align);
6986 MaxVal -= APInt(BitWidth, Rem);
6987 APInt MinVal = APInt::getZero(numBits: BitWidth);
6988 if (llvm::isKnownNonZero(V, Q: DL))
6989 MinVal = Align;
6990 ConservativeResult = ConservativeResult.intersectWith(
6991 CR: ConstantRange::getNonEmpty(Lower: MinVal, Upper: MaxVal + 1), Type: RangeType);
6992 }
6993 }
6994
6995 // A range of Phi is a subset of union of all ranges of its input.
6996 if (PHINode *Phi = dyn_cast<PHINode>(Val: V)) {
6997 // SCEVExpander sometimes creates SCEVUnknowns that are secretly
6998 // AddRecs; return the range for the corresponding AddRec.
6999 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: getSCEV(V)))
7000 return getRangeRef(S: AR, SignHint, Depth: Depth + 1);
7001
7002 // Make sure that we do not run over cycled Phis.
7003 if (RangeRefPHIAllowedOperands(DT, PHI: Phi)) {
7004 ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
7005
7006 for (const auto &Op : Phi->operands()) {
7007 auto OpRange = getRangeRef(S: getSCEV(V: Op), SignHint, Depth: Depth + 1);
7008 RangeFromOps = RangeFromOps.unionWith(CR: OpRange);
7009 // No point to continue if we already have a full set.
7010 if (RangeFromOps.isFullSet())
7011 break;
7012 }
7013 ConservativeResult =
7014 ConservativeResult.intersectWith(CR: RangeFromOps, Type: RangeType);
7015 }
7016 }
7017
7018 // vscale can't be equal to zero
7019 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V))
7020 if (II->getIntrinsicID() == Intrinsic::vscale) {
7021 ConstantRange Disallowed = APInt::getZero(numBits: BitWidth);
7022 ConservativeResult = ConservativeResult.difference(CR: Disallowed);
7023 }
7024
7025 return setRange(S: U, Hint: SignHint, CR: std::move(ConservativeResult));
7026 }
7027 case scCouldNotCompute:
7028 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
7029 }
7030
7031 return setRange(S, Hint: SignHint, CR: std::move(ConservativeResult));
7032}
7033
7034// Given a StartRange, Step and MaxBECount for an expression compute a range of
7035// values that the expression can take. Initially, the expression has a value
7036// from StartRange and then is changed by Step up to MaxBECount times. Signed
7037// argument defines if we treat Step as signed or unsigned. The second return
7038// value indicates that no wrapping occurred.
7039static std::pair<ConstantRange, bool>
7040getRangeForAffineARHelper(APInt Step, const ConstantRange &StartRange,
7041 const APInt &MaxBECount, bool Signed) {
7042 unsigned BitWidth = Step.getBitWidth();
7043 assert(BitWidth == StartRange.getBitWidth() &&
7044 BitWidth == MaxBECount.getBitWidth() && "mismatched bit widths");
7045 // If either Step or MaxBECount is 0, then the expression won't change, and we
7046 // just need to return the initial range.
7047 if (Step == 0 || MaxBECount == 0)
7048 return {StartRange, true};
7049
7050 // If we don't know anything about the initial value (i.e. StartRange is
7051 // FullRange), then we don't know anything about the final range either.
7052 // Return FullRange.
7053 if (StartRange.isFullSet())
7054 return {ConstantRange::getFull(BitWidth), false};
7055
7056 // If Step is signed and negative, then we use its absolute value, but we also
7057 // note that we're moving in the opposite direction.
7058 bool Descending = Signed && Step.isNegative();
7059
7060 if (Signed)
7061 // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
7062 // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
7063 // This equations hold true due to the well-defined wrap-around behavior of
7064 // APInt.
7065 Step = Step.abs();
7066
7067 // Check if Offset is more than full span of BitWidth. If it is, the
7068 // expression is guaranteed to overflow.
7069 if (APInt::getMaxValue(numBits: StartRange.getBitWidth()).udiv(RHS: Step).ult(RHS: MaxBECount))
7070 return {ConstantRange::getFull(BitWidth), false};
7071
7072 // Offset is by how much the expression can change. Checks above guarantee no
7073 // overflow here.
7074 APInt Offset = Step * MaxBECount;
7075
7076 // Minimum value of the final range will match the minimal value of StartRange
7077 // if the expression is increasing and will be decreased by Offset otherwise.
7078 // Maximum value of the final range will match the maximal value of StartRange
7079 // if the expression is decreasing and will be increased by Offset otherwise.
7080 APInt StartLower = StartRange.getLower();
7081 APInt StartUpper = StartRange.getUpper() - 1;
7082 bool Overflow;
7083 APInt MovedBoundary;
7084 if (Signed) {
7085 // This does not use sadd_ov, as we want to check overflow for a signed
7086 // start with an unsigned offset.
7087 if (Descending) {
7088 MovedBoundary = StartLower - std::move(Offset);
7089 Overflow = MovedBoundary.sgt(RHS: StartLower) || StartRange.isSignWrappedSet();
7090 } else {
7091 MovedBoundary = StartUpper + std::move(Offset);
7092 Overflow = MovedBoundary.slt(RHS: StartUpper) || StartRange.isSignWrappedSet();
7093 }
7094 } else {
7095 MovedBoundary = StartUpper.uadd_ov(RHS: std::move(Offset), Overflow);
7096 Overflow |= StartRange.isWrappedSet();
7097 }
7098
7099 // It's possible that the new minimum/maximum value will fall into the initial
7100 // range (due to wrap around). This means that the expression can take any
7101 // value in this bitwidth, and we have to return full range.
7102 if (StartRange.contains(Val: MovedBoundary))
7103 return {ConstantRange::getFull(BitWidth), false};
7104
7105 APInt NewLower =
7106 Descending ? std::move(MovedBoundary) : std::move(StartLower);
7107 APInt NewUpper =
7108 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
7109 NewUpper += 1;
7110
7111 // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
7112 return {ConstantRange::getNonEmpty(Lower: std::move(NewLower), Upper: std::move(NewUpper)),
7113 !Overflow};
7114}
7115
7116std::pair<ConstantRange, SCEV::NoWrapFlags>
7117ScalarEvolution::getRangeForAffineAR(const SCEV *Start, const SCEV *Step,
7118 const APInt &MaxBECount) {
7119 assert(getTypeSizeInBits(Start->getType()) ==
7120 getTypeSizeInBits(Step->getType()) &&
7121 getTypeSizeInBits(Start->getType()) == MaxBECount.getBitWidth() &&
7122 "mismatched bit widths");
7123
7124 // First, consider step signed.
7125 ConstantRange StartSRange = getSignedRange(S: Start);
7126 ConstantRange StepSRange = getSignedRange(S: Step);
7127
7128 // If Step can be both positive and negative, we need to find ranges for the
7129 // maximum absolute step values in both directions and union them.
7130 auto [SR1, NSW1] = getRangeForAffineARHelper(
7131 Step: StepSRange.getSignedMin(), StartRange: StartSRange, MaxBECount, /*Signed=*/true);
7132 auto [SR2, NSW2] = getRangeForAffineARHelper(Step: StepSRange.getSignedMax(),
7133 StartRange: StartSRange, MaxBECount,
7134 /*Signed=*/true);
7135 ConstantRange SR = SR1.unionWith(CR: SR2);
7136
7137 // Next, consider step unsigned.
7138 auto [UR, NUW] = getRangeForAffineARHelper(
7139 Step: getUnsignedRangeMax(S: Step), StartRange: getUnsignedRange(S: Start), MaxBECount,
7140 /*Signed=*/false);
7141
7142 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7143 if (NUW)
7144 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7145 if (NSW1 && NSW2)
7146 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7147
7148 // Finally, intersect signed and unsigned ranges.
7149 return {SR.intersectWith(CR: UR, Type: ConstantRange::Smallest), Flags};
7150}
7151
7152ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
7153 const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
7154 ScalarEvolution::RangeSignHint SignHint) {
7155 assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n");
7156 assert(AddRec->hasNoSelfWrap() &&
7157 "This only works for non-self-wrapping AddRecs!");
7158 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
7159 const SCEV *Step = AddRec->getStepRecurrence(SE&: *this);
7160 // Only deal with constant step to save compile time.
7161 if (!isa<SCEVConstant>(Val: Step))
7162 return ConstantRange::getFull(BitWidth);
7163 // Let's make sure that we can prove that we do not self-wrap during
7164 // MaxBECount iterations. We need this because MaxBECount is a maximum
7165 // iteration count estimate, and we might infer nw from some exit for which we
7166 // do not know max exit count (or any other side reasoning).
7167 // TODO: Turn into assert at some point.
7168 if (getTypeSizeInBits(Ty: MaxBECount->getType()) >
7169 getTypeSizeInBits(Ty: AddRec->getType()))
7170 return ConstantRange::getFull(BitWidth);
7171 MaxBECount = getNoopOrZeroExtend(V: MaxBECount, Ty: AddRec->getType());
7172 const SCEV *RangeWidth = getMinusOne(Ty: AddRec->getType());
7173 const SCEV *StepAbs = getUMinExpr(LHS: Step, RHS: getNegativeSCEV(V: Step));
7174 const SCEV *MaxItersWithoutWrap = getUDivExpr(LHS: RangeWidth, RHS: StepAbs);
7175 if (!isKnownPredicateViaConstantRanges(Pred: ICmpInst::ICMP_ULE, LHS: MaxBECount,
7176 RHS: MaxItersWithoutWrap))
7177 return ConstantRange::getFull(BitWidth);
7178
7179 ICmpInst::Predicate LEPred =
7180 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
7181 ICmpInst::Predicate GEPred =
7182 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
7183 const SCEV *End = AddRec->evaluateAtIteration(It: MaxBECount, SE&: *this);
7184
7185 // We know that there is no self-wrap. Let's take Start and End values and
7186 // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
7187 // the iteration. They either lie inside the range [Min(Start, End),
7188 // Max(Start, End)] or outside it:
7189 //
7190 // Case 1: RangeMin ... Start V1 ... VN End ... RangeMax;
7191 // Case 2: RangeMin Vk ... V1 Start ... End Vn ... Vk + 1 RangeMax;
7192 //
7193 // No self wrap flag guarantees that the intermediate values cannot be BOTH
7194 // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
7195 // knowledge, let's try to prove that we are dealing with Case 1. It is so if
7196 // Start <= End and step is positive, or Start >= End and step is negative.
7197 const SCEV *Start = applyLoopGuards(Expr: AddRec->getStart(), L: AddRec->getLoop());
7198 ConstantRange StartRange = getRangeRef(S: Start, SignHint);
7199 ConstantRange EndRange = getRangeRef(S: End, SignHint);
7200 ConstantRange RangeBetween = StartRange.unionWith(CR: EndRange);
7201 // If they already cover full iteration space, we will know nothing useful
7202 // even if we prove what we want to prove.
7203 if (RangeBetween.isFullSet())
7204 return RangeBetween;
7205 // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
7206 bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
7207 : RangeBetween.isWrappedSet();
7208 if (IsWrappedSet)
7209 return ConstantRange::getFull(BitWidth);
7210
7211 if (isKnownPositive(S: Step) &&
7212 isKnownPredicateViaConstantRanges(Pred: LEPred, LHS: Start, RHS: End))
7213 return RangeBetween;
7214 if (isKnownNegative(S: Step) &&
7215 isKnownPredicateViaConstantRanges(Pred: GEPred, LHS: Start, RHS: End))
7216 return RangeBetween;
7217 return ConstantRange::getFull(BitWidth);
7218}
7219
7220ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
7221 const SCEV *Step,
7222 const APInt &MaxBECount) {
7223 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
7224 // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
7225
7226 unsigned BitWidth = MaxBECount.getBitWidth();
7227 assert(getTypeSizeInBits(Start->getType()) == BitWidth &&
7228 getTypeSizeInBits(Step->getType()) == BitWidth &&
7229 "mismatched bit widths");
7230
7231 struct SelectPattern {
7232 Value *Condition = nullptr;
7233 APInt TrueValue;
7234 APInt FalseValue;
7235
7236 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
7237 const SCEV *S) {
7238 std::optional<unsigned> CastOp;
7239 APInt Offset(BitWidth, 0);
7240
7241 assert(SE.getTypeSizeInBits(S->getType()) == BitWidth &&
7242 "Should be!");
7243
7244 // Peel off a constant offset. In the future we could consider being
7245 // smarter here and handle {Start+Step,+,Step} too.
7246 const APInt *Off;
7247 if (match(S, P: m_scev_Add(Op0: m_scev_APInt(C&: Off), Op1: m_SCEV(V&: S))))
7248 Offset = *Off;
7249
7250 // Peel off a cast operation
7251 if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(Val: S)) {
7252 CastOp = SCast->getSCEVType();
7253 S = SCast->getOperand();
7254 }
7255
7256 using namespace llvm::PatternMatch;
7257
7258 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
7259 const APInt *TrueVal, *FalseVal;
7260 if (!SU ||
7261 !match(V: SU->getValue(), P: m_Select(C: m_Value(V&: Condition), L: m_APInt(Res&: TrueVal),
7262 R: m_APInt(Res&: FalseVal)))) {
7263 Condition = nullptr;
7264 return;
7265 }
7266
7267 TrueValue = *TrueVal;
7268 FalseValue = *FalseVal;
7269
7270 // Re-apply the cast we peeled off earlier
7271 if (CastOp)
7272 switch (*CastOp) {
7273 default:
7274 llvm_unreachable("Unknown SCEV cast type!");
7275
7276 case scTruncate:
7277 TrueValue = TrueValue.trunc(width: BitWidth);
7278 FalseValue = FalseValue.trunc(width: BitWidth);
7279 break;
7280 case scZeroExtend:
7281 TrueValue = TrueValue.zext(width: BitWidth);
7282 FalseValue = FalseValue.zext(width: BitWidth);
7283 break;
7284 case scSignExtend:
7285 TrueValue = TrueValue.sext(width: BitWidth);
7286 FalseValue = FalseValue.sext(width: BitWidth);
7287 break;
7288 }
7289
7290 // Re-apply the constant offset we peeled off earlier
7291 TrueValue += Offset;
7292 FalseValue += Offset;
7293 }
7294
7295 bool isRecognized() { return Condition != nullptr; }
7296 };
7297
7298 SelectPattern StartPattern(*this, BitWidth, Start);
7299 if (!StartPattern.isRecognized())
7300 return ConstantRange::getFull(BitWidth);
7301
7302 SelectPattern StepPattern(*this, BitWidth, Step);
7303 if (!StepPattern.isRecognized())
7304 return ConstantRange::getFull(BitWidth);
7305
7306 if (StartPattern.Condition != StepPattern.Condition) {
7307 // We don't handle this case today; but we could, by considering four
7308 // possibilities below instead of two. I'm not sure if there are cases where
7309 // that will help over what getRange already does, though.
7310 return ConstantRange::getFull(BitWidth);
7311 }
7312
7313 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
7314 // construct arbitrary general SCEV expressions here. This function is called
7315 // from deep in the call stack, and calling getSCEV (on a sext instruction,
7316 // say) can end up caching a suboptimal value.
7317
7318 // FIXME: without the explicit `this` receiver below, MSVC errors out with
7319 // C2352 and C2512 (otherwise it isn't needed).
7320
7321 const SCEV *TrueStart = this->getConstant(Val: StartPattern.TrueValue);
7322 const SCEV *TrueStep = this->getConstant(Val: StepPattern.TrueValue);
7323 const SCEV *FalseStart = this->getConstant(Val: StartPattern.FalseValue);
7324 const SCEV *FalseStep = this->getConstant(Val: StepPattern.FalseValue);
7325
7326 ConstantRange TrueRange =
7327 this->getRangeForAffineAR(Start: TrueStart, Step: TrueStep, MaxBECount).first;
7328 ConstantRange FalseRange =
7329 this->getRangeForAffineAR(Start: FalseStart, Step: FalseStep, MaxBECount).first;
7330
7331 return TrueRange.unionWith(CR: FalseRange);
7332}
7333
7334SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
7335 if (isa<ConstantExpr>(Val: V)) return SCEV::FlagAnyWrap;
7336 const BinaryOperator *BinOp = cast<BinaryOperator>(Val: V);
7337
7338 // Return early if there are no flags to propagate to the SCEV.
7339 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7340 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: BinOp);
7341 PDI && PDI->isDisjoint()) {
7342 Flags = ScalarEvolution::setFlags(Flags: SCEV::FlagNUW, OnFlags: SCEV::FlagNSW);
7343 } else {
7344 if (BinOp->hasNoUnsignedWrap())
7345 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7346 if (BinOp->hasNoSignedWrap())
7347 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7348 }
7349 if (Flags == SCEV::FlagAnyWrap)
7350 return SCEV::FlagAnyWrap;
7351
7352 return isSCEVExprNeverPoison(I: BinOp) ? Flags : SCEV::FlagAnyWrap;
7353}
7354
7355const Instruction *
7356ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
7357 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
7358 return &*AddRec->getLoop()->getHeader()->begin();
7359 if (auto *U = dyn_cast<SCEVUnknown>(Val: S))
7360 if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
7361 return I;
7362 return nullptr;
7363}
7364
7365const Instruction *ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops,
7366 bool &Precise) {
7367 Precise = true;
7368 // Do a bounded search of the def relation of the requested SCEVs.
7369 SmallPtrSet<const SCEV *, 16> Visited;
7370 SmallVector<SCEVUse> Worklist;
7371 auto pushOp = [&](const SCEV *S) {
7372 if (!Visited.insert(Ptr: S).second)
7373 return;
7374 // Threshold of 30 here is arbitrary.
7375 if (Visited.size() > 30) {
7376 Precise = false;
7377 return;
7378 }
7379 Worklist.push_back(Elt: S);
7380 };
7381
7382 for (SCEVUse S : Ops)
7383 pushOp(S);
7384
7385 const Instruction *Bound = nullptr;
7386 while (!Worklist.empty()) {
7387 SCEVUse S = Worklist.pop_back_val();
7388 if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
7389 if (!Bound || DT.dominates(Def: Bound, User: DefI))
7390 Bound = DefI;
7391 } else {
7392 for (SCEVUse Op : S->operands())
7393 pushOp(Op);
7394 }
7395 }
7396 return Bound ? Bound : &*F.getEntryBlock().begin();
7397}
7398
7399const Instruction *
7400ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops) {
7401 bool Discard;
7402 return getDefiningScopeBound(Ops, Precise&: Discard);
7403}
7404
7405bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
7406 const Instruction *B) {
7407 if (A->getParent() == B->getParent() &&
7408 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7409 End: B->getIterator()))
7410 return true;
7411
7412 auto *BLoop = LI.getLoopFor(BB: B->getParent());
7413 if (BLoop && BLoop->getHeader() == B->getParent() &&
7414 BLoop->getLoopPreheader() == A->getParent() &&
7415 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7416 End: A->getParent()->end()) &&
7417 isGuaranteedToTransferExecutionToSuccessor(Begin: B->getParent()->begin(),
7418 End: B->getIterator()))
7419 return true;
7420 return false;
7421}
7422
7423bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
7424 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
7425 visitAll(Root: Op, Visitor&: PC);
7426 return PC.MaybePoison.empty();
7427}
7428
7429bool ScalarEvolution::isGuaranteedNotToCauseUB(const SCEV *Op) {
7430 return !SCEVExprContains(Root: Op, Pred: [this](const SCEV *S) {
7431 const SCEV *Op1;
7432 bool M = match(S, P: m_scev_UDiv(Op0: m_SCEV(), Op1: m_SCEV(V&: Op1)));
7433 // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
7434 // is a non-zero constant, we have to assume the UDiv may be UB.
7435 return M && (!isKnownNonZero(S: Op1) || !isGuaranteedNotToBePoison(Op: Op1));
7436 });
7437}
7438
7439bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
7440 // Only proceed if we can prove that I does not yield poison.
7441 if (!programUndefinedIfPoison(Inst: I))
7442 return false;
7443
7444 // At this point we know that if I is executed, then it does not wrap
7445 // according to at least one of NSW or NUW. If I is not executed, then we do
7446 // not know if the calculation that I represents would wrap. Multiple
7447 // instructions can map to the same SCEV. If we apply NSW or NUW from I to
7448 // the SCEV, we must guarantee no wrapping for that SCEV also when it is
7449 // derived from other instructions that map to the same SCEV. We cannot make
7450 // that guarantee for cases where I is not executed. So we need to find a
7451 // upper bound on the defining scope for the SCEV, and prove that I is
7452 // executed every time we enter that scope. When the bounding scope is a
7453 // loop (the common case), this is equivalent to proving I executes on every
7454 // iteration of that loop.
7455 SmallVector<SCEVUse> SCEVOps;
7456 for (const Use &Op : I->operands()) {
7457 // I could be an extractvalue from a call to an overflow intrinsic.
7458 // TODO: We can do better here in some cases.
7459 if (isSCEVable(Ty: Op->getType()))
7460 SCEVOps.push_back(Elt: getSCEV(V: Op));
7461 }
7462 auto *DefI = getDefiningScopeBound(Ops: SCEVOps);
7463 return isGuaranteedToTransferExecutionTo(A: DefI, B: I);
7464}
7465
7466bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
7467 // If we know that \c I can never be poison period, then that's enough.
7468 if (isSCEVExprNeverPoison(I))
7469 return true;
7470
7471 // If the loop only has one exit, then we know that, if the loop is entered,
7472 // any instruction dominating that exit will be executed. If any such
7473 // instruction would result in UB, the addrec cannot be poison.
7474 //
7475 // This is basically the same reasoning as in isSCEVExprNeverPoison(), but
7476 // also handles uses outside the loop header (they just need to dominate the
7477 // single exit).
7478
7479 auto *ExitingBB = L->getExitingBlock();
7480 if (!ExitingBB || !loopHasNoAbnormalExits(L))
7481 return false;
7482
7483 SmallPtrSet<const Value *, 16> KnownPoison;
7484 SmallVector<const Instruction *, 8> Worklist;
7485
7486 // We start by assuming \c I, the post-inc add recurrence, is poison. Only
7487 // things that are known to be poison under that assumption go on the
7488 // Worklist.
7489 KnownPoison.insert(Ptr: I);
7490 Worklist.push_back(Elt: I);
7491
7492 while (!Worklist.empty()) {
7493 const Instruction *Poison = Worklist.pop_back_val();
7494
7495 for (const Use &U : Poison->uses()) {
7496 const Instruction *PoisonUser = cast<Instruction>(Val: U.getUser());
7497 if (mustTriggerUB(I: PoisonUser, KnownPoison) &&
7498 DT.dominates(A: PoisonUser->getParent(), B: ExitingBB))
7499 return true;
7500
7501 if (propagatesPoison(PoisonOp: U) && L->contains(Inst: PoisonUser))
7502 if (KnownPoison.insert(Ptr: PoisonUser).second)
7503 Worklist.push_back(Elt: PoisonUser);
7504 }
7505 }
7506
7507 return false;
7508}
7509
7510ScalarEvolution::LoopProperties
7511ScalarEvolution::getLoopProperties(const Loop *L) {
7512 using LoopProperties = ScalarEvolution::LoopProperties;
7513
7514 auto Itr = LoopPropertiesCache.find(Val: L);
7515 if (Itr == LoopPropertiesCache.end()) {
7516 auto HasSideEffects = [](Instruction *I) {
7517 if (auto *SI = dyn_cast<StoreInst>(Val: I))
7518 return !SI->isSimple();
7519
7520 if (I->mayThrow())
7521 return true;
7522
7523 // Non-volatile memset / memcpy do not count as side-effect for forward
7524 // progress.
7525 if (isa<MemIntrinsic>(Val: I) && !I->isVolatile())
7526 return false;
7527
7528 return I->mayWriteToMemory();
7529 };
7530
7531 LoopProperties LP = {/* HasNoAbnormalExits */ true,
7532 /*HasNoSideEffects*/ true};
7533
7534 for (auto *BB : L->getBlocks())
7535 for (auto &I : *BB) {
7536 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7537 LP.HasNoAbnormalExits = false;
7538 if (HasSideEffects(&I))
7539 LP.HasNoSideEffects = false;
7540 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7541 break; // We're already as pessimistic as we can get.
7542 }
7543
7544 auto InsertPair = LoopPropertiesCache.insert(KV: {L, LP});
7545 assert(InsertPair.second && "We just checked!");
7546 Itr = InsertPair.first;
7547 }
7548
7549 return Itr->second;
7550}
7551
7552bool ScalarEvolution::loopIsFiniteByAssumption(const Loop *L) {
7553 // A mustprogress loop without side effects must be finite.
7554 // TODO: The check used here is very conservative. It's only *specific*
7555 // side effects which are well defined in infinite loops.
7556 return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L));
7557}
7558
7559const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
7560 // Worklist item with a Value and a bool indicating whether all operands have
7561 // been visited already.
7562 using PointerTy = PointerIntPair<Value *, 1, bool>;
7563 SmallVector<PointerTy> Stack;
7564
7565 Stack.emplace_back(Args&: V, Args: false);
7566 while (!Stack.empty()) {
7567 auto E = Stack.back();
7568 Value *CurV = E.getPointer();
7569
7570 if (getExistingSCEV(V: CurV)) {
7571 Stack.pop_back();
7572 continue;
7573 }
7574
7575 SmallVector<Value *> Ops;
7576 const SCEV *CreatedSCEV = nullptr;
7577 // If all operands have been visited already, create the SCEV.
7578 if (E.getInt()) {
7579 CreatedSCEV = createSCEV(V: CurV);
7580 } else {
7581 // Otherwise get the operands we need to create SCEV's for before creating
7582 // the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
7583 // just use it.
7584 CreatedSCEV = getOperandsToCreate(V: CurV, Ops);
7585 }
7586
7587 if (CreatedSCEV) {
7588 insertValueToMap(V: CurV, S: CreatedSCEV);
7589 Stack.pop_back();
7590 } else {
7591 Stack.back().setInt(true);
7592 // Queue its operands which need to be constructed.
7593 for (Value *Op : Ops)
7594 Stack.emplace_back(Args&: Op, Args: false);
7595 }
7596 }
7597
7598 return getExistingSCEV(V);
7599}
7600
7601const SCEV *
7602ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
7603 if (!isSCEVable(Ty: V->getType()))
7604 return getUnknown(V);
7605
7606 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7607 // Don't attempt to analyze instructions in blocks that aren't
7608 // reachable. Such instructions don't matter, and they aren't required
7609 // to obey basic rules for definitions dominating uses which this
7610 // analysis depends on.
7611 if (!DT.isReachableFromEntry(A: I->getParent()))
7612 return getUnknown(V: PoisonValue::get(T: V->getType()));
7613 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7614 return getConstant(V: CI);
7615 else if (isa<GlobalAlias>(Val: V))
7616 return getUnknown(V);
7617 else if (!isa<ConstantExpr>(Val: V))
7618 return getUnknown(V);
7619
7620 Operator *U = cast<Operator>(Val: V);
7621 if (auto BO =
7622 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CxtI: dyn_cast<Instruction>(Val: V))) {
7623 bool IsConstArg = isa<ConstantInt>(Val: BO->RHS);
7624 switch (BO->Opcode) {
7625 case Instruction::Add:
7626 case Instruction::Mul: {
7627 // For additions and multiplications, traverse add/mul chains for which we
7628 // can potentially create a single SCEV, to reduce the number of
7629 // get{Add,Mul}Expr calls.
7630 do {
7631 if (BO->Op) {
7632 if (BO->Op != V && getExistingSCEV(V: BO->Op)) {
7633 Ops.push_back(Elt: BO->Op);
7634 break;
7635 }
7636 }
7637 Ops.push_back(Elt: BO->RHS);
7638 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7639 CxtI: dyn_cast<Instruction>(Val: V));
7640 if (!NewBO ||
7641 (BO->Opcode == Instruction::Add &&
7642 (NewBO->Opcode != Instruction::Add &&
7643 NewBO->Opcode != Instruction::Sub)) ||
7644 (BO->Opcode == Instruction::Mul &&
7645 NewBO->Opcode != Instruction::Mul)) {
7646 Ops.push_back(Elt: BO->LHS);
7647 break;
7648 }
7649 // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
7650 // requires a SCEV for the LHS.
7651 if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
7652 auto *I = dyn_cast<Instruction>(Val: BO->Op);
7653 if (I && programUndefinedIfPoison(Inst: I)) {
7654 Ops.push_back(Elt: BO->LHS);
7655 break;
7656 }
7657 }
7658 BO = NewBO;
7659 } while (true);
7660 return nullptr;
7661 }
7662 case Instruction::Sub:
7663 case Instruction::UDiv:
7664 case Instruction::URem:
7665 break;
7666 case Instruction::AShr:
7667 case Instruction::Shl:
7668 case Instruction::Xor:
7669 if (!IsConstArg)
7670 return nullptr;
7671 break;
7672 case Instruction::And:
7673 case Instruction::Or:
7674 if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(BitWidth: 1))
7675 return nullptr;
7676 break;
7677 case Instruction::LShr:
7678 return getUnknown(V);
7679 default:
7680 llvm_unreachable("Unhandled binop");
7681 break;
7682 }
7683
7684 Ops.push_back(Elt: BO->LHS);
7685 Ops.push_back(Elt: BO->RHS);
7686 return nullptr;
7687 }
7688
7689 switch (U->getOpcode()) {
7690 case Instruction::Trunc:
7691 case Instruction::ZExt:
7692 case Instruction::SExt:
7693 case Instruction::PtrToAddr:
7694 case Instruction::PtrToInt:
7695 Ops.push_back(Elt: U->getOperand(i: 0));
7696 return nullptr;
7697
7698 case Instruction::BitCast:
7699 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType())) {
7700 Ops.push_back(Elt: U->getOperand(i: 0));
7701 return nullptr;
7702 }
7703 return getUnknown(V);
7704
7705 case Instruction::SDiv:
7706 case Instruction::SRem:
7707 Ops.push_back(Elt: U->getOperand(i: 0));
7708 Ops.push_back(Elt: U->getOperand(i: 1));
7709 return nullptr;
7710
7711 case Instruction::GetElementPtr:
7712 assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() &&
7713 "GEP source element type must be sized");
7714 llvm::append_range(C&: Ops, R: U->operands());
7715 return nullptr;
7716
7717 case Instruction::IntToPtr:
7718 return getUnknown(V);
7719
7720 case Instruction::PHI:
7721 // getNodeForPHI has four ways to turn a PHI into a SCEV; retrieve the
7722 // relevant nodes for each of them.
7723 //
7724 // The first is just to call simplifyInstruction, and get something back
7725 // that isn't a PHI.
7726 if (Value *V = simplifyInstruction(
7727 I: cast<PHINode>(Val: U),
7728 Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
7729 /*UseInstrInfo=*/true, /*CanUseUndef=*/false})) {
7730 assert(V);
7731 Ops.push_back(Elt: V);
7732 return nullptr;
7733 }
7734 // The second is createNodeForPHIWithIdenticalOperands: this looks for
7735 // operands which all perform the same operation, but haven't been
7736 // CSE'ed for whatever reason.
7737 if (BinaryOperator *BO = getCommonInstForPHI(PN: cast<PHINode>(Val: U))) {
7738 assert(BO);
7739 Ops.push_back(Elt: BO);
7740 return nullptr;
7741 }
7742 // The third is createNodeFromSelectLikePHI; this takes a PHI which
7743 // is equivalent to a select, and analyzes it like a select.
7744 {
7745 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
7746 if (getOperandsForSelectLikePHI(DT, PN: cast<PHINode>(Val: U), Cond, LHS, RHS)) {
7747 assert(Cond);
7748 assert(LHS);
7749 assert(RHS);
7750 if (auto *CondICmp = dyn_cast<ICmpInst>(Val: Cond)) {
7751 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 0));
7752 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 1));
7753 }
7754 Ops.push_back(Elt: Cond);
7755 Ops.push_back(Elt: LHS);
7756 Ops.push_back(Elt: RHS);
7757 return nullptr;
7758 }
7759 }
7760 // The fourth way is createAddRecFromPHI. It's complicated to handle here,
7761 // so just construct it recursively.
7762 //
7763 // In addition to getNodeForPHI, also construct nodes which might be needed
7764 // by getRangeRef.
7765 if (RangeRefPHIAllowedOperands(DT, PHI: cast<PHINode>(Val: U))) {
7766 for (Value *V : cast<PHINode>(Val: U)->operands())
7767 Ops.push_back(Elt: V);
7768 return nullptr;
7769 }
7770 return nullptr;
7771
7772 case Instruction::Select: {
7773 // Check if U is a select that can be simplified to a SCEVUnknown.
7774 auto CanSimplifyToUnknown = [this, U]() {
7775 if (U->getType()->isIntegerTy(BitWidth: 1) || isa<ConstantInt>(Val: U->getOperand(i: 0)))
7776 return false;
7777
7778 auto *ICI = dyn_cast<ICmpInst>(Val: U->getOperand(i: 0));
7779 if (!ICI)
7780 return false;
7781 Value *LHS = ICI->getOperand(i_nocapture: 0);
7782 Value *RHS = ICI->getOperand(i_nocapture: 1);
7783 if (ICI->getPredicate() == CmpInst::ICMP_EQ ||
7784 ICI->getPredicate() == CmpInst::ICMP_NE) {
7785 if (!(isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()))
7786 return true;
7787 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
7788 getTypeSizeInBits(Ty: U->getType()))
7789 return true;
7790 return false;
7791 };
7792 if (CanSimplifyToUnknown())
7793 return getUnknown(V: U);
7794
7795 llvm::append_range(C&: Ops, R: U->operands());
7796 return nullptr;
7797 break;
7798 }
7799 case Instruction::Call:
7800 case Instruction::Invoke:
7801 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand()) {
7802 Ops.push_back(Elt: RV);
7803 return nullptr;
7804 }
7805
7806 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
7807 switch (II->getIntrinsicID()) {
7808 case Intrinsic::abs:
7809 Ops.push_back(Elt: II->getArgOperand(i: 0));
7810 return nullptr;
7811 case Intrinsic::umax:
7812 case Intrinsic::umin:
7813 case Intrinsic::smax:
7814 case Intrinsic::smin:
7815 case Intrinsic::usub_sat:
7816 case Intrinsic::uadd_sat:
7817 Ops.push_back(Elt: II->getArgOperand(i: 0));
7818 Ops.push_back(Elt: II->getArgOperand(i: 1));
7819 return nullptr;
7820 case Intrinsic::start_loop_iterations:
7821 case Intrinsic::annotation:
7822 case Intrinsic::ptr_annotation:
7823 Ops.push_back(Elt: II->getArgOperand(i: 0));
7824 return nullptr;
7825 default:
7826 break;
7827 }
7828 }
7829 break;
7830 }
7831
7832 return nullptr;
7833}
7834
7835const SCEV *ScalarEvolution::createSCEV(Value *V) {
7836 if (!isSCEVable(Ty: V->getType()))
7837 return getUnknown(V);
7838
7839 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7840 // Don't attempt to analyze instructions in blocks that aren't
7841 // reachable. Such instructions don't matter, and they aren't required
7842 // to obey basic rules for definitions dominating uses which this
7843 // analysis depends on.
7844 if (!DT.isReachableFromEntry(A: I->getParent()))
7845 return getUnknown(V: PoisonValue::get(T: V->getType()));
7846 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7847 return getConstant(V: CI);
7848 else if (isa<GlobalAlias>(Val: V))
7849 return getUnknown(V);
7850 else if (!isa<ConstantExpr>(Val: V))
7851 return getUnknown(V);
7852
7853 const SCEV *LHS;
7854 const SCEV *RHS;
7855
7856 Operator *U = cast<Operator>(Val: V);
7857 if (auto BO =
7858 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CxtI: dyn_cast<Instruction>(Val: V))) {
7859 switch (BO->Opcode) {
7860 case Instruction::Add: {
7861 // The simple thing to do would be to just call getSCEV on both operands
7862 // and call getAddExpr with the result. However if we're looking at a
7863 // bunch of things all added together, this can be quite inefficient,
7864 // because it leads to N-1 getAddExpr calls for N ultimate operands.
7865 // Instead, gather up all the operands and make a single getAddExpr call.
7866 // LLVM IR canonical form means we need only traverse the left operands.
7867 SmallVector<SCEVUse, 4> AddOps;
7868 do {
7869 if (BO->Op) {
7870 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
7871 AddOps.push_back(Elt: OpSCEV);
7872 break;
7873 }
7874
7875 // If a NUW or NSW flag can be applied to the SCEV for this
7876 // addition, then compute the SCEV for this addition by itself
7877 // with a separate call to getAddExpr. We need to do that
7878 // instead of pushing the operands of the addition onto AddOps,
7879 // since the flags are only known to apply to this particular
7880 // addition - they may not apply to other additions that can be
7881 // formed with operands from AddOps.
7882 const SCEV *RHS = getSCEV(V: BO->RHS);
7883 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
7884 if (Flags != SCEV::FlagAnyWrap) {
7885 const SCEV *LHS = getSCEV(V: BO->LHS);
7886 if (BO->Opcode == Instruction::Sub)
7887 AddOps.push_back(Elt: getMinusSCEV(LHS, RHS, Flags));
7888 else
7889 AddOps.push_back(Elt: getAddExpr(LHS, RHS, Flags));
7890 break;
7891 }
7892 }
7893
7894 if (BO->Opcode == Instruction::Sub)
7895 AddOps.push_back(Elt: getNegativeSCEV(V: getSCEV(V: BO->RHS)));
7896 else
7897 AddOps.push_back(Elt: getSCEV(V: BO->RHS));
7898
7899 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7900 CxtI: dyn_cast<Instruction>(Val: V));
7901 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
7902 NewBO->Opcode != Instruction::Sub)) {
7903 AddOps.push_back(Elt: getSCEV(V: BO->LHS));
7904 break;
7905 }
7906 BO = NewBO;
7907 } while (true);
7908
7909 return getAddExpr(Ops&: AddOps);
7910 }
7911
7912 case Instruction::Mul: {
7913 SmallVector<SCEVUse, 4> MulOps;
7914 do {
7915 if (BO->Op) {
7916 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
7917 MulOps.push_back(Elt: OpSCEV);
7918 break;
7919 }
7920
7921 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
7922 if (Flags != SCEV::FlagAnyWrap) {
7923 LHS = getSCEV(V: BO->LHS);
7924 RHS = getSCEV(V: BO->RHS);
7925 MulOps.push_back(Elt: getMulExpr(LHS, RHS, Flags));
7926 break;
7927 }
7928 }
7929
7930 MulOps.push_back(Elt: getSCEV(V: BO->RHS));
7931 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7932 CxtI: dyn_cast<Instruction>(Val: V));
7933 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
7934 MulOps.push_back(Elt: getSCEV(V: BO->LHS));
7935 break;
7936 }
7937 BO = NewBO;
7938 } while (true);
7939
7940 return getMulExpr(Ops&: MulOps);
7941 }
7942 case Instruction::UDiv:
7943 LHS = getSCEV(V: BO->LHS);
7944 RHS = getSCEV(V: BO->RHS);
7945 return getUDivExpr(LHS, RHS);
7946 case Instruction::URem:
7947 LHS = getSCEV(V: BO->LHS);
7948 RHS = getSCEV(V: BO->RHS);
7949 return getURemExpr(LHS, RHS);
7950 case Instruction::Sub: {
7951 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7952 if (BO->Op)
7953 Flags = getNoWrapFlagsFromUB(V: BO->Op);
7954
7955 // Try to use ptrtoaddr for subtracts with at least one ptrtoint
7956 // operand. While we don't model ptrtoint directly in SCEV, the
7957 // difference between two pointer addresses is well-defined.
7958 Value *PtrLHS = nullptr, *PtrRHS = nullptr;
7959 bool HasPtrLHS = match(V: BO->LHS, P: m_PtrToInt(Op: m_Value(V&: PtrLHS)));
7960 bool HasPtrRHS = match(V: BO->RHS, P: m_PtrToInt(Op: m_Value(V&: PtrRHS)));
7961 if (HasPtrLHS || HasPtrRHS) {
7962 // Convert a ptrtoint operand (OrigOp) to ptrtoaddr of its pointer
7963 // PtrOp. When only one side is ptrtoint (BothPtr is false), skip
7964 // SCEVUnknown pointers since wrapping them in ptrtoaddr adds no
7965 // useful structure.
7966 auto GetOp = [&](bool HasPtr, Value *PtrOp, Value *OrigOp,
7967 bool BothPtr) -> const SCEV * {
7968 if (!HasPtr)
7969 return getSCEV(V: OrigOp);
7970 const SCEV *PtrSCEV = getSCEV(V: PtrOp);
7971 if (BothPtr || !isa<SCEVUnknown>(Val: PtrSCEV)) {
7972 const SCEV *Addr = getPtrToAddrExpr(Op: PtrSCEV);
7973 if (!isa<SCEVCouldNotCompute>(Val: Addr) &&
7974 getTypeSizeInBits(Ty: OrigOp->getType()) <=
7975 getTypeSizeInBits(Ty: Addr->getType()))
7976 return getTruncateOrNoop(V: Addr, Ty: OrigOp->getType());
7977 }
7978 return getSCEV(V: OrigOp);
7979 };
7980 const SCEV *L = GetOp(HasPtrLHS, PtrLHS, BO->LHS, HasPtrRHS);
7981 const SCEV *R = GetOp(HasPtrRHS, PtrRHS, BO->RHS, HasPtrLHS);
7982 return getMinusSCEV(LHS: L, RHS: R, Flags);
7983 }
7984
7985 LHS = getSCEV(V: BO->LHS);
7986 RHS = getSCEV(V: BO->RHS);
7987 return getMinusSCEV(LHS, RHS, Flags);
7988 }
7989 case Instruction::And:
7990 // For an expression like x&255 that merely masks off the high bits,
7991 // use zext(trunc(x)) as the SCEV expression.
7992 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
7993 if (CI->isZero())
7994 return getSCEV(V: BO->RHS);
7995 if (CI->isMinusOne())
7996 return getSCEV(V: BO->LHS);
7997 const APInt &A = CI->getValue();
7998
7999 // Instcombine's ShrinkDemandedConstant may strip bits out of
8000 // constants, obscuring what would otherwise be a low-bits mask.
8001 // Use computeKnownBits to compute what ShrinkDemandedConstant
8002 // knew about to reconstruct a low-bits mask value.
8003 unsigned LZ = A.countl_zero();
8004 unsigned TZ = A.countr_zero();
8005 unsigned BitWidth = A.getBitWidth();
8006 KnownBits Known(BitWidth);
8007 computeKnownBits(V: BO->LHS, Known, DL: getDataLayout(), AC: &AC, CxtI: nullptr, DT: &DT);
8008
8009 APInt EffectiveMask =
8010 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - LZ - TZ).shl(shiftAmt: TZ);
8011 if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
8012 const SCEV *MulCount = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ));
8013 const SCEV *LHS = getSCEV(V: BO->LHS);
8014 const SCEV *ShiftedLHS = nullptr;
8015 if (auto *LHSMul = dyn_cast<SCEVMulExpr>(Val: LHS)) {
8016 if (auto *OpC = dyn_cast<SCEVConstant>(Val: LHSMul->getOperand(i: 0))) {
8017 // For an expression like (x * 8) & 8, simplify the multiply.
8018 unsigned MulZeros = OpC->getAPInt().countr_zero();
8019 unsigned GCD = std::min(a: MulZeros, b: TZ);
8020 APInt DivAmt = APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ - GCD);
8021 SmallVector<SCEVUse, 4> MulOps;
8022 MulOps.push_back(Elt: getConstant(Val: OpC->getAPInt().ashr(ShiftAmt: GCD)));
8023 append_range(C&: MulOps, R: LHSMul->operands().drop_front());
8024 const SCEV *NewMul = getMulExpr(Ops&: MulOps, Flags: LHSMul->getNoWrapFlags());
8025 ShiftedLHS = getUDivExpr(LHS: NewMul, RHS: getConstant(Val: DivAmt));
8026 }
8027 }
8028 if (!ShiftedLHS)
8029 ShiftedLHS = getUDivExpr(LHS, RHS: MulCount);
8030 return getMulExpr(
8031 LHS: getZeroExtendExpr(
8032 Op: getTruncateExpr(Op: ShiftedLHS,
8033 Ty: IntegerType::get(C&: getContext(), NumBits: BitWidth - LZ - TZ)),
8034 Ty: BO->LHS->getType()),
8035 RHS: MulCount);
8036 }
8037 }
8038 // Binary `and` is a bit-wise `umin`.
8039 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
8040 LHS = getSCEV(V: BO->LHS);
8041 RHS = getSCEV(V: BO->RHS);
8042 return getUMinExpr(LHS, RHS);
8043 }
8044 break;
8045
8046 case Instruction::Or:
8047 // Binary `or` is a bit-wise `umax`.
8048 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
8049 LHS = getSCEV(V: BO->LHS);
8050 RHS = getSCEV(V: BO->RHS);
8051 return getUMaxExpr(LHS, RHS);
8052 }
8053 break;
8054
8055 case Instruction::Xor:
8056 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8057 // If the RHS of xor is -1, then this is a not operation.
8058 if (CI->isMinusOne())
8059 return getNotSCEV(V: getSCEV(V: BO->LHS));
8060
8061 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
8062 // This is a variant of the check for xor with -1, and it handles
8063 // the case where instcombine has trimmed non-demanded bits out
8064 // of an xor with -1.
8065 if (auto *LBO = dyn_cast<BinaryOperator>(Val: BO->LHS))
8066 if (ConstantInt *LCI = dyn_cast<ConstantInt>(Val: LBO->getOperand(i_nocapture: 1)))
8067 if (LBO->getOpcode() == Instruction::And &&
8068 LCI->getValue() == CI->getValue())
8069 if (const SCEVZeroExtendExpr *Z =
8070 dyn_cast<SCEVZeroExtendExpr>(Val: getSCEV(V: BO->LHS))) {
8071 Type *UTy = BO->LHS->getType();
8072 const SCEV *Z0 = Z->getOperand();
8073 Type *Z0Ty = Z0->getType();
8074 unsigned Z0TySize = getTypeSizeInBits(Ty: Z0Ty);
8075
8076 // If C is a low-bits mask, the zero extend is serving to
8077 // mask off the high bits. Complement the operand and
8078 // re-apply the zext.
8079 if (CI->getValue().isMask(numBits: Z0TySize))
8080 return getZeroExtendExpr(Op: getNotSCEV(V: Z0), Ty: UTy);
8081
8082 // If C is a single bit, it may be in the sign-bit position
8083 // before the zero-extend. In this case, represent the xor
8084 // using an add, which is equivalent, and re-apply the zext.
8085 APInt Trunc = CI->getValue().trunc(width: Z0TySize);
8086 if (Trunc.zext(width: getTypeSizeInBits(Ty: UTy)) == CI->getValue() &&
8087 Trunc.isSignMask())
8088 return getZeroExtendExpr(Op: getAddExpr(LHS: Z0, RHS: getConstant(Val: Trunc)),
8089 Ty: UTy);
8090 }
8091 }
8092 break;
8093
8094 case Instruction::Shl:
8095 // Turn shift left of a constant amount into a multiply.
8096 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8097 uint32_t BitWidth = cast<IntegerType>(Val: SA->getType())->getBitWidth();
8098
8099 // If the shift count is not less than the bitwidth, the result of
8100 // the shift is undefined. Don't try to analyze it, because the
8101 // resolution chosen here may differ from the resolution chosen in
8102 // other parts of the compiler.
8103 if (SA->getValue().uge(RHS: BitWidth))
8104 break;
8105
8106 // We can safely preserve the nuw flag in all cases. It's also safe to
8107 // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
8108 // requires special handling. It can be preserved as long as we're not
8109 // left shifting by bitwidth - 1.
8110 auto Flags = SCEV::FlagAnyWrap;
8111 if (BO->Op) {
8112 auto MulFlags = getNoWrapFlagsFromUB(V: BO->Op);
8113 if (any(Val: MulFlags & SCEV::FlagNSW) &&
8114 (any(Val: MulFlags & SCEV::FlagNUW) ||
8115 SA->getValue().ult(RHS: BitWidth - 1)))
8116 Flags = Flags | SCEV::FlagNSW;
8117 if (any(Val: MulFlags & SCEV::FlagNUW))
8118 Flags = Flags | SCEV::FlagNUW;
8119 }
8120
8121 ConstantInt *X = ConstantInt::get(
8122 Context&: getContext(), V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
8123 return getMulExpr(LHS: getSCEV(V: BO->LHS), RHS: getConstant(V: X), Flags);
8124 }
8125 break;
8126
8127 case Instruction::AShr:
8128 // AShr X, C, where C is a constant.
8129 ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS);
8130 if (!CI)
8131 break;
8132
8133 Type *OuterTy = BO->LHS->getType();
8134 uint64_t BitWidth = getTypeSizeInBits(Ty: OuterTy);
8135 // If the shift count is not less than the bitwidth, the result of
8136 // the shift is undefined. Don't try to analyze it, because the
8137 // resolution chosen here may differ from the resolution chosen in
8138 // other parts of the compiler.
8139 if (CI->getValue().uge(RHS: BitWidth))
8140 break;
8141
8142 if (CI->isZero())
8143 return getSCEV(V: BO->LHS); // shift by zero --> noop
8144
8145 uint64_t AShrAmt = CI->getZExtValue();
8146 Type *TruncTy = IntegerType::get(C&: getContext(), NumBits: BitWidth - AShrAmt);
8147
8148 Operator *L = dyn_cast<Operator>(Val: BO->LHS);
8149 const SCEV *AddTruncateExpr = nullptr;
8150 ConstantInt *ShlAmtCI = nullptr;
8151 const SCEV *AddConstant = nullptr;
8152
8153 if (L && L->getOpcode() == Instruction::Add) {
8154 // X = Shl A, n
8155 // Y = Add X, c
8156 // Z = AShr Y, m
8157 // n, c and m are constants.
8158
8159 Operator *LShift = dyn_cast<Operator>(Val: L->getOperand(i: 0));
8160 ConstantInt *AddOperandCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8161 if (LShift && LShift->getOpcode() == Instruction::Shl) {
8162 if (AddOperandCI) {
8163 const SCEV *ShlOp0SCEV = getSCEV(V: LShift->getOperand(i: 0));
8164 ShlAmtCI = dyn_cast<ConstantInt>(Val: LShift->getOperand(i: 1));
8165 // since we truncate to TruncTy, the AddConstant should be of the
8166 // same type, so create a new Constant with type same as TruncTy.
8167 // Also, the Add constant should be shifted right by AShr amount.
8168 APInt AddOperand = AddOperandCI->getValue().ashr(ShiftAmt: AShrAmt);
8169 AddConstant = getConstant(Val: AddOperand.trunc(width: BitWidth - AShrAmt));
8170 // we model the expression as sext(add(trunc(A), c << n)), since the
8171 // sext(trunc) part is already handled below, we create a
8172 // AddExpr(TruncExp) which will be used later.
8173 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8174 }
8175 }
8176 } else if (L && L->getOpcode() == Instruction::Shl) {
8177 // X = Shl A, n
8178 // Y = AShr X, m
8179 // Both n and m are constant.
8180
8181 const SCEV *ShlOp0SCEV = getSCEV(V: L->getOperand(i: 0));
8182 ShlAmtCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8183 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8184 }
8185
8186 if (AddTruncateExpr && ShlAmtCI) {
8187 // We can merge the two given cases into a single SCEV statement,
8188 // incase n = m, the mul expression will be 2^0, so it gets resolved to
8189 // a simpler case. The following code handles the two cases:
8190 //
8191 // 1) For a two-shift sext-inreg, i.e. n = m,
8192 // use sext(trunc(x)) as the SCEV expression.
8193 //
8194 // 2) When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
8195 // expression. We already checked that ShlAmt < BitWidth, so
8196 // the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
8197 // ShlAmt - AShrAmt < Amt.
8198 const APInt &ShlAmt = ShlAmtCI->getValue();
8199 if (ShlAmt.ult(RHS: BitWidth) && ShlAmt.uge(RHS: AShrAmt)) {
8200 APInt Mul = APInt::getOneBitSet(numBits: BitWidth - AShrAmt,
8201 BitNo: ShlAmtCI->getZExtValue() - AShrAmt);
8202 const SCEV *CompositeExpr =
8203 getMulExpr(LHS: AddTruncateExpr, RHS: getConstant(Val: Mul));
8204 if (L->getOpcode() != Instruction::Shl)
8205 CompositeExpr = getAddExpr(LHS: CompositeExpr, RHS: AddConstant);
8206
8207 return getSignExtendExpr(Op: CompositeExpr, Ty: OuterTy);
8208 }
8209 }
8210 break;
8211 }
8212 }
8213
8214 switch (U->getOpcode()) {
8215 case Instruction::Trunc:
8216 return getTruncateExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8217
8218 case Instruction::ZExt:
8219 return getZeroExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8220
8221 case Instruction::SExt:
8222 if (auto BO = MatchBinaryOp(V: U->getOperand(i: 0), DL: getDataLayout(), AC, DT,
8223 CxtI: dyn_cast<Instruction>(Val: V))) {
8224 // The NSW flag of a subtract does not always survive the conversion to
8225 // A + (-1)*B. By pushing sign extension onto its operands we are much
8226 // more likely to preserve NSW and allow later AddRec optimisations.
8227 //
8228 // NOTE: This is effectively duplicating this logic from getSignExtend:
8229 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
8230 // but by that point the NSW information has potentially been lost.
8231 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
8232 Type *Ty = U->getType();
8233 auto *V1 = getSignExtendExpr(Op: getSCEV(V: BO->LHS), Ty);
8234 auto *V2 = getSignExtendExpr(Op: getSCEV(V: BO->RHS), Ty);
8235 return getMinusSCEV(LHS: V1, RHS: V2, Flags: SCEV::FlagNSW);
8236 }
8237 }
8238 return getSignExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8239
8240 case Instruction::BitCast:
8241 // BitCasts are no-op casts so we just eliminate the cast.
8242 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType()))
8243 return getSCEV(V: U->getOperand(i: 0));
8244 break;
8245
8246 case Instruction::PtrToAddr: {
8247 const SCEV *IntOp = getPtrToAddrExpr(Op: getSCEV(V: U->getOperand(i: 0)));
8248 if (isa<SCEVCouldNotCompute>(Val: IntOp))
8249 return getUnknown(V);
8250 return IntOp;
8251 }
8252
8253 case Instruction::PtrToInt:
8254 // SCEV only models ptrtoaddr.
8255 return getUnknown(V);
8256
8257 case Instruction::IntToPtr:
8258 // Just don't deal with inttoptr casts.
8259 return getUnknown(V);
8260
8261 case Instruction::SDiv:
8262 // If both operands are non-negative, this is just an udiv.
8263 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8264 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8265 return getUDivExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8266 break;
8267
8268 case Instruction::SRem:
8269 // If both operands are non-negative, this is just an urem.
8270 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8271 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8272 return getURemExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8273 break;
8274
8275 case Instruction::GetElementPtr:
8276 return createNodeForGEP(GEP: cast<GEPOperator>(Val: U));
8277
8278 case Instruction::PHI:
8279 return createNodeForPHI(PN: cast<PHINode>(Val: U));
8280
8281 case Instruction::Select:
8282 return createNodeForSelectOrPHI(V: U, Cond: U->getOperand(i: 0), TrueVal: U->getOperand(i: 1),
8283 FalseVal: U->getOperand(i: 2));
8284
8285 case Instruction::Call:
8286 case Instruction::Invoke:
8287 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand())
8288 return getSCEV(V: RV);
8289
8290 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
8291 switch (II->getIntrinsicID()) {
8292 case Intrinsic::abs:
8293 return getAbsExpr(
8294 Op: getSCEV(V: II->getArgOperand(i: 0)),
8295 /*IsNSW=*/cast<ConstantInt>(Val: II->getArgOperand(i: 1))->isOne());
8296 case Intrinsic::umax:
8297 LHS = getSCEV(V: II->getArgOperand(i: 0));
8298 RHS = getSCEV(V: II->getArgOperand(i: 1));
8299 return getUMaxExpr(LHS, RHS);
8300 case Intrinsic::umin:
8301 LHS = getSCEV(V: II->getArgOperand(i: 0));
8302 RHS = getSCEV(V: II->getArgOperand(i: 1));
8303 return getUMinExpr(LHS, RHS);
8304 case Intrinsic::smax:
8305 LHS = getSCEV(V: II->getArgOperand(i: 0));
8306 RHS = getSCEV(V: II->getArgOperand(i: 1));
8307 return getSMaxExpr(LHS, RHS);
8308 case Intrinsic::smin:
8309 LHS = getSCEV(V: II->getArgOperand(i: 0));
8310 RHS = getSCEV(V: II->getArgOperand(i: 1));
8311 return getSMinExpr(LHS, RHS);
8312 case Intrinsic::usub_sat: {
8313 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8314 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8315 const SCEV *ClampedY = getUMinExpr(LHS: X, RHS: Y);
8316 return getMinusSCEV(LHS: X, RHS: ClampedY, Flags: SCEV::FlagNUW);
8317 }
8318 case Intrinsic::uadd_sat: {
8319 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8320 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8321 const SCEV *ClampedX = getUMinExpr(LHS: X, RHS: getNotSCEV(V: Y));
8322 return getAddExpr(LHS: ClampedX, RHS: Y, Flags: SCEV::FlagNUW);
8323 }
8324 case Intrinsic::start_loop_iterations:
8325 case Intrinsic::annotation:
8326 case Intrinsic::ptr_annotation:
8327 // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is
8328 // just eqivalent to the first operand for SCEV purposes.
8329 return getSCEV(V: II->getArgOperand(i: 0));
8330 case Intrinsic::vscale:
8331 return getVScale(Ty: II->getType());
8332 default:
8333 break;
8334 }
8335 }
8336 break;
8337 }
8338
8339 return getUnknown(V);
8340}
8341
8342//===----------------------------------------------------------------------===//
8343// Iteration Count Computation Code
8344//
8345
8346const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount) {
8347 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8348 return getCouldNotCompute();
8349
8350 auto *ExitCountType = ExitCount->getType();
8351 assert(ExitCountType->isIntegerTy());
8352 auto *EvalTy = Type::getIntNTy(C&: ExitCountType->getContext(),
8353 N: 1 + ExitCountType->getScalarSizeInBits());
8354 return getTripCountFromExitCount(ExitCount, EvalTy, L: nullptr);
8355}
8356
8357const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
8358 Type *EvalTy,
8359 const Loop *L) {
8360 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8361 return getCouldNotCompute();
8362
8363 unsigned ExitCountSize = getTypeSizeInBits(Ty: ExitCount->getType());
8364 unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
8365
8366 auto CanAddOneWithoutOverflow = [&]() {
8367 ConstantRange ExitCountRange =
8368 getRangeRef(S: ExitCount, SignHint: RangeSignHint::HINT_RANGE_UNSIGNED);
8369 if (!ExitCountRange.contains(Val: APInt::getMaxValue(numBits: ExitCountSize)))
8370 return true;
8371
8372 return L && isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: ExitCount,
8373 RHS: getMinusOne(Ty: ExitCount->getType()));
8374 };
8375
8376 // If we need to zero extend the backedge count, check if we can add one to
8377 // it prior to zero extending without overflow. Provided this is safe, it
8378 // allows better simplification of the +1.
8379 if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
8380 return getZeroExtendExpr(
8381 Op: getAddExpr(LHS: ExitCount, RHS: getOne(Ty: ExitCount->getType())), Ty: EvalTy);
8382
8383 // Get the total trip count from the count by adding 1. This may wrap.
8384 return getAddExpr(LHS: getTruncateOrZeroExtend(V: ExitCount, Ty: EvalTy), RHS: getOne(Ty: EvalTy));
8385}
8386
8387static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
8388 if (!ExitCount)
8389 return 0;
8390
8391 ConstantInt *ExitConst = ExitCount->getValue();
8392
8393 // Guard against huge trip counts.
8394 if (ExitConst->getValue().getActiveBits() > 32)
8395 return 0;
8396
8397 // In case of integer overflow, this returns 0, which is correct.
8398 return ((unsigned)ExitConst->getZExtValue()) + 1;
8399}
8400
8401unsigned ScalarEvolution::getSmallConstantTripCount(const Loop *L) {
8402 auto *ExitCount = dyn_cast<SCEVConstant>(Val: getBackedgeTakenCount(L, Kind: Exact));
8403 return getConstantTripCount(ExitCount);
8404}
8405
8406unsigned
8407ScalarEvolution::getSmallConstantTripCount(const Loop *L,
8408 const BasicBlock *ExitingBlock) {
8409 assert(ExitingBlock && "Must pass a non-null exiting block!");
8410 assert(L->isLoopExiting(ExitingBlock) &&
8411 "Exiting block must actually branch out of the loop!");
8412 const SCEVConstant *ExitCount =
8413 dyn_cast<SCEVConstant>(Val: getExitCount(L, ExitingBlock));
8414 return getConstantTripCount(ExitCount);
8415}
8416
8417unsigned ScalarEvolution::getSmallConstantMaxTripCount(
8418 const Loop *L, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8419
8420 const auto *MaxExitCount =
8421 Predicates ? getPredicatedConstantMaxBackedgeTakenCount(L, Predicates&: *Predicates)
8422 : getConstantMaxBackedgeTakenCount(L);
8423 return getConstantTripCount(ExitCount: dyn_cast<SCEVConstant>(Val: MaxExitCount));
8424}
8425
8426unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L) {
8427 SmallVector<BasicBlock *, 8> ExitingBlocks;
8428 L->getExitingBlocks(ExitingBlocks);
8429
8430 // An exit with an uncomputable exit count makes the result 1.
8431 if (ExitingBlocks.empty() ||
8432 any_of(Range&: ExitingBlocks, P: [this, L](BasicBlock *ExitingBB) {
8433 return isa<SCEVCouldNotCompute>(Val: getExitCount(L, ExitingBlock: ExitingBB));
8434 }))
8435 return 1;
8436
8437 LoopGuards Guards = LoopGuards::collect(L, SE&: *this);
8438 unsigned Res = 0;
8439 for (BasicBlock *ExitingBB : ExitingBlocks)
8440 Res = std::gcd(
8441 m: Res, n: getSmallConstantTripMultiple(ExitCount: getExitCount(L, ExitingBlock: ExitingBB), Guards));
8442 return Res;
8443}
8444
8445unsigned
8446ScalarEvolution::getSmallConstantTripMultiple(const SCEV *ExitCount,
8447 const LoopGuards &Guards) {
8448 assert(!isa<SCEVCouldNotCompute>(ExitCount) && "Must be computable!");
8449
8450 // Get the trip count
8451 const SCEV *TCExpr =
8452 getTripCountFromExitCount(ExitCount: applyLoopGuards(Expr: ExitCount, Guards));
8453
8454 APInt Multiple = getNonZeroConstantMultiple(S: TCExpr);
8455 // If a trip multiple is huge (>=2^32), the trip count is still divisible by
8456 // the greatest power of 2 divisor less than 2^32.
8457 return Multiple.getActiveBits() > 32
8458 ? 1U << std::min(a: 31U, b: Multiple.countTrailingZeros())
8459 : (unsigned)Multiple.getZExtValue();
8460}
8461
8462unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8463 const SCEV *ExitCount) {
8464 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8465 return 1;
8466
8467 return getSmallConstantTripMultiple(ExitCount, Guards: LoopGuards::collect(L, SE&: *this));
8468}
8469
8470/// Returns the largest constant divisor of the trip count of this loop as a
8471/// normal unsigned value, if possible. This means that the actual trip count is
8472/// always a multiple of the returned value (don't forget the trip count could
8473/// very well be zero as well!).
8474///
8475/// Returns 1 if the trip count is unknown or not guaranteed to be the
8476/// multiple of a constant (which is also the case if the trip count is simply
8477/// constant, use getSmallConstantTripCount for that case), Will also return 1
8478/// if the trip count is very large (>= 2^32).
8479///
8480/// As explained in the comments for getSmallConstantTripCount, this assumes
8481/// that control exits the loop via ExitingBlock.
8482unsigned
8483ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8484 const BasicBlock *ExitingBlock) {
8485 assert(ExitingBlock && "Must pass a non-null exiting block!");
8486 assert(L->isLoopExiting(ExitingBlock) &&
8487 "Exiting block must actually branch out of the loop!");
8488 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
8489 return getSmallConstantTripMultiple(L, ExitCount);
8490}
8491
8492const SCEV *ScalarEvolution::getExitCount(const Loop *L,
8493 const BasicBlock *ExitingBlock,
8494 ExitCountKind Kind) {
8495 switch (Kind) {
8496 case Exact:
8497 return getBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this);
8498 case SymbolicMaximum:
8499 return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this);
8500 case ConstantMaximum:
8501 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this);
8502 };
8503 llvm_unreachable("Invalid ExitCountKind!");
8504}
8505
8506const SCEV *ScalarEvolution::getPredicatedExitCount(
8507 const Loop *L, const BasicBlock *ExitingBlock,
8508 SmallVectorImpl<const SCEVPredicate *> *Predicates, ExitCountKind Kind) {
8509 switch (Kind) {
8510 case Exact:
8511 return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this,
8512 Predicates);
8513 case SymbolicMaximum:
8514 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this,
8515 Predicates);
8516 case ConstantMaximum:
8517 return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this,
8518 Predicates);
8519 };
8520 llvm_unreachable("Invalid ExitCountKind!");
8521}
8522
8523const SCEV *ScalarEvolution::getPredicatedBackedgeTakenCount(
8524 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8525 return getPredicatedBackedgeTakenInfo(L).getExact(L, SE: this, Predicates: &Preds);
8526}
8527
8528const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
8529 ExitCountKind Kind) {
8530 switch (Kind) {
8531 case Exact:
8532 return getBackedgeTakenInfo(L).getExact(L, SE: this);
8533 case ConstantMaximum:
8534 return getBackedgeTakenInfo(L).getConstantMax(SE: this);
8535 case SymbolicMaximum:
8536 return getBackedgeTakenInfo(L).getSymbolicMax(L, SE: this);
8537 };
8538 llvm_unreachable("Invalid ExitCountKind!");
8539}
8540
8541const SCEV *ScalarEvolution::getPredicatedSymbolicMaxBackedgeTakenCount(
8542 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8543 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, SE: this, Predicates: &Preds);
8544}
8545
8546const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
8547 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8548 return getPredicatedBackedgeTakenInfo(L).getConstantMax(SE: this, Predicates: &Preds);
8549}
8550
8551bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
8552 return getBackedgeTakenInfo(L).isConstantMaxOrZero(SE: this);
8553}
8554
8555/// Push PHI nodes in the header of the given loop onto the given Worklist.
8556static void PushLoopPHIs(const Loop *L,
8557 SmallVectorImpl<Instruction *> &Worklist,
8558 SmallPtrSetImpl<Instruction *> &Visited) {
8559 BasicBlock *Header = L->getHeader();
8560
8561 // Push all Loop-header PHIs onto the Worklist stack.
8562 for (PHINode &PN : Header->phis())
8563 if (Visited.insert(Ptr: &PN).second)
8564 Worklist.push_back(Elt: &PN);
8565}
8566
8567ScalarEvolution::BackedgeTakenInfo &
8568ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
8569 auto &BTI = getBackedgeTakenInfo(L);
8570 if (BTI.hasFullInfo())
8571 return BTI;
8572
8573 auto Pair = PredicatedBackedgeTakenCounts.try_emplace(Key: L);
8574
8575 if (!Pair.second)
8576 return Pair.first->second;
8577
8578 BackedgeTakenInfo Result =
8579 computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
8580
8581 return PredicatedBackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8582}
8583
8584ScalarEvolution::BackedgeTakenInfo &
8585ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
8586 // Initially insert an invalid entry for this loop. If the insertion
8587 // succeeds, proceed to actually compute a backedge-taken count and
8588 // update the value. The temporary CouldNotCompute value tells SCEV
8589 // code elsewhere that it shouldn't attempt to request a new
8590 // backedge-taken count, which could result in infinite recursion.
8591 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
8592 BackedgeTakenCounts.try_emplace(Key: L);
8593 if (!Pair.second)
8594 return Pair.first->second;
8595
8596 // computeBackedgeTakenCount may allocate memory for its result. Inserting it
8597 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
8598 // must be cleared in this scope.
8599 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
8600
8601 // Now that we know more about the trip count for this loop, forget any
8602 // existing SCEV values for PHI nodes in this loop since they are only
8603 // conservative estimates made without the benefit of trip count
8604 // information. This invalidation is not necessary for correctness, and is
8605 // only done to produce more precise results.
8606 if (Result.hasAnyInfo()) {
8607 // Invalidate any expression using an addrec in this loop.
8608 SmallVector<SCEVUse, 8> ToForget;
8609 auto LoopUsersIt = LoopUsers.find(Val: L);
8610 if (LoopUsersIt != LoopUsers.end())
8611 append_range(C&: ToForget, R&: LoopUsersIt->second);
8612 forgetMemoizedResults(SCEVs: ToForget);
8613
8614 // Invalidate constant-evolved loop header phis.
8615 for (PHINode &PN : L->getHeader()->phis())
8616 ConstantEvolutionLoopExitValue.erase(Val: &PN);
8617 }
8618
8619 // Re-lookup the insert position, since the call to
8620 // computeBackedgeTakenCount above could result in a
8621 // recusive call to getBackedgeTakenInfo (on a different
8622 // loop), which would invalidate the iterator computed
8623 // earlier.
8624 return BackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8625}
8626
8627void ScalarEvolution::forgetAllLoops() {
8628 // This method is intended to forget all info about loops. It should
8629 // invalidate caches as if the following happened:
8630 // - The trip counts of all loops have changed arbitrarily
8631 // - Every llvm::Value has been updated in place to produce a different
8632 // result.
8633 BackedgeTakenCounts.clear();
8634 PredicatedBackedgeTakenCounts.clear();
8635 BECountUsers.clear();
8636 LoopPropertiesCache.clear();
8637 ConstantEvolutionLoopExitValue.clear();
8638 ValueExprMap.clear();
8639 ValuesAtScopes.clear();
8640 ValuesAtScopesUsers.clear();
8641 LoopDispositions.clear();
8642 BlockDispositions.clear();
8643 UnsignedRanges.clear();
8644 SignedRanges.clear();
8645 ExprValueMap.clear();
8646 HasRecMap.clear();
8647 ConstantMultipleCache.clear();
8648 PredicatedSCEVRewrites.clear();
8649 FoldCache.clear();
8650 FoldCacheUser.clear();
8651}
8652void ScalarEvolution::visitAndClearUsers(
8653 SmallVectorImpl<Instruction *> &Worklist,
8654 SmallPtrSetImpl<Instruction *> &Visited,
8655 SmallVectorImpl<SCEVUse> &ToForget) {
8656 while (!Worklist.empty()) {
8657 Instruction *I = Worklist.pop_back_val();
8658 if (!isSCEVable(Ty: I->getType()) && !isa<WithOverflowInst>(Val: I))
8659 continue;
8660
8661 ValueExprMapType::iterator It =
8662 ValueExprMap.find_as(Val: static_cast<Value *>(I));
8663 if (It != ValueExprMap.end()) {
8664 ToForget.push_back(Elt: It->second);
8665 eraseValueFromMap(V: It->first);
8666 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
8667 ConstantEvolutionLoopExitValue.erase(Val: PN);
8668 }
8669
8670 PushDefUseChildren(I, Worklist, Visited);
8671 }
8672}
8673
8674void ScalarEvolution::forgetLoop(const Loop *L) {
8675 SmallVector<const Loop *, 16> LoopWorklist(1, L);
8676 SmallVector<Instruction *, 32> Worklist;
8677 SmallPtrSet<Instruction *, 16> Visited;
8678 SmallVector<SCEVUse, 16> ToForget;
8679
8680 // Iterate over all the loops and sub-loops to drop SCEV information.
8681 while (!LoopWorklist.empty()) {
8682 auto *CurrL = LoopWorklist.pop_back_val();
8683
8684 // Drop any stored trip count value.
8685 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ false);
8686 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ true);
8687
8688 // Drop information about predicated SCEV rewrites for this loop.
8689 PredicatedSCEVRewrites.remove_if(
8690 Pred: [&](const auto &Entry) { return Entry.first.second == CurrL; });
8691
8692 auto LoopUsersItr = LoopUsers.find(Val: CurrL);
8693 if (LoopUsersItr != LoopUsers.end())
8694 llvm::append_range(C&: ToForget, R&: LoopUsersItr->second);
8695
8696 // Drop information about expressions based on loop-header PHIs.
8697 PushLoopPHIs(L: CurrL, Worklist, Visited);
8698 visitAndClearUsers(Worklist, Visited, ToForget);
8699
8700 LoopPropertiesCache.erase(Val: CurrL);
8701 // Forget all contained loops too, to avoid dangling entries in the
8702 // ValuesAtScopes map.
8703 LoopWorklist.append(in_start: CurrL->begin(), in_end: CurrL->end());
8704 }
8705 forgetMemoizedResults(SCEVs: ToForget);
8706}
8707
8708void ScalarEvolution::forgetTopmostLoop(const Loop *L) {
8709 forgetLoop(L: L->getOutermostLoop());
8710}
8711
8712void ScalarEvolution::forgetValue(Value *V) {
8713 Instruction *I = dyn_cast<Instruction>(Val: V);
8714 if (!I) return;
8715
8716 // Drop information about expressions based on loop-header PHIs.
8717 SmallVector<Instruction *, 16> Worklist;
8718 SmallPtrSet<Instruction *, 8> Visited;
8719 SmallVector<SCEVUse, 8> ToForget;
8720 Worklist.push_back(Elt: I);
8721 Visited.insert(Ptr: I);
8722 visitAndClearUsers(Worklist, Visited, ToForget);
8723
8724 forgetMemoizedResults(SCEVs: ToForget);
8725}
8726
8727void ScalarEvolution::forgetValues(ArrayRef<Value *> Values) {
8728 SmallVector<Instruction *, 16> Worklist;
8729 SmallPtrSet<Instruction *, 8> Visited;
8730 SmallVector<SCEVUse, 8> ToForget;
8731 for (Value *V : Values)
8732 if (auto *I = dyn_cast<Instruction>(Val: V))
8733 if (Visited.insert(Ptr: I).second)
8734 Worklist.push_back(Elt: I);
8735 visitAndClearUsers(Worklist, Visited, ToForget);
8736
8737 forgetMemoizedResults(SCEVs: ToForget);
8738}
8739
8740void ScalarEvolution::forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V) {
8741 // If SCEV looked through a trivial LCSSA phi node, we might have SCEV's
8742 // directly using a SCEVUnknown/SCEVAddRec defined in the loop. After an
8743 // extra predecessor is added, this is no longer valid. Find all Unknowns and
8744 // AddRecs defined in the loop and invalidate any SCEV's making use of them.
8745 auto InvalidateValue = [&](Value *Val) {
8746 if (!isSCEVable(Ty: Val->getType()))
8747 return;
8748 if (const SCEV *S = getExistingSCEV(V: Val)) {
8749 struct InvalidationRootCollector {
8750 Loop *L;
8751 SmallVector<SCEVUse, 8> Roots;
8752
8753 InvalidationRootCollector(Loop *L) : L(L) {}
8754
8755 bool follow(const SCEV *S) {
8756 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
8757 if (auto *I = dyn_cast<Instruction>(Val: SU->getValue()))
8758 if (L->contains(Inst: I))
8759 Roots.push_back(Elt: S);
8760 } else if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S)) {
8761 if (L->contains(L: AddRec->getLoop()))
8762 Roots.push_back(Elt: S);
8763 }
8764 return true;
8765 }
8766 bool isDone() const { return false; }
8767 };
8768
8769 InvalidationRootCollector C(L);
8770 visitAll(Root: S, Visitor&: C);
8771 forgetMemoizedResults(SCEVs: C.Roots);
8772 }
8773 };
8774
8775 InvalidateValue(V);
8776
8777 // If V has a non-SCEV-able type (e.g. {i64, i1} from a with.overflow
8778 // intrinsic), its users (e.g. extractvalue) may have stale SCEV
8779 // expressions referencing loop-internal values.
8780 if (!isSCEVable(Ty: V->getType()) &&
8781 any_of(Range: V->incoming_values(), P: IsaPred<WithOverflowInst>))
8782 for (User *U : V->users())
8783 InvalidateValue(U);
8784 // Also perform the normal invalidation.
8785 forgetValue(V);
8786}
8787
8788void ScalarEvolution::forgetLoopDispositions() { LoopDispositions.clear(); }
8789
8790void ScalarEvolution::forgetBlockAndLoopDispositions(Value *V) {
8791 // Unless a specific value is passed to invalidation, completely clear both
8792 // caches.
8793 if (!V) {
8794 BlockDispositions.clear();
8795 LoopDispositions.clear();
8796 return;
8797 }
8798
8799 if (!isSCEVable(Ty: V->getType()))
8800 return;
8801
8802 const SCEV *S = getExistingSCEV(V);
8803 if (!S)
8804 return;
8805
8806 // Invalidate the block and loop dispositions cached for S. Dispositions of
8807 // S's users may change if S's disposition changes (i.e. a user may change to
8808 // loop-invariant, if S changes to loop invariant), so also invalidate
8809 // dispositions of S's users recursively.
8810 SmallVector<SCEVUse, 8> Worklist = {S};
8811 SmallPtrSet<const SCEV *, 8> Seen = {S};
8812 while (!Worklist.empty()) {
8813 const SCEV *Curr = Worklist.pop_back_val();
8814 bool LoopDispoRemoved = LoopDispositions.erase(Val: Curr);
8815 bool BlockDispoRemoved = BlockDispositions.erase(Val: Curr);
8816 if (!LoopDispoRemoved && !BlockDispoRemoved)
8817 continue;
8818 auto Users = SCEVUsers.find(Val: Curr);
8819 if (Users != SCEVUsers.end())
8820 for (const auto *User : Users->second)
8821 if (Seen.insert(Ptr: User).second)
8822 Worklist.push_back(Elt: User);
8823 }
8824}
8825
8826/// Get the exact loop backedge taken count considering all loop exits. A
8827/// computable result can only be returned for loops with all exiting blocks
8828/// dominating the latch. howFarToZero assumes that the limit of each loop test
8829/// is never skipped. This is a valid assumption as long as the loop exits via
8830/// that test. For precise results, it is the caller's responsibility to specify
8831/// the relevant loop exiting block using getExact(ExitingBlock, SE).
8832const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
8833 const Loop *L, ScalarEvolution *SE,
8834 SmallVectorImpl<const SCEVPredicate *> *Preds) const {
8835 // If any exits were not computable, the loop is not computable.
8836 if (!isComplete() || ExitNotTaken.empty())
8837 return SE->getCouldNotCompute();
8838
8839 const BasicBlock *Latch = L->getLoopLatch();
8840 // All exiting blocks we have collected must dominate the only backedge.
8841 if (!Latch)
8842 return SE->getCouldNotCompute();
8843
8844 // All exiting blocks we have gathered dominate loop's latch, so exact trip
8845 // count is simply a minimum out of all these calculated exit counts.
8846 SmallVector<SCEVUse, 2> Ops;
8847 for (const auto &ENT : ExitNotTaken) {
8848 const SCEV *BECount = ENT.ExactNotTaken;
8849 assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!");
8850 assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&
8851 "We should only have known counts for exiting blocks that dominate "
8852 "latch!");
8853
8854 Ops.push_back(Elt: BECount);
8855
8856 if (Preds)
8857 append_range(C&: *Preds, R: ENT.Predicates);
8858
8859 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
8860 "Predicate should be always true!");
8861 }
8862
8863 // If an earlier exit exits on the first iteration (exit count zero), then
8864 // a later poison exit count should not propagate into the result. This are
8865 // exactly the semantics provided by umin_seq.
8866 return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true);
8867}
8868
8869const ScalarEvolution::ExitNotTakenInfo *
8870ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
8871 const BasicBlock *ExitingBlock,
8872 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8873 for (const auto &ENT : ExitNotTaken)
8874 if (ENT.ExitingBlock == ExitingBlock) {
8875 if (ENT.hasAlwaysTruePredicate())
8876 return &ENT;
8877 else if (Predicates) {
8878 append_range(C&: *Predicates, R: ENT.Predicates);
8879 return &ENT;
8880 }
8881 }
8882
8883 return nullptr;
8884}
8885
8886/// getConstantMax - Get the constant max backedge taken count for the loop.
8887const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
8888 ScalarEvolution *SE,
8889 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8890 if (!getConstantMax())
8891 return SE->getCouldNotCompute();
8892
8893 for (const auto &ENT : ExitNotTaken)
8894 if (!ENT.hasAlwaysTruePredicate()) {
8895 if (!Predicates)
8896 return SE->getCouldNotCompute();
8897 append_range(C&: *Predicates, R: ENT.Predicates);
8898 }
8899
8900 assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||
8901 isa<SCEVConstant>(getConstantMax())) &&
8902 "No point in having a non-constant max backedge taken count!");
8903 return getConstantMax();
8904}
8905
8906const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
8907 const Loop *L, ScalarEvolution *SE,
8908 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8909 if (!SymbolicMax) {
8910 // Form an expression for the maximum exit count possible for this loop. We
8911 // merge the max and exact information to approximate a version of
8912 // getConstantMaxBackedgeTakenCount which isn't restricted to just
8913 // constants.
8914 SmallVector<SCEVUse, 4> ExitCounts;
8915
8916 for (const auto &ENT : ExitNotTaken) {
8917 const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
8918 if (!isa<SCEVCouldNotCompute>(Val: ExitCount)) {
8919 assert(SE->DT.dominates(ENT.ExitingBlock, L->getLoopLatch()) &&
8920 "We should only have known counts for exiting blocks that "
8921 "dominate latch!");
8922 ExitCounts.push_back(Elt: ExitCount);
8923 if (Predicates)
8924 append_range(C&: *Predicates, R: ENT.Predicates);
8925
8926 assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
8927 "Predicate should be always true!");
8928 }
8929 }
8930 if (ExitCounts.empty())
8931 SymbolicMax = SE->getCouldNotCompute();
8932 else
8933 SymbolicMax =
8934 SE->getUMinFromMismatchedTypes(Ops&: ExitCounts, /*Sequential*/ true);
8935 }
8936 return SymbolicMax;
8937}
8938
8939bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
8940 ScalarEvolution *SE) const {
8941 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
8942 return !ENT.hasAlwaysTruePredicate();
8943 };
8944 return MaxOrZero && !any_of(Range: ExitNotTaken, P: PredicateNotAlwaysTrue);
8945}
8946
8947ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E)
8948 : ExitLimit(E, E, E, false) {}
8949
8950ScalarEvolution::ExitLimit::ExitLimit(
8951 const SCEV *E, const SCEV *ConstantMaxNotTaken,
8952 const SCEV *SymbolicMaxNotTaken, bool MaxOrZero,
8953 ArrayRef<ArrayRef<const SCEVPredicate *>> PredLists)
8954 : ExactNotTaken(E), ConstantMaxNotTaken(ConstantMaxNotTaken),
8955 SymbolicMaxNotTaken(SymbolicMaxNotTaken), MaxOrZero(MaxOrZero) {
8956 // If we prove the max count is zero, so is the symbolic bound. This happens
8957 // in practice due to differences in a) how context sensitive we've chosen
8958 // to be and b) how we reason about bounds implied by UB.
8959 if (ConstantMaxNotTaken->isZero()) {
8960 this->ExactNotTaken = E = ConstantMaxNotTaken;
8961 this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
8962 }
8963
8964 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
8965 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
8966 "Exact is not allowed to be less precise than Constant Max");
8967 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
8968 !isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken)) &&
8969 "Exact is not allowed to be less precise than Symbolic Max");
8970 assert((isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken) ||
8971 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
8972 "Symbolic Max is not allowed to be less precise than Constant Max");
8973 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
8974 isa<SCEVConstant>(ConstantMaxNotTaken)) &&
8975 "No point in having a non-constant max backedge taken count!");
8976 SmallPtrSet<const SCEVPredicate *, 4> SeenPreds;
8977 for (const auto PredList : PredLists)
8978 for (const auto *P : PredList) {
8979 if (SeenPreds.contains(Ptr: P))
8980 continue;
8981 assert(!isa<SCEVUnionPredicate>(P) && "Only add leaf predicates here!");
8982 SeenPreds.insert(Ptr: P);
8983 Predicates.push_back(Elt: P);
8984 }
8985 assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&
8986 "Backedge count should be int");
8987 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
8988 !ConstantMaxNotTaken->getType()->isPointerTy()) &&
8989 "Max backedge count should be int");
8990}
8991
8992ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E,
8993 const SCEV *ConstantMaxNotTaken,
8994 const SCEV *SymbolicMaxNotTaken,
8995 bool MaxOrZero,
8996 ArrayRef<const SCEVPredicate *> PredList)
8997 : ExitLimit(E, ConstantMaxNotTaken, SymbolicMaxNotTaken, MaxOrZero,
8998 ArrayRef({PredList})) {}
8999
9000/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
9001/// computable exit into a persistent ExitNotTakenInfo array.
9002ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
9003 ArrayRef<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo> ExitCounts,
9004 bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
9005 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
9006 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
9007
9008 ExitNotTaken.reserve(N: ExitCounts.size());
9009 std::transform(first: ExitCounts.begin(), last: ExitCounts.end(),
9010 result: std::back_inserter(x&: ExitNotTaken),
9011 unary_op: [&](const EdgeExitInfo &EEI) {
9012 BasicBlock *ExitBB = EEI.first;
9013 const ExitLimit &EL = EEI.second;
9014 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
9015 EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
9016 EL.Predicates);
9017 });
9018 assert((isa<SCEVCouldNotCompute>(ConstantMax) ||
9019 isa<SCEVConstant>(ConstantMax)) &&
9020 "No point in having a non-constant max backedge taken count!");
9021}
9022
9023/// Compute the number of times the backedge of the specified loop will execute.
9024ScalarEvolution::BackedgeTakenInfo
9025ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
9026 bool AllowPredicates) {
9027 SmallVector<BasicBlock *, 8> ExitingBlocks;
9028 L->getExitingBlocks(ExitingBlocks);
9029
9030 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
9031
9032 SmallVector<EdgeExitInfo, 4> ExitCounts;
9033 bool CouldComputeBECount = true;
9034 BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
9035 const SCEV *MustExitMaxBECount = nullptr;
9036 const SCEV *MayExitMaxBECount = nullptr;
9037 bool MustExitMaxOrZero = false;
9038 bool IsOnlyExit = ExitingBlocks.size() == 1;
9039
9040 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
9041 // and compute maxBECount.
9042 // Do a union of all the predicates here.
9043 for (BasicBlock *ExitBB : ExitingBlocks) {
9044 // We canonicalize untaken exits to br (constant), ignore them so that
9045 // proving an exit untaken doesn't negatively impact our ability to reason
9046 // about the loop as whole.
9047 if (auto *BI = dyn_cast<CondBrInst>(Val: ExitBB->getTerminator()))
9048 if (auto *CI = dyn_cast<ConstantInt>(Val: BI->getCondition())) {
9049 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9050 if (ExitIfTrue == CI->isZero())
9051 continue;
9052 }
9053
9054 ExitLimit EL = computeExitLimit(L, ExitingBlock: ExitBB, IsOnlyExit, AllowPredicates);
9055
9056 assert((AllowPredicates || EL.Predicates.empty()) &&
9057 "Predicated exit limit when predicates are not allowed!");
9058
9059 // 1. For each exit that can be computed, add an entry to ExitCounts.
9060 // CouldComputeBECount is true only if all exits can be computed.
9061 if (EL.ExactNotTaken != getCouldNotCompute())
9062 ++NumExitCountsComputed;
9063 else
9064 // We couldn't compute an exact value for this exit, so
9065 // we won't be able to compute an exact value for the loop.
9066 CouldComputeBECount = false;
9067 // Remember exit count if either exact or symbolic is known. Because
9068 // Exact always implies symbolic, only check symbolic.
9069 if (EL.SymbolicMaxNotTaken != getCouldNotCompute())
9070 ExitCounts.emplace_back(Args&: ExitBB, Args&: EL);
9071 else {
9072 assert(EL.ExactNotTaken == getCouldNotCompute() &&
9073 "Exact is known but symbolic isn't?");
9074 ++NumExitCountsNotComputed;
9075 }
9076
9077 // 2. Derive the loop's MaxBECount from each exit's max number of
9078 // non-exiting iterations. Partition the loop exits into two kinds:
9079 // LoopMustExits and LoopMayExits.
9080 //
9081 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
9082 // is a LoopMayExit. If any computable LoopMustExit is found, then
9083 // MaxBECount is the minimum EL.ConstantMaxNotTaken of computable
9084 // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
9085 // EL.ConstantMaxNotTaken, where CouldNotCompute is considered greater than
9086 // any
9087 // computable EL.ConstantMaxNotTaken.
9088 if (EL.ConstantMaxNotTaken != getCouldNotCompute() && Latch &&
9089 DT.dominates(A: ExitBB, B: Latch)) {
9090 if (!MustExitMaxBECount) {
9091 MustExitMaxBECount = EL.ConstantMaxNotTaken;
9092 MustExitMaxOrZero = EL.MaxOrZero;
9093 } else {
9094 MustExitMaxBECount = getUMinFromMismatchedTypes(LHS: MustExitMaxBECount,
9095 RHS: EL.ConstantMaxNotTaken);
9096 }
9097 } else if (MayExitMaxBECount != getCouldNotCompute()) {
9098 if (!MayExitMaxBECount || EL.ConstantMaxNotTaken == getCouldNotCompute())
9099 MayExitMaxBECount = EL.ConstantMaxNotTaken;
9100 else {
9101 MayExitMaxBECount = getUMaxFromMismatchedTypes(LHS: MayExitMaxBECount,
9102 RHS: EL.ConstantMaxNotTaken);
9103 }
9104 }
9105 }
9106 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
9107 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
9108 // The loop backedge will be taken the maximum or zero times if there's
9109 // a single exit that must be taken the maximum or zero times.
9110 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
9111
9112 // Remember which SCEVs are used in exit limits for invalidation purposes.
9113 // We only care about non-constant SCEVs here, so we can ignore
9114 // EL.ConstantMaxNotTaken
9115 // and MaxBECount, which must be SCEVConstant.
9116 for (const auto &Pair : ExitCounts) {
9117 if (!isa<SCEVConstant>(Val: Pair.second.ExactNotTaken))
9118 BECountUsers[Pair.second.ExactNotTaken].insert(Ptr: {L, AllowPredicates});
9119 if (!isa<SCEVConstant>(Val: Pair.second.SymbolicMaxNotTaken))
9120 BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
9121 Ptr: {L, AllowPredicates});
9122 }
9123 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
9124 MaxBECount, MaxOrZero);
9125}
9126
9127ScalarEvolution::ExitLimit
9128ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
9129 bool IsOnlyExit, bool AllowPredicates) {
9130 assert(L->contains(ExitingBlock) && "Exit count for non-loop block?");
9131 // If our exiting block does not dominate the latch, then its connection with
9132 // loop's exit limit may be far from trivial.
9133 const BasicBlock *Latch = L->getLoopLatch();
9134 if (!Latch || !DT.dominates(A: ExitingBlock, B: Latch))
9135 return getCouldNotCompute();
9136
9137 Instruction *Term = ExitingBlock->getTerminator();
9138 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Term)) {
9139 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9140 assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&
9141 "It should have one successor in loop and one exit block!");
9142 // Proceed to the next level to examine the exit condition expression.
9143 return computeExitLimitFromCond(L, ExitCond: BI->getCondition(), ExitIfTrue,
9144 /*ControlsOnlyExit=*/IsOnlyExit,
9145 AllowPredicates);
9146 }
9147
9148 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: Term)) {
9149 // For switch, make sure that there is a single exit from the loop.
9150 BasicBlock *Exit = nullptr;
9151 for (auto *SBB : successors(BB: ExitingBlock))
9152 if (!L->contains(BB: SBB)) {
9153 if (Exit) // Multiple exit successors.
9154 return getCouldNotCompute();
9155 Exit = SBB;
9156 }
9157 assert(Exit && "Exiting block must have at least one exit");
9158 return computeExitLimitFromSingleExitSwitch(
9159 L, Switch: SI, ExitingBB: Exit, /*ControlsOnlyExit=*/IsSubExpr: IsOnlyExit);
9160 }
9161
9162 return getCouldNotCompute();
9163}
9164
9165ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCond(
9166 const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9167 bool AllowPredicates) {
9168 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
9169 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
9170 ControlsOnlyExit, AllowPredicates);
9171}
9172
9173std::optional<ScalarEvolution::ExitLimit>
9174ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
9175 bool ExitIfTrue, bool ControlsOnlyExit,
9176 bool AllowPredicates) {
9177 (void)this->L;
9178 (void)this->ExitIfTrue;
9179 (void)this->AllowPredicates;
9180
9181 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9182 this->AllowPredicates == AllowPredicates &&
9183 "Variance in assumed invariant key components!");
9184 auto Itr = TripCountMap.find(Val: {ExitCond, ControlsOnlyExit});
9185 if (Itr == TripCountMap.end())
9186 return std::nullopt;
9187 return Itr->second;
9188}
9189
9190void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond,
9191 bool ExitIfTrue,
9192 bool ControlsOnlyExit,
9193 bool AllowPredicates,
9194 const ExitLimit &EL) {
9195 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9196 this->AllowPredicates == AllowPredicates &&
9197 "Variance in assumed invariant key components!");
9198
9199 auto InsertResult = TripCountMap.insert(KV: {{ExitCond, ControlsOnlyExit}, EL});
9200 assert(InsertResult.second && "Expected successful insertion!");
9201 (void)InsertResult;
9202 (void)ExitIfTrue;
9203}
9204
9205ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
9206 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9207 bool ControlsOnlyExit, bool AllowPredicates) {
9208
9209 if (auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit,
9210 AllowPredicates))
9211 return *MaybeEL;
9212
9213 ExitLimit EL = computeExitLimitFromCondImpl(
9214 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates);
9215 Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL);
9216 return EL;
9217}
9218
9219ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
9220 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9221 bool ControlsOnlyExit, bool AllowPredicates) {
9222 // Handle BinOp conditions (And, Or).
9223 if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
9224 Cache, L, ExitCond, ExitIfTrue, AllowPredicates))
9225 return *LimitFromBinOp;
9226
9227 // With an icmp, it may be feasible to compute an exact backedge-taken count.
9228 // Proceed to the next level to examine the icmp.
9229 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(Val: ExitCond)) {
9230 ExitLimit EL =
9231 computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue, IsSubExpr: ControlsOnlyExit);
9232 if (EL.hasFullInfo() || !AllowPredicates)
9233 return EL;
9234
9235 // Try again, but use SCEV predicates this time.
9236 return computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue,
9237 IsSubExpr: ControlsOnlyExit,
9238 /*AllowPredicates=*/true);
9239 }
9240
9241 // Check for a constant condition. These are normally stripped out by
9242 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
9243 // preserve the CFG and is temporarily leaving constant conditions
9244 // in place.
9245 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: ExitCond)) {
9246 if (ExitIfTrue == !CI->getZExtValue())
9247 // The backedge is always taken.
9248 return getCouldNotCompute();
9249 // The backedge is never taken.
9250 return getZero(Ty: CI->getType());
9251 }
9252
9253 // If we're exiting based on the overflow flag of an x.with.overflow intrinsic
9254 // with a constant step, we can form an equivalent icmp predicate and figure
9255 // out how many iterations will be taken before we exit.
9256 const WithOverflowInst *WO;
9257 const APInt *C;
9258 if (match(V: ExitCond, P: m_ExtractValue<1>(V: m_WithOverflowInst(I&: WO))) &&
9259 match(V: WO->getRHS(), P: m_APInt(Res&: C))) {
9260 ConstantRange NWR =
9261 ConstantRange::makeExactNoWrapRegion(BinOp: WO->getBinaryOp(), Other: *C,
9262 NoWrapKind: WO->getNoWrapKind());
9263 CmpInst::Predicate Pred;
9264 APInt NewRHSC, Offset;
9265 NWR.getEquivalentICmp(Pred, RHS&: NewRHSC, Offset);
9266 if (!ExitIfTrue)
9267 Pred = ICmpInst::getInversePredicate(pred: Pred);
9268 auto *LHS = getSCEV(V: WO->getLHS());
9269 if (Offset != 0)
9270 LHS = getAddExpr(LHS, RHS: getConstant(Val: Offset));
9271 auto EL = computeExitLimitFromICmp(L, Pred, LHS, RHS: getConstant(Val: NewRHSC),
9272 IsSubExpr: ControlsOnlyExit, AllowPredicates);
9273 if (EL.hasAnyInfo())
9274 return EL;
9275 }
9276
9277 // If it's not an integer or pointer comparison then compute it the hard way.
9278 return computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9279}
9280
9281std::optional<ScalarEvolution::ExitLimit>
9282ScalarEvolution::computeExitLimitFromCondFromBinOp(ExitLimitCacheTy &Cache,
9283 const Loop *L,
9284 Value *ExitCond,
9285 bool ExitIfTrue,
9286 bool AllowPredicates) {
9287 // Check if the controlling expression for this loop is an And or Or.
9288 Value *Op0, *Op1;
9289 bool IsAnd;
9290 if (match(V: ExitCond, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9291 IsAnd = true;
9292 else if (match(V: ExitCond, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9293 IsAnd = false;
9294 else
9295 return std::nullopt;
9296
9297 // A sub-condition of a non-trivial binop never solely controls the exit,
9298 // whether we exit always depends on both conditions.
9299 ExitLimit EL0 = computeExitLimitFromCondCached(
9300 Cache, L, ExitCond: Op0, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9301 ExitLimit EL1 = computeExitLimitFromCondCached(
9302 Cache, L, ExitCond: Op1, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9303
9304 // EitherMayExit is true in these two cases:
9305 // br (and Op0 Op1), loop, exit
9306 // br (or Op0 Op1), exit, loop
9307 bool EitherMayExit = IsAnd ^ ExitIfTrue;
9308
9309 const SCEV *BECount = getCouldNotCompute();
9310 const SCEV *ConstantMaxBECount = getCouldNotCompute();
9311 const SCEV *SymbolicMaxBECount = getCouldNotCompute();
9312 if (EitherMayExit) {
9313 bool UseSequentialUMin = !isa<BinaryOperator>(Val: ExitCond);
9314 // Both conditions must be same for the loop to continue executing.
9315 // Choose the less conservative count.
9316 if (EL0.ExactNotTaken != getCouldNotCompute() &&
9317 EL1.ExactNotTaken != getCouldNotCompute()) {
9318 BECount = getUMinFromMismatchedTypes(LHS: EL0.ExactNotTaken, RHS: EL1.ExactNotTaken,
9319 Sequential: UseSequentialUMin);
9320 }
9321 if (EL0.ConstantMaxNotTaken == getCouldNotCompute())
9322 ConstantMaxBECount = EL1.ConstantMaxNotTaken;
9323 else if (EL1.ConstantMaxNotTaken == getCouldNotCompute())
9324 ConstantMaxBECount = EL0.ConstantMaxNotTaken;
9325 else
9326 ConstantMaxBECount = getUMinFromMismatchedTypes(LHS: EL0.ConstantMaxNotTaken,
9327 RHS: EL1.ConstantMaxNotTaken);
9328 if (EL0.SymbolicMaxNotTaken == getCouldNotCompute())
9329 SymbolicMaxBECount = EL1.SymbolicMaxNotTaken;
9330 else if (EL1.SymbolicMaxNotTaken == getCouldNotCompute())
9331 SymbolicMaxBECount = EL0.SymbolicMaxNotTaken;
9332 else
9333 SymbolicMaxBECount = getUMinFromMismatchedTypes(
9334 LHS: EL0.SymbolicMaxNotTaken, RHS: EL1.SymbolicMaxNotTaken, Sequential: UseSequentialUMin);
9335 } else {
9336 // Both conditions must be same at the same time for the loop to exit.
9337 // For now, be conservative.
9338 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
9339 BECount = EL0.ExactNotTaken;
9340 }
9341
9342 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able
9343 // to be more aggressive when computing BECount than when computing
9344 // ConstantMaxBECount. In these cases it is possible for EL0.ExactNotTaken
9345 // and
9346 // EL1.ExactNotTaken to match, but for EL0.ConstantMaxNotTaken and
9347 // EL1.ConstantMaxNotTaken to not.
9348 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
9349 !isa<SCEVCouldNotCompute>(Val: BECount))
9350 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
9351 if (isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount))
9352 SymbolicMaxBECount =
9353 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
9354 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
9355 {ArrayRef(EL0.Predicates), ArrayRef(EL1.Predicates)});
9356}
9357
9358ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9359 const Loop *L, ICmpInst *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9360 bool AllowPredicates) {
9361 // If the condition was exit on true, convert the condition to exit on false
9362 CmpPredicate Pred;
9363 if (!ExitIfTrue)
9364 Pred = ExitCond->getCmpPredicate();
9365 else
9366 Pred = ExitCond->getInverseCmpPredicate();
9367 const ICmpInst::Predicate OriginalPred = Pred;
9368
9369 const SCEV *LHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 0));
9370 const SCEV *RHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 1));
9371
9372 ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, IsSubExpr: ControlsOnlyExit,
9373 AllowPredicates);
9374 if (EL.hasAnyInfo())
9375 return EL;
9376
9377 auto *ExhaustiveCount =
9378 computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9379
9380 if (!isa<SCEVCouldNotCompute>(Val: ExhaustiveCount))
9381 return ExhaustiveCount;
9382
9383 return computeShiftCompareExitLimit(LHS: ExitCond->getOperand(i_nocapture: 0),
9384 RHS: ExitCond->getOperand(i_nocapture: 1), L, Pred: OriginalPred);
9385}
9386ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9387 const Loop *L, CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS,
9388 bool ControlsOnlyExit, bool AllowPredicates) {
9389
9390 // Try to evaluate any dependencies out of the loop.
9391 LHS = getSCEVAtScope(S: LHS, L);
9392 RHS = getSCEVAtScope(S: RHS, L);
9393
9394 // At this point, we would like to compute how many iterations of the
9395 // loop the predicate will return true for these inputs.
9396 if (isLoopInvariant(S: LHS, L) && !isLoopInvariant(S: RHS, L)) {
9397 // If there is a loop-invariant, force it into the RHS.
9398 std::swap(a&: LHS, b&: RHS);
9399 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
9400 }
9401
9402 bool ControllingFiniteLoop = ControlsOnlyExit && loopHasNoAbnormalExits(L) &&
9403 loopIsFiniteByAssumption(L);
9404 // Simplify the operands before analyzing them.
9405 (void)SimplifyICmpOperands(Pred, LHS, RHS, /*Depth=*/0);
9406
9407 // If we have a comparison of a chrec against a constant, try to use value
9408 // ranges to answer this query.
9409 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS))
9410 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
9411 if (AddRec->getLoop() == L) {
9412 // Form the constant range.
9413 ConstantRange CompRange =
9414 ConstantRange::makeExactICmpRegion(Pred, Other: RHSC->getAPInt());
9415
9416 const SCEV *Ret = AddRec->getNumIterationsInRange(Range: CompRange, SE&: *this);
9417 if (!isa<SCEVCouldNotCompute>(Val: Ret)) return Ret;
9418 }
9419
9420 // If this loop must exit based on this condition (or execute undefined
9421 // behaviour), see if we can improve wrap flags. This is essentially
9422 // a must execute style proof.
9423 if (ControllingFiniteLoop && isLoopInvariant(S: RHS, L)) {
9424 // If we can prove the test sequence produced must repeat the same values
9425 // on self-wrap of the IV, then we can infer that IV doesn't self wrap
9426 // because if it did, we'd have an infinite (undefined) loop.
9427 // TODO: We can peel off any functions which are invertible *in L*. Loop
9428 // invariant terms are effectively constants for our purposes here.
9429 SCEVUse InnerLHS = LHS;
9430 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val&: LHS))
9431 InnerLHS = ZExt->getOperand();
9432 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: InnerLHS);
9433 AR && !AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() &&
9434 isKnownToBeAPowerOfTwo(S: AR->getStepRecurrence(SE&: *this), /*OrZero=*/true,
9435 /*OrNegative=*/true)) {
9436 auto Flags = AR->getNoWrapFlags();
9437 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
9438 SmallVector<SCEVUse> Operands{AR->operands()};
9439 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9440 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9441 }
9442
9443 // For a slt/ult condition with a positive step, can we prove nsw/nuw?
9444 // From no-self-wrap, this follows trivially from the fact that every
9445 // (un)signed-wrapped, but not self-wrapped value must be LT than the
9446 // last value before (un)signed wrap. Since we know that last value
9447 // didn't exit, nor will any smaller one.
9448 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT) {
9449 auto WrapType = Pred == ICmpInst::ICMP_SLT ? SCEV::FlagNSW : SCEV::FlagNUW;
9450 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS);
9451 AR && AR->getLoop() == L && AR->isAffine() &&
9452 !AR->getNoWrapFlags(Mask: WrapType) && AR->hasNoSelfWrap() &&
9453 isKnownPositive(S: AR->getStepRecurrence(SE&: *this))) {
9454 auto Flags = AR->getNoWrapFlags();
9455 Flags = setFlags(Flags, OnFlags: WrapType);
9456 SmallVector<SCEVUse> Operands{AR->operands()};
9457 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9458 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9459 }
9460 }
9461 }
9462
9463 switch (Pred) {
9464 case ICmpInst::ICMP_NE: { // while (X != Y)
9465 // Convert to: while (X-Y != 0)
9466 if (LHS->getType()->isPointerTy()) {
9467 LHS = getPtrToAddrExpr(Op: LHS);
9468 if (isa<SCEVCouldNotCompute>(Val: LHS))
9469 return LHS;
9470 }
9471 if (RHS->getType()->isPointerTy()) {
9472 RHS = getPtrToAddrExpr(Op: RHS);
9473 if (isa<SCEVCouldNotCompute>(Val: RHS))
9474 return RHS;
9475 }
9476 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit,
9477 AllowPredicates);
9478 if (EL.hasAnyInfo())
9479 return EL;
9480 break;
9481 }
9482 case ICmpInst::ICMP_EQ: { // while (X == Y)
9483 // Convert to: while (X-Y == 0)
9484 if (LHS->getType()->isPointerTy()) {
9485 LHS = getPtrToAddrExpr(Op: LHS);
9486 if (isa<SCEVCouldNotCompute>(Val: LHS))
9487 return LHS;
9488 }
9489 if (RHS->getType()->isPointerTy()) {
9490 RHS = getPtrToAddrExpr(Op: RHS);
9491 if (isa<SCEVCouldNotCompute>(Val: RHS))
9492 return RHS;
9493 }
9494 ExitLimit EL = howFarToNonZero(V: getMinusSCEV(LHS, RHS), L);
9495 if (EL.hasAnyInfo()) return EL;
9496 break;
9497 }
9498 case ICmpInst::ICMP_SLE:
9499 case ICmpInst::ICMP_ULE:
9500 // Since the loop is finite, an invariant RHS cannot include the boundary
9501 // value, otherwise it would loop forever.
9502 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9503 !isLoopInvariant(S: RHS, L)) {
9504 // Otherwise, perform the addition in a wider type, to avoid overflow.
9505 // If the LHS is an addrec with the appropriate nowrap flag, the
9506 // extension will be sunk into it and the exit count can be analyzed.
9507 auto *OldType = dyn_cast<IntegerType>(Val: LHS->getType());
9508 if (!OldType)
9509 break;
9510 // Prefer doubling the bitwidth over adding a single bit to make it more
9511 // likely that we use a legal type.
9512 auto *NewType =
9513 Type::getIntNTy(C&: OldType->getContext(), N: OldType->getBitWidth() * 2);
9514 if (ICmpInst::isSigned(Pred)) {
9515 LHS = getSignExtendExpr(Op: LHS, Ty: NewType);
9516 RHS = getSignExtendExpr(Op: RHS, Ty: NewType);
9517 } else {
9518 LHS = getZeroExtendExpr(Op: LHS, Ty: NewType);
9519 RHS = getZeroExtendExpr(Op: RHS, Ty: NewType);
9520 }
9521 }
9522 RHS = getAddExpr(LHS: getOne(Ty: RHS->getType()), RHS);
9523 [[fallthrough]];
9524 case ICmpInst::ICMP_SLT:
9525 case ICmpInst::ICMP_ULT: { // while (X < Y)
9526 bool IsSigned = ICmpInst::isSigned(Pred);
9527 ExitLimit EL = howManyLessThans(LHS, RHS, L, isSigned: IsSigned, ControlsOnlyExit,
9528 AllowPredicates);
9529 if (EL.hasAnyInfo())
9530 return EL;
9531 break;
9532 }
9533 case ICmpInst::ICMP_SGE:
9534 case ICmpInst::ICMP_UGE:
9535 // Since the loop is finite, an invariant RHS cannot include the boundary
9536 // value, otherwise it would loop forever.
9537 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9538 !isLoopInvariant(S: RHS, L))
9539 break;
9540 RHS = getAddExpr(LHS: getMinusOne(Ty: RHS->getType()), RHS);
9541 [[fallthrough]];
9542 case ICmpInst::ICMP_SGT:
9543 case ICmpInst::ICMP_UGT: { // while (X > Y)
9544 bool IsSigned = ICmpInst::isSigned(Pred);
9545 ExitLimit EL = howManyGreaterThans(LHS, RHS, L, isSigned: IsSigned, IsSubExpr: ControlsOnlyExit,
9546 AllowPredicates);
9547 if (EL.hasAnyInfo())
9548 return EL;
9549 break;
9550 }
9551 default:
9552 break;
9553 }
9554
9555 return getCouldNotCompute();
9556}
9557
9558ScalarEvolution::ExitLimit
9559ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L,
9560 SwitchInst *Switch,
9561 BasicBlock *ExitingBlock,
9562 bool ControlsOnlyExit) {
9563 assert(!L->contains(ExitingBlock) && "Not an exiting block!");
9564
9565 // Give up if the exit is the default dest of a switch.
9566 if (Switch->getDefaultDest() == ExitingBlock)
9567 return getCouldNotCompute();
9568
9569 assert(L->contains(Switch->getDefaultDest()) &&
9570 "Default case must not exit the loop!");
9571 const SCEV *LHS = getSCEVAtScope(V: Switch->getCondition(), L);
9572 const SCEV *RHS = getConstant(V: Switch->findCaseDest(BB: ExitingBlock));
9573
9574 // while (X != Y) --> while (X-Y != 0)
9575 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit);
9576 if (EL.hasAnyInfo())
9577 return EL;
9578
9579 return getCouldNotCompute();
9580}
9581
9582static ConstantInt *
9583EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
9584 ScalarEvolution &SE) {
9585 const SCEV *InVal = SE.getConstant(V: C);
9586 const SCEV *Val = AddRec->evaluateAtIteration(It: InVal, SE);
9587 assert(isa<SCEVConstant>(Val) &&
9588 "Evaluation of SCEV at constant didn't fold correctly?");
9589 return cast<SCEVConstant>(Val)->getValue();
9590}
9591
9592ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
9593 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) {
9594 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: RHSV);
9595 if (!RHS)
9596 return getCouldNotCompute();
9597
9598 const BasicBlock *Latch = L->getLoopLatch();
9599 if (!Latch)
9600 return getCouldNotCompute();
9601
9602 const BasicBlock *Predecessor = L->getLoopPredecessor();
9603 if (!Predecessor)
9604 return getCouldNotCompute();
9605
9606 // Return true if V is of the form "LHS `shift_op` <positive constant>".
9607 // Return LHS in OutLHS, shift_op in OutOpCode, and the shift amount in
9608 // OutShiftAmt.
9609 auto MatchPositiveShift = [](Value *V, Value *&OutLHS,
9610 Instruction::BinaryOps &OutOpCode,
9611 unsigned &OutShiftAmt) {
9612 using namespace PatternMatch;
9613
9614 ConstantInt *ShiftAmt;
9615 if (match(V, P: m_LShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9616 OutOpCode = Instruction::LShr;
9617 else if (match(V, P: m_AShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9618 OutOpCode = Instruction::AShr;
9619 else if (match(V, P: m_Shl(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9620 OutOpCode = Instruction::Shl;
9621 else
9622 return false;
9623
9624 uint64_t Amt = ShiftAmt->getValue().getLimitedValue();
9625 if (Amt == 0 || Amt >= OutLHS->getType()->getScalarSizeInBits())
9626 return false;
9627 OutShiftAmt = Amt;
9628 return true;
9629 };
9630
9631 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in
9632 //
9633 // loop:
9634 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ]
9635 // %iv.shifted = lshr i32 %iv, <positive constant>
9636 //
9637 // Return true on a successful match. Return the corresponding PHI node (%iv
9638 // above) in PNOut, the opcode of the shift operation in OpCodeOut, and the
9639 // shift amount in ShiftAmtOut.
9640 auto MatchShiftRecurrence = [&](Value *V, PHINode *&PNOut,
9641 Instruction::BinaryOps &OpCodeOut,
9642 unsigned &ShiftAmtOut) {
9643 std::optional<Instruction::BinaryOps> PostShiftOpCode;
9644
9645 {
9646 Instruction::BinaryOps OpC;
9647 Value *V;
9648 unsigned Amt;
9649
9650 // If we encounter a shift instruction, "peel off" the shift operation,
9651 // and remember that we did so. Later when we inspect %iv's backedge
9652 // value, we will make sure that the backedge value uses the same
9653 // operation.
9654 //
9655 // Note: the peeled shift operation does not have to be the same
9656 // instruction as the one feeding into the PHI's backedge value. We only
9657 // really care about it being the same *kind* of shift instruction --
9658 // that's all that is required for our later inferences to hold.
9659 if (MatchPositiveShift(LHS, V, OpC, Amt)) {
9660 PostShiftOpCode = OpC;
9661 LHS = V;
9662 }
9663 }
9664
9665 PNOut = dyn_cast<PHINode>(Val: LHS);
9666 if (!PNOut || PNOut->getParent() != L->getHeader())
9667 return false;
9668
9669 Value *BEValue = PNOut->getIncomingValueForBlock(BB: Latch);
9670 Value *OpLHS;
9671
9672 return
9673 // The backedge value for the PHI node must be a shift by a positive
9674 // amount
9675 MatchPositiveShift(BEValue, OpLHS, OpCodeOut, ShiftAmtOut) &&
9676
9677 // of the PHI node itself
9678 OpLHS == PNOut &&
9679
9680 // and the kind of shift should be match the kind of shift we peeled
9681 // off, if any.
9682 (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut);
9683 };
9684
9685 PHINode *PN;
9686 Instruction::BinaryOps OpCode;
9687 unsigned ShiftAmt;
9688 if (!MatchShiftRecurrence(LHS, PN, OpCode, ShiftAmt))
9689 return getCouldNotCompute();
9690
9691 const DataLayout &DL = getDataLayout();
9692
9693 // The key rationale for this optimization is that for some kinds of shift
9694 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1
9695 // within a finite number of iterations. If the condition guarding the
9696 // backedge (in the sense that the backedge is taken if the condition is true)
9697 // is false for the value the shift recurrence stabilizes to, then we know
9698 // that the backedge is taken only a finite number of times.
9699
9700 ConstantInt *StableValue = nullptr;
9701 switch (OpCode) {
9702 default:
9703 llvm_unreachable("Impossible case!");
9704
9705 case Instruction::AShr: {
9706 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
9707 // bitwidth(K) iterations.
9708 Value *FirstValue = PN->getIncomingValueForBlock(BB: Predecessor);
9709 KnownBits Known = computeKnownBits(V: FirstValue, DL, AC: &AC,
9710 CxtI: Predecessor->getTerminator(), DT: &DT);
9711 auto *Ty = cast<IntegerType>(Val: RHS->getType());
9712 if (Known.isNonNegative())
9713 StableValue = ConstantInt::get(Ty, V: 0);
9714 else if (Known.isNegative())
9715 StableValue = ConstantInt::get(Ty, V: -1, IsSigned: true);
9716 else
9717 return getCouldNotCompute();
9718
9719 break;
9720 }
9721 case Instruction::LShr:
9722 case Instruction::Shl:
9723 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>}
9724 // stabilize to 0 in at most bitwidth(K) iterations.
9725 StableValue = ConstantInt::get(Ty: cast<IntegerType>(Val: RHS->getType()), V: 0);
9726 break;
9727 }
9728
9729 auto *Result =
9730 ConstantFoldCompareInstOperands(Predicate: Pred, LHS: StableValue, RHS, DL, TLI: &TLI);
9731 assert(Result->getType()->isIntegerTy(1) &&
9732 "Otherwise cannot be an operand to a branch instruction");
9733
9734 if (Result->isNullValue()) {
9735 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
9736 unsigned MaxBTC = BitWidth;
9737
9738 // For right-shift recurrences (lshr/ashr with non-negative start), we can
9739 // compute a tighter max backedge-taken count from the range of the start
9740 // value. After k shifts of ShiftAmt, value = start >> (k * ShiftAmt).
9741 // The value reaches 0 (the stable value) when k * ShiftAmt >=
9742 // activeBits(start), so max BTC = ceil(activeBits(maxStart) / ShiftAmt).
9743 if (OpCode == Instruction::LShr || OpCode == Instruction::AShr) {
9744 Value *StartValue = PN->getIncomingValueForBlock(BB: Predecessor);
9745 const SCEV *StartSCEV = getSCEV(V: StartValue);
9746 APInt MaxStart = getUnsignedRangeMax(S: StartSCEV);
9747 if (MaxStart.isStrictlyPositive()) {
9748 unsigned ActiveBits = MaxStart.getActiveBits();
9749 unsigned RangeBTC = divideCeil(Numerator: ActiveBits, Denominator: ShiftAmt);
9750 MaxBTC = std::min(a: MaxBTC, b: RangeBTC);
9751 }
9752 }
9753
9754 const SCEV *UpperBound =
9755 getConstant(Ty: getEffectiveSCEVType(Ty: RHS->getType()), V: MaxBTC);
9756 return ExitLimit(getCouldNotCompute(), UpperBound, UpperBound, false);
9757 }
9758
9759 return getCouldNotCompute();
9760}
9761
9762/// Return true if we can constant fold an instruction of the specified type,
9763/// assuming that all operands were constants.
9764static bool canConstantFold(const Instruction *I,
9765 const TargetLibraryInfo *TLI) {
9766 if (isa<BinaryOperator, UnaryOperator, GEPOperator, FreezeInst, CmpInst,
9767 SelectInst, CastInst, LoadInst, ExtractElementInst, InsertElementInst,
9768 ExtractValueInst, InsertValueInst>(Val: I))
9769 return true;
9770
9771 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
9772 if (const Function *F = CI->getCalledFunction())
9773 return canConstantFoldCallTo(Call: CI, F, TLI);
9774 return false;
9775}
9776
9777/// Determine whether this instruction can constant evolve within this loop
9778/// assuming its operands can all constant evolve.
9779static bool canConstantEvolve(Instruction *I, const Loop *L,
9780 const TargetLibraryInfo *TLI) {
9781 // An instruction outside of the loop can't be derived from a loop PHI.
9782 if (!L->contains(Inst: I)) return false;
9783
9784 if (isa<PHINode>(Val: I)) {
9785 // We don't currently keep track of the control flow needed to evaluate
9786 // PHIs, so we cannot handle PHIs inside of loops.
9787 return L->getHeader() == I->getParent();
9788 }
9789
9790 // If we won't be able to constant fold this expression even if the operands
9791 // are constants, bail early.
9792 return canConstantFold(I, TLI);
9793}
9794
9795/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
9796/// recursing through each instruction operand until reaching a loop header phi.
9797static PHINode *
9798getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
9799 DenseMap<Instruction *, PHINode *> &PHIMap,
9800 const TargetLibraryInfo *TLI, unsigned Depth) {
9801 if (Depth > MaxConstantEvolvingDepth)
9802 return nullptr;
9803
9804 // Otherwise, we can evaluate this instruction if all of its operands are
9805 // constant or derived from a PHI node themselves.
9806 PHINode *PHI = nullptr;
9807 for (Value *Op : UseInst->operands()) {
9808 if (isa<Constant>(Val: Op)) continue;
9809
9810 Instruction *OpInst = dyn_cast<Instruction>(Val: Op);
9811 if (!OpInst || !canConstantEvolve(I: OpInst, L, TLI))
9812 return nullptr;
9813
9814 PHINode *P = dyn_cast<PHINode>(Val: OpInst);
9815 if (!P)
9816 // If this operand is already visited, reuse the prior result.
9817 // We may have P != PHI if this is the deepest point at which the
9818 // inconsistent paths meet.
9819 P = PHIMap.lookup(Val: OpInst);
9820 if (!P) {
9821 // Recurse and memoize the results, whether a phi is found or not.
9822 // This recursive call invalidates pointers into PHIMap.
9823 P = getConstantEvolvingPHIOperands(UseInst: OpInst, L, PHIMap, TLI, Depth: Depth + 1);
9824 PHIMap[OpInst] = P;
9825 }
9826 if (!P)
9827 return nullptr; // Not evolving from PHI
9828 if (PHI && PHI != P)
9829 return nullptr; // Evolving from multiple different PHIs.
9830 PHI = P;
9831 }
9832 // This is a expression evolving from a constant PHI!
9833 return PHI;
9834}
9835
9836/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
9837/// in the loop that V is derived from. We allow arbitrary operations along the
9838/// way, but the operands of an operation must either be constants or a value
9839/// derived from a constant PHI. If this expression does not fit with these
9840/// constraints, return null.
9841static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L,
9842 const TargetLibraryInfo *TLI) {
9843 Instruction *I = dyn_cast<Instruction>(Val: V);
9844 if (!I || !canConstantEvolve(I, L, TLI))
9845 return nullptr;
9846
9847 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
9848 return PN;
9849
9850 // Record non-constant instructions contained by the loop.
9851 DenseMap<Instruction *, PHINode *> PHIMap;
9852 return getConstantEvolvingPHIOperands(UseInst: I, L, PHIMap, TLI, Depth: 0);
9853}
9854
9855/// EvaluateExpression - Given an expression that passes the
9856/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
9857/// in the loop has the value PHIVal. If we can't fold this expression for some
9858/// reason, return null.
9859static Constant *EvaluateExpression(Value *V, const Loop *L,
9860 DenseMap<Instruction *, Constant *> &Vals,
9861 const DataLayout &DL,
9862 const TargetLibraryInfo *TLI) {
9863 // Convenient constant check, but redundant for recursive calls.
9864 if (Constant *C = dyn_cast<Constant>(Val: V)) return C;
9865 Instruction *I = dyn_cast<Instruction>(Val: V);
9866 if (!I) return nullptr;
9867
9868 if (Constant *C = Vals.lookup(Val: I)) return C;
9869
9870 // An instruction inside the loop depends on a value outside the loop that we
9871 // weren't given a mapping for, or a value such as a call inside the loop.
9872 if (!canConstantEvolve(I, L, TLI))
9873 return nullptr;
9874
9875 // An unmapped PHI can be due to a branch or another loop inside this loop,
9876 // or due to this not being the initial iteration through a loop where we
9877 // couldn't compute the evolution of this particular PHI last time.
9878 if (isa<PHINode>(Val: I)) return nullptr;
9879
9880 std::vector<Constant*> Operands(I->getNumOperands());
9881
9882 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
9883 Instruction *Operand = dyn_cast<Instruction>(Val: I->getOperand(i));
9884 if (!Operand) {
9885 Operands[i] = dyn_cast<Constant>(Val: I->getOperand(i));
9886 if (!Operands[i]) return nullptr;
9887 continue;
9888 }
9889 Constant *C = EvaluateExpression(V: Operand, L, Vals, DL, TLI);
9890 Vals[Operand] = C;
9891 if (!C) return nullptr;
9892 Operands[i] = C;
9893 }
9894
9895 return ConstantFoldInstOperands(I, Ops: Operands, DL, TLI,
9896 /*AllowNonDeterministic=*/false);
9897}
9898
9899
9900// If every incoming value to PN except the one for BB is a specific Constant,
9901// return that, else return nullptr.
9902static Constant *getOtherIncomingValue(PHINode *PN, BasicBlock *BB) {
9903 Constant *IncomingVal = nullptr;
9904
9905 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
9906 if (PN->getIncomingBlock(i) == BB)
9907 continue;
9908
9909 auto *CurrentVal = dyn_cast<Constant>(Val: PN->getIncomingValue(i));
9910 if (!CurrentVal)
9911 return nullptr;
9912
9913 if (IncomingVal != CurrentVal) {
9914 if (IncomingVal)
9915 return nullptr;
9916 IncomingVal = CurrentVal;
9917 }
9918 }
9919
9920 return IncomingVal;
9921}
9922
9923/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
9924/// in the header of its containing loop, we know the loop executes a
9925/// constant number of times, and the PHI node is just a recurrence
9926/// involving constants, fold it.
9927Constant *
9928ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
9929 const APInt &BEs,
9930 const Loop *L) {
9931 auto [I, Inserted] = ConstantEvolutionLoopExitValue.try_emplace(Key: PN);
9932 if (!Inserted)
9933 return I->second;
9934
9935 if (BEs.ugt(RHS: MaxBruteForceIterations))
9936 return nullptr; // Not going to evaluate it.
9937
9938 Constant *&RetVal = I->second;
9939
9940 DenseMap<Instruction *, Constant *> CurrentIterVals;
9941 BasicBlock *Header = L->getHeader();
9942 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9943
9944 BasicBlock *Latch = L->getLoopLatch();
9945 if (!Latch)
9946 return nullptr;
9947
9948 for (PHINode &PHI : Header->phis()) {
9949 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
9950 CurrentIterVals[&PHI] = StartCST;
9951 }
9952 if (!CurrentIterVals.count(Val: PN))
9953 return RetVal = nullptr;
9954
9955 Value *BEValue = PN->getIncomingValueForBlock(BB: Latch);
9956
9957 // Execute the loop symbolically to determine the exit value.
9958 assert(BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) &&
9959 "BEs is <= MaxBruteForceIterations which is an 'unsigned'!");
9960
9961 unsigned NumIterations = BEs.getZExtValue(); // must be in range
9962 unsigned IterationNum = 0;
9963 const DataLayout &DL = getDataLayout();
9964 for (; ; ++IterationNum) {
9965 if (IterationNum == NumIterations)
9966 return RetVal = CurrentIterVals[PN]; // Got exit value!
9967
9968 // Compute the value of the PHIs for the next iteration.
9969 // EvaluateExpression adds non-phi values to the CurrentIterVals map.
9970 DenseMap<Instruction *, Constant *> NextIterVals;
9971 Constant *NextPHI =
9972 EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9973 if (!NextPHI)
9974 return nullptr; // Couldn't evaluate!
9975 NextIterVals[PN] = NextPHI;
9976
9977 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
9978
9979 // Also evaluate the other PHI nodes. However, we don't get to stop if we
9980 // cease to be able to evaluate one of them or if they stop evolving,
9981 // because that doesn't necessarily prevent us from computing PN.
9982 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
9983 for (const auto &I : CurrentIterVals) {
9984 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
9985 if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
9986 PHIsToCompute.emplace_back(Args&: PHI, Args: I.second);
9987 }
9988 // We use two distinct loops because EvaluateExpression may invalidate any
9989 // iterators into CurrentIterVals.
9990 for (const auto &I : PHIsToCompute) {
9991 PHINode *PHI = I.first;
9992 Constant *&NextPHI = NextIterVals[PHI];
9993 if (!NextPHI) { // Not already computed.
9994 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
9995 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9996 }
9997 if (NextPHI != I.second)
9998 StoppedEvolving = false;
9999 }
10000
10001 // If all entries in CurrentIterVals == NextIterVals then we can stop
10002 // iterating, the loop can't continue to change.
10003 if (StoppedEvolving)
10004 return RetVal = CurrentIterVals[PN];
10005
10006 CurrentIterVals.swap(RHS&: NextIterVals);
10007 }
10008}
10009
10010const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L,
10011 Value *Cond,
10012 bool ExitWhen) {
10013 PHINode *PN = getConstantEvolvingPHI(V: Cond, L, TLI: &TLI);
10014 if (!PN) return getCouldNotCompute();
10015
10016 // If the loop is canonicalized, the PHI will have exactly two entries.
10017 // That's the only form we support here.
10018 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
10019
10020 DenseMap<Instruction *, Constant *> CurrentIterVals;
10021 BasicBlock *Header = L->getHeader();
10022 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
10023
10024 BasicBlock *Latch = L->getLoopLatch();
10025 assert(Latch && "Should follow from NumIncomingValues == 2!");
10026
10027 for (PHINode &PHI : Header->phis()) {
10028 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
10029 CurrentIterVals[&PHI] = StartCST;
10030 }
10031 if (!CurrentIterVals.count(Val: PN))
10032 return getCouldNotCompute();
10033
10034 // Okay, we find a PHI node that defines the trip count of this loop. Execute
10035 // the loop symbolically to determine when the condition gets a value of
10036 // "ExitWhen".
10037 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis.
10038 const DataLayout &DL = getDataLayout();
10039 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
10040 auto *CondVal = dyn_cast_or_null<ConstantInt>(
10041 Val: EvaluateExpression(V: Cond, L, Vals&: CurrentIterVals, DL, TLI: &TLI));
10042
10043 // Couldn't symbolically evaluate.
10044 if (!CondVal) return getCouldNotCompute();
10045
10046 if (CondVal->getValue() == uint64_t(ExitWhen)) {
10047 ++NumBruteForceTripCountsComputed;
10048 return getConstant(Ty: Type::getInt32Ty(C&: getContext()), V: IterationNum);
10049 }
10050
10051 // Update all the PHI nodes for the next iteration.
10052 DenseMap<Instruction *, Constant *> NextIterVals;
10053
10054 // Create a list of which PHIs we need to compute. We want to do this before
10055 // calling EvaluateExpression on them because that may invalidate iterators
10056 // into CurrentIterVals.
10057 SmallVector<PHINode *, 8> PHIsToCompute;
10058 for (const auto &I : CurrentIterVals) {
10059 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
10060 if (!PHI || PHI->getParent() != Header) continue;
10061 PHIsToCompute.push_back(Elt: PHI);
10062 }
10063 for (PHINode *PHI : PHIsToCompute) {
10064 Constant *&NextPHI = NextIterVals[PHI];
10065 if (NextPHI) continue; // Already computed!
10066
10067 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
10068 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
10069 }
10070 CurrentIterVals.swap(RHS&: NextIterVals);
10071 }
10072
10073 // Too many iterations were needed to evaluate.
10074 return getCouldNotCompute();
10075}
10076
10077SCEVUse ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
10078 auto &Values = ValuesAtScopes[V];
10079 // Check to see if we've folded this expression at this loop before.
10080 for (auto &LS : Values)
10081 if (LS.first == L)
10082 return LS.second ? LS.second : SCEVUse(V);
10083
10084 Values.emplace_back(Args&: L, Args: nullptr);
10085
10086 // Otherwise compute it.
10087 SCEVUse C = computeSCEVAtScope(S: V, L);
10088 for (auto &LS : reverse(C&: ValuesAtScopes[V]))
10089 if (LS.first == L) {
10090 LS.second = C;
10091 // Record the dependency under the bare expression: invalidation walks
10092 // expressions, and any use flags on C do not change which expression
10093 // this is the value at scope of.
10094 if (!isa<SCEVConstant>(Val: C))
10095 ValuesAtScopesUsers[C.getPointer()].push_back(Elt: {L, V});
10096 break;
10097 }
10098 return C;
10099}
10100
10101/// This builds up a Constant using the ConstantExpr interface. That way, we
10102/// will return Constants for objects which aren't represented by a
10103/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
10104/// Returns NULL if the SCEV isn't representable as a Constant.
10105static Constant *BuildConstantFromSCEV(const SCEV *V) {
10106 switch (V->getSCEVType()) {
10107 case scCouldNotCompute:
10108 case scAddRecExpr:
10109 case scVScale:
10110 return nullptr;
10111 case scConstant:
10112 return cast<SCEVConstant>(Val: V)->getValue();
10113 case scUnknown:
10114 return dyn_cast<Constant>(Val: cast<SCEVUnknown>(Val: V)->getValue());
10115 case scPtrToAddr: {
10116 const SCEVPtrToAddrExpr *P2I = cast<SCEVPtrToAddrExpr>(Val: V);
10117 if (Constant *CastOp = BuildConstantFromSCEV(V: P2I->getOperand()))
10118 return ConstantExpr::getPtrToAddr(C: CastOp, Ty: P2I->getType());
10119
10120 return nullptr;
10121 }
10122 case scTruncate: {
10123 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(Val: V);
10124 if (Constant *CastOp = BuildConstantFromSCEV(V: ST->getOperand()))
10125 return ConstantExpr::getTrunc(C: CastOp, Ty: ST->getType());
10126 return nullptr;
10127 }
10128 case scAddExpr: {
10129 const SCEVAddExpr *SA = cast<SCEVAddExpr>(Val: V);
10130 Constant *C = nullptr;
10131 for (const SCEV *Op : SA->operands()) {
10132 Constant *OpC = BuildConstantFromSCEV(V: Op);
10133 if (!OpC)
10134 return nullptr;
10135 if (!C) {
10136 C = OpC;
10137 continue;
10138 }
10139 assert(!C->getType()->isPointerTy() &&
10140 "Can only have one pointer, and it must be last");
10141 if (OpC->getType()->isPointerTy()) {
10142 // The offsets have been converted to bytes. We can add bytes using
10143 // an i8 GEP.
10144 C = ConstantExpr::getPtrAdd(Ptr: OpC, Offset: C);
10145 } else {
10146 C = ConstantExpr::getAdd(C1: C, C2: OpC);
10147 }
10148 }
10149 return C;
10150 }
10151 case scMulExpr:
10152 case scSignExtend:
10153 case scZeroExtend:
10154 case scUDivExpr:
10155 case scSMaxExpr:
10156 case scUMaxExpr:
10157 case scSMinExpr:
10158 case scUMinExpr:
10159 case scSequentialUMinExpr:
10160 return nullptr;
10161 }
10162 llvm_unreachable("Unknown SCEV kind!");
10163}
10164
10165const SCEV *ScalarEvolution::getWithOperands(const SCEV *S,
10166 SmallVectorImpl<SCEVUse> &NewOps) {
10167 switch (S->getSCEVType()) {
10168 case scTruncate:
10169 case scZeroExtend:
10170 case scSignExtend:
10171 case scPtrToAddr:
10172 return getCastExpr(Kind: S->getSCEVType(), Op: NewOps[0], Ty: S->getType());
10173 case scAddRecExpr: {
10174 auto *AddRec = cast<SCEVAddRecExpr>(Val: S);
10175 return getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), NWFlags: AddRec->getNoWrapFlags());
10176 }
10177 case scAddExpr:
10178 return getAddExpr(Ops&: NewOps, Flags: cast<SCEVAddExpr>(Val: S)->getNoWrapFlags());
10179 case scMulExpr:
10180 return getMulExpr(Ops&: NewOps, Flags: cast<SCEVMulExpr>(Val: S)->getNoWrapFlags());
10181 case scUDivExpr:
10182 return getUDivExpr(LHS: NewOps[0], RHS: NewOps[1]);
10183 case scUMaxExpr:
10184 case scSMaxExpr:
10185 case scUMinExpr:
10186 case scSMinExpr:
10187 return getMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10188 case scSequentialUMinExpr:
10189 return getSequentialMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10190 case scConstant:
10191 case scVScale:
10192 case scUnknown:
10193 return S;
10194 case scCouldNotCompute:
10195 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10196 }
10197 llvm_unreachable("Unknown SCEV kind!");
10198}
10199
10200SCEVUse ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
10201 switch (V->getSCEVType()) {
10202 case scConstant:
10203 case scVScale:
10204 return V;
10205 case scAddRecExpr: {
10206 // If this is a loop recurrence for a loop that does not contain L, then we
10207 // are dealing with the final value computed by the loop.
10208 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: V);
10209 // First, attempt to evaluate each operand.
10210 // Avoid performing the look-up in the common case where the specified
10211 // expression has no loop-variant portions.
10212 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
10213 SCEVUse OpAtScope = getSCEVAtScope(V: AddRec->getOperand(i), L);
10214 if (OpAtScope == AddRec->getOperand(i))
10215 continue;
10216
10217 // Okay, at least one of these operands is loop variant but might be
10218 // foldable. Build a new instance of the folded commutative expression.
10219 SmallVector<SCEVUse, 8> NewOps;
10220 NewOps.reserve(N: AddRec->getNumOperands());
10221 append_range(C&: NewOps, R: AddRec->operands().take_front(N: i));
10222 NewOps.push_back(Elt: OpAtScope);
10223 for (++i; i != e; ++i)
10224 NewOps.push_back(Elt: getSCEVAtScope(V: AddRec->getOperand(i), L));
10225
10226 const SCEV *FoldedRec = getAddRecExpr(
10227 Operands&: NewOps, L: AddRec->getLoop(), NWFlags: AddRec->getNoWrapFlags(Mask: SCEV::FlagNW));
10228 AddRec = dyn_cast<SCEVAddRecExpr>(Val: FoldedRec);
10229 // The addrec may be folded to a nonrecurrence, for example, if the
10230 // induction variable is multiplied by zero after constant folding. Go
10231 // ahead and return the folded value.
10232 if (!AddRec)
10233 return FoldedRec;
10234 break;
10235 }
10236
10237 // If the scope is outside the addrec's loop, evaluate it by using the
10238 // loop exit value of the addrec.
10239 if (!AddRec->getLoop()->contains(L)) {
10240 SCEVUse ExitValue = AddRec->getExitValue(SE&: *this);
10241 if (isa<SCEVCouldNotCompute>(Val: ExitValue))
10242 return AddRec;
10243 return ExitValue;
10244 }
10245
10246 return AddRec;
10247 }
10248 case scTruncate:
10249 case scZeroExtend:
10250 case scSignExtend:
10251 case scPtrToAddr:
10252 case scAddExpr:
10253 case scMulExpr:
10254 case scUDivExpr:
10255 case scUMaxExpr:
10256 case scSMaxExpr:
10257 case scUMinExpr:
10258 case scSMinExpr:
10259 case scSequentialUMinExpr: {
10260 ArrayRef<SCEVUse> Ops = V->operands();
10261 // Avoid performing the look-up in the common case where the specified
10262 // expression has no loop-variant portions.
10263 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
10264 SCEVUse OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10265 if (OpAtScope != Ops[i].getPointer()) {
10266 // Okay, at least one of these operands is loop variant but might be
10267 // foldable. Build a new instance of the folded commutative expression.
10268 SmallVector<SCEVUse, 8> NewOps;
10269 NewOps.reserve(N: Ops.size());
10270 append_range(C&: NewOps, R: Ops.take_front(N: i));
10271 NewOps.push_back(Elt: OpAtScope);
10272
10273 for (++i; i != e; ++i) {
10274 OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10275 NewOps.push_back(Elt: OpAtScope);
10276 }
10277
10278 return getWithOperands(S: V, NewOps);
10279 }
10280 }
10281 // If we got here, all operands are loop invariant.
10282 return V;
10283 }
10284 case scUnknown: {
10285 // If this instruction is evolved from a constant-evolving PHI, compute the
10286 // exit value from the loop without using SCEVs.
10287 const SCEVUnknown *SU = cast<SCEVUnknown>(Val: V);
10288 Instruction *I = dyn_cast<Instruction>(Val: SU->getValue());
10289 if (!I)
10290 return V; // This is some other type of SCEVUnknown, just return it.
10291
10292 if (PHINode *PN = dyn_cast<PHINode>(Val: I)) {
10293 const Loop *CurrLoop = this->LI[I->getParent()];
10294 // Looking for loop exit value.
10295 if (CurrLoop && CurrLoop->getParentLoop() == L &&
10296 PN->getParent() == CurrLoop->getHeader()) {
10297 // Okay, there is no closed form solution for the PHI node. Check
10298 // to see if the loop that contains it has a known backedge-taken
10299 // count. If so, we may be able to force computation of the exit
10300 // value.
10301 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: CurrLoop);
10302 // This trivial case can show up in some degenerate cases where
10303 // the incoming IR has not yet been fully simplified.
10304 if (BackedgeTakenCount->isZero()) {
10305 Value *InitValue = nullptr;
10306 bool MultipleInitValues = false;
10307 for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
10308 if (!CurrLoop->contains(BB: PN->getIncomingBlock(i))) {
10309 if (!InitValue)
10310 InitValue = PN->getIncomingValue(i);
10311 else if (InitValue != PN->getIncomingValue(i)) {
10312 MultipleInitValues = true;
10313 break;
10314 }
10315 }
10316 }
10317 if (!MultipleInitValues && InitValue)
10318 return getSCEV(V: InitValue);
10319 }
10320 // Do we have a loop invariant value flowing around the backedge
10321 // for a loop which must execute the backedge?
10322 if (!isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount) &&
10323 isKnownNonZero(S: BackedgeTakenCount) &&
10324 PN->getNumIncomingValues() == 2) {
10325
10326 unsigned InLoopPred =
10327 CurrLoop->contains(BB: PN->getIncomingBlock(i: 0)) ? 0 : 1;
10328 Value *BackedgeVal = PN->getIncomingValue(i: InLoopPred);
10329 if (CurrLoop->isLoopInvariant(V: BackedgeVal))
10330 return getSCEV(V: BackedgeVal);
10331 }
10332 if (auto *BTCC = dyn_cast<SCEVConstant>(Val: BackedgeTakenCount)) {
10333 // Okay, we know how many times the containing loop executes. If
10334 // this is a constant evolving PHI node, get the final value at
10335 // the specified iteration number.
10336 Constant *RV =
10337 getConstantEvolutionLoopExitValue(PN, BEs: BTCC->getAPInt(), L: CurrLoop);
10338 if (RV)
10339 return getSCEV(V: RV);
10340 }
10341 }
10342 }
10343
10344 // Okay, this is an expression that we cannot symbolically evaluate
10345 // into a SCEV. Check to see if it's possible to symbolically evaluate
10346 // the arguments into constants, and if so, try to constant propagate the
10347 // result. This is particularly useful for computing loop exit values.
10348 if (!canConstantFold(I, TLI: &TLI))
10349 return V; // This is some other type of SCEVUnknown, just return it.
10350
10351 SmallVector<Constant *, 4> Operands;
10352 Operands.reserve(N: I->getNumOperands());
10353 bool MadeImprovement = false;
10354 for (Value *Op : I->operands()) {
10355 if (Constant *C = dyn_cast<Constant>(Val: Op)) {
10356 Operands.push_back(Elt: C);
10357 continue;
10358 }
10359
10360 // If any of the operands is non-constant and if they are
10361 // non-integer and non-pointer, don't even try to analyze them
10362 // with scev techniques.
10363 if (!isSCEVable(Ty: Op->getType()))
10364 return V;
10365
10366 const SCEV *OrigV = getSCEV(V: Op);
10367 const SCEV *OpV = getSCEVAtScope(V: OrigV, L);
10368 MadeImprovement |= OrigV != OpV;
10369
10370 Constant *C = BuildConstantFromSCEV(V: OpV);
10371 if (!C)
10372 return V;
10373 assert(C->getType() == Op->getType() && "Type mismatch");
10374 Operands.push_back(Elt: C);
10375 }
10376
10377 // Check to see if getSCEVAtScope actually made an improvement.
10378 if (!MadeImprovement)
10379 return V; // This is some other type of SCEVUnknown, just return it.
10380
10381 Constant *C = nullptr;
10382 const DataLayout &DL = getDataLayout();
10383 C = ConstantFoldInstOperands(I, Ops: Operands, DL, TLI: &TLI,
10384 /*AllowNonDeterministic=*/false);
10385 if (!C)
10386 return V;
10387 return getSCEV(V: C);
10388 }
10389 case scCouldNotCompute:
10390 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10391 }
10392 llvm_unreachable("Unknown SCEV type!");
10393}
10394
10395SCEVUse ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
10396 return getSCEVAtScope(V: getSCEV(V), L);
10397}
10398
10399const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const {
10400 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: S))
10401 return stripInjectiveFunctions(S: ZExt->getOperand());
10402 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
10403 return stripInjectiveFunctions(S: SExt->getOperand());
10404 return S;
10405}
10406
10407/// Finds the minimum unsigned root of the following equation:
10408///
10409/// A * X = B (mod N)
10410///
10411/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
10412/// A and B isn't important.
10413///
10414/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
10415static const SCEV *
10416SolveLinEquationWithOverflow(const APInt &A, const SCEV *B,
10417 SmallVectorImpl<const SCEVPredicate *> *Predicates,
10418 ScalarEvolution &SE, const Loop *L) {
10419 uint32_t BW = A.getBitWidth();
10420 assert(BW == SE.getTypeSizeInBits(B->getType()));
10421 assert(A != 0 && "A must be non-zero.");
10422
10423 // 1. D = gcd(A, N)
10424 //
10425 // The gcd of A and N may have only one prime factor: 2. The number of
10426 // trailing zeros in A is its multiplicity
10427 uint32_t Mult2 = A.countr_zero();
10428 // D = 2^Mult2
10429
10430 // 2. Check if B is divisible by D.
10431 //
10432 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
10433 // is not less than multiplicity of this prime factor for D.
10434 unsigned MinTZ = SE.getMinTrailingZeros(S: B);
10435 // Try again with the terminator of the loop predecessor for context-specific
10436 // result, if MinTZ s too small.
10437 if (MinTZ < Mult2 && L->getLoopPredecessor())
10438 MinTZ = SE.getMinTrailingZeros(S: B, CtxI: L->getLoopPredecessor()->getTerminator());
10439 if (MinTZ < Mult2) {
10440 // Check if we can prove there's no remainder using URem.
10441 const SCEV *URem =
10442 SE.getURemExpr(LHS: B, RHS: SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2)));
10443 const SCEV *Zero = SE.getZero(Ty: B->getType());
10444 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: URem, RHS: Zero)) {
10445 // Try to add a predicate ensuring B is a multiple of 1 << Mult2.
10446 if (!Predicates)
10447 return SE.getCouldNotCompute();
10448
10449 // Avoid adding a predicate that is known to be false.
10450 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_NE, LHS: URem, RHS: Zero))
10451 return SE.getCouldNotCompute();
10452 Predicates->push_back(Elt: SE.getEqualPredicate(LHS: URem, RHS: Zero));
10453 }
10454 }
10455
10456 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
10457 // modulo (N / D).
10458 //
10459 // If D == 1, (N / D) == N == 2^BW, so we need one extra bit to represent
10460 // (N / D) in general. The inverse itself always fits into BW bits, though,
10461 // so we immediately truncate it.
10462 APInt AD = A.lshr(shiftAmt: Mult2).trunc(width: BW - Mult2); // AD = A / D
10463 APInt I = AD.multiplicativeInverse().zext(width: BW);
10464
10465 // 4. Compute the minimum unsigned root of the equation:
10466 // I * (B / D) mod (N / D)
10467 // To simplify the computation, we factor out the divide by D:
10468 // (I * B mod N) / D
10469 const SCEV *D = SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2));
10470 return SE.getUDivExactExpr(LHS: SE.getMulExpr(LHS: B, RHS: SE.getConstant(Val: I)), RHS: D);
10471}
10472
10473/// For a given quadratic addrec, generate coefficients of the corresponding
10474/// quadratic equation, multiplied by a common value to ensure that they are
10475/// integers.
10476/// The returned value is a tuple { A, B, C, M, BitWidth }, where
10477/// Ax^2 + Bx + C is the quadratic function, M is the value that A, B and C
10478/// were multiplied by, and BitWidth is the bit width of the original addrec
10479/// coefficients.
10480/// This function returns std::nullopt if the addrec coefficients are not
10481/// compile- time constants.
10482static std::optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
10483GetQuadraticEquation(const SCEVAddRecExpr *AddRec) {
10484 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
10485 const SCEVConstant *LC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 0));
10486 const SCEVConstant *MC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 1));
10487 const SCEVConstant *NC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 2));
10488 LLVM_DEBUG(dbgs() << __func__ << ": analyzing quadratic addrec: "
10489 << *AddRec << '\n');
10490
10491 // We currently can only solve this if the coefficients are constants.
10492 if (!LC || !MC || !NC) {
10493 LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n");
10494 return std::nullopt;
10495 }
10496
10497 APInt L = LC->getAPInt();
10498 APInt M = MC->getAPInt();
10499 APInt N = NC->getAPInt();
10500 assert(!N.isZero() && "This is not a quadratic addrec");
10501
10502 unsigned BitWidth = LC->getAPInt().getBitWidth();
10503 unsigned NewWidth = BitWidth + 1;
10504 LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "
10505 << BitWidth << '\n');
10506 // The sign-extension (as opposed to a zero-extension) here matches the
10507 // extension used in SolveQuadraticEquationWrap (with the same motivation).
10508 N = N.sext(width: NewWidth);
10509 M = M.sext(width: NewWidth);
10510 L = L.sext(width: NewWidth);
10511
10512 // The increments are M, M+N, M+2N, ..., so the accumulated values are
10513 // L+M, (L+M)+(M+N), (L+M)+(M+N)+(M+2N), ..., that is,
10514 // L+M, L+2M+N, L+3M+3N, ...
10515 // After n iterations the accumulated value Acc is L + nM + n(n-1)/2 N.
10516 //
10517 // The equation Acc = 0 is then
10518 // L + nM + n(n-1)/2 N = 0, or 2L + 2M n + n(n-1) N = 0.
10519 // In a quadratic form it becomes:
10520 // N n^2 + (2M-N) n + 2L = 0.
10521
10522 APInt A = N;
10523 APInt B = 2 * M - A;
10524 APInt C = 2 * L;
10525 APInt T = APInt(NewWidth, 2);
10526 LLVM_DEBUG(dbgs() << __func__ << ": equation " << A << "x^2 + " << B
10527 << "x + " << C << ", coeff bw: " << NewWidth
10528 << ", multiplied by " << T << '\n');
10529 return std::make_tuple(args&: A, args&: B, args&: C, args&: T, args&: BitWidth);
10530}
10531
10532/// Helper function to compare optional APInts:
10533/// (a) if X and Y both exist, return min(X, Y),
10534/// (b) if neither X nor Y exist, return std::nullopt,
10535/// (c) if exactly one of X and Y exists, return that value.
10536static std::optional<APInt> MinOptional(std::optional<APInt> X,
10537 std::optional<APInt> Y) {
10538 if (X && Y) {
10539 unsigned W = std::max(a: X->getBitWidth(), b: Y->getBitWidth());
10540 APInt XW = X->sext(width: W);
10541 APInt YW = Y->sext(width: W);
10542 return XW.slt(RHS: YW) ? *X : *Y;
10543 }
10544 if (!X && !Y)
10545 return std::nullopt;
10546 return X ? *X : *Y;
10547}
10548
10549/// Helper function to truncate an optional APInt to a given BitWidth.
10550/// When solving addrec-related equations, it is preferable to return a value
10551/// that has the same bit width as the original addrec's coefficients. If the
10552/// solution fits in the original bit width, truncate it (except for i1).
10553/// Returning a value of a different bit width may inhibit some optimizations.
10554///
10555/// In general, a solution to a quadratic equation generated from an addrec
10556/// may require BW+1 bits, where BW is the bit width of the addrec's
10557/// coefficients. The reason is that the coefficients of the quadratic
10558/// equation are BW+1 bits wide (to avoid truncation when converting from
10559/// the addrec to the equation).
10560static std::optional<APInt> TruncIfPossible(std::optional<APInt> X,
10561 unsigned BitWidth) {
10562 if (!X)
10563 return std::nullopt;
10564 unsigned W = X->getBitWidth();
10565 if (BitWidth > 1 && BitWidth < W && X->isIntN(N: BitWidth))
10566 return X->trunc(width: BitWidth);
10567 return X;
10568}
10569
10570/// Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n
10571/// iterations. The values L, M, N are assumed to be signed, and they
10572/// should all have the same bit widths.
10573/// Find the least n >= 0 such that c(n) = 0 in the arithmetic modulo 2^BW,
10574/// where BW is the bit width of the addrec's coefficients.
10575/// If the calculated value is a BW-bit integer (for BW > 1), it will be
10576/// returned as such, otherwise the bit width of the returned value may
10577/// be greater than BW.
10578///
10579/// This function returns std::nullopt if
10580/// (a) the addrec coefficients are not constant, or
10581/// (b) SolveQuadraticEquationWrap was unable to find a solution. For cases
10582/// like x^2 = 5, no integer solutions exist, in other cases an integer
10583/// solution may exist, but SolveQuadraticEquationWrap may fail to find it.
10584static std::optional<APInt>
10585SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
10586 APInt A, B, C, M;
10587 unsigned BitWidth;
10588 auto T = GetQuadraticEquation(AddRec);
10589 if (!T)
10590 return std::nullopt;
10591
10592 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10593 LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n");
10594 std::optional<APInt> X =
10595 APIntOps::SolveQuadraticEquationWrap(A, B, C, RangeWidth: BitWidth + 1);
10596 if (!X)
10597 return std::nullopt;
10598
10599 ConstantInt *CX = ConstantInt::get(Context&: SE.getContext(), V: *X);
10600 ConstantInt *V = EvaluateConstantChrecAtConstant(AddRec, C: CX, SE);
10601 if (!V->isZero())
10602 return std::nullopt;
10603
10604 return TruncIfPossible(X, BitWidth);
10605}
10606
10607/// Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n
10608/// iterations. The values M, N are assumed to be signed, and they
10609/// should all have the same bit widths.
10610/// Find the least n such that c(n) does not belong to the given range,
10611/// while c(n-1) does.
10612///
10613/// This function returns std::nullopt if
10614/// (a) the addrec coefficients are not constant, or
10615/// (b) SolveQuadraticEquationWrap was unable to find a solution for the
10616/// bounds of the range.
10617static std::optional<APInt>
10618SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec,
10619 const ConstantRange &Range, ScalarEvolution &SE) {
10620 assert(AddRec->getOperand(0)->isZero() &&
10621 "Starting value of addrec should be 0");
10622 LLVM_DEBUG(dbgs() << __func__ << ": solving boundary crossing for range "
10623 << Range << ", addrec " << *AddRec << '\n');
10624 // This case is handled in getNumIterationsInRange. Here we can assume that
10625 // we start in the range.
10626 assert(Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) &&
10627 "Addrec's initial value should be in range");
10628
10629 APInt A, B, C, M;
10630 unsigned BitWidth;
10631 auto T = GetQuadraticEquation(AddRec);
10632 if (!T)
10633 return std::nullopt;
10634
10635 // Be careful about the return value: there can be two reasons for not
10636 // returning an actual number. First, if no solutions to the equations
10637 // were found, and second, if the solutions don't leave the given range.
10638 // The first case means that the actual solution is "unknown", the second
10639 // means that it's known, but not valid. If the solution is unknown, we
10640 // cannot make any conclusions.
10641 // Return a pair: the optional solution and a flag indicating if the
10642 // solution was found.
10643 auto SolveForBoundary =
10644 [&](APInt Bound) -> std::pair<std::optional<APInt>, bool> {
10645 // Solve for signed overflow and unsigned overflow, pick the lower
10646 // solution.
10647 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: checking boundary "
10648 << Bound << " (before multiplying by " << M << ")\n");
10649 Bound *= M; // The quadratic equation multiplier.
10650
10651 std::optional<APInt> SO;
10652 if (BitWidth > 1) {
10653 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10654 "signed overflow\n");
10655 SO = APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth);
10656 }
10657 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10658 "unsigned overflow\n");
10659 std::optional<APInt> UO =
10660 APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth + 1);
10661
10662 auto LeavesRange = [&] (const APInt &X) {
10663 ConstantInt *C0 = ConstantInt::get(Context&: SE.getContext(), V: X);
10664 ConstantInt *V0 = EvaluateConstantChrecAtConstant(AddRec, C: C0, SE);
10665 if (Range.contains(Val: V0->getValue()))
10666 return false;
10667 // X should be at least 1, so X-1 is non-negative.
10668 ConstantInt *C1 = ConstantInt::get(Context&: SE.getContext(), V: X-1);
10669 ConstantInt *V1 = EvaluateConstantChrecAtConstant(AddRec, C: C1, SE);
10670 if (Range.contains(Val: V1->getValue()))
10671 return true;
10672 return false;
10673 };
10674
10675 // If SolveQuadraticEquationWrap returns std::nullopt, it means that there
10676 // can be a solution, but the function failed to find it. We cannot treat it
10677 // as "no solution".
10678 if (!SO || !UO)
10679 return {std::nullopt, false};
10680
10681 // Check the smaller value first to see if it leaves the range.
10682 // At this point, both SO and UO must have values.
10683 std::optional<APInt> Min = MinOptional(X: SO, Y: UO);
10684 if (LeavesRange(*Min))
10685 return { Min, true };
10686 std::optional<APInt> Max = Min == SO ? UO : SO;
10687 if (LeavesRange(*Max))
10688 return { Max, true };
10689
10690 // Solutions were found, but were eliminated, hence the "true".
10691 return {std::nullopt, true};
10692 };
10693
10694 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10695 // Lower bound is inclusive, subtract 1 to represent the exiting value.
10696 APInt Lower = Range.getLower().sext(width: A.getBitWidth()) - 1;
10697 APInt Upper = Range.getUpper().sext(width: A.getBitWidth());
10698 auto SL = SolveForBoundary(Lower);
10699 auto SU = SolveForBoundary(Upper);
10700 // If any of the solutions was unknown, no meaninigful conclusions can
10701 // be made.
10702 if (!SL.second || !SU.second)
10703 return std::nullopt;
10704
10705 // Claim: The correct solution is not some value between Min and Max.
10706 //
10707 // Justification: Assuming that Min and Max are different values, one of
10708 // them is when the first signed overflow happens, the other is when the
10709 // first unsigned overflow happens. Crossing the range boundary is only
10710 // possible via an overflow (treating 0 as a special case of it, modeling
10711 // an overflow as crossing k*2^W for some k).
10712 //
10713 // The interesting case here is when Min was eliminated as an invalid
10714 // solution, but Max was not. The argument is that if there was another
10715 // overflow between Min and Max, it would also have been eliminated if
10716 // it was considered.
10717 //
10718 // For a given boundary, it is possible to have two overflows of the same
10719 // type (signed/unsigned) without having the other type in between: this
10720 // can happen when the vertex of the parabola is between the iterations
10721 // corresponding to the overflows. This is only possible when the two
10722 // overflows cross k*2^W for the same k. In such case, if the second one
10723 // left the range (and was the first one to do so), the first overflow
10724 // would have to enter the range, which would mean that either we had left
10725 // the range before or that we started outside of it. Both of these cases
10726 // are contradictions.
10727 //
10728 // Claim: In the case where SolveForBoundary returns std::nullopt, the correct
10729 // solution is not some value between the Max for this boundary and the
10730 // Min of the other boundary.
10731 //
10732 // Justification: Assume that we had such Max_A and Min_B corresponding
10733 // to range boundaries A and B and such that Max_A < Min_B. If there was
10734 // a solution between Max_A and Min_B, it would have to be caused by an
10735 // overflow corresponding to either A or B. It cannot correspond to B,
10736 // since Min_B is the first occurrence of such an overflow. If it
10737 // corresponded to A, it would have to be either a signed or an unsigned
10738 // overflow that is larger than both eliminated overflows for A. But
10739 // between the eliminated overflows and this overflow, the values would
10740 // cover the entire value space, thus crossing the other boundary, which
10741 // is a contradiction.
10742
10743 return TruncIfPossible(X: MinOptional(X: SL.first, Y: SU.first), BitWidth);
10744}
10745
10746ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V,
10747 const Loop *L,
10748 bool ControlsOnlyExit,
10749 bool AllowPredicates) {
10750
10751 // This is only used for loops with a "x != y" exit test. The exit condition
10752 // is now expressed as a single expression, V = x-y. So the exit test is
10753 // effectively V != 0. We know and take advantage of the fact that this
10754 // expression only being used in a comparison by zero context.
10755
10756 SmallVector<const SCEVPredicate *> Predicates;
10757 // If the value is a constant
10758 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10759 // If the value is already zero, the branch will execute zero times.
10760 if (C->getValue()->isZero()) return C;
10761 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10762 }
10763
10764 const SCEVAddRecExpr *AddRec =
10765 dyn_cast<SCEVAddRecExpr>(Val: stripInjectiveFunctions(S: V));
10766
10767 if (!AddRec && AllowPredicates)
10768 // Try to make this an AddRec using runtime tests, in the first X
10769 // iterations of this loop, where X is the SCEV expression found by the
10770 // algorithm below.
10771 AddRec = convertSCEVToAddRecWithPredicates(S: V, L, Preds&: Predicates);
10772
10773 if (!AddRec || AddRec->getLoop() != L)
10774 return getCouldNotCompute();
10775
10776 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
10777 // the quadratic equation to solve it.
10778 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
10779 // We can only use this value if the chrec ends up with an exact zero
10780 // value at this index. When solving for "X*X != 5", for example, we
10781 // should not accept a root of 2.
10782 if (auto S = SolveQuadraticAddRecExact(AddRec, SE&: *this)) {
10783 const auto *R = cast<SCEVConstant>(Val: getConstant(Val: *S));
10784 return ExitLimit(R, R, R, false, Predicates);
10785 }
10786 return getCouldNotCompute();
10787 }
10788
10789 // Otherwise we can only handle this if it is affine.
10790 if (!AddRec->isAffine())
10791 return getCouldNotCompute();
10792
10793 // If this is an affine expression, the execution count of this branch is
10794 // the minimum unsigned root of the following equation:
10795 //
10796 // Start + Step*N = 0 (mod 2^BW)
10797 //
10798 // equivalent to:
10799 //
10800 // Step*N = -Start (mod 2^BW)
10801 //
10802 // where BW is the common bit width of Start and Step.
10803
10804 // Get the initial value for the loop.
10805 const SCEV *Start = getSCEVAtScope(V: AddRec->getStart(), L: L->getParentLoop());
10806 const SCEV *Step = getSCEVAtScope(V: AddRec->getOperand(i: 1), L: L->getParentLoop());
10807
10808 if (!isLoopInvariant(S: Step, L))
10809 return getCouldNotCompute();
10810
10811 LoopGuards Guards = LoopGuards::collect(L, SE&: *this);
10812 // Specialize step for this loop so we get context sensitive facts below.
10813 const SCEV *StepWLG = applyLoopGuards(Expr: Step, Guards);
10814
10815 // For positive steps (counting up until unsigned overflow):
10816 // N = -Start/Step (as unsigned)
10817 // For negative steps (counting down to zero):
10818 // N = Start/-Step
10819 // First compute the unsigned distance from zero in the direction of Step.
10820 bool CountDown = isKnownNegative(S: StepWLG);
10821 if (!CountDown && !isKnownNonNegative(S: StepWLG))
10822 return getCouldNotCompute();
10823
10824 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(V: Start);
10825 // Handle unitary steps, which cannot wraparound.
10826 // 1*N = -Start; -1*N = Start (mod 2^BW), so:
10827 // N = Distance (as unsigned)
10828
10829 if (match(S: Step, P: m_CombineOr(Ps: m_scev_One(), Ps: m_scev_AllOnes()))) {
10830 APInt MaxBECount = getUnsignedRangeMax(S: applyLoopGuards(Expr: Distance, Guards));
10831 MaxBECount = APIntOps::umin(A: MaxBECount, B: getUnsignedRangeMax(S: Distance));
10832
10833 // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated,
10834 // we end up with a loop whose backedge-taken count is n - 1. Detect this
10835 // case, and see if we can improve the bound.
10836 //
10837 // Explicitly handling this here is necessary because getUnsignedRange
10838 // isn't context-sensitive; it doesn't know that we only care about the
10839 // range inside the loop.
10840 const SCEV *Zero = getZero(Ty: Distance->getType());
10841 const SCEV *One = getOne(Ty: Distance->getType());
10842 const SCEV *DistancePlusOne = getAddExpr(LHS: Distance, RHS: One);
10843 if (isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: DistancePlusOne, RHS: Zero)) {
10844 // If Distance + 1 doesn't overflow, we can compute the maximum distance
10845 // as "unsigned_max(Distance + 1) - 1". Also apply the loop guards to
10846 // Distance + 1; the range of Distance itself may be a wrapped set even
10847 // when the guards bound Distance + 1 tightly.
10848 APInt Max = APIntOps::umin(
10849 A: getUnsignedRangeMax(S: applyLoopGuards(Expr: DistancePlusOne, Guards)),
10850 B: getUnsignedRangeMax(S: DistancePlusOne));
10851 MaxBECount = APIntOps::umin(A: MaxBECount, B: Max - 1);
10852 }
10853 return ExitLimit(Distance, getConstant(Val: MaxBECount), Distance, false,
10854 Predicates);
10855 }
10856
10857 // If the condition controls loop exit (the loop exits only if the expression
10858 // is true) and the addition is no-wrap we can use unsigned divide to
10859 // compute the backedge count. In this case, the step may not divide the
10860 // distance, but we don't care because if the condition is "missed" the loop
10861 // will have undefined behavior due to wrapping.
10862 if (ControlsOnlyExit && AddRec->hasNoSelfWrap() &&
10863 loopHasNoAbnormalExits(L: AddRec->getLoop())) {
10864
10865 // If the stride is zero and the start is non-zero, the loop must be
10866 // infinite. In C++, most loops are finite by assumption, in which case the
10867 // step being zero implies UB must execute if the loop is entered.
10868 if (!(loopIsFiniteByAssumption(L) && isKnownNonZero(S: Start)) &&
10869 !isKnownNonZero(S: StepWLG))
10870 return getCouldNotCompute();
10871
10872 const SCEV *Exact =
10873 getUDivExpr(LHS: Distance, RHS: CountDown ? getNegativeSCEV(V: Step) : Step);
10874 const SCEV *ConstantMax = getCouldNotCompute();
10875 if (Exact != getCouldNotCompute()) {
10876 APInt MaxInt = getUnsignedRangeMax(S: applyLoopGuards(Expr: Exact, Guards));
10877 ConstantMax =
10878 getConstant(Val: APIntOps::umin(A: MaxInt, B: getUnsignedRangeMax(S: Exact)));
10879 }
10880 const SCEV *SymbolicMax =
10881 isa<SCEVCouldNotCompute>(Val: Exact) ? ConstantMax : Exact;
10882 return ExitLimit(Exact, ConstantMax, SymbolicMax, false, Predicates);
10883 }
10884
10885 // Solve the general equation.
10886 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Val: Step);
10887 if (!StepC || StepC->getValue()->isZero())
10888 return getCouldNotCompute();
10889 const SCEV *E = SolveLinEquationWithOverflow(
10890 A: StepC->getAPInt(), B: getNegativeSCEV(V: Start),
10891 Predicates: AllowPredicates ? &Predicates : nullptr, SE&: *this, L);
10892
10893 const SCEV *M = E;
10894 if (E != getCouldNotCompute()) {
10895 APInt MaxWithGuards = getUnsignedRangeMax(S: applyLoopGuards(Expr: E, Guards));
10896 M = getConstant(Val: APIntOps::umin(A: MaxWithGuards, B: getUnsignedRangeMax(S: E)));
10897 }
10898 auto *S = isa<SCEVCouldNotCompute>(Val: E) ? M : E;
10899 return ExitLimit(E, M, S, false, Predicates);
10900}
10901
10902ScalarEvolution::ExitLimit
10903ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) {
10904 // Loops that look like: while (X == 0) are very strange indeed. We don't
10905 // handle them yet except for the trivial case. This could be expanded in the
10906 // future as needed.
10907
10908 // If the value is a constant, check to see if it is known to be non-zero
10909 // already. If so, the backedge will execute zero times.
10910 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10911 if (!C->getValue()->isZero())
10912 return getZero(Ty: C->getType());
10913 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10914 }
10915
10916 // We could implement others, but I really doubt anyone writes loops like
10917 // this, and if they did, they would already be constant folded.
10918 return getCouldNotCompute();
10919}
10920
10921std::pair<const BasicBlock *, const BasicBlock *>
10922ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(const BasicBlock *BB)
10923 const {
10924 // If the block has a unique predecessor, then there is no path from the
10925 // predecessor to the block that does not go through the direct edge
10926 // from the predecessor to the block.
10927 if (const BasicBlock *Pred = BB->getSinglePredecessor())
10928 return {Pred, BB};
10929
10930 // A loop's header is defined to be a block that dominates the loop.
10931 // If the header has a unique predecessor outside the loop, it must be
10932 // a block that has exactly one successor that can reach the loop.
10933 if (const Loop *L = LI.getLoopFor(BB))
10934 return {L->getLoopPredecessor(), L->getHeader()};
10935
10936 return {nullptr, BB};
10937}
10938
10939/// SCEV structural equivalence is usually sufficient for testing whether two
10940/// expressions are equal, however for the purposes of looking for a condition
10941/// guarding a loop, it can be useful to be a little more general, since a
10942/// front-end may have replicated the controlling expression.
10943static bool HasSameValue(const SCEV *A, const SCEV *B) {
10944 // Quick check to see if they are the same SCEV.
10945 if (A == B) return true;
10946
10947 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) {
10948 // Not all instructions that are "identical" compute the same value. For
10949 // instance, two distinct alloca instructions allocating the same type are
10950 // identical and do not read memory; but compute distinct values.
10951 return A->isIdenticalTo(I: B) && (isa<BinaryOperator>(Val: A) || isa<GetElementPtrInst>(Val: A));
10952 };
10953
10954 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
10955 // two different instructions with the same value. Check for this case.
10956 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(Val: A))
10957 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(Val: B))
10958 if (const Instruction *AI = dyn_cast<Instruction>(Val: AU->getValue()))
10959 if (const Instruction *BI = dyn_cast<Instruction>(Val: BU->getValue()))
10960 if (ComputesEqualValues(AI, BI))
10961 return true;
10962
10963 // Otherwise assume they may have a different value.
10964 return false;
10965}
10966
10967static bool MatchBinarySub(const SCEV *S, SCEVUse &LHS, SCEVUse &RHS) {
10968 const SCEV *Op0, *Op1;
10969 if (!match(S, P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))))
10970 return false;
10971 if (match(S: Op0, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
10972 LHS = Op1;
10973 return true;
10974 }
10975 if (match(S: Op1, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
10976 LHS = Op0;
10977 return true;
10978 }
10979 return false;
10980}
10981
10982bool ScalarEvolution::SimplifyICmpOperands(CmpPredicate &Pred, SCEVUse &LHS,
10983 SCEVUse &RHS, unsigned Depth) {
10984 bool Changed = false;
10985 // Simplifies ICMP to trivial true or false by turning it into '0 == 0' or
10986 // '0 != 0'.
10987 auto TrivialCase = [&](bool TriviallyTrue) {
10988 LHS = RHS = getConstant(V: ConstantInt::getFalse(Context&: getContext()));
10989 Pred = TriviallyTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
10990 return true;
10991 };
10992 // If we hit the max recursion limit bail out.
10993 if (Depth >= 3)
10994 return false;
10995
10996 const SCEV *NewLHS, *NewRHS;
10997 if (match(U: LHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
10998 match(U: RHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale()))) {
10999 const SCEVMulExpr *LMul = cast<SCEVMulExpr>(Val&: LHS);
11000 const SCEVMulExpr *RMul = cast<SCEVMulExpr>(Val&: RHS);
11001
11002 // (X * vscale) pred (Y * vscale) ==> X pred Y
11003 // when both multiples are NSW.
11004 // (X * vscale) uicmp/eq/ne (Y * vscale) ==> X uicmp/eq/ne Y
11005 // when both multiples are NUW.
11006 if ((LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap()) ||
11007 (LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap() &&
11008 !ICmpInst::isSigned(Pred))) {
11009 LHS = NewLHS;
11010 RHS = NewRHS;
11011 Changed = true;
11012 }
11013 }
11014
11015 // Canonicalize a constant to the right side.
11016 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS)) {
11017 // Check for both operands constant.
11018 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
11019 if (!ICmpInst::compare(LHS: LHSC->getAPInt(), RHS: RHSC->getAPInt(), Pred))
11020 return TrivialCase(false);
11021 return TrivialCase(true);
11022 }
11023 // Otherwise swap the operands to put the constant on the right.
11024 std::swap(a&: LHS, b&: RHS);
11025 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11026 Changed = true;
11027 }
11028
11029 // (K + A) pred (K + B) --> A pred B
11030 // For equality, no flags are needed.
11031 // For signed, both adds must be NSW. For unsigned, both must be NUW.
11032 {
11033 const SCEVConstant *C = nullptr;
11034 if (match(U: LHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: NewLHS))) &&
11035 match(U: RHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: NewRHS)))) {
11036 const auto *LAdd = cast<SCEVAddExpr>(Val&: LHS);
11037 const auto *RAdd = cast<SCEVAddExpr>(Val&: RHS);
11038 if (ICmpInst::isEquality(P: Pred) ||
11039 (ICmpInst::isSigned(Pred) && LAdd->hasNoSignedWrap() &&
11040 RAdd->hasNoSignedWrap()) ||
11041 (ICmpInst::isUnsigned(Pred) && LAdd->hasNoUnsignedWrap() &&
11042 RAdd->hasNoUnsignedWrap())) {
11043 LHS = NewLHS;
11044 RHS = NewRHS;
11045 Changed = true;
11046 }
11047 }
11048 }
11049
11050 // (C * A) pred (C * B) --> A pred B
11051 // For equality predicates, both muls must be NUW or both must be NSW
11052 // (either suffices to make multiplication by C injective; C == 0 is
11053 // impossible because SCEV folds 0 * X to 0).
11054 // For signed ordering, C must be positive and both muls must be NSW.
11055 // For unsigned ordering, both muls must be NUW.
11056 {
11057 const SCEVConstant *C = nullptr;
11058 if (match(U: LHS, P: m_scev_Mul(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: NewLHS))) &&
11059 match(U: RHS, P: m_scev_Mul(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: NewRHS)))) {
11060 const auto *LMul = cast<SCEVMulExpr>(Val&: LHS);
11061 const auto *RMul = cast<SCEVMulExpr>(Val&: RHS);
11062 bool BothNUW = LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap();
11063 bool BothNSW = LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap();
11064 if ((ICmpInst::isEquality(P: Pred) && (BothNUW || BothNSW)) ||
11065 (ICmpInst::isSigned(Pred) && BothNSW &&
11066 C->getAPInt().isStrictlyPositive()) ||
11067 (ICmpInst::isUnsigned(Pred) && BothNUW)) {
11068 LHS = NewLHS;
11069 RHS = NewRHS;
11070 Changed = true;
11071 }
11072 }
11073 }
11074
11075 // If we're comparing an addrec with a value which is loop-invariant in the
11076 // addrec's loop, put the addrec on the left. Also make a dominance check,
11077 // as both operands could be addrecs loop-invariant in each other's loop.
11078 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: RHS)) {
11079 const Loop *L = AR->getLoop();
11080 if (isLoopInvariant(S: LHS, L) && properlyDominates(S: LHS, BB: L->getHeader())) {
11081 std::swap(a&: LHS, b&: RHS);
11082 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11083 Changed = true;
11084 }
11085 }
11086
11087 // If there's a constant operand, canonicalize comparisons with boundary
11088 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
11089 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(Val&: RHS)) {
11090 const APInt &RA = RC->getAPInt();
11091
11092 bool SimplifiedByConstantRange = false;
11093
11094 if (!ICmpInst::isEquality(P: Pred)) {
11095 ConstantRange ExactCR = ConstantRange::makeExactICmpRegion(Pred, Other: RA);
11096 if (ExactCR.isFullSet())
11097 return TrivialCase(true);
11098 if (ExactCR.isEmptySet())
11099 return TrivialCase(false);
11100
11101 APInt NewRHS;
11102 CmpInst::Predicate NewPred;
11103 if (ExactCR.getEquivalentICmp(Pred&: NewPred, RHS&: NewRHS) &&
11104 ICmpInst::isEquality(P: NewPred)) {
11105 // We were able to convert an inequality to an equality.
11106 Pred = NewPred;
11107 RHS = getConstant(Val: NewRHS);
11108 Changed = SimplifiedByConstantRange = true;
11109 }
11110 }
11111
11112 if (!SimplifiedByConstantRange) {
11113 switch (Pred) {
11114 default:
11115 break;
11116 case ICmpInst::ICMP_EQ:
11117 case ICmpInst::ICMP_NE:
11118 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
11119 if (RA.isZero() && MatchBinarySub(S: LHS, LHS, RHS))
11120 Changed = true;
11121 break;
11122
11123 // The "Should have been caught earlier!" messages refer to the fact
11124 // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above
11125 // should have fired on the corresponding cases, and canonicalized the
11126 // check to trivial case.
11127
11128 case ICmpInst::ICMP_UGE:
11129 assert(!RA.isMinValue() && "Should have been caught earlier!");
11130 Pred = ICmpInst::ICMP_UGT;
11131 RHS = getConstant(Val: RA - 1);
11132 Changed = true;
11133 break;
11134 case ICmpInst::ICMP_ULE:
11135 assert(!RA.isMaxValue() && "Should have been caught earlier!");
11136 Pred = ICmpInst::ICMP_ULT;
11137 RHS = getConstant(Val: RA + 1);
11138 Changed = true;
11139 break;
11140 case ICmpInst::ICMP_SGE:
11141 assert(!RA.isMinSignedValue() && "Should have been caught earlier!");
11142 Pred = ICmpInst::ICMP_SGT;
11143 RHS = getConstant(Val: RA - 1);
11144 Changed = true;
11145 break;
11146 case ICmpInst::ICMP_SLE:
11147 assert(!RA.isMaxSignedValue() && "Should have been caught earlier!");
11148 Pred = ICmpInst::ICMP_SLT;
11149 RHS = getConstant(Val: RA + 1);
11150 Changed = true;
11151 break;
11152 }
11153 }
11154 }
11155
11156 // a /u b == 0 => a < b
11157 // a /u b != 0 => a >= b
11158 if (ICmpInst::isEquality(P: Pred) && RHS->isZero() &&
11159 match(U: LHS, P: m_scev_UDiv(Op0: m_SCEV(V&: LHS), Op1: m_SCEV(V&: RHS)))) {
11160 Pred = Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE;
11161 Changed = true;
11162 }
11163
11164 // Check for obvious equality.
11165 if (HasSameValue(A: LHS, B: RHS)) {
11166 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
11167 return TrivialCase(true);
11168 if (ICmpInst::isFalseWhenEqual(predicate: Pred))
11169 return TrivialCase(false);
11170 }
11171
11172 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
11173 // adding or subtracting 1 from one of the operands.
11174 switch (Pred) {
11175 case ICmpInst::ICMP_SLE:
11176 if (!getSignedRangeMax(S: RHS).isMaxSignedValue()) {
11177 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11178 Flags: SCEV::FlagNSW);
11179 Pred = ICmpInst::ICMP_SLT;
11180 Changed = true;
11181 } else if (!getSignedRangeMin(S: LHS).isMinSignedValue()) {
11182 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS,
11183 Flags: SCEV::FlagNSW);
11184 Pred = ICmpInst::ICMP_SLT;
11185 Changed = true;
11186 }
11187 break;
11188 case ICmpInst::ICMP_SGE:
11189 if (!getSignedRangeMin(S: RHS).isMinSignedValue()) {
11190 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS,
11191 Flags: SCEV::FlagNSW);
11192 Pred = ICmpInst::ICMP_SGT;
11193 Changed = true;
11194 } else if (!getSignedRangeMax(S: LHS).isMaxSignedValue()) {
11195 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11196 Flags: SCEV::FlagNSW);
11197 Pred = ICmpInst::ICMP_SGT;
11198 Changed = true;
11199 }
11200 break;
11201 case ICmpInst::ICMP_ULE:
11202 if (!getUnsignedRangeMax(S: RHS).isMaxValue()) {
11203 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11204 Flags: SCEV::FlagNUW);
11205 Pred = ICmpInst::ICMP_ULT;
11206 Changed = true;
11207 } else if (!getUnsignedRangeMin(S: LHS).isMinValue()) {
11208 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS);
11209 Pred = ICmpInst::ICMP_ULT;
11210 Changed = true;
11211 }
11212 break;
11213 case ICmpInst::ICMP_UGE:
11214 // If RHS is an op we can fold the -1, try that first.
11215 // Otherwise prefer LHS to preserve the nuw flag.
11216 if ((isa<SCEVConstant>(Val: RHS) ||
11217 (isa<SCEVAddExpr, SCEVAddRecExpr>(Val: RHS) &&
11218 isa<SCEVConstant>(Val: cast<SCEVNAryExpr>(Val&: RHS)->getOperand(i: 0)))) &&
11219 !getUnsignedRangeMin(S: RHS).isMinValue()) {
11220 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11221 Pred = ICmpInst::ICMP_UGT;
11222 Changed = true;
11223 } else if (!getUnsignedRangeMax(S: LHS).isMaxValue()) {
11224 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11225 Flags: SCEV::FlagNUW);
11226 Pred = ICmpInst::ICMP_UGT;
11227 Changed = true;
11228 } else if (!getUnsignedRangeMin(S: RHS).isMinValue()) {
11229 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11230 Pred = ICmpInst::ICMP_UGT;
11231 Changed = true;
11232 }
11233 break;
11234 default:
11235 break;
11236 }
11237
11238 // TODO: More simplifications are possible here.
11239
11240 // Recursively simplify until we either hit a recursion limit or nothing
11241 // changes.
11242 if (Changed)
11243 (void)SimplifyICmpOperands(Pred, LHS, RHS, Depth: Depth + 1);
11244
11245 return Changed;
11246}
11247
11248bool ScalarEvolution::isKnownNegative(const SCEV *S) {
11249 return getSignedRangeMax(S).isNegative();
11250}
11251
11252bool ScalarEvolution::isKnownPositive(const SCEV *S) {
11253 return getSignedRangeMin(S).isStrictlyPositive();
11254}
11255
11256bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
11257 return !getSignedRangeMin(S).isNegative();
11258}
11259
11260bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
11261 return !getSignedRangeMax(S).isStrictlyPositive();
11262}
11263
11264bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
11265 // Query push down for cases where the unsigned range is
11266 // less than sufficient.
11267 if (const auto *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
11268 return isKnownNonZero(S: SExt->getOperand(i: 0));
11269 return getUnsignedRangeMin(S) != 0;
11270}
11271
11272bool ScalarEvolution::isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero,
11273 bool OrNegative) {
11274 auto NonRecursive = [OrNegative](const SCEV *S) {
11275 if (auto *C = dyn_cast<SCEVConstant>(Val: S))
11276 return C->getAPInt().isPowerOf2() ||
11277 (OrNegative && C->getAPInt().isNegatedPowerOf2());
11278
11279 // vscale is a power-of-two.
11280 return isa<SCEVVScale>(Val: S);
11281 };
11282
11283 if (NonRecursive(S))
11284 return true;
11285
11286 auto *Mul = dyn_cast<SCEVMulExpr>(Val: S);
11287 if (!Mul)
11288 return false;
11289 return all_of(Range: Mul->operands(), P: NonRecursive) && (OrZero || isKnownNonZero(S));
11290}
11291
11292bool ScalarEvolution::isKnownMultipleOf(
11293 const SCEV *S, uint64_t M,
11294 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
11295 if (M == 0)
11296 return false;
11297 if (M == 1)
11298 return true;
11299
11300 // For a constant, check that "S % M == 0".
11301 if (auto *Cst = dyn_cast<SCEVConstant>(Val: S)) {
11302 APInt C = Cst->getAPInt();
11303 return C.urem(RHS: M) == 0;
11304 }
11305
11306 // Basic tests have failed.
11307 // Check "S % M == 0" at compile time and record runtime Assumptions.
11308 auto *STy = dyn_cast<IntegerType>(Val: S->getType());
11309 const SCEV *SmodM =
11310 getURemExpr(LHS: S, RHS: getConstant(V: ConstantInt::get(Ty: STy, V: M, IsSigned: false)));
11311 const SCEV *Zero = getZero(Ty: STy);
11312
11313 // Check whether "S % M == 0" is known at compile time.
11314 if (isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero))
11315 return true;
11316
11317 // Check whether "S % M != 0" is known at compile time.
11318 if (isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: SmodM, RHS: Zero))
11319 return false;
11320
11321 if (!Predicates)
11322 return false;
11323
11324 // Look through Add and AddRec expressions with nuw to improve the
11325 // precision of added predicates. S is a multiple of M if S starts with a
11326 // multiple of M and at every iteration step S only adds multiples of M.
11327 if (isa<SCEVAddExpr, SCEVAddRecExpr>(Val: S) &&
11328 cast<SCEVNAryExpr>(Val: S)->hasNoUnsignedWrap() &&
11329 all_of(Range: S->operands(),
11330 P: [&](SCEVUse Op) { return isKnownMultipleOf(S: Op, M, Predicates); }))
11331 return true;
11332
11333 // Similarly, look through Mul with nuw, where any operand being a
11334 // known-multiple is sufficient.
11335 if (auto *Mul = dyn_cast<SCEVMulExpr>(Val: S))
11336 if (Mul->hasNoUnsignedWrap() && any_of(Range: S->operands(), P: [&](SCEVUse Op) {
11337 return isKnownMultipleOf(S: Op, M, Predicates);
11338 }))
11339 return true;
11340
11341 // Similarly, look through MinMax, with no wrapping arithmetic to consider.
11342 if (isa<SCEVMinMaxExpr>(Val: S) && all_of(Range: S->operands(), P: [&](SCEVUse Op) {
11343 return isKnownMultipleOf(S: Op, M, Predicates);
11344 }))
11345 return true;
11346
11347 const SCEVPredicate *P = getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero);
11348
11349 // Detect redundant predicates.
11350 for (auto *A : *Predicates)
11351 if (A->implies(N: P, SE&: *this))
11352 return true;
11353
11354 // Only record non-redundant predicates.
11355 Predicates->push_back(Elt: P);
11356 return true;
11357}
11358
11359bool ScalarEvolution::haveSameSign(const SCEV *S1, const SCEV *S2) {
11360 return ((isKnownNonNegative(S: S1) && isKnownNonNegative(S: S2)) ||
11361 (isKnownNegative(S: S1) && isKnownNegative(S: S2)));
11362}
11363
11364std::pair<const SCEV *, const SCEV *>
11365ScalarEvolution::SplitIntoInitAndPostInc(const Loop *L, const SCEV *S) {
11366 // Compute SCEV on entry of loop L.
11367 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, SE&: *this);
11368 if (Start == getCouldNotCompute())
11369 return { Start, Start };
11370 // Compute post increment SCEV for loop L.
11371 const SCEV *PostInc = SCEVPostIncRewriter::rewrite(S, L, SE&: *this);
11372 assert(PostInc != getCouldNotCompute() && "Unexpected could not compute");
11373 return { Start, PostInc };
11374}
11375
11376bool ScalarEvolution::isKnownViaInduction(CmpPredicate Pred, SCEVUse LHS,
11377 SCEVUse RHS) {
11378 // First collect all loops.
11379 SmallPtrSet<const Loop *, 8> LoopsUsed;
11380 getUsedLoops(S: LHS, LoopsUsed);
11381 getUsedLoops(S: RHS, LoopsUsed);
11382
11383 if (LoopsUsed.empty())
11384 return false;
11385
11386 // Domination relationship must be a linear order on collected loops.
11387#ifndef NDEBUG
11388 for (const auto *L1 : LoopsUsed)
11389 for (const auto *L2 : LoopsUsed)
11390 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||
11391 DT.dominates(L2->getHeader(), L1->getHeader())) &&
11392 "Domination relationship is not a linear order");
11393#endif
11394
11395 const Loop *MDL =
11396 *llvm::max_element(Range&: LoopsUsed, C: [&](const Loop *L1, const Loop *L2) {
11397 return DT.properlyDominates(A: L1->getHeader(), B: L2->getHeader());
11398 });
11399
11400 // Get init and post increment value for LHS.
11401 auto SplitLHS = SplitIntoInitAndPostInc(L: MDL, S: LHS);
11402 // if LHS contains unknown non-invariant SCEV then bail out.
11403 if (SplitLHS.first == getCouldNotCompute())
11404 return false;
11405 assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
11406 // Get init and post increment value for RHS.
11407 auto SplitRHS = SplitIntoInitAndPostInc(L: MDL, S: RHS);
11408 // if RHS contains unknown non-invariant SCEV then bail out.
11409 if (SplitRHS.first == getCouldNotCompute())
11410 return false;
11411 assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
11412 // It is possible that init SCEV contains an invariant load but it does
11413 // not dominate MDL and is not available at MDL loop entry, so we should
11414 // check it here.
11415 if (!isAvailableAtLoopEntry(S: SplitLHS.first, L: MDL) ||
11416 !isAvailableAtLoopEntry(S: SplitRHS.first, L: MDL))
11417 return false;
11418
11419 // It seems backedge guard check is faster than entry one so in some cases
11420 // it can speed up whole estimation by short circuit
11421 return isLoopBackedgeGuardedByCond(L: MDL, Pred, LHS: SplitLHS.second,
11422 RHS: SplitRHS.second) &&
11423 isLoopEntryGuardedByCond(L: MDL, Pred, LHS: SplitLHS.first, RHS: SplitRHS.first);
11424}
11425
11426bool ScalarEvolution::isKnownPredicate(CmpPredicate Pred, SCEVUse LHS,
11427 SCEVUse RHS) {
11428 // Canonicalize the inputs first.
11429 (void)SimplifyICmpOperands(Pred, LHS, RHS);
11430
11431 return isKnownViaInduction(Pred, LHS, RHS) ||
11432 isKnownPredicateViaSplitting(Pred, LHS, RHS) ||
11433 isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
11434}
11435
11436std::optional<bool> ScalarEvolution::evaluatePredicate(CmpPredicate Pred,
11437 const SCEV *LHS,
11438 const SCEV *RHS) {
11439 if (isKnownPredicate(Pred, LHS, RHS))
11440 return true;
11441 if (isKnownPredicate(Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11442 return false;
11443 return std::nullopt;
11444}
11445
11446bool ScalarEvolution::isKnownPredicateAt(CmpPredicate Pred, const SCEV *LHS,
11447 const SCEV *RHS,
11448 const Instruction *CtxI) {
11449 // TODO: Analyze guards and assumes from Context's block.
11450 return isKnownPredicate(Pred, LHS, RHS) ||
11451 isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS);
11452}
11453
11454std::optional<bool>
11455ScalarEvolution::evaluatePredicateAt(CmpPredicate Pred, const SCEV *LHS,
11456 const SCEV *RHS, const Instruction *CtxI) {
11457 std::optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
11458 if (KnownWithoutContext)
11459 return KnownWithoutContext;
11460
11461 if (isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS))
11462 return true;
11463 if (isBasicBlockEntryGuardedByCond(
11464 BB: CtxI->getParent(), Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11465 return false;
11466 return std::nullopt;
11467}
11468
11469bool ScalarEvolution::isKnownOnEveryIteration(CmpPredicate Pred,
11470 const SCEVAddRecExpr *LHS,
11471 const SCEV *RHS) {
11472 const Loop *L = LHS->getLoop();
11473 return isLoopEntryGuardedByCond(L, Pred, LHS: LHS->getStart(), RHS) &&
11474 isLoopBackedgeGuardedByCond(L, Pred, LHS: LHS->getPostIncExpr(SE&: *this), RHS);
11475}
11476
11477std::optional<ScalarEvolution::MonotonicPredicateType>
11478ScalarEvolution::getMonotonicPredicateType(const SCEVAddRecExpr *LHS,
11479 ICmpInst::Predicate Pred) {
11480 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
11481
11482#ifndef NDEBUG
11483 // Verify an invariant: inverting the predicate should turn a monotonically
11484 // increasing change to a monotonically decreasing one, and vice versa.
11485 if (Result) {
11486 auto ResultSwapped =
11487 getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred));
11488
11489 assert(*ResultSwapped != *Result &&
11490 "monotonicity should flip as we flip the predicate");
11491 }
11492#endif
11493
11494 return Result;
11495}
11496
11497std::optional<ScalarEvolution::MonotonicPredicateType>
11498ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
11499 ICmpInst::Predicate Pred) {
11500 // A zero step value for LHS means the induction variable is essentially a
11501 // loop invariant value. We don't really depend on the predicate actually
11502 // flipping from false to true (for increasing predicates, and the other way
11503 // around for decreasing predicates), all we care about is that *if* the
11504 // predicate changes then it only changes from false to true.
11505 //
11506 // A zero step value in itself is not very useful, but there may be places
11507 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
11508 // as general as possible.
11509
11510 // Only handle LE/LT/GE/GT predicates.
11511 if (!ICmpInst::isRelational(P: Pred))
11512 return std::nullopt;
11513
11514 bool IsGreater = ICmpInst::isGE(P: Pred) || ICmpInst::isGT(P: Pred);
11515 assert((IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) &&
11516 "Should be greater or less!");
11517
11518 // Check that AR does not wrap.
11519 if (ICmpInst::isUnsigned(Pred)) {
11520 if (!LHS->hasNoUnsignedWrap())
11521 return std::nullopt;
11522 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11523 }
11524 assert(ICmpInst::isSigned(Pred) &&
11525 "Relational predicate is either signed or unsigned!");
11526 if (!LHS->hasNoSignedWrap())
11527 return std::nullopt;
11528
11529 const SCEV *Step = LHS->getStepRecurrence(SE&: *this);
11530
11531 if (isKnownNonNegative(S: Step))
11532 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11533
11534 if (isKnownNonPositive(S: Step))
11535 return !IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11536
11537 return std::nullopt;
11538}
11539
11540std::optional<ScalarEvolution::LoopInvariantPredicate>
11541ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
11542 const SCEV *RHS, const Loop *L,
11543 const Instruction *CtxI) {
11544 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11545 if (!isLoopInvariant(S: RHS, L)) {
11546 if (!isLoopInvariant(S: LHS, L))
11547 return std::nullopt;
11548
11549 std::swap(a&: LHS, b&: RHS);
11550 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11551 }
11552
11553 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11554 if (!ArLHS || ArLHS->getLoop() != L)
11555 return std::nullopt;
11556
11557 auto MonotonicType = getMonotonicPredicateType(LHS: ArLHS, Pred);
11558 if (!MonotonicType)
11559 return std::nullopt;
11560 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
11561 // true as the loop iterates, and the backedge is control dependent on
11562 // "ArLHS `Pred` RHS" == true then we can reason as follows:
11563 //
11564 // * if the predicate was false in the first iteration then the predicate
11565 // is never evaluated again, since the loop exits without taking the
11566 // backedge.
11567 // * if the predicate was true in the first iteration then it will
11568 // continue to be true for all future iterations since it is
11569 // monotonically increasing.
11570 //
11571 // For both the above possibilities, we can replace the loop varying
11572 // predicate with its value on the first iteration of the loop (which is
11573 // loop invariant).
11574 //
11575 // A similar reasoning applies for a monotonically decreasing predicate, by
11576 // replacing true with false and false with true in the above two bullets.
11577 bool Increasing = *MonotonicType == ScalarEvolution::MonotonicallyIncreasing;
11578 auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
11579
11580 if (isLoopBackedgeGuardedByCond(L, Pred: P, LHS, RHS))
11581 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11582 RHS);
11583
11584 if (!CtxI)
11585 return std::nullopt;
11586 // Try to prove via context.
11587 // TODO: Support other cases.
11588 switch (Pred) {
11589 default:
11590 break;
11591 case ICmpInst::ICMP_ULE:
11592 case ICmpInst::ICMP_ULT: {
11593 assert(ArLHS->hasNoUnsignedWrap() && "Is a requirement of monotonicity!");
11594 // Given preconditions
11595 // (1) ArLHS does not cross the border of positive and negative parts of
11596 // range because of:
11597 // - Positive step; (TODO: lift this limitation)
11598 // - nuw - does not cross zero boundary;
11599 // - nsw - does not cross SINT_MAX boundary;
11600 // (2) ArLHS <s RHS
11601 // (3) RHS >=s 0
11602 // we can replace the loop variant ArLHS <u RHS condition with loop
11603 // invariant Start(ArLHS) <u RHS.
11604 //
11605 // Because of (1) there are two options:
11606 // - ArLHS is always negative. It means that ArLHS <u RHS is always false;
11607 // - ArLHS is always non-negative. Because of (3) RHS is also non-negative.
11608 // It means that ArLHS <s RHS <=> ArLHS <u RHS.
11609 // Because of (2) ArLHS <u RHS is trivially true.
11610 // All together it means that ArLHS <u RHS <=> Start(ArLHS) >=s 0.
11611 // We can strengthen this to Start(ArLHS) <u RHS.
11612 auto SignFlippedPred = ICmpInst::getFlippedSignednessPredicate(Pred);
11613 if (ArLHS->hasNoSignedWrap() && ArLHS->isAffine() &&
11614 isKnownPositive(S: ArLHS->getStepRecurrence(SE&: *this)) &&
11615 isKnownNonNegative(S: RHS) &&
11616 isKnownPredicateAt(Pred: SignFlippedPred, LHS: ArLHS, RHS, CtxI))
11617 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11618 RHS);
11619 }
11620 }
11621
11622 return std::nullopt;
11623}
11624
11625std::optional<ScalarEvolution::LoopInvariantPredicate>
11626ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations(
11627 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11628 const Instruction *CtxI, const SCEV *MaxIter) {
11629 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11630 Pred, LHS, RHS, L, CtxI, MaxIter))
11631 return LIP;
11632 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: MaxIter))
11633 // Number of iterations expressed as UMIN isn't always great for expressing
11634 // the value on the last iteration. If the straightforward approach didn't
11635 // work, try the following trick: if the a predicate is invariant for X, it
11636 // is also invariant for umin(X, ...). So try to find something that works
11637 // among subexpressions of MaxIter expressed as umin.
11638 for (SCEVUse Op : UMin->operands())
11639 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11640 Pred, LHS, RHS, L, CtxI, MaxIter: Op))
11641 return LIP;
11642 return std::nullopt;
11643}
11644
11645std::optional<ScalarEvolution::LoopInvariantPredicate>
11646ScalarEvolution::getLoopInvariantExitCondDuringFirstIterationsImpl(
11647 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11648 const Instruction *CtxI, const SCEV *MaxIter) {
11649 // Try to prove the following set of facts:
11650 // - The predicate is monotonic in the iteration space.
11651 // - If the check does not fail on the 1st iteration:
11652 // - No overflow will happen during first MaxIter iterations;
11653 // - It will not fail on the MaxIter'th iteration.
11654 // If the check does fail on the 1st iteration, we leave the loop and no
11655 // other checks matter.
11656
11657 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11658 if (!isLoopInvariant(S: RHS, L)) {
11659 if (!isLoopInvariant(S: LHS, L))
11660 return std::nullopt;
11661
11662 std::swap(a&: LHS, b&: RHS);
11663 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11664 }
11665
11666 auto *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11667 if (!AR || AR->getLoop() != L)
11668 return std::nullopt;
11669
11670 // Even if both are valid, we need to consistently chose the unsigned or the
11671 // signed predicate below, not mixtures of both. For now, prefer the unsigned
11672 // predicate.
11673 Pred = Pred.dropSameSign();
11674
11675 // The predicate must be relational (i.e. <, <=, >=, >).
11676 if (!ICmpInst::isRelational(P: Pred))
11677 return std::nullopt;
11678
11679 // TODO: Support steps other than +/- 1.
11680 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
11681 auto *One = getOne(Ty: Step->getType());
11682 auto *MinusOne = getNegativeSCEV(V: One);
11683 if (Step != One && Step != MinusOne)
11684 return std::nullopt;
11685
11686 // Type mismatch here means that MaxIter is potentially larger than max
11687 // unsigned value in start type, which mean we cannot prove no wrap for the
11688 // indvar.
11689 if (AR->getType() != MaxIter->getType())
11690 return std::nullopt;
11691
11692 // Value of IV on suggested last iteration.
11693 const SCEV *Last = AR->evaluateAtIteration(It: MaxIter, SE&: *this);
11694 // Does it still meet the requirement?
11695 if (!isLoopBackedgeGuardedByCond(L, Pred, LHS: Last, RHS))
11696 return std::nullopt;
11697 // Because step is +/- 1 and MaxIter has same type as Start (i.e. it does
11698 // not exceed max unsigned value of this type), this effectively proves
11699 // that there is no wrap during the iteration. To prove that there is no
11700 // signed/unsigned wrap, we need to check that
11701 // Start <= Last for step = 1 or Start >= Last for step = -1.
11702 ICmpInst::Predicate NoOverflowPred =
11703 CmpInst::isSigned(Pred) ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
11704 if (Step == MinusOne)
11705 NoOverflowPred = ICmpInst::getSwappedPredicate(pred: NoOverflowPred);
11706 const SCEV *Start = AR->getStart();
11707 if (!isKnownPredicateAt(Pred: NoOverflowPred, LHS: Start, RHS: Last, CtxI))
11708 return std::nullopt;
11709
11710 // Everything is fine.
11711 return ScalarEvolution::LoopInvariantPredicate(Pred, Start, RHS);
11712}
11713
11714bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
11715 SCEVUse LHS,
11716 SCEVUse RHS) {
11717 if (HasSameValue(A: LHS, B: RHS))
11718 return ICmpInst::isTrueWhenEqual(predicate: Pred);
11719
11720 auto CheckRange = [&](bool IsSigned) {
11721 auto RangeLHS = IsSigned ? getSignedRange(S: LHS) : getUnsignedRange(S: LHS);
11722 auto RangeRHS = IsSigned ? getSignedRange(S: RHS) : getUnsignedRange(S: RHS);
11723 return RangeLHS.icmp(Pred, Other: RangeRHS);
11724 };
11725
11726 // The check at the top of the function catches the case where the values are
11727 // known to be equal.
11728 if (Pred == CmpInst::ICMP_EQ)
11729 return false;
11730
11731 if (Pred == CmpInst::ICMP_NE) {
11732 if (CheckRange(true) || CheckRange(false))
11733 return true;
11734 auto *Diff = getMinusSCEV(LHS, RHS);
11735 return !isa<SCEVCouldNotCompute>(Val: Diff) && isKnownNonZero(S: Diff);
11736 }
11737
11738 return CheckRange(CmpInst::isSigned(Pred));
11739}
11740
11741bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
11742 SCEVUse LHS, SCEVUse RHS) {
11743 // Match X to (A + C1)<ExpectedFlags> and Y to (A + C2)<ExpectedFlags>, where
11744 // C1 and C2 are constant integers. If either X or Y are not add expressions,
11745 // consider them as X + 0 and Y + 0 respectively. C1 and C2 are returned via
11746 // OutC1 and OutC2.
11747 auto MatchBinaryAddToConst = [this](SCEVUse X, SCEVUse Y, APInt &OutC1,
11748 APInt &OutC2,
11749 SCEV::NoWrapFlags ExpectedFlags) {
11750 SCEVUse XNonConstOp, XConstOp;
11751 SCEVUse YNonConstOp, YConstOp;
11752 SCEV::NoWrapFlags XFlagsPresent;
11753 SCEV::NoWrapFlags YFlagsPresent;
11754
11755 if (!splitBinaryAdd(Expr: X, L&: XConstOp, R&: XNonConstOp, Flags&: XFlagsPresent)) {
11756 XConstOp = getZero(Ty: X->getType());
11757 XNonConstOp = X;
11758 XFlagsPresent = ExpectedFlags;
11759 }
11760 if (!isa<SCEVConstant>(Val: XConstOp))
11761 return false;
11762
11763 if (!splitBinaryAdd(Expr: Y, L&: YConstOp, R&: YNonConstOp, Flags&: YFlagsPresent)) {
11764 YConstOp = getZero(Ty: Y->getType());
11765 YNonConstOp = Y;
11766 YFlagsPresent = ExpectedFlags;
11767 }
11768
11769 if (YNonConstOp != XNonConstOp)
11770 return false;
11771
11772 if (!isa<SCEVConstant>(Val: YConstOp))
11773 return false;
11774
11775 // When matching ADDs with NUW flags (and unsigned predicates), only the
11776 // second ADD (with the larger constant) requires NUW.
11777 if ((YFlagsPresent & ExpectedFlags) != ExpectedFlags)
11778 return false;
11779 if (ExpectedFlags != SCEV::FlagNUW &&
11780 (XFlagsPresent & ExpectedFlags) != ExpectedFlags) {
11781 return false;
11782 }
11783
11784 OutC1 = cast<SCEVConstant>(Val&: XConstOp)->getAPInt();
11785 OutC2 = cast<SCEVConstant>(Val&: YConstOp)->getAPInt();
11786
11787 return true;
11788 };
11789
11790 APInt C1;
11791 APInt C2;
11792
11793 switch (Pred) {
11794 default:
11795 break;
11796
11797 case ICmpInst::ICMP_SGE:
11798 std::swap(a&: LHS, b&: RHS);
11799 [[fallthrough]];
11800 case ICmpInst::ICMP_SLE:
11801 // (X + C1)<nsw> s<= (X + C2)<nsw> if C1 s<= C2.
11802 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.sle(RHS: C2))
11803 return true;
11804
11805 break;
11806
11807 case ICmpInst::ICMP_SGT:
11808 std::swap(a&: LHS, b&: RHS);
11809 [[fallthrough]];
11810 case ICmpInst::ICMP_SLT:
11811 // (X + C1)<nsw> s< (X + C2)<nsw> if C1 s< C2.
11812 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.slt(RHS: C2))
11813 return true;
11814
11815 break;
11816
11817 case ICmpInst::ICMP_UGE:
11818 std::swap(a&: LHS, b&: RHS);
11819 [[fallthrough]];
11820 case ICmpInst::ICMP_ULE:
11821 // (X + C1) u<= (X + C2)<nuw> for C1 u<= C2.
11822 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(RHS: C2))
11823 return true;
11824
11825 break;
11826
11827 case ICmpInst::ICMP_UGT:
11828 std::swap(a&: LHS, b&: RHS);
11829 [[fallthrough]];
11830 case ICmpInst::ICMP_ULT:
11831 // (X + C1) u< (X + C2)<nuw> if C1 u< C2.
11832 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(RHS: C2))
11833 return true;
11834 break;
11835 }
11836
11837 return false;
11838}
11839
11840bool ScalarEvolution::isKnownPredicateViaSplitting(CmpPredicate Pred,
11841 SCEVUse LHS, SCEVUse RHS) {
11842 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate)
11843 return false;
11844
11845 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on
11846 // the stack can result in exponential time complexity.
11847 SaveAndRestore Restore(ProvingSplitPredicate, true);
11848
11849 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L
11850 //
11851 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use
11852 // isKnownPredicate. isKnownPredicate is more powerful, but also more
11853 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the
11854 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to
11855 // use isKnownPredicate later if needed.
11856 return isKnownNonNegative(S: RHS) &&
11857 isKnownPredicate(Pred: CmpInst::ICMP_SGE, LHS, RHS: getZero(Ty: LHS->getType())) &&
11858 isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS, RHS);
11859}
11860
11861bool ScalarEvolution::isImpliedViaGuard(const BasicBlock *BB, CmpPredicate Pred,
11862 const SCEV *LHS, const SCEV *RHS) {
11863 // No need to even try if we know the module has no guards.
11864 if (!HasGuards)
11865 return false;
11866
11867 return any_of(Range: *BB, P: [&](const Instruction &I) {
11868 using namespace llvm::PatternMatch;
11869
11870 Value *Condition;
11871 return match(V: &I, P: m_Intrinsic<Intrinsic::experimental_guard>(
11872 Ops: m_Value(V&: Condition))) &&
11873 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse: false);
11874 });
11875}
11876
11877/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
11878/// protected by a conditional between LHS and RHS. This is used to
11879/// to eliminate casts.
11880bool ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
11881 CmpPredicate Pred,
11882 const SCEV *LHS,
11883 const SCEV *RHS) {
11884 // Interpret a null as meaning no loop, where there is obviously no guard
11885 // (interprocedural conditions notwithstanding). Do not bother about
11886 // unreachable loops.
11887 if (!L || !DT.isReachableFromEntry(A: L->getHeader()))
11888 return true;
11889
11890 if (VerifyIR)
11891 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) &&
11892 "This cannot be done on broken IR!");
11893
11894
11895 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11896 return true;
11897
11898 BasicBlock *Latch = L->getLoopLatch();
11899 if (!Latch)
11900 return false;
11901
11902 CondBrInst *LoopContinuePredicate =
11903 dyn_cast<CondBrInst>(Val: Latch->getTerminator());
11904 if (LoopContinuePredicate &&
11905 isImpliedCond(Pred, LHS, RHS, FoundCondValue: LoopContinuePredicate->getCondition(),
11906 Inverse: LoopContinuePredicate->getSuccessor(i: 0) != L->getHeader()))
11907 return true;
11908
11909 // We don't want more than one activation of the following loops on the stack
11910 // -- that can lead to O(n!) time complexity.
11911 if (WalkingBEDominatingConds)
11912 return false;
11913
11914 SaveAndRestore ClearOnExit(WalkingBEDominatingConds, true);
11915
11916 // See if we can exploit a trip count to prove the predicate.
11917 const auto &BETakenInfo = getBackedgeTakenInfo(L);
11918 const SCEV *LatchBECount = BETakenInfo.getExact(ExitingBlock: Latch, SE: this);
11919 if (LatchBECount != getCouldNotCompute()) {
11920 // We know that Latch branches back to the loop header exactly
11921 // LatchBECount times. This means the backdege condition at Latch is
11922 // equivalent to "{0,+,1} u< LatchBECount".
11923 Type *Ty = LatchBECount->getType();
11924 auto NoWrapFlags = SCEV::NoWrapFlags(SCEV::FlagNUW | SCEV::FlagNW);
11925 const SCEV *LoopCounter =
11926 getAddRecExpr(Start: getZero(Ty), Step: getOne(Ty), L, Flags: NoWrapFlags);
11927 if (isImpliedCond(Pred, LHS, RHS, FoundPred: ICmpInst::ICMP_ULT, FoundLHS: LoopCounter,
11928 FoundRHS: LatchBECount))
11929 return true;
11930 }
11931
11932 // Check conditions due to any @llvm.assume intrinsics.
11933 for (auto &AssumeVH : AC.assumptions()) {
11934 if (!AssumeVH)
11935 continue;
11936 auto *CI = cast<CallInst>(Val&: AssumeVH);
11937 if (!DT.dominates(Def: CI, User: Latch->getTerminator()))
11938 continue;
11939
11940 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: CI->getArgOperand(i: 0), Inverse: false))
11941 return true;
11942 }
11943
11944 if (isImpliedViaGuard(BB: Latch, Pred, LHS, RHS))
11945 return true;
11946
11947 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
11948 DTN != HeaderDTN; DTN = DTN->getIDom()) {
11949 assert(DTN && "should reach the loop header before reaching the root!");
11950
11951 BasicBlock *BB = DTN->getBlock();
11952 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
11953 return true;
11954
11955 BasicBlock *PBB = BB->getSinglePredecessor();
11956 if (!PBB)
11957 continue;
11958
11959 CondBrInst *ContBr = dyn_cast<CondBrInst>(Val: PBB->getTerminator());
11960 if (!ContBr || ContBr->getSuccessor(i: 0) == ContBr->getSuccessor(i: 1))
11961 continue;
11962
11963 // If we have an edge `E` within the loop body that dominates the only
11964 // latch, the condition guarding `E` also guards the backedge. This
11965 // reasoning works only for loops with a single latch.
11966 // We're constructively (and conservatively) enumerating edges within the
11967 // loop body that dominate the latch. The dominator tree better agree
11968 // with us on this:
11969 assert(DT.dominates(BasicBlockEdge(PBB, BB), Latch) && "should be!");
11970 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: ContBr->getCondition(),
11971 Inverse: BB != ContBr->getSuccessor(i: 0)))
11972 return true;
11973 }
11974
11975 return false;
11976}
11977
11978bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
11979 CmpPredicate Pred,
11980 const SCEV *LHS,
11981 const SCEV *RHS) {
11982 // Do not bother proving facts for unreachable code.
11983 if (!DT.isReachableFromEntry(A: BB))
11984 return true;
11985 if (VerifyIR)
11986 assert(!verifyFunction(*BB->getParent(), &dbgs()) &&
11987 "This cannot be done on broken IR!");
11988
11989 // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
11990 // the facts (a >= b && a != b) separately. A typical situation is when the
11991 // non-strict comparison is known from ranges and non-equality is known from
11992 // dominating predicates. If we are proving strict comparison, we always try
11993 // to prove non-equality and non-strict comparison separately.
11994 CmpPredicate NonStrictPredicate = ICmpInst::getNonStrictCmpPredicate(Pred);
11995 const bool ProvingStrictComparison =
11996 Pred != NonStrictPredicate.dropSameSign();
11997 bool ProvedNonStrictComparison = false;
11998 bool ProvedNonEquality = false;
11999
12000 auto SplitAndProve = [&](std::function<bool(CmpPredicate)> Fn) -> bool {
12001 if (!ProvedNonStrictComparison)
12002 ProvedNonStrictComparison = Fn(NonStrictPredicate);
12003 if (!ProvedNonEquality)
12004 ProvedNonEquality = Fn(ICmpInst::ICMP_NE);
12005 if (ProvedNonStrictComparison && ProvedNonEquality)
12006 return true;
12007 return false;
12008 };
12009
12010 if (ProvingStrictComparison) {
12011 auto ProofFn = [&](CmpPredicate P) {
12012 return isKnownViaNonRecursiveReasoning(Pred: P, LHS, RHS);
12013 };
12014 if (SplitAndProve(ProofFn))
12015 return true;
12016 }
12017
12018 // Try to prove (Pred, LHS, RHS) using isImpliedCond.
12019 auto ProveViaCond = [&](const Value *Condition, bool Inverse) {
12020 const Instruction *CtxI = &BB->front();
12021 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI))
12022 return true;
12023 if (ProvingStrictComparison) {
12024 auto ProofFn = [&](CmpPredicate P) {
12025 return isImpliedCond(Pred: P, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI);
12026 };
12027 if (SplitAndProve(ProofFn))
12028 return true;
12029 }
12030 return false;
12031 };
12032
12033 // Starting at the block's predecessor, climb up the predecessor chain, as long
12034 // as there are predecessors that can be found that have unique successors
12035 // leading to the original block.
12036 const Loop *ContainingLoop = LI.getLoopFor(BB);
12037 const BasicBlock *PredBB;
12038 if (ContainingLoop && ContainingLoop->getHeader() == BB)
12039 PredBB = ContainingLoop->getLoopPredecessor();
12040 else
12041 PredBB = BB->getSinglePredecessor();
12042 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
12043 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
12044 const CondBrInst *BlockEntryPredicate =
12045 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
12046 if (!BlockEntryPredicate)
12047 continue;
12048
12049 if (ProveViaCond(BlockEntryPredicate->getCondition(),
12050 BlockEntryPredicate->getSuccessor(i: 0) != Pair.second))
12051 return true;
12052 }
12053
12054 // Check conditions due to any @llvm.assume intrinsics.
12055 for (auto &AssumeVH : AC.assumptions()) {
12056 if (!AssumeVH)
12057 continue;
12058 auto *CI = cast<CallInst>(Val&: AssumeVH);
12059 if (!DT.dominates(Def: CI, BB))
12060 continue;
12061
12062 if (ProveViaCond(CI->getArgOperand(i: 0), false))
12063 return true;
12064 }
12065
12066 // Check conditions due to any @llvm.experimental.guard intrinsics.
12067 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
12068 M: F.getParent(), id: Intrinsic::experimental_guard);
12069 if (GuardDecl)
12070 for (const auto *GU : GuardDecl->users())
12071 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
12072 if (Guard->getFunction() == BB->getParent() && DT.dominates(Def: Guard, BB))
12073 if (ProveViaCond(Guard->getArgOperand(i: 0), false))
12074 return true;
12075 return false;
12076}
12077
12078bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred,
12079 const SCEV *LHS,
12080 const SCEV *RHS) {
12081 // Interpret a null as meaning no loop, where there is obviously no guard
12082 // (interprocedural conditions notwithstanding).
12083 if (!L)
12084 return false;
12085
12086 // Both LHS and RHS must be available at loop entry.
12087 assert(isAvailableAtLoopEntry(LHS, L) &&
12088 "LHS is not available at Loop Entry");
12089 assert(isAvailableAtLoopEntry(RHS, L) &&
12090 "RHS is not available at Loop Entry");
12091
12092 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
12093 return true;
12094
12095 return isBasicBlockEntryGuardedByCond(BB: L->getHeader(), Pred, LHS, RHS);
12096}
12097
12098bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12099 const SCEV *RHS,
12100 const Value *FoundCondValue, bool Inverse,
12101 const Instruction *CtxI) {
12102 // False conditions implies anything. Do not bother analyzing it further.
12103 if (FoundCondValue ==
12104 ConstantInt::getBool(Context&: FoundCondValue->getContext(), V: Inverse))
12105 return true;
12106
12107 if (!PendingLoopPredicates.insert(Ptr: FoundCondValue).second)
12108 return false;
12109
12110 llvm::scope_exit ClearOnExit(
12111 [&]() { PendingLoopPredicates.erase(Ptr: FoundCondValue); });
12112
12113 // Recursively handle And and Or conditions.
12114 const Value *Op0, *Op1;
12115 if (match(V: FoundCondValue, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12116 if (!Inverse)
12117 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12118 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12119 } else if (match(V: FoundCondValue, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12120 if (Inverse)
12121 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12122 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12123 }
12124
12125 const ICmpInst *ICI = dyn_cast<ICmpInst>(Val: FoundCondValue);
12126 if (!ICI) return false;
12127
12128 // Now that we found a conditional branch that dominates the loop or controls
12129 // the loop latch. Check to see if it is the comparison we are looking for.
12130 CmpPredicate FoundPred;
12131 if (Inverse)
12132 FoundPred = ICI->getInverseCmpPredicate();
12133 else
12134 FoundPred = ICI->getCmpPredicate();
12135
12136 const SCEV *FoundLHS = getSCEV(V: ICI->getOperand(i_nocapture: 0));
12137 const SCEV *FoundRHS = getSCEV(V: ICI->getOperand(i_nocapture: 1));
12138
12139 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, Context: CtxI);
12140}
12141
12142bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12143 const SCEV *RHS, CmpPredicate FoundPred,
12144 const SCEV *FoundLHS, const SCEV *FoundRHS,
12145 const Instruction *CtxI) {
12146 // Balance the types.
12147 if (getTypeSizeInBits(Ty: LHS->getType()) <
12148 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12149 // For unsigned and equality predicates, try to prove that both found
12150 // operands fit into narrow unsigned range. If so, try to prove facts in
12151 // narrow types.
12152 if (!CmpInst::isSigned(Pred: FoundPred) && !FoundLHS->getType()->isPointerTy() &&
12153 !FoundRHS->getType()->isPointerTy()) {
12154 auto *NarrowType = LHS->getType();
12155 auto *WideType = FoundLHS->getType();
12156 auto BitWidth = getTypeSizeInBits(Ty: NarrowType);
12157 const SCEV *MaxValue = getZeroExtendExpr(
12158 Op: getConstant(Val: APInt::getMaxValue(numBits: BitWidth)), Ty: WideType);
12159 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundLHS,
12160 RHS: MaxValue) &&
12161 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundRHS,
12162 RHS: MaxValue)) {
12163 const SCEV *TruncFoundLHS = getTruncateExpr(Op: FoundLHS, Ty: NarrowType);
12164 const SCEV *TruncFoundRHS = getTruncateExpr(Op: FoundRHS, Ty: NarrowType);
12165 // We cannot preserve samesign after truncation.
12166 if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred: FoundPred.dropSameSign(),
12167 FoundLHS: TruncFoundLHS, FoundRHS: TruncFoundRHS, CtxI))
12168 return true;
12169 }
12170 }
12171
12172 if (LHS->getType()->isPointerTy() || RHS->getType()->isPointerTy())
12173 return false;
12174 if (CmpInst::isSigned(Pred)) {
12175 LHS = getSignExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12176 RHS = getSignExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12177 } else {
12178 LHS = getZeroExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12179 RHS = getZeroExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12180 }
12181 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
12182 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12183 if (FoundLHS->getType()->isPointerTy() || FoundRHS->getType()->isPointerTy())
12184 return false;
12185 if (CmpInst::isSigned(Pred: FoundPred)) {
12186 FoundLHS = getSignExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12187 FoundRHS = getSignExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12188 } else {
12189 FoundLHS = getZeroExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12190 FoundRHS = getZeroExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12191 }
12192 }
12193 return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS,
12194 FoundRHS, CtxI);
12195}
12196
12197bool ScalarEvolution::isImpliedCondBalancedTypes(
12198 CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS, CmpPredicate FoundPred,
12199 SCEVUse FoundLHS, SCEVUse FoundRHS, const Instruction *CtxI) {
12200 assert(getTypeSizeInBits(LHS->getType()) ==
12201 getTypeSizeInBits(FoundLHS->getType()) &&
12202 "Types should be balanced!");
12203 // Canonicalize the query to match the way instcombine will have
12204 // canonicalized the comparison.
12205 if (SimplifyICmpOperands(Pred, LHS, RHS))
12206 if (LHS == RHS)
12207 return CmpInst::isTrueWhenEqual(predicate: Pred);
12208 if (SimplifyICmpOperands(Pred&: FoundPred, LHS&: FoundLHS, RHS&: FoundRHS))
12209 if (FoundLHS == FoundRHS)
12210 return CmpInst::isFalseWhenEqual(predicate: FoundPred);
12211
12212 // Check to see if we can make the LHS or RHS match.
12213 if (LHS == FoundRHS || RHS == FoundLHS) {
12214 if (isa<SCEVConstant>(Val: RHS)) {
12215 std::swap(a&: FoundLHS, b&: FoundRHS);
12216 FoundPred = ICmpInst::getSwappedCmpPredicate(Pred: FoundPred);
12217 } else {
12218 std::swap(a&: LHS, b&: RHS);
12219 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12220 }
12221 }
12222
12223 // Check whether the found predicate is the same as the desired predicate.
12224 if (auto P = CmpPredicate::getMatching(A: FoundPred, B: Pred))
12225 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12226
12227 // Check whether swapping the found predicate makes it the same as the
12228 // desired predicate.
12229 if (auto P = CmpPredicate::getMatching(
12230 A: ICmpInst::getSwappedCmpPredicate(Pred: FoundPred), B: Pred)) {
12231 // We can write the implication
12232 // 0. LHS Pred RHS <- FoundLHS SwapPred FoundRHS
12233 // using one of the following ways:
12234 // 1. LHS Pred RHS <- FoundRHS Pred FoundLHS
12235 // 2. RHS SwapPred LHS <- FoundLHS SwapPred FoundRHS
12236 // 3. LHS Pred RHS <- ~FoundLHS Pred ~FoundRHS
12237 // 4. ~LHS SwapPred ~RHS <- FoundLHS SwapPred FoundRHS
12238 // Forms 1. and 2. require swapping the operands of one condition. Don't
12239 // do this if it would break canonical constant/addrec ordering.
12240 if (!isa<SCEVConstant>(Val: RHS) && !isa<SCEVAddRecExpr>(Val: LHS))
12241 return isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P), LHS: RHS,
12242 RHS: LHS, FoundLHS, FoundRHS, Context: CtxI);
12243 if (!isa<SCEVConstant>(Val: FoundRHS) && !isa<SCEVAddRecExpr>(Val: FoundLHS))
12244 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: FoundRHS, FoundRHS: FoundLHS, Context: CtxI);
12245
12246 // There's no clear preference between forms 3. and 4., try both. Avoid
12247 // forming getNotSCEV of pointer values as the resulting subtract is
12248 // not legal.
12249 if (!LHS->getType()->isPointerTy() && !RHS->getType()->isPointerTy() &&
12250 isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P),
12251 LHS: getNotSCEV(V: LHS), RHS: getNotSCEV(V: RHS), FoundLHS,
12252 FoundRHS, Context: CtxI))
12253 return true;
12254
12255 if (!FoundLHS->getType()->isPointerTy() &&
12256 !FoundRHS->getType()->isPointerTy() &&
12257 isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: getNotSCEV(V: FoundLHS),
12258 FoundRHS: getNotSCEV(V: FoundRHS), Context: CtxI))
12259 return true;
12260
12261 return false;
12262 }
12263
12264 auto IsSignFlippedPredicate = [](CmpInst::Predicate P1,
12265 CmpInst::Predicate P2) {
12266 assert(P1 != P2 && "Handled earlier!");
12267 return CmpInst::isRelational(P: P2) &&
12268 P1 == ICmpInst::getFlippedSignednessPredicate(Pred: P2);
12269 };
12270 if (IsSignFlippedPredicate(Pred, FoundPred)) {
12271 // Unsigned comparison is the same as signed comparison when both the
12272 // operands are non-negative or negative.
12273 if (haveSameSign(S1: FoundLHS, S2: FoundRHS))
12274 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12275 // Create local copies that we can freely swap and canonicalize our
12276 // conditions to "le/lt".
12277 CmpPredicate CanonicalPred = Pred, CanonicalFoundPred = FoundPred;
12278 const SCEV *CanonicalLHS = LHS, *CanonicalRHS = RHS,
12279 *CanonicalFoundLHS = FoundLHS, *CanonicalFoundRHS = FoundRHS;
12280 if (ICmpInst::isGT(P: CanonicalPred) || ICmpInst::isGE(P: CanonicalPred)) {
12281 CanonicalPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalPred);
12282 CanonicalFoundPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalFoundPred);
12283 std::swap(a&: CanonicalLHS, b&: CanonicalRHS);
12284 std::swap(a&: CanonicalFoundLHS, b&: CanonicalFoundRHS);
12285 }
12286 assert((ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) &&
12287 "Must be!");
12288 assert((ICmpInst::isLT(CanonicalFoundPred) ||
12289 ICmpInst::isLE(CanonicalFoundPred)) &&
12290 "Must be!");
12291 if (ICmpInst::isSigned(Pred: CanonicalPred) && isKnownNonNegative(S: CanonicalRHS))
12292 // Use implication:
12293 // x <u y && y >=s 0 --> x <s y.
12294 // If we can prove the left part, the right part is also proven.
12295 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
12296 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
12297 FoundRHS: CanonicalFoundRHS);
12298 if (ICmpInst::isUnsigned(Pred: CanonicalPred) && isKnownNegative(S: CanonicalRHS))
12299 // Use implication:
12300 // x <s y && y <s 0 --> x <u y.
12301 // If we can prove the left part, the right part is also proven.
12302 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
12303 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
12304 FoundRHS: CanonicalFoundRHS);
12305 }
12306
12307 // Check if we can make progress by sharpening ranges.
12308 if (FoundPred == ICmpInst::ICMP_NE &&
12309 (isa<SCEVConstant>(Val: FoundLHS) || isa<SCEVConstant>(Val: FoundRHS))) {
12310
12311 const SCEVConstant *C = nullptr;
12312 const SCEV *V = nullptr;
12313
12314 if (isa<SCEVConstant>(Val: FoundLHS)) {
12315 C = cast<SCEVConstant>(Val&: FoundLHS);
12316 V = FoundRHS;
12317 } else {
12318 C = cast<SCEVConstant>(Val&: FoundRHS);
12319 V = FoundLHS;
12320 }
12321
12322 // The guarding predicate tells us that C != V. If the known range
12323 // of V is [C, t), we can sharpen the range to [C + 1, t). The
12324 // range we consider has to correspond to same signedness as the
12325 // predicate we're interested in folding.
12326
12327 APInt Min = ICmpInst::isSigned(Pred) ?
12328 getSignedRangeMin(S: V) : getUnsignedRangeMin(S: V);
12329
12330 if (Min == C->getAPInt()) {
12331 // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
12332 // This is true even if (Min + 1) wraps around -- in case of
12333 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
12334
12335 APInt SharperMin = Min + 1;
12336
12337 switch (Pred) {
12338 case ICmpInst::ICMP_SGE:
12339 case ICmpInst::ICMP_UGE:
12340 // We know V `Pred` SharperMin. If this implies LHS `Pred`
12341 // RHS, we're done.
12342 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin),
12343 Context: CtxI))
12344 return true;
12345 [[fallthrough]];
12346
12347 case ICmpInst::ICMP_SGT:
12348 case ICmpInst::ICMP_UGT:
12349 // We know from the range information that (V `Pred` Min ||
12350 // V == Min). We know from the guarding condition that !(V
12351 // == Min). This gives us
12352 //
12353 // V `Pred` Min || V == Min && !(V == Min)
12354 // => V `Pred` Min
12355 //
12356 // If V `Pred` Min implies LHS `Pred` RHS, we're done.
12357
12358 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12359 return true;
12360 break;
12361
12362 // `LHS < RHS` and `LHS <= RHS` are handled in the same way as `RHS > LHS` and `RHS >= LHS` respectively.
12363 case ICmpInst::ICMP_SLE:
12364 case ICmpInst::ICMP_ULE:
12365 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12366 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin), Context: CtxI))
12367 return true;
12368 [[fallthrough]];
12369
12370 case ICmpInst::ICMP_SLT:
12371 case ICmpInst::ICMP_ULT:
12372 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12373 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12374 return true;
12375 break;
12376
12377 default:
12378 // No change
12379 break;
12380 }
12381 }
12382 }
12383
12384 // Check whether the actual condition is beyond sufficient.
12385 if (FoundPred == ICmpInst::ICMP_EQ)
12386 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
12387 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12388 return true;
12389 if (Pred == ICmpInst::ICMP_NE)
12390 if (!ICmpInst::isTrueWhenEqual(predicate: FoundPred))
12391 if (isImpliedCondOperands(Pred: FoundPred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12392 return true;
12393
12394 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS))
12395 return true;
12396
12397 // Otherwise assume the worst.
12398 return false;
12399}
12400
12401bool ScalarEvolution::splitBinaryAdd(SCEVUse Expr, SCEVUse &L, SCEVUse &R,
12402 SCEV::NoWrapFlags &Flags) {
12403 if (!match(U: Expr, P: m_scev_Add(Op0: m_SCEV(V&: L), Op1: m_SCEV(V&: R))))
12404 return false;
12405
12406 Flags = cast<SCEVAddExpr>(Val&: Expr)->getNoWrapFlags();
12407 return true;
12408}
12409
12410std::optional<APInt>
12411ScalarEvolution::computeConstantDifference(const SCEV *More, const SCEV *Less) {
12412 // We avoid subtracting expressions here because this function is usually
12413 // fairly deep in the call stack (i.e. is called many times).
12414
12415 unsigned BW = getTypeSizeInBits(Ty: More->getType());
12416 APInt Diff(BW, 0);
12417 APInt DiffMul(BW, 1);
12418 // Try various simplifications to reduce the difference to a constant. Limit
12419 // the number of allowed simplifications to keep compile-time low.
12420 for (unsigned I = 0; I < 8; ++I) {
12421 if (More == Less)
12422 return Diff;
12423
12424 // Reduce addrecs with identical steps to their start value.
12425 if (isa<SCEVAddRecExpr>(Val: Less) && isa<SCEVAddRecExpr>(Val: More)) {
12426 const auto *LAR = cast<SCEVAddRecExpr>(Val: Less);
12427 const auto *MAR = cast<SCEVAddRecExpr>(Val: More);
12428
12429 if (LAR->getLoop() != MAR->getLoop())
12430 return std::nullopt;
12431
12432 // We look at affine expressions only; not for correctness but to keep
12433 // getStepRecurrence cheap.
12434 if (!LAR->isAffine() || !MAR->isAffine())
12435 return std::nullopt;
12436
12437 if (LAR->getStepRecurrence(SE&: *this) != MAR->getStepRecurrence(SE&: *this))
12438 return std::nullopt;
12439
12440 Less = LAR->getStart();
12441 More = MAR->getStart();
12442 continue;
12443 }
12444
12445 // Try to match a common constant multiply.
12446 auto MatchConstMul =
12447 [](const SCEV *S) -> std::optional<std::pair<const SCEV *, APInt>> {
12448 const APInt *C;
12449 const SCEV *Op;
12450 if (match(S, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEV(V&: Op))))
12451 return {{Op, *C}};
12452 return std::nullopt;
12453 };
12454 if (auto MatchedMore = MatchConstMul(More)) {
12455 if (auto MatchedLess = MatchConstMul(Less)) {
12456 if (MatchedMore->second == MatchedLess->second) {
12457 More = MatchedMore->first;
12458 Less = MatchedLess->first;
12459 DiffMul *= MatchedMore->second;
12460 continue;
12461 }
12462 }
12463 }
12464
12465 // Try to cancel out common factors in two add expressions.
12466 SmallDenseMap<const SCEV *, int, 8> Multiplicity;
12467 auto Add = [&](const SCEV *S, int Mul) {
12468 if (auto *C = dyn_cast<SCEVConstant>(Val: S)) {
12469 if (Mul == 1) {
12470 Diff += C->getAPInt() * DiffMul;
12471 } else {
12472 assert(Mul == -1);
12473 Diff -= C->getAPInt() * DiffMul;
12474 }
12475 } else
12476 Multiplicity[S] += Mul;
12477 };
12478 auto Decompose = [&](const SCEV *S, int Mul) {
12479 if (isa<SCEVAddExpr>(Val: S)) {
12480 for (const SCEV *Op : S->operands())
12481 Add(Op, Mul);
12482 } else
12483 Add(S, Mul);
12484 };
12485 Decompose(More, 1);
12486 Decompose(Less, -1);
12487
12488 // Check whether all the non-constants cancel out, or reduce to new
12489 // More/Less values.
12490 const SCEV *NewMore = nullptr, *NewLess = nullptr;
12491 for (const auto &[S, Mul] : Multiplicity) {
12492 if (Mul == 0)
12493 continue;
12494 if (Mul == 1) {
12495 if (NewMore)
12496 return std::nullopt;
12497 NewMore = S;
12498 } else if (Mul == -1) {
12499 if (NewLess)
12500 return std::nullopt;
12501 NewLess = S;
12502 } else
12503 return std::nullopt;
12504 }
12505
12506 // Values stayed the same, no point in trying further.
12507 if (NewMore == More || NewLess == Less)
12508 return std::nullopt;
12509
12510 More = NewMore;
12511 Less = NewLess;
12512
12513 // Reduced to constant.
12514 if (!More && !Less)
12515 return Diff;
12516
12517 // Left with variable on only one side, bail out.
12518 if (!More || !Less)
12519 return std::nullopt;
12520 }
12521
12522 // Did not reduce to constant.
12523 return std::nullopt;
12524}
12525
12526bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
12527 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12528 const SCEV *FoundRHS, const Instruction *CtxI) {
12529 // Try to recognize the following pattern:
12530 //
12531 // FoundRHS = ...
12532 // ...
12533 // loop:
12534 // FoundLHS = {Start,+,W}
12535 // context_bb: // Basic block from the same loop
12536 // known(Pred, FoundLHS, FoundRHS)
12537 //
12538 // If some predicate is known in the context of a loop, it is also known on
12539 // each iteration of this loop, including the first iteration. Therefore, in
12540 // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to
12541 // prove the original pred using this fact.
12542 if (!CtxI)
12543 return false;
12544 const BasicBlock *ContextBB = CtxI->getParent();
12545 // Make sure AR varies in the context block.
12546 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS)) {
12547 const Loop *L = AR->getLoop();
12548 const auto *Latch = L->getLoopLatch();
12549 // Make sure that context belongs to the loop and executes on 1st iteration
12550 // (if it ever executes at all).
12551 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12552 return false;
12553 if (!isAvailableAtLoopEntry(S: FoundRHS, L: AR->getLoop()))
12554 return false;
12555 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: AR->getStart(), FoundRHS);
12556 }
12557
12558 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundRHS)) {
12559 const Loop *L = AR->getLoop();
12560 const auto *Latch = L->getLoopLatch();
12561 // Make sure that context belongs to the loop and executes on 1st iteration
12562 // (if it ever executes at all).
12563 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12564 return false;
12565 if (!isAvailableAtLoopEntry(S: FoundLHS, L: AR->getLoop()))
12566 return false;
12567 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS: AR->getStart());
12568 }
12569
12570 return false;
12571}
12572
12573bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(CmpPredicate Pred,
12574 const SCEV *LHS,
12575 const SCEV *RHS,
12576 const SCEV *FoundLHS,
12577 const SCEV *FoundRHS) {
12578 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT)
12579 return false;
12580
12581 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12582 if (!AddRecLHS)
12583 return false;
12584
12585 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12586 if (!AddRecFoundLHS)
12587 return false;
12588
12589 // We'd like to let SCEV reason about control dependencies, so we constrain
12590 // both the inequalities to be about add recurrences on the same loop. This
12591 // way we can use isLoopEntryGuardedByCond later.
12592
12593 const Loop *L = AddRecFoundLHS->getLoop();
12594 if (L != AddRecLHS->getLoop())
12595 return false;
12596
12597 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1)
12598 //
12599 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C)
12600 // ... (2)
12601 //
12602 // Informal proof for (2), assuming (1) [*]:
12603 //
12604 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**]
12605 //
12606 // Then
12607 //
12608 // FoundLHS s< FoundRHS s< INT_MIN - C
12609 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ]
12610 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ]
12611 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s<
12612 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ]
12613 // <=> FoundLHS + C s< FoundRHS + C
12614 //
12615 // [*]: (1) can be proved by ruling out overflow.
12616 //
12617 // [**]: This can be proved by analyzing all the four possibilities:
12618 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and
12619 // (A s>= 0, B s>= 0).
12620 //
12621 // Note:
12622 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C"
12623 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS
12624 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS
12625 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is
12626 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS +
12627 // C)".
12628
12629 std::optional<APInt> LDiff = computeConstantDifference(More: LHS, Less: FoundLHS);
12630 if (!LDiff)
12631 return false;
12632 std::optional<APInt> RDiff = computeConstantDifference(More: RHS, Less: FoundRHS);
12633 if (!RDiff || *LDiff != *RDiff)
12634 return false;
12635
12636 if (LDiff->isMinValue())
12637 return true;
12638
12639 APInt FoundRHSLimit;
12640
12641 if (Pred == CmpInst::ICMP_ULT) {
12642 FoundRHSLimit = -(*RDiff);
12643 } else {
12644 assert(Pred == CmpInst::ICMP_SLT && "Checked above!");
12645 FoundRHSLimit = APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: RHS->getType())) - *RDiff;
12646 }
12647
12648 // Try to prove (1) or (2), as needed.
12649 return isAvailableAtLoopEntry(S: FoundRHS, L) &&
12650 isLoopEntryGuardedByCond(L, Pred, LHS: FoundRHS,
12651 RHS: getConstant(Val: FoundRHSLimit));
12652}
12653
12654bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred, const SCEV *LHS,
12655 const SCEV *RHS, const SCEV *FoundLHS,
12656 const SCEV *FoundRHS, unsigned Depth) {
12657 const PHINode *LPhi = nullptr, *RPhi = nullptr;
12658
12659 llvm::scope_exit ClearOnExit([&]() {
12660 if (LPhi) {
12661 bool Erased = PendingMerges.erase(Ptr: LPhi);
12662 assert(Erased && "Failed to erase LPhi!");
12663 (void)Erased;
12664 }
12665 if (RPhi) {
12666 bool Erased = PendingMerges.erase(Ptr: RPhi);
12667 assert(Erased && "Failed to erase RPhi!");
12668 (void)Erased;
12669 }
12670 });
12671
12672 // Find respective Phis and check that they are not being pending.
12673 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(Val: LHS))
12674 if (auto *Phi = dyn_cast<PHINode>(Val: LU->getValue())) {
12675 if (!PendingMerges.insert(Ptr: Phi).second)
12676 return false;
12677 LPhi = Phi;
12678 }
12679 if (const SCEVUnknown *RU = dyn_cast<SCEVUnknown>(Val: RHS))
12680 if (auto *Phi = dyn_cast<PHINode>(Val: RU->getValue())) {
12681 // If we detect a loop of Phi nodes being processed by this method, for
12682 // example:
12683 //
12684 // %a = phi i32 [ %some1, %preheader ], [ %b, %latch ]
12685 // %b = phi i32 [ %some2, %preheader ], [ %a, %latch ]
12686 //
12687 // we don't want to deal with a case that complex, so return conservative
12688 // answer false.
12689 if (!PendingMerges.insert(Ptr: Phi).second)
12690 return false;
12691 RPhi = Phi;
12692 }
12693
12694 // If none of LHS, RHS is a Phi, nothing to do here.
12695 if (!LPhi && !RPhi)
12696 return false;
12697
12698 // If there is a SCEVUnknown Phi we are interested in, make it left.
12699 if (!LPhi) {
12700 std::swap(a&: LHS, b&: RHS);
12701 std::swap(a&: FoundLHS, b&: FoundRHS);
12702 std::swap(a&: LPhi, b&: RPhi);
12703 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12704 }
12705
12706 assert(LPhi && "LPhi should definitely be a SCEVUnknown Phi!");
12707 const BasicBlock *LBB = LPhi->getParent();
12708 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(Val: RHS);
12709
12710 auto ProvedEasily = [&](const SCEV *S1, const SCEV *S2) {
12711 return isKnownViaNonRecursiveReasoning(Pred, LHS: S1, RHS: S2) ||
12712 isImpliedCondOperandsViaRanges(Pred, LHS: S1, RHS: S2, FoundPred: Pred, FoundLHS, FoundRHS) ||
12713 isImpliedViaOperations(Pred, LHS: S1, RHS: S2, FoundLHS, FoundRHS, Depth);
12714 };
12715
12716 if (RPhi && RPhi->getParent() == LBB) {
12717 // Case one: RHS is also a SCEVUnknown Phi from the same basic block.
12718 // If we compare two Phis from the same block, and for each entry block
12719 // the predicate is true for incoming values from this block, then the
12720 // predicate is also true for the Phis.
12721 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12722 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12723 const SCEV *R = getSCEV(V: RPhi->getIncomingValueForBlock(BB: IncBB));
12724 if (!ProvedEasily(L, R))
12725 return false;
12726 }
12727 } else if (RAR && RAR->getLoop()->getHeader() == LBB) {
12728 // Case two: RHS is also a Phi from the same basic block, and it is an
12729 // AddRec. It means that there is a loop which has both AddRec and Unknown
12730 // PHIs, for it we can compare incoming values of AddRec from above the loop
12731 // and latch with their respective incoming values of LPhi.
12732 // TODO: Generalize to handle loops with many inputs in a header.
12733 if (LPhi->getNumIncomingValues() != 2) return false;
12734
12735 auto *RLoop = RAR->getLoop();
12736 auto *Predecessor = RLoop->getLoopPredecessor();
12737 assert(Predecessor && "Loop with AddRec with no predecessor?");
12738 const SCEV *L1 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Predecessor));
12739 if (!ProvedEasily(L1, RAR->getStart()))
12740 return false;
12741 auto *Latch = RLoop->getLoopLatch();
12742 assert(Latch && "Loop with AddRec with no latch?");
12743 const SCEV *L2 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Latch));
12744 if (!ProvedEasily(L2, RAR->getPostIncExpr(SE&: *this)))
12745 return false;
12746 } else {
12747 // In all other cases go over inputs of LHS and compare each of them to RHS,
12748 // the predicate is true for (LHS, RHS) if it is true for all such pairs.
12749 // At this point RHS is either a non-Phi, or it is a Phi from some block
12750 // different from LBB.
12751 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12752 // Check that RHS is available in this block.
12753 if (!dominates(S: RHS, BB: IncBB))
12754 return false;
12755 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12756 // Make sure L does not refer to a value from a potentially previous
12757 // iteration of a loop.
12758 if (!properlyDominates(S: L, BB: LBB))
12759 return false;
12760 // Addrecs are considered to properly dominate their loop, so are missed
12761 // by the previous check. Discard any values that have computable
12762 // evolution in this loop.
12763 if (auto *Loop = LI.getLoopFor(BB: LBB))
12764 if (hasComputableLoopEvolution(S: L, L: Loop))
12765 return false;
12766 if (!ProvedEasily(L, RHS))
12767 return false;
12768 }
12769 }
12770 return true;
12771}
12772
12773bool ScalarEvolution::isImpliedCondOperandsViaShift(CmpPredicate Pred,
12774 const SCEV *LHS,
12775 const SCEV *RHS,
12776 const SCEV *FoundLHS,
12777 const SCEV *FoundRHS) {
12778 // We want to imply LHS < RHS from LHS < (RHS >> shiftvalue). First, make
12779 // sure that we are dealing with same LHS.
12780 if (RHS == FoundRHS) {
12781 std::swap(a&: LHS, b&: RHS);
12782 std::swap(a&: FoundLHS, b&: FoundRHS);
12783 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12784 }
12785 if (LHS != FoundLHS)
12786 return false;
12787
12788 auto *SUFoundRHS = dyn_cast<SCEVUnknown>(Val: FoundRHS);
12789 if (!SUFoundRHS)
12790 return false;
12791
12792 Value *Shiftee, *ShiftValue;
12793
12794 using namespace PatternMatch;
12795 if (match(V: SUFoundRHS->getValue(),
12796 P: m_LShr(L: m_Value(V&: Shiftee), R: m_Value(V&: ShiftValue)))) {
12797 auto *ShifteeS = getSCEV(V: Shiftee);
12798 // Prove one of the following:
12799 // LHS <u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <u RHS
12800 // LHS <=u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <=u RHS
12801 // LHS <s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12802 // ---> LHS <s RHS
12803 // LHS <=s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12804 // ---> LHS <=s RHS
12805 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
12806 return isKnownPredicate(Pred: ICmpInst::ICMP_ULE, LHS: ShifteeS, RHS);
12807 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
12808 if (isKnownNonNegative(S: ShifteeS))
12809 return isKnownPredicate(Pred: ICmpInst::ICMP_SLE, LHS: ShifteeS, RHS);
12810 }
12811
12812 return false;
12813}
12814
12815bool ScalarEvolution::isImpliedCondOperandsViaMatchingDiff(
12816 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12817 const SCEV *FoundRHS) {
12818 // Only valid for equality predicates: (A == B) implies (C == D) when
12819 // the SCEV difference A - B equals C - D (they check the same
12820 // underlying relationship at every iteration).
12821 if (!ICmpInst::isEquality(P: Pred))
12822 return false;
12823
12824 // Restrict to cases involving loop recurrences - that's where this
12825 // pattern arises (correlated IV comparisons). This avoids calling
12826 // getMinusSCEV on arbitrary non-loop expressions.
12827 if ((!isa<SCEVAddRecExpr>(Val: LHS) && !isa<SCEVAddRecExpr>(Val: RHS)) ||
12828 (!isa<SCEVAddRecExpr>(Val: FoundLHS) && !isa<SCEVAddRecExpr>(Val: FoundRHS)))
12829 return false;
12830
12831 // AddRecs from different loops can never produce matching differences.
12832 const SCEVAddRecExpr *QueryAddRec = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12833 if (!QueryAddRec)
12834 QueryAddRec = cast<SCEVAddRecExpr>(Val: RHS);
12835 const SCEVAddRecExpr *FoundAddRec = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12836 if (!FoundAddRec)
12837 FoundAddRec = cast<SCEVAddRecExpr>(Val: FoundRHS);
12838 if (QueryAddRec->getLoop() != FoundAddRec->getLoop())
12839 return false;
12840
12841 // If the strides differ, the differences can never match.
12842 if (QueryAddRec->getStepRecurrence(SE&: *this) !=
12843 FoundAddRec->getStepRecurrence(SE&: *this))
12844 return false;
12845
12846 // Compute differences. For pointer-typed operands sharing the same base,
12847 // getMinusSCEV strips the common base and returns an integer SCEV.
12848 // For example, {base,+,8} - (base+8*n) = {-8n,+,8}
12849 const SCEV *FoundDiff = getMinusSCEV(LHS: FoundLHS, RHS: FoundRHS);
12850 if (isa<SCEVCouldNotCompute>(Val: FoundDiff))
12851 return false;
12852
12853 const SCEV *Diff = getMinusSCEV(LHS, RHS);
12854 if (isa<SCEVCouldNotCompute>(Val: Diff))
12855 return false;
12856
12857 return Diff == FoundDiff;
12858}
12859
12860bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred, const SCEV *LHS,
12861 const SCEV *RHS,
12862 const SCEV *FoundLHS,
12863 const SCEV *FoundRHS,
12864 const Instruction *CtxI) {
12865 return isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred: Pred, FoundLHS,
12866 FoundRHS) ||
12867 isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS,
12868 FoundRHS) ||
12869 isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS) ||
12870 isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
12871 CtxI) ||
12872 isImpliedCondOperandsViaMatchingDiff(Pred, LHS, RHS, FoundLHS,
12873 FoundRHS) ||
12874 isImpliedCondOperandsHelper(Pred, LHS, RHS, FoundLHS, FoundRHS);
12875}
12876
12877/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
12878template <typename MinMaxExprType>
12879static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
12880 const SCEV *Candidate) {
12881 const MinMaxExprType *MinMaxExpr = dyn_cast<MinMaxExprType>(MaybeMinMaxExpr);
12882 if (!MinMaxExpr)
12883 return false;
12884
12885 return is_contained(MinMaxExpr->operands(), Candidate);
12886}
12887
12888static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE,
12889 CmpPredicate Pred, const SCEV *LHS,
12890 const SCEV *RHS) {
12891 // If both sides are affine addrecs for the same loop, with equal
12892 // steps, and we know the recurrences don't wrap, then we only
12893 // need to check the predicate on the starting values.
12894
12895 if (!ICmpInst::isRelational(P: Pred))
12896 return false;
12897
12898 const SCEV *LStart, *RStart, *Step;
12899 const Loop *L;
12900 if (!match(S: LHS,
12901 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: LStart), Op1: m_SCEV(V&: Step), L: m_Loop(L))) ||
12902 !match(S: RHS, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: RStart), Op1: m_scev_Specific(S: Step),
12903 L: m_SpecificLoop(L))))
12904 return false;
12905 const SCEVAddRecExpr *LAR = cast<SCEVAddRecExpr>(Val: LHS);
12906 const SCEVAddRecExpr *RAR = cast<SCEVAddRecExpr>(Val: RHS);
12907 SCEV::NoWrapFlags NW = ICmpInst::isSigned(Pred) ?
12908 SCEV::FlagNSW : SCEV::FlagNUW;
12909 if (!LAR->getNoWrapFlags(Mask: NW) || !RAR->getNoWrapFlags(Mask: NW))
12910 return false;
12911
12912 return SE.isKnownPredicate(Pred, LHS: LStart, RHS: RStart);
12913}
12914
12915/// Is LHS `Pred` RHS true because one of them is an AddRec that is known not to
12916/// go below its own start value?
12917static bool IsKnownPredicateViaAddRecMonotonicity(ScalarEvolution &SE,
12918 CmpPredicate Pred,
12919 const SCEV *LHS,
12920 const SCEV *RHS) {
12921 // Normalize to (AddRec Pred Start).
12922 if (!isa<SCEVAddRecExpr>(Val: LHS) && isa<SCEVAddRecExpr>(Val: RHS)) {
12923 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12924 std::swap(a&: LHS, b&: RHS);
12925 }
12926
12927 // The recurrence is equal to Start in the first iteration, so only the
12928 // non-strict predicate holds.
12929 if (Pred != ICmpInst::ICMP_UGE && Pred != ICmpInst::ICMP_SGE)
12930 return false;
12931
12932 const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12933 if (!AR || AR->getStart() != RHS)
12934 return false;
12935
12936 return SE.getMonotonicPredicateType(LHS: AR, Pred) ==
12937 ScalarEvolution::MonotonicallyIncreasing;
12938}
12939
12940/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
12941/// expression?
12942static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, CmpPredicate Pred,
12943 const SCEV *LHS, const SCEV *RHS) {
12944 switch (Pred) {
12945 default:
12946 return false;
12947
12948 case ICmpInst::ICMP_SGE:
12949 std::swap(a&: LHS, b&: RHS);
12950 [[fallthrough]];
12951 case ICmpInst::ICMP_SLE:
12952 return
12953 // min(A, ...) <= A
12954 IsMinMaxConsistingOf<SCEVSMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12955 // A <= max(A, ...)
12956 IsMinMaxConsistingOf<SCEVSMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12957
12958 case ICmpInst::ICMP_UGE:
12959 std::swap(a&: LHS, b&: RHS);
12960 [[fallthrough]];
12961 case ICmpInst::ICMP_ULE:
12962 return
12963 // min(A, ...) <= A
12964 // FIXME: what about umin_seq?
12965 IsMinMaxConsistingOf<SCEVUMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12966 // A <= max(A, ...)
12967 IsMinMaxConsistingOf<SCEVUMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12968
12969 case ICmpInst::ICMP_UGT:
12970 std::swap(a&: LHS, b&: RHS);
12971 [[fallthrough]];
12972 case ICmpInst::ICMP_ULT:
12973 // umin(Ops) u<= each Op, so proving Op u< RHS for any Op proves
12974 // umin(Ops) u< RHS.
12975 //
12976 // Use computeConstantDifference instead of the more powerful
12977 // isKnownPredicate to keep this check cheap: isKnownPredicateViaMinOrMax
12978 // is called from isKnownViaNonRecursiveReasoning, so recursing into
12979 // the full predicate prover would be expensive.
12980 if (const auto *Min = dyn_cast<SCEVUMinExpr>(Val: LHS)) {
12981 for (SCEVUse Op : Min->operands()) {
12982 std::optional<APInt> Diff = SE.computeConstantDifference(More: RHS, Less: Op);
12983 // When Op and RHS share a common base differing by a
12984 // constant offset D (RHS - Op = D), Op u< RHS holds iff D != 0 and
12985 // RHS >= D (unsigned), i.e. the subtraction doesn't underflow.
12986 if (Diff && !Diff->isZero() && SE.getUnsignedRangeMin(S: RHS).uge(RHS: *Diff))
12987 return true;
12988 }
12989 }
12990 return false;
12991 }
12992
12993 llvm_unreachable("covered switch fell through?!");
12994}
12995
12996bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
12997 const SCEV *RHS,
12998 const SCEV *FoundLHS,
12999 const SCEV *FoundRHS,
13000 unsigned Depth) {
13001 assert(getTypeSizeInBits(LHS->getType()) ==
13002 getTypeSizeInBits(RHS->getType()) &&
13003 "LHS and RHS have different sizes?");
13004 assert(getTypeSizeInBits(FoundLHS->getType()) ==
13005 getTypeSizeInBits(FoundRHS->getType()) &&
13006 "FoundLHS and FoundRHS have different sizes?");
13007 // We want to avoid hurting the compile time with analysis of too big trees.
13008 if (Depth > MaxSCEVOperationsImplicationDepth)
13009 return false;
13010
13011 // We only want to work with GT comparison so far.
13012 if (ICmpInst::isLT(P: Pred)) {
13013 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
13014 std::swap(a&: LHS, b&: RHS);
13015 std::swap(a&: FoundLHS, b&: FoundRHS);
13016 }
13017
13018 CmpInst::Predicate P = Pred.getPreferredSignedPredicate();
13019
13020 // For unsigned, try to reduce it to corresponding signed comparison.
13021 if (P == ICmpInst::ICMP_UGT)
13022 // We can replace unsigned predicate with its signed counterpart if all
13023 // involved values are non-negative.
13024 // TODO: We could have better support for unsigned.
13025 if (isKnownNonNegative(S: FoundLHS) && isKnownNonNegative(S: FoundRHS)) {
13026 // Knowing that both FoundLHS and FoundRHS are non-negative, and knowing
13027 // FoundLHS >u FoundRHS, we also know that FoundLHS >s FoundRHS. Let us
13028 // use this fact to prove that LHS and RHS are non-negative.
13029 const SCEV *MinusOne = getMinusOne(Ty: LHS->getType());
13030 if (isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS, RHS: MinusOne, FoundLHS,
13031 FoundRHS) &&
13032 isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS: RHS, RHS: MinusOne, FoundLHS,
13033 FoundRHS))
13034 P = ICmpInst::ICMP_SGT;
13035 }
13036
13037 if (P != ICmpInst::ICMP_SGT)
13038 return false;
13039
13040 auto GetOpFromSExt = [&](const SCEV *S) -> const SCEV * {
13041 if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(Val: S))
13042 return Ext->getOperand();
13043 // TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
13044 // the constant in some cases.
13045 return S;
13046 };
13047
13048 // Acquire values from extensions.
13049 auto *OrigLHS = LHS;
13050 auto *OrigFoundLHS = FoundLHS;
13051 LHS = GetOpFromSExt(LHS);
13052 FoundLHS = GetOpFromSExt(FoundLHS);
13053
13054 // Is the SGT predicate can be proved trivially or using the found context.
13055 auto IsSGTViaContext = [&](const SCEV *S1, const SCEV *S2) {
13056 return isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2) ||
13057 isImpliedViaOperations(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2, FoundLHS: OrigFoundLHS,
13058 FoundRHS, Depth: Depth + 1);
13059 };
13060
13061 if (auto *LHSAddExpr = dyn_cast<SCEVAddExpr>(Val: LHS)) {
13062 // We want to avoid creation of any new non-constant SCEV. Since we are
13063 // going to compare the operands to RHS, we should be certain that we don't
13064 // need any size extensions for this. So let's decline all cases when the
13065 // sizes of types of LHS and RHS do not match.
13066 // TODO: Maybe try to get RHS from sext to catch more cases?
13067 if (getTypeSizeInBits(Ty: LHS->getType()) != getTypeSizeInBits(Ty: RHS->getType()))
13068 return false;
13069
13070 // Should not overflow.
13071 if (!LHSAddExpr->hasNoSignedWrap())
13072 return false;
13073
13074 SCEVUse LL = LHSAddExpr->getOperand(i: 0);
13075 SCEVUse LR = LHSAddExpr->getOperand(i: 1);
13076 auto *MinusOne = getMinusOne(Ty: RHS->getType());
13077
13078 // Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
13079 auto IsSumGreaterThanRHS = [&](const SCEV *S1, const SCEV *S2) {
13080 return IsSGTViaContext(S1, MinusOne) && IsSGTViaContext(S2, RHS);
13081 };
13082 // Try to prove the following rule:
13083 // (LHS = LL + LR) && (LL >= 0) && (LR > RHS) => (LHS > RHS).
13084 // (LHS = LL + LR) && (LR >= 0) && (LL > RHS) => (LHS > RHS).
13085 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
13086 return true;
13087 } else if (auto *LHSUnknownExpr = dyn_cast<SCEVUnknown>(Val: LHS)) {
13088 Value *LL, *LR;
13089 // FIXME: Once we have SDiv implemented, we can get rid of this matching.
13090
13091 using namespace llvm::PatternMatch;
13092
13093 if (match(V: LHSUnknownExpr->getValue(), P: m_SDiv(L: m_Value(V&: LL), R: m_Value(V&: LR)))) {
13094 // Rules for division.
13095 // We are going to perform some comparisons with Denominator and its
13096 // derivative expressions. In general case, creating a SCEV for it may
13097 // lead to a complex analysis of the entire graph, and in particular it
13098 // can request trip count recalculation for the same loop. This would
13099 // cache as SCEVCouldNotCompute to avoid the infinite recursion. To avoid
13100 // this, we only want to create SCEVs that are constants in this section.
13101 // So we bail if Denominator is not a constant.
13102 if (!isa<ConstantInt>(Val: LR))
13103 return false;
13104
13105 auto *Denominator = cast<SCEVConstant>(Val: getSCEV(V: LR));
13106
13107 // We want to make sure that LHS = FoundLHS / Denominator. If it is so,
13108 // then a SCEV for the numerator already exists and matches with FoundLHS.
13109 auto *Numerator = getExistingSCEV(V: LL);
13110 if (!Numerator || Numerator->getType() != FoundLHS->getType())
13111 return false;
13112
13113 // Make sure that the numerator matches with FoundLHS and the denominator
13114 // is positive.
13115 if (!HasSameValue(A: Numerator, B: FoundLHS) || !isKnownPositive(S: Denominator))
13116 return false;
13117
13118 auto *DTy = Denominator->getType();
13119 auto *FRHSTy = FoundRHS->getType();
13120 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
13121 // One of types is a pointer and another one is not. We cannot extend
13122 // them properly to a wider type, so let us just reject this case.
13123 // TODO: Usage of getEffectiveSCEVType for DTy, FRHSTy etc should help
13124 // to avoid this check.
13125 return false;
13126
13127 // Given that:
13128 // FoundLHS > FoundRHS, LHS = FoundLHS / Denominator, Denominator > 0.
13129 auto *WTy = getWiderType(T1: DTy, T2: FRHSTy);
13130 auto *DenominatorExt = getNoopOrSignExtend(V: Denominator, Ty: WTy);
13131 auto *FoundRHSExt = getNoopOrSignExtend(V: FoundRHS, Ty: WTy);
13132
13133 // Try to prove the following rule:
13134 // (FoundRHS > Denominator - 2) && (RHS <= 0) => (LHS > RHS).
13135 // For example, given that FoundLHS > 2. It means that FoundLHS is at
13136 // least 3. If we divide it by Denominator < 4, we will have at least 1.
13137 auto *DenomMinusTwo = getMinusSCEV(LHS: DenominatorExt, RHS: getConstant(Ty: WTy, V: 2));
13138 if (isKnownNonPositive(S: RHS) &&
13139 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
13140 return true;
13141
13142 // Try to prove the following rule:
13143 // (FoundRHS > -1 - Denominator) && (RHS < 0) => (LHS > RHS).
13144 // For example, given that FoundLHS > -3. Then FoundLHS is at least -2.
13145 // If we divide it by Denominator > 2, then:
13146 // 1. If FoundLHS is negative, then the result is 0.
13147 // 2. If FoundLHS is non-negative, then the result is non-negative.
13148 // Anyways, the result is non-negative.
13149 auto *MinusOne = getMinusOne(Ty: WTy);
13150 auto *NegDenomMinusOne = getMinusSCEV(LHS: MinusOne, RHS: DenominatorExt);
13151 if (isKnownNegative(S: RHS) &&
13152 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
13153 return true;
13154 }
13155 }
13156
13157 // If our expression contained SCEVUnknown Phis, and we split it down and now
13158 // need to prove something for them, try to prove the predicate for every
13159 // possible incoming values of those Phis.
13160 if (isImpliedViaMerge(Pred, LHS: OrigLHS, RHS, FoundLHS: OrigFoundLHS, FoundRHS, Depth: Depth + 1))
13161 return true;
13162
13163 return false;
13164}
13165
13166static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS,
13167 const SCEV *RHS) {
13168 // zext x u<= sext x, sext x s<= zext x
13169 const SCEV *Op;
13170 switch (Pred) {
13171 case ICmpInst::ICMP_SGE:
13172 std::swap(a&: LHS, b&: RHS);
13173 [[fallthrough]];
13174 case ICmpInst::ICMP_SLE: {
13175 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
13176 return match(S: LHS, P: m_scev_SExt(Op0: m_SCEV(V&: Op))) &&
13177 match(S: RHS, P: m_scev_ZExt(Op0: m_scev_Specific(S: Op)));
13178 }
13179 case ICmpInst::ICMP_UGE:
13180 std::swap(a&: LHS, b&: RHS);
13181 [[fallthrough]];
13182 case ICmpInst::ICMP_ULE: {
13183 // If operand >=u 0 then ZExt == SExt. If operand <u 0 then ZExt <u SExt.
13184 return match(S: LHS, P: m_scev_ZExt(Op0: m_SCEV(V&: Op))) &&
13185 match(S: RHS, P: m_scev_SExt(Op0: m_scev_Specific(S: Op)));
13186 }
13187 default:
13188 return false;
13189 };
13190 llvm_unreachable("unhandled case");
13191}
13192
13193bool ScalarEvolution::isKnownViaNonRecursiveReasoning(CmpPredicate Pred,
13194 SCEVUse LHS,
13195 SCEVUse RHS) {
13196 return isKnownPredicateExtendIdiom(Pred, LHS, RHS) ||
13197 isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
13198 IsKnownPredicateViaMinOrMax(SE&: *this, Pred, LHS, RHS) ||
13199 IsKnownPredicateViaAddRecStart(SE&: *this, Pred, LHS, RHS) ||
13200 IsKnownPredicateViaAddRecMonotonicity(SE&: *this, Pred, LHS, RHS) ||
13201 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
13202}
13203
13204bool ScalarEvolution::isImpliedCondOperandsHelper(CmpPredicate Pred,
13205 const SCEV *LHS,
13206 const SCEV *RHS,
13207 const SCEV *FoundLHS,
13208 const SCEV *FoundRHS) {
13209 switch (Pred) {
13210 default:
13211 llvm_unreachable("Unexpected CmpPredicate value!");
13212 case ICmpInst::ICMP_EQ:
13213 case ICmpInst::ICMP_NE:
13214 if (HasSameValue(A: LHS, B: FoundLHS) && HasSameValue(A: RHS, B: FoundRHS))
13215 return true;
13216 break;
13217 case ICmpInst::ICMP_SLT:
13218 case ICmpInst::ICMP_SLE:
13219 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS, RHS: FoundLHS) &&
13220 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS: RHS, RHS: FoundRHS))
13221 return true;
13222 break;
13223 case ICmpInst::ICMP_SGT:
13224 case ICmpInst::ICMP_SGE:
13225 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS, RHS: FoundLHS) &&
13226 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS: RHS, RHS: FoundRHS))
13227 return true;
13228 break;
13229 case ICmpInst::ICMP_ULT:
13230 case ICmpInst::ICMP_ULE:
13231 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS, RHS: FoundLHS) &&
13232 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS: RHS, RHS: FoundRHS))
13233 return true;
13234 break;
13235 case ICmpInst::ICMP_UGT:
13236 case ICmpInst::ICMP_UGE:
13237 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS, RHS: FoundLHS) &&
13238 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: RHS, RHS: FoundRHS))
13239 return true;
13240 break;
13241 }
13242
13243 // Maybe it can be proved via operations?
13244 if (isImpliedViaOperations(Pred, LHS, RHS, FoundLHS, FoundRHS))
13245 return true;
13246
13247 return false;
13248}
13249
13250bool ScalarEvolution::isImpliedCondOperandsViaRanges(
13251 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
13252 const SCEV *FoundLHS, const SCEV *FoundRHS) {
13253 if (!isa<SCEVConstant>(Val: RHS) || !isa<SCEVConstant>(Val: FoundRHS))
13254 // The restriction on `FoundRHS` be lifted easily -- it exists only to
13255 // reduce the compile time impact of this optimization.
13256 return false;
13257
13258 std::optional<APInt> Addend = computeConstantDifference(More: LHS, Less: FoundLHS);
13259 if (!Addend)
13260 return false;
13261
13262 const APInt &ConstFoundRHS = cast<SCEVConstant>(Val: FoundRHS)->getAPInt();
13263
13264 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the
13265 // antecedent "`FoundLHS` `FoundPred` `FoundRHS`".
13266 ConstantRange FoundLHSRange =
13267 ConstantRange::makeExactICmpRegion(Pred: FoundPred, Other: ConstFoundRHS);
13268
13269 // Since `LHS` is `FoundLHS` + `Addend`, we can compute a range for `LHS`:
13270 ConstantRange LHSRange = FoundLHSRange.add(Other: ConstantRange(*Addend));
13271
13272 // We can also compute the range of values for `LHS` that satisfy the
13273 // consequent, "`LHS` `Pred` `RHS`":
13274 const APInt &ConstRHS = cast<SCEVConstant>(Val: RHS)->getAPInt();
13275 // The antecedent implies the consequent if every value of `LHS` that
13276 // satisfies the antecedent also satisfies the consequent.
13277 return LHSRange.icmp(Pred, Other: ConstRHS);
13278}
13279
13280bool ScalarEvolution::canIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
13281 bool IsSigned) {
13282 assert(isKnownPositive(Stride) && "Positive stride expected!");
13283
13284 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
13285 const SCEV *One = getOne(Ty: Stride->getType());
13286
13287 if (IsSigned) {
13288 APInt MaxRHS = getSignedRangeMax(S: RHS);
13289 APInt MaxValue = APInt::getSignedMaxValue(numBits: BitWidth);
13290 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13291
13292 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
13293 return (std::move(MaxValue) - MaxStrideMinusOne).slt(RHS: MaxRHS);
13294 }
13295
13296 APInt MaxRHS = getUnsignedRangeMax(S: RHS);
13297 APInt MaxValue = APInt::getMaxValue(numBits: BitWidth);
13298 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13299
13300 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
13301 return (std::move(MaxValue) - MaxStrideMinusOne).ult(RHS: MaxRHS);
13302}
13303
13304bool ScalarEvolution::canIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride,
13305 bool IsSigned) {
13306
13307 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
13308 const SCEV *One = getOne(Ty: Stride->getType());
13309
13310 if (IsSigned) {
13311 APInt MinRHS = getSignedRangeMin(S: RHS);
13312 APInt MinValue = APInt::getSignedMinValue(numBits: BitWidth);
13313 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13314
13315 // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow!
13316 return (std::move(MinValue) + MaxStrideMinusOne).sgt(RHS: MinRHS);
13317 }
13318
13319 APInt MinRHS = getUnsignedRangeMin(S: RHS);
13320 APInt MinValue = APInt::getMinValue(numBits: BitWidth);
13321 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13322
13323 // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow!
13324 return (std::move(MinValue) + MaxStrideMinusOne).ugt(RHS: MinRHS);
13325}
13326
13327const SCEV *ScalarEvolution::getUDivCeilSCEV(const SCEV *N, const SCEV *D) {
13328 // umin(N, 1) + floor((N - umin(N, 1)) / D)
13329 // This is equivalent to "1 + floor((N - 1) / D)" for N != 0. The umin
13330 // expression fixes the case of N=0.
13331 const SCEV *MinNOne = getUMinExpr(LHS: N, RHS: getOne(Ty: N->getType()));
13332 const SCEV *NMinusOne = getMinusSCEV(LHS: N, RHS: MinNOne);
13333 return getAddExpr(LHS: MinNOne, RHS: getUDivExpr(LHS: NMinusOne, RHS: D));
13334}
13335
13336const SCEV *ScalarEvolution::computeMaxBECountForLT(const SCEV *Start,
13337 const SCEV *Stride,
13338 const SCEV *End,
13339 unsigned BitWidth,
13340 bool IsSigned) {
13341 // The logic in this function assumes we can represent a positive stride.
13342 // If we can't, the backedge-taken count must be zero.
13343 if (IsSigned && BitWidth == 1)
13344 return getZero(Ty: Stride->getType());
13345
13346 // This code below only been closely audited for negative strides in the
13347 // unsigned comparison case, it may be correct for signed comparison, but
13348 // that needs to be established.
13349 if (IsSigned && isKnownNegative(S: Stride))
13350 return getCouldNotCompute();
13351
13352 // Calculate the maximum backedge count based on the range of values
13353 // permitted by Start, End, and Stride.
13354 APInt MinStart =
13355 IsSigned ? getSignedRangeMin(S: Start) : getUnsignedRangeMin(S: Start);
13356
13357 APInt MinStride =
13358 IsSigned ? getSignedRangeMin(S: Stride) : getUnsignedRangeMin(S: Stride);
13359
13360 // We assume either the stride is positive, or the backedge-taken count
13361 // is zero. So force StrideForMaxBECount to be at least one.
13362 APInt One(BitWidth, 1);
13363 APInt StrideForMaxBECount = IsSigned ? APIntOps::smax(A: One, B: MinStride)
13364 : APIntOps::umax(A: One, B: MinStride);
13365
13366 APInt MaxValue = IsSigned ? APInt::getSignedMaxValue(numBits: BitWidth)
13367 : APInt::getMaxValue(numBits: BitWidth);
13368 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
13369
13370 // Although End can be a MAX expression we estimate MaxEnd considering only
13371 // the case End = RHS of the loop termination condition. This is safe because
13372 // in the other case (End - Start) is zero, leading to a zero maximum backedge
13373 // taken count.
13374 APInt MaxEnd = IsSigned ? APIntOps::smin(A: getSignedRangeMax(S: End), B: Limit)
13375 : APIntOps::umin(A: getUnsignedRangeMax(S: End), B: Limit);
13376
13377 // MaxBECount = ceil((max(MaxEnd, MinStart) - MinStart) / Stride)
13378 MaxEnd = IsSigned ? APIntOps::smax(A: MaxEnd, B: MinStart)
13379 : APIntOps::umax(A: MaxEnd, B: MinStart);
13380
13381 APInt Delta = MaxEnd - MinStart;
13382
13383 // Try to refine Delta in case End - Start gives a tighter bound after
13384 // folding.
13385 Delta = APIntOps::umin(A: Delta, B: getUnsignedRangeMax(S: getMinusSCEV(LHS: End, RHS: Start)));
13386
13387 return getUDivCeilSCEV(N: getConstant(Val: Delta), D: getConstant(Val: StrideForMaxBECount));
13388}
13389
13390ScalarEvolution::ExitLimit
13391ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
13392 const Loop *L, bool IsSigned,
13393 bool ControlsOnlyExit, bool AllowPredicates) {
13394 SmallVector<const SCEVPredicate *> Predicates;
13395
13396 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13397 bool PredicatedIV = false;
13398 if (!IV) {
13399 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: LHS)) {
13400 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: ZExt->getOperand());
13401 if (AR && AR->getLoop() == L && AR->isAffine()) {
13402 auto canProveNUW = [&]() {
13403 // We can use the comparison to infer no-wrap flags only if it fully
13404 // controls the loop exit.
13405 if (!ControlsOnlyExit)
13406 return false;
13407
13408 if (!isLoopInvariant(S: RHS, L))
13409 return false;
13410
13411 if (!isKnownNonZero(S: AR->getStepRecurrence(SE&: *this)))
13412 // We need the sequence defined by AR to strictly increase in the
13413 // unsigned integer domain for the logic below to hold.
13414 return false;
13415
13416 const unsigned InnerBitWidth = getTypeSizeInBits(Ty: AR->getType());
13417 const unsigned OuterBitWidth = getTypeSizeInBits(Ty: RHS->getType());
13418 // If RHS <=u Limit, then there must exist a value V in the sequence
13419 // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and
13420 // V <=u UINT_MAX. Thus, we must exit the loop before unsigned
13421 // overflow occurs. This limit also implies that a signed comparison
13422 // (in the wide bitwidth) is equivalent to an unsigned comparison as
13423 // the high bits on both sides must be zero.
13424 APInt StrideMax = getUnsignedRangeMax(S: AR->getStepRecurrence(SE&: *this));
13425 APInt Limit = APInt::getMaxValue(numBits: InnerBitWidth) - (StrideMax - 1);
13426 Limit = Limit.zext(width: OuterBitWidth);
13427 return getUnsignedRangeMax(S: applyLoopGuards(Expr: RHS, L)).ule(RHS: Limit);
13428 };
13429 auto Flags = AR->getNoWrapFlags();
13430 if (!hasFlags(Flags, TestFlags: SCEV::FlagNUW) && canProveNUW())
13431 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
13432
13433 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
13434 if (AR->hasNoUnsignedWrap()) {
13435 // Emulate what getZeroExtendExpr would have done during construction
13436 // if we'd been able to infer the fact just above at that time.
13437 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
13438 Type *Ty = ZExt->getType();
13439 const SCEV *S = getAddRecExpr(
13440 Start: getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: 0),
13441 Step: getZeroExtendExpr(Op: Step, Ty, Depth: 0), L, Flags: AR->getNoWrapFlags());
13442 IV = dyn_cast<SCEVAddRecExpr>(Val: S);
13443 }
13444 }
13445 }
13446 }
13447
13448
13449 if (!IV && AllowPredicates) {
13450 // Try to make this an AddRec using runtime tests, in the first X
13451 // iterations of this loop, where X is the SCEV expression found by the
13452 // algorithm below.
13453 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13454 PredicatedIV = true;
13455 }
13456
13457 // Avoid weird loops
13458 if (!IV || IV->getLoop() != L || !IV->isAffine())
13459 return getCouldNotCompute();
13460
13461 // A precondition of this method is that the condition being analyzed
13462 // reaches an exiting branch which dominates the latch. Given that, we can
13463 // assume that an increment which violates the nowrap specification and
13464 // produces poison must cause undefined behavior when the resulting poison
13465 // value is branched upon and thus we can conclude that the backedge is
13466 // taken no more often than would be required to produce that poison value.
13467 // Note that a well defined loop can exit on the iteration which violates
13468 // the nowrap specification if there is another exit (either explicit or
13469 // implicit/exceptional) which causes the loop to execute before the
13470 // exiting instruction we're analyzing would trigger UB.
13471 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13472 bool NoWrap = ControlsOnlyExit && any(Val: IV->getNoWrapFlags(Mask: WrapType));
13473 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13474
13475 const SCEV *Stride = IV->getStepRecurrence(SE&: *this);
13476 const SCEV *GuardedStride = Stride;
13477
13478 // Whether the IV may reach the maximum value before the exit is taken.
13479 bool IVMayOverflow = true;
13480
13481 bool PositiveStride = isKnownPositive(S: Stride);
13482 // A dominating guard may prove the stride positive.
13483 if (!PositiveStride) {
13484 const SCEV *LoopGuardedStride = applyLoopGuards(Expr: Stride, L);
13485 if (isKnownPositive(S: LoopGuardedStride)) {
13486 GuardedStride = LoopGuardedStride;
13487 PositiveStride = true;
13488 // Encode the context-sensitive stride > 0 fact into the expression
13489 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13490 }
13491 }
13492
13493 // Avoid negative or zero stride values.
13494 if (!PositiveStride) {
13495 // We can compute the correct backedge taken count for loops with unknown
13496 // strides if we can prove that the loop is not an infinite loop with side
13497 // effects. Here's the loop structure we are trying to handle -
13498 //
13499 // i = start
13500 // do {
13501 // A[i] = i;
13502 // i += s;
13503 // } while (i < end);
13504 //
13505 // The backedge taken count for such loops is evaluated as -
13506 // (max(end, start + stride) - start - 1) /u stride
13507 //
13508 // The additional preconditions that we need to check to prove correctness
13509 // of the above formula is as follows -
13510 //
13511 // a) IV is either nuw or nsw depending upon signedness (indicated by the
13512 // NoWrap flag).
13513 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13514 // no side effects within the loop)
13515 // c) loop has a single static exit (with no abnormal exits)
13516 //
13517 // Precondition a) implies that if the stride is negative, this is a single
13518 // trip loop. The backedge taken count formula reduces to zero in this case.
13519 //
13520 // Precondition b) and c) combine to imply that if rhs is invariant in L,
13521 // then a zero stride means the backedge can't be taken without executing
13522 // undefined behavior.
13523 //
13524 // The positive stride case is the same as isKnownPositive(Stride) returning
13525 // true (original behavior of the function).
13526 //
13527 if (PredicatedIV || !NoWrap || !loopIsFiniteByAssumption(L) ||
13528 !loopHasNoAbnormalExits(L))
13529 return getCouldNotCompute();
13530
13531 if (!isKnownNonZero(S: Stride)) {
13532 // If we have a step of zero, and RHS isn't invariant in L, we don't know
13533 // if it might eventually be greater than start and if so, on which
13534 // iteration. We can't even produce a useful upper bound.
13535 if (!isLoopInvariant(S: RHS, L))
13536 return getCouldNotCompute();
13537
13538 // We allow a potentially zero stride, but we need to divide by stride
13539 // below. Since the loop can't be infinite and this check must control
13540 // the sole exit, we can infer the exit must be taken on the first
13541 // iteration (e.g. backedge count = 0) if the stride is zero. Given that,
13542 // we know the numerator in the divides below must be zero, so we can
13543 // pick an arbitrary non-zero value for the denominator (e.g. stride)
13544 // and produce the right result.
13545 // FIXME: Handle the case where Stride is poison?
13546 auto wouldZeroStrideBeUB = [&]() {
13547 // Proof by contradiction. Suppose the stride were zero. If we can
13548 // prove that the backedge *is* taken on the first iteration, then since
13549 // we know this condition controls the sole exit, we must have an
13550 // infinite loop. We can't have a (well defined) infinite loop per
13551 // check just above.
13552 // Note: The (Start - Stride) term is used to get the start' term from
13553 // (start' + stride,+,stride). Remember that we only care about the
13554 // result of this expression when stride == 0 at runtime.
13555 auto *StartIfZero = getMinusSCEV(LHS: IV->getStart(), RHS: Stride);
13556 return isLoopEntryGuardedByCond(L, Pred: Cond, LHS: StartIfZero, RHS);
13557 };
13558 if (!wouldZeroStrideBeUB()) {
13559 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13560 }
13561 }
13562 } else {
13563 // Avoid proven overflow cases: this will ensure that the backedge taken
13564 // count will not generate any unsigned overflow.
13565 IVMayOverflow = canIVOverflowOnLT(RHS, Stride: GuardedStride, IsSigned);
13566 if (IVMayOverflow && !NoWrap)
13567 return getCouldNotCompute();
13568 }
13569
13570 // On all paths just preceeding, we established the following invariant:
13571 // IV can be assumed not to overflow up to and including the exiting
13572 // iteration. We proved this in one of two ways:
13573 // 1) We can show overflow doesn't occur before the exiting iteration
13574 // 1a) canIVOverflowOnLT, and b) step of one
13575 // 2) We can show that if overflow occurs, the loop must execute UB
13576 // before any possible exit.
13577 // Note that we have not yet proved RHS invariant (in general).
13578
13579 const SCEV *Start = IV->getStart();
13580
13581 // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond.
13582 // If we convert to integers, isLoopEntryGuardedByCond will miss some cases.
13583 // Use integer-typed versions for actual computation; we can't subtract
13584 // pointers in general.
13585 const SCEV *OrigStart = Start;
13586 const SCEV *OrigRHS = RHS;
13587 if (Start->getType()->isPointerTy()) {
13588 Start = getPtrToAddrExpr(Op: Start);
13589 if (isa<SCEVCouldNotCompute>(Val: Start))
13590 return Start;
13591 }
13592 if (RHS->getType()->isPointerTy()) {
13593 RHS = getPtrToAddrExpr(Op: RHS);
13594 if (isa<SCEVCouldNotCompute>(Val: RHS))
13595 return RHS;
13596 }
13597
13598 const SCEV *End = nullptr, *BECount = getCouldNotCompute(),
13599 *BECountIfBackedgeTaken = getCouldNotCompute();
13600 if (!isLoopInvariant(S: RHS, L)) {
13601 const auto *RHSAddRec = dyn_cast<SCEVAddRecExpr>(Val: RHS);
13602 if (PositiveStride && RHSAddRec != nullptr && RHSAddRec->getLoop() == L &&
13603 any(Val: RHSAddRec->getNoWrapFlags())) {
13604 // The structure of loop we are trying to calculate backedge count of:
13605 //
13606 // left = left_start
13607 // right = right_start
13608 //
13609 // while(left < right){
13610 // ... do something here ...
13611 // left += s1; // stride of left is s1 (s1 > 0)
13612 // right += s2; // stride of right is s2 (s2 < 0)
13613 // }
13614 //
13615
13616 const SCEV *RHSStart = RHSAddRec->getStart();
13617 const SCEV *RHSStride = RHSAddRec->getStepRecurrence(SE&: *this);
13618
13619 // If Stride - RHSStride is positive and does not overflow, we can write
13620 // backedge count as ->
13621 // ceil((End - Start) /u (Stride - RHSStride))
13622 // Where, End = max(RHSStart, Start)
13623
13624 // Check if RHSStride < 0 and Stride - RHSStride will not overflow.
13625 if (isKnownNegative(S: RHSStride) &&
13626 willNotOverflow(BinOp: Instruction::Sub, /*Signed=*/true, LHS: Stride,
13627 RHS: RHSStride)) {
13628
13629 const SCEV *Denominator = getMinusSCEV(LHS: Stride, RHS: RHSStride);
13630 if (isKnownPositive(S: Denominator)) {
13631 End = IsSigned ? getSMaxExpr(LHS: RHSStart, RHS: Start)
13632 : getUMaxExpr(LHS: RHSStart, RHS: Start);
13633
13634 // We can do this because End >= Start, as End = max(RHSStart, Start)
13635 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13636
13637 BECount = getUDivCeilSCEV(N: Delta, D: Denominator);
13638 BECountIfBackedgeTaken =
13639 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHSStart, RHS: Start), D: Denominator);
13640 }
13641 }
13642 }
13643 } else {
13644 // Let End = max(RHS,Start). We use the expression (End-Start)/Stride to
13645 // describe the backedge count: if the backedge is taken at least once then
13646 // End is RHS, and if not End is Start so we get a backedge count of zero.
13647 //
13648 // AddingStrideMinusOneMayOverflow has the following preconditions:
13649 //
13650 // 1. If IsSigned, Start <=s End; otherwise, Start <=u End
13651 // 2. The index variable doesn't overflow.
13652 //
13653 // Therefore, we know N exists such that
13654 // (Start + Stride * N) >= End, and computing "(Start + Stride * N)"
13655 // doesn't overflow.
13656 //
13657 // Using this information, try to prove whether the addition in
13658 // "(End - Start) + (Stride - 1)" has unsigned overflow.
13659 //
13660 // If the IV cannot overflow, RHS is at least Stride - 1 below the maximum
13661 // value, so the distance End - Start is at most UMAX - (Stride - 1) and
13662 // the (Stride - 1) addition below cannot overflow.
13663 const SCEV *One = getOne(Ty: Stride->getType());
13664 bool AddingStrideMinusOneMayOverflow = IVMayOverflow && [&] {
13665 if (isKnownToBeAPowerOfTwo(S: Stride)) {
13666 // Suppose Stride is a power of two, and Start/End are unsigned
13667 // integers. Let UMAX be the largest representable unsigned
13668 // integer.
13669 //
13670 // By the preconditions of this function, we know
13671 // "(Start + Stride * N) >= End", and this doesn't overflow.
13672 // As a formula:
13673 //
13674 // End <= (Start + Stride * N) <= UMAX
13675 //
13676 // Subtracting Start from all the terms:
13677 //
13678 // End - Start <= Stride * N <= UMAX - Start
13679 //
13680 // Since Start is unsigned, UMAX - Start <= UMAX. Therefore:
13681 //
13682 // End - Start <= Stride * N <= UMAX
13683 //
13684 // Stride * N is a multiple of Stride. Therefore,
13685 //
13686 // End - Start <= Stride * N <= UMAX - (UMAX mod Stride)
13687 //
13688 // Since Stride is a power of two, UMAX + 1 is divisible by
13689 // Stride. Therefore, UMAX mod Stride == Stride - 1. So we can
13690 // write:
13691 //
13692 // End - Start <= Stride * N <= UMAX - Stride - 1
13693 //
13694 // Dropping the middle term:
13695 //
13696 // End - Start <= UMAX - Stride - 1
13697 //
13698 // Adding Stride - 1 to both sides:
13699 //
13700 // (End - Start) + (Stride - 1) <= UMAX
13701 //
13702 // In other words, the addition doesn't have unsigned overflow.
13703 //
13704 // A similar proof works if we treat Start/End as signed values.
13705 // Just rewrite steps before "End - Start <= Stride * N <= UMAX"
13706 // to use signed max instead of unsigned max. Note that we're
13707 // trying to prove a lack of unsigned overflow in either case.
13708 return false;
13709 }
13710 if (Start == Stride || Start == getMinusSCEV(LHS: Stride, RHS: One)) {
13711 // If Start is equal to Stride, (End - Start) + (Stride - 1) == End
13712 // - 1. If !IsSigned, 0 <u Stride == Start <=u End; so 0 <u End - 1
13713 // <u End. If IsSigned, 0 <s Stride == Start <=s End; so 0 <s End -
13714 // 1 <s End.
13715 //
13716 // If Start is equal to Stride - 1, (End - Start) + Stride - 1 ==
13717 // End.
13718 return false;
13719 }
13720 return true;
13721 }();
13722
13723 auto *OrigStartMinusStride = getMinusSCEV(LHS: OrigStart, RHS: Stride);
13724 assert(isAvailableAtLoopEntry(OrigStartMinusStride, L) && "Must be!");
13725 assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!");
13726 assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!");
13727 // Can we prove Start - Stride < RHS, and either Start - Stride < Start or
13728 // (via !AddingStrideMinusOneMayOverflow) that (RHS - Start) + (Stride - 1)
13729 // does not overflow?
13730 if ((!AddingStrideMinusOneMayOverflow ||
13731 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigStartMinusStride, RHS: OrigStart)) &&
13732 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigStartMinusStride, RHS: OrigRHS)) {
13733 // In this case, we can use a refined formula for computing backedge
13734 // taken count. The general formula remains:
13735 // "End-Start /uceiling Stride"
13736 // We want to use the alternate formula:
13737 // "((RHS - 1) - (Start - Stride)) /u Stride"
13738 // Let's do a quick case analysis to show these are equivalent under
13739 // our preconditions.
13740 // * For RHS <= Start (End is Start), the backedge-taken count must be
13741 // zero. Together with the precondition "Start - Stride < RHS", we have
13742 // "Start - Stride < RHS <= Start". Subtracting Start - Stride from
13743 // all sides we get "0 < RHS - (Start - Stride) <= Stride".
13744 // Subtracting 1 we get "0 <= (RHS - 1) - (Start - Stride) < Stride".
13745 // So dividing that by Stride gives zero.
13746 //
13747 // * For RHS > Start (End is RHS), the backedge count must be
13748 // "RHS-Start /uceil Stride", so it is sufficient to show that the
13749 // numerator "((RHS - 1) - (Start - Stride))" does not overflow.
13750 //
13751 // If "Start - Stride < Start" holds, we have
13752 // "RHS > Start > Start - Stride". As such
13753 // "RHS - (Start - Stride) - 1" does not overflow, which is the
13754 // reassociated numerator.
13755 //
13756 // Otherwise !AddingStrideMinusOneMayOverflow guarantees that
13757 // "(End - Start) + (Stride - 1)" does not overflow unsigned. Here
13758 // "End" is "RHS", as "RHS > Start", so this is the reassociated
13759 // numerator. Neither sub-term wraps unsigned: "RHS - Start"
13760 // due to "RHS > Start", and "Stride - 1", as Stride is non-zero.
13761 const SCEV *MinusOne = getMinusOne(Ty: Stride->getType());
13762 const SCEV *Numerator =
13763 getMinusSCEV(LHS: getAddExpr(LHS: RHS, RHS: MinusOne), RHS: getMinusSCEV(LHS: Start, RHS: Stride));
13764 BECount = getUDivExpr(LHS: Numerator, RHS: Stride);
13765 }
13766
13767 if (isa<SCEVCouldNotCompute>(Val: BECount)) {
13768 auto canProveRHSGreaterThanEqualStart = [&]() {
13769 auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13770 const SCEV *GuardedRHS = applyLoopGuards(Expr: OrigRHS, L);
13771 const SCEV *GuardedStart = applyLoopGuards(Expr: OrigStart, L);
13772
13773 if (isLoopEntryGuardedByCond(L, Pred: CondGE, LHS: OrigRHS, RHS: OrigStart) ||
13774 isKnownPredicate(Pred: CondGE, LHS: GuardedRHS, RHS: GuardedStart))
13775 return true;
13776
13777 // (RHS > Start - 1) implies RHS >= Start.
13778 // * "RHS >= Start" is trivially equivalent to "RHS > Start - 1" if
13779 // "Start - 1" doesn't overflow.
13780 // * For signed comparison, if Start - 1 does overflow, it's equal
13781 // to INT_MAX, and "RHS >s INT_MAX" is trivially false.
13782 // * For unsigned comparison, if Start - 1 does overflow, it's equal
13783 // to UINT_MAX, and "RHS >u UINT_MAX" is trivially false.
13784 //
13785 // FIXME: Should isLoopEntryGuardedByCond do this for us?
13786 auto CondGT = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13787 const SCEV *StartMinusOne =
13788 getAddExpr(LHS: OrigStart, RHS: getMinusOne(Ty: OrigStart->getType()));
13789 return isLoopEntryGuardedByCond(L, Pred: CondGT, LHS: OrigRHS, RHS: StartMinusOne);
13790 };
13791
13792 // If we know that RHS >= Start in the context of loop, then we know
13793 // that max(RHS, Start) = RHS at this point.
13794 if (canProveRHSGreaterThanEqualStart()) {
13795 End = RHS;
13796 } else {
13797 // If RHS < Start, the backedge will be taken zero times. So in
13798 // general, we can write the backedge-taken count as:
13799 //
13800 // RHS >= Start ? ceil(RHS - Start) / Stride : 0
13801 //
13802 // We convert it to the following to make it more convenient for SCEV:
13803 //
13804 // ceil(max(RHS, Start) - Start) / Stride
13805 End = IsSigned ? getSMaxExpr(LHS: RHS, RHS: Start) : getUMaxExpr(LHS: RHS, RHS: Start);
13806
13807 // See what would happen if we assume the backedge is taken. This is
13808 // used to compute MaxBECount.
13809 BECountIfBackedgeTaken =
13810 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHS, RHS: Start), D: Stride);
13811 }
13812
13813 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13814 if (!AddingStrideMinusOneMayOverflow) {
13815 // floor((D + (S - 1)) / S)
13816 // We prefer this formulation if it's legal because it's fewer
13817 // operations.
13818 BECount =
13819 getUDivExpr(LHS: getAddExpr(LHS: Delta, RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13820 } else {
13821 BECount = getUDivCeilSCEV(N: Delta, D: Stride);
13822 }
13823 }
13824 }
13825
13826 const SCEV *ConstantMaxBECount;
13827 bool MaxOrZero = false;
13828 if (isa<SCEVConstant>(Val: BECount)) {
13829 ConstantMaxBECount = BECount;
13830 } else if (isa<SCEVConstant>(Val: BECountIfBackedgeTaken)) {
13831 // If we know exactly how many times the backedge will be taken if it's
13832 // taken at least once, then the backedge count will either be that or
13833 // zero.
13834 ConstantMaxBECount = BECountIfBackedgeTaken;
13835 MaxOrZero = true;
13836 } else {
13837 ConstantMaxBECount = computeMaxBECountForLT(
13838 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned);
13839 }
13840
13841 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
13842 !isa<SCEVCouldNotCompute>(Val: BECount))
13843 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
13844
13845 const SCEV *SymbolicMaxBECount =
13846 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13847 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, MaxOrZero,
13848 Predicates);
13849}
13850
13851ScalarEvolution::ExitLimit ScalarEvolution::howManyGreaterThans(
13852 const SCEV *LHS, const SCEV *RHS, const Loop *L, bool IsSigned,
13853 bool ControlsOnlyExit, bool AllowPredicates) {
13854 SmallVector<const SCEVPredicate *> Predicates;
13855 // We handle only IV > Invariant
13856 if (!isLoopInvariant(S: RHS, L))
13857 return getCouldNotCompute();
13858
13859 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13860 if (!IV && AllowPredicates)
13861 // Try to make this an AddRec using runtime tests, in the first X
13862 // iterations of this loop, where X is the SCEV expression found by the
13863 // algorithm below.
13864 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13865
13866 // Avoid weird loops
13867 if (!IV || IV->getLoop() != L || !IV->isAffine())
13868 return getCouldNotCompute();
13869
13870 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13871 bool NoWrap = ControlsOnlyExit && any(Val: IV->getNoWrapFlags(Mask: WrapType));
13872 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13873
13874 const SCEV *Stride = getNegativeSCEV(V: IV->getStepRecurrence(SE&: *this));
13875
13876 // Avoid negative or zero stride values
13877 if (!isKnownPositive(S: Stride))
13878 return getCouldNotCompute();
13879
13880 // Avoid proven overflow cases: this will ensure that the backedge taken count
13881 // will not generate any unsigned overflow. Relaxed no-overflow conditions
13882 // exploit NoWrapFlags, allowing to optimize in presence of undefined
13883 // behaviors like the case of C language.
13884 bool MayAddOverflow = false;
13885 const SCEV *Start = IV->getStart();
13886 const SCEV *End = RHS;
13887 if (!Stride->isOne() && canIVOverflowOnGT(RHS, Stride, IsSigned)) {
13888 if (!NoWrap)
13889 return getCouldNotCompute();
13890 MayAddOverflow = true;
13891 }
13892
13893 if (!isLoopEntryGuardedByCond(L, Pred: Cond, LHS: getAddExpr(LHS: Start, RHS: Stride), RHS)) {
13894 // If we know that Start >= RHS in the context of loop, then we know that
13895 // min(RHS, Start) = RHS at this point.
13896 if (isLoopEntryGuardedByCond(
13897 L, Pred: IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, LHS: Start, RHS))
13898 End = RHS;
13899 else
13900 End = IsSigned ? getSMinExpr(LHS: RHS, RHS: Start) : getUMinExpr(LHS: RHS, RHS: Start);
13901 }
13902
13903 if (Start->getType()->isPointerTy()) {
13904 Start = getPtrToAddrExpr(Op: Start);
13905 if (isa<SCEVCouldNotCompute>(Val: Start))
13906 return Start;
13907 }
13908 if (End->getType()->isPointerTy()) {
13909 End = getPtrToAddrExpr(Op: End);
13910 if (isa<SCEVCouldNotCompute>(Val: End))
13911 return End;
13912 }
13913
13914 const SCEV *Delta = getMinusSCEV(LHS: Start, RHS: End);
13915 const SCEV *BECount;
13916 if (MayAddOverflow) {
13917 // The ceiling division instead needs Start >= End, so that (Start - End) is
13918 // the exact unsigned distance between them.
13919 if (!isLoopEntryGuardedByCond(
13920 L, Pred: IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, LHS: Start, RHS: End))
13921 return getCouldNotCompute();
13922 BECount = getUDivCeilSCEV(N: Delta, D: Stride);
13923 } else {
13924 // Compute ((Start - End) + (Stride - 1)) / Stride, if the IV cannot
13925 // overflow as it requires fewer operations.
13926 const SCEV *One = getOne(Ty: Stride->getType());
13927 BECount = getUDivExpr(LHS: getAddExpr(LHS: Delta, RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13928 }
13929
13930 APInt MaxStart = IsSigned ? getSignedRangeMax(S: Start)
13931 : getUnsignedRangeMax(S: Start);
13932
13933 APInt MinStride = IsSigned ? getSignedRangeMin(S: Stride)
13934 : getUnsignedRangeMin(S: Stride);
13935
13936 unsigned BitWidth = getTypeSizeInBits(Ty: LHS->getType());
13937 APInt Limit = IsSigned ? APInt::getSignedMinValue(numBits: BitWidth) + (MinStride - 1)
13938 : APInt::getMinValue(numBits: BitWidth) + (MinStride - 1);
13939
13940 // Although End can be a MIN expression we estimate MinEnd considering only
13941 // the case End = RHS. This is safe because in the other case (Start - End)
13942 // is zero, leading to a zero maximum backedge taken count.
13943 APInt MinEnd =
13944 IsSigned ? APIntOps::smax(A: getSignedRangeMin(S: RHS), B: Limit)
13945 : APIntOps::umax(A: getUnsignedRangeMin(S: RHS), B: Limit);
13946
13947 const SCEV *ConstantMaxBECount =
13948 isa<SCEVConstant>(Val: BECount)
13949 ? BECount
13950 : getUDivCeilSCEV(N: getConstant(Val: MaxStart - MinEnd),
13951 D: getConstant(Val: MinStride));
13952
13953 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount))
13954 ConstantMaxBECount = BECount;
13955 const SCEV *SymbolicMaxBECount =
13956 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13957
13958 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
13959 Predicates);
13960}
13961
13962const SCEV *SCEVAddRecExpr::getNumIterationsInRange(const ConstantRange &Range,
13963 ScalarEvolution &SE) const {
13964 if (Range.isFullSet()) // Infinite loop.
13965 return SE.getCouldNotCompute();
13966
13967 // If the start is a non-zero constant, shift the range to simplify things.
13968 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: getStart()))
13969 if (!SC->getValue()->isZero()) {
13970 SmallVector<SCEVUse, 4> Operands(operands());
13971 Operands[0] = SE.getZero(Ty: SC->getType());
13972 const SCEV *Shifted = SE.getAddRecExpr(Operands, L: getLoop(),
13973 NWFlags: getNoWrapFlags(Mask: FlagNW));
13974 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Val: Shifted))
13975 return ShiftedAddRec->getNumIterationsInRange(
13976 Range: Range.subtract(CI: SC->getAPInt()), SE);
13977 // This is strange and shouldn't happen.
13978 return SE.getCouldNotCompute();
13979 }
13980
13981 // The only time we can solve this is when we have all constant indices.
13982 // Otherwise, we cannot determine the overflow conditions.
13983 if (!all_of(Range: operands(), P: IsaPred<SCEVConstant>))
13984 return SE.getCouldNotCompute();
13985
13986 // Okay at this point we know that all elements of the chrec are constants and
13987 // that the start element is zero.
13988
13989 // First check to see if the range contains zero. If not, the first
13990 // iteration exits.
13991 unsigned BitWidth = SE.getTypeSizeInBits(Ty: getType());
13992 if (!Range.contains(Val: APInt(BitWidth, 0)))
13993 return SE.getZero(Ty: getType());
13994
13995 if (isAffine()) {
13996 // If this is an affine expression then we have this situation:
13997 // Solve {0,+,A} in Range === Ax in Range
13998
13999 // We know that zero is in the range. If A is positive then we know that
14000 // the upper value of the range must be the first possible exit value.
14001 // If A is negative then the lower of the range is the last possible loop
14002 // value. Also note that we already checked for a full range.
14003 APInt A = cast<SCEVConstant>(Val: getOperand(i: 1))->getAPInt();
14004 APInt End = A.sge(RHS: 1) ? (Range.getUpper() - 1) : Range.getLower();
14005
14006 // The exit value should be (End+A)/A.
14007 APInt ExitVal = (End + A).udiv(RHS: A);
14008 ConstantInt *ExitValue = ConstantInt::get(Context&: SE.getContext(), V: ExitVal);
14009
14010 // Evaluate at the exit value. If we really did fall out of the valid
14011 // range, then we computed our trip count, otherwise wrap around or other
14012 // things must have happened.
14013 ConstantInt *Val = EvaluateConstantChrecAtConstant(AddRec: this, C: ExitValue, SE);
14014 if (Range.contains(Val: Val->getValue()))
14015 return SE.getCouldNotCompute(); // Something strange happened
14016
14017 // Ensure that the previous value is in the range.
14018 assert(Range.contains(
14019 EvaluateConstantChrecAtConstant(this,
14020 ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) &&
14021 "Linear scev computation is off in a bad way!");
14022 return SE.getConstant(V: ExitValue);
14023 }
14024
14025 if (isQuadratic()) {
14026 if (auto S = SolveQuadraticAddRecRange(AddRec: this, Range, SE))
14027 return SE.getConstant(Val: *S);
14028 }
14029
14030 return SE.getCouldNotCompute();
14031}
14032
14033const SCEVAddRecExpr *
14034SCEVAddRecExpr::getPostIncExpr(ScalarEvolution &SE) const {
14035 assert(getNumOperands() > 1 && "AddRec with zero step?");
14036 // There is a temptation to just call getAddExpr(this, getStepRecurrence(SE)),
14037 // but in this case we cannot guarantee that the value returned will be an
14038 // AddRec because SCEV does not have a fixed point where it stops
14039 // simplification: it is legal to return ({rec1} + {rec2}). For example, it
14040 // may happen if we reach arithmetic depth limit while simplifying. So we
14041 // construct the returned value explicitly.
14042 SmallVector<SCEVUse, 3> Ops;
14043 // If this is {A,+,B,+,C,...,+,N}, then its step is {B,+,C,+,...,+,N}, and
14044 // (this + Step) is {A+B,+,B+C,+...,+,N}.
14045 for (unsigned i = 0, e = getNumOperands() - 1; i < e; ++i)
14046 Ops.push_back(Elt: SE.getAddExpr(LHS: getOperand(i), RHS: getOperand(i: i + 1)));
14047 // We know that the last operand is not a constant zero (otherwise it would
14048 // have been popped out earlier). This guarantees us that if the result has
14049 // the same last operand, then it will also not be popped out, meaning that
14050 // the returned value will be an AddRec.
14051 const SCEV *Last = getOperand(i: getNumOperands() - 1);
14052 assert(!Last->isZero() && "Recurrency with zero step?");
14053 Ops.push_back(Elt: Last);
14054 return cast<SCEVAddRecExpr>(Val: SE.getAddRecExpr(Operands&: Ops, L: getLoop(),
14055 NWFlags: SCEV::FlagAnyWrap));
14056}
14057
14058// Return true when S contains at least an undef value.
14059bool ScalarEvolution::containsUndefs(const SCEV *S) const {
14060 return SCEVExprContains(
14061 Root: S, Pred: [](const SCEV *S) { return match(S, P: m_scev_UndefOrPoison()); });
14062}
14063
14064// Return true when S contains a value that is a nullptr.
14065bool ScalarEvolution::containsErasedValue(const SCEV *S) const {
14066 return SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
14067 if (const auto *SU = dyn_cast<SCEVUnknown>(Val: S))
14068 return SU->getValue() == nullptr;
14069 return false;
14070 });
14071}
14072
14073/// Return the size of an element read or written by Inst.
14074const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) {
14075 if (!isa<LoadInst, StoreInst>(Val: Inst))
14076 return nullptr;
14077 Type *ETy = getEffectiveSCEVType(Ty: getLoadStorePointerOperand(V: Inst)->getType());
14078 return getSizeOfExpr(IntTy: ETy, AllocTy: getLoadStoreType(I: Inst));
14079}
14080
14081//===----------------------------------------------------------------------===//
14082// SCEVCallbackVH Class Implementation
14083//===----------------------------------------------------------------------===//
14084
14085void ScalarEvolution::SCEVCallbackVH::deleted() {
14086 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
14087 if (PHINode *PN = dyn_cast<PHINode>(Val: getValPtr()))
14088 SE->ConstantEvolutionLoopExitValue.erase(Val: PN);
14089 SE->eraseValueFromMap(V: getValPtr());
14090 // this now dangles!
14091}
14092
14093void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
14094 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
14095
14096 // Forget all the expressions associated with users of the old value,
14097 // so that future queries will recompute the expressions using the new
14098 // value.
14099 SE->forgetValue(V: getValPtr());
14100 // this now dangles!
14101}
14102
14103ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
14104 : CallbackVH(V), SE(se) {}
14105
14106//===----------------------------------------------------------------------===//
14107// ScalarEvolution Class Implementation
14108//===----------------------------------------------------------------------===//
14109
14110ScalarEvolution::ScalarEvolution(Function &F, TargetLibraryInfo &TLI,
14111 AssumptionCache &AC, DominatorTree &DT,
14112 LoopInfo &LI)
14113 : F(F), DL(F.getDataLayout()), TLI(TLI), AC(AC), DT(DT), LI(LI),
14114 CouldNotCompute(new SCEVCouldNotCompute()), ValuesAtScopes(64),
14115 LoopDispositions(64), BlockDispositions(64) {
14116 // To use guards for proving predicates, we need to scan every instruction in
14117 // relevant basic blocks, and not just terminators. Doing this is a waste of
14118 // time if the IR does not actually contain any calls to
14119 // @llvm.experimental.guard, so do a quick check and remember this beforehand.
14120 //
14121 // This pessimizes the case where a pass that preserves ScalarEvolution wants
14122 // to _add_ guards to the module when there weren't any before, and wants
14123 // ScalarEvolution to optimize based on those guards. For now we prefer to be
14124 // efficient in lieu of being smart in that rather obscure case.
14125
14126 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
14127 M: F.getParent(), id: Intrinsic::experimental_guard);
14128 HasGuards = GuardDecl && !GuardDecl->use_empty();
14129}
14130
14131ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg)
14132 : F(Arg.F), DL(Arg.DL), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC),
14133 DT(Arg.DT), LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)),
14134 ValueExprMap(std::move(Arg.ValueExprMap)),
14135 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
14136 PendingMerges(std::move(Arg.PendingMerges)),
14137 ConstantMultipleCache(std::move(Arg.ConstantMultipleCache)),
14138 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
14139 PredicatedBackedgeTakenCounts(
14140 std::move(Arg.PredicatedBackedgeTakenCounts)),
14141 BECountUsers(std::move(Arg.BECountUsers)),
14142 ConstantEvolutionLoopExitValue(
14143 std::move(Arg.ConstantEvolutionLoopExitValue)),
14144 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
14145 ValuesAtScopesUsers(std::move(Arg.ValuesAtScopesUsers)),
14146 LoopDispositions(std::move(Arg.LoopDispositions)),
14147 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
14148 BlockDispositions(std::move(Arg.BlockDispositions)),
14149 SCEVUsers(std::move(Arg.SCEVUsers)),
14150 UnsignedRanges(std::move(Arg.UnsignedRanges)),
14151 SignedRanges(std::move(Arg.SignedRanges)),
14152 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
14153 UniquePreds(std::move(Arg.UniquePreds)),
14154 SCEVAllocator(std::move(Arg.SCEVAllocator)),
14155 ConstantSCEVs(std::move(Arg.ConstantSCEVs)),
14156 LoopUsers(std::move(Arg.LoopUsers)),
14157 PredicatedSCEVRewrites(std::move(Arg.PredicatedSCEVRewrites)),
14158 FirstUnknown(Arg.FirstUnknown) {
14159 Arg.FirstUnknown = nullptr;
14160}
14161
14162ScalarEvolution::~ScalarEvolution() {
14163 // Iterate through all the SCEVUnknown instances and call their
14164 // destructors, so that they release their references to their values.
14165 for (SCEVUnknown *U = FirstUnknown; U;) {
14166 SCEVUnknown *Tmp = U;
14167 U = U->Next;
14168 Tmp->~SCEVUnknown();
14169 }
14170 FirstUnknown = nullptr;
14171
14172 ExprValueMap.clear();
14173 ValueExprMap.clear();
14174 HasRecMap.clear();
14175 BackedgeTakenCounts.clear();
14176 PredicatedBackedgeTakenCounts.clear();
14177
14178 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage");
14179 assert(PendingMerges.empty() && "isImpliedViaMerge garbage");
14180 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!");
14181 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!");
14182}
14183
14184bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) {
14185 return !isa<SCEVCouldNotCompute>(Val: getBackedgeTakenCount(L));
14186}
14187
14188/// When printing a top-level SCEV for trip counts, it's helpful to include
14189/// a type for constants which are otherwise hard to disambiguate.
14190static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV* S) {
14191 if (isa<SCEVConstant>(Val: S))
14192 OS << *S->getType() << " ";
14193 OS << *S;
14194}
14195
14196static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE,
14197 const Loop *L) {
14198 // Print all inner loops first
14199 for (Loop *I : *L)
14200 PrintLoopInfo(OS, SE, L: I);
14201
14202 OS << "Loop ";
14203 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14204 OS << ": ";
14205
14206 SmallVector<BasicBlock *, 8> ExitingBlocks;
14207 L->getExitingBlocks(ExitingBlocks);
14208 if (ExitingBlocks.size() != 1)
14209 OS << "<multiple exits> ";
14210
14211 auto *BTC = SE->getBackedgeTakenCount(L);
14212 if (!isa<SCEVCouldNotCompute>(Val: BTC)) {
14213 OS << "backedge-taken count is ";
14214 PrintSCEVWithTypeHint(OS, S: BTC);
14215 } else
14216 OS << "Unpredictable backedge-taken count.";
14217 OS << "\n";
14218
14219 if (ExitingBlocks.size() > 1)
14220 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14221 OS << " exit count for " << ExitingBlock->getName() << ": ";
14222 const SCEV *EC = SE->getExitCount(L, ExitingBlock);
14223 PrintSCEVWithTypeHint(OS, S: EC);
14224 if (isa<SCEVCouldNotCompute>(Val: EC)) {
14225 // Retry with predicates.
14226 SmallVector<const SCEVPredicate *> Predicates;
14227 EC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates);
14228 if (!isa<SCEVCouldNotCompute>(Val: EC)) {
14229 OS << "\n predicated exit count for " << ExitingBlock->getName()
14230 << ": ";
14231 PrintSCEVWithTypeHint(OS, S: EC);
14232 OS << "\n Predicates:\n";
14233 for (const auto *P : Predicates)
14234 P->print(OS, Depth: 4);
14235 }
14236 }
14237 OS << "\n";
14238 }
14239
14240 OS << "Loop ";
14241 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14242 OS << ": ";
14243
14244 auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L);
14245 if (!isa<SCEVCouldNotCompute>(Val: ConstantBTC)) {
14246 OS << "constant max backedge-taken count is ";
14247 PrintSCEVWithTypeHint(OS, S: ConstantBTC);
14248 if (SE->isBackedgeTakenCountMaxOrZero(L))
14249 OS << ", actual taken count either this or zero.";
14250 } else {
14251 OS << "Unpredictable constant max backedge-taken count. ";
14252 }
14253
14254 OS << "\n"
14255 "Loop ";
14256 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14257 OS << ": ";
14258
14259 auto *SymbolicBTC = SE->getSymbolicMaxBackedgeTakenCount(L);
14260 if (!isa<SCEVCouldNotCompute>(Val: SymbolicBTC)) {
14261 OS << "symbolic max backedge-taken count is ";
14262 PrintSCEVWithTypeHint(OS, S: SymbolicBTC);
14263 if (SE->isBackedgeTakenCountMaxOrZero(L))
14264 OS << ", actual taken count either this or zero.";
14265 } else {
14266 OS << "Unpredictable symbolic max backedge-taken count. ";
14267 }
14268 OS << "\n";
14269
14270 if (ExitingBlocks.size() > 1)
14271 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14272 OS << " symbolic max exit count for " << ExitingBlock->getName() << ": ";
14273 auto *ExitBTC = SE->getExitCount(L, ExitingBlock,
14274 Kind: ScalarEvolution::SymbolicMaximum);
14275 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14276 if (isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14277 // Retry with predicates.
14278 SmallVector<const SCEVPredicate *> Predicates;
14279 ExitBTC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates,
14280 Kind: ScalarEvolution::SymbolicMaximum);
14281 if (!isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14282 OS << "\n predicated symbolic max exit count for "
14283 << ExitingBlock->getName() << ": ";
14284 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14285 OS << "\n Predicates:\n";
14286 for (const auto *P : Predicates)
14287 P->print(OS, Depth: 4);
14288 }
14289 }
14290 OS << "\n";
14291 }
14292
14293 SmallVector<const SCEVPredicate *, 4> Preds;
14294 auto *PBT = SE->getPredicatedBackedgeTakenCount(L, Preds);
14295 if (PBT != BTC) {
14296 OS << "Loop ";
14297 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14298 OS << ": ";
14299 if (!isa<SCEVCouldNotCompute>(Val: PBT)) {
14300 OS << "Predicated backedge-taken count is ";
14301 PrintSCEVWithTypeHint(OS, S: PBT);
14302 } else
14303 OS << "Unpredictable predicated backedge-taken count.";
14304 OS << "\n";
14305 OS << " Predicates:\n";
14306 for (const auto *P : Preds)
14307 P->print(OS, Depth: 4);
14308 }
14309 Preds.clear();
14310
14311 auto *PredConstantMax =
14312 SE->getPredicatedConstantMaxBackedgeTakenCount(L, Preds);
14313 if (PredConstantMax != ConstantBTC) {
14314 OS << "Loop ";
14315 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14316 OS << ": ";
14317 if (!isa<SCEVCouldNotCompute>(Val: PredConstantMax)) {
14318 OS << "Predicated constant max backedge-taken count is ";
14319 PrintSCEVWithTypeHint(OS, S: PredConstantMax);
14320 } else
14321 OS << "Unpredictable predicated constant max backedge-taken count.";
14322 OS << "\n";
14323 OS << " Predicates:\n";
14324 for (const auto *P : Preds)
14325 P->print(OS, Depth: 4);
14326 }
14327 Preds.clear();
14328
14329 auto *PredSymbolicMax =
14330 SE->getPredicatedSymbolicMaxBackedgeTakenCount(L, Preds);
14331 if (SymbolicBTC != PredSymbolicMax) {
14332 OS << "Loop ";
14333 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14334 OS << ": ";
14335 if (!isa<SCEVCouldNotCompute>(Val: PredSymbolicMax)) {
14336 OS << "Predicated symbolic max backedge-taken count is ";
14337 PrintSCEVWithTypeHint(OS, S: PredSymbolicMax);
14338 } else
14339 OS << "Unpredictable predicated symbolic max backedge-taken count.";
14340 OS << "\n";
14341 OS << " Predicates:\n";
14342 for (const auto *P : Preds)
14343 P->print(OS, Depth: 4);
14344 }
14345
14346 if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
14347 OS << "Loop ";
14348 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14349 OS << ": ";
14350 OS << "Trip multiple is " << SE->getSmallConstantTripMultiple(L) << "\n";
14351 }
14352}
14353
14354namespace llvm {
14355// Note: these overloaded operators need to be in the llvm namespace for them
14356// to be resolved correctly. If we put them outside the llvm namespace, the
14357//
14358// OS << ": " << SE.getLoopDisposition(SV, InnerL);
14359//
14360// code below "breaks" and start printing raw enum values as opposed to the
14361// string values.
14362static raw_ostream &operator<<(raw_ostream &OS,
14363 ScalarEvolution::LoopDisposition LD) {
14364 switch (LD) {
14365 case ScalarEvolution::LoopVariant:
14366 OS << "Variant";
14367 break;
14368 case ScalarEvolution::LoopInvariant:
14369 OS << "Invariant";
14370 break;
14371 case ScalarEvolution::LoopUniform:
14372 OS << "Uniform";
14373 break;
14374 case ScalarEvolution::LoopComputable:
14375 OS << "Computable";
14376 break;
14377 }
14378 return OS;
14379}
14380
14381static raw_ostream &operator<<(raw_ostream &OS,
14382 llvm::ScalarEvolution::BlockDisposition BD) {
14383 switch (BD) {
14384 case ScalarEvolution::DoesNotDominateBlock:
14385 OS << "DoesNotDominate";
14386 break;
14387 case ScalarEvolution::DominatesBlock:
14388 OS << "Dominates";
14389 break;
14390 case ScalarEvolution::ProperlyDominatesBlock:
14391 OS << "ProperlyDominates";
14392 break;
14393 }
14394 return OS;
14395}
14396} // namespace llvm
14397
14398void ScalarEvolution::print(raw_ostream &OS) const {
14399 // ScalarEvolution's implementation of the print method is to print
14400 // out SCEV values of all instructions that are interesting. Doing
14401 // this potentially causes it to create new SCEV objects though,
14402 // which technically conflicts with the const qualifier. This isn't
14403 // observable from outside the class though, so casting away the
14404 // const isn't dangerous.
14405 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14406
14407 if (ClassifyExpressions) {
14408 OS << "Classifying expressions for: ";
14409 F.printAsOperand(O&: OS, /*PrintType=*/false);
14410 OS << "\n";
14411 for (Instruction &I : instructions(F))
14412 if (isSCEVable(Ty: I.getType()) && !isa<CmpInst>(Val: I)) {
14413 OS << I << '\n';
14414 OS << " --> ";
14415 const SCEV *SV = SE.getSCEV(V: &I);
14416 SV->print(OS);
14417 if (!isa<SCEVCouldNotCompute>(Val: SV)) {
14418 OS << " U: ";
14419 SE.getUnsignedRange(S: SV).print(OS);
14420 OS << " S: ";
14421 SE.getSignedRange(S: SV).print(OS);
14422 }
14423
14424 const Loop *L = LI.getLoopFor(BB: I.getParent());
14425
14426 SCEVUse AtUse = SE.getSCEVAtScope(V: SV, L);
14427 if (AtUse != SV) {
14428 OS << " --> ";
14429 OS << AtUse;
14430 if (!isa<SCEVCouldNotCompute>(Val: AtUse)) {
14431 OS << " U: ";
14432 SE.getUnsignedRange(S: AtUse).print(OS);
14433 OS << " S: ";
14434 SE.getSignedRange(S: AtUse).print(OS);
14435 }
14436 }
14437
14438 if (L) {
14439 OS << "\t\t" "Exits: ";
14440 SCEVUse ExitValue = SE.getSCEVAtScope(V: SV, L: L->getParentLoop());
14441 if (!SE.isLoopInvariant(S: ExitValue, L)) {
14442 OS << "<<Unknown>>";
14443 } else {
14444 OS << ExitValue;
14445 }
14446
14447 ListSeparator LS(", ", "\t\tLoopDispositions: { ");
14448 for (const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
14449 OS << LS;
14450 Iter->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14451 OS << ": " << SE.getLoopDisposition(S: SV, L: Iter);
14452 }
14453
14454 for (const auto *InnerL : depth_first(G: L)) {
14455 if (InnerL == L)
14456 continue;
14457 OS << LS;
14458 InnerL->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14459 OS << ": " << SE.getLoopDisposition(S: SV, L: InnerL);
14460 }
14461
14462 OS << " }";
14463 }
14464
14465 OS << "\n";
14466 }
14467 }
14468
14469 OS << "Determining loop execution counts for: ";
14470 F.printAsOperand(O&: OS, /*PrintType=*/false);
14471 OS << "\n";
14472 for (Loop *I : LI)
14473 PrintLoopInfo(OS, SE: &SE, L: I);
14474}
14475
14476ScalarEvolution::LoopDisposition
14477ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) {
14478 auto &Values = LoopDispositions[S];
14479 for (auto &V : Values) {
14480 if (V.getPointer() == L)
14481 return V.getInt();
14482 }
14483 Values.emplace_back(Args&: L, Args: LoopVariant);
14484 LoopDisposition D = computeLoopDisposition(S, L);
14485 auto &Values2 = LoopDispositions[S];
14486 for (auto &V : llvm::reverse(C&: Values2)) {
14487 if (V.getPointer() == L) {
14488 V.setInt(D);
14489 break;
14490 }
14491 }
14492 return D;
14493}
14494
14495ScalarEvolution::LoopDisposition
14496ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
14497 switch (S->getSCEVType()) {
14498 case scConstant:
14499 case scVScale:
14500 return LoopInvariant;
14501 case scAddRecExpr: {
14502 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14503
14504 // If L is the addrec's loop, it's computable.
14505 if (AR->getLoop() == L)
14506 return LoopComputable;
14507
14508 // Add recurrences are never invariant in the function-body (null loop).
14509 if (!L)
14510 return LoopVariant;
14511
14512 // Everything that is not defined at loop entry is variant.
14513 if (DT.dominates(A: L->getHeader(), B: AR->getLoop()->getHeader())) {
14514 if (L->contains(L: AR->getLoop()) &&
14515 llvm::all_of(Range: AR->operands(),
14516 P: [&](const SCEV *Op) { return isLoopUniform(S: Op, L); }))
14517 return LoopUniform;
14518
14519 return LoopVariant;
14520 }
14521 assert(!L->contains(AR->getLoop()) && "Containing loop's header does not"
14522 " dominate the contained loop's header?");
14523
14524 // This recurrence is invariant w.r.t. L if AR's loop contains L.
14525 if (AR->getLoop()->contains(L))
14526 return LoopInvariant;
14527
14528 // This recurrence is variant w.r.t. L if any of its operands
14529 // are variant.
14530 for (SCEVUse Op : AR->operands())
14531 if (!isLoopInvariant(S: Op, L))
14532 return LoopVariant;
14533
14534 // Otherwise it's loop-invariant.
14535 return LoopInvariant;
14536 }
14537 case scTruncate:
14538 case scZeroExtend:
14539 case scSignExtend:
14540 case scPtrToAddr:
14541 case scAddExpr:
14542 case scMulExpr:
14543 case scUDivExpr:
14544 case scUMaxExpr:
14545 case scSMaxExpr:
14546 case scUMinExpr:
14547 case scSMinExpr:
14548 case scSequentialUMinExpr: {
14549 bool HasVarying = false;
14550 bool HasUniform = false;
14551 for (SCEVUse Op : S->operands()) {
14552 LoopDisposition D = getLoopDisposition(S: Op, L);
14553 if (D == LoopVariant)
14554 return LoopVariant;
14555 if (D == LoopComputable)
14556 HasVarying = true;
14557 if (D == LoopUniform)
14558 HasUniform = true;
14559 }
14560 return HasVarying ? (HasUniform ? LoopVariant : LoopComputable)
14561 : (HasUniform ? LoopUniform : LoopInvariant);
14562 }
14563 case scUnknown:
14564 // All non-instruction values are loop invariant. All instructions are loop
14565 // invariant if they are not contained in the specified loop.
14566 // Instructions are never considered invariant in the function body
14567 // (null loop) because they are defined within the "loop".
14568 if (auto *I = dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue()))
14569 return (L && !L->contains(Inst: I)) ? LoopInvariant : LoopVariant;
14570 return LoopInvariant;
14571 case scCouldNotCompute:
14572 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14573 }
14574 llvm_unreachable("Unknown SCEV kind!");
14575}
14576
14577bool ScalarEvolution::isLoopUniform(const SCEV *S, const Loop *L) {
14578 LoopDisposition D = getLoopDisposition(S, L);
14579 return D == LoopUniform || D == LoopInvariant;
14580}
14581
14582bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) {
14583 return getLoopDisposition(S, L) == LoopInvariant;
14584}
14585
14586bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) {
14587 return getLoopDisposition(S, L) == LoopComputable;
14588}
14589
14590ScalarEvolution::BlockDisposition
14591ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14592 auto &Values = BlockDispositions[S];
14593 for (auto &V : Values) {
14594 if (V.getPointer() == BB)
14595 return V.getInt();
14596 }
14597 Values.emplace_back(Args&: BB, Args: DoesNotDominateBlock);
14598 BlockDisposition D = computeBlockDisposition(S, BB);
14599 auto &Values2 = BlockDispositions[S];
14600 for (auto &V : llvm::reverse(C&: Values2)) {
14601 if (V.getPointer() == BB) {
14602 V.setInt(D);
14603 break;
14604 }
14605 }
14606 return D;
14607}
14608
14609ScalarEvolution::BlockDisposition
14610ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14611 switch (S->getSCEVType()) {
14612 case scConstant:
14613 case scVScale:
14614 return ProperlyDominatesBlock;
14615 case scAddRecExpr: {
14616 // This uses a "dominates" query instead of "properly dominates" query
14617 // to test for proper dominance too, because the instruction which
14618 // produces the addrec's value is a PHI, and a PHI effectively properly
14619 // dominates its entire containing block.
14620 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14621 if (!DT.dominates(A: AR->getLoop()->getHeader(), B: BB))
14622 return DoesNotDominateBlock;
14623
14624 // Fall through into SCEVNAryExpr handling.
14625 [[fallthrough]];
14626 }
14627 case scTruncate:
14628 case scZeroExtend:
14629 case scSignExtend:
14630 case scPtrToAddr:
14631 case scAddExpr:
14632 case scMulExpr:
14633 case scUDivExpr:
14634 case scUMaxExpr:
14635 case scSMaxExpr:
14636 case scUMinExpr:
14637 case scSMinExpr:
14638 case scSequentialUMinExpr: {
14639 bool Proper = true;
14640 for (const SCEV *NAryOp : S->operands()) {
14641 BlockDisposition D = getBlockDisposition(S: NAryOp, BB);
14642 if (D == DoesNotDominateBlock)
14643 return DoesNotDominateBlock;
14644 if (D == DominatesBlock)
14645 Proper = false;
14646 }
14647 return Proper ? ProperlyDominatesBlock : DominatesBlock;
14648 }
14649 case scUnknown:
14650 if (Instruction *I =
14651 dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue())) {
14652 if (I->getParent() == BB)
14653 return DominatesBlock;
14654 if (DT.properlyDominates(A: I->getParent(), B: BB))
14655 return ProperlyDominatesBlock;
14656 return DoesNotDominateBlock;
14657 }
14658 return ProperlyDominatesBlock;
14659 case scCouldNotCompute:
14660 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14661 }
14662 llvm_unreachable("Unknown SCEV kind!");
14663}
14664
14665bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
14666 return getBlockDisposition(S, BB) >= DominatesBlock;
14667}
14668
14669bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) {
14670 return getBlockDisposition(S, BB) == ProperlyDominatesBlock;
14671}
14672
14673bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const {
14674 return SCEVExprContains(Root: S, Pred: [&](const SCEV *Expr) { return Expr == Op; });
14675}
14676
14677void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
14678 bool Predicated) {
14679 auto &BECounts =
14680 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14681 auto It = BECounts.find(Val: L);
14682 if (It != BECounts.end()) {
14683 for (const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
14684 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14685 if (!isa<SCEVConstant>(Val: S)) {
14686 auto UserIt = BECountUsers.find(Val: S);
14687 assert(UserIt != BECountUsers.end());
14688 UserIt->second.erase(Ptr: {L, Predicated});
14689 }
14690 }
14691 }
14692 BECounts.erase(I: It);
14693 }
14694}
14695
14696void ScalarEvolution::forgetMemoizedResults(ArrayRef<SCEVUse> SCEVs) {
14697 SmallPtrSet<const SCEV *, 8> ToForget(llvm::from_range, SCEVs);
14698 SmallVector<SCEVUse, 8> Worklist(ToForget.begin(), ToForget.end());
14699
14700 while (!Worklist.empty()) {
14701 const SCEV *Curr = Worklist.pop_back_val();
14702 auto Users = SCEVUsers.find(Val: Curr);
14703 if (Users != SCEVUsers.end())
14704 for (const auto *User : Users->second)
14705 if (ToForget.insert(Ptr: User).second)
14706 Worklist.push_back(Elt: User);
14707 }
14708
14709 for (const auto *S : ToForget)
14710 forgetMemoizedResultsImpl(S);
14711
14712 PredicatedSCEVRewrites.remove_if(
14713 Pred: [&](const auto &Entry) { return ToForget.count(Ptr: Entry.first.first); });
14714}
14715
14716void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) {
14717 LoopDispositions.erase(Val: S);
14718 BlockDispositions.erase(Val: S);
14719 UnsignedRanges.erase(Val: S);
14720 SignedRanges.erase(Val: S);
14721 HasRecMap.erase(Val: S);
14722 ConstantMultipleCache.erase(Val: S);
14723
14724 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
14725 UnsignedWrapViaInductionTried.erase(Ptr: AR);
14726 SignedWrapViaInductionTried.erase(Ptr: AR);
14727 }
14728
14729 auto ExprIt = ExprValueMap.find(Val: S);
14730 if (ExprIt != ExprValueMap.end()) {
14731 for (Value *V : ExprIt->second) {
14732 auto ValueIt = ValueExprMap.find_as(Val: V);
14733 if (ValueIt != ValueExprMap.end())
14734 ValueExprMap.erase(I: ValueIt);
14735 }
14736 ExprValueMap.erase(I: ExprIt);
14737 }
14738
14739 auto ScopeIt = ValuesAtScopes.find(Val: S);
14740 if (ScopeIt != ValuesAtScopes.end()) {
14741 for (const auto &Pair : ScopeIt->second)
14742 if (!isa_and_nonnull<SCEVConstant>(Val: Pair.second))
14743 llvm::erase(C&: ValuesAtScopesUsers[Pair.second.getPointer()],
14744 V: std::make_pair(x: Pair.first, y&: S));
14745 ValuesAtScopes.erase(I: ScopeIt);
14746 }
14747
14748 auto ScopeUserIt = ValuesAtScopesUsers.find(Val: S);
14749 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
14750 for (const auto &Pair : ScopeUserIt->second)
14751 // The recorded value at scope is a use of S, which may carry no-wrap
14752 // flags that are not part of this key.
14753 llvm::erase_if(C&: ValuesAtScopes[Pair.second], P: [&](const auto &LS) {
14754 return LS.first == Pair.first && LS.second.getPointer() == S;
14755 });
14756 ValuesAtScopesUsers.erase(I: ScopeUserIt);
14757 }
14758
14759 auto BEUsersIt = BECountUsers.find(Val: S);
14760 if (BEUsersIt != BECountUsers.end()) {
14761 // Work on a copy, as forgetBackedgeTakenCounts() will modify the original.
14762 auto Copy = BEUsersIt->second;
14763 for (const auto &Pair : Copy)
14764 forgetBackedgeTakenCounts(L: Pair.getPointer(), Predicated: Pair.getInt());
14765 BECountUsers.erase(I: BEUsersIt);
14766 }
14767
14768 auto FoldUser = FoldCacheUser.find(Val: S);
14769 if (FoldUser != FoldCacheUser.end())
14770 for (auto &KV : FoldUser->second)
14771 FoldCache.erase(Val: KV);
14772 FoldCacheUser.erase(Val: S);
14773}
14774
14775void
14776ScalarEvolution::getUsedLoops(const SCEV *S,
14777 SmallPtrSetImpl<const Loop *> &LoopsUsed) {
14778 struct FindUsedLoops {
14779 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
14780 : LoopsUsed(LoopsUsed) {}
14781 SmallPtrSetImpl<const Loop *> &LoopsUsed;
14782 bool follow(const SCEV *S) {
14783 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S))
14784 LoopsUsed.insert(Ptr: AR->getLoop());
14785 return true;
14786 }
14787
14788 bool isDone() const { return false; }
14789 };
14790
14791 FindUsedLoops F(LoopsUsed);
14792 SCEVTraversal<FindUsedLoops>(F).visitAll(Root: S);
14793}
14794
14795void ScalarEvolution::getReachableBlocks(
14796 SmallPtrSetImpl<BasicBlock *> &Reachable, Function &F) {
14797 SmallVector<BasicBlock *> Worklist;
14798 Worklist.push_back(Elt: &F.getEntryBlock());
14799 while (!Worklist.empty()) {
14800 BasicBlock *BB = Worklist.pop_back_val();
14801 if (!Reachable.insert(Ptr: BB).second)
14802 continue;
14803
14804 Value *Cond;
14805 BasicBlock *TrueBB, *FalseBB;
14806 if (match(V: BB->getTerminator(), P: m_Br(C: m_Value(V&: Cond), T: m_BasicBlock(V&: TrueBB),
14807 F: m_BasicBlock(V&: FalseBB)))) {
14808 if (auto *C = dyn_cast<ConstantInt>(Val: Cond)) {
14809 Worklist.push_back(Elt: C->isOne() ? TrueBB : FalseBB);
14810 continue;
14811 }
14812
14813 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
14814 const SCEV *L = getSCEV(V: Cmp->getOperand(i_nocapture: 0));
14815 const SCEV *R = getSCEV(V: Cmp->getOperand(i_nocapture: 1));
14816 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getCmpPredicate(), LHS: L, RHS: R)) {
14817 Worklist.push_back(Elt: TrueBB);
14818 continue;
14819 }
14820 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getInverseCmpPredicate(), LHS: L,
14821 RHS: R)) {
14822 Worklist.push_back(Elt: FalseBB);
14823 continue;
14824 }
14825 }
14826 }
14827
14828 append_range(C&: Worklist, R: successors(BB));
14829 }
14830}
14831
14832void ScalarEvolution::verify() const {
14833 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14834 ScalarEvolution SE2(F, TLI, AC, DT, LI);
14835
14836 SmallVector<Loop *, 8> LoopStack(LI.begin(), LI.end());
14837
14838 // Map's SCEV expressions from one ScalarEvolution "universe" to another.
14839 struct SCEVMapper : public SCEVRewriteVisitor<SCEVMapper> {
14840 SCEVMapper(ScalarEvolution &SE) : SCEVRewriteVisitor<SCEVMapper>(SE) {}
14841
14842 const SCEV *visitConstant(const SCEVConstant *Constant) {
14843 return SE.getConstant(Val: Constant->getAPInt());
14844 }
14845
14846 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14847 return SE.getUnknown(V: Expr->getValue());
14848 }
14849
14850 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
14851 return SE.getCouldNotCompute();
14852 }
14853 };
14854
14855 SCEVMapper SCM(SE2);
14856 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
14857 SE2.getReachableBlocks(Reachable&: ReachableBlocks, F);
14858
14859 auto GetDelta = [&](const SCEV *Old, const SCEV *New) -> const SCEV * {
14860 if (containsUndefs(S: Old) || containsUndefs(S: New)) {
14861 // SCEV treats "undef" as an unknown but consistent value (i.e. it does
14862 // not propagate undef aggressively). This means we can (and do) fail
14863 // verification in cases where a transform makes a value go from "undef"
14864 // to "undef+1" (say). The transform is fine, since in both cases the
14865 // result is "undef", but SCEV thinks the value increased by 1.
14866 return nullptr;
14867 }
14868
14869 // Unless VerifySCEVStrict is set, we only compare constant deltas.
14870 const SCEV *Delta = SE2.getMinusSCEV(LHS: Old, RHS: New);
14871 if (!VerifySCEVStrict && !isa<SCEVConstant>(Val: Delta))
14872 return nullptr;
14873
14874 return Delta;
14875 };
14876
14877 while (!LoopStack.empty()) {
14878 auto *L = LoopStack.pop_back_val();
14879 llvm::append_range(C&: LoopStack, R&: *L);
14880
14881 // Only verify BECounts in reachable loops. For an unreachable loop,
14882 // any BECount is legal.
14883 if (!ReachableBlocks.contains(Ptr: L->getHeader()))
14884 continue;
14885
14886 // Only verify cached BECounts. Computing new BECounts may change the
14887 // results of subsequent SCEV uses.
14888 auto It = BackedgeTakenCounts.find(Val: L);
14889 if (It == BackedgeTakenCounts.end())
14890 continue;
14891
14892 auto *CurBECount =
14893 SCM.visit(S: It->second.getExact(L, SE: const_cast<ScalarEvolution *>(this)));
14894 auto *NewBECount = SE2.getBackedgeTakenCount(L);
14895
14896 if (CurBECount == SE2.getCouldNotCompute() ||
14897 NewBECount == SE2.getCouldNotCompute()) {
14898 // NB! This situation is legal, but is very suspicious -- whatever pass
14899 // change the loop to make a trip count go from could not compute to
14900 // computable or vice-versa *should have* invalidated SCEV. However, we
14901 // choose not to assert here (for now) since we don't want false
14902 // positives.
14903 continue;
14904 }
14905
14906 if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) >
14907 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14908 NewBECount = SE2.getZeroExtendExpr(Op: NewBECount, Ty: CurBECount->getType());
14909 else if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) <
14910 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14911 CurBECount = SE2.getZeroExtendExpr(Op: CurBECount, Ty: NewBECount->getType());
14912
14913 const SCEV *Delta = GetDelta(CurBECount, NewBECount);
14914 if (Delta && !Delta->isZero()) {
14915 dbgs() << "Trip Count for " << *L << " Changed!\n";
14916 dbgs() << "Old: " << *CurBECount << "\n";
14917 dbgs() << "New: " << *NewBECount << "\n";
14918 dbgs() << "Delta: " << *Delta << "\n";
14919 std::abort();
14920 }
14921 }
14922
14923 // Collect all valid loops currently in LoopInfo.
14924 SmallPtrSet<Loop *, 32> ValidLoops;
14925 SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end());
14926 while (!Worklist.empty()) {
14927 Loop *L = Worklist.pop_back_val();
14928 if (ValidLoops.insert(Ptr: L).second)
14929 Worklist.append(in_start: L->begin(), in_end: L->end());
14930 }
14931 for (const auto &KV : ValueExprMap) {
14932#ifndef NDEBUG
14933 // Check for SCEV expressions referencing invalid/deleted loops.
14934 if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
14935 assert(ValidLoops.contains(AR->getLoop()) &&
14936 "AddRec references invalid loop");
14937 }
14938#endif
14939
14940 // Check that the value is also part of the reverse map.
14941 auto It = ExprValueMap.find(Val: KV.second);
14942 if (It == ExprValueMap.end() || !It->second.contains(key: KV.first)) {
14943 dbgs() << "Value " << *KV.first
14944 << " is in ValueExprMap but not in ExprValueMap\n";
14945 std::abort();
14946 }
14947
14948 if (auto *I = dyn_cast<Instruction>(Val: &*KV.first)) {
14949 if (!ReachableBlocks.contains(Ptr: I->getParent()))
14950 continue;
14951 const SCEV *OldSCEV = SCM.visit(S: KV.second);
14952 const SCEV *NewSCEV = SE2.getSCEV(V: I);
14953 const SCEV *Delta = GetDelta(OldSCEV, NewSCEV);
14954 if (Delta && !Delta->isZero()) {
14955 dbgs() << "SCEV for value " << *I << " changed!\n"
14956 << "Old: " << *OldSCEV << "\n"
14957 << "New: " << *NewSCEV << "\n"
14958 << "Delta: " << *Delta << "\n";
14959 std::abort();
14960 }
14961 }
14962 }
14963
14964 for (const auto &KV : ExprValueMap) {
14965 for (Value *V : KV.second) {
14966 const SCEV *S = ValueExprMap.lookup(Val: V);
14967 if (!S) {
14968 dbgs() << "Value " << *V
14969 << " is in ExprValueMap but not in ValueExprMap\n";
14970 std::abort();
14971 }
14972 if (S != KV.first) {
14973 dbgs() << "Value " << *V << " mapped to " << *S << " rather than "
14974 << *KV.first << "\n";
14975 std::abort();
14976 }
14977 }
14978 }
14979
14980 // Verify integrity of SCEV users.
14981 for (const auto &S : UniqueSCEVs) {
14982 for (SCEVUse Op : S.operands()) {
14983 // We do not store dependencies of constants.
14984 if (isa<SCEVConstant>(Val: Op))
14985 continue;
14986 auto It = SCEVUsers.find(Val: Op);
14987 if (It != SCEVUsers.end() && It->second.count(Ptr: &S))
14988 continue;
14989 dbgs() << "Use of operand " << *Op << " by user " << S
14990 << " is not being tracked!\n";
14991 std::abort();
14992 }
14993 }
14994
14995 // Verify integrity of ValuesAtScopes users.
14996 for (const auto &ValueAndVec : ValuesAtScopes) {
14997 const SCEV *Value = ValueAndVec.first;
14998 for (const auto &LoopAndValueAtScope : ValueAndVec.second) {
14999 const Loop *L = LoopAndValueAtScope.first;
15000 SCEVUse ValueAtScope = LoopAndValueAtScope.second;
15001 if (!isa<SCEVConstant>(Val: ValueAtScope)) {
15002 auto It = ValuesAtScopesUsers.find(Val: ValueAtScope.getPointer());
15003 if (It != ValuesAtScopesUsers.end() &&
15004 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: Value)))
15005 continue;
15006 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
15007 << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
15008 std::abort();
15009 }
15010 }
15011 }
15012
15013 for (const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
15014 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
15015 for (const auto &LoopAndValue : ValueAtScopeAndVec.second) {
15016 const Loop *L = LoopAndValue.first;
15017 const SCEV *Value = LoopAndValue.second;
15018 assert(!isa<SCEVConstant>(Value));
15019 auto It = ValuesAtScopes.find(Val: Value);
15020 // The recorded value at scope may carry no-wrap flags that are not part
15021 // of the key it is recorded under.
15022 if (It != ValuesAtScopes.end() && any_of(Range: It->second, P: [&](const auto &LS) {
15023 return LS.first == L && LS.second.getPointer() == ValueAtScope;
15024 }))
15025 continue;
15026 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
15027 << *ValueAtScope << " missing in ValuesAtScopes\n";
15028 std::abort();
15029 }
15030 }
15031
15032 // Verify integrity of BECountUsers.
15033 auto VerifyBECountUsers = [&](bool Predicated) {
15034 auto &BECounts =
15035 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
15036 for (const auto &LoopAndBEInfo : BECounts) {
15037 for (const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
15038 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
15039 if (!isa<SCEVConstant>(Val: S)) {
15040 auto UserIt = BECountUsers.find(Val: S);
15041 if (UserIt != BECountUsers.end() &&
15042 UserIt->second.contains(Ptr: { LoopAndBEInfo.first, Predicated }))
15043 continue;
15044 dbgs() << "Value " << *S << " for loop " << *LoopAndBEInfo.first
15045 << " missing from BECountUsers\n";
15046 std::abort();
15047 }
15048 }
15049 }
15050 }
15051 };
15052 VerifyBECountUsers(/* Predicated */ false);
15053 VerifyBECountUsers(/* Predicated */ true);
15054
15055 // Verify intergity of loop disposition cache.
15056 for (auto &[S, Values] : LoopDispositions) {
15057 for (auto [Loop, CachedDisposition] : Values) {
15058 const auto RecomputedDisposition = SE2.getLoopDisposition(S, L: Loop);
15059 if (CachedDisposition != RecomputedDisposition) {
15060 dbgs() << "Cached disposition of " << *S << " for loop " << *Loop
15061 << " is incorrect: cached " << CachedDisposition << ", actual "
15062 << RecomputedDisposition << "\n";
15063 std::abort();
15064 }
15065 }
15066 }
15067
15068 // Verify integrity of the block disposition cache.
15069 for (auto &[S, Values] : BlockDispositions) {
15070 for (auto [BB, CachedDisposition] : Values) {
15071 const auto RecomputedDisposition = SE2.getBlockDisposition(S, BB);
15072 if (CachedDisposition != RecomputedDisposition) {
15073 dbgs() << "Cached disposition of " << *S << " for block %"
15074 << BB->getName() << " is incorrect: cached " << CachedDisposition
15075 << ", actual " << RecomputedDisposition << "\n";
15076 std::abort();
15077 }
15078 }
15079 }
15080
15081 // Verify FoldCache/FoldCacheUser caches.
15082 for (auto [FoldID, Expr] : FoldCache) {
15083 auto I = FoldCacheUser.find(Val: Expr);
15084 if (I == FoldCacheUser.end()) {
15085 dbgs() << "Missing entry in FoldCacheUser for cached expression " << *Expr
15086 << "!\n";
15087 std::abort();
15088 }
15089 if (!is_contained(Range: I->second, Element: FoldID)) {
15090 dbgs() << "Missing FoldID in cached users of " << *Expr << "!\n";
15091 std::abort();
15092 }
15093 }
15094 for (auto [Expr, IDs] : FoldCacheUser) {
15095 for (auto &FoldID : IDs) {
15096 const SCEV *S = FoldCache.lookup(Val: FoldID);
15097 if (!S) {
15098 dbgs() << "Missing entry in FoldCache for expression " << *Expr
15099 << "!\n";
15100 std::abort();
15101 }
15102 if (S != Expr) {
15103 dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: " << *S
15104 << " != " << *Expr << "!\n";
15105 std::abort();
15106 }
15107 }
15108 }
15109
15110 // Verify that ConstantMultipleCache computations are correct. We check that
15111 // cached multiples and recomputed multiples are multiples of each other to
15112 // verify correctness. It is possible that a recomputed multiple is different
15113 // from the cached multiple due to strengthened no wrap flags or changes in
15114 // KnownBits computations.
15115 for (auto [S, Multiple] : ConstantMultipleCache) {
15116 APInt RecomputedMultiple = SE2.getConstantMultiple(S);
15117 if ((Multiple != 0 && RecomputedMultiple != 0 &&
15118 Multiple.urem(RHS: RecomputedMultiple) != 0 &&
15119 RecomputedMultiple.urem(RHS: Multiple) != 0)) {
15120 dbgs() << "Incorrect cached computation in ConstantMultipleCache for "
15121 << *S << " : Computed " << RecomputedMultiple
15122 << " but cache contains " << Multiple << "!\n";
15123 std::abort();
15124 }
15125 }
15126}
15127
15128bool ScalarEvolution::invalidate(
15129 Function &F, const PreservedAnalyses &PA,
15130 FunctionAnalysisManager::Invalidator &Inv) {
15131 // Invalidate the ScalarEvolution object whenever it isn't preserved or one
15132 // of its dependencies is invalidated.
15133 auto PAC = PA.getChecker<ScalarEvolutionAnalysis>();
15134 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) ||
15135 Inv.invalidate<AssumptionAnalysis>(IR&: F, PA) ||
15136 Inv.invalidate<DominatorTreeAnalysis>(IR&: F, PA) ||
15137 Inv.invalidate<LoopAnalysis>(IR&: F, PA);
15138}
15139
15140AnalysisKey ScalarEvolutionAnalysis::Key;
15141
15142ScalarEvolution ScalarEvolutionAnalysis::run(Function &F,
15143 FunctionAnalysisManager &AM) {
15144 auto &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
15145 auto &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
15146 auto &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
15147 auto &LI = AM.getResult<LoopAnalysis>(IR&: F);
15148 return ScalarEvolution(F, TLI, AC, DT, LI);
15149}
15150
15151PreservedAnalyses
15152ScalarEvolutionVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
15153 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).verify();
15154 return PreservedAnalyses::all();
15155}
15156
15157PreservedAnalyses
15158ScalarEvolutionPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
15159 // For compatibility with opt's -analyze feature under legacy pass manager
15160 // which was not ported to NPM. This keeps tests using
15161 // update_analyze_test_checks.py working.
15162 OS << "Printing analysis 'Scalar Evolution Analysis' for function '"
15163 << F.getName() << "':\n";
15164 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).print(OS);
15165 return PreservedAnalyses::all();
15166}
15167
15168INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass, "scalar-evolution",
15169 "Scalar Evolution Analysis", false, true)
15170INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
15171INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
15172INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
15173INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
15174INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution",
15175 "Scalar Evolution Analysis", false, true)
15176
15177char ScalarEvolutionWrapperPass::ID = 0;
15178
15179ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) {}
15180
15181bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) {
15182 SE.reset(p: new ScalarEvolution(
15183 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F),
15184 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
15185 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
15186 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
15187 return false;
15188}
15189
15190void ScalarEvolutionWrapperPass::releaseMemory() { SE.reset(); }
15191
15192void ScalarEvolutionWrapperPass::print(raw_ostream &OS, const Module *) const {
15193 SE->print(OS);
15194}
15195
15196void ScalarEvolutionWrapperPass::verifyAnalysis() const {
15197 if (!VerifySCEV)
15198 return;
15199
15200 SE->verify();
15201}
15202
15203void ScalarEvolutionWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
15204 AU.setPreservesAll();
15205 AU.addRequiredTransitive<AssumptionCacheTracker>();
15206 AU.addRequiredTransitive<LoopInfoWrapperPass>();
15207 AU.addRequiredTransitive<DominatorTreeWrapperPass>();
15208 AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>();
15209}
15210
15211const SCEVPredicate *ScalarEvolution::getEqualPredicate(const SCEV *LHS,
15212 const SCEV *RHS) {
15213 return getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS, RHS);
15214}
15215
15216const SCEVPredicate *
15217ScalarEvolution::getComparePredicate(const ICmpInst::Predicate Pred,
15218 const SCEV *LHS, const SCEV *RHS) {
15219 FoldingSetNodeID ID;
15220 assert(LHS->getType() == RHS->getType() &&
15221 "Type mismatch between LHS and RHS");
15222 // Unique this node based on the arguments
15223 ID.AddInteger(I: SCEVPredicate::P_Compare);
15224 ID.AddInteger(I: Pred);
15225 ID.AddPointer(Ptr: LHS);
15226 ID.AddPointer(Ptr: RHS);
15227 FoldingSetInsertToken Token;
15228 if (const auto *S = UniquePreds.lookup(ID, Token))
15229 return S;
15230 SCEVComparePredicate *Eq = new (SCEVAllocator)
15231 SCEVComparePredicate(ID.Intern(Allocator&: SCEVAllocator), Pred, LHS, RHS);
15232 UniquePreds.insert(N: Eq, Token);
15233 return Eq;
15234}
15235
15236const SCEVPredicate *ScalarEvolution::getWrapPredicate(
15237 const SCEVAddRecExpr *AR,
15238 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15239 FoldingSetNodeID ID;
15240 // Unique this node based on the arguments
15241 ID.AddInteger(I: SCEVPredicate::P_Wrap);
15242 ID.AddPointer(Ptr: AR);
15243 ID.AddInteger(I: AddedFlags);
15244 FoldingSetInsertToken Token;
15245 if (const auto *S = UniquePreds.lookup(ID, Token))
15246 return S;
15247 auto *OF = new (SCEVAllocator)
15248 SCEVWrapPredicate(ID.Intern(Allocator&: SCEVAllocator), AR, AddedFlags);
15249 UniquePreds.insert(N: OF, Token);
15250 return OF;
15251}
15252
15253namespace {
15254
15255class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
15256public:
15257
15258 /// Rewrites \p S in the context of a loop L and the SCEV predication
15259 /// infrastructure.
15260 ///
15261 /// If \p Pred is non-null, the SCEV expression is rewritten to respect the
15262 /// equivalences present in \p Pred.
15263 ///
15264 /// If \p NewPreds is non-null, rewrite is free to add further predicates to
15265 /// \p NewPreds such that the result will be an AddRecExpr.
15266 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
15267 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15268 const SCEVPredicate *Pred) {
15269 SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred);
15270 return Rewriter.visit(S);
15271 }
15272
15273 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
15274 if (Pred) {
15275 if (auto *U = dyn_cast<SCEVUnionPredicate>(Val: Pred)) {
15276 for (const auto *Pred : U->getPredicates())
15277 if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred))
15278 if (IPred->getLHS() == Expr &&
15279 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15280 return IPred->getRHS();
15281 } else if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred)) {
15282 if (IPred->getLHS() == Expr &&
15283 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15284 return IPred->getRHS();
15285 }
15286 }
15287 return convertToAddRecWithPreds(Expr);
15288 }
15289
15290 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
15291 const SCEV *Operand = visit(S: Expr->getOperand());
15292 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15293 if (AR && AR->getLoop() == L && AR->isAffine()) {
15294 // This couldn't be folded because the operand didn't have the nuw
15295 // flag. Add the nusw flag as an assumption that we could make.
15296 const SCEV *Step = AR->getStepRecurrence(SE);
15297 Type *Ty = Expr->getType();
15298 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNUSW))
15299 return SE.getAddRecExpr(Start: SE.getZeroExtendExpr(Op: AR->getStart(), Ty),
15300 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15301 Flags: AR->getNoWrapFlags());
15302 }
15303 return SE.getZeroExtendExpr(Op: Operand, Ty: Expr->getType());
15304 }
15305
15306 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
15307 const SCEV *Operand = visit(S: Expr->getOperand());
15308 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15309 if (AR && AR->getLoop() == L && AR->isAffine()) {
15310 // This couldn't be folded because the operand didn't have the nsw
15311 // flag. Add the nssw flag as an assumption that we could make.
15312 const SCEV *Step = AR->getStepRecurrence(SE);
15313 Type *Ty = Expr->getType();
15314 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNSSW))
15315 return SE.getAddRecExpr(Start: SE.getSignExtendExpr(Op: AR->getStart(), Ty),
15316 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15317 Flags: AR->getNoWrapFlags());
15318 }
15319 return SE.getSignExtendExpr(Op: Operand, Ty: Expr->getType());
15320 }
15321
15322private:
15323 explicit SCEVPredicateRewriter(
15324 const Loop *L, ScalarEvolution &SE,
15325 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15326 const SCEVPredicate *Pred)
15327 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {}
15328
15329 bool addOverflowAssumption(const SCEVPredicate *P) {
15330 if (!NewPreds) {
15331 // Check if we've already made this assumption.
15332 return Pred && Pred->implies(N: P, SE);
15333 }
15334 NewPreds->push_back(Elt: P);
15335 return true;
15336 }
15337
15338 bool addOverflowAssumption(const SCEVAddRecExpr *AR,
15339 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15340 auto *A = SE.getWrapPredicate(AR, AddedFlags);
15341 return addOverflowAssumption(P: A);
15342 }
15343
15344 // If \p Expr represents a PHINode, we try to see if it can be represented
15345 // as an AddRec, possibly under a predicate (PHISCEVPred). If it is possible
15346 // to add this predicate as a runtime overflow check, we return the AddRec.
15347 // If \p Expr does not meet these conditions (is not a PHI node, or we
15348 // couldn't create an AddRec for it, or couldn't add the predicate), we just
15349 // return \p Expr.
15350 const SCEV *convertToAddRecWithPreds(const SCEVUnknown *Expr) {
15351 if (!isa<PHINode>(Val: Expr->getValue()))
15352 return Expr;
15353 std::optional<
15354 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
15355 PredicatedRewrite = SE.createAddRecFromPHIWithCasts(SymbolicPHI: Expr);
15356 if (!PredicatedRewrite)
15357 return Expr;
15358 for (const auto *P : PredicatedRewrite->second){
15359 // Wrap predicates from outer loops are not supported.
15360 if (auto *WP = dyn_cast<const SCEVWrapPredicate>(Val: P)) {
15361 if (L != WP->getExpr()->getLoop())
15362 return Expr;
15363 }
15364 if (!addOverflowAssumption(P))
15365 return Expr;
15366 }
15367 return PredicatedRewrite->first;
15368 }
15369
15370 SmallVectorImpl<const SCEVPredicate *> *NewPreds;
15371 const SCEVPredicate *Pred;
15372 const Loop *L;
15373};
15374
15375} // end anonymous namespace
15376
15377const SCEV *
15378ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L,
15379 const SCEVPredicate &Preds) {
15380 return SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: nullptr, Pred: &Preds);
15381}
15382
15383const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
15384 const SCEV *S, const Loop *L,
15385 SmallVectorImpl<const SCEVPredicate *> &Preds) {
15386 SmallVector<const SCEVPredicate *> TransformPreds;
15387 S = SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: &TransformPreds, Pred: nullptr);
15388 auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S);
15389
15390 if (!AddRec)
15391 return nullptr;
15392
15393 // Check if any of the transformed predicates is known to be false. In that
15394 // case, it doesn't make sense to convert to a predicated AddRec, as the
15395 // versioned loop will never execute.
15396 for (const SCEVPredicate *Pred : TransformPreds) {
15397 auto *WrapPred = dyn_cast<SCEVWrapPredicate>(Val: Pred);
15398 if (!WrapPred || WrapPred->getFlags() != SCEVWrapPredicate::IncrementNSSW)
15399 continue;
15400
15401 const SCEVAddRecExpr *AddRecToCheck = WrapPred->getExpr();
15402 const SCEV *ExitCount = getBackedgeTakenCount(L: AddRecToCheck->getLoop());
15403 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
15404 continue;
15405
15406 const SCEV *Step = AddRecToCheck->getStepRecurrence(SE&: *this);
15407 if (!Step->isOne())
15408 continue;
15409
15410 ExitCount = getTruncateOrSignExtend(V: ExitCount, Ty: Step->getType());
15411 const SCEV *Add = getAddExpr(LHS: AddRecToCheck->getStart(), RHS: ExitCount);
15412 if (isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS: Add, RHS: AddRecToCheck->getStart()))
15413 return nullptr;
15414 }
15415
15416 // Since the transformation was successful, we can now transfer the SCEV
15417 // predicates.
15418 Preds.append(in_start: TransformPreds.begin(), in_end: TransformPreds.end());
15419
15420 return AddRec;
15421}
15422
15423/// SCEV predicates
15424SCEVPredicate::SCEVPredicate(const FoldingSetNodeIDRef ID,
15425 SCEVPredicateKind Kind)
15426 : FastID(ID), Kind(Kind) {}
15427
15428SCEVComparePredicate::SCEVComparePredicate(const FoldingSetNodeIDRef ID,
15429 const ICmpInst::Predicate Pred,
15430 const SCEV *LHS, const SCEV *RHS)
15431 : SCEVPredicate(ID, P_Compare), Pred(Pred), LHS(LHS), RHS(RHS) {
15432 assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match");
15433 assert(LHS != RHS && "LHS and RHS are the same SCEV");
15434}
15435
15436bool SCEVComparePredicate::implies(const SCEVPredicate *N,
15437 ScalarEvolution &SE) const {
15438 const auto *Op = dyn_cast<SCEVComparePredicate>(Val: N);
15439
15440 if (!Op)
15441 return false;
15442
15443 if (Pred != ICmpInst::ICMP_EQ)
15444 return false;
15445
15446 return Op->LHS == LHS && Op->RHS == RHS;
15447}
15448
15449bool SCEVComparePredicate::isAlwaysTrue() const { return false; }
15450
15451void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
15452 if (Pred == ICmpInst::ICMP_EQ)
15453 OS.indent(NumSpaces: Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n";
15454 else
15455 OS.indent(NumSpaces: Depth) << "Compare predicate: " << *LHS << " " << Pred << ") "
15456 << *RHS << "\n";
15457
15458}
15459
15460SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
15461 const SCEVAddRecExpr *AR,
15462 IncrementWrapFlags Flags)
15463 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {}
15464
15465const SCEVAddRecExpr *SCEVWrapPredicate::getExpr() const { return AR; }
15466
15467bool SCEVWrapPredicate::implies(const SCEVPredicate *N,
15468 ScalarEvolution &SE) const {
15469 const auto *Op = dyn_cast<SCEVWrapPredicate>(Val: N);
15470 if (!Op || setFlags(Flags, OnFlags: Op->Flags) != Flags)
15471 return false;
15472
15473 if (Op->AR == AR)
15474 return true;
15475
15476 if (Flags != SCEVWrapPredicate::IncrementNSSW &&
15477 Flags != SCEVWrapPredicate::IncrementNUSW)
15478 return false;
15479
15480 const SCEV *Start = AR->getStart();
15481 const SCEV *OpStart = Op->AR->getStart();
15482 if (Start->getType()->isPointerTy() != OpStart->getType()->isPointerTy())
15483 return false;
15484
15485 // Reject pointers to different address spaces.
15486 if (Start->getType()->isPointerTy() && Start->getType() != OpStart->getType())
15487 return false;
15488
15489 // NUSW/NSSW on a wider-type AddRec does not imply the same on a
15490 // narrower-type AddRec.
15491 if (SE.getTypeSizeInBits(Ty: AR->getType()) >
15492 SE.getTypeSizeInBits(Ty: Op->AR->getType()))
15493 return false;
15494
15495 const SCEV *Step = AR->getStepRecurrence(SE);
15496 const SCEV *OpStep = Op->AR->getStepRecurrence(SE);
15497 if (!SE.isKnownPositive(S: Step) || !SE.isKnownPositive(S: OpStep))
15498 return false;
15499
15500 // If both steps are positive, this implies N, if N's start and step are
15501 // ULE/SLE (for NSUW/NSSW) than this'.
15502 Type *WiderTy = SE.getWiderType(T1: Step->getType(), T2: OpStep->getType());
15503 Step = SE.getNoopOrZeroExtend(V: Step, Ty: WiderTy);
15504 OpStep = SE.getNoopOrZeroExtend(V: OpStep, Ty: WiderTy);
15505
15506 bool IsNUW = Flags == SCEVWrapPredicate::IncrementNUSW;
15507 OpStart = IsNUW ? SE.getNoopOrZeroExtend(V: OpStart, Ty: WiderTy)
15508 : SE.getNoopOrSignExtend(V: OpStart, Ty: WiderTy);
15509 Start = IsNUW ? SE.getNoopOrZeroExtend(V: Start, Ty: WiderTy)
15510 : SE.getNoopOrSignExtend(V: Start, Ty: WiderTy);
15511 CmpInst::Predicate Pred = IsNUW ? CmpInst::ICMP_ULE : CmpInst::ICMP_SLE;
15512 return SE.isKnownPredicate(Pred, LHS: OpStep, RHS: Step) &&
15513 SE.isKnownPredicate(Pred, LHS: OpStart, RHS: Start);
15514}
15515
15516bool SCEVWrapPredicate::isAlwaysTrue() const {
15517 SCEV::NoWrapFlags ScevFlags = AR->getNoWrapFlags();
15518 IncrementWrapFlags IFlags = Flags;
15519
15520 if (ScalarEvolution::setFlags(Flags: ScevFlags, OnFlags: SCEV::FlagNSW) == ScevFlags)
15521 IFlags = clearFlags(Flags: IFlags, OffFlags: IncrementNSSW);
15522
15523 return IFlags == IncrementAnyWrap;
15524}
15525
15526void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const {
15527 OS.indent(NumSpaces: Depth) << *getExpr() << " Added Flags: ";
15528 if (SCEVWrapPredicate::IncrementNUSW & getFlags())
15529 OS << "<nusw>";
15530 if (SCEVWrapPredicate::IncrementNSSW & getFlags())
15531 OS << "<nssw>";
15532 OS << "\n";
15533}
15534
15535SCEVWrapPredicate::IncrementWrapFlags
15536SCEVWrapPredicate::getImpliedFlags(const SCEVAddRecExpr *AR,
15537 ScalarEvolution &SE) {
15538 IncrementWrapFlags ImpliedFlags = IncrementAnyWrap;
15539 SCEV::NoWrapFlags StaticFlags = AR->getNoWrapFlags();
15540
15541 // We can safely transfer the NSW flag as NSSW.
15542 if (ScalarEvolution::setFlags(Flags: StaticFlags, OnFlags: SCEV::FlagNSW) == StaticFlags)
15543 ImpliedFlags = IncrementNSSW;
15544
15545 if (ScalarEvolution::setFlags(Flags: StaticFlags, OnFlags: SCEV::FlagNUW) == StaticFlags) {
15546 // If the increment is positive, the SCEV NUW flag will also imply the
15547 // WrapPredicate NUSW flag.
15548 if (const auto *Step = dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE)))
15549 if (Step->getValue()->getValue().isNonNegative())
15550 ImpliedFlags = setFlags(Flags: ImpliedFlags, OnFlags: IncrementNUSW);
15551 }
15552
15553 return ImpliedFlags;
15554}
15555
15556/// Union predicates don't get cached so create a dummy set ID for it.
15557SCEVUnionPredicate::SCEVUnionPredicate(ArrayRef<const SCEVPredicate *> Preds,
15558 ScalarEvolution &SE)
15559 : SCEVPredicate(FoldingSetNodeIDRef(), P_Union) {
15560 for (const auto *P : Preds)
15561 add(N: P, SE);
15562}
15563
15564bool SCEVUnionPredicate::isAlwaysTrue() const {
15565 return all_of(Range: Preds,
15566 P: [](const SCEVPredicate *I) { return I->isAlwaysTrue(); });
15567}
15568
15569bool SCEVUnionPredicate::implies(const SCEVPredicate *N,
15570 ScalarEvolution &SE) const {
15571 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N))
15572 return all_of(Range: Set->Preds, P: [this, &SE](const SCEVPredicate *I) {
15573 return this->implies(N: I, SE);
15574 });
15575
15576 if (any_of(Range: Preds,
15577 P: [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); }))
15578 return true;
15579
15580 // A wrap predicate may be implied by a wrap predicate in Preds after applying
15581 // equal predicates.
15582 const auto *NWrap = dyn_cast<SCEVWrapPredicate>(Val: N);
15583 if (!NWrap)
15584 return false;
15585 const Loop *L = NWrap->getExpr()->getLoop();
15586 return any_of(Range: Preds, P: [&](const SCEVPredicate *I) {
15587 const auto *IWrap = dyn_cast<SCEVWrapPredicate>(Val: I);
15588 if (!IWrap)
15589 return false;
15590 const auto *RewrittenAR = dyn_cast<SCEVAddRecExpr>(
15591 Val: SE.rewriteUsingPredicate(S: IWrap->getExpr(), L, Preds: *this));
15592 return RewrittenAR &&
15593 SE.getWrapPredicate(AR: RewrittenAR, AddedFlags: IWrap->getFlags())->implies(N, SE);
15594 });
15595}
15596
15597void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const {
15598 for (const auto *Pred : Preds)
15599 Pred->print(OS, Depth);
15600}
15601
15602void SCEVUnionPredicate::add(const SCEVPredicate *N, ScalarEvolution &SE) {
15603 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N)) {
15604 for (const auto *Pred : Set->Preds)
15605 add(N: Pred, SE);
15606 return;
15607 }
15608
15609 // Implication checks are quadratic in the number of predicates. Stop doing
15610 // them if there are many predicates, as they should be too expensive to use
15611 // anyway at that point.
15612 bool CheckImplies = Preds.size() < 16;
15613
15614 // Only add predicate if it is not already implied by this union predicate.
15615 if (CheckImplies && implies(N, SE))
15616 return;
15617
15618 // Build a new vector containing the current predicates, except the ones that
15619 // are implied by the new predicate N.
15620 SmallVector<const SCEVPredicate *> PrunedPreds;
15621 for (auto *P : Preds) {
15622 if (CheckImplies && N->implies(N: P, SE))
15623 continue;
15624 PrunedPreds.push_back(Elt: P);
15625 }
15626 Preds = std::move(PrunedPreds);
15627 Preds.push_back(Elt: N);
15628}
15629
15630PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE,
15631 Loop &L)
15632 : SE(SE), L(L) {
15633 SmallVector<const SCEVPredicate*, 4> Empty;
15634 Preds = std::make_unique<SCEVUnionPredicate>(args&: Empty, args&: SE);
15635}
15636
15637void ScalarEvolution::registerUser(const SCEV *User, ArrayRef<SCEVUse> Ops) {
15638 for (const SCEV *Op : Ops)
15639 // We do not expect that forgetting cached data for SCEVConstants will ever
15640 // open any prospects for sharpening or introduce any correctness issues,
15641 // so we don't bother storing their dependencies.
15642 if (!isa<SCEVConstant>(Val: Op))
15643 SCEVUsers[Op].insert(Ptr: User);
15644}
15645
15646const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) {
15647 const SCEV *Expr = SE.getSCEV(V);
15648 return getPredicatedSCEV(Expr);
15649}
15650
15651const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
15652 RewriteEntry &Entry = RewriteMap[Expr];
15653
15654 // If we already have an entry and the version matches, return it.
15655 if (Entry.second && Generation == Entry.first)
15656 return Entry.second;
15657
15658 // We found an entry but it's stale. Rewrite the stale entry
15659 // according to the current predicate.
15660 if (Entry.second)
15661 Expr = Entry.second;
15662
15663 const SCEV *NewSCEV = SE.rewriteUsingPredicate(S: Expr, L: &L, Preds: *Preds);
15664 Entry = {Generation, NewSCEV};
15665
15666 return NewSCEV;
15667}
15668
15669const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
15670 if (!BackedgeCount) {
15671 SmallVector<const SCEVPredicate *, 4> Preds;
15672 BackedgeCount = SE.getPredicatedBackedgeTakenCount(L: &L, Preds);
15673 for (const auto *P : Preds)
15674 addPredicate(Pred: *P);
15675 }
15676 return BackedgeCount;
15677}
15678
15679const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
15680 if (!SymbolicMaxBackedgeCount) {
15681 SmallVector<const SCEVPredicate *, 4> Preds;
15682 SymbolicMaxBackedgeCount =
15683 SE.getPredicatedSymbolicMaxBackedgeTakenCount(L: &L, Preds);
15684 for (const auto *P : Preds)
15685 addPredicate(Pred: *P);
15686 }
15687 return SymbolicMaxBackedgeCount;
15688}
15689
15690unsigned PredicatedScalarEvolution::getSmallConstantMaxTripCount() {
15691 if (!SmallConstantMaxTripCount) {
15692 SmallVector<const SCEVPredicate *, 4> Preds;
15693 SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(L: &L, Predicates: &Preds);
15694 for (const auto *P : Preds)
15695 addPredicate(Pred: *P);
15696 }
15697 return *SmallConstantMaxTripCount;
15698}
15699
15700void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
15701 if (Preds->implies(N: &Pred, SE))
15702 return;
15703
15704 SmallVector<const SCEVPredicate *, 4> NewPreds(Preds->getPredicates());
15705 NewPreds.push_back(Elt: &Pred);
15706 Preds = std::make_unique<SCEVUnionPredicate>(args&: NewPreds, args&: SE);
15707 updateGeneration();
15708}
15709
15710void PredicatedScalarEvolution::addPredicates(
15711 ArrayRef<const SCEVPredicate *> Preds) {
15712 for (const SCEVPredicate *P : Preds)
15713 addPredicate(Pred: *P);
15714}
15715
15716const SCEVPredicate &PredicatedScalarEvolution::getPredicate() const {
15717 return *Preds;
15718}
15719
15720void PredicatedScalarEvolution::updateGeneration() {
15721 // If the generation number wrapped recompute everything.
15722 if (++Generation == 0) {
15723 for (auto &II : RewriteMap) {
15724 const SCEV *Rewritten = II.second.second;
15725 II.second = {Generation, SE.rewriteUsingPredicate(S: Rewritten, L: &L, Preds: *Preds)};
15726 }
15727 }
15728}
15729
15730bool PredicatedScalarEvolution::hasNoOverflow(
15731 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) {
15732 const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: getSCEV(V));
15733 if (!AR)
15734 return false;
15735
15736 Flags = SCEVWrapPredicate::clearFlags(
15737 Flags, OffFlags: SCEVWrapPredicate::getImpliedFlags(AR, SE));
15738
15739 return Flags == SCEVWrapPredicate::IncrementAnyWrap;
15740}
15741
15742const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(
15743 Value *V, SmallVectorImpl<const SCEVPredicate *> *ExtraPreds) {
15744 const SCEV *Expr = this->getSCEV(V);
15745 SmallVector<const SCEVPredicate *, 4> NewPreds;
15746 auto *New = SE.convertSCEVToAddRecWithPredicates(S: Expr, L: &L, Preds&: NewPreds);
15747
15748 if (!New)
15749 return nullptr;
15750
15751 if (ExtraPreds) {
15752 ExtraPreds->append(RHS: NewPreds);
15753 return New;
15754 }
15755
15756 addPredicates(Preds: NewPreds);
15757
15758 RewriteMap[SE.getSCEV(V)] = {Generation, New};
15759 return New;
15760}
15761
15762PredicatedScalarEvolution::PredicatedScalarEvolution(
15763 const PredicatedScalarEvolution &Init)
15764 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L),
15765 Preds(std::make_unique<SCEVUnionPredicate>(args: Init.Preds->getPredicates(),
15766 args&: SE)),
15767 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {}
15768
15769void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const {
15770 // For each block.
15771 for (auto *BB : L.getBlocks())
15772 for (auto &I : *BB) {
15773 if (!SE.isSCEVable(Ty: I.getType()))
15774 continue;
15775
15776 auto *Expr = SE.getSCEV(V: &I);
15777 auto II = RewriteMap.find(Val: Expr);
15778
15779 if (II == RewriteMap.end())
15780 continue;
15781
15782 // Don't print things that are not interesting.
15783 if (II->second.second == Expr)
15784 continue;
15785
15786 OS.indent(NumSpaces: Depth) << "[PSE]" << I << ":\n";
15787 OS.indent(NumSpaces: Depth + 2) << *Expr << "\n";
15788 OS.indent(NumSpaces: Depth + 2) << "--> " << *II->second.second << "\n";
15789 }
15790}
15791
15792ScalarEvolution::LoopGuards
15793ScalarEvolution::LoopGuards::collect(const Loop *L, ScalarEvolution &SE) {
15794 BasicBlock *Header = L->getHeader();
15795 BasicBlock *Pred = L->getLoopPredecessor();
15796 LoopGuards Guards(SE);
15797 if (!Pred)
15798 return Guards;
15799 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
15800 collectFromBlock(SE, Guards, Block: Header, Pred, VisitedBlocks);
15801 return Guards;
15802}
15803
15804void ScalarEvolution::LoopGuards::collectFromPHI(
15805 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15806 const PHINode &Phi, SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks,
15807 SmallDenseMap<const BasicBlock *, LoopGuards> &IncomingGuards,
15808 unsigned Depth) {
15809 if (!SE.isSCEVable(Ty: Phi.getType()))
15810 return;
15811
15812 using MinMaxPattern = std::pair<const SCEVConstant *, SCEVTypes>;
15813 auto GetMinMaxConst = [&](unsigned IncomingIdx) -> MinMaxPattern {
15814 const BasicBlock *InBlock = Phi.getIncomingBlock(i: IncomingIdx);
15815 if (!VisitedBlocks.insert(Ptr: InBlock).second)
15816 return {nullptr, scCouldNotCompute};
15817
15818 // Avoid analyzing unreachable blocks so that we don't get trapped
15819 // traversing cycles with ill-formed dominance or infinite cycles
15820 if (!SE.DT.isReachableFromEntry(A: InBlock))
15821 return {nullptr, scCouldNotCompute};
15822
15823 auto [G, Inserted] = IncomingGuards.try_emplace(Key: InBlock, Args: LoopGuards(SE));
15824 if (Inserted)
15825 collectFromBlock(SE, Guards&: G->second, Block: Phi.getParent(), Pred: InBlock, VisitedBlocks,
15826 Depth: Depth + 1);
15827 auto &RewriteMap = G->second.RewriteMap;
15828 if (RewriteMap.empty())
15829 return {nullptr, scCouldNotCompute};
15830 auto S = RewriteMap.find(Val: SE.getSCEV(V: Phi.getIncomingValue(i: IncomingIdx)));
15831 if (S == RewriteMap.end())
15832 return {nullptr, scCouldNotCompute};
15833 auto *SM = dyn_cast_if_present<SCEVMinMaxExpr>(Val: S->second);
15834 if (!SM)
15835 return {nullptr, scCouldNotCompute};
15836 if (const SCEVConstant *C0 = dyn_cast<SCEVConstant>(Val: SM->getOperand(i: 0)))
15837 return {C0, SM->getSCEVType()};
15838 return {nullptr, scCouldNotCompute};
15839 };
15840 auto MergeMinMaxConst = [](MinMaxPattern P1,
15841 MinMaxPattern P2) -> MinMaxPattern {
15842 auto [C1, T1] = P1;
15843 auto [C2, T2] = P2;
15844 if (!C1 || !C2 || T1 != T2)
15845 return {nullptr, scCouldNotCompute};
15846 switch (T1) {
15847 case scUMaxExpr:
15848 return {C1->getAPInt().ult(RHS: C2->getAPInt()) ? C1 : C2, T1};
15849 case scSMaxExpr:
15850 return {C1->getAPInt().slt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15851 case scUMinExpr:
15852 return {C1->getAPInt().ugt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15853 case scSMinExpr:
15854 return {C1->getAPInt().sgt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15855 default:
15856 llvm_unreachable("Trying to merge non-MinMaxExpr SCEVs.");
15857 }
15858 };
15859 auto P = GetMinMaxConst(0);
15860 for (unsigned int In = 1; In < Phi.getNumIncomingValues(); In++) {
15861 if (!P.first)
15862 break;
15863 P = MergeMinMaxConst(P, GetMinMaxConst(In));
15864 }
15865 if (P.first) {
15866 const SCEV *LHS = SE.getSCEV(V: const_cast<PHINode *>(&Phi));
15867 SmallVector<SCEVUse, 2> Ops({P.first, LHS});
15868 const SCEV *RHS = SE.getMinMaxExpr(Kind: P.second, Ops);
15869 Guards.RewriteMap.insert(KV: {LHS, RHS});
15870 }
15871}
15872
15873// Return a new SCEV that modifies \p Expr to the closest number divides by
15874// \p Divisor and less or equal than Expr. For now, only handle constant
15875// Expr.
15876static const SCEV *getPreviousSCEVDivisibleByDivisor(const SCEV *Expr,
15877 const APInt &DivisorVal,
15878 ScalarEvolution &SE) {
15879 const APInt *ExprVal;
15880 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15881 DivisorVal.isNonPositive())
15882 return Expr;
15883 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15884 // return the SCEV: Expr - Expr % Divisor
15885 return SE.getConstant(Val: *ExprVal - Rem);
15886}
15887
15888// Return a new SCEV that modifies \p Expr to the closest number divides by
15889// \p Divisor and greater or equal than Expr. For now, only handle constant
15890// Expr.
15891static const SCEV *getNextSCEVDivisibleByDivisor(const SCEV *Expr,
15892 const APInt &DivisorVal,
15893 ScalarEvolution &SE) {
15894 const APInt *ExprVal;
15895 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15896 DivisorVal.isNonPositive())
15897 return Expr;
15898 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15899 if (Rem.isZero())
15900 return Expr;
15901 // return the SCEV: Expr + Divisor - Expr % Divisor
15902 return SE.getConstant(Val: *ExprVal + DivisorVal - Rem);
15903}
15904
15905static bool collectDivisibilityInformation(
15906 ICmpInst::Predicate Predicate, const SCEV *LHS, const SCEV *RHS,
15907 DenseMap<const SCEV *, const SCEV *> &DivInfo,
15908 DenseMap<const SCEV *, APInt> &Multiples, ScalarEvolution &SE) {
15909 // If we have LHS == 0, check if LHS is computing a property of some unknown
15910 // SCEV %v which we can rewrite %v to express explicitly.
15911 if (Predicate != CmpInst::ICMP_EQ || !match(S: RHS, P: m_scev_Zero()))
15912 return false;
15913 // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to
15914 // explicitly express that.
15915 const SCEVUnknown *URemLHS = nullptr;
15916 const SCEV *URemRHS = nullptr;
15917 if (!match(S: LHS, P: m_scev_URem(LHS: m_SCEVUnknown(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE)))
15918 return false;
15919
15920 const SCEV *Multiple =
15921 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: URemLHS, RHS: URemRHS), RHS: URemRHS);
15922 DivInfo[URemLHS] = Multiple;
15923 if (auto *C = dyn_cast<SCEVConstant>(Val: URemRHS))
15924 Multiples[URemLHS] = C->getAPInt();
15925 return true;
15926}
15927
15928// Check if the condition is a divisibility guard (A % B == 0).
15929static bool isDivisibilityGuard(const SCEV *LHS, const SCEV *RHS,
15930 ScalarEvolution &SE) {
15931 const SCEV *X, *Y;
15932 return match(S: LHS, P: m_scev_URem(LHS: m_SCEV(V&: X), RHS: m_SCEV(V&: Y), SE)) && RHS->isZero();
15933}
15934
15935// Apply divisibility by \p Divisor on MinMaxExpr with constant values,
15936// recursively. This is done by aligning up/down the constant value to the
15937// Divisor.
15938static const SCEV *applyDivisibilityOnMinMaxExpr(const SCEV *MinMaxExpr,
15939 APInt Divisor,
15940 ScalarEvolution &SE) {
15941 // Return true if \p Expr is a MinMax SCEV expression with a non-negative
15942 // constant operand. If so, return in \p SCTy the SCEV type and in \p RHS
15943 // the non-constant operand and in \p LHS the constant operand.
15944 auto IsMinMaxSCEVWithNonNegativeConstant =
15945 [&](const SCEV *Expr, SCEVTypes &SCTy, const SCEV *&LHS,
15946 const SCEV *&RHS) {
15947 if (auto *MinMax = dyn_cast<SCEVMinMaxExpr>(Val: Expr)) {
15948 if (MinMax->getNumOperands() != 2)
15949 return false;
15950 if (auto *C = dyn_cast<SCEVConstant>(Val: MinMax->getOperand(i: 0))) {
15951 if (C->getAPInt().isNegative())
15952 return false;
15953 SCTy = MinMax->getSCEVType();
15954 LHS = MinMax->getOperand(i: 0);
15955 RHS = MinMax->getOperand(i: 1);
15956 return true;
15957 }
15958 }
15959 return false;
15960 };
15961
15962 const SCEV *MinMaxLHS = nullptr, *MinMaxRHS = nullptr;
15963 SCEVTypes SCTy;
15964 if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS,
15965 MinMaxRHS))
15966 return MinMaxExpr;
15967 auto IsMin = isa<SCEVSMinExpr>(Val: MinMaxExpr) || isa<SCEVUMinExpr>(Val: MinMaxExpr);
15968 assert(SE.isKnownNonNegative(MinMaxLHS) && "Expected non-negative operand!");
15969 auto *DivisibleExpr =
15970 IsMin ? getPreviousSCEVDivisibleByDivisor(Expr: MinMaxLHS, DivisorVal: Divisor, SE)
15971 : getNextSCEVDivisibleByDivisor(Expr: MinMaxLHS, DivisorVal: Divisor, SE);
15972 SmallVector<SCEVUse> Ops = {
15973 applyDivisibilityOnMinMaxExpr(MinMaxExpr: MinMaxRHS, Divisor, SE), DivisibleExpr};
15974 return SE.getMinMaxExpr(Kind: SCTy, Ops);
15975}
15976
15977void ScalarEvolution::LoopGuards::collectFromBlock(
15978 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15979 const BasicBlock *Block, const BasicBlock *Pred,
15980 SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks, unsigned Depth) {
15981
15982 assert(SE.DT.isReachableFromEntry(Block) && SE.DT.isReachableFromEntry(Pred));
15983
15984 SmallVector<SCEVUse> ExprsToRewrite;
15985 auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS,
15986 const SCEV *RHS,
15987 DenseMap<const SCEV *, const SCEV *> &RewriteMap,
15988 const LoopGuards &DivGuards) {
15989 // WARNING: It is generally unsound to apply any wrap flags to the proposed
15990 // replacement SCEV which isn't directly implied by the structure of that
15991 // SCEV. In particular, using contextual facts to imply flags is *NOT*
15992 // legal. See the scoping rules for flags in the header to understand why.
15993
15994 // Puts rewrite rule \p From -> \p To into the rewrite map. Also if \p From
15995 // and \p FromRewritten are the same (i.e. there has been no rewrite
15996 // registered for \p From), then puts this value in the list of rewritten
15997 // expressions.
15998 auto AddRewrite = [&](const SCEV *From, const SCEV *FromRewritten,
15999 const SCEV *To) {
16000 if (From == FromRewritten)
16001 ExprsToRewrite.push_back(Elt: From);
16002 RewriteMap[From] = To;
16003 };
16004
16005 // Checks whether \p S has already been rewritten. In that case returns the
16006 // existing rewrite because we want to chain further rewrites onto the
16007 // already rewritten value. Otherwise returns \p S.
16008 auto GetMaybeRewritten = [&](const SCEV *S) {
16009 return RewriteMap.lookup_or(Val: S, Default&: S);
16010 };
16011
16012 // Check for a condition of the form (-C1 + X < C2). InstCombine will
16013 // create this form when combining two checks of the form (X u< C2 + C1) and
16014 // (X >=u C1).
16015 auto MatchRangeCheckIdiom = [&](ICmpInst::Predicate Pred,
16016 const SCEV *MatchLHS,
16017 const SCEV *MatchRHS) {
16018 const SCEVConstant *C1;
16019 const SCEVUnknown *LHSUnknown;
16020 auto *C2 = dyn_cast<SCEVConstant>(Val: MatchRHS);
16021 if (!match(S: MatchLHS,
16022 P: m_scev_Add(Op0: m_SCEVConstant(V&: C1), Op1: m_SCEVUnknown(V&: LHSUnknown))) ||
16023 !C2)
16024 return false;
16025
16026 auto ExactRegion =
16027 ConstantRange::makeExactICmpRegion(Pred, Other: C2->getAPInt())
16028 .sub(Other: C1->getAPInt());
16029
16030 // Tighten the raw range with what we already know about LHSUnknown
16031 // from prior guards recorded in RewriteMap, or from SCEV's own range
16032 // analysis.
16033 const SCEV *RewrittenLHS = GetMaybeRewritten(LHSUnknown);
16034 ExactRegion = ExactRegion.intersectWith(CR: SE.getUnsignedRange(S: RewrittenLHS),
16035 Type: ConstantRange::Unsigned);
16036
16037 // Bail if the guard is inconsistent with prior facts, or if the range
16038 // is still not a monotonic non-wrapping interval after tightening.
16039 if (ExactRegion.isEmptySet() || ExactRegion.isWrappedSet() ||
16040 ExactRegion.isFullSet())
16041 return false;
16042
16043 const SCEV *RegionMin = SE.getConstant(Val: ExactRegion.getUnsignedMin());
16044 const SCEV *RegionMax = SE.getConstant(Val: ExactRegion.getUnsignedMax());
16045 const SCEV *ClampedLHS =
16046 SE.getUMaxExpr(LHS: RegionMin, RHS: SE.getUMinExpr(LHS: RewrittenLHS, RHS: RegionMax));
16047 AddRewrite(LHSUnknown, RewrittenLHS, ClampedLHS);
16048 return true;
16049 };
16050 if (MatchRangeCheckIdiom(Predicate, LHS, RHS))
16051 return;
16052
16053 // Do not apply information for constants or if RHS contains an AddRec.
16054 if (isa<SCEVConstant>(Val: LHS) || SE.containsAddRecurrence(S: RHS))
16055 return;
16056
16057 // If RHS is SCEVUnknown, make sure the information is applied to it.
16058 if (!isa<SCEVUnknown>(Val: LHS) && isa<SCEVUnknown>(Val: RHS)) {
16059 std::swap(a&: LHS, b&: RHS);
16060 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
16061 }
16062
16063 const SCEV *RewrittenLHS = GetMaybeRewritten(LHS);
16064 // Apply divisibility information when computing the constant multiple.
16065 const APInt &DividesBy =
16066 SE.getConstantMultiple(S: DivGuards.rewrite(Expr: RewrittenLHS));
16067
16068 // Collect rewrites for LHS and its transitive operands based on the
16069 // condition.
16070 // For min/max expressions, also apply the guard to its operands:
16071 // 'min(a, b) >= c' -> '(a >= c) and (b >= c)',
16072 // 'min(a, b) > c' -> '(a > c) and (b > c)',
16073 // 'max(a, b) <= c' -> '(a <= c) and (b <= c)',
16074 // 'max(a, b) < c' -> '(a < c) and (b < c)'.
16075
16076 // We cannot express strict predicates in SCEV, so instead we replace them
16077 // with non-strict ones against plus or minus one of RHS depending on the
16078 // predicate.
16079 const SCEV *One = SE.getOne(Ty: RHS->getType());
16080 switch (Predicate) {
16081 case CmpInst::ICMP_ULT:
16082 if (RHS->getType()->isPointerTy())
16083 return;
16084 RHS = SE.getUMaxExpr(LHS: RHS, RHS: One);
16085 [[fallthrough]];
16086 case CmpInst::ICMP_SLT: {
16087 RHS = SE.getMinusSCEV(LHS: RHS, RHS: One);
16088 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16089 break;
16090 }
16091 case CmpInst::ICMP_UGT:
16092 case CmpInst::ICMP_SGT:
16093 RHS = SE.getAddExpr(LHS: RHS, RHS: One);
16094 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16095 break;
16096 case CmpInst::ICMP_ULE:
16097 case CmpInst::ICMP_SLE:
16098 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16099 break;
16100 case CmpInst::ICMP_UGE:
16101 case CmpInst::ICMP_SGE:
16102 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16103 break;
16104 default:
16105 break;
16106 }
16107
16108 SmallVector<SCEVUse, 16> Worklist(1, LHS);
16109 SmallPtrSet<const SCEV *, 16> Visited;
16110
16111 auto EnqueueOperands = [&Worklist](const SCEVNAryExpr *S) {
16112 append_range(C&: Worklist, R: S->operands());
16113 };
16114
16115 while (!Worklist.empty()) {
16116 const SCEV *From = Worklist.pop_back_val();
16117 if (isa<SCEVConstant>(Val: From))
16118 continue;
16119 if (!Visited.insert(Ptr: From).second)
16120 continue;
16121 const SCEV *FromRewritten = GetMaybeRewritten(From);
16122 const SCEV *To = nullptr;
16123
16124 switch (Predicate) {
16125 case CmpInst::ICMP_ULT:
16126 case CmpInst::ICMP_ULE:
16127 To = SE.getUMinExpr(LHS: FromRewritten, RHS);
16128 if (auto *UMax = dyn_cast<SCEVUMaxExpr>(Val: FromRewritten))
16129 EnqueueOperands(UMax);
16130 break;
16131 case CmpInst::ICMP_SLT:
16132 case CmpInst::ICMP_SLE:
16133 To = SE.getSMinExpr(LHS: FromRewritten, RHS);
16134 if (auto *SMax = dyn_cast<SCEVSMaxExpr>(Val: FromRewritten))
16135 EnqueueOperands(SMax);
16136 break;
16137 case CmpInst::ICMP_UGT:
16138 case CmpInst::ICMP_UGE:
16139 To = SE.getUMaxExpr(LHS: FromRewritten, RHS);
16140 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: FromRewritten))
16141 EnqueueOperands(UMin);
16142 break;
16143 case CmpInst::ICMP_SGT:
16144 case CmpInst::ICMP_SGE:
16145 To = SE.getSMaxExpr(LHS: FromRewritten, RHS);
16146 if (auto *SMin = dyn_cast<SCEVSMinExpr>(Val: FromRewritten))
16147 EnqueueOperands(SMin);
16148 break;
16149 case CmpInst::ICMP_EQ:
16150 if (isa<SCEVConstant>(Val: RHS))
16151 To = RHS;
16152 break;
16153 case CmpInst::ICMP_NE:
16154 if (match(S: RHS, P: m_scev_Zero())) {
16155 const SCEV *OneAlignedUp =
16156 getNextSCEVDivisibleByDivisor(Expr: One, DivisorVal: DividesBy, SE);
16157 To = SE.getUMaxExpr(LHS: FromRewritten, RHS: OneAlignedUp);
16158 } else {
16159 // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS),
16160 // but creating the subtraction eagerly is expensive. Track the
16161 // inequalities in a separate map, and materialize the rewrite lazily
16162 // when encountering a suitable subtraction while re-writing.
16163 if (LHS->getType()->isPointerTy()) {
16164 LHS = SE.getPtrToAddrExpr(Op: LHS);
16165 RHS = SE.getPtrToAddrExpr(Op: RHS);
16166 if (isa<SCEVCouldNotCompute>(Val: LHS) || isa<SCEVCouldNotCompute>(Val: RHS))
16167 break;
16168 }
16169 const SCEVConstant *C;
16170 const SCEV *A, *B;
16171 if (match(S: RHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: A))) &&
16172 match(S: LHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: B)))) {
16173 RHS = A;
16174 LHS = B;
16175 }
16176 if (LHS > RHS)
16177 std::swap(a&: LHS, b&: RHS);
16178 Guards.NotEqual.insert(V: {LHS, RHS});
16179 continue;
16180 }
16181 break;
16182 default:
16183 break;
16184 }
16185
16186 if (To)
16187 AddRewrite(From, FromRewritten, To);
16188 }
16189 };
16190
16191 SmallVector<PointerIntPair<Value *, 1, bool>> Terms;
16192 // First, collect information from assumptions dominating the loop.
16193 for (auto &AssumeVH : SE.AC.assumptions()) {
16194 if (!AssumeVH)
16195 continue;
16196 auto *AssumeI = cast<CallInst>(Val&: AssumeVH);
16197 if (!SE.DT.dominates(Def: AssumeI, BB: Block))
16198 continue;
16199 Terms.emplace_back(Args: AssumeI->getOperand(i_nocapture: 0), Args: true);
16200 }
16201
16202 // Second, collect information from llvm.experimental.guards dominating the loop.
16203 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
16204 M: SE.F.getParent(), id: Intrinsic::experimental_guard);
16205 if (GuardDecl)
16206 for (const auto *GU : GuardDecl->users())
16207 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
16208 if (Guard->getFunction() == Block->getParent() &&
16209 SE.DT.dominates(Def: Guard, BB: Block))
16210 Terms.emplace_back(Args: Guard->getArgOperand(i: 0), Args: true);
16211
16212 // Third, collect conditions from dominating branches. Starting at the loop
16213 // predecessor, climb up the predecessor chain, as long as there are
16214 // predecessors that can be found that have unique successors leading to the
16215 // original header.
16216 // TODO: share this logic with isLoopEntryGuardedByCond.
16217 unsigned NumCollectedConditions = 0;
16218 VisitedBlocks.insert(Ptr: Block);
16219 std::pair<const BasicBlock *, const BasicBlock *> Pair(Pred, Block);
16220 for (; Pair.first;
16221 Pair = SE.getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
16222 VisitedBlocks.insert(Ptr: Pair.second);
16223 const CondBrInst *LoopEntryPredicate =
16224 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
16225 if (!LoopEntryPredicate)
16226 continue;
16227
16228 Terms.emplace_back(Args: LoopEntryPredicate->getCondition(),
16229 Args: LoopEntryPredicate->getSuccessor(i: 0) == Pair.second);
16230 NumCollectedConditions++;
16231
16232 // If we are recursively collecting guards stop after 2
16233 // conditions to limit compile-time impact for now.
16234 if (Depth > 0 && NumCollectedConditions == 2)
16235 break;
16236 }
16237 // Finally, if we stopped climbing the predecessor chain because
16238 // there wasn't a unique one to continue, try to collect conditions
16239 // for PHINodes by recursively following all of their incoming
16240 // blocks and try to merge the found conditions to build a new one
16241 // for the Phi.
16242 if (Pair.second->hasNPredecessorsOrMore(N: 2) &&
16243 Depth < MaxLoopGuardCollectionDepth) {
16244 SmallDenseMap<const BasicBlock *, LoopGuards> IncomingGuards;
16245 for (auto &Phi : Pair.second->phis())
16246 collectFromPHI(SE, Guards, Phi, VisitedBlocks, IncomingGuards, Depth);
16247 }
16248
16249 // Now apply the information from the collected conditions to
16250 // Guards.RewriteMap. Conditions are processed in reverse order, so the
16251 // earliest conditions is processed first, except guards with divisibility
16252 // information, which are moved to the back. This ensures the SCEVs with the
16253 // shortest dependency chains are constructed first.
16254 SmallVector<std::tuple<CmpInst::Predicate, const SCEV *, const SCEV *>>
16255 GuardsToProcess;
16256 for (auto [Term, EnterIfTrue] : reverse(C&: Terms)) {
16257 SmallVector<Value *, 8> Worklist;
16258 SmallPtrSet<Value *, 8> Visited;
16259 Worklist.push_back(Elt: Term);
16260 while (!Worklist.empty()) {
16261 Value *Cond = Worklist.pop_back_val();
16262 if (!Visited.insert(Ptr: Cond).second)
16263 continue;
16264
16265 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
16266 auto Predicate =
16267 EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate();
16268 const auto *LHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 0));
16269 const auto *RHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 1));
16270 // If LHS is a constant, apply information to the other expression.
16271 // TODO: If LHS is not a constant, check if using CompareSCEVComplexity
16272 // can improve results.
16273 if (isa<SCEVConstant>(Val: LHS)) {
16274 std::swap(a&: LHS, b&: RHS);
16275 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
16276 }
16277 GuardsToProcess.emplace_back(Args&: Predicate, Args&: LHS, Args&: RHS);
16278 continue;
16279 }
16280
16281 Value *L, *R;
16282 if (EnterIfTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: L), R: m_Value(V&: R)))
16283 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: L), R: m_Value(V&: R)))) {
16284 Worklist.push_back(Elt: L);
16285 Worklist.push_back(Elt: R);
16286 }
16287 }
16288 }
16289
16290 // Process divisibility guards in reverse order to populate DivGuards early.
16291 DenseMap<const SCEV *, APInt> Multiples;
16292 LoopGuards DivGuards(SE);
16293 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess) {
16294 if (!isDivisibilityGuard(LHS, RHS, SE))
16295 continue;
16296 collectDivisibilityInformation(Predicate, LHS, RHS, DivInfo&: DivGuards.RewriteMap,
16297 Multiples, SE);
16298 }
16299
16300 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess)
16301 CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap, DivGuards);
16302
16303 // Apply divisibility information last. This ensures it is applied to the
16304 // outermost expression after other rewrites for the given value.
16305 for (const auto &[K, Divisor] : Multiples) {
16306 const SCEV *DivisorSCEV = SE.getConstant(Val: Divisor);
16307 Guards.RewriteMap[K] =
16308 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: applyDivisibilityOnMinMaxExpr(
16309 MinMaxExpr: Guards.rewrite(Expr: K), Divisor, SE),
16310 RHS: DivisorSCEV),
16311 RHS: DivisorSCEV);
16312 ExprsToRewrite.push_back(Elt: K);
16313 }
16314
16315 // Let the rewriter preserve NUW/NSW flags if the unsigned/signed ranges of
16316 // the replacement expressions are contained in the ranges of the replaced
16317 // expressions.
16318 Guards.PreserveNUW = true;
16319 Guards.PreserveNSW = true;
16320 for (const SCEV *Expr : ExprsToRewrite) {
16321 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16322 Guards.PreserveNUW &=
16323 SE.getUnsignedRange(S: Expr).contains(CR: SE.getUnsignedRange(S: RewriteTo));
16324 Guards.PreserveNSW &=
16325 SE.getSignedRange(S: Expr).contains(CR: SE.getSignedRange(S: RewriteTo));
16326 }
16327
16328 // Now that all rewrite information is collect, rewrite the collected
16329 // expressions with the information in the map. This applies information to
16330 // sub-expressions.
16331 if (ExprsToRewrite.size() > 1) {
16332 for (const SCEV *Expr : ExprsToRewrite) {
16333 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16334 Guards.RewriteMap.erase(Val: Expr);
16335 Guards.RewriteMap.insert(KV: {Expr, Guards.rewrite(Expr: RewriteTo)});
16336 }
16337 }
16338}
16339
16340const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
16341 /// A rewriter to replace SCEV expressions in Map with the corresponding entry
16342 /// in the map. It skips AddRecExpr because we cannot guarantee that the
16343 /// replacement is loop invariant in the loop of the AddRec.
16344 class SCEVLoopGuardRewriter
16345 : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> {
16346 const DenseMap<const SCEV *, const SCEV *> &Map;
16347 const SmallDenseSet<std::pair<const SCEV *, const SCEV *>> &NotEqual;
16348
16349 SCEV::NoWrapFlags FlagMask = SCEV::FlagAnyWrap;
16350
16351 public:
16352 SCEVLoopGuardRewriter(ScalarEvolution &SE,
16353 const ScalarEvolution::LoopGuards &Guards)
16354 : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap),
16355 NotEqual(Guards.NotEqual) {
16356 if (Guards.PreserveNUW)
16357 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNUW);
16358 if (Guards.PreserveNSW)
16359 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNSW);
16360 }
16361
16362 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
16363
16364 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
16365 return Map.lookup_or(Val: Expr, Default&: Expr);
16366 }
16367
16368 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
16369 if (const SCEV *S = Map.lookup(Val: Expr))
16370 return S;
16371
16372 // If we didn't find the extact ZExt expr in the map, check if there's
16373 // an entry for a smaller ZExt we can use instead.
16374 Type *Ty = Expr->getType();
16375 const SCEV *Op = Expr->getOperand(i: 0);
16376 unsigned Bitwidth = Ty->getScalarSizeInBits() / 2;
16377 while (Bitwidth % 8 == 0 && Bitwidth >= 8 &&
16378 Bitwidth > Op->getType()->getScalarSizeInBits()) {
16379 Type *NarrowTy = IntegerType::get(C&: SE.getContext(), NumBits: Bitwidth);
16380 auto *NarrowExt = SE.getZeroExtendExpr(Op, Ty: NarrowTy);
16381 if (const SCEV *S = Map.lookup(Val: NarrowExt))
16382 return SE.getZeroExtendExpr(Op: S, Ty);
16383 Bitwidth = Bitwidth / 2;
16384 }
16385
16386 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitZeroExtendExpr(
16387 Expr);
16388 }
16389
16390 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
16391 if (const SCEV *S = Map.lookup(Val: Expr))
16392 return S;
16393 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSignExtendExpr(
16394 Expr);
16395 }
16396
16397 const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) {
16398 if (const SCEV *S = Map.lookup(Val: Expr))
16399 return S;
16400 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitUMinExpr(Expr);
16401 }
16402
16403 const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) {
16404 if (const SCEV *S = Map.lookup(Val: Expr))
16405 return S;
16406 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSMinExpr(Expr);
16407 }
16408
16409 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
16410 if (const SCEV *S = Map.lookup(Val: Expr))
16411 return S;
16412
16413 // Helper to check if S is a subtraction (A - B) where A != B, and if so,
16414 // return UMax(S, 1).
16415 auto RewriteSubtraction = [&](const SCEV *S) -> const SCEV * {
16416 SCEVUse LHS, RHS;
16417 if (MatchBinarySub(S, LHS, RHS)) {
16418 if (LHS > RHS)
16419 std::swap(a&: LHS, b&: RHS);
16420 if (NotEqual.contains(V: {LHS, RHS})) {
16421 const SCEV *OneAlignedUp = getNextSCEVDivisibleByDivisor(
16422 Expr: SE.getOne(Ty: S->getType()), DivisorVal: SE.getConstantMultiple(S), SE);
16423 return SE.getUMaxExpr(LHS: OneAlignedUp, RHS: S);
16424 }
16425 }
16426 return nullptr;
16427 };
16428
16429 // Check if Expr itself is a subtraction pattern with guard info.
16430 if (const SCEV *Rewritten = RewriteSubtraction(Expr))
16431 return Rewritten;
16432
16433 // Trip count expressions sometimes consist of adding 3 operands, i.e.
16434 // (Const + A + B). There may be guard info for A + B, and if so, apply
16435 // it.
16436 // TODO: Could more generally apply guards to Add sub-expressions.
16437 if (isa<SCEVConstant>(Val: Expr->getOperand(i: 0))) {
16438 if (Expr->getNumOperands() == 3) {
16439 const SCEV *Add =
16440 SE.getAddExpr(LHS: Expr->getOperand(i: 1), RHS: Expr->getOperand(i: 2));
16441 if (const SCEV *Rewritten = RewriteSubtraction(Add))
16442 return SE.getAddExpr(
16443 LHS: Expr->getOperand(i: 0), RHS: Rewritten,
16444 Flags: ScalarEvolution::maskFlags(Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16445 if (const SCEV *S = Map.lookup(Val: Add))
16446 return SE.getAddExpr(LHS: Expr->getOperand(i: 0), RHS: S);
16447 }
16448
16449 // For expressions of the form (Const + A), check if we have guard info
16450 // for (Const + 1 + A), and rewrite to ((Const + 1 + A) - 1). This makes
16451 // sure we don't lose information when rewriting expressions based on
16452 // back-edge taken counts in some cases.
16453 if (Expr->getNumOperands() == 2) {
16454 const SCEV *S = nullptr;
16455 // Handle (-1 + 1 + A) without constructing SCEVs.
16456 if (match(U: Expr->getOperand(i: 0), P: m_scev_AllOnes())) {
16457 S = Map.lookup(Val: Expr->getOperand(i: 1));
16458 } else {
16459 const SCEV *NewC =
16460 SE.getAddExpr(LHS: Expr->getOperand(i: 0), RHS: SE.getOne(Ty: Expr->getType()));
16461 S = Map.lookup(Val: SE.getAddExpr(LHS: NewC, RHS: Expr->getOperand(i: 1)));
16462 }
16463 if (S)
16464 return SE.getAddExpr(LHS: S, RHS: SE.getMinusOne(Ty: Expr->getType()));
16465 }
16466 }
16467 SmallVector<SCEVUse, 2> Operands;
16468 bool Changed = false;
16469 for (SCEVUse Op : Expr->operands()) {
16470 Operands.push_back(
16471 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
16472 Changed |= Op != Operands.back();
16473 }
16474 // We are only replacing operands with equivalent values, so transfer the
16475 // flags from the original expression.
16476 return !Changed ? Expr
16477 : SE.getAddExpr(Ops&: Operands,
16478 Flags: ScalarEvolution::maskFlags(
16479 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16480 }
16481
16482 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
16483 SmallVector<SCEVUse, 2> Operands;
16484 bool Changed = false;
16485 for (SCEVUse Op : Expr->operands()) {
16486 Operands.push_back(
16487 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
16488 Changed |= Op != Operands.back();
16489 }
16490 // We are only replacing operands with equivalent values, so transfer the
16491 // flags from the original expression.
16492 return !Changed ? Expr
16493 : SE.getMulExpr(Ops&: Operands,
16494 Flags: ScalarEvolution::maskFlags(
16495 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16496 }
16497 };
16498
16499 if (RewriteMap.empty() && NotEqual.empty())
16500 return Expr;
16501
16502 SCEVLoopGuardRewriter Rewriter(SE, *this);
16503 return Rewriter.visit(S: Expr);
16504}
16505
16506const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
16507 return applyLoopGuards(Expr, Guards: LoopGuards::collect(L, SE&: *this));
16508}
16509
16510const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr,
16511 const LoopGuards &Guards) {
16512 return Guards.rewrite(Expr);
16513}
16514