1//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the scalar evolution analysis
10// engine, which is used primarily to analyze expressions involving induction
11// variables in loops.
12//
13// There are several aspects to this library. First is the representation of
14// scalar expressions, which are represented as subclasses of the SCEV class.
15// These classes are used to represent certain types of subexpressions that we
16// can handle. We only create one SCEV of a particular shape, so
17// pointer-comparisons for equality are legal.
18//
19// One important aspect of the SCEV objects is that they are never cyclic, even
20// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
21// the PHI node is one of the idioms that we can represent (e.g., a polynomial
22// recurrence) then we represent it directly as a recurrence node, otherwise we
23// represent it as a SCEVUnknown node.
24//
25// In addition to being able to represent expressions of various types, we also
26// have folders that are used to build the *canonical* representation for a
27// particular expression. These folders are capable of using a variety of
28// rewrite rules to simplify the expressions.
29//
30// Once the folders are defined, we can implement the more interesting
31// higher-level code, such as the code that recognizes PHI nodes of various
32// types, computes the execution count of a loop, etc.
33//
34// TODO: We should use these routines and value representations to implement
35// dependence analysis!
36//
37//===----------------------------------------------------------------------===//
38//
39// There are several good references for the techniques used in this analysis.
40//
41// Chains of recurrences -- a method to expedite the evaluation
42// of closed-form functions
43// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
44//
45// On computational properties of chains of recurrences
46// Eugene V. Zima
47//
48// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
49// Robert A. van Engelen
50//
51// Efficient Symbolic Analysis for Optimizing Compilers
52// Robert A. van Engelen
53//
54// Using the chains of recurrences algebra for data dependence testing and
55// induction variable substitution
56// MS Thesis, Johnie Birch
57//
58//===----------------------------------------------------------------------===//
59
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/ADT/APInt.h"
62#include "llvm/ADT/ArrayRef.h"
63#include "llvm/ADT/DenseMap.h"
64#include "llvm/ADT/DepthFirstIterator.h"
65#include "llvm/ADT/FoldingSet.h"
66#include "llvm/ADT/STLExtras.h"
67#include "llvm/ADT/ScopeExit.h"
68#include "llvm/ADT/Sequence.h"
69#include "llvm/ADT/SmallPtrSet.h"
70#include "llvm/ADT/SmallVector.h"
71#include "llvm/ADT/Statistic.h"
72#include "llvm/ADT/StringExtras.h"
73#include "llvm/ADT/StringRef.h"
74#include "llvm/Analysis/AssumptionCache.h"
75#include "llvm/Analysis/ConstantFolding.h"
76#include "llvm/Analysis/InstructionSimplify.h"
77#include "llvm/Analysis/LoopInfo.h"
78#include "llvm/Analysis/MemoryBuiltins.h"
79#include "llvm/Analysis/ScalarEvolutionExpressions.h"
80#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
81#include "llvm/Analysis/TargetLibraryInfo.h"
82#include "llvm/Analysis/ValueTracking.h"
83#include "llvm/Config/llvm-config.h"
84#include "llvm/IR/Argument.h"
85#include "llvm/IR/BasicBlock.h"
86#include "llvm/IR/CFG.h"
87#include "llvm/IR/Constant.h"
88#include "llvm/IR/ConstantRange.h"
89#include "llvm/IR/Constants.h"
90#include "llvm/IR/DataLayout.h"
91#include "llvm/IR/DerivedTypes.h"
92#include "llvm/IR/Dominators.h"
93#include "llvm/IR/Function.h"
94#include "llvm/IR/GlobalAlias.h"
95#include "llvm/IR/GlobalValue.h"
96#include "llvm/IR/InstIterator.h"
97#include "llvm/IR/InstrTypes.h"
98#include "llvm/IR/Instruction.h"
99#include "llvm/IR/Instructions.h"
100#include "llvm/IR/IntrinsicInst.h"
101#include "llvm/IR/Intrinsics.h"
102#include "llvm/IR/LLVMContext.h"
103#include "llvm/IR/Operator.h"
104#include "llvm/IR/PatternMatch.h"
105#include "llvm/IR/Type.h"
106#include "llvm/IR/Use.h"
107#include "llvm/IR/User.h"
108#include "llvm/IR/Value.h"
109#include "llvm/IR/Verifier.h"
110#include "llvm/InitializePasses.h"
111#include "llvm/Pass.h"
112#include "llvm/Support/Casting.h"
113#include "llvm/Support/CommandLine.h"
114#include "llvm/Support/Compiler.h"
115#include "llvm/Support/Debug.h"
116#include "llvm/Support/ErrorHandling.h"
117#include "llvm/Support/InterleavedRange.h"
118#include "llvm/Support/KnownBits.h"
119#include "llvm/Support/SaveAndRestore.h"
120#include "llvm/Support/raw_ostream.h"
121#include <algorithm>
122#include <cassert>
123#include <climits>
124#include <cstdint>
125#include <cstdlib>
126#include <map>
127#include <memory>
128#include <numeric>
129#include <optional>
130#include <tuple>
131#include <utility>
132#include <vector>
133
134using namespace llvm;
135using namespace PatternMatch;
136using namespace SCEVPatternMatch;
137
138#define DEBUG_TYPE "scalar-evolution"
139
140STATISTIC(NumExitCountsComputed,
141 "Number of loop exits with predictable exit counts");
142STATISTIC(NumExitCountsNotComputed,
143 "Number of loop exits without predictable exit counts");
144STATISTIC(NumBruteForceTripCountsComputed,
145 "Number of loops with trip counts computed by force");
146
147#ifdef EXPENSIVE_CHECKS
148bool llvm::VerifySCEV = true;
149#else
150bool llvm::VerifySCEV = false;
151#endif
152
153static cl::opt<unsigned>
154 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
155 cl::desc("Maximum number of iterations SCEV will "
156 "symbolically execute a constant "
157 "derived loop"),
158 cl::init(Val: 100));
159
160static cl::opt<bool, true> VerifySCEVOpt(
161 "verify-scev", cl::Hidden, cl::location(L&: VerifySCEV),
162 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
163static cl::opt<bool> VerifySCEVStrict(
164 "verify-scev-strict", cl::Hidden,
165 cl::desc("Enable stricter verification with -verify-scev is passed"));
166
167static cl::opt<bool> VerifyIR(
168 "scev-verify-ir", cl::Hidden,
169 cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"),
170 cl::init(Val: false));
171
172static cl::opt<unsigned> MulOpsInlineThreshold(
173 "scev-mulops-inline-threshold", cl::Hidden,
174 cl::desc("Threshold for inlining multiplication operands into a SCEV"),
175 cl::init(Val: 32));
176
177static cl::opt<unsigned> AddOpsInlineThreshold(
178 "scev-addops-inline-threshold", cl::Hidden,
179 cl::desc("Threshold for inlining addition operands into a SCEV"),
180 cl::init(Val: 500));
181
182static cl::opt<unsigned> MaxSCEVCompareDepth(
183 "scalar-evolution-max-scev-compare-depth", cl::Hidden,
184 cl::desc("Maximum depth of recursive SCEV complexity comparisons"),
185 cl::init(Val: 32));
186
187static cl::opt<unsigned> MaxSCEVOperationsImplicationDepth(
188 "scalar-evolution-max-scev-operations-implication-depth", cl::Hidden,
189 cl::desc("Maximum depth of recursive SCEV operations implication analysis"),
190 cl::init(Val: 2));
191
192static cl::opt<unsigned> MaxValueCompareDepth(
193 "scalar-evolution-max-value-compare-depth", cl::Hidden,
194 cl::desc("Maximum depth of recursive value complexity comparisons"),
195 cl::init(Val: 2));
196
197static cl::opt<unsigned>
198 MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden,
199 cl::desc("Maximum depth of recursive arithmetics"),
200 cl::init(Val: 32));
201
202static cl::opt<unsigned> MaxConstantEvolvingDepth(
203 "scalar-evolution-max-constant-evolving-depth", cl::Hidden,
204 cl::desc("Maximum depth of recursive constant evolving"), cl::init(Val: 32));
205
206static cl::opt<unsigned>
207 MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden,
208 cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"),
209 cl::init(Val: 8));
210
211static cl::opt<unsigned>
212 MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden,
213 cl::desc("Max coefficients in AddRec during evolving"),
214 cl::init(Val: 8));
215
216static cl::opt<unsigned>
217 HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden,
218 cl::desc("Size of the expression which is considered huge"),
219 cl::init(Val: 4096));
220
221static cl::opt<unsigned> RangeIterThreshold(
222 "scev-range-iter-threshold", cl::Hidden,
223 cl::desc("Threshold for switching to iteratively computing SCEV ranges"),
224 cl::init(Val: 32));
225
226static cl::opt<unsigned> MaxLoopGuardCollectionDepth(
227 "scalar-evolution-max-loop-guard-collection-depth", cl::Hidden,
228 cl::desc("Maximum depth for recursive loop guard collection"), cl::init(Val: 1));
229
230static cl::opt<bool>
231ClassifyExpressions("scalar-evolution-classify-expressions",
232 cl::Hidden, cl::init(Val: true),
233 cl::desc("When printing analysis, include information on every instruction"));
234
235static cl::opt<bool> UseExpensiveRangeSharpening(
236 "scalar-evolution-use-expensive-range-sharpening", cl::Hidden,
237 cl::init(Val: false),
238 cl::desc("Use more powerful methods of sharpening expression ranges. May "
239 "be costly in terms of compile time"));
240
241static cl::opt<bool>
242 EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden,
243 cl::desc("Handle <= and >= in finite loops"),
244 cl::init(Val: true));
245
246static cl::opt<bool> UseContextForNoWrapFlagInference(
247 "scalar-evolution-use-context-for-no-wrap-flag-strenghening", cl::Hidden,
248 cl::desc("Infer nuw/nsw flags using context where suitable"),
249 cl::init(Val: true));
250
251//===----------------------------------------------------------------------===//
252// SCEV class definitions
253//===----------------------------------------------------------------------===//
254
255void SCEV::computeAndSetCanonical(ScalarEvolution &SE) {
256 // Leaf nodes are always their own canonical.
257 switch (getSCEVType()) {
258 case scConstant:
259 case scVScale:
260 case scUnknown:
261 CanonicalSCEV = this;
262 return;
263 default:
264 break;
265 }
266
267 // For all other expressions, check whether any immediate operand has a
268 // different canonical. Since operands are always created before their parent,
269 // their canonical pointers are already set — no recursion needed.
270 if (all_of(Range: operands(), P: [](SCEVUse Op) { return Op.isCanonical(); })) {
271 CanonicalSCEV = this;
272 return;
273 }
274
275 SmallVector<SCEVUse, 4> CanonOps(
276 map_range(C: operands(), F: [](SCEVUse Op) { return Op.getCanonical(); }));
277 // Rebuild the expression from the canonical operands, stripping use flags.
278 CanonicalSCEV = SE.getWithOperands(S: this, NewOps&: CanonOps);
279}
280
281//===----------------------------------------------------------------------===//
282// Implementation of the SCEV class.
283//
284
285#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
286LLVM_DUMP_METHOD void SCEV::dump() const {
287 print(dbgs());
288 dbgs() << '\n';
289}
290#endif
291
292void SCEV::print(raw_ostream &OS) const {
293 switch (getSCEVType()) {
294 case scConstant:
295 cast<SCEVConstant>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
296 return;
297 case scVScale:
298 OS << "vscale";
299 return;
300 case scPtrToAddr: {
301 const SCEVCastExpr *PtrCast = cast<SCEVCastExpr>(Val: this);
302 SCEVUse Op = PtrCast->getOperand();
303 OS << "(ptrtoaddr " << *Op->getType() << " " << Op << " to "
304 << *PtrCast->getType() << ")";
305 return;
306 }
307 case scTruncate: {
308 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: this);
309 SCEVUse Op = Trunc->getOperand();
310 OS << "(trunc " << *Op->getType() << " " << Op << " to "
311 << *Trunc->getType() << ")";
312 return;
313 }
314 case scZeroExtend: {
315 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: this);
316 SCEVUse Op = ZExt->getOperand();
317 OS << "(zext " << *Op->getType() << " " << Op << " to " << *ZExt->getType()
318 << ")";
319 return;
320 }
321 case scSignExtend: {
322 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: this);
323 SCEVUse Op = SExt->getOperand();
324 OS << "(sext " << *Op->getType() << " " << Op << " to " << *SExt->getType()
325 << ")";
326 return;
327 }
328 case scAddRecExpr: {
329 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: this);
330 OS << "{" << AR->getOperand(i: 0);
331 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
332 OS << ",+," << AR->getOperand(i);
333 OS << "}<";
334 if (AR->hasNoUnsignedWrap())
335 OS << "nuw><";
336 if (AR->hasNoSignedWrap())
337 OS << "nsw><";
338 if (AR->hasNoSelfWrap() && !AR->hasNoUnsignedWrap() &&
339 !AR->hasNoSignedWrap())
340 OS << "nw><";
341 AR->getLoop()->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
342 OS << ">";
343 return;
344 }
345 case scAddExpr:
346 case scMulExpr:
347 case scUMaxExpr:
348 case scSMaxExpr:
349 case scUMinExpr:
350 case scSMinExpr:
351 case scSequentialUMinExpr: {
352 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(Val: this);
353 const char *OpStr = nullptr;
354 switch (NAry->getSCEVType()) {
355 case scAddExpr: OpStr = " + "; break;
356 case scMulExpr: OpStr = " * "; break;
357 case scUMaxExpr: OpStr = " umax "; break;
358 case scSMaxExpr: OpStr = " smax "; break;
359 case scUMinExpr:
360 OpStr = " umin ";
361 break;
362 case scSMinExpr:
363 OpStr = " smin ";
364 break;
365 case scSequentialUMinExpr:
366 OpStr = " umin_seq ";
367 break;
368 default:
369 llvm_unreachable("There are no other nary expression types.");
370 }
371 OS << "(" << llvm::interleaved(R: NAry->operands(), Separator: OpStr) << ")";
372 switch (NAry->getSCEVType()) {
373 case scAddExpr:
374 case scMulExpr:
375 if (NAry->hasNoUnsignedWrap())
376 OS << "<nuw>";
377 if (NAry->hasNoSignedWrap())
378 OS << "<nsw>";
379 break;
380 default:
381 // Nothing to print for other nary expressions.
382 break;
383 }
384 return;
385 }
386 case scUDivExpr: {
387 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: this);
388 OS << "(" << UDiv->getLHS() << " /u " << UDiv->getRHS() << ")";
389 return;
390 }
391 case scUnknown:
392 cast<SCEVUnknown>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
393 return;
394 case scCouldNotCompute:
395 OS << "***COULDNOTCOMPUTE***";
396 return;
397 }
398 llvm_unreachable("Unknown SCEV kind!");
399}
400
401ArrayRef<SCEVUse> SCEV::operands() const {
402 switch (getSCEVType()) {
403 case scConstant:
404 case scVScale:
405 case scUnknown:
406 return {};
407 case scPtrToAddr:
408 case scTruncate:
409 case scZeroExtend:
410 case scSignExtend:
411 return cast<SCEVCastExpr>(Val: this)->operands();
412 case scAddRecExpr:
413 case scAddExpr:
414 case scMulExpr:
415 case scUMaxExpr:
416 case scSMaxExpr:
417 case scUMinExpr:
418 case scSMinExpr:
419 case scSequentialUMinExpr:
420 return cast<SCEVNAryExpr>(Val: this)->operands();
421 case scUDivExpr:
422 return cast<SCEVUDivExpr>(Val: this)->operands();
423 case scCouldNotCompute:
424 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
425 }
426 llvm_unreachable("Unknown SCEV kind!");
427}
428
429bool SCEV::isZero() const { return match(S: this, P: m_scev_Zero()); }
430
431bool SCEV::isOne() const { return match(S: this, P: m_scev_One()); }
432
433bool SCEV::isAllOnesValue() const { return match(S: this, P: m_scev_AllOnes()); }
434
435bool SCEV::isNonConstantNegative() const {
436 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: this);
437 if (!Mul) return false;
438
439 // If there is a constant factor, it will be first.
440 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: 0));
441 if (!SC) return false;
442
443 // Return true if the value is negative, this matches things like (-42 * V).
444 return SC->getAPInt().isNegative();
445}
446
447SCEVCouldNotCompute::SCEVCouldNotCompute()
448 : SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0, nullptr) {}
449
450bool SCEVCouldNotCompute::classof(const SCEV *S) {
451 return S->getSCEVType() == scCouldNotCompute;
452}
453
454const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
455 auto &Entry = ConstantSCEVs[V];
456 if (Entry)
457 return Entry;
458
459 FoldingSetNodeID ID;
460 ID.AddInteger(I: scConstant);
461 ID.AddPointer(Ptr: V);
462 FoldingSetInsertToken Token;
463 if (SCEVConstant *S =
464 static_cast<SCEVConstant *>(UniqueSCEVs.lookup(ID, Token)))
465 return Entry = S;
466 SCEVConstant *S =
467 new (SCEVAllocator) SCEVConstant(ID.Intern(Allocator&: SCEVAllocator), V);
468 UniqueSCEVs.insert(N: S, Token);
469 S->computeAndSetCanonical(SE&: *this);
470 return Entry = S;
471}
472
473const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
474 return getConstant(V: ConstantInt::get(Context&: getContext(), V: Val));
475}
476
477const SCEV *
478ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
479 IntegerType *ITy = cast<IntegerType>(Val: getEffectiveSCEVType(Ty));
480 // TODO: Avoid implicit trunc?
481 // See https://github.com/llvm/llvm-project/issues/112510.
482 return getConstant(
483 V: ConstantInt::get(Ty: ITy, V, IsSigned: isSigned, /*ImplicitTrunc=*/true));
484}
485
486const SCEV *ScalarEvolution::getVScale(Type *Ty) {
487 FoldingSetNodeID ID;
488 ID.AddInteger(I: scVScale);
489 ID.AddPointer(Ptr: Ty);
490 FoldingSetInsertToken Token;
491 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
492 return S;
493 SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(Allocator&: SCEVAllocator), Ty);
494 UniqueSCEVs.insert(N: S, Token);
495 S->computeAndSetCanonical(SE&: *this);
496 return S;
497}
498
499const SCEV *ScalarEvolution::getElementCount(Type *Ty, ElementCount EC,
500 SCEVFlags Flags) {
501 const SCEV *Res = getConstant(Ty, V: EC.getKnownMinValue());
502 if (EC.isScalable())
503 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty), Flags);
504 return Res;
505}
506
507SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
508 SCEVUse op, Type *ty)
509 : SCEV(ID, SCEVTy, computeExpressionSize(Args: op), ty), Op(op) {}
510
511SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID,
512 const SCEV *Op, Type *ITy)
513 : SCEVCastExpr(ID, scPtrToAddr, Op, ITy) {
514 assert(getOperand()->getType()->isPointerTy() && getType()->isIntegerTy() &&
515 "Must be a non-bit-width-changing pointer-to-integer cast!");
516}
517
518SCEVIntegralCastExpr::SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID,
519 SCEVTypes SCEVTy, SCEVUse op,
520 Type *ty)
521 : SCEVCastExpr(ID, SCEVTy, op, ty) {}
522
523SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
524 Type *ty)
525 : SCEVIntegralCastExpr(ID, scTruncate, op, ty) {
526 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
527 "Cannot truncate non-integer value!");
528}
529
530SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
531 Type *ty)
532 : SCEVIntegralCastExpr(ID, scZeroExtend, op, ty) {
533 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
534 "Cannot zero extend non-integer value!");
535}
536
537SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
538 Type *ty)
539 : SCEVIntegralCastExpr(ID, scSignExtend, op, ty) {
540 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
541 "Cannot sign extend non-integer value!");
542}
543
544void SCEVUnknown::deleted() {
545 // Clear this SCEVUnknown from various maps.
546 SE->forgetMemoizedResults(SCEVs: {this});
547
548 // Remove this SCEVUnknown from the uniquing map.
549 SE->UniqueSCEVs.erase(N: this);
550
551 // Release the value.
552 setValPtr(nullptr);
553}
554
555void SCEVUnknown::allUsesReplacedWith(Value *New) {
556 // Clear this SCEVUnknown from various maps.
557 SE->forgetMemoizedResults(SCEVs: {this});
558
559 // Remove this SCEVUnknown from the uniquing map.
560 SE->UniqueSCEVs.erase(N: this);
561
562 // Replace the value pointer in case someone is still using this SCEVUnknown.
563 setValPtr(New);
564}
565
566//===----------------------------------------------------------------------===//
567// SCEV Utilities
568//===----------------------------------------------------------------------===//
569
570/// Compare the two values \p LV and \p RV in terms of their "complexity" where
571/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
572/// operands in SCEV expressions.
573static int CompareValueComplexity(const LoopInfo *const LI, Value *LV,
574 Value *RV, unsigned Depth) {
575 if (Depth > MaxValueCompareDepth)
576 return 0;
577
578 // Order pointer values after integer values. This helps SCEVExpander form
579 // GEPs.
580 bool LIsPointer = LV->getType()->isPointerTy(),
581 RIsPointer = RV->getType()->isPointerTy();
582 if (LIsPointer != RIsPointer)
583 return (int)LIsPointer - (int)RIsPointer;
584
585 // Compare getValueID values.
586 unsigned LID = LV->getValueID(), RID = RV->getValueID();
587 if (LID != RID)
588 return (int)LID - (int)RID;
589
590 // Sort arguments by their position.
591 if (const auto *LA = dyn_cast<Argument>(Val: LV)) {
592 const auto *RA = cast<Argument>(Val: RV);
593 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
594 return (int)LArgNo - (int)RArgNo;
595 }
596
597 if (const auto *LGV = dyn_cast<GlobalValue>(Val: LV)) {
598 const auto *RGV = cast<GlobalValue>(Val: RV);
599
600 if (auto L = LGV->getLinkage() - RGV->getLinkage())
601 return L;
602
603 const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
604 auto LT = GV->getLinkage();
605 return !(GlobalValue::isPrivateLinkage(Linkage: LT) ||
606 GlobalValue::isInternalLinkage(Linkage: LT));
607 };
608
609 // Use the names to distinguish the two values, but only if the
610 // names are semantically important.
611 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
612 return LGV->getName().compare(RHS: RGV->getName());
613 }
614
615 // For instructions, compare their loop depth, and their operand count. This
616 // is pretty loose.
617 if (const auto *LInst = dyn_cast<Instruction>(Val: LV)) {
618 const auto *RInst = cast<Instruction>(Val: RV);
619
620 // Compare loop depths.
621 const BasicBlock *LParent = LInst->getParent(),
622 *RParent = RInst->getParent();
623 if (LParent != RParent) {
624 unsigned LDepth = LI->getLoopDepth(BB: LParent),
625 RDepth = LI->getLoopDepth(BB: RParent);
626 if (LDepth != RDepth)
627 return (int)LDepth - (int)RDepth;
628 }
629
630 // Compare the number of operands.
631 unsigned LNumOps = LInst->getNumOperands(),
632 RNumOps = RInst->getNumOperands();
633 if (LNumOps != RNumOps)
634 return (int)LNumOps - (int)RNumOps;
635
636 for (unsigned Idx : seq(Size: LNumOps)) {
637 int Result = CompareValueComplexity(LI, LV: LInst->getOperand(i: Idx),
638 RV: RInst->getOperand(i: Idx), Depth: Depth + 1);
639 if (Result != 0)
640 return Result;
641 }
642 }
643
644 return 0;
645}
646
647// Return negative, zero, or positive, if LHS is less than, equal to, or greater
648// than RHS, respectively. A three-way result allows recursive comparisons to be
649// more efficient.
650// If the max analysis depth was reached, return std::nullopt, assuming we do
651// not know if they are equivalent for sure.
652static std::optional<int>
653CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
654 const SCEV *RHS, DominatorTree &DT, unsigned Depth = 0) {
655 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
656 if (LHS == RHS)
657 return 0;
658
659 // Primarily, sort the SCEVs by their getSCEVType().
660 SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
661 if (LType != RType)
662 return (int)LType - (int)RType;
663
664 if (Depth > MaxSCEVCompareDepth)
665 return std::nullopt;
666
667 // Aside from the getSCEVType() ordering, the particular ordering
668 // isn't very important except that it's beneficial to be consistent,
669 // so that (a + b) and (b + a) don't end up as different expressions.
670 switch (LType) {
671 case scUnknown: {
672 const SCEVUnknown *LU = cast<SCEVUnknown>(Val: LHS);
673 const SCEVUnknown *RU = cast<SCEVUnknown>(Val: RHS);
674
675 int X =
676 CompareValueComplexity(LI, LV: LU->getValue(), RV: RU->getValue(), Depth: Depth + 1);
677 return X;
678 }
679
680 case scConstant: {
681 const SCEVConstant *LC = cast<SCEVConstant>(Val: LHS);
682 const SCEVConstant *RC = cast<SCEVConstant>(Val: RHS);
683
684 // Compare constant values.
685 const APInt &LA = LC->getAPInt();
686 const APInt &RA = RC->getAPInt();
687 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
688 if (LBitWidth != RBitWidth)
689 return (int)LBitWidth - (int)RBitWidth;
690 return LA.ult(RHS: RA) ? -1 : 1;
691 }
692
693 case scVScale: {
694 const auto *LTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: LHS)->getType());
695 const auto *RTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: RHS)->getType());
696 return LTy->getBitWidth() - RTy->getBitWidth();
697 }
698
699 case scAddRecExpr: {
700 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(Val: LHS);
701 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(Val: RHS);
702
703 // There is always a dominance between two recs that are used by one SCEV,
704 // so we can safely sort recs by loop header dominance. We require such
705 // order in getAddExpr.
706 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
707 if (LLoop != RLoop) {
708 const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
709 assert(LHead != RHead && "Two loops share the same header?");
710 if (DT.dominates(A: LHead, B: RHead))
711 return 1;
712 assert(DT.dominates(RHead, LHead) &&
713 "No dominance between recurrences used by one SCEV?");
714 return -1;
715 }
716
717 [[fallthrough]];
718 }
719
720 case scTruncate:
721 case scZeroExtend:
722 case scSignExtend:
723 case scPtrToAddr:
724 case scAddExpr:
725 case scMulExpr:
726 case scUDivExpr:
727 case scSMaxExpr:
728 case scUMaxExpr:
729 case scSMinExpr:
730 case scUMinExpr:
731 case scSequentialUMinExpr: {
732 ArrayRef<SCEVUse> LOps = LHS->operands();
733 ArrayRef<SCEVUse> ROps = RHS->operands();
734
735 // Lexicographically compare n-ary-like expressions.
736 unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
737 if (LNumOps != RNumOps)
738 return (int)LNumOps - (int)RNumOps;
739
740 for (unsigned i = 0; i != LNumOps; ++i) {
741 auto X = CompareSCEVComplexity(LI, LHS: LOps[i].getPointer(),
742 RHS: ROps[i].getPointer(), DT, Depth: Depth + 1);
743 if (X != 0)
744 return X;
745 }
746 return 0;
747 }
748
749 case scCouldNotCompute:
750 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
751 }
752 llvm_unreachable("Unknown SCEV kind!");
753}
754
755/// Given a list of SCEV objects, order them by their complexity, and group
756/// objects of the same complexity together by value. When this routine is
757/// finished, we know that any duplicates in the vector are consecutive and that
758/// complexity is monotonically increasing.
759///
760/// Note that we go take special precautions to ensure that we get deterministic
761/// results from this routine. In other words, we don't want the results of
762/// this to depend on where the addresses of various SCEV objects happened to
763/// land in memory.
764static void GroupByComplexity(SmallVectorImpl<SCEVUse> &Ops, LoopInfo *LI,
765 DominatorTree &DT) {
766 if (Ops.size() < 2) return; // Noop
767
768 // Whether LHS has provably less complexity than RHS.
769 auto IsLessComplex = [&](SCEVUse LHS, SCEVUse RHS) {
770 auto Complexity = CompareSCEVComplexity(LI, LHS, RHS, DT);
771 return Complexity && *Complexity < 0;
772 };
773 if (Ops.size() == 2) {
774 // This is the common case, which also happens to be trivially simple.
775 // Special case it.
776 SCEVUse &LHS = Ops[0], &RHS = Ops[1];
777 if (IsLessComplex(RHS, LHS))
778 std::swap(a&: LHS, b&: RHS);
779 return;
780 }
781
782 // Do the rough sort by complexity.
783 llvm::stable_sort(
784 Range&: Ops, C: [&](SCEVUse LHS, SCEVUse RHS) { return IsLessComplex(LHS, RHS); });
785
786 // Now that we are sorted by complexity, group elements of the same
787 // complexity. Note that this is, at worst, N^2, but the vector is likely to
788 // be extremely short in practice. Note that we take this approach because we
789 // do not want to depend on the addresses of the objects we are grouping.
790 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
791 const SCEV *S = Ops[i];
792 unsigned Complexity = S->getSCEVType();
793
794 // If there are any objects of the same complexity and same value as this
795 // one, group them.
796 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
797 if (Ops[j] == S) { // Found a duplicate.
798 // Move it to immediately after i'th element.
799 std::swap(a&: Ops[i+1], b&: Ops[j]);
800 ++i; // no need to rescan it.
801 if (i == e-2) return; // Done!
802 }
803 }
804 }
805}
806
807/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
808/// least HugeExprThreshold nodes).
809static bool hasHugeExpression(ArrayRef<SCEVUse> Ops) {
810 return any_of(Range&: Ops, P: [](const SCEV *S) {
811 return S->getExpressionSize() >= HugeExprThreshold;
812 });
813}
814
815/// Performs a number of common optimizations on the passed \p Ops. If the
816/// whole expression reduces down to a single operand, it will be returned.
817///
818/// The following optimizations are performed:
819/// * Fold constants using the \p Fold function.
820/// * Remove identity constants satisfying \p IsIdentity.
821/// * If a constant satisfies \p IsAbsorber, return it.
822/// * Sort operands by complexity.
823template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
824static const SCEV *
825constantFoldAndGroupOps(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT,
826 SmallVectorImpl<SCEVUse> &Ops, FoldT Fold,
827 IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
828 const SCEVConstant *Folded = nullptr;
829 for (unsigned Idx = 0; Idx < Ops.size();) {
830 const SCEV *Op = Ops[Idx];
831 if (const auto *C = dyn_cast<SCEVConstant>(Val: Op)) {
832 if (!Folded)
833 Folded = C;
834 else
835 Folded = cast<SCEVConstant>(
836 SE.getConstant(Fold(Folded->getAPInt(), C->getAPInt())));
837 Ops.erase(CI: Ops.begin() + Idx);
838 continue;
839 }
840 ++Idx;
841 }
842
843 if (Ops.empty()) {
844 assert(Folded && "Must have folded value");
845 return Folded;
846 }
847
848 if (Folded && IsAbsorber(Folded->getAPInt()))
849 return Folded;
850
851 GroupByComplexity(Ops, LI: &LI, DT);
852 if (Folded && !IsIdentity(Folded->getAPInt()))
853 Ops.insert(I: Ops.begin(), Elt: Folded);
854
855 return Ops.size() == 1 ? Ops[0] : nullptr;
856}
857
858//===----------------------------------------------------------------------===//
859// Simple SCEV method implementations
860//===----------------------------------------------------------------------===//
861
862/// Compute BC(It, K). The result has width W. Assume, K > 0.
863static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
864 ScalarEvolution &SE,
865 Type *ResultTy) {
866 // Handle the simplest case efficiently.
867 if (K == 1)
868 return SE.getTruncateOrZeroExtend(V: It, Ty: ResultTy);
869
870 // We are using the following formula for BC(It, K):
871 //
872 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
873 //
874 // Suppose, W is the bitwidth of the return value. We must be prepared for
875 // overflow. Hence, we must assure that the result of our computation is
876 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
877 // safe in modular arithmetic.
878 //
879 // However, this code doesn't use exactly that formula; the formula it uses
880 // is something like the following, where T is the number of factors of 2 in
881 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
882 // exponentiation:
883 //
884 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
885 //
886 // This formula is trivially equivalent to the previous formula. However,
887 // this formula can be implemented much more efficiently. The trick is that
888 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
889 // arithmetic. To do exact division in modular arithmetic, all we have
890 // to do is multiply by the inverse. Therefore, this step can be done at
891 // width W.
892 //
893 // The next issue is how to safely do the division by 2^T. The way this
894 // is done is by doing the multiplication step at a width of at least W + T
895 // bits. This way, the bottom W+T bits of the product are accurate. Then,
896 // when we perform the division by 2^T (which is equivalent to a right shift
897 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
898 // truncated out after the division by 2^T.
899 //
900 // In comparison to just directly using the first formula, this technique
901 // is much more efficient; using the first formula requires W * K bits,
902 // but this formula less than W + K bits. Also, the first formula requires
903 // a division step, whereas this formula only requires multiplies and shifts.
904 //
905 // It doesn't matter whether the subtraction step is done in the calculation
906 // width or the input iteration count's width; if the subtraction overflows,
907 // the result must be zero anyway. We prefer here to do it in the width of
908 // the induction variable because it helps a lot for certain cases; CodeGen
909 // isn't smart enough to ignore the overflow, which leads to much less
910 // efficient code if the width of the subtraction is wider than the native
911 // register width.
912 //
913 // (It's possible to not widen at all by pulling out factors of 2 before
914 // the multiplication; for example, K=2 can be calculated as
915 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
916 // extra arithmetic, so it's not an obvious win, and it gets
917 // much more complicated for K > 3.)
918
919 // Protection from insane SCEVs; this bound is conservative,
920 // but it probably doesn't matter.
921 if (K > 1000)
922 return SE.getCouldNotCompute();
923
924 unsigned W = SE.getTypeSizeInBits(Ty: ResultTy);
925
926 // Calculate K! / 2^T and T; we divide out the factors of two before
927 // multiplying for calculating K! / 2^T to avoid overflow.
928 // Other overflow doesn't matter because we only care about the bottom
929 // W bits of the result.
930 APInt OddFactorial(W, 1);
931 unsigned T = 1;
932 for (unsigned i = 3; i <= K; ++i) {
933 unsigned TwoFactors = countr_zero(Val: i);
934 T += TwoFactors;
935 OddFactorial *= (i >> TwoFactors);
936 }
937
938 // We need at least W + T bits for the multiplication step
939 unsigned CalculationBits = W + T;
940
941 // Calculate 2^T, at width T+W.
942 APInt DivFactor = APInt::getOneBitSet(numBits: CalculationBits, BitNo: T);
943
944 // Calculate the multiplicative inverse of K! / 2^T;
945 // this multiplication factor will perform the exact division by
946 // K! / 2^T.
947 APInt MultiplyFactor = OddFactorial.multiplicativeInverse();
948
949 // Calculate the product, at width T+W
950 IntegerType *CalculationTy = IntegerType::get(C&: SE.getContext(),
951 NumBits: CalculationBits);
952 const SCEV *Dividend = SE.getTruncateOrZeroExtend(V: It, Ty: CalculationTy);
953 for (unsigned i = 1; i != K; ++i) {
954 const SCEV *S = SE.getMinusSCEV(LHS: It, RHS: SE.getConstant(Ty: It->getType(), V: i));
955 Dividend = SE.getMulExpr(LHS: Dividend,
956 RHS: SE.getTruncateOrZeroExtend(V: S, Ty: CalculationTy));
957 }
958
959 // Divide by 2^T
960 const SCEV *DivResult = SE.getUDivExpr(LHS: Dividend, RHS: SE.getConstant(Val: DivFactor));
961
962 // Truncate the result, and divide by K! / 2^T.
963
964 return SE.getMulExpr(LHS: SE.getConstant(Val: MultiplyFactor),
965 RHS: SE.getTruncateOrZeroExtend(V: DivResult, Ty: ResultTy));
966}
967
968/// Return the value of this chain of recurrences at the specified iteration
969/// number. We can evaluate this recurrence by multiplying each element in the
970/// chain by the binomial coefficient corresponding to it. In other words, we
971/// can evaluate {A,+,B,+,C,+,D} as:
972///
973/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
974///
975/// where BC(It, k) stands for binomial coefficient.
976const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
977 ScalarEvolution &SE) const {
978 return evaluateAtIteration(Operands: operands(), It, SE);
979}
980
981SCEVUse SCEVAddRecExpr::evaluateAtIteration(ArrayRef<SCEVUse> Operands,
982 const SCEV *It, ScalarEvolution &SE,
983 SCEVFlags UseFlags) {
984 assert(Operands.size() > 0);
985 assert((Operands.size() == 2 || UseFlags == SCEV::FlagNone) &&
986 "use-specific flags only supported for affine AddRecs");
987 SCEVUse Result = Operands[0].getPointer();
988 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
989 // The computation is correct in the face of overflow provided that the
990 // multiplication is performed _after_ the evaluation of the binomial
991 // coefficient.
992 const SCEV *Coeff = BinomialCoefficient(It, K: i, SE, ResultTy: Result->getType());
993 if (isa<SCEVCouldNotCompute>(Val: Coeff))
994 return Coeff;
995
996 SCEVUse Mul = SE.getMulExpr(LHS: Operands[i].getPointer(), RHS: Coeff,
997 Flags: {SCEV::FlagNone, UseFlags});
998 Result = SE.getAddExpr(LHS: Result, RHS: Mul, Flags: {SCEV::FlagNone, UseFlags});
999 }
1000 return Result;
1001}
1002
1003SCEVUse SCEVAddRecExpr::getExitValue(ScalarEvolution &SE) const {
1004 const SCEV *BTC = SE.getBackedgeTakenCount(L: getLoop());
1005 if (isa<SCEVCouldNotCompute>(Val: BTC))
1006 return BTC;
1007 // The loop reaches iteration BTC, so the value this recurrence computes there
1008 // is the value it had, and that did not wrap.
1009 return evaluateAtIteration(Operands: operands(), It: BTC, SE,
1010 UseFlags: isAffine() ? getNoWrapFlags(Mask: SCEV::FlagNUW)
1011 : SCEV::FlagNone);
1012}
1013
1014//===----------------------------------------------------------------------===//
1015// SCEV Expression folder implementations
1016//===----------------------------------------------------------------------===//
1017
1018/// The SCEVCastSinkingRewriter takes a scalar evolution expression,
1019/// which computes a pointer-typed value, and rewrites the whole expression
1020/// tree so that *all* the computations are done on integers, and the only
1021/// pointer-typed operands in the expression are SCEVUnknown.
1022/// The CreatePtrCast callback is invoked to create the actual conversion
1023/// (ptrtoint or ptrtoaddr) at the SCEVUnknown leaves.
1024class SCEVCastSinkingRewriter
1025 : public SCEVRewriteVisitor<SCEVCastSinkingRewriter> {
1026 using Base = SCEVRewriteVisitor<SCEVCastSinkingRewriter>;
1027 using ConversionFn = function_ref<const SCEV *(const SCEVUnknown *)>;
1028 Type *TargetTy;
1029 ConversionFn CreatePtrCast;
1030
1031public:
1032 SCEVCastSinkingRewriter(ScalarEvolution &SE, Type *TargetTy,
1033 ConversionFn CreatePtrCast)
1034 : Base(SE), TargetTy(TargetTy), CreatePtrCast(std::move(CreatePtrCast)) {}
1035
1036 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE,
1037 Type *TargetTy, ConversionFn CreatePtrCast) {
1038 SCEVCastSinkingRewriter Rewriter(SE, TargetTy, std::move(CreatePtrCast));
1039 return Rewriter.visit(S: Scev);
1040 }
1041
1042 const SCEV *visit(const SCEV *S) {
1043 Type *STy = S->getType();
1044 // If the expression is not pointer-typed, just keep it as-is.
1045 if (!STy->isPointerTy())
1046 return S;
1047 // Else, recursively sink the cast down into it.
1048 return Base::visit(S);
1049 }
1050
1051 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
1052 // Preserve wrap flags on rewritten SCEVAddExpr, which the default
1053 // implementation drops.
1054 SmallVector<SCEVUse, 2> Operands;
1055 bool Changed = false;
1056 for (SCEVUse Op : Expr->operands()) {
1057 Operands.push_back(Elt: visit(S: Op.getPointer()));
1058 Changed |= Op.getPointer() != Operands.back();
1059 }
1060 return !Changed ? Expr : SE.getAddExpr(Ops&: Operands, Flags: Expr->getNoWrapFlags());
1061 }
1062
1063 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
1064 assert(Expr->getType()->isPointerTy() &&
1065 "Should only reach pointer-typed SCEVUnknown's.");
1066 // Perform some basic constant folding. If the operand of the cast is a
1067 // null pointer, don't create a cast SCEV expression (that will be left
1068 // as-is), but produce a zero constant.
1069 if (isa<ConstantPointerNull>(Val: Expr->getValue()))
1070 return SE.getZero(Ty: TargetTy);
1071 return CreatePtrCast(Expr);
1072 }
1073};
1074
1075const SCEV *ScalarEvolution::getPtrToAddrExpr(const SCEV *Op) {
1076 assert(Op->getType()->isPointerTy() && "Op must be a pointer");
1077
1078 // Treat pointers with unstable representation conservatively, since the
1079 // address bits may change.
1080 if (DL.hasUnstableRepresentation(Ty: Op->getType()))
1081 return getCouldNotCompute();
1082
1083 Type *Ty = DL.getAddressType(PtrTy: Op->getType());
1084
1085 // Use the rewriter to sink the cast down to SCEVUnknown leaves.
1086 // The rewriter handles null pointer constant folding.
1087 const SCEV *IntOp = SCEVCastSinkingRewriter::rewrite(
1088 Scev: Op, SE&: *this, TargetTy: Ty, CreatePtrCast: [this, Ty](const SCEVUnknown *U) {
1089 FoldingSetNodeID ID;
1090 ID.AddInteger(I: scPtrToAddr);
1091 ID.AddPointer(Ptr: U);
1092 ID.AddPointer(Ptr: Ty);
1093 FoldingSetInsertToken Token;
1094 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1095 return S;
1096 SCEV *S = new (SCEVAllocator)
1097 SCEVPtrToAddrExpr(ID.Intern(Allocator&: SCEVAllocator), U, Ty);
1098 UniqueSCEVs.insert(N: S, Token);
1099 S->computeAndSetCanonical(SE&: *this);
1100 registerUser(User: S, Ops: {U});
1101 return static_cast<const SCEV *>(S);
1102 });
1103 assert(IntOp->getType()->isIntegerTy() &&
1104 "We must have succeeded in sinking the cast, "
1105 "and ending up with an integer-typed expression!");
1106 return IntOp;
1107}
1108
1109const SCEV *ScalarEvolution::getTruncateExpr(SCEVUse Op, Type *Ty,
1110 unsigned Depth) {
1111 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
1112 "This is not a truncating conversion!");
1113 assert(isSCEVable(Ty) &&
1114 "This is not a conversion to a SCEVable type!");
1115 assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!");
1116 Ty = getEffectiveSCEVType(Ty);
1117
1118 FoldingSetNodeID ID;
1119 ID.AddInteger(I: scTruncate);
1120 ID.AddPointer(Ptr: Op.getOpaqueValue());
1121 ID.AddPointer(Ptr: Ty);
1122 FoldingSetInsertToken Token;
1123 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1124 return S;
1125
1126 // Fold if the operand is constant.
1127 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1128 return getConstant(
1129 V: cast<ConstantInt>(Val: ConstantExpr::getTrunc(C: SC->getValue(), Ty)));
1130
1131 // trunc(trunc(x)) --> trunc(x)
1132 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op))
1133 return getTruncateExpr(Op: ST->getOperand(), Ty, Depth: Depth + 1);
1134
1135 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
1136 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val&: Op))
1137 return getTruncateOrSignExtend(V: SS->getOperand(), Ty, Depth: Depth + 1);
1138
1139 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
1140 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1141 return getTruncateOrZeroExtend(V: SZ->getOperand(), Ty, Depth: Depth + 1);
1142
1143 if (Depth > MaxCastDepth) {
1144 SCEV *S =
1145 new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator), Op, Ty);
1146 UniqueSCEVs.insert(N: S, Token);
1147 S->computeAndSetCanonical(SE&: *this);
1148 registerUser(User: S, Ops: Op);
1149 return S;
1150 }
1151
1152 // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
1153 // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
1154 // if after transforming we have at most one truncate, not counting truncates
1155 // that replace other casts.
1156 if (isa<SCEVAddExpr>(Val: Op) || isa<SCEVMulExpr>(Val: Op)) {
1157 auto *CommOp = cast<SCEVCommutativeExpr>(Val&: Op);
1158 SmallVector<SCEVUse, 4> Operands;
1159 unsigned numTruncs = 0;
1160 for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1161 ++i) {
1162 const SCEV *S = getTruncateExpr(Op: CommOp->getOperand(i), Ty, Depth: Depth + 1);
1163 if (!isa<SCEVIntegralCastExpr>(Val: CommOp->getOperand(i)) &&
1164 isa<SCEVTruncateExpr>(Val: S))
1165 numTruncs++;
1166 Operands.push_back(Elt: S);
1167 }
1168 if (numTruncs < 2) {
1169 if (isa<SCEVAddExpr>(Val: Op))
1170 return getAddExpr(Ops&: Operands);
1171 if (isa<SCEVMulExpr>(Val: Op))
1172 return getMulExpr(Ops&: Operands);
1173 llvm_unreachable("Unexpected SCEV type for Op.");
1174 }
1175 // Although we checked in the beginning that ID is not in the cache, it is
1176 // possible that during recursion and different modification ID was inserted
1177 // into the cache. So if we find it, just return it.
1178 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1179 return S;
1180 }
1181
1182 // If the input value is a chrec scev, truncate the chrec's operands.
1183 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: Op)) {
1184 SmallVector<SCEVUse, 4> Operands;
1185 for (const SCEV *Op : AddRec->operands())
1186 Operands.push_back(Elt: getTruncateExpr(Op, Ty, Depth: Depth + 1));
1187 return getAddRecExpr(Operands, L: AddRec->getLoop(), Flags: SCEV::FlagNone);
1188 }
1189
1190 // Return zero if truncating to known zeros.
1191 uint32_t MinTrailingZeros = getMinTrailingZeros(S: Op);
1192 if (MinTrailingZeros >= getTypeSizeInBits(Ty))
1193 return getZero(Ty);
1194
1195 // The cast wasn't folded; create an explicit cast node. We can reuse
1196 // the existing insert position since if we get here, we won't have
1197 // made any changes which would invalidate it.
1198 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator),
1199 Op, Ty);
1200 UniqueSCEVs.insert(N: S, Token);
1201 S->computeAndSetCanonical(SE&: *this);
1202 registerUser(User: S, Ops: Op);
1203 return S;
1204}
1205
1206// Get the limit of a recurrence such that incrementing by Step cannot cause
1207// signed overflow as long as the value of the recurrence within the
1208// loop does not exceed this limit before incrementing.
1209static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
1210 ICmpInst::Predicate *Pred,
1211 ScalarEvolution *SE) {
1212 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1213 if (SE->isKnownPositive(S: Step)) {
1214 *Pred = ICmpInst::ICMP_SLT;
1215 return SE->getConstant(Val: APInt::getSignedMinValue(numBits: BitWidth) -
1216 SE->getSignedRangeMax(S: Step));
1217 }
1218 if (SE->isKnownNegative(S: Step)) {
1219 *Pred = ICmpInst::ICMP_SGT;
1220 return SE->getConstant(Val: APInt::getSignedMaxValue(numBits: BitWidth) -
1221 SE->getSignedRangeMin(S: Step));
1222 }
1223 return nullptr;
1224}
1225
1226// Get the limit of a recurrence such that incrementing by Step cannot cause
1227// unsigned overflow as long as the value of the recurrence within the loop does
1228// not exceed this limit before incrementing.
1229static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step,
1230 ICmpInst::Predicate *Pred,
1231 ScalarEvolution *SE) {
1232 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1233 *Pred = ICmpInst::ICMP_ULT;
1234
1235 return SE->getConstant(Val: APInt::getMinValue(numBits: BitWidth) -
1236 SE->getUnsignedRangeMax(S: Step));
1237}
1238
1239namespace {
1240
1241struct ExtendOpTraitsBase {
1242 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(SCEVUse, Type *,
1243 unsigned);
1244};
1245
1246// Used to make code generic over signed and unsigned overflow.
1247template <typename ExtendOp> struct ExtendOpTraits {
1248 // Members present:
1249 //
1250 // static const SCEVFlags WrapType;
1251 //
1252 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
1253 //
1254 // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1255 // ICmpInst::Predicate *Pred,
1256 // ScalarEvolution *SE);
1257};
1258
1259template <>
1260struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
1261 static const SCEVFlags WrapType = SCEV::FlagNSW;
1262
1263 static const GetExtendExprTy GetExtendExpr;
1264
1265 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1266 ICmpInst::Predicate *Pred,
1267 ScalarEvolution *SE) {
1268 return getSignedOverflowLimitForStep(Step, Pred, SE);
1269 }
1270};
1271
1272const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1273 SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr;
1274
1275template <>
1276struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
1277 static const SCEVFlags WrapType = SCEV::FlagNUW;
1278
1279 static const GetExtendExprTy GetExtendExpr;
1280
1281 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1282 ICmpInst::Predicate *Pred,
1283 ScalarEvolution *SE) {
1284 return getUnsignedOverflowLimitForStep(Step, Pred, SE);
1285 }
1286};
1287
1288const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1289 SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr;
1290
1291} // end anonymous namespace
1292
1293// The recurrence AR has been shown to have no signed/unsigned wrap or something
1294// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
1295// easily prove NSW/NUW for its preincrement or postincrement sibling. This
1296// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
1297// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
1298// expression "Step + sext/zext(PreIncAR)" is congruent with
1299// "sext/zext(PostIncAR)"
1300template <typename ExtendOpTy>
1301static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR,
1302 ScalarEvolution *SE, unsigned Depth) {
1303 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1304 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1305
1306 const Loop *L = AR->getLoop();
1307 const SCEV *Start = AR->getStart();
1308 const SCEV *Step = AR->getStepRecurrence(SE&: *SE);
1309
1310 // Check for a simple looking step prior to loop entry.
1311 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Val: Start);
1312 if (!SA)
1313 return nullptr;
1314
1315 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
1316 // subtraction is expensive. For this purpose, perform a quick and dirty
1317 // difference, by checking for Step in the operand list. Note, that
1318 // SA might have repeated ops, like %a + %a + ..., so only remove one.
1319 SmallVector<SCEVUse, 4> DiffOps(SA->operands());
1320 for (auto It = DiffOps.begin(); It != DiffOps.end(); ++It)
1321 if (*It == Step) {
1322 DiffOps.erase(CI: It);
1323 break;
1324 }
1325
1326 if (DiffOps.size() == SA->getNumOperands())
1327 return nullptr;
1328
1329 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
1330 // `Step`:
1331
1332 // 1. NSW/NUW flags on the step increment.
1333 auto PreStartFlags =
1334 ScalarEvolution::maskFlags(Flags: SA->getNoWrapFlags(), Mask: SCEV::FlagNUW);
1335 const SCEV *PreStart = SE->getAddExpr(Ops&: DiffOps, Flags: PreStartFlags);
1336 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
1337 Val: SE->getAddRecExpr(Start: PreStart, Step, L, Flags: SCEV::FlagNone));
1338
1339 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
1340 // "S+X does not sign/unsign-overflow".
1341 //
1342
1343 const SCEV *BECount = SE->getBackedgeTakenCount(L);
1344 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType)) &&
1345 !isa<SCEVCouldNotCompute>(Val: BECount) && SE->isKnownPositive(S: BECount))
1346 return PreStart;
1347
1348 // 2. Direct overflow check on the step operation's expression.
1349 unsigned BitWidth = SE->getTypeSizeInBits(Ty: AR->getType());
1350 Type *WideTy = IntegerType::get(C&: SE->getContext(), NumBits: BitWidth * 2);
1351 const SCEV *OperandExtendedStart =
1352 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
1353 (SE->*GetExtendExpr)(Step, WideTy, Depth));
1354 if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
1355 if (PreAR && any(AR->getNoWrapFlags(Mask: WrapType))) {
1356 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
1357 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
1358 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
1359 SE->setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(PreAR), Flags: WrapType);
1360 }
1361 return PreStart;
1362 }
1363
1364 // 3. Loop precondition.
1365 ICmpInst::Predicate Pred;
1366 const SCEV *OverflowLimit =
1367 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1368
1369 if (OverflowLimit &&
1370 SE->isLoopEntryGuardedByCond(L, Pred, LHS: PreStart, RHS: OverflowLimit))
1371 return PreStart;
1372
1373 return nullptr;
1374}
1375
1376// Get the normalized zero or sign extended expression for this AddRec's Start.
1377template <typename ExtendOpTy>
1378static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
1379 ScalarEvolution *SE,
1380 unsigned Depth) {
1381 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1382
1383 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, SE, Depth);
1384 if (!PreStart)
1385 return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
1386
1387 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(SE&: *SE), Ty,
1388 Depth),
1389 (SE->*GetExtendExpr)(PreStart, Ty, Depth));
1390}
1391
1392// Try to prove away overflow by looking at "nearby" add recurrences. A
1393// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
1394// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
1395//
1396// Formally:
1397//
1398// {S,+,X} == {S-T,+,X} + T
1399// => Ext({S,+,X}) == Ext({S-T,+,X} + T)
1400//
1401// If ({S-T,+,X} + T) does not overflow ... (1)
1402//
1403// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
1404//
1405// If {S-T,+,X} does not overflow ... (2)
1406//
1407// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
1408// == {Ext(S-T)+Ext(T),+,Ext(X)}
1409//
1410// If (S-T)+T does not overflow ... (3)
1411//
1412// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
1413// == {Ext(S),+,Ext(X)} == LHS
1414//
1415// Thus, if (1), (2) and (3) are true for some T, then
1416// Ext({S,+,X}) == {Ext(S),+,Ext(X)}
1417//
1418// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
1419// does not overflow" restricted to the 0th iteration. Therefore we only need
1420// to check for (1) and (2).
1421//
1422// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
1423// is `Delta` (defined below).
1424template <typename ExtendOpTy>
1425bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
1426 const SCEV *Step,
1427 const Loop *L) {
1428 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1429
1430 // We restrict `Start` to a constant to prevent SCEV from spending too much
1431 // time here. It is correct (but more expensive) to continue with a
1432 // non-constant `Start` and do a general SCEV subtraction to compute
1433 // `PreStart` below.
1434 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Val: Start);
1435 if (!StartC)
1436 return false;
1437
1438 APInt StartAI = StartC->getAPInt();
1439
1440 for (unsigned Delta : {-2, -1, 1, 2}) {
1441 const SCEV *PreStart = getConstant(Val: StartAI - Delta);
1442 const auto *PreAR = static_cast<SCEVAddRecExpr *>(
1443 findExistingSCEVInCache(SCEVType: scAddRecExpr, Ops: {PreStart, Step}, L));
1444
1445 // Give up if we don't already have the add recurrence we need because
1446 // actually constructing an add recurrence is relatively expensive.
1447 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType))) { // proves (2)
1448 const SCEV *DeltaS = getConstant(Ty: StartC->getType(), V: Delta);
1449 ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
1450 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1451 DeltaS, &Pred, this);
1452 if (Limit && isKnownPredicate(Pred, LHS: PreAR, RHS: Limit)) // proves (1)
1453 return true;
1454 }
1455 }
1456
1457 return false;
1458}
1459
1460// Finds an integer D for an expression (C + x + y + ...) such that the top
1461// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
1462// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
1463// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
1464// the (C + x + y + ...) expression is \p WholeAddExpr.
1465static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1466 const SCEVConstant *ConstantTerm,
1467 const SCEVAddExpr *WholeAddExpr) {
1468 const APInt &C = ConstantTerm->getAPInt();
1469 const unsigned BitWidth = C.getBitWidth();
1470 // Find number of trailing zeros of (x + y + ...) w/o the C first:
1471 uint32_t TZ = BitWidth;
1472 for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
1473 TZ = std::min(a: TZ, b: SE.getMinTrailingZeros(S: WholeAddExpr->getOperand(i: I)));
1474 if (TZ) {
1475 // Set D to be as many least significant bits of C as possible while still
1476 // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
1477 return TZ < BitWidth ? C.trunc(width: TZ).zext(width: BitWidth) : C;
1478 }
1479 return APInt(BitWidth, 0);
1480}
1481
1482// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
1483// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
1484// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
1485// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
1486static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1487 const APInt &ConstantStart,
1488 const SCEV *Step) {
1489 const unsigned BitWidth = ConstantStart.getBitWidth();
1490 const uint32_t TZ = SE.getMinTrailingZeros(S: Step);
1491 if (TZ)
1492 return TZ < BitWidth ? ConstantStart.trunc(width: TZ).zext(width: BitWidth)
1493 : ConstantStart;
1494 return APInt(BitWidth, 0);
1495}
1496
1497static void insertFoldCacheEntry(
1498 const ScalarEvolution::FoldID &ID, const SCEV *S,
1499 DenseMap<ScalarEvolution::FoldID, const SCEV *> &FoldCache,
1500 DenseMap<const SCEV *, SmallVector<ScalarEvolution::FoldID, 2>>
1501 &FoldCacheUser) {
1502 auto I = FoldCache.insert(KV: {ID, S});
1503 if (!I.second) {
1504 // Remove FoldCacheUser entry for ID when replacing an existing FoldCache
1505 // entry.
1506 auto &UserIDs = FoldCacheUser[I.first->second];
1507 assert(count(UserIDs, ID) == 1 && "unexpected duplicates in UserIDs");
1508 for (unsigned I = 0; I != UserIDs.size(); ++I)
1509 if (UserIDs[I] == ID) {
1510 std::swap(a&: UserIDs[I], b&: UserIDs.back());
1511 break;
1512 }
1513 UserIDs.pop_back();
1514 I.first->second = S;
1515 }
1516 FoldCacheUser[S].push_back(Elt: ID);
1517}
1518
1519const SCEV *ScalarEvolution::getZeroExtendExpr(SCEVUse Op, Type *Ty,
1520 unsigned Depth) {
1521 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1522 "This is not an extending conversion!");
1523 assert(isSCEVable(Ty) &&
1524 "This is not a conversion to a SCEVable type!");
1525 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1526 Ty = getEffectiveSCEVType(Ty);
1527
1528 FoldID ID(scZeroExtend, Op, Ty);
1529 if (const SCEV *S = FoldCache.lookup(Val: ID))
1530 return S;
1531
1532 const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
1533 if (!isa<SCEVZeroExtendExpr>(Val: S))
1534 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1535 return S;
1536}
1537
1538const SCEV *ScalarEvolution::getZeroExtendExprImpl(SCEVUse Op, Type *Ty,
1539 unsigned Depth) {
1540 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1541 "This is not an extending conversion!");
1542 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1543 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1544
1545 // Fold if the operand is constant.
1546 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1547 return getConstant(Val: SC->getAPInt().zext(width: getTypeSizeInBits(Ty)));
1548
1549 // zext(zext(x)) --> zext(x)
1550 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1551 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1552
1553 // If the operand is an affine AddRec with the no-unsigned-wrap flag, the
1554 // zero-extension distributes over the recurrence.
1555 const SCEV *Start, *Step;
1556 const Loop *L;
1557 if (Depth <= MaxCastDepth &&
1558 match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1559 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1560 if (AR->hasNoUnsignedWrap()) {
1561 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1562 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1563 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1564 }
1565 }
1566
1567 // Before doing any expensive analysis, check to see if we've already
1568 // computed a SCEV for this Op and Ty.
1569 FoldingSetNodeID ID;
1570 ID.AddInteger(I: scZeroExtend);
1571 ID.AddPointer(Ptr: Op.getOpaqueValue());
1572 ID.AddPointer(Ptr: Ty);
1573 FoldingSetInsertToken Token;
1574 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1575 return S;
1576 if (Depth > MaxCastDepth) {
1577 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1578 Op, Ty);
1579 UniqueSCEVs.insert(N: S, Token);
1580 S->computeAndSetCanonical(SE&: *this);
1581 registerUser(User: S, Ops: Op);
1582 return S;
1583 }
1584
1585 // zext(trunc(x)) --> zext(x) or x or trunc(x)
1586 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
1587 // It's possible the bits taken off by the truncate were all zero bits. If
1588 // so, we should be able to simplify this further.
1589 const SCEV *X = ST->getOperand();
1590 ConstantRange CR = getUnsignedRange(S: X);
1591 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1592 unsigned NewBits = getTypeSizeInBits(Ty);
1593 if (CR.truncate(BitWidth: TruncBits).zeroExtend(BitWidth: NewBits).contains(
1594 CR: CR.zextOrTrunc(BitWidth: NewBits)))
1595 return getTruncateOrZeroExtend(V: X, Ty, Depth);
1596 }
1597
1598 // If the input value is a chrec scev, and we can prove that the value
1599 // did not overflow the old, smaller, value, we can zero extend all of the
1600 // operands (often constants). This allows analysis of something like
1601 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
1602 if (match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1603 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1604 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1605
1606 // The no-unsigned-wrap case is handled before the uniquing lookup above.
1607
1608 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1609 // Note that this serves two purposes: It filters out loops that are
1610 // simply not analyzable, and it covers the case where this code is
1611 // being called from within backedge-taken count analysis, such that
1612 // attempting to ask for the backedge-taken count would likely result
1613 // in infinite recursion. In the later case, the analysis code will
1614 // cope with a conservative value, and it will take care to purge
1615 // that value once it has finished.
1616 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1617 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
1618 // Manually compute the final value for AR, checking for overflow.
1619
1620 // Check whether the backedge-taken count can be losslessly casted to
1621 // the addrec's type. The count is always unsigned.
1622 const SCEV *CastedMaxBECount =
1623 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
1624 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1625 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
1626 if (MaxBECount == RecastedMaxBECount) {
1627 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
1628 // Check whether Start+Step*MaxBECount has no unsigned overflow.
1629 const SCEV *ZMul =
1630 getMulExpr(LHS: CastedMaxBECount, RHS: Step, Flags: SCEV::FlagNone, Depth: Depth + 1);
1631 const SCEV *ZAdd = getZeroExtendExpr(
1632 Op: getAddExpr(LHS: Start, RHS: ZMul, Flags: SCEV::FlagNone, Depth: Depth + 1), Ty: WideTy,
1633 Depth: Depth + 1);
1634 const SCEV *WideStart = getZeroExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
1635 const SCEV *WideMaxBECount =
1636 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
1637 const SCEV *OperandExtendedAdd =
1638 getAddExpr(LHS: WideStart,
1639 RHS: getMulExpr(LHS: WideMaxBECount,
1640 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1641 Flags: SCEV::FlagNone, Depth: Depth + 1),
1642 Flags: SCEV::FlagNone, Depth: Depth + 1);
1643 if (ZAdd == OperandExtendedAdd) {
1644 // Cache knowledge of AR NUW, which is propagated to this AddRec.
1645 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1646 // Return the expression with the addrec on the outside.
1647 Start =
1648 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1649 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1650 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1651 }
1652 // Similar to above, only this time treat the step value as signed.
1653 // This covers loops that count down.
1654 OperandExtendedAdd =
1655 getAddExpr(LHS: WideStart,
1656 RHS: getMulExpr(LHS: WideMaxBECount,
1657 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1658 Flags: SCEV::FlagNone, Depth: Depth + 1),
1659 Flags: SCEV::FlagNone, Depth: Depth + 1);
1660 if (ZAdd == OperandExtendedAdd) {
1661 // Cache knowledge of AR NW, which is propagated to this AddRec.
1662 // Negative step causes unsigned wrap, but it still can't self-wrap.
1663 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1664 // Return the expression with the addrec on the outside.
1665 Start =
1666 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1667 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1668 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1669 }
1670 }
1671 }
1672
1673 // Normally, in the cases we can prove no-overflow via a
1674 // backedge guarding condition, we can also compute a backedge
1675 // taken count for the loop. The exceptions are assumptions and
1676 // guards present in the loop -- SCEV is not great at exploiting
1677 // these to compute max backedge taken counts, but can still use
1678 // these to prove lack of overflow. Use this fact to avoid
1679 // doing extra work that may not pay off.
1680 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount) || HasGuards ||
1681 !AC.assumptions().empty()) {
1682
1683 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1684 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
1685 if (AR->hasNoUnsignedWrap()) {
1686 // Same as nuw case above - duplicated here to avoid a compile time
1687 // issue. It's not clear that the order of checks does matter, but
1688 // it's one of two issue possible causes for a change which was
1689 // reverted. Be conservative for the moment.
1690 Start =
1691 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1692 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1693 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1694 }
1695
1696 // For a negative step, we can extend the operands iff doing so only
1697 // traverses values in the range zext([0,UINT_MAX]).
1698 if (isKnownNegative(S: Step)) {
1699 const SCEV *N =
1700 getConstant(Val: APInt::getMaxValue(numBits: BitWidth) - getSignedRangeMin(S: Step));
1701 if (isLoopBackedgeGuardedByCond(L, Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N) ||
1702 isKnownOnEveryIteration(Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N)) {
1703 // Cache knowledge of AR NW, which is propagated to this
1704 // AddRec. Negative step causes unsigned wrap, but it
1705 // still can't self-wrap.
1706 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1707 // Return the expression with the addrec on the outside.
1708 Start =
1709 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1710 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1711 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1712 }
1713 }
1714 }
1715
1716 // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
1717 // if D + (C - D + Step * n) could be proven to not unsigned wrap
1718 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1719 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
1720 const APInt &C = SC->getAPInt();
1721 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
1722 if (D != 0) {
1723 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1724 const SCEV *SResidual =
1725 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
1726 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1727 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: SCEV::FlagNSW | SCEV::FlagNUW,
1728 Depth: Depth + 1);
1729 }
1730 }
1731
1732 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1733 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1734 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1735 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1736 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1737 }
1738 }
1739
1740 // zext(A % B) --> zext(A) % zext(B)
1741 {
1742 const SCEV *LHS;
1743 const SCEV *RHS;
1744 if (match(U: Op, P: m_scev_URem(LHS: m_SCEV(V&: LHS), RHS: m_SCEV(V&: RHS), SE&: *this)))
1745 return getURemExpr(LHS: getZeroExtendExpr(Op: LHS, Ty, Depth: Depth + 1),
1746 RHS: getZeroExtendExpr(Op: RHS, Ty, Depth: Depth + 1));
1747 }
1748
1749 // zext(A / B) --> zext(A) / zext(B).
1750 if (auto *Div = dyn_cast<SCEVUDivExpr>(Val&: Op))
1751 return getUDivExpr(LHS: getZeroExtendExpr(Op: Div->getLHS(), Ty, Depth: Depth + 1),
1752 RHS: getZeroExtendExpr(Op: Div->getRHS(), Ty, Depth: Depth + 1));
1753
1754 if (auto *SA = dyn_cast<SCEVAddExpr>(Val&: Op)) {
1755 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
1756 if (SA->hasNoUnsignedWrap()) {
1757 // If the addition does not unsign overflow then we can, by definition,
1758 // commute the zero extension with the addition operation.
1759 SmallVector<SCEVUse, 4> Ops;
1760 for (SCEVUse Op : SA->operands())
1761 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1762 return getAddExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1763 }
1764
1765 const APInt *C, *C2;
1766 // zext (C + A)<nsw> -> (sext(C) + sext(A))<nsw> if zext (C + A)<nsw> >=s 0.
1767 // Currently the non-negative check is done manually, as isKnownNonNegative
1768 // is too expensive.
1769 if (SA->hasNoSignedWrap() &&
1770 match(V: SA, P: m_scev_Add(Op0: m_scev_APInt(C),
1771 Op1: m_scev_SMax(Op0: m_scev_APInt(C&: C2), Op1: m_SCEV()))) &&
1772 C->isNegative() && !C->isMinSignedValue() && C2->sge(RHS: C->abs())) {
1773 assert(isKnownNonNegative(SA) && "incorrectly determined non-negative");
1774 return getAddExpr(LHS: getSignExtendExpr(Op: SA->getOperand(i: 0), Ty, Depth: Depth + 1),
1775 RHS: getSignExtendExpr(Op: SA->getOperand(i: 1), Ty, Depth: Depth + 1),
1776 Flags: SCEV::FlagNSW, Depth: Depth + 1);
1777 }
1778
1779 // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
1780 // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
1781 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1782 //
1783 // Often address arithmetics contain expressions like
1784 // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
1785 // This transformation is useful while proving that such expressions are
1786 // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
1787 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1788 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1789 if (D != 0) {
1790 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1791 const SCEV *SResidual =
1792 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagNone, Depth);
1793 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1794 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1795 Depth: Depth + 1);
1796 }
1797 }
1798 }
1799
1800 if (auto *SM = dyn_cast<SCEVMulExpr>(Val&: Op)) {
1801 // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
1802 if (SM->hasNoUnsignedWrap()) {
1803 // If the multiply does not unsign overflow then we can, by definition,
1804 // commute the zero extension with the multiply operation.
1805 SmallVector<SCEVUse, 4> Ops;
1806 for (SCEVUse Op : SM->operands())
1807 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1808 return getMulExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1809 }
1810
1811 // zext(2^K * (trunc X to iN)) to iM ->
1812 // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
1813 //
1814 // Proof:
1815 //
1816 // zext(2^K * (trunc X to iN)) to iM
1817 // = zext((trunc X to iN) << K) to iM
1818 // = zext((trunc X to i{N-K}) << K)<nuw> to iM
1819 // (because shl removes the top K bits)
1820 // = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
1821 // = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
1822 //
1823 const APInt *C;
1824 const SCEV *TruncRHS;
1825 if (match(V: SM,
1826 P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_scev_Trunc(Op0: m_SCEV(V&: TruncRHS)))) &&
1827 C->isPowerOf2()) {
1828 int NewTruncBits =
1829 getTypeSizeInBits(Ty: SM->getOperand(i: 1)->getType()) - C->logBase2();
1830 Type *NewTruncTy = IntegerType::get(C&: getContext(), NumBits: NewTruncBits);
1831 return getMulExpr(
1832 LHS: getZeroExtendExpr(Op: SM->getOperand(i: 0), Ty),
1833 RHS: getZeroExtendExpr(Op: getTruncateExpr(Op: TruncRHS, Ty: NewTruncTy), Ty),
1834 Flags: SCEV::FlagNUW, Depth: Depth + 1);
1835 }
1836 }
1837
1838 // zext(umin(x, y)) -> umin(zext(x), zext(y))
1839 // zext(umax(x, y)) -> umax(zext(x), zext(y))
1840 if (isa<SCEVUMinExpr>(Val: Op) || isa<SCEVUMaxExpr>(Val: Op)) {
1841 auto *MinMax = cast<SCEVMinMaxExpr>(Val&: Op);
1842 SmallVector<SCEVUse, 4> Operands;
1843 for (SCEVUse Operand : MinMax->operands())
1844 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1845 if (isa<SCEVUMinExpr>(Val: MinMax))
1846 return getUMinExpr(Operands);
1847 return getUMaxExpr(Operands);
1848 }
1849
1850 // zext(umin_seq(x, y)) -> umin_seq(zext(x), zext(y))
1851 if (auto *MinMax = dyn_cast<SCEVSequentialMinMaxExpr>(Val&: Op)) {
1852 assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
1853 SmallVector<SCEVUse, 4> Operands;
1854 for (SCEVUse Operand : MinMax->operands())
1855 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1856 return getUMinExpr(Operands, /*Sequential*/ true);
1857 }
1858
1859 // The cast wasn't folded; create an explicit cast node.
1860 // Recompute the insert position, as it may have been invalidated.
1861 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1862 return S;
1863 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1864 Op, Ty);
1865 UniqueSCEVs.insert(N: S, Token);
1866 S->computeAndSetCanonical(SE&: *this);
1867 registerUser(User: S, Ops: Op);
1868 return S;
1869}
1870
1871const SCEV *ScalarEvolution::getSignExtendExpr(SCEVUse Op, Type *Ty,
1872 unsigned Depth) {
1873 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1874 "This is not an extending conversion!");
1875 assert(isSCEVable(Ty) &&
1876 "This is not a conversion to a SCEVable type!");
1877 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1878 Ty = getEffectiveSCEVType(Ty);
1879
1880 FoldID ID(scSignExtend, Op, Ty);
1881 if (const SCEV *S = FoldCache.lookup(Val: ID))
1882 return S;
1883
1884 const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
1885 if (!isa<SCEVSignExtendExpr>(Val: S))
1886 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1887 return S;
1888}
1889
1890const SCEV *ScalarEvolution::getSignExtendExprImpl(SCEVUse Op, Type *Ty,
1891 unsigned Depth) {
1892 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1893 "This is not an extending conversion!");
1894 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1895 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1896 Ty = getEffectiveSCEVType(Ty);
1897
1898 // Fold if the operand is constant.
1899 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1900 return getConstant(Val: SC->getAPInt().sext(width: getTypeSizeInBits(Ty)));
1901
1902 // sext(sext(x)) --> sext(x)
1903 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val&: Op))
1904 return getSignExtendExpr(Op: SS->getOperand(), Ty, Depth: Depth + 1);
1905
1906 // sext(zext(x)) --> zext(x)
1907 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1908 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1909
1910 // If the operand is an affine AddRec with the no-signed-wrap flag, the
1911 // sign-extension distributes over the recurrence.
1912 const SCEV *Start, *Step;
1913 const Loop *L;
1914 if (Depth <= MaxCastDepth &&
1915 match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1916 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1917 if (AR->hasNoSignedWrap()) {
1918 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1919 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1920 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1921 }
1922 }
1923
1924 // Before doing any expensive analysis, check to see if we've already
1925 // computed a SCEV for this Op and Ty.
1926 FoldingSetNodeID ID;
1927 ID.AddInteger(I: scSignExtend);
1928 ID.AddPointer(Ptr: Op.getOpaqueValue());
1929 ID.AddPointer(Ptr: Ty);
1930 FoldingSetInsertToken Token;
1931 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1932 return S;
1933 // Limit recursion depth.
1934 if (Depth > MaxCastDepth) {
1935 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1936 Op, Ty);
1937 UniqueSCEVs.insert(N: S, Token);
1938 S->computeAndSetCanonical(SE&: *this);
1939 registerUser(User: S, Ops: Op);
1940 return S;
1941 }
1942
1943 // sext(trunc(x)) --> sext(x) or x or trunc(x)
1944 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
1945 // It's possible the bits taken off by the truncate were all sign bits. If
1946 // so, we should be able to simplify this further.
1947 const SCEV *X = ST->getOperand();
1948 ConstantRange CR = getSignedRange(S: X);
1949 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1950 unsigned NewBits = getTypeSizeInBits(Ty);
1951 if (CR.truncate(BitWidth: TruncBits).signExtend(BitWidth: NewBits).contains(
1952 CR: CR.sextOrTrunc(BitWidth: NewBits)))
1953 return getTruncateOrSignExtend(V: X, Ty, Depth);
1954 }
1955
1956 if (auto *SA = dyn_cast<SCEVAddExpr>(Val&: Op)) {
1957 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
1958 if (SA->hasNoSignedWrap()) {
1959 // If the addition does not sign overflow then we can, by definition,
1960 // commute the sign extension with the addition operation.
1961 SmallVector<SCEVUse, 4> Ops;
1962 for (SCEVUse Op : SA->operands())
1963 Ops.push_back(Elt: getSignExtendExpr(Op, Ty, Depth: Depth + 1));
1964 return getAddExpr(Ops, Flags: SCEV::FlagNSW, Depth: Depth + 1);
1965 }
1966
1967 // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
1968 // if D + (C - D + x + y + ...) could be proven to not signed wrap
1969 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1970 //
1971 // For instance, this will bring two seemingly different expressions:
1972 // 1 + sext(5 + 20 * %x + 24 * %y) and
1973 // sext(6 + 20 * %x + 24 * %y)
1974 // to the same form:
1975 // 2 + sext(4 + 20 * %x + 24 * %y)
1976 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1977 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1978 if (D != 0) {
1979 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1980 const SCEV *SResidual =
1981 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagNone, Depth);
1982 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1983 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1984 Depth: Depth + 1);
1985 }
1986 }
1987 }
1988 // If the input value is a chrec scev, and we can prove that the value
1989 // did not overflow the old, smaller, value, we can sign extend all of the
1990 // operands (often constants). This allows analysis of something like
1991 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
1992 if (match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1993 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1994 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1995
1996 // The no-signed-wrap case is handled before the uniquing lookup above.
1997
1998 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1999 // Note that this serves two purposes: It filters out loops that are
2000 // simply not analyzable, and it covers the case where this code is
2001 // being called from within backedge-taken count analysis, such that
2002 // attempting to ask for the backedge-taken count would likely result
2003 // in infinite recursion. In the later case, the analysis code will
2004 // cope with a conservative value, and it will take care to purge
2005 // that value once it has finished.
2006 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
2007 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
2008 // Manually compute the final value for AR, checking for
2009 // overflow.
2010
2011 // Check whether the backedge-taken count can be losslessly casted to
2012 // the addrec's type. The count is always unsigned.
2013 const SCEV *CastedMaxBECount =
2014 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
2015 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
2016 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
2017 if (MaxBECount == RecastedMaxBECount) {
2018 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
2019 // Check whether Start+Step*MaxBECount has no signed overflow.
2020 const SCEV *SMul =
2021 getMulExpr(LHS: CastedMaxBECount, RHS: Step, Flags: SCEV::FlagNone, Depth: Depth + 1);
2022 const SCEV *SAdd = getSignExtendExpr(
2023 Op: getAddExpr(LHS: Start, RHS: SMul, Flags: SCEV::FlagNone, Depth: Depth + 1), Ty: WideTy,
2024 Depth: Depth + 1);
2025 const SCEV *WideStart = getSignExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
2026 const SCEV *WideMaxBECount =
2027 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
2028 const SCEV *OperandExtendedAdd =
2029 getAddExpr(LHS: WideStart,
2030 RHS: getMulExpr(LHS: WideMaxBECount,
2031 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2032 Flags: SCEV::FlagNone, Depth: Depth + 1),
2033 Flags: SCEV::FlagNone, Depth: Depth + 1);
2034 if (SAdd == OperandExtendedAdd) {
2035 // Cache knowledge of AR NSW, which is propagated to this AddRec.
2036 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2037 // Return the expression with the addrec on the outside.
2038 Start =
2039 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2040 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2041 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2042 }
2043 // Similar to above, only this time treat the step value as unsigned.
2044 // This covers loops that count up with an unsigned step.
2045 OperandExtendedAdd =
2046 getAddExpr(LHS: WideStart,
2047 RHS: getMulExpr(LHS: WideMaxBECount,
2048 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2049 Flags: SCEV::FlagNone, Depth: Depth + 1),
2050 Flags: SCEV::FlagNone, Depth: Depth + 1);
2051 if (SAdd == OperandExtendedAdd) {
2052 // If AR wraps around then
2053 //
2054 // abs(Step) * MaxBECount > unsigned-max(AR->getType())
2055 // => SAdd != OperandExtendedAdd
2056 //
2057 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
2058 // (SAdd == OperandExtendedAdd => AR is NW)
2059
2060 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
2061
2062 // Return the expression with the addrec on the outside.
2063 Start =
2064 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2065 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2066 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2067 }
2068 }
2069 }
2070
2071 auto NewFlags = proveNoSignedWrapViaInduction(AR);
2072 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
2073 if (AR->hasNoSignedWrap()) {
2074 // Same as nsw case above - duplicated here to avoid a compile time
2075 // issue. It's not clear that the order of checks does matter, but
2076 // it's one of two issue possible causes for a change which was
2077 // reverted. Be conservative for the moment.
2078 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2079 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2080 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2081 }
2082
2083 // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
2084 // if D + (C - D + Step * n) could be proven to not signed wrap
2085 // where D maximizes the number of trailing zeros of (C - D + Step * n)
2086 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
2087 const APInt &C = SC->getAPInt();
2088 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
2089 if (D != 0) {
2090 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
2091 const SCEV *SResidual =
2092 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
2093 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
2094 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
2095 Depth: Depth + 1);
2096 }
2097 }
2098
2099 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
2100 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2101 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2102 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2103 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2104 }
2105 }
2106
2107 // If the input value is provably positive and we could not simplify
2108 // away the sext build a zext instead.
2109 if (isKnownNonNegative(S: Op))
2110 return getZeroExtendExpr(Op, Ty, Depth: Depth + 1);
2111
2112 // sext(smin(x, y)) -> smin(sext(x), sext(y))
2113 // sext(smax(x, y)) -> smax(sext(x), sext(y))
2114 if (isa<SCEVSMinExpr>(Val: Op) || isa<SCEVSMaxExpr>(Val: Op)) {
2115 auto *MinMax = cast<SCEVMinMaxExpr>(Val&: Op);
2116 SmallVector<SCEVUse, 4> Operands;
2117 for (SCEVUse Operand : MinMax->operands())
2118 Operands.push_back(Elt: getSignExtendExpr(Op: Operand, Ty));
2119 if (isa<SCEVSMinExpr>(Val: MinMax))
2120 return getSMinExpr(Operands);
2121 return getSMaxExpr(Operands);
2122 }
2123
2124 // The cast wasn't folded; create an explicit cast node.
2125 // Recompute the insert position, as it may have been invalidated.
2126 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
2127 return S;
2128 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2129 Op, Ty);
2130 UniqueSCEVs.insert(N: S, Token);
2131 S->computeAndSetCanonical(SE&: *this);
2132 registerUser(User: S, Ops: Op);
2133 return S;
2134}
2135
2136const SCEV *ScalarEvolution::getCastExpr(SCEVTypes Kind, SCEVUse Op, Type *Ty) {
2137 switch (Kind) {
2138 case scTruncate:
2139 return getTruncateExpr(Op, Ty);
2140 case scZeroExtend:
2141 return getZeroExtendExpr(Op, Ty);
2142 case scSignExtend:
2143 return getSignExtendExpr(Op, Ty);
2144 case scPtrToAddr: {
2145 const SCEV *Expr = getPtrToAddrExpr(Op);
2146 assert(Expr->getType() == Ty && "requested type must match");
2147 return Expr;
2148 }
2149 default:
2150 llvm_unreachable("Not a SCEV cast expression!");
2151 }
2152}
2153
2154/// getAnyExtendExpr - Return a SCEV for the given operand extended with
2155/// unspecified bits out to the given type.
2156const SCEV *ScalarEvolution::getAnyExtendExpr(SCEVUse Op, Type *Ty) {
2157 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2158 "This is not an extending conversion!");
2159 assert(isSCEVable(Ty) &&
2160 "This is not a conversion to a SCEVable type!");
2161 Ty = getEffectiveSCEVType(Ty);
2162
2163 // Sign-extend negative constants.
2164 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
2165 if (SC->getAPInt().isNegative())
2166 return getSignExtendExpr(Op, Ty);
2167
2168 // Peel off a truncate cast.
2169 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
2170 const SCEV *NewOp = T->getOperand();
2171 if (getTypeSizeInBits(Ty: NewOp->getType()) < getTypeSizeInBits(Ty))
2172 return getAnyExtendExpr(Op: NewOp, Ty);
2173 return getTruncateOrNoop(V: NewOp, Ty);
2174 }
2175
2176 // Next try a zext cast. If the cast is folded, use it.
2177 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
2178 if (!isa<SCEVZeroExtendExpr>(Val: ZExt))
2179 return ZExt;
2180
2181 // Next try a sext cast. If the cast is folded, use it.
2182 const SCEV *SExt = getSignExtendExpr(Op, Ty);
2183 if (!isa<SCEVSignExtendExpr>(Val: SExt))
2184 return SExt;
2185
2186 // Force the cast to be folded into the operands of an addrec.
2187 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: Op)) {
2188 SmallVector<SCEVUse, 4> Ops;
2189 for (const SCEV *Op : AR->operands())
2190 Ops.push_back(Elt: getAnyExtendExpr(Op, Ty));
2191 return getAddRecExpr(Operands&: Ops, L: AR->getLoop(), Flags: SCEV::FlagNW);
2192 }
2193
2194 // If the expression is obviously signed, use the sext cast value.
2195 if (isa<SCEVSMaxExpr>(Val: Op))
2196 return SExt;
2197
2198 // Absent any other information, use the zext cast value.
2199 return ZExt;
2200}
2201
2202/// Process the given Ops list, which is a list of operands to be added under
2203/// the given scale, update the given map. This is a helper function for
2204/// getAddRecExpr. As an example of what it does, given a sequence of operands
2205/// that would form an add expression like this:
2206///
2207/// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
2208///
2209/// where A and B are constants, update the map with these values:
2210///
2211/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
2212///
2213/// and add 13 + A*B*29 to AccumulatedConstant.
2214/// This will allow getAddRecExpr to produce this:
2215///
2216/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
2217///
2218/// This form often exposes folding opportunities that are hidden in
2219/// the original operand list.
2220///
2221/// Return true iff it appears that any interesting folding opportunities
2222/// may be exposed. This helps getAddRecExpr short-circuit extra work in
2223/// the common case where no interesting opportunities are present, and
2224/// is also used as a check to avoid infinite recursion.
2225static bool CollectAddOperandsWithScales(SmallDenseMap<SCEVUse, APInt, 16> &M,
2226 SmallVectorImpl<SCEVUse> &NewOps,
2227 APInt &AccumulatedConstant,
2228 ArrayRef<SCEVUse> Ops,
2229 const APInt &Scale,
2230 ScalarEvolution &SE) {
2231 bool Interesting = false;
2232
2233 // Iterate over the add operands. They are sorted, with constants first.
2234 unsigned i = 0;
2235 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Ops[i])) {
2236 ++i;
2237 // Pull a buried constant out to the outside.
2238 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
2239 Interesting = true;
2240 AccumulatedConstant += Scale * C->getAPInt();
2241 }
2242
2243 // Next comes everything else. We're especially interested in multiplies
2244 // here, but they're in the middle, so just visit the rest with one loop.
2245 for (; i != Ops.size(); ++i) {
2246 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: Ops[i]);
2247 if (Mul && isa<SCEVConstant>(Val: Mul->getOperand(i: 0))) {
2248 APInt NewScale =
2249 Scale * cast<SCEVConstant>(Val: Mul->getOperand(i: 0))->getAPInt();
2250 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Val: Mul->getOperand(i: 1))) {
2251 // A multiplication of a constant with another add; recurse.
2252 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: Mul->getOperand(i: 1));
2253 Interesting |= CollectAddOperandsWithScales(
2254 M, NewOps, AccumulatedConstant, Ops: Add->operands(), Scale: NewScale, SE);
2255 } else {
2256 // A multiplication of a constant with some other value. Update
2257 // the map.
2258 SmallVector<SCEVUse, 4> MulOps(drop_begin(RangeOrContainer: Mul->operands()));
2259 const SCEV *Key = SE.getMulExpr(Ops&: MulOps);
2260 auto Pair = M.insert(KV: {Key, NewScale});
2261 if (Pair.second) {
2262 NewOps.push_back(Elt: Pair.first->first);
2263 } else {
2264 Pair.first->second += NewScale;
2265 // The map already had an entry for this value, which may indicate
2266 // a folding opportunity.
2267 Interesting = true;
2268 }
2269 }
2270 } else {
2271 // An ordinary operand. Update the map.
2272 auto Pair = M.insert(KV: {Ops[i], Scale});
2273 if (Pair.second) {
2274 NewOps.push_back(Elt: Pair.first->first);
2275 } else {
2276 Pair.first->second += Scale;
2277 // The map already had an entry for this value, which may indicate
2278 // a folding opportunity.
2279 Interesting = true;
2280 }
2281 }
2282 }
2283
2284 return Interesting;
2285}
2286
2287bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed,
2288 const SCEV *LHS, const SCEV *RHS,
2289 const Instruction *CtxI) {
2290 auto Operation = [this, BinOp](SCEVUse L, SCEVUse R) -> const SCEV * {
2291 switch (BinOp) {
2292 default:
2293 llvm_unreachable("Unsupported binary op");
2294 case Instruction::Add:
2295 return getAddExpr(LHS: L, RHS: R);
2296 case Instruction::Sub:
2297 return getMinusSCEV(LHS: L, RHS: R);
2298 case Instruction::Mul:
2299 return getMulExpr(LHS: L, RHS: R);
2300 }
2301 };
2302
2303 const SCEV *(ScalarEvolution::*Extension)(SCEVUse, Type *, unsigned) =
2304 Signed ? &ScalarEvolution::getSignExtendExpr
2305 : &ScalarEvolution::getZeroExtendExpr;
2306
2307 // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
2308 auto *NarrowTy = cast<IntegerType>(Val: LHS->getType());
2309 auto *WideTy =
2310 IntegerType::get(C&: NarrowTy->getContext(), NumBits: NarrowTy->getBitWidth() * 2);
2311
2312 const SCEV *A = (this->*Extension)(Operation(LHS, RHS), WideTy, 0);
2313 const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
2314 const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
2315 const SCEV *B = Operation(LHSB, RHSB);
2316 if (A == B)
2317 return true;
2318 // Can we use context to prove the fact we need?
2319 if (!CtxI)
2320 return false;
2321 // TODO: Support mul.
2322 if (BinOp == Instruction::Mul)
2323 return false;
2324 auto *RHSC = dyn_cast<SCEVConstant>(Val: RHS);
2325 // TODO: Lift this limitation.
2326 if (!RHSC)
2327 return false;
2328 APInt C = RHSC->getAPInt();
2329 unsigned NumBits = C.getBitWidth();
2330 bool IsSub = (BinOp == Instruction::Sub);
2331 bool IsNegativeConst = (Signed && C.isNegative());
2332 // Compute the direction and magnitude by which we need to check overflow.
2333 bool OverflowDown = IsSub ^ IsNegativeConst;
2334 APInt Magnitude = C;
2335 if (IsNegativeConst) {
2336 if (C == APInt::getSignedMinValue(numBits: NumBits))
2337 // TODO: SINT_MIN on inversion gives the same negative value, we don't
2338 // want to deal with that.
2339 return false;
2340 Magnitude = -C;
2341 }
2342
2343 ICmpInst::Predicate Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2344 if (OverflowDown) {
2345 // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
2346 APInt Min = Signed ? APInt::getSignedMinValue(numBits: NumBits)
2347 : APInt::getMinValue(numBits: NumBits);
2348 APInt Limit = Min + Magnitude;
2349 return isKnownPredicateAt(Pred, LHS: getConstant(Val: Limit), RHS: LHS, CtxI);
2350 } else {
2351 // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
2352 APInt Max = Signed ? APInt::getSignedMaxValue(numBits: NumBits)
2353 : APInt::getMaxValue(numBits: NumBits);
2354 APInt Limit = Max - Magnitude;
2355 return isKnownPredicateAt(Pred, LHS, RHS: getConstant(Val: Limit), CtxI);
2356 }
2357}
2358
2359std::optional<SCEVFlags> ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp(
2360 const OverflowingBinaryOperator *OBO) {
2361 // It cannot be done any better.
2362 if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
2363 return std::nullopt;
2364
2365 SCEVFlags Flags = SCEVFlags::FlagNone;
2366
2367 if (OBO->hasNoUnsignedWrap())
2368 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2369 if (OBO->hasNoSignedWrap())
2370 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2371
2372 bool Deduced = false;
2373
2374 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)OBO->getOpcode();
2375 const SCEV *LHS = getSCEV(V: OBO->getOperand(i_nocapture: 0));
2376 const SCEV *RHS = getSCEV(V: OBO->getOperand(i_nocapture: 1));
2377
2378 bool CanUseNSW = true;
2379 const APInt *ShiftAmt;
2380 // Treat `shl %a, C` as `mul %a, 1 << C`.
2381 if (match(V: OBO, P: m_Shl(L: m_Value(), R: m_APInt(Res&: ShiftAmt)))) {
2382 unsigned BitWidth = ShiftAmt->getBitWidth();
2383 if (ShiftAmt->uge(RHS: BitWidth))
2384 return std::nullopt;
2385 // NSW only transfers if the shift amount is < BitWidth - 1, as INT_MIN * -1
2386 // overflows.
2387 CanUseNSW = ShiftAmt->ult(RHS: BitWidth - 1);
2388 Opcode = Instruction::Mul;
2389 RHS = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: ShiftAmt->getZExtValue()));
2390 } else if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
2391 Opcode != Instruction::Mul) {
2392 return std::nullopt;
2393 }
2394
2395 const Instruction *CtxI =
2396 UseContextForNoWrapFlagInference ? dyn_cast<Instruction>(Val: OBO) : nullptr;
2397 if (!OBO->hasNoUnsignedWrap() &&
2398 willNotOverflow(BinOp: Opcode, /* Signed */ false, LHS, RHS, CtxI)) {
2399 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2400 Deduced = true;
2401 }
2402
2403 if (CanUseNSW && !OBO->hasNoSignedWrap() &&
2404 willNotOverflow(BinOp: Opcode, /* Signed */ true, LHS, RHS, CtxI)) {
2405 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2406 Deduced = true;
2407 }
2408
2409 if (Deduced)
2410 return Flags;
2411 return std::nullopt;
2412}
2413
2414// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
2415// `OldFlags' as can't-wrap behavior. Infer a more aggressive set of
2416// can't-overflow flags for the operation if possible.
2417static SCEVFlags StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type,
2418 ArrayRef<SCEVUse> Ops, SCEVFlags Flags) {
2419 using namespace std::placeholders;
2420
2421 using OBO = OverflowingBinaryOperator;
2422
2423 bool CanAnalyze =
2424 Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr;
2425 (void)CanAnalyze;
2426 assert(CanAnalyze && "don't call from other places!");
2427
2428 SCEVFlags SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
2429 SCEVFlags SignOrUnsignWrap =
2430 ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2431
2432 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
2433 auto IsKnownNonNegative = [&](SCEVUse U) {
2434 return SE->isKnownNonNegative(S: U);
2435 };
2436
2437 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Range&: Ops, P: IsKnownNonNegative))
2438 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SignOrUnsignMask);
2439
2440 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2441
2442 if (SignOrUnsignWrap != SignOrUnsignMask &&
2443 (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
2444 isa<SCEVConstant>(Val: Ops[0])) {
2445
2446 auto Opcode = [&] {
2447 switch (Type) {
2448 case scAddExpr:
2449 return Instruction::Add;
2450 case scMulExpr:
2451 return Instruction::Mul;
2452 default:
2453 llvm_unreachable("Unexpected SCEV op.");
2454 }
2455 }();
2456
2457 const APInt &C = cast<SCEVConstant>(Val: Ops[0])->getAPInt();
2458
2459 // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
2460 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
2461 auto NSWRegion =
2462 ConstantRange::makeExactNoWrapRegion(BinOp: Opcode, Other: C, NoWrapKind: OBO::NoSignedWrap);
2463 if (NSWRegion.contains(CR: SE->getSignedRange(S: Ops[1])))
2464 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2465 }
2466
2467 // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
2468 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
2469 auto NUWRegion =
2470 ConstantRange::makeExactNoWrapRegion(BinOp: Opcode, Other: C, NoWrapKind: OBO::NoUnsignedWrap);
2471 if (NUWRegion.contains(CR: SE->getUnsignedRange(S: Ops[1])))
2472 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2473 }
2474 }
2475
2476 // <0,+,nonnegative><nw> is also nuw
2477 // TODO: Add corresponding nsw case
2478 if (Type == scAddRecExpr && ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNW) &&
2479 !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) && Ops.size() == 2 &&
2480 Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
2481 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2482
2483 // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
2484 if (Type == scMulExpr && !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) &&
2485 Ops.size() == 2) {
2486 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[0]))
2487 if (UDiv->getOperand(i: 1) == Ops[1])
2488 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2489 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[1]))
2490 if (UDiv->getOperand(i: 1) == Ops[0])
2491 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2492 }
2493
2494 return Flags;
2495}
2496
2497bool ScalarEvolution::isAvailableAtLoopEntry(const SCEV *S, const Loop *L) {
2498 return isLoopInvariant(S, L) && properlyDominates(S, BB: L->getHeader());
2499}
2500
2501/// Get a canonical add expression, or something simpler if possible.
2502SCEVUse ScalarEvolution::getAddExpr(SmallVectorImpl<SCEVUse> &Ops,
2503 SCEVFlagsPair Flags, unsigned Depth) {
2504 SCEVFlags ExprFlags = Flags.ExprFlags;
2505 SCEVFlags UseFlags = Flags.UseFlags;
2506 assert(!(ExprFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2507 "only nuw or nsw allowed");
2508 assert(!(UseFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2509 "only nuw or nsw allowed");
2510 assert(!Ops.empty() && "Cannot get empty add!");
2511 if (Ops.size() == 1) return Ops[0];
2512#ifndef NDEBUG
2513 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
2514 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
2515 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
2516 "SCEVAddExpr operand types don't match!");
2517 unsigned NumPtrs = count_if(
2518 Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
2519 assert(NumPtrs <= 1 && "add has at most one pointer operand");
2520#endif
2521
2522 const SCEV *Folded = constantFoldAndGroupOps(
2523 SE&: *this, LI, DT, Ops,
2524 Fold: [](const APInt &C1, const APInt &C2) { return C1 + C2; },
2525 IsIdentity: [](const APInt &C) { return C.isZero(); }, // identity
2526 IsAbsorber: [](const APInt &C) { return false; }); // absorber
2527 if (Folded)
2528 return Folded;
2529
2530#ifndef NDEBUG
2531 // Keep track of operands after constant folding, for verification when adding
2532 // use-specific flags.
2533 const SmallVector<SCEVUse, 8> OrigOps(Ops.begin(), Ops.end());
2534#endif
2535
2536 unsigned Idx = isa<SCEVConstant>(Val: Ops[0]) ? 1 : 0;
2537
2538 // Delay expensive flag strengthening until necessary.
2539 auto ComputeFlags = [this, ExprFlags](ArrayRef<SCEVUse> Ops) {
2540 return StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops, Flags: ExprFlags);
2541 };
2542
2543 // Limit recursion calls depth.
2544 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
2545 return {getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
2546
2547 if (SCEV *S = findExistingSCEVInCache(SCEVType: scAddExpr, Ops)) {
2548 // Don't strengthen flags if we have no new information.
2549 SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
2550 if (Add->getNoWrapFlags(Mask: ExprFlags) != ExprFlags)
2551 Add->setNoWrapFlags(ComputeFlags(Ops));
2552 return {S, UseFlags};
2553 }
2554
2555 // Okay, check to see if the same value occurs in the operand list more than
2556 // once. If so, merge them together into an multiply expression. Since we
2557 // sorted the list, these values are required to be adjacent.
2558 Type *Ty = Ops[0]->getType();
2559 bool FoundMatch = false;
2560 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
2561 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2
2562 // Scan ahead to count how many equal operands there are.
2563 unsigned Count = 2;
2564 while (i+Count != e && Ops[i+Count] == Ops[i])
2565 ++Count;
2566 // Merge the values into a multiply.
2567 SCEVUse Scale = getConstant(Ty, V: Count);
2568 const SCEV *Mul = getMulExpr(LHS: Scale, RHS: Ops[i], Flags: SCEV::FlagNone, Depth: Depth + 1);
2569 if (Ops.size() == Count)
2570 return Mul;
2571 Ops[i] = Mul;
2572 Ops.erase(CS: Ops.begin()+i+1, CE: Ops.begin()+i+Count);
2573 --i; e -= Count - 1;
2574 FoundMatch = true;
2575 }
2576 if (FoundMatch)
2577 return getAddExpr(Ops, Flags: ExprFlags, Depth: Depth + 1);
2578
2579 // Check for truncates. If all the operands are truncated from the same
2580 // type, see if factoring out the truncate would permit the result to be
2581 // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
2582 // if the contents of the resulting outer trunc fold to something simple.
2583 auto FindTruncSrcType = [&]() -> Type * {
2584 // We're ultimately looking to fold an addrec of truncs and muls of only
2585 // constants and truncs, so if we find any other types of SCEV
2586 // as operands of the addrec then we bail and return nullptr here.
2587 // Otherwise, we return the type of the operand of a trunc that we find.
2588 if (auto *T = dyn_cast<SCEVTruncateExpr>(Val&: Ops[Idx]))
2589 return T->getOperand()->getType();
2590 if (const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
2591 SCEVUse LastOp = Mul->getOperand(i: Mul->getNumOperands() - 1);
2592 if (const auto *T = dyn_cast<SCEVTruncateExpr>(Val&: LastOp))
2593 return T->getOperand()->getType();
2594 }
2595 return nullptr;
2596 };
2597 if (auto *SrcType = FindTruncSrcType()) {
2598 SmallVector<SCEVUse, 8> LargeOps;
2599 bool Ok = true;
2600 // Check all the operands to see if they can be represented in the
2601 // source type of the truncate.
2602 for (const SCEV *Op : Ops) {
2603 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2604 if (T->getOperand()->getType() != SrcType) {
2605 Ok = false;
2606 break;
2607 }
2608 LargeOps.push_back(Elt: T->getOperand());
2609 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Op)) {
2610 LargeOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2611 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val: Op)) {
2612 SmallVector<SCEVUse, 8> LargeMulOps;
2613 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2614 if (const SCEVTruncateExpr *T =
2615 dyn_cast<SCEVTruncateExpr>(Val: M->getOperand(i: j))) {
2616 if (T->getOperand()->getType() != SrcType) {
2617 Ok = false;
2618 break;
2619 }
2620 LargeMulOps.push_back(Elt: T->getOperand());
2621 } else if (const auto *C = dyn_cast<SCEVConstant>(Val: M->getOperand(i: j))) {
2622 LargeMulOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2623 } else {
2624 Ok = false;
2625 break;
2626 }
2627 }
2628 if (Ok)
2629 LargeOps.push_back(
2630 Elt: getMulExpr(Ops&: LargeMulOps, Flags: SCEV::FlagNone, Depth: Depth + 1));
2631 } else {
2632 Ok = false;
2633 break;
2634 }
2635 }
2636 if (Ok) {
2637 // Evaluate the expression in the larger type.
2638 const SCEV *Fold = getAddExpr(Ops&: LargeOps, Flags: SCEV::FlagNone, Depth: Depth + 1);
2639 // If it folds to something simple, use it. Otherwise, don't.
2640 if (isa<SCEVConstant>(Val: Fold) || isa<SCEVUnknown>(Val: Fold))
2641 return getTruncateExpr(Op: Fold, Ty);
2642 }
2643 }
2644
2645 if (Ops.size() == 2) {
2646 // Check if we have an expression of the form ((X + C1) - C2), where C1 and
2647 // C2 can be folded in a way that allows retaining wrapping flags of (X +
2648 // C1).
2649 const SCEV *A = Ops[0];
2650 const SCEV *B = Ops[1];
2651 auto *AddExpr = dyn_cast<SCEVAddExpr>(Val: B);
2652 auto *C = dyn_cast<SCEVConstant>(Val: A);
2653 if (AddExpr && C && isa<SCEVConstant>(Val: AddExpr->getOperand(i: 0))) {
2654 auto C1 = cast<SCEVConstant>(Val: AddExpr->getOperand(i: 0))->getAPInt();
2655 auto C2 = C->getAPInt();
2656 SCEVFlags PreservedFlags = SCEV::FlagNone;
2657
2658 APInt ConstAdd = C1 + C2;
2659 auto AddFlags = AddExpr->getNoWrapFlags();
2660 // Adding a smaller constant is NUW if the original AddExpr was NUW.
2661 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNUW) &&
2662 ConstAdd.ule(RHS: C1)) {
2663 PreservedFlags =
2664 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNUW);
2665 }
2666
2667 // Adding a constant with the same sign and small magnitude is NSW, if the
2668 // original AddExpr was NSW.
2669 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNSW) &&
2670 C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
2671 ConstAdd.abs().ule(RHS: C1.abs())) {
2672 PreservedFlags =
2673 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNSW);
2674 }
2675
2676 if (PreservedFlags != SCEV::FlagNone) {
2677 SmallVector<SCEVUse, 4> NewOps(AddExpr->operands());
2678 NewOps[0] = getConstant(Val: ConstAdd);
2679 return getAddExpr(Ops&: NewOps, Flags: PreservedFlags);
2680 }
2681 }
2682
2683 // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext
2684 // (B), if trunc (A) + -A + B does not unsigned-wrap.
2685 const SCEVAddExpr *InnerAdd;
2686 if (match(S: B, P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
2687 const SCEV *NarrowA = getTruncateExpr(Op: A, Ty: InnerAdd->getType());
2688 if (NarrowA == getNegativeSCEV(V: InnerAdd->getOperand(i: 0)) &&
2689 getZeroExtendExpr(Op: NarrowA, Ty: B->getType()) == A &&
2690 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops: {NarrowA, InnerAdd},
2691 Flags: SCEV::FlagNone),
2692 TestFlags: SCEV::FlagNUW)) {
2693 return getZeroExtendExpr(Op: getAddExpr(LHS: NarrowA, RHS: InnerAdd), Ty: B->getType());
2694 }
2695 }
2696 }
2697
2698 // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
2699 const SCEV *Y;
2700 if (Ops.size() == 2 &&
2701 match(U: Ops[0],
2702 P: m_scev_Mul(Op0: m_scev_AllOnes(),
2703 Op1: m_scev_URem(LHS: m_scev_Specific(S: Ops[1]), RHS: m_SCEV(V&: Y), SE&: *this))))
2704 return getMulExpr(LHS: Y, RHS: getUDivExpr(LHS: Ops[1], RHS: Y));
2705
2706 // Skip past any other cast SCEVs.
2707 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
2708 ++Idx;
2709
2710 // If there are add operands they would be next.
2711 if (Idx < Ops.size()) {
2712 bool DeletedAdd = false;
2713 // If the original flags and all inlined SCEVAddExprs are NUW, use the
2714 // common NUW flag for expression after inlining. Other flags cannot be
2715 // preserved, because they may depend on the original order of operations.
2716 SCEVFlags CommonFlags = maskFlags(Flags: ExprFlags, Mask: SCEV::FlagNUW);
2717 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[Idx])) {
2718 if (Ops.size() > AddOpsInlineThreshold ||
2719 Add->getNumOperands() > AddOpsInlineThreshold)
2720 break;
2721 // If we have an add, expand the add operands onto the end of the operands
2722 // list.
2723 Ops.erase(CI: Ops.begin()+Idx);
2724 append_range(C&: Ops, R: Add->operands());
2725 DeletedAdd = true;
2726 CommonFlags = maskFlags(Flags: CommonFlags, Mask: Add->getNoWrapFlags());
2727 }
2728
2729 // If we deleted at least one add, we added operands to the end of the list,
2730 // and they are not necessarily sorted. Recurse to resort and resimplify
2731 // any operands we just acquired.
2732 if (DeletedAdd)
2733 return getAddExpr(Ops, Flags: CommonFlags, Depth: Depth + 1);
2734 }
2735
2736 // Skip over the add expression until we get to a multiply.
2737 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
2738 ++Idx;
2739
2740 // Check to see if there are any folding opportunities present with
2741 // operands multiplied by constant values.
2742 if (Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx])) {
2743 uint64_t BitWidth = getTypeSizeInBits(Ty);
2744 SmallDenseMap<SCEVUse, APInt, 16> M;
2745 SmallVector<SCEVUse, 8> NewOps;
2746 APInt AccumulatedConstant(BitWidth, 0);
2747 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2748 Ops, Scale: APInt(BitWidth, 1), SE&: *this)) {
2749 struct APIntCompare {
2750 bool operator()(const APInt &LHS, const APInt &RHS) const {
2751 return LHS.ult(RHS);
2752 }
2753 };
2754
2755 // Some interesting folding opportunity is present, so its worthwhile to
2756 // re-generate the operands list. Group the operands by constant scale,
2757 // to avoid multiplying by the same constant scale multiple times.
2758 std::map<APInt, SmallVector<SCEVUse, 4>, APIntCompare> MulOpLists;
2759 for (SCEVUse NewOp : NewOps)
2760 MulOpLists[M.find(Val: NewOp)->second].push_back(Elt: NewOp);
2761 // Re-generate the operands list.
2762 Ops.clear();
2763 if (AccumulatedConstant != 0)
2764 Ops.push_back(Elt: getConstant(Val: AccumulatedConstant));
2765 for (auto &MulOp : MulOpLists) {
2766 if (MulOp.first == 1) {
2767 Ops.push_back(Elt: getAddExpr(Ops&: MulOp.second, Flags: SCEV::FlagNone, Depth: Depth + 1));
2768 } else if (MulOp.first != 0) {
2769 Ops.push_back(
2770 Elt: getMulExpr(LHS: getConstant(Val: MulOp.first),
2771 RHS: getAddExpr(Ops&: MulOp.second, Flags: SCEV::FlagNone, Depth: Depth + 1),
2772 Flags: SCEV::FlagNone, Depth: Depth + 1));
2773 }
2774 }
2775 if (Ops.empty())
2776 return getZero(Ty);
2777 if (Ops.size() == 1)
2778 return Ops[0];
2779 return getAddExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
2780 }
2781 }
2782
2783 // Given a SCEVMulExpr and an operand index, return the product of all
2784 // operands except the one at OpIdx.
2785 auto StripFactor = [&](const SCEVMulExpr *M, unsigned OpIdx) -> SCEVUse {
2786 if (M->getNumOperands() == 2)
2787 return M->getOperand(i: OpIdx == 0);
2788 SmallVector<SCEVUse, 4> Remaining(M->operands().take_front(N: OpIdx));
2789 append_range(C&: Remaining, R: M->operands().drop_front(N: OpIdx + 1));
2790 return getMulExpr(Ops&: Remaining, Flags: SCEV::FlagNone, Depth: Depth + 1);
2791 };
2792
2793 // If we are adding something to a multiply expression, make sure the
2794 // something is not already an operand of the multiply. If so, merge it into
2795 // the multiply.
2796 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx]); ++Idx) {
2797 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val&: Ops[Idx]);
2798 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
2799 // Scan all terms to find every occurrence of common factor MulOpSCEV
2800 // and fold them in one shot:
2801 // A1*X + A2*X + ... + An*X --> X * (A1 + A2 + ... + An)
2802 const SCEV *MulOpSCEV = Mul->getOperand(i: MulOp);
2803 if (isa<SCEVConstant>(Val: MulOpSCEV))
2804 continue;
2805
2806 // Cofactors: 1 for bare addends matching MulOpSCEV, or the
2807 // remaining product for multiply terms containing MulOpSCEV.
2808 SmallVector<SCEVUse, 4> Cofactors;
2809 SmallVector<unsigned, 4> DeadIndices;
2810 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) {
2811 if (MulOpSCEV == Ops[AddOp]) {
2812 // W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
2813 Cofactors.push_back(Elt: getOne(Ty));
2814 DeadIndices.push_back(Elt: AddOp);
2815 continue;
2816 }
2817
2818 if (AddOp <= Idx || !isa<SCEVMulExpr>(Val: Ops[AddOp]))
2819 continue;
2820
2821 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Val&: Ops[AddOp]);
2822 for (unsigned OMulOp = 0, OE = OtherMul->getNumOperands(); OMulOp != OE;
2823 ++OMulOp) {
2824 if (OtherMul->getOperand(i: OMulOp) == MulOpSCEV) {
2825 // (A*B*C) + (A*D*E) --> A * (B*C + D*E)
2826 Cofactors.push_back(Elt: StripFactor(OtherMul, OMulOp));
2827 DeadIndices.push_back(Elt: AddOp);
2828 break;
2829 }
2830 }
2831 }
2832
2833 // Fold all collected cofactors with the anchor multiply's cofactor:
2834 // MulOpSCEV * (Cofactor_1 + ... + Cofactor_n + AnchorCofactor)
2835 if (!Cofactors.empty()) {
2836 Cofactors.push_back(Elt: StripFactor(Mul, MulOp));
2837
2838 SCEVUse InnerSum = getAddExpr(Ops&: Cofactors, Flags: SCEV::FlagNone, Depth: Depth + 1);
2839 SCEVUse OuterMul =
2840 getMulExpr(LHS: MulOpSCEV, RHS: InnerSum, Flags: SCEV::FlagNone, Depth: Depth + 1);
2841
2842 // DeadIndices does not include Idx (the anchor), hence +1.
2843 if (Ops.size() == DeadIndices.size() + 1)
2844 return OuterMul;
2845
2846 // Erase Ops[Idx] first, then erase DeadIndices in reverse order.
2847 // The -1 adjustment accounts for the shift from removing Idx;
2848 // reverse order means each erasure only shifts later positions,
2849 // which have already been processed.
2850 Ops.erase(CI: Ops.begin() + Idx);
2851 for (unsigned Dead : reverse(C&: DeadIndices))
2852 Ops.erase(CI: Ops.begin() + (Dead > Idx ? Dead - 1 : Dead));
2853
2854 Ops.push_back(Elt: OuterMul);
2855 return getAddExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
2856 }
2857 }
2858 }
2859
2860 // If there are any add recurrences in the operands list, see if any other
2861 // added values are loop invariant. If so, we can fold them into the
2862 // recurrence.
2863 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
2864 ++Idx;
2865
2866 // Scan over all recurrences, trying to fold loop invariants into them.
2867 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
2868 // Scan all of the other operands to this add and add them to the vector if
2869 // they are loop invariant w.r.t. the recurrence.
2870 SmallVector<SCEVUse, 8> LIOps;
2871 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
2872 const Loop *AddRecLoop = AddRec->getLoop();
2873 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2874 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRecLoop)) {
2875 LIOps.push_back(Elt: Ops[i]);
2876 Ops.erase(CI: Ops.begin()+i);
2877 --i; --e;
2878 }
2879
2880 // If we found some loop invariants, fold them into the recurrence.
2881 if (!LIOps.empty()) {
2882 // Compute nowrap flags for the addition of the loop-invariant ops and
2883 // the addrec. Temporarily push it as an operand for that purpose. These
2884 // flags are valid in the scope of the addrec only.
2885 LIOps.push_back(Elt: AddRec);
2886 SCEVFlags Flags = ComputeFlags(LIOps);
2887 LIOps.pop_back();
2888
2889 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
2890 LIOps.push_back(Elt: AddRec->getStart());
2891
2892 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
2893
2894 // It is not in general safe to propagate flags valid on an add within
2895 // the addrec scope to one outside it. We must prove that the inner
2896 // scope is guaranteed to execute if the outer one does to be able to
2897 // safely propagate. We know the program is undefined if poison is
2898 // produced on the inner scoped addrec. We also know that *for this use*
2899 // the outer scoped add can't overflow (because of the flags we just
2900 // computed for the inner scoped add) without the program being undefined.
2901 // Proving that entry to the outer scope neccesitates entry to the inner
2902 // scope, thus proves the program undefined if the flags would be violated
2903 // in the outer scope.
2904 SCEVFlags AddFlags = Flags;
2905 if (AddFlags != SCEV::FlagNone) {
2906 auto *DefI = getDefiningScopeBound(Ops: LIOps);
2907 auto *ReachI = &*AddRecLoop->getHeader()->begin();
2908 if (!isGuaranteedToTransferExecutionTo(A: DefI, B: ReachI))
2909 AddFlags = SCEV::FlagNone;
2910 }
2911 AddRecOps[0] = getAddExpr(Ops&: LIOps, Flags: AddFlags, Depth: Depth + 1);
2912
2913 // Build the new addrec. Propagate the NUW and NSW flags if both the
2914 // outer add and the inner addrec are guaranteed to have no overflow.
2915 // Always propagate NW.
2916 Flags = AddRec->getNoWrapFlags(Mask: setFlags(Flags, OnFlags: SCEV::FlagNW));
2917 const SCEV *NewRec = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags);
2918
2919 // If all of the other operands were loop invariant, we are done.
2920 if (Ops.size() == 1) return NewRec;
2921
2922 // Otherwise, add the folded AddRec by the non-invariant parts.
2923 for (unsigned i = 0;; ++i)
2924 if (Ops[i] == AddRec) {
2925 Ops[i] = NewRec;
2926 break;
2927 }
2928 return getAddExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
2929 }
2930
2931 // Okay, if there weren't any loop invariants to be folded, check to see if
2932 // there are multiple AddRec's with the same loop induction variable being
2933 // added together. If so, we can fold them.
2934 for (unsigned OtherIdx = Idx+1;
2935 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2936 ++OtherIdx) {
2937 // We expect the AddRecExpr's to be sorted in reverse dominance order,
2938 // so that the 1st found AddRecExpr is dominated by all others.
2939 assert(DT.dominates(
2940 cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),
2941 AddRec->getLoop()->getHeader()) &&
2942 "AddRecExprs are not sorted in reverse dominance order?");
2943 if (AddRecLoop == cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx])->getLoop()) {
2944 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L>
2945 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
2946 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2947 ++OtherIdx) {
2948 const auto *OtherAddRec = cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
2949 if (OtherAddRec->getLoop() == AddRecLoop) {
2950 for (unsigned i = 0, e = OtherAddRec->getNumOperands();
2951 i != e; ++i) {
2952 if (i >= AddRecOps.size()) {
2953 append_range(C&: AddRecOps, R: OtherAddRec->operands().drop_front(N: i));
2954 break;
2955 }
2956 AddRecOps[i] =
2957 getAddExpr(LHS: AddRecOps[i], RHS: OtherAddRec->getOperand(i),
2958 Flags: SCEV::FlagNone, Depth: Depth + 1);
2959 }
2960 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
2961 }
2962 }
2963 // Step size has changed, so we cannot guarantee no self-wraparound.
2964 Ops[Idx] = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags: SCEV::FlagNone);
2965 return getAddExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
2966 }
2967 }
2968
2969 // Otherwise couldn't fold anything into this recurrence. Move onto the
2970 // next one.
2971 }
2972
2973 // Okay, it looks like we really DO need an add expr. Check to see if we
2974 // already have one, otherwise create a new one.
2975 assert((UseFlags == SCEV::FlagNone || equal(OrigOps, Ops)) &&
2976 "Tried to add SCEVUse flags after operands changed");
2977 return {getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
2978}
2979
2980const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
2981 SCEVFlags Flags) {
2982 FoldingSetNodeID ID;
2983 ID.AddInteger(I: scAddExpr);
2984 for (SCEVUse Op : Ops)
2985 ID.AddPointer(Ptr: Op.getOpaqueValue());
2986 FoldingSetInsertToken Token;
2987 SCEVAddExpr *S = static_cast<SCEVAddExpr *>(UniqueSCEVs.lookup(ID, Token));
2988 if (!S) {
2989 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
2990 llvm::uninitialized_copy(Src&: Ops, Dst: O);
2991 S = new (SCEVAllocator)
2992 SCEVAddExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size());
2993 UniqueSCEVs.insert(N: S, Token);
2994 S->computeAndSetCanonical(SE&: *this);
2995 registerUser(User: S, Ops);
2996 }
2997 S->setNoWrapFlags(Flags);
2998 return S;
2999}
3000
3001const SCEV *ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<SCEVUse> Ops,
3002 const Loop *L,
3003 SCEVFlags Flags) {
3004 FoldingSetNodeID ID;
3005 ID.AddInteger(I: scAddRecExpr);
3006 for (SCEVUse Op : Ops)
3007 ID.AddPointer(Ptr: Op.getOpaqueValue());
3008 ID.AddPointer(Ptr: L);
3009 FoldingSetInsertToken Token;
3010 SCEVAddRecExpr *S =
3011 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.lookup(ID, Token));
3012 if (!S) {
3013 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3014 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3015 S = new (SCEVAllocator)
3016 SCEVAddRecExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size(), L);
3017 UniqueSCEVs.insert(N: S, Token);
3018 S->computeAndSetCanonical(SE&: *this);
3019 LoopUsers[L].push_back(Elt: S);
3020 registerUser(User: S, Ops);
3021 }
3022 setNoWrapFlags(AddRec: S, Flags);
3023 return S;
3024}
3025
3026const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
3027 SCEVFlags Flags) {
3028 FoldingSetNodeID ID;
3029 ID.AddInteger(I: scMulExpr);
3030 for (SCEVUse Op : Ops)
3031 ID.AddPointer(Ptr: Op.getOpaqueValue());
3032 FoldingSetInsertToken Token;
3033 SCEVMulExpr *S = static_cast<SCEVMulExpr *>(UniqueSCEVs.lookup(ID, Token));
3034 if (!S) {
3035 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3036 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3037 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(Allocator&: SCEVAllocator),
3038 O, Ops.size());
3039 UniqueSCEVs.insert(N: S, Token);
3040 S->computeAndSetCanonical(SE&: *this);
3041 registerUser(User: S, Ops);
3042 }
3043 S->setNoWrapFlags(Flags);
3044 return S;
3045}
3046
3047const SCEV *ScalarEvolution::getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS) {
3048 FoldingSetNodeID ID;
3049 ID.AddInteger(I: scUDivExpr);
3050 ID.AddPointer(Ptr: LHS.getOpaqueValue());
3051 ID.AddPointer(Ptr: RHS.getOpaqueValue());
3052 FoldingSetInsertToken Token;
3053 SCEV *S = UniqueSCEVs.lookup(ID, Token);
3054 if (!S) {
3055 S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(Allocator&: SCEVAllocator), LHS, RHS);
3056 UniqueSCEVs.insert(N: S, Token);
3057 S->computeAndSetCanonical(SE&: *this);
3058 registerUser(User: S, Ops: {LHS, RHS});
3059 }
3060 return S;
3061}
3062
3063static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
3064 uint64_t k = i*j;
3065 if (j > 1 && k / j != i) Overflow = true;
3066 return k;
3067}
3068
3069/// Compute the result of "n choose k", the binomial coefficient. If an
3070/// intermediate computation overflows, Overflow will be set and the return will
3071/// be garbage. Overflow is not cleared on absence of overflow.
3072static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
3073 // We use the multiplicative formula:
3074 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
3075 // At each iteration, we take the n-th term of the numeral and divide by the
3076 // (k-n)th term of the denominator. This division will always produce an
3077 // integral result, and helps reduce the chance of overflow in the
3078 // intermediate computations. However, we can still overflow even when the
3079 // final result would fit.
3080
3081 if (n == 0 || n == k) return 1;
3082 if (k > n) return 0;
3083
3084 if (k > n/2)
3085 k = n-k;
3086
3087 uint64_t r = 1;
3088 for (uint64_t i = 1; i <= k; ++i) {
3089 r = umul_ov(i: r, j: n-(i-1), Overflow);
3090 r /= i;
3091 }
3092 return r;
3093}
3094
3095/// Determine if any of the operands in this SCEV are a constant or if
3096/// any of the add or multiply expressions in this SCEV contain a constant.
3097static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
3098 struct FindConstantInAddMulChain {
3099 bool FoundConstant = false;
3100
3101 bool follow(const SCEV *S) {
3102 FoundConstant |= isa<SCEVConstant>(Val: S);
3103 return isa<SCEVAddExpr>(Val: S) || isa<SCEVMulExpr>(Val: S);
3104 }
3105
3106 bool isDone() const {
3107 return FoundConstant;
3108 }
3109 };
3110
3111 FindConstantInAddMulChain F;
3112 SCEVTraversal<FindConstantInAddMulChain> ST(F);
3113 ST.visitAll(Root: StartExpr);
3114 return F.FoundConstant;
3115}
3116
3117/// Get a canonical multiply expression, or something simpler if possible.
3118SCEVUse ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
3119 SCEVFlagsPair Flags, unsigned Depth) {
3120 SCEVFlags ExprFlags = Flags.ExprFlags;
3121 SCEVFlags UseFlags = Flags.UseFlags;
3122 assert(ExprFlags == maskFlags(ExprFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3123 "only nuw or nsw allowed");
3124 assert(UseFlags == maskFlags(UseFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3125 "only nuw or nsw allowed");
3126 assert(!Ops.empty() && "Cannot get empty mul!");
3127 if (Ops.size() == 1) return Ops[0];
3128#ifndef NDEBUG
3129 Type *ETy = Ops[0]->getType();
3130 assert(!ETy->isPointerTy());
3131 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
3132 assert(Ops[i]->getType() == ETy &&
3133 "SCEVMulExpr operand types don't match!");
3134#endif
3135
3136 const SCEV *Folded = constantFoldAndGroupOps(
3137 SE&: *this, LI, DT, Ops,
3138 Fold: [](const APInt &C1, const APInt &C2) { return C1 * C2; },
3139 IsIdentity: [](const APInt &C) { return C.isOne(); }, // identity
3140 IsAbsorber: [](const APInt &C) { return C.isZero(); }); // absorber
3141 if (Folded)
3142 return Folded;
3143
3144#ifndef NDEBUG
3145 // Keep track of operands after constant folding, for verification when adding
3146 // use-specific flags.
3147 const SmallVector<SCEVUse, 8> OrigOps(Ops.begin(), Ops.end());
3148#endif
3149
3150 // Delay expensive flag strengthening until necessary.
3151 auto ComputeFlags = [this, ExprFlags](const ArrayRef<SCEVUse> Ops) {
3152 return StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops, Flags: ExprFlags);
3153 };
3154
3155 // Limit recursion calls depth.
3156 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
3157 return {getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
3158
3159 if (SCEV *S = findExistingSCEVInCache(SCEVType: scMulExpr, Ops)) {
3160 // Don't strengthen flags if we have no new information.
3161 SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
3162 if (Mul->getNoWrapFlags(Mask: ExprFlags) != ExprFlags)
3163 Mul->setNoWrapFlags(ComputeFlags(Ops));
3164 return {S, UseFlags};
3165 }
3166
3167 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: Ops[0])) {
3168 if (Ops.size() == 2) {
3169 // C1*(C2+V) -> C1*C2 + C1*V
3170 // If any of Add's ops are Adds or Muls with a constant, apply this
3171 // transformation as well.
3172 //
3173 // TODO: There are some cases where this transformation is not
3174 // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of
3175 // this transformation should be narrowed down.
3176 const SCEV *Op0, *Op1;
3177 if (match(U: Ops[1], P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))) &&
3178 containsConstantInAddMulChain(StartExpr: Ops[1])) {
3179 const SCEV *LHS = getMulExpr(LHS: LHSC, RHS: Op0, Flags: SCEV::FlagNone, Depth: Depth + 1);
3180 const SCEV *RHS = getMulExpr(LHS: LHSC, RHS: Op1, Flags: SCEV::FlagNone, Depth: Depth + 1);
3181 return getAddExpr(LHS, RHS, Flags: SCEV::FlagNone, Depth: Depth + 1);
3182 }
3183
3184 if (Ops[0]->isAllOnesValue()) {
3185 // If we have a mul by -1 of an add, try distributing the -1 among the
3186 // add operands.
3187 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[1])) {
3188 SmallVector<SCEVUse, 4> NewOps;
3189 bool AnyFolded = false;
3190 for (const SCEV *AddOp : Add->operands()) {
3191 const SCEV *Mul =
3192 getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddOp), Flags: SCEV::FlagNone, Depth: Depth + 1);
3193 if (!isa<SCEVMulExpr>(Val: Mul)) AnyFolded = true;
3194 NewOps.push_back(Elt: Mul);
3195 }
3196 if (AnyFolded)
3197 return getAddExpr(Ops&: NewOps, Flags: SCEV::FlagNone, Depth: Depth + 1);
3198 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: Ops[1])) {
3199 // Negation preserves a recurrence's no self-wrap property.
3200 SmallVector<SCEVUse, 4> Operands;
3201 for (const SCEV *AddRecOp : AddRec->operands())
3202 Operands.push_back(Elt: getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddRecOp),
3203 Flags: SCEV::FlagNone, Depth: Depth + 1));
3204 // Let M be the minimum representable signed value. AddRec with nsw
3205 // multiplied by -1 can have signed overflow if and only if it takes a
3206 // value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
3207 // maximum signed value. In all other cases signed overflow is
3208 // impossible.
3209 auto FlagsMask = SCEV::FlagNW;
3210 if (AddRec->hasNoSignedWrap()) {
3211 auto MinInt =
3212 APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: AddRec->getType()));
3213 if (getSignedRangeMin(S: AddRec) != MinInt)
3214 FlagsMask = setFlags(Flags: FlagsMask, OnFlags: SCEV::FlagNSW);
3215 }
3216 return getAddRecExpr(Operands, L: AddRec->getLoop(),
3217 Flags: AddRec->getNoWrapFlags(Mask: FlagsMask));
3218 }
3219 }
3220
3221 // Try to push the constant operand into a ZExt: C * zext (A + B) ->
3222 // zext (C*A + C*B) if trunc (C) * (A + B) does not unsigned-wrap.
3223 const SCEVAddExpr *InnerAdd;
3224 if (match(U: Ops[1], P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
3225 const SCEV *NarrowC = getTruncateExpr(Op: LHSC, Ty: InnerAdd->getType());
3226 if (isa<SCEVConstant>(Val: InnerAdd->getOperand(i: 0)) &&
3227 getZeroExtendExpr(Op: NarrowC, Ty: Ops[1]->getType()) == LHSC &&
3228 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops: {NarrowC, InnerAdd},
3229 Flags: SCEV::FlagNone),
3230 TestFlags: SCEV::FlagNUW)) {
3231 const SCEV *Res =
3232 getMulExpr(LHS: NarrowC, RHS: InnerAdd, Flags: SCEV::FlagNUW, Depth: Depth + 1);
3233 return getZeroExtendExpr(Op: Res, Ty: Ops[1]->getType(), Depth: Depth + 1);
3234 };
3235 }
3236
3237 // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2,
3238 // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple
3239 // of C1, fold to (D /u (C2 /u C1)).
3240 const SCEV *D;
3241 APInt C1V = LHSC->getAPInt();
3242 // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1
3243 // as -1 * 1, as it won't enable additional folds.
3244 if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes())
3245 C1V = C1V.abs();
3246 const SCEVConstant *C2;
3247 if (C1V.isPowerOf2() &&
3248 match(U: Ops[1], P: m_scev_UDiv(Op0: m_SCEV(V&: D), Op1: m_SCEVConstant(V&: C2))) &&
3249 C2->getAPInt().isPowerOf2() &&
3250 C1V.logBase2() <= getMinTrailingZeros(S: D)) {
3251 const SCEV *NewMul = nullptr;
3252 if (C1V.uge(RHS: C2->getAPInt())) {
3253 NewMul = getMulExpr(LHS: getUDivExpr(LHS: getConstant(Val: C1V), RHS: C2), RHS: D);
3254 } else if (C2->getAPInt().logBase2() <= getMinTrailingZeros(S: D)) {
3255 assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier");
3256 NewMul = getUDivExpr(LHS: D, RHS: getUDivExpr(LHS: C2, RHS: getConstant(Val: C1V)));
3257 }
3258 if (NewMul)
3259 return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(V: NewMul);
3260 }
3261 }
3262 }
3263
3264 // Skip over the add expression until we get to a multiply.
3265 unsigned Idx = 0;
3266 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
3267 ++Idx;
3268
3269 // If there are mul operands inline them all into this expression.
3270 if (Idx < Ops.size()) {
3271 bool DeletedMul = false;
3272 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
3273 if (Ops.size() > MulOpsInlineThreshold)
3274 break;
3275 // If we have an mul, expand the mul operands onto the end of the
3276 // operands list.
3277 Ops.erase(CI: Ops.begin()+Idx);
3278 append_range(C&: Ops, R: Mul->operands());
3279 DeletedMul = true;
3280 }
3281
3282 // If we deleted at least one mul, we added operands to the end of the
3283 // list, and they are not necessarily sorted. Recurse to resort and
3284 // resimplify any operands we just acquired.
3285 if (DeletedMul)
3286 return getMulExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
3287 }
3288
3289 // If there are any add recurrences in the operands list, see if any other
3290 // added values are loop invariant. If so, we can fold them into the
3291 // recurrence.
3292 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3293 ++Idx;
3294
3295 // Scan over all recurrences, trying to fold loop invariants into them.
3296 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
3297 // Scan all of the other operands to this mul and add them to the vector
3298 // if they are loop invariant w.r.t. the recurrence.
3299 SmallVector<SCEVUse, 8> LIOps;
3300 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
3301 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3302 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRec->getLoop())) {
3303 LIOps.push_back(Elt: Ops[i]);
3304 Ops.erase(CI: Ops.begin()+i);
3305 --i; --e;
3306 }
3307
3308 // If we found some loop invariants, fold them into the recurrence.
3309 if (!LIOps.empty()) {
3310 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
3311 SmallVector<SCEVUse, 4> NewOps;
3312 NewOps.reserve(N: AddRec->getNumOperands());
3313 const SCEV *Scale = getMulExpr(Ops&: LIOps, Flags: SCEV::FlagNone, Depth: Depth + 1);
3314
3315 // If both the mul and addrec are nuw, we can preserve nuw.
3316 // If both the mul and addrec are nsw, we can only preserve nsw if either
3317 // a) they are also nuw, or
3318 // b) all multiplications of addrec operands with scale are nsw.
3319 SCEVFlags Flags = AddRec->getNoWrapFlags(Mask: ComputeFlags({Scale, AddRec}));
3320
3321 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
3322 NewOps.push_back(Elt: getMulExpr(LHS: Scale, RHS: AddRec->getOperand(i),
3323 Flags: SCEV::FlagNone, Depth: Depth + 1));
3324
3325 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW) && !hasFlags(Flags, TestFlags: SCEV::FlagNUW)) {
3326 ConstantRange NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
3327 BinOp: Instruction::Mul, Other: getSignedRange(S: Scale),
3328 NoWrapKind: OverflowingBinaryOperator::NoSignedWrap);
3329 if (!NSWRegion.contains(CR: getSignedRange(S: AddRec->getOperand(i))))
3330 Flags = clearFlags(Flags, OffFlags: SCEV::FlagNSW);
3331 }
3332 }
3333
3334 const SCEV *NewRec = getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags);
3335
3336 // If all of the other operands were loop invariant, we are done.
3337 if (Ops.size() == 1) return NewRec;
3338
3339 // Otherwise, multiply the folded AddRec by the non-invariant parts.
3340 for (unsigned i = 0;; ++i)
3341 if (Ops[i] == AddRec) {
3342 Ops[i] = NewRec;
3343 break;
3344 }
3345 return getMulExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
3346 }
3347
3348 // Okay, if there weren't any loop invariants to be folded, check to see
3349 // if there are multiple AddRec's with the same loop induction variable
3350 // being multiplied together. If so, we can fold them.
3351
3352 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
3353 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
3354 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
3355 // ]]],+,...up to x=2n}.
3356 // Note that the arguments to choose() are always integers with values
3357 // known at compile time, never SCEV objects.
3358 //
3359 // The implementation avoids pointless extra computations when the two
3360 // addrec's are of different length (mathematically, it's equivalent to
3361 // an infinite stream of zeros on the right).
3362 bool OpsModified = false;
3363 for (unsigned OtherIdx = Idx+1;
3364 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3365 ++OtherIdx) {
3366 const SCEVAddRecExpr *OtherAddRec =
3367 dyn_cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
3368 if (!OtherAddRec || OtherAddRec->getLoop() != AddRec->getLoop())
3369 continue;
3370
3371 // Limit max number of arguments to avoid creation of unreasonably big
3372 // SCEVAddRecs with very complex operands.
3373 if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
3374 MaxAddRecSize || hasHugeExpression(Ops: {AddRec, OtherAddRec}))
3375 continue;
3376
3377 bool Overflow = false;
3378 Type *Ty = AddRec->getType();
3379 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
3380 SmallVector<SCEVUse, 7> AddRecOps;
3381 for (int x = 0, xe = AddRec->getNumOperands() +
3382 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
3383 SmallVector<SCEVUse, 7> SumOps;
3384 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3385 uint64_t Coeff1 = Choose(n: x, k: 2*x - y, Overflow);
3386 for (int z = std::max(a: y-x, b: y-(int)AddRec->getNumOperands()+1),
3387 ze = std::min(a: x+1, b: (int)OtherAddRec->getNumOperands());
3388 z < ze && !Overflow; ++z) {
3389 uint64_t Coeff2 = Choose(n: 2*x - y, k: x-z, Overflow);
3390 uint64_t Coeff;
3391 if (LargerThan64Bits)
3392 Coeff = umul_ov(i: Coeff1, j: Coeff2, Overflow);
3393 else
3394 Coeff = Coeff1*Coeff2;
3395 const SCEV *CoeffTerm = getConstant(Ty, V: Coeff);
3396 const SCEV *Term1 = AddRec->getOperand(i: y-z);
3397 const SCEV *Term2 = OtherAddRec->getOperand(i: z);
3398 SumOps.push_back(
3399 Elt: getMulExpr(Op0: CoeffTerm, Op1: Term1, Op2: Term2, Flags: SCEV::FlagNone, Depth: Depth + 1));
3400 }
3401 }
3402 if (SumOps.empty())
3403 SumOps.push_back(Elt: getZero(Ty));
3404 AddRecOps.push_back(Elt: getAddExpr(Ops&: SumOps, Flags: SCEV::FlagNone, Depth: Depth + 1));
3405 }
3406 if (!Overflow) {
3407 const SCEV *NewAddRec =
3408 getAddRecExpr(Operands&: AddRecOps, L: AddRec->getLoop(), Flags: SCEV::FlagNone);
3409 if (Ops.size() == 2) return NewAddRec;
3410 Ops[Idx] = NewAddRec;
3411 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
3412 OpsModified = true;
3413 AddRec = dyn_cast<SCEVAddRecExpr>(Val: NewAddRec);
3414 if (!AddRec)
3415 break;
3416 }
3417 }
3418 if (OpsModified)
3419 return getMulExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
3420
3421 // Otherwise couldn't fold anything into this recurrence. Move onto the
3422 // next one.
3423 }
3424
3425 // Okay, it looks like we really DO need an mul expr. Check to see if we
3426 // already have one, otherwise create a new one.
3427 assert((UseFlags == SCEV::FlagNone || equal(OrigOps, Ops)) &&
3428 "Tried to add SCEVUse flags after operands changed");
3429 return {getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
3430}
3431
3432/// Represents an unsigned remainder expression based on unsigned division.
3433const SCEV *ScalarEvolution::getURemExpr(SCEVUse LHS, SCEVUse RHS) {
3434 assert(getEffectiveSCEVType(LHS->getType()) ==
3435 getEffectiveSCEVType(RHS->getType()) &&
3436 "SCEVURemExpr operand types don't match!");
3437
3438 // Short-circuit easy cases
3439 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3440 // If constant is one, the result is trivial
3441 if (RHSC->getValue()->isOne())
3442 return getZero(Ty: LHS->getType()); // X urem 1 --> 0
3443
3444 // If constant is a power of two, fold into a zext(trunc(LHS)).
3445 if (RHSC->getAPInt().isPowerOf2()) {
3446 Type *FullTy = LHS->getType();
3447 Type *TruncTy =
3448 IntegerType::get(C&: getContext(), NumBits: RHSC->getAPInt().logBase2());
3449 return getZeroExtendExpr(Op: getTruncateExpr(Op: LHS, Ty: TruncTy), Ty: FullTy);
3450 }
3451 }
3452
3453 // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
3454 const SCEV *UDiv = getUDivExpr(LHS, RHS);
3455 const SCEV *Mult = getMulExpr(LHS: UDiv, RHS, Flags: SCEV::FlagNUW);
3456 return getMinusSCEV(LHS, RHS: Mult, Flags: SCEV::FlagNUW);
3457}
3458
3459/// Get a canonical unsigned division expression, or something simpler if
3460/// possible.
3461const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
3462 assert(!LHS->getType()->isPointerTy() &&
3463 "SCEVUDivExpr operand can't be pointer!");
3464 assert(LHS->getType() == RHS->getType() &&
3465 "SCEVUDivExpr operand types don't match!");
3466
3467 if (SCEV *S = findExistingSCEVInCache(SCEVType: scUDivExpr, Ops: {LHS, RHS}))
3468 return S;
3469
3470 // 0 udiv Y == 0
3471 if (match(U: LHS, P: m_scev_Zero()))
3472 return LHS;
3473
3474 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3475 if (RHSC->getValue()->isOne())
3476 return LHS; // X udiv 1 --> x
3477 // If the denominator is zero, the result of the udiv is undefined. Don't
3478 // try to analyze it, because the resolution chosen here may differ from
3479 // the resolution chosen in other parts of the compiler.
3480 if (!RHSC->getValue()->isZero()) {
3481 // Determine if the division can be folded into the operands of
3482 // its operands.
3483 // TODO: Generalize this to non-constants by using known-bits information.
3484 Type *Ty = LHS->getType();
3485 unsigned LZ = RHSC->getAPInt().countl_zero();
3486 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
3487 // For non-power-of-two values, effectively round the value up to the
3488 // nearest power of two.
3489 if (!RHSC->getAPInt().isPowerOf2())
3490 ++MaxShiftAmt;
3491 IntegerType *ExtTy =
3492 IntegerType::get(C&: getContext(), NumBits: getTypeSizeInBits(Ty) + MaxShiftAmt);
3493 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
3494 if (const SCEVConstant *Step =
3495 dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE&: *this))) {
3496 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
3497 const APInt &StepInt = Step->getAPInt();
3498 const APInt &DivInt = RHSC->getAPInt();
3499 if (!StepInt.urem(RHS: DivInt) &&
3500 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3501 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3502 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy), L: AR->getLoop(),
3503 Flags: SCEV::FlagNone)) {
3504 SmallVector<SCEVUse, 4> Operands;
3505 for (const SCEV *Op : AR->operands())
3506 Operands.push_back(Elt: getUDivExpr(LHS: Op, RHS));
3507 return getAddRecExpr(Operands, L: AR->getLoop(), Flags: SCEV::FlagNW);
3508 }
3509 /// Get a canonical UDivExpr for a recurrence.
3510 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
3511 const APInt *StartRem;
3512 if (!DivInt.urem(RHS: StepInt) && match(S: getURemExpr(LHS: AR->getStart(), RHS: Step),
3513 P: m_scev_APInt(C&: StartRem))) {
3514 bool NoWrap =
3515 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3516 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3517 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy), L: AR->getLoop(),
3518 Flags: SCEV::FlagNone);
3519
3520 // With N <= C and both N, C as powers-of-2, the transformation
3521 // {X,+,N}/C => {(X - X%N),+,N}/C preserves division results even
3522 // if wrapping occurs, as the division results remain equivalent for
3523 // all offsets in [[(X - X%N), X).
3524 bool CanFoldWithWrap = StepInt.ule(RHS: DivInt) && // N <= C
3525 StepInt.isPowerOf2() && DivInt.isPowerOf2();
3526 // Only fold if the subtraction can be folded in the start
3527 // expression.
3528 const SCEV *NewStart =
3529 getMinusSCEV(LHS: AR->getStart(), RHS: getConstant(Val: *StartRem));
3530 if (*StartRem != 0 && (NoWrap || CanFoldWithWrap) &&
3531 !isa<SCEVAddExpr>(Val: NewStart)) {
3532 const SCEV *NewLHS =
3533 getAddRecExpr(Start: NewStart, Step, L: AR->getLoop(),
3534 Flags: NoWrap ? SCEV::FlagNW : SCEV::FlagNone);
3535 if (LHS != NewLHS)
3536 return getUDivExpr(LHS: NewLHS, RHS);
3537 }
3538 }
3539 }
3540 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
3541 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val&: LHS)) {
3542 if (M->hasNoUnsignedWrap()) {
3543 // Find an operand that's safely divisible.
3544 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3545 const SCEV *Op = M->getOperand(i);
3546 const SCEV *Div = getUDivExpr(LHS: Op, RHS: RHSC);
3547 if (!isa<SCEVUDivExpr>(Val: Div) && getMulExpr(LHS: Div, RHS: RHSC) == Op) {
3548 SmallVector<SCEVUse, 4> Operands(M->operands());
3549 Operands[i] = Div;
3550 return getMulExpr(Ops&: Operands);
3551 }
3552 }
3553
3554 // Even if it's not divisible, try to remove a common factor.
3555 if (const auto *LHSC = dyn_cast<SCEVConstant>(Val: M->getOperand(i: 0))) {
3556 APInt Factor = APIntOps::GreatestCommonDivisor(A: LHSC->getAPInt(),
3557 B: RHSC->getAPInt());
3558 if (!Factor.isIntN(N: 1)) {
3559 SmallVector<SCEVUse, 2> NewOperands;
3560 NewOperands.push_back(Elt: getConstant(Val: LHSC->getAPInt().udiv(RHS: Factor)));
3561 append_range(C&: NewOperands, R: M->operands().drop_front());
3562 const SCEV *NewMul = getMulExpr(Ops&: NewOperands);
3563 return getUDivExpr(LHS: NewMul,
3564 RHS: getConstant(Val: RHSC->getAPInt().udiv(RHS: Factor)));
3565 }
3566 }
3567 }
3568 }
3569
3570 // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
3571 if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(Val&: LHS)) {
3572 if (auto *DivisorConstant =
3573 dyn_cast<SCEVConstant>(Val: OtherDiv->getRHS())) {
3574 bool Overflow = false;
3575 APInt NewRHS =
3576 DivisorConstant->getAPInt().umul_ov(RHS: RHSC->getAPInt(), Overflow);
3577 if (Overflow) {
3578 return getConstant(Ty: RHSC->getType(), V: 0, isSigned: false);
3579 }
3580 return getUDivExpr(LHS: OtherDiv->getLHS(), RHS: getConstant(Val: NewRHS));
3581 }
3582 }
3583
3584 // (A+B)/C --> (A/C + B/C) if the add does not unsigned wrap and A/C and
3585 // B/C can be folded.
3586 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(Val&: LHS)) {
3587 if (A->hasNoUnsignedWrap()) {
3588 SmallVector<SCEVUse, 4> Operands;
3589 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
3590 const SCEV *Op = getUDivExpr(LHS: A->getOperand(i), RHS);
3591 if (isa<SCEVUDivExpr>(Val: Op) ||
3592 getMulExpr(LHS: Op, RHS) != A->getOperand(i))
3593 break;
3594 Operands.push_back(Elt: Op);
3595 }
3596 if (Operands.size() == A->getNumOperands())
3597 return getAddExpr(Ops&: Operands);
3598 }
3599 }
3600
3601 // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
3602 // This is an idiom for rounding A up to the next multiple of N, where A
3603 // is aready known to be a multiple of M. In this case, instcombine can
3604 // see that some low bits of the added constant are unused, so can clear
3605 // them, but we want to canonicalise to set the low bits. This makes the
3606 // pattern easier to match, without needing to check for known bits in
3607 // A*M.
3608 const APInt &N = RHSC->getAPInt();
3609 const APInt *NMinusM, *M;
3610 const SCEV *A;
3611 if (match(U: LHS, P: m_scev_Add(Op0: m_scev_APInt(C&: NMinusM),
3612 Op1: m_scev_Mul(Op0: m_scev_APInt(C&: M), Op1: m_SCEV(V&: A))))) {
3613 if (N.isPowerOf2() && M->isPowerOf2() && M->ult(RHS: N) &&
3614 *NMinusM == N - *M) {
3615 return getUDivExpr(
3616 LHS: getAddExpr(LHS: getConstant(Val: N - 1), RHS: getMulExpr(LHS: getConstant(Val: *M), RHS: A)),
3617 RHS);
3618 }
3619 }
3620
3621 // Fold if both operands are constant.
3622 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS))
3623 return getConstant(Val: LHSC->getAPInt().udiv(RHS: RHSC->getAPInt()));
3624 }
3625 }
3626
3627 // ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
3628 const APInt *NegC, *C;
3629 if (match(U: LHS,
3630 P: m_scev_Add(Op0: m_scev_APInt(C&: NegC),
3631 Op1: m_scev_SMax(Op0: m_scev_APInt(C), Op1: m_scev_Specific(S: RHS)))) &&
3632 NegC->isNegative() && !NegC->isMinSignedValue() && *C == -*NegC)
3633 return getZero(Ty: LHS->getType());
3634
3635 // (%a * %b)<nuw> / %b -> %a
3636 const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: LHS);
3637 if (Mul && Mul->hasNoUnsignedWrap()) {
3638 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
3639 if (Mul->getOperand(i) == RHS) {
3640 SmallVector<SCEVUse, 2> Operands;
3641 append_range(C&: Operands, R: Mul->operands().take_front(N: i));
3642 append_range(C&: Operands, R: Mul->operands().drop_front(N: i + 1));
3643 return getMulExpr(Ops&: Operands);
3644 }
3645 }
3646 }
3647
3648 // TODO: Generalize to handle any common factors.
3649 // udiv (mul nuw a, vscale), (mul nuw b, vscale) --> udiv a, b
3650 const SCEV *NewLHS, *NewRHS;
3651 if (match(U: LHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
3652 match(U: RHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale())))
3653 return getUDivExpr(LHS: NewLHS, RHS: NewRHS);
3654
3655 return getOrCreateUDivExpr(LHS, RHS);
3656}
3657
3658/// Get a canonical unsigned division expression, or something simpler if
3659/// possible. There is no representation for an exact udiv in SCEV IR, but we
3660/// can attempt to optimize it prior to construction.
3661const SCEV *ScalarEvolution::getUDivExactExpr(SCEVUse LHS, SCEVUse RHS) {
3662 // Currently there is no exact specific logic.
3663
3664 return getUDivExpr(LHS, RHS);
3665}
3666
3667/// Get an add recurrence expression for the specified loop. Simplify the
3668/// expression as much as possible.
3669SCEVUse ScalarEvolution::getAddRecExpr(SCEVUse Start, SCEVUse Step,
3670 const Loop *L, SCEVFlagsPair Flags) {
3671 SmallVector<SCEVUse, 4> Operands;
3672 Operands.push_back(Elt: Start);
3673 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Val&: Step))
3674 if (StepChrec->getLoop() == L) {
3675 append_range(C&: Operands, R: StepChrec->operands());
3676 // The use flags describe the two-operand recurrence, not the flattened
3677 // one built here, so drop them just like the expression's NUW/NSW.
3678 return getAddRecExpr(Operands, L,
3679 Flags: maskFlags(Flags: Flags.ExprFlags, Mask: SCEV::FlagNW));
3680 }
3681
3682 Operands.push_back(Elt: Step);
3683 return getAddRecExpr(Operands, L, Flags);
3684}
3685
3686/// Get an add recurrence expression for the specified loop. Simplify the
3687/// expression as much as possible.
3688SCEVUse ScalarEvolution::getAddRecExpr(SmallVectorImpl<SCEVUse> &Operands,
3689 const Loop *L, SCEVFlagsPair NWFlags) {
3690 SCEVFlags ExprFlags = NWFlags.ExprFlags;
3691 SCEVFlags UseFlags = NWFlags.UseFlags;
3692 assert(!(UseFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
3693 "only nuw or nsw allowed");
3694 if (Operands.size() == 1) return Operands[0];
3695#ifndef NDEBUG
3696 Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
3697 for (const SCEV *Op : llvm::drop_begin(Operands)) {
3698 assert(getEffectiveSCEVType(Op->getType()) == ETy &&
3699 "SCEVAddRecExpr operand types don't match!");
3700 assert(!Op->getType()->isPointerTy() && "Step must be integer");
3701 }
3702 for (const SCEV *Op : Operands)
3703 assert(isAvailableAtLoopEntry(Op, L) &&
3704 "SCEVAddRecExpr operand is not available at loop entry!");
3705
3706 // Keep track of the original operands, for verification when adding
3707 // use-specific flags.
3708 const SmallVector<SCEVUse, 4> OrigOperands(Operands.begin(), Operands.end());
3709#endif
3710
3711 if (Operands.back()->isZero()) {
3712 Operands.pop_back();
3713 return getAddRecExpr(Operands, L, NWFlags: SCEV::FlagNone); // {X,+,0} --> X
3714 }
3715
3716 // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
3717 // use that information to infer NUW and NSW flags. However, computing a
3718 // BE count requires calling getAddRecExpr, so we may not yet have a
3719 // meaningful BE count at this point (and if we don't, we'd be stuck
3720 // with a SCEVCouldNotCompute as the cached BE count).
3721
3722 ExprFlags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags: ExprFlags);
3723
3724 // Canonicalize nested AddRecs in by nesting them in order of loop depth.
3725 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Val&: Operands[0])) {
3726 const Loop *NestedLoop = NestedAR->getLoop();
3727 if (L->contains(L: NestedLoop)
3728 ? (L->getLoopDepth() < NestedLoop->getLoopDepth())
3729 : (!NestedLoop->contains(L) &&
3730 DT.dominates(A: L->getHeader(), B: NestedLoop->getHeader()))) {
3731 SmallVector<SCEVUse, 4> NestedOperands(NestedAR->operands());
3732 Operands[0] = NestedAR->getStart();
3733 // AddRecs require their operands be loop-invariant with respect to their
3734 // loops. Don't perform this transformation if it would break this
3735 // requirement.
3736 bool AllInvariant = all_of(
3737 Range&: Operands, P: [&](const SCEV *Op) { return isLoopInvariant(S: Op, L); });
3738
3739 if (AllInvariant) {
3740 // Create a recurrence for the outer loop with the same step size.
3741 //
3742 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
3743 // inner recurrence has the same property.
3744 SCEVFlags OuterFlags =
3745 maskFlags(Flags: ExprFlags, Mask: SCEV::FlagNW | NestedAR->getNoWrapFlags());
3746
3747 NestedOperands[0] = getAddRecExpr(Operands, L, NWFlags: OuterFlags);
3748 AllInvariant = all_of(Range&: NestedOperands, P: [&](const SCEV *Op) {
3749 return isLoopInvariant(S: Op, L: NestedLoop);
3750 });
3751
3752 if (AllInvariant) {
3753 // Ok, both add recurrences are valid after the transformation.
3754 //
3755 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
3756 // the outer recurrence has the same property.
3757 SCEVFlags InnerFlags =
3758 maskFlags(Flags: NestedAR->getNoWrapFlags(), Mask: SCEV::FlagNW | ExprFlags);
3759 return getAddRecExpr(Operands&: NestedOperands, L: NestedLoop, NWFlags: InnerFlags);
3760 }
3761 }
3762 // Reset Operands to its original state.
3763 Operands[0] = NestedAR;
3764 }
3765 }
3766
3767 // Okay, it looks like we really DO need an addrec expr. Check to see if we
3768 // already have one, otherwise create a new one.
3769 assert((UseFlags == SCEV::FlagNone || equal(OrigOperands, Operands)) &&
3770 "Tried to add SCEVUse flags after operands changed");
3771 return {getOrCreateAddRecExpr(Ops: Operands, L, Flags: ExprFlags), UseFlags};
3772}
3773
3774const SCEV *ScalarEvolution::getGEPExpr(GEPOperator *GEP,
3775 ArrayRef<SCEVUse> IndexExprs) {
3776 const SCEV *BaseExpr = getSCEV(V: GEP->getPointerOperand());
3777 // getSCEV(Base)->getType() has the same address space as Base->getType()
3778 // because SCEV::getType() preserves the address space.
3779 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
3780 if (NW != GEPNoWrapFlags::none()) {
3781 // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
3782 // but to do that, we have to ensure that said flag is valid in the entire
3783 // defined scope of the SCEV.
3784 // TODO: non-instructions have global scope. We might be able to prove
3785 // some global scope cases
3786 auto *GEPI = dyn_cast<Instruction>(Val: GEP);
3787 if (!GEPI || !isSCEVExprNeverPoison(I: GEPI))
3788 NW = GEPNoWrapFlags::none();
3789 }
3790
3791 return getGEPExpr(BaseExpr, IndexExprs, SrcElementTy: GEP->getSourceElementType(), NW);
3792}
3793
3794const SCEV *ScalarEvolution::getGEPExpr(SCEVUse BaseExpr,
3795 ArrayRef<SCEVUse> IndexExprs,
3796 Type *SrcElementTy, GEPNoWrapFlags NW) {
3797 SCEVFlags OffsetWrap = SCEV::FlagNone;
3798 if (NW.hasNoUnsignedSignedWrap())
3799 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNSW);
3800 if (NW.hasNoUnsignedWrap())
3801 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNUW);
3802
3803 Type *CurTy = BaseExpr->getType();
3804 Type *IntIdxTy = getEffectiveSCEVType(Ty: BaseExpr->getType());
3805 bool FirstIter = true;
3806 SmallVector<SCEVUse, 4> Offsets;
3807 for (SCEVUse IndexExpr : IndexExprs) {
3808 // Compute the (potentially symbolic) offset in bytes for this index.
3809 if (StructType *STy = dyn_cast<StructType>(Val: CurTy)) {
3810 // For a struct, add the member offset.
3811 ConstantInt *Index = cast<SCEVConstant>(Val&: IndexExpr)->getValue();
3812 unsigned FieldNo = Index->getZExtValue();
3813 const SCEV *FieldOffset = getOffsetOfExpr(IntTy: IntIdxTy, STy, FieldNo);
3814 Offsets.push_back(Elt: FieldOffset);
3815
3816 // Update CurTy to the type of the field at Index.
3817 CurTy = STy->getTypeAtIndex(V: Index);
3818 } else {
3819 // Update CurTy to its element type.
3820 if (FirstIter) {
3821 assert(isa<PointerType>(CurTy) &&
3822 "The first index of a GEP indexes a pointer");
3823 CurTy = SrcElementTy;
3824 FirstIter = false;
3825 } else {
3826 CurTy = GetElementPtrInst::getTypeAtIndex(Ty: CurTy, Idx: (uint64_t)0);
3827 }
3828 // For an array, add the element offset, explicitly scaled.
3829 const SCEV *ElementSize = getSizeOfExpr(IntTy: IntIdxTy, AllocTy: CurTy);
3830 // Getelementptr indices are signed.
3831 IndexExpr = getTruncateOrSignExtend(V: IndexExpr, Ty: IntIdxTy);
3832
3833 // Multiply the index by the element size to compute the element offset.
3834 const SCEV *LocalOffset = getMulExpr(LHS: IndexExpr, RHS: ElementSize, Flags: OffsetWrap);
3835 Offsets.push_back(Elt: LocalOffset);
3836 }
3837 }
3838
3839 // Handle degenerate case of GEP without offsets.
3840 if (Offsets.empty())
3841 return BaseExpr;
3842
3843 // Add the offsets together, assuming nsw if inbounds.
3844 const SCEV *Offset = getAddExpr(Ops&: Offsets, Flags: OffsetWrap);
3845 // Add the base address and the offset. We cannot use the nsw flag, as the
3846 // base address is unsigned. However, if we know that the offset is
3847 // non-negative, we can use nuw.
3848 bool NUW = NW.hasNoUnsignedWrap() ||
3849 (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Offset));
3850 SCEVFlags BaseWrap = NUW ? SCEV::FlagNUW : SCEV::FlagNone;
3851 const SCEV *GEPExpr = getAddExpr(LHS: BaseExpr, RHS: Offset, Flags: BaseWrap);
3852 assert(BaseExpr->getType() == GEPExpr->getType() &&
3853 "GEP should not change type mid-flight.");
3854 return GEPExpr;
3855}
3856
3857SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
3858 ArrayRef<SCEVUse> Ops,
3859 const Loop *L) {
3860 assert((SCEVType != scAddRecExpr || L) &&
3861 "L must be passed to find existing AddRecs");
3862 FoldingSetNodeID ID;
3863 ID.AddInteger(I: SCEVType);
3864 for (SCEVUse Op : Ops)
3865 ID.AddPointer(Ptr: Op.getOpaqueValue());
3866 if (L)
3867 ID.AddPointer(Ptr: L);
3868 FoldingSetInsertToken Token;
3869 return UniqueSCEVs.lookup(ID, Token);
3870}
3871
3872const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
3873 SCEVFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagNone;
3874 return getSMaxExpr(LHS: Op, RHS: getNegativeSCEV(V: Op, Flags));
3875}
3876
3877const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
3878 SmallVectorImpl<SCEVUse> &Ops) {
3879 assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
3880 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
3881 if (Ops.size() == 1) return Ops[0];
3882#ifndef NDEBUG
3883 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
3884 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
3885 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
3886 "Operand types don't match!");
3887 assert(Ops[0]->getType()->isPointerTy() ==
3888 Ops[i]->getType()->isPointerTy() &&
3889 "min/max should be consistently pointerish");
3890 }
3891#endif
3892
3893 bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
3894 bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
3895
3896 const SCEV *Folded = constantFoldAndGroupOps(
3897 SE&: *this, LI, DT, Ops,
3898 Fold: [&](const APInt &C1, const APInt &C2) {
3899 switch (Kind) {
3900 case scSMaxExpr:
3901 return APIntOps::smax(A: C1, B: C2);
3902 case scSMinExpr:
3903 return APIntOps::smin(A: C1, B: C2);
3904 case scUMaxExpr:
3905 return APIntOps::umax(A: C1, B: C2);
3906 case scUMinExpr:
3907 return APIntOps::umin(A: C1, B: C2);
3908 default:
3909 llvm_unreachable("Unknown SCEV min/max opcode");
3910 }
3911 },
3912 IsIdentity: [&](const APInt &C) {
3913 // identity
3914 if (IsMax)
3915 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3916 else
3917 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3918 },
3919 IsAbsorber: [&](const APInt &C) {
3920 // absorber
3921 if (IsMax)
3922 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3923 else
3924 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3925 });
3926 if (Folded)
3927 return Folded;
3928
3929 // Check if we have created the same expression before.
3930 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops)) {
3931 return S;
3932 }
3933
3934 // Find the first operation of the same kind
3935 unsigned Idx = 0;
3936 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
3937 ++Idx;
3938
3939 // Check to see if one of the operands is of the same kind. If so, expand its
3940 // operands onto our operand list, and recurse to simplify.
3941 if (Idx < Ops.size()) {
3942 bool DeletedAny = false;
3943 while (Ops[Idx]->getSCEVType() == Kind) {
3944 const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Val&: Ops[Idx]);
3945 Ops.erase(CI: Ops.begin()+Idx);
3946 append_range(C&: Ops, R: SMME->operands());
3947 DeletedAny = true;
3948 }
3949
3950 if (DeletedAny)
3951 return getMinMaxExpr(Kind, Ops);
3952 }
3953
3954 // Okay, check to see if the same value occurs in the operand list twice. If
3955 // so, delete one. Since we sorted the list, these values are required to
3956 // be adjacent.
3957 llvm::CmpInst::Predicate GEPred =
3958 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
3959 llvm::CmpInst::Predicate LEPred =
3960 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
3961 llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
3962 llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
3963 for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
3964 if (Ops[i] == Ops[i + 1] ||
3965 isKnownViaNonRecursiveReasoning(Pred: FirstPred, LHS: Ops[i], RHS: Ops[i + 1])) {
3966 // X op Y op Y --> X op Y
3967 // X op Y --> X, if we know X, Y are ordered appropriately
3968 Ops.erase(CS: Ops.begin() + i + 1, CE: Ops.begin() + i + 2);
3969 --i;
3970 --e;
3971 } else if (isKnownViaNonRecursiveReasoning(Pred: SecondPred, LHS: Ops[i],
3972 RHS: Ops[i + 1])) {
3973 // X op Y --> Y, if we know X, Y are ordered appropriately
3974 Ops.erase(CS: Ops.begin() + i, CE: Ops.begin() + i + 1);
3975 --i;
3976 --e;
3977 }
3978 }
3979
3980 if (Ops.size() == 1) return Ops[0];
3981
3982 assert(!Ops.empty() && "Reduced smax down to nothing!");
3983
3984 // Okay, it looks like we really DO need an expr. Check to see if we
3985 // already have one, otherwise create a new one.
3986 FoldingSetNodeID ID;
3987 ID.AddInteger(I: Kind);
3988 for (SCEVUse Op : Ops)
3989 ID.AddPointer(Ptr: Op.getOpaqueValue());
3990 FoldingSetInsertToken Token;
3991 const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
3992 if (ExistingSCEV)
3993 return ExistingSCEV;
3994 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3995 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3996 SCEV *S = new (SCEVAllocator)
3997 SCEVMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
3998
3999 UniqueSCEVs.insert(N: S, Token);
4000 S->computeAndSetCanonical(SE&: *this);
4001 registerUser(User: S, Ops);
4002 return S;
4003}
4004
4005namespace {
4006
4007class SCEVSequentialMinMaxDeduplicatingVisitor final
4008 : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
4009 std::optional<const SCEV *>> {
4010 using RetVal = std::optional<const SCEV *>;
4011
4012 ScalarEvolution &SE;
4013 const SCEVTypes RootKind; // Must be a sequential min/max expression.
4014 const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
4015 SmallPtrSet<const SCEV *, 16> SeenOps;
4016
4017 bool canRecurseInto(SCEVTypes Kind) const {
4018 // We can only recurse into the SCEV expression of the same effective type
4019 // as the type of our root SCEV expression.
4020 return RootKind == Kind || NonSequentialRootKind == Kind;
4021 };
4022
4023 RetVal visit(const SCEV *S) {
4024 // Has the whole operand been seen already?
4025 if (!SeenOps.insert(Ptr: S).second)
4026 return std::nullopt;
4027 if (isa<SCEVMinMaxExpr, SCEVSequentialMinMaxExpr>(Val: S)) {
4028 SCEVTypes Kind = S->getSCEVType();
4029
4030 if (!canRecurseInto(Kind))
4031 return S;
4032
4033 auto *NAry = cast<SCEVNAryExpr>(Val: S);
4034 SmallVector<SCEVUse> NewOps;
4035 bool Changed = visit(Kind, OrigOps: NAry->operands(), NewOps);
4036
4037 if (!Changed)
4038 return S;
4039 if (NewOps.empty())
4040 return std::nullopt;
4041
4042 return isa<SCEVSequentialMinMaxExpr>(Val: S)
4043 ? SE.getSequentialMinMaxExpr(Kind, Operands&: NewOps)
4044 : SE.getMinMaxExpr(Kind, Ops&: NewOps);
4045 }
4046 return S;
4047 }
4048
4049public:
4050 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
4051 SCEVTypes RootKind)
4052 : SE(SE), RootKind(RootKind),
4053 NonSequentialRootKind(
4054 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
4055 Ty: RootKind)) {}
4056
4057 bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<SCEVUse> OrigOps,
4058 SmallVectorImpl<SCEVUse> &NewOps) {
4059 bool Changed = false;
4060 SmallVector<SCEVUse> Ops;
4061 Ops.reserve(N: OrigOps.size());
4062
4063 for (const SCEV *Op : OrigOps) {
4064 RetVal NewOp = visit(S: Op);
4065 if (NewOp != Op)
4066 Changed = true;
4067 if (NewOp)
4068 Ops.emplace_back(Args&: *NewOp);
4069 }
4070
4071 if (Changed)
4072 NewOps = std::move(Ops);
4073 return Changed;
4074 }
4075};
4076
4077} // namespace
4078
4079static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind) {
4080 switch (Kind) {
4081 case scConstant:
4082 case scVScale:
4083 case scTruncate:
4084 case scZeroExtend:
4085 case scSignExtend:
4086 case scPtrToAddr:
4087 case scAddExpr:
4088 case scMulExpr:
4089 case scUDivExpr:
4090 case scAddRecExpr:
4091 case scUMaxExpr:
4092 case scSMaxExpr:
4093 case scUMinExpr:
4094 case scSMinExpr:
4095 case scUnknown:
4096 // If any operand is poison, the whole expression is poison.
4097 return true;
4098 case scSequentialUMinExpr:
4099 // FIXME: if the *first* operand is poison, the whole expression is poison.
4100 return false; // Pessimistically, say that it does not propagate poison.
4101 case scCouldNotCompute:
4102 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
4103 }
4104 llvm_unreachable("Unknown SCEV kind!");
4105}
4106
4107namespace {
4108// The only way poison may be introduced in a SCEV expression is from a
4109// poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown,
4110// not SCEVConstant). Notably, SCEVFlags on SCEV nodes can *not*
4111// introduce poison -- they encode guaranteed, non-speculated knowledge.
4112//
4113// Additionally, all SCEV nodes propagate poison from inputs to outputs,
4114// with the notable exception of umin_seq, where only poison from the first
4115// operand is (unconditionally) propagated.
4116struct SCEVPoisonCollector {
4117 bool LookThroughMaybePoisonBlocking;
4118 SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
4119 SCEVPoisonCollector(bool LookThroughMaybePoisonBlocking)
4120 : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
4121
4122 bool follow(const SCEV *S) {
4123 if (!LookThroughMaybePoisonBlocking &&
4124 !scevUnconditionallyPropagatesPoisonFromOperands(Kind: S->getSCEVType()))
4125 return false;
4126
4127 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
4128 if (!isGuaranteedNotToBePoison(V: SU->getValue()))
4129 MaybePoison.insert(Ptr: SU);
4130 }
4131 return true;
4132 }
4133 bool isDone() const { return false; }
4134};
4135} // namespace
4136
4137/// Return true if V is poison given that AssumedPoison is already poison.
4138static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) {
4139 // First collect all SCEVs that might result in AssumedPoison to be poison.
4140 // We need to look through potentially poison-blocking operations here,
4141 // because we want to find all SCEVs that *might* result in poison, not only
4142 // those that are *required* to.
4143 SCEVPoisonCollector PC1(/* LookThroughMaybePoisonBlocking */ true);
4144 visitAll(Root: AssumedPoison, Visitor&: PC1);
4145
4146 // AssumedPoison is never poison. As the assumption is false, the implication
4147 // is true. Don't bother walking the other SCEV in this case.
4148 if (PC1.MaybePoison.empty())
4149 return true;
4150
4151 // Collect all SCEVs in S that, if poison, *will* result in S being poison
4152 // as well. We cannot look through potentially poison-blocking operations
4153 // here, as their arguments only *may* make the result poison.
4154 SCEVPoisonCollector PC2(/* LookThroughMaybePoisonBlocking */ false);
4155 visitAll(Root: S, Visitor&: PC2);
4156
4157 // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison,
4158 // it will also make S poison by being part of PC2.MaybePoison.
4159 return llvm::set_is_subset(S1: PC1.MaybePoison, S2: PC2.MaybePoison);
4160}
4161
4162void ScalarEvolution::getPoisonGeneratingValues(
4163 SmallPtrSetImpl<const Value *> &Result, const SCEV *S) {
4164 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ false);
4165 visitAll(Root: S, Visitor&: PC);
4166 for (const SCEVUnknown *SU : PC.MaybePoison)
4167 Result.insert(Ptr: SU->getValue());
4168}
4169
4170bool ScalarEvolution::canReuseInstruction(
4171 const SCEV *S, Instruction *I,
4172 SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts) {
4173 // If the instruction cannot be poison, it's always safe to reuse.
4174 if (programUndefinedIfPoison(Inst: I))
4175 return true;
4176
4177 // Otherwise, it is possible that I is more poisonous that S. Collect the
4178 // poison-contributors of S, and then check whether I has any additional
4179 // poison-contributors. Poison that is contributed through poison-generating
4180 // flags is handled by dropping those flags instead.
4181 SmallPtrSet<const Value *, 8> PoisonVals;
4182 getPoisonGeneratingValues(Result&: PoisonVals, S);
4183
4184 SmallVector<Value *> Worklist;
4185 SmallPtrSet<Value *, 8> Visited;
4186 Worklist.push_back(Elt: I);
4187 unsigned NumVisitedInsts = 0;
4188 const unsigned InstLimit = std::max<unsigned>(a: 16, b: S->getExpressionSize());
4189 while (!Worklist.empty()) {
4190 Value *V = Worklist.pop_back_val();
4191 if (!Visited.insert(Ptr: V).second)
4192 continue;
4193
4194 // Either the value can't be poison, or the S would also be poison if it
4195 // is.
4196 if (PoisonVals.contains(Ptr: V) || ::isGuaranteedNotToBePoison(V))
4197 continue;
4198
4199 // Avoid walking large instruction graphs.
4200 if (++NumVisitedInsts > InstLimit)
4201 return false;
4202
4203 auto *I = dyn_cast<Instruction>(Val: V);
4204 if (!I)
4205 return false;
4206
4207 // Disjoint or instructions are interpreted as adds by SCEV. However, we
4208 // can't replace an arbitrary add with disjoint or, even if we drop the
4209 // flag. We would need to convert the or into an add.
4210 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: I))
4211 if (PDI->isDisjoint())
4212 return false;
4213
4214 // FIXME: Ignore vscale, even though it technically could be poison. Do this
4215 // because SCEV currently assumes it can't be poison. Remove this special
4216 // case once we proper model when vscale can be poison.
4217 if (auto *II = dyn_cast<IntrinsicInst>(Val: I);
4218 II && II->getIntrinsicID() == Intrinsic::vscale)
4219 continue;
4220
4221 if (canCreatePoison(Op: cast<Operator>(Val: I), /*ConsiderFlagsAndMetadata*/ false))
4222 return false;
4223
4224 // If the instruction can't create poison, we can recurse to its operands.
4225 if (I->hasPoisonGeneratingAnnotations())
4226 DropPoisonGeneratingInsts.push_back(Elt: I);
4227
4228 llvm::append_range(C&: Worklist, R: I->operands());
4229 }
4230 return true;
4231}
4232
4233const SCEV *
4234ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
4235 SmallVectorImpl<SCEVUse> &Ops) {
4236 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
4237 "Not a SCEVSequentialMinMaxExpr!");
4238 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4239 if (Ops.size() == 1)
4240 return Ops[0];
4241#ifndef NDEBUG
4242 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4243 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4244 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4245 "Operand types don't match!");
4246 assert(Ops[0]->getType()->isPointerTy() ==
4247 Ops[i]->getType()->isPointerTy() &&
4248 "min/max should be consistently pointerish");
4249 }
4250#endif
4251
4252 // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
4253 // so we can *NOT* do any kind of sorting of the expressions!
4254
4255 // Check if we have created the same expression before.
4256 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops))
4257 return S;
4258
4259 // FIXME: there are *some* simplifications that we can do here.
4260
4261 // Keep only the first instance of an operand.
4262 {
4263 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
4264 bool Changed = Deduplicator.visit(Kind, OrigOps: Ops, NewOps&: Ops);
4265 if (Changed)
4266 return getSequentialMinMaxExpr(Kind, Ops);
4267 }
4268
4269 // Check to see if one of the operands is of the same kind. If so, expand its
4270 // operands onto our operand list, and recurse to simplify.
4271 {
4272 unsigned Idx = 0;
4273 bool DeletedAny = false;
4274 while (Idx < Ops.size()) {
4275 if (Ops[Idx]->getSCEVType() != Kind) {
4276 ++Idx;
4277 continue;
4278 }
4279 const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Val&: Ops[Idx]);
4280 Ops.erase(CI: Ops.begin() + Idx);
4281 Ops.insert(I: Ops.begin() + Idx, From: SMME->operands().begin(),
4282 To: SMME->operands().end());
4283 DeletedAny = true;
4284 }
4285
4286 if (DeletedAny)
4287 return getSequentialMinMaxExpr(Kind, Ops);
4288 }
4289
4290 const SCEV *SaturationPoint;
4291 ICmpInst::Predicate Pred;
4292 switch (Kind) {
4293 case scSequentialUMinExpr:
4294 SaturationPoint = getZero(Ty: Ops[0]->getType());
4295 Pred = ICmpInst::ICMP_ULE;
4296 break;
4297 default:
4298 llvm_unreachable("Not a sequential min/max type.");
4299 }
4300
4301 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4302 if (!isGuaranteedNotToCauseUB(Op: Ops[i]))
4303 continue;
4304 // We can replace %x umin_seq %y with %x umin %y if either:
4305 // * %y being poison implies %x is also poison.
4306 // * %x cannot be the saturating value (e.g. zero for umin).
4307 if (::impliesPoison(AssumedPoison: Ops[i], S: Ops[i - 1]) ||
4308 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_NE, LHS: Ops[i - 1],
4309 RHS: SaturationPoint)) {
4310 SmallVector<SCEVUse, 2> SeqOps = {Ops[i - 1], Ops[i]};
4311 Ops[i - 1] = getMinMaxExpr(
4312 Kind: SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Ty: Kind),
4313 Ops&: SeqOps);
4314 Ops.erase(CI: Ops.begin() + i);
4315 return getSequentialMinMaxExpr(Kind, Ops);
4316 }
4317 // Fold %x umin_seq %y to %x if %x ule %y.
4318 // TODO: We might be able to prove the predicate for a later operand.
4319 if (isKnownViaNonRecursiveReasoning(Pred, LHS: Ops[i - 1], RHS: Ops[i])) {
4320 Ops.erase(CI: Ops.begin() + i);
4321 return getSequentialMinMaxExpr(Kind, Ops);
4322 }
4323 }
4324
4325 // Okay, it looks like we really DO need an expr. Check to see if we
4326 // already have one, otherwise create a new one.
4327 FoldingSetNodeID ID;
4328 ID.AddInteger(I: Kind);
4329 for (SCEVUse Op : Ops)
4330 ID.AddPointer(Ptr: Op.getOpaqueValue());
4331 FoldingSetInsertToken Token;
4332 const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
4333 if (ExistingSCEV)
4334 return ExistingSCEV;
4335
4336 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
4337 llvm::uninitialized_copy(Src&: Ops, Dst: O);
4338 SCEV *S = new (SCEVAllocator)
4339 SCEVSequentialMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
4340
4341 UniqueSCEVs.insert(N: S, Token);
4342 S->computeAndSetCanonical(SE&: *this);
4343 registerUser(User: S, Ops);
4344 return S;
4345}
4346
4347const SCEV *ScalarEvolution::getSMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4348 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4349 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4350}
4351
4352const SCEV *ScalarEvolution::getSMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4353 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4354}
4355
4356const SCEV *ScalarEvolution::getUMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4357 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4358 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4359}
4360
4361const SCEV *ScalarEvolution::getUMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4362 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4363}
4364
4365const SCEV *ScalarEvolution::getSMinExpr(SCEVUse LHS, SCEVUse RHS) {
4366 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4367 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4368}
4369
4370const SCEV *ScalarEvolution::getSMinExpr(SmallVectorImpl<SCEVUse> &Ops) {
4371 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4372}
4373
4374const SCEV *ScalarEvolution::getUMinExpr(SCEVUse LHS, SCEVUse RHS,
4375 bool Sequential) {
4376 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4377 return getUMinExpr(Operands&: Ops, Sequential);
4378}
4379
4380const SCEV *ScalarEvolution::getUMinExpr(SmallVectorImpl<SCEVUse> &Ops,
4381 bool Sequential) {
4382 return Sequential ? getSequentialMinMaxExpr(Kind: scSequentialUMinExpr, Ops)
4383 : getMinMaxExpr(Kind: scUMinExpr, Ops);
4384}
4385
4386const SCEV *
4387ScalarEvolution::getSizeOfExpr(Type *IntTy, TypeSize Size) {
4388 const SCEV *Res = getConstant(Ty: IntTy, V: Size.getKnownMinValue());
4389 if (Size.isScalable())
4390 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty: IntTy));
4391 return Res;
4392}
4393
4394const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
4395 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeAllocSize(Ty: AllocTy));
4396}
4397
4398const SCEV *ScalarEvolution::getStoreSizeOfExpr(Type *IntTy, Type *StoreTy) {
4399 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeStoreSize(Ty: StoreTy));
4400}
4401
4402const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
4403 StructType *STy,
4404 unsigned FieldNo) {
4405 // We can bypass creating a target-independent constant expression and then
4406 // folding it back into a ConstantInt. This is just a compile-time
4407 // optimization.
4408 const StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
4409 assert(!SL->getSizeInBits().isScalable() &&
4410 "Cannot get offset for structure containing scalable vector types");
4411 return getConstant(Ty: IntTy, V: SL->getElementOffset(Idx: FieldNo));
4412}
4413
4414const SCEV *ScalarEvolution::getUnknown(Value *V) {
4415 // Don't attempt to do anything other than create a SCEVUnknown object
4416 // here. createSCEV only calls getUnknown after checking for all other
4417 // interesting possibilities, and any other code that calls getUnknown
4418 // is doing so in order to hide a value from SCEV canonicalization.
4419
4420 FoldingSetNodeID ID;
4421 ID.AddInteger(I: scUnknown);
4422 ID.AddPointer(Ptr: V);
4423 FoldingSetInsertToken Token;
4424 if (SCEV *S = UniqueSCEVs.lookup(ID, Token)) {
4425 assert(cast<SCEVUnknown>(S)->getValue() == V &&
4426 "Stale SCEVUnknown in uniquing map!");
4427 return S;
4428 }
4429 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(Allocator&: SCEVAllocator), V, this,
4430 FirstUnknown);
4431 FirstUnknown = cast<SCEVUnknown>(Val: S);
4432 UniqueSCEVs.insert(N: S, Token);
4433 S->computeAndSetCanonical(SE&: *this);
4434 return S;
4435}
4436
4437//===----------------------------------------------------------------------===//
4438// Basic SCEV Analysis and PHI Idiom Recognition Code
4439//
4440
4441/// Test if values of the given type are analyzable within the SCEV
4442/// framework. This primarily includes integer types, and it can optionally
4443/// include pointer types if the ScalarEvolution class has access to
4444/// target-specific information.
4445bool ScalarEvolution::isSCEVable(Type *Ty) const {
4446 // Integers and pointers are always SCEVable.
4447 return Ty->isIntOrPtrTy();
4448}
4449
4450/// Return the size in bits of the specified type, for which isSCEVable must
4451/// return true.
4452uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
4453 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4454 if (Ty->isPointerTy())
4455 return getDataLayout().getIndexTypeSizeInBits(Ty);
4456 return getDataLayout().getTypeSizeInBits(Ty);
4457}
4458
4459/// Return a type with the same bitwidth as the given type and which represents
4460/// how SCEV will treat the given type, for which isSCEVable must return
4461/// true. For pointer types, this is the pointer index sized integer type.
4462Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
4463 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4464
4465 if (Ty->isIntegerTy())
4466 return Ty;
4467
4468 // The only other support type is pointer.
4469 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
4470 return getDataLayout().getIndexType(PtrTy: Ty);
4471}
4472
4473Type *ScalarEvolution::getWiderType(Type *T1, Type *T2) const {
4474 return getTypeSizeInBits(Ty: T1) >= getTypeSizeInBits(Ty: T2) ? T1 : T2;
4475}
4476
4477bool ScalarEvolution::instructionCouldExistWithOperands(const SCEV *A,
4478 const SCEV *B) {
4479 /// For a valid use point to exist, the defining scope of one operand
4480 /// must dominate the other.
4481 bool PreciseA, PreciseB;
4482 auto *ScopeA = getDefiningScopeBound(Ops: {A}, Precise&: PreciseA);
4483 auto *ScopeB = getDefiningScopeBound(Ops: {B}, Precise&: PreciseB);
4484 if (!PreciseA || !PreciseB)
4485 // Can't tell.
4486 return false;
4487 return (ScopeA == ScopeB) || DT.dominates(Def: ScopeA, User: ScopeB) ||
4488 DT.dominates(Def: ScopeB, User: ScopeA);
4489}
4490
4491const SCEV *ScalarEvolution::getCouldNotCompute() {
4492 return CouldNotCompute.get();
4493}
4494
4495bool ScalarEvolution::checkValidity(const SCEV *S) const {
4496 bool ContainsNulls = SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
4497 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
4498 return SU && SU->getValue() == nullptr;
4499 });
4500
4501 return !ContainsNulls;
4502}
4503
4504bool ScalarEvolution::containsAddRecurrence(const SCEV *S) {
4505 HasRecMapType::iterator I = HasRecMap.find(Val: S);
4506 if (I != HasRecMap.end())
4507 return I->second;
4508
4509 bool FoundAddRec =
4510 SCEVExprContains(Root: S, Pred: [](const SCEV *S) { return isa<SCEVAddRecExpr>(Val: S); });
4511 HasRecMap.insert(KV: {S, FoundAddRec});
4512 return FoundAddRec;
4513}
4514
4515/// Return the ValueOffsetPair set for \p S. \p S can be represented
4516/// by the value and offset from any ValueOffsetPair in the set.
4517ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) {
4518 ExprValueMapType::iterator SI = ExprValueMap.find_as(Val: S);
4519 if (SI == ExprValueMap.end())
4520 return {};
4521 return SI->second.getArrayRef();
4522}
4523
4524/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
4525/// cannot be used separately. eraseValueFromMap should be used to remove
4526/// V from ValueExprMap and ExprValueMap at the same time.
4527void ScalarEvolution::eraseValueFromMap(Value *V) {
4528 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4529 if (I != ValueExprMap.end()) {
4530 auto EVIt = ExprValueMap.find(Val: I->second);
4531 bool Removed = EVIt->second.remove(X: V);
4532 (void) Removed;
4533 assert(Removed && "Value not in ExprValueMap?");
4534 ValueExprMap.erase(I);
4535 }
4536}
4537
4538void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
4539 // A recursive query may have already computed the SCEV. It should be
4540 // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
4541 // inferred nowrap flags.
4542 auto It = ValueExprMap.find_as(Val: V);
4543 if (It == ValueExprMap.end()) {
4544 ValueExprMap.insert(KV: {SCEVCallbackVH(V, this), S});
4545 ExprValueMap[S].insert(X: V);
4546 }
4547}
4548
4549/// Return an existing SCEV if it exists, otherwise analyze the expression and
4550/// create a new one.
4551const SCEV *ScalarEvolution::getSCEV(Value *V) {
4552 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4553
4554 if (const SCEV *S = getExistingSCEV(V))
4555 return S;
4556 return createSCEVIter(V);
4557}
4558
4559const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
4560 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4561
4562 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4563 if (I != ValueExprMap.end()) {
4564 const SCEV *S = I->second;
4565 assert(checkValidity(S) &&
4566 "existing SCEV has not been properly invalidated");
4567 return S;
4568 }
4569 return nullptr;
4570}
4571
4572/// Return a SCEV corresponding to -V = -1*V
4573const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V, SCEVFlags Flags) {
4574 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4575 return getConstant(
4576 V: cast<ConstantInt>(Val: ConstantExpr::getNeg(C: VC->getValue())));
4577
4578 Type *Ty = V->getType();
4579 Ty = getEffectiveSCEVType(Ty);
4580 return getMulExpr(LHS: V, RHS: getMinusOne(Ty), Flags);
4581}
4582
4583/// If Expr computes ~A, return A else return nullptr
4584static const SCEV *MatchNotExpr(const SCEV *Expr) {
4585 const SCEV *MulOp;
4586 if (match(S: Expr, P: m_scev_Add(Op0: m_scev_AllOnes(),
4587 Op1: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: MulOp)))))
4588 return MulOp;
4589 return nullptr;
4590}
4591
4592/// Return a SCEV corresponding to ~V = -1-V
4593const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
4594 assert(!V->getType()->isPointerTy() && "Can't negate pointer");
4595
4596 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4597 return getConstant(
4598 V: cast<ConstantInt>(Val: ConstantExpr::getNot(C: VC->getValue())));
4599
4600 // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
4601 if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(Val: V)) {
4602 auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
4603 SmallVector<SCEVUse, 2> MatchedOperands;
4604 for (const SCEV *Operand : MME->operands()) {
4605 const SCEV *Matched = MatchNotExpr(Expr: Operand);
4606 if (!Matched)
4607 return (const SCEV *)nullptr;
4608 MatchedOperands.push_back(Elt: Matched);
4609 }
4610 return getMinMaxExpr(Kind: SCEVMinMaxExpr::negate(T: MME->getSCEVType()),
4611 Ops&: MatchedOperands);
4612 };
4613 if (const SCEV *Replaced = MatchMinMaxNegation(MME))
4614 return Replaced;
4615 }
4616
4617 Type *Ty = V->getType();
4618 Ty = getEffectiveSCEVType(Ty);
4619 return getMinusSCEV(LHS: getMinusOne(Ty), RHS: V);
4620}
4621
4622const SCEV *ScalarEvolution::removePointerBase(const SCEV *P) {
4623 assert(P->getType()->isPointerTy());
4624
4625 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: P)) {
4626 // The base of an AddRec is the first operand.
4627 SmallVector<SCEVUse> Ops{AddRec->operands()};
4628 Ops[0] = removePointerBase(P: Ops[0]);
4629 // Don't try to transfer nowrap flags for now. We could in some cases
4630 // (for example, if pointer operand of the AddRec is a SCEVUnknown).
4631 return getAddRecExpr(Operands&: Ops, L: AddRec->getLoop(), NWFlags: SCEV::FlagNone);
4632 }
4633 if (auto *Add = dyn_cast<SCEVAddExpr>(Val: P)) {
4634 // The base of an Add is the pointer operand.
4635 SmallVector<SCEVUse> Ops{Add->operands()};
4636 SCEVUse *PtrOp = nullptr;
4637 for (SCEVUse &AddOp : Ops) {
4638 if (AddOp->getType()->isPointerTy()) {
4639 assert(!PtrOp && "Cannot have multiple pointer ops");
4640 PtrOp = &AddOp;
4641 }
4642 }
4643 *PtrOp = removePointerBase(P: *PtrOp);
4644 // Don't try to transfer nowrap flags for now. We could in some cases
4645 // (for example, if the pointer operand of the Add is a SCEVUnknown).
4646 return getAddExpr(Ops);
4647 }
4648 // Any other expression must be a pointer base.
4649 return getZero(Ty: P->getType());
4650}
4651
4652const SCEV *ScalarEvolution::getMinusSCEV(SCEVUse LHS, SCEVUse RHS,
4653 SCEVFlags Flags, unsigned Depth) {
4654 // Fast path: X - X --> 0.
4655 if (LHS == RHS)
4656 return getZero(Ty: LHS->getType());
4657
4658 // If we subtract two pointers with different pointer bases, bail.
4659 // Eventually, we're going to add an assertion to getMulExpr that we
4660 // can't multiply by a pointer.
4661 if (RHS->getType()->isPointerTy()) {
4662 if (!LHS->getType()->isPointerTy() ||
4663 getPointerBase(V: LHS) != getPointerBase(V: RHS))
4664 return getCouldNotCompute();
4665 LHS = removePointerBase(P: LHS);
4666 RHS = removePointerBase(P: RHS);
4667 }
4668
4669 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
4670 // makes it so that we cannot make much use of NUW.
4671 auto AddFlags = SCEV::FlagNone;
4672 const bool RHSIsNotMinSigned =
4673 !getSignedRangeMin(S: RHS).isMinSignedValue();
4674 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW)) {
4675 // Let M be the minimum representable signed value. Then (-1)*RHS
4676 // signed-wraps if and only if RHS is M. That can happen even for
4677 // a NSW subtraction because e.g. (-1)*M signed-wraps even though
4678 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
4679 // (-1)*RHS, we need to prove that RHS != M.
4680 //
4681 // If LHS is non-negative and we know that LHS - RHS does not
4682 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
4683 // either by proving that RHS > M or that LHS >= 0.
4684 if (RHSIsNotMinSigned || isKnownNonNegative(S: LHS)) {
4685 AddFlags = SCEV::FlagNSW;
4686 }
4687 }
4688
4689 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
4690 // RHS is NSW and LHS >= 0.
4691 //
4692 // The difficulty here is that the NSW flag may have been proven
4693 // relative to a loop that is to be found in a recurrence in LHS and
4694 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
4695 // larger scope than intended.
4696 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagNone;
4697
4698 return getAddExpr(LHS, RHS: getNegativeSCEV(V: RHS, Flags: NegFlags), Flags: AddFlags, Depth);
4699}
4700
4701const SCEV *ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty,
4702 unsigned Depth) {
4703 Type *SrcTy = V->getType();
4704 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4705 "Cannot truncate or zero extend with non-integer arguments!");
4706 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4707 return V; // No conversion
4708 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4709 return getTruncateExpr(Op: V, Ty, Depth);
4710 return getZeroExtendExpr(Op: V, Ty, Depth);
4711}
4712
4713const SCEV *ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, Type *Ty,
4714 unsigned Depth) {
4715 Type *SrcTy = V->getType();
4716 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4717 "Cannot truncate or zero extend with non-integer arguments!");
4718 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4719 return V; // No conversion
4720 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4721 return getTruncateExpr(Op: V, Ty, Depth);
4722 return getSignExtendExpr(Op: V, Ty, Depth);
4723}
4724
4725const SCEV *ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
4726 Type *SrcTy = V->getType();
4727 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4728 "Cannot noop or zero extend with non-integer arguments!");
4729 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4730 "getNoopOrZeroExtend cannot truncate!");
4731 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4732 return V; // No conversion
4733 return getZeroExtendExpr(Op: V, Ty);
4734}
4735
4736const SCEV *ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
4737 Type *SrcTy = V->getType();
4738 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4739 "Cannot noop or sign extend with non-integer arguments!");
4740 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4741 "getNoopOrSignExtend cannot truncate!");
4742 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4743 return V; // No conversion
4744 return getSignExtendExpr(Op: V, Ty);
4745}
4746
4747const SCEV *ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
4748 Type *SrcTy = V->getType();
4749 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4750 "Cannot noop or any extend with non-integer arguments!");
4751 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4752 "getNoopOrAnyExtend cannot truncate!");
4753 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4754 return V; // No conversion
4755 return getAnyExtendExpr(Op: V, Ty);
4756}
4757
4758const SCEV *ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
4759 Type *SrcTy = V->getType();
4760 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4761 "Cannot truncate or noop with non-integer arguments!");
4762 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
4763 "getTruncateOrNoop cannot extend!");
4764 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4765 return V; // No conversion
4766 return getTruncateExpr(Op: V, Ty);
4767}
4768
4769const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
4770 const SCEV *RHS) {
4771 const SCEV *PromotedLHS = LHS;
4772 const SCEV *PromotedRHS = RHS;
4773
4774 if (getTypeSizeInBits(Ty: LHS->getType()) > getTypeSizeInBits(Ty: RHS->getType()))
4775 PromotedRHS = getZeroExtendExpr(Op: RHS, Ty: LHS->getType());
4776 else
4777 PromotedLHS = getNoopOrZeroExtend(V: LHS, Ty: RHS->getType());
4778
4779 return getUMaxExpr(LHS: PromotedLHS, RHS: PromotedRHS);
4780}
4781
4782const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
4783 const SCEV *RHS,
4784 bool Sequential) {
4785 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4786 return getUMinFromMismatchedTypes(Ops, Sequential);
4787}
4788
4789const SCEV *
4790ScalarEvolution::getUMinFromMismatchedTypes(SmallVectorImpl<SCEVUse> &Ops,
4791 bool Sequential) {
4792 assert(!Ops.empty() && "At least one operand must be!");
4793 // Trivial case.
4794 if (Ops.size() == 1)
4795 return Ops[0];
4796
4797 // Find the max type first.
4798 Type *MaxType = nullptr;
4799 for (SCEVUse S : Ops)
4800 if (MaxType)
4801 MaxType = getWiderType(T1: MaxType, T2: S->getType());
4802 else
4803 MaxType = S->getType();
4804 assert(MaxType && "Failed to find maximum type!");
4805
4806 // Extend all ops to max type.
4807 SmallVector<SCEVUse, 2> PromotedOps;
4808 for (SCEVUse S : Ops)
4809 PromotedOps.push_back(Elt: getNoopOrZeroExtend(V: S, Ty: MaxType));
4810
4811 // Generate umin.
4812 return getUMinExpr(Ops&: PromotedOps, Sequential);
4813}
4814
4815const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
4816 // A pointer operand may evaluate to a nonpointer expression, such as null.
4817 if (!V->getType()->isPointerTy())
4818 return V;
4819
4820 while (true) {
4821 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: V)) {
4822 V = AddRec->getStart();
4823 } else if (auto *Add = dyn_cast<SCEVAddExpr>(Val: V)) {
4824 const SCEV *PtrOp = nullptr;
4825 for (const SCEV *AddOp : Add->operands()) {
4826 if (AddOp->getType()->isPointerTy()) {
4827 assert(!PtrOp && "Cannot have multiple pointer ops");
4828 PtrOp = AddOp;
4829 }
4830 }
4831 assert(PtrOp && "Must have pointer op");
4832 V = PtrOp;
4833 } else // Not something we can look further into.
4834 return V;
4835 }
4836}
4837
4838/// Push users of the given Instruction onto the given Worklist.
4839static void PushDefUseChildren(Instruction *I,
4840 SmallVectorImpl<Instruction *> &Worklist,
4841 SmallPtrSetImpl<Instruction *> &Visited) {
4842 // Push the def-use children onto the Worklist stack.
4843 for (User *U : I->users()) {
4844 auto *UserInsn = cast<Instruction>(Val: U);
4845 if (Visited.insert(Ptr: UserInsn).second)
4846 Worklist.push_back(Elt: UserInsn);
4847 }
4848}
4849
4850namespace {
4851
4852/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
4853/// expression in case its Loop is L. If it is not L then
4854/// if IgnoreOtherLoops is true then use AddRec itself
4855/// otherwise rewrite cannot be done.
4856/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4857class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
4858public:
4859 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
4860 bool IgnoreOtherLoops = true) {
4861 SCEVInitRewriter Rewriter(L, SE);
4862 const SCEV *Result = Rewriter.visit(S);
4863 if (Rewriter.hasSeenLoopVariantSCEVUnknown())
4864 return SE.getCouldNotCompute();
4865 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
4866 ? SE.getCouldNotCompute()
4867 : Result;
4868 }
4869
4870 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4871 if (!SE.isLoopInvariant(S: Expr, L))
4872 SeenLoopVariantSCEVUnknown = true;
4873 return Expr;
4874 }
4875
4876 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4877 // Only re-write AddRecExprs for this loop.
4878 if (Expr->getLoop() == L)
4879 return Expr->getStart();
4880 SeenOtherLoops = true;
4881 return Expr;
4882 }
4883
4884 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4885
4886 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4887
4888private:
4889 explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
4890 : SCEVRewriteVisitor(SE), L(L) {}
4891
4892 const Loop *L;
4893 bool SeenLoopVariantSCEVUnknown = false;
4894 bool SeenOtherLoops = false;
4895};
4896
4897/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
4898/// increment expression in case its Loop is L. If it is not L then
4899/// use AddRec itself.
4900/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4901class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
4902public:
4903 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
4904 SCEVPostIncRewriter Rewriter(L, SE);
4905 const SCEV *Result = Rewriter.visit(S);
4906 return Rewriter.hasSeenLoopVariantSCEVUnknown()
4907 ? SE.getCouldNotCompute()
4908 : Result;
4909 }
4910
4911 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4912 if (!SE.isLoopInvariant(S: Expr, L))
4913 SeenLoopVariantSCEVUnknown = true;
4914 return Expr;
4915 }
4916
4917 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4918 // Only re-write AddRecExprs for this loop.
4919 if (Expr->getLoop() == L)
4920 return Expr->getPostIncExpr(SE);
4921 SeenOtherLoops = true;
4922 return Expr;
4923 }
4924
4925 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4926
4927 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4928
4929private:
4930 explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
4931 : SCEVRewriteVisitor(SE), L(L) {}
4932
4933 const Loop *L;
4934 bool SeenLoopVariantSCEVUnknown = false;
4935 bool SeenOtherLoops = false;
4936};
4937
4938/// This class evaluates the compare condition by matching it against the
4939/// condition of loop latch. If there is a match we assume a true value
4940/// for the condition while building SCEV nodes.
4941class SCEVBackedgeConditionFolder
4942 : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
4943public:
4944 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4945 ScalarEvolution &SE) {
4946 bool IsPosBECond = false;
4947 Value *BECond = nullptr;
4948 if (BasicBlock *Latch = L->getLoopLatch()) {
4949 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Latch->getTerminator())) {
4950 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
4951 "Both outgoing branches should not target same header!");
4952 BECond = BI->getCondition();
4953 IsPosBECond = BI->getSuccessor(i: 0) == L->getHeader();
4954 } else {
4955 return S;
4956 }
4957 }
4958 SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
4959 return Rewriter.visit(S);
4960 }
4961
4962 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4963 const SCEV *Result = Expr;
4964 bool InvariantF = SE.isLoopInvariant(S: Expr, L);
4965
4966 if (!InvariantF) {
4967 Instruction *I = cast<Instruction>(Val: Expr->getValue());
4968 switch (I->getOpcode()) {
4969 case Instruction::Select: {
4970 SelectInst *SI = cast<SelectInst>(Val: I);
4971 std::optional<const SCEV *> Res =
4972 compareWithBackedgeCondition(IC: SI->getCondition());
4973 if (Res) {
4974 bool IsOne = cast<SCEVConstant>(Val: *Res)->getValue()->isOne();
4975 Result = SE.getSCEV(V: IsOne ? SI->getTrueValue() : SI->getFalseValue());
4976 }
4977 break;
4978 }
4979 default: {
4980 std::optional<const SCEV *> Res = compareWithBackedgeCondition(IC: I);
4981 if (Res)
4982 Result = *Res;
4983 break;
4984 }
4985 }
4986 }
4987 return Result;
4988 }
4989
4990private:
4991 explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
4992 bool IsPosBECond, ScalarEvolution &SE)
4993 : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
4994 IsPositiveBECond(IsPosBECond) {}
4995
4996 std::optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
4997
4998 const Loop *L;
4999 /// Loop back condition.
5000 Value *BackedgeCond = nullptr;
5001 /// Set to true if loop back is on positive branch condition.
5002 bool IsPositiveBECond;
5003};
5004
5005std::optional<const SCEV *>
5006SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
5007
5008 // If value matches the backedge condition for loop latch,
5009 // then return a constant evolution node based on loopback
5010 // branch taken.
5011 if (BackedgeCond == IC)
5012 return IsPositiveBECond ? SE.getOne(Ty: Type::getInt1Ty(C&: SE.getContext()))
5013 : SE.getZero(Ty: Type::getInt1Ty(C&: SE.getContext()));
5014 return std::nullopt;
5015}
5016
5017class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
5018public:
5019 static const SCEV *rewrite(const SCEV *S, const Loop *L,
5020 ScalarEvolution &SE) {
5021 SCEVShiftRewriter Rewriter(L, SE);
5022 const SCEV *Result = Rewriter.visit(S);
5023 return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
5024 }
5025
5026 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5027 // Only allow AddRecExprs for this loop.
5028 if (!SE.isLoopInvariant(S: Expr, L))
5029 Valid = false;
5030 return Expr;
5031 }
5032
5033 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5034 if (Expr->getLoop() == L && Expr->isAffine())
5035 return SE.getMinusSCEV(LHS: Expr, RHS: Expr->getStepRecurrence(SE));
5036 Valid = false;
5037 return Expr;
5038 }
5039
5040 bool isValid() { return Valid; }
5041
5042private:
5043 explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
5044 : SCEVRewriteVisitor(SE), L(L) {}
5045
5046 const Loop *L;
5047 bool Valid = true;
5048};
5049
5050} // end anonymous namespace
5051
5052void ScalarEvolution::inferNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
5053 if (!AR->isAffine())
5054 return;
5055
5056 // Force computation of ranges, which will also perform range-based flag
5057 // inference.
5058 if (!AR->hasNoSignedWrap())
5059 (void)getSignedRange(S: AR);
5060
5061 if (!AR->hasNoUnsignedWrap())
5062 (void)getUnsignedRange(S: AR);
5063
5064 if (!AR->hasNoSelfWrap()) {
5065 const SCEV *BECount = getConstantMaxBackedgeTakenCount(L: AR->getLoop());
5066 if (const SCEVConstant *BECountMax = dyn_cast<SCEVConstant>(Val: BECount)) {
5067 ConstantRange StepCR = getSignedRange(S: AR->getStepRecurrence(SE&: *this));
5068 const APInt &BECountAP = BECountMax->getAPInt();
5069 unsigned NoOverflowBitWidth =
5070 BECountAP.getActiveBits() + StepCR.getMinSignedBits();
5071 if (NoOverflowBitWidth <= getTypeSizeInBits(Ty: AR->getType()))
5072 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
5073 }
5074 }
5075}
5076
5077SCEVFlags
5078ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5079 SCEVFlags Result = AR->getNoWrapFlags();
5080
5081 if (AR->hasNoSignedWrap())
5082 return Result;
5083
5084 if (!AR->isAffine())
5085 return Result;
5086
5087 // This function can be expensive, only try to prove NSW once per AddRec.
5088 if (!SignedWrapViaInductionTried.insert(Ptr: AR).second)
5089 return Result;
5090
5091 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5092 const Loop *L = AR->getLoop();
5093
5094 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5095 // Note that this serves two purposes: It filters out loops that are
5096 // simply not analyzable, and it covers the case where this code is
5097 // being called from within backedge-taken count analysis, such that
5098 // attempting to ask for the backedge-taken count would likely result
5099 // in infinite recursion. In the later case, the analysis code will
5100 // cope with a conservative value, and it will take care to purge
5101 // that value once it has finished.
5102 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5103
5104 // Normally, in the cases we can prove no-overflow via a
5105 // backedge guarding condition, we can also compute a backedge
5106 // taken count for the loop. The exceptions are assumptions and
5107 // guards present in the loop -- SCEV is not great at exploiting
5108 // these to compute max backedge taken counts, but can still use
5109 // these to prove lack of overflow. Use this fact to avoid
5110 // doing extra work that may not pay off.
5111
5112 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5113 AC.assumptions().empty())
5114 return Result;
5115
5116 // If the backedge is guarded by a comparison with the pre-inc value the
5117 // addrec is safe. Also, if the entry is guarded by a comparison with the
5118 // start value and the backedge is guarded by a comparison with the post-inc
5119 // value, the addrec is safe.
5120 ICmpInst::Predicate Pred;
5121 const SCEV *OverflowLimit =
5122 getSignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
5123 if (OverflowLimit &&
5124 (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
5125 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))) {
5126 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNSW);
5127 }
5128 return Result;
5129}
5130SCEVFlags
5131ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5132 SCEVFlags Result = AR->getNoWrapFlags();
5133
5134 if (AR->hasNoUnsignedWrap())
5135 return Result;
5136
5137 if (!AR->isAffine())
5138 return Result;
5139
5140 // This function can be expensive, only try to prove NUW once per AddRec.
5141 if (!UnsignedWrapViaInductionTried.insert(Ptr: AR).second)
5142 return Result;
5143
5144 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5145 const Loop *L = AR->getLoop();
5146
5147 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5148 // Note that this serves two purposes: It filters out loops that are
5149 // simply not analyzable, and it covers the case where this code is
5150 // being called from within backedge-taken count analysis, such that
5151 // attempting to ask for the backedge-taken count would likely result
5152 // in infinite recursion. In the later case, the analysis code will
5153 // cope with a conservative value, and it will take care to purge
5154 // that value once it has finished.
5155 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5156
5157 // Normally, in the cases we can prove no-overflow via a
5158 // backedge guarding condition, we can also compute a backedge
5159 // taken count for the loop. The exceptions are assumptions and
5160 // guards present in the loop -- SCEV is not great at exploiting
5161 // these to compute max backedge taken counts, but can still use
5162 // these to prove lack of overflow. Use this fact to avoid
5163 // doing extra work that may not pay off.
5164
5165 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5166 AC.assumptions().empty())
5167 return Result;
5168
5169 // If the backedge is guarded by a comparison with the pre-inc value the
5170 // addrec is safe. Also, if the entry is guarded by a comparison with the
5171 // start value and the backedge is guarded by a comparison with the post-inc
5172 // value, the addrec is safe.
5173 if (isKnownPositive(S: Step)) {
5174 ICmpInst::Predicate Pred;
5175 const SCEV *OverflowLimit =
5176 getUnsignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
5177 if (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
5178 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))
5179 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNUW);
5180 }
5181 return Result;
5182}
5183
5184namespace {
5185
5186/// Represents an abstract binary operation. This may exist as a
5187/// normal instruction or constant expression, or may have been
5188/// derived from an expression tree.
5189struct BinaryOp {
5190 unsigned Opcode;
5191 Value *LHS;
5192 Value *RHS;
5193 bool IsNSW = false;
5194 bool IsNUW = false;
5195
5196 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
5197 /// constant expression.
5198 Operator *Op = nullptr;
5199
5200 explicit BinaryOp(Operator *Op)
5201 : Opcode(Op->getOpcode()), LHS(Op->getOperand(i: 0)), RHS(Op->getOperand(i: 1)),
5202 Op(Op) {
5203 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: Op)) {
5204 IsNSW = OBO->hasNoSignedWrap();
5205 IsNUW = OBO->hasNoUnsignedWrap();
5206 }
5207 }
5208
5209 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
5210 bool IsNUW = false)
5211 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5212};
5213
5214} // end anonymous namespace
5215
5216/// Try to map \p V into a BinaryOp, and return \c std::nullopt on failure.
5217static std::optional<BinaryOp> MatchBinaryOp(Value *V, const DataLayout &DL,
5218 AssumptionCache &AC,
5219 const DominatorTree &DT,
5220 const Instruction *CtxI) {
5221 auto *Op = dyn_cast<Operator>(Val: V);
5222 if (!Op)
5223 return std::nullopt;
5224
5225 // Implementation detail: all the cleverness here should happen without
5226 // creating new SCEV expressions -- our caller knowns tricks to avoid creating
5227 // SCEV expressions when possible, and we should not break that.
5228
5229 switch (Op->getOpcode()) {
5230 case Instruction::Add:
5231 case Instruction::Sub:
5232 case Instruction::Mul:
5233 case Instruction::UDiv:
5234 case Instruction::URem:
5235 case Instruction::And:
5236 case Instruction::AShr:
5237 case Instruction::Shl:
5238 return BinaryOp(Op);
5239
5240 case Instruction::Or: {
5241 // Convert or disjoint into add nuw nsw.
5242 if (cast<PossiblyDisjointInst>(Val: Op)->isDisjoint()) {
5243 BinaryOp BinOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1),
5244 /*IsNSW=*/true, /*IsNUW=*/true);
5245 // Keep the reference to the original instruction so that we can later
5246 // check whether it can produce poison value or not.
5247 BinOp.Op = Op;
5248 return BinOp;
5249 }
5250 return BinaryOp(Op);
5251 }
5252
5253 case Instruction::Xor:
5254 if (auto *RHSC = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1)))
5255 // If the RHS of the xor is a signmask, then this is just an add.
5256 // Instcombine turns add of signmask into xor as a strength reduction step.
5257 if (RHSC->getValue().isSignMask())
5258 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5259 // Binary `xor` is a bit-wise `add`.
5260 if (V->getType()->isIntegerTy(BitWidth: 1))
5261 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5262 return BinaryOp(Op);
5263
5264 case Instruction::LShr:
5265 // Turn logical shift right of a constant into a unsigned divide.
5266 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1))) {
5267 uint32_t BitWidth = cast<IntegerType>(Val: Op->getType())->getBitWidth();
5268
5269 // If the shift count is not less than the bitwidth, the result of
5270 // the shift is undefined. Don't try to analyze it, because the
5271 // resolution chosen here may differ from the resolution chosen in
5272 // other parts of the compiler.
5273 if (SA->getValue().ult(RHS: BitWidth)) {
5274 Constant *X =
5275 ConstantInt::get(Context&: SA->getContext(),
5276 V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
5277 return BinaryOp(Instruction::UDiv, Op->getOperand(i: 0), X);
5278 }
5279 }
5280 return BinaryOp(Op);
5281
5282 case Instruction::ExtractValue: {
5283 auto *EVI = cast<ExtractValueInst>(Val: Op);
5284 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5285 break;
5286
5287 auto *WO = dyn_cast<WithOverflowInst>(Val: EVI->getAggregateOperand());
5288 if (!WO)
5289 break;
5290
5291 Instruction::BinaryOps BinOp = WO->getBinaryOp();
5292 bool Signed = WO->isSigned();
5293 // TODO: Should add nuw/nsw flags for mul as well.
5294 if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
5295 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5296
5297 // Now that we know that all uses of the arithmetic-result component of
5298 // CI are guarded by the overflow check, we can go ahead and pretend
5299 // that the arithmetic is non-overflowing.
5300 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5301 /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
5302 }
5303
5304 default:
5305 break;
5306 }
5307
5308 // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
5309 // semantics as a Sub, return a binary sub expression.
5310 if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
5311 if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5312 return BinaryOp(Instruction::Sub, II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1));
5313
5314 return std::nullopt;
5315}
5316
5317/// Helper function to createAddRecFromPHIWithCasts. We have a phi
5318/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
5319/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
5320/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
5321/// follows one of the following patterns:
5322/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5323/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5324/// If the SCEV expression of \p Op conforms with one of the expected patterns
5325/// we return the type of the truncation operation, and indicate whether the
5326/// truncated type should be treated as signed/unsigned by setting
5327/// \p Signed to true/false, respectively.
5328static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
5329 bool &Signed, ScalarEvolution &SE) {
5330 // The case where Op == SymbolicPHI (that is, with no type conversions on
5331 // the way) is handled by the regular add recurrence creating logic and
5332 // would have already been triggered in createAddRecForPHI. Reaching it here
5333 // means that createAddRecFromPHI had failed for this PHI before (e.g.,
5334 // because one of the other operands of the SCEVAddExpr updating this PHI is
5335 // not invariant).
5336 //
5337 // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
5338 // this case predicates that allow us to prove that Op == SymbolicPHI will
5339 // be added.
5340 if (Op == SymbolicPHI)
5341 return nullptr;
5342
5343 unsigned SourceBits = SE.getTypeSizeInBits(Ty: SymbolicPHI->getType());
5344 unsigned NewBits = SE.getTypeSizeInBits(Ty: Op->getType());
5345 if (SourceBits != NewBits)
5346 return nullptr;
5347
5348 if (match(S: Op, P: m_scev_SExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5349 Signed = true;
5350 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5351 }
5352 if (match(S: Op, P: m_scev_ZExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5353 Signed = false;
5354 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5355 }
5356 return nullptr;
5357}
5358
5359static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
5360 if (!PN->getType()->isIntegerTy())
5361 return nullptr;
5362 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5363 if (!L || L->getHeader() != PN->getParent())
5364 return nullptr;
5365 return L;
5366}
5367
5368// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
5369// computation that updates the phi follows the following pattern:
5370// (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
5371// which correspond to a phi->trunc->sext/zext->add->phi update chain.
5372// If so, try to see if it can be rewritten as an AddRecExpr under some
5373// Predicates. If successful, return them as a pair. Also cache the results
5374// of the analysis.
5375//
5376// Example usage scenario:
5377// Say the Rewriter is called for the following SCEV:
5378// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5379// where:
5380// %X = phi i64 (%Start, %BEValue)
5381// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
5382// and call this function with %SymbolicPHI = %X.
5383//
5384// The analysis will find that the value coming around the backedge has
5385// the following SCEV:
5386// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5387// Upon concluding that this matches the desired pattern, the function
5388// will return the pair {NewAddRec, SmallPredsVec} where:
5389// NewAddRec = {%Start,+,%Step}
5390// SmallPredsVec = {P1, P2, P3} as follows:
5391// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
5392// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
5393// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
5394// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
5395// under the predicates {P1,P2,P3}.
5396// This predicated rewrite will be cached in PredicatedSCEVRewrites:
5397// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
5398//
5399// TODO's:
5400//
5401// 1) Extend the Induction descriptor to also support inductions that involve
5402// casts: When needed (namely, when we are called in the context of the
5403// vectorizer induction analysis), a Set of cast instructions will be
5404// populated by this method, and provided back to isInductionPHI. This is
5405// needed to allow the vectorizer to properly record them to be ignored by
5406// the cost model and to avoid vectorizing them (otherwise these casts,
5407// which are redundant under the runtime overflow checks, will be
5408// vectorized, which can be costly).
5409//
5410// 2) Support additional induction/PHISCEV patterns: We also want to support
5411// inductions where the sext-trunc / zext-trunc operations (partly) occur
5412// after the induction update operation (the induction increment):
5413//
5414// (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
5415// which correspond to a phi->add->trunc->sext/zext->phi update chain.
5416//
5417// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
5418// which correspond to a phi->trunc->add->sext/zext->phi update chain.
5419//
5420// 3) Outline common code with createAddRecFromPHI to avoid duplication.
5421std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5422ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
5423 SmallVector<const SCEVPredicate *, 3> Predicates;
5424
5425 // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
5426 // return an AddRec expression under some predicate.
5427
5428 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5429 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5430 assert(L && "Expecting an integer loop header phi");
5431
5432 // The loop may have multiple entrances or multiple exits; we can analyze
5433 // this phi as an addrec if it has a unique entry value and a unique
5434 // backedge value.
5435 Value *BEValueV = nullptr, *StartValueV = nullptr;
5436 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5437 Value *V = PN->getIncomingValue(i);
5438 if (L->contains(BB: PN->getIncomingBlock(i))) {
5439 if (!BEValueV) {
5440 BEValueV = V;
5441 } else if (BEValueV != V) {
5442 BEValueV = nullptr;
5443 break;
5444 }
5445 } else if (!StartValueV) {
5446 StartValueV = V;
5447 } else if (StartValueV != V) {
5448 StartValueV = nullptr;
5449 break;
5450 }
5451 }
5452 if (!BEValueV || !StartValueV)
5453 return std::nullopt;
5454
5455 const SCEV *BEValue = getSCEV(V: BEValueV);
5456
5457 // If the value coming around the backedge is an add with the symbolic
5458 // value we just inserted, possibly with casts that we can ignore under
5459 // an appropriate runtime guard, then we found a simple induction variable!
5460 const auto *Add = dyn_cast<SCEVAddExpr>(Val: BEValue);
5461 if (!Add)
5462 return std::nullopt;
5463
5464 // If there is a single occurrence of the symbolic value, possibly
5465 // casted, replace it with a recurrence.
5466 unsigned FoundIndex = Add->getNumOperands();
5467 Type *TruncTy = nullptr;
5468 bool Signed;
5469 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5470 if ((TruncTy =
5471 isSimpleCastedPHI(Op: Add->getOperand(i), SymbolicPHI, Signed, SE&: *this)))
5472 if (FoundIndex == e) {
5473 FoundIndex = i;
5474 break;
5475 }
5476
5477 if (FoundIndex == Add->getNumOperands())
5478 return std::nullopt;
5479
5480 // Create an add with everything but the specified operand.
5481 SmallVector<SCEVUse, 8> Ops;
5482 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5483 if (i != FoundIndex)
5484 Ops.push_back(Elt: Add->getOperand(i));
5485 const SCEV *Accum = getAddExpr(Ops);
5486
5487 // The runtime checks will not be valid if the step amount is
5488 // varying inside the loop.
5489 if (!isLoopInvariant(S: Accum, L))
5490 return std::nullopt;
5491
5492 // *** Part2: Create the predicates
5493
5494 // Analysis was successful: we have a phi-with-cast pattern for which we
5495 // can return an AddRec expression under the following predicates:
5496 //
5497 // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
5498 // fits within the truncated type (does not overflow) for i = 0 to n-1.
5499 // P2: An Equal predicate that guarantees that
5500 // Start = (Ext ix (Trunc iy (Start) to ix) to iy)
5501 // P3: An Equal predicate that guarantees that
5502 // Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
5503 //
5504 // As we next prove, the above predicates guarantee that:
5505 // Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
5506 //
5507 //
5508 // More formally, we want to prove that:
5509 // Expr(i+1) = Start + (i+1) * Accum
5510 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5511 //
5512 // Given that:
5513 // 1) Expr(0) = Start
5514 // 2) Expr(1) = Start + Accum
5515 // = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
5516 // 3) Induction hypothesis (step i):
5517 // Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
5518 //
5519 // Proof:
5520 // Expr(i+1) =
5521 // = Start + (i+1)*Accum
5522 // = (Start + i*Accum) + Accum
5523 // = Expr(i) + Accum
5524 // = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
5525 // :: from step i
5526 //
5527 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
5528 //
5529 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
5530 // + (Ext ix (Trunc iy (Accum) to ix) to iy)
5531 // + Accum :: from P3
5532 //
5533 // = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
5534 // + Accum :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
5535 //
5536 // = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
5537 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5538 //
5539 // By induction, the same applies to all iterations 1<=i<n:
5540 //
5541
5542 // Create a truncated addrec for which we will add a no overflow check (P1).
5543 const SCEV *StartVal = getSCEV(V: StartValueV);
5544 const SCEV *PHISCEV =
5545 getAddRecExpr(Start: getTruncateExpr(Op: StartVal, Ty: TruncTy),
5546 Step: getTruncateExpr(Op: Accum, Ty: TruncTy), L, Flags: SCEV::FlagNone);
5547
5548 // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
5549 // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
5550 // will be constant.
5551 //
5552 // If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
5553 // add P1.
5554 if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV)) {
5555 SCEVWrapPredicate::IncrementWrapFlags AddedFlags =
5556 Signed ? SCEVWrapPredicate::IncrementNSSW
5557 : SCEVWrapPredicate::IncrementNUSW;
5558 const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
5559 Predicates.push_back(Elt: AddRecPred);
5560 }
5561
5562 // Create the Equal Predicates P2,P3:
5563
5564 // It is possible that the predicates P2 and/or P3 are computable at
5565 // compile time due to StartVal and/or Accum being constants.
5566 // If either one is, then we can check that now and escape if either P2
5567 // or P3 is false.
5568
5569 // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
5570 // for each of StartVal and Accum
5571 auto getExtendedExpr = [&](const SCEV *Expr,
5572 bool CreateSignExtend) -> const SCEV * {
5573 assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant");
5574 const SCEV *TruncatedExpr = getTruncateExpr(Op: Expr, Ty: TruncTy);
5575 const SCEV *ExtendedExpr =
5576 CreateSignExtend ? getSignExtendExpr(Op: TruncatedExpr, Ty: Expr->getType())
5577 : getZeroExtendExpr(Op: TruncatedExpr, Ty: Expr->getType());
5578 return ExtendedExpr;
5579 };
5580
5581 // Given:
5582 // ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
5583 // = getExtendedExpr(Expr)
5584 // Determine whether the predicate P: Expr == ExtendedExpr
5585 // is known to be false at compile time
5586 auto PredIsKnownFalse = [&](const SCEV *Expr,
5587 const SCEV *ExtendedExpr) -> bool {
5588 return Expr != ExtendedExpr &&
5589 isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: Expr, RHS: ExtendedExpr);
5590 };
5591
5592 const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
5593 if (PredIsKnownFalse(StartVal, StartExtended)) {
5594 LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";);
5595 return std::nullopt;
5596 }
5597
5598 // The Step is always Signed (because the overflow checks are either
5599 // NSSW or NUSW)
5600 const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
5601 if (PredIsKnownFalse(Accum, AccumExtended)) {
5602 LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";);
5603 return std::nullopt;
5604 }
5605
5606 auto AppendPredicate = [&](const SCEV *Expr,
5607 const SCEV *ExtendedExpr) -> void {
5608 if (Expr != ExtendedExpr &&
5609 !isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: Expr, RHS: ExtendedExpr)) {
5610 const SCEVPredicate *Pred = getEqualPredicate(LHS: Expr, RHS: ExtendedExpr);
5611 LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred);
5612 Predicates.push_back(Elt: Pred);
5613 }
5614 };
5615
5616 AppendPredicate(StartVal, StartExtended);
5617 AppendPredicate(Accum, AccumExtended);
5618
5619 // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
5620 // which the casts had been folded away. The caller can rewrite SymbolicPHI
5621 // into NewAR if it will also add the runtime overflow checks specified in
5622 // Predicates.
5623 const SCEV *NewAR = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags: SCEV::FlagNone);
5624
5625 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5626 std::make_pair(x&: NewAR, y&: Predicates);
5627 // Remember the result of the analysis for this SCEV at this locayyytion.
5628 PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
5629 return PredRewrite;
5630}
5631
5632std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5633ScalarEvolution::createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI) {
5634 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5635 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5636 if (!L)
5637 return std::nullopt;
5638
5639 // Check to see if we already analyzed this PHI.
5640 auto I = PredicatedSCEVRewrites.find(Val: {SymbolicPHI, L});
5641 if (I != PredicatedSCEVRewrites.end()) {
5642 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5643 I->second;
5644 // Analysis was done before and failed to create an AddRec:
5645 if (Rewrite.first == SymbolicPHI)
5646 return std::nullopt;
5647 // Analysis was done before and succeeded to create an AddRec under
5648 // a predicate:
5649 assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec");
5650 assert(!(Rewrite.second).empty() && "Expected to find Predicates");
5651 return Rewrite;
5652 }
5653
5654 std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5655 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5656
5657 // Record in the cache that the analysis failed
5658 if (!Rewrite) {
5659 SmallVector<const SCEVPredicate *, 3> Predicates;
5660 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5661 return std::nullopt;
5662 }
5663
5664 return Rewrite;
5665}
5666
5667// FIXME: This utility is currently required because the Rewriter currently
5668// does not rewrite this expression:
5669// {0, +, (sext ix (trunc iy to ix) to iy)}
5670// into {0, +, %step},
5671// even when the following Equal predicate exists:
5672// "%step == (sext ix (trunc iy to ix) to iy)".
5673bool PredicatedScalarEvolution::areAddRecsEqualWithPreds(
5674 const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2,
5675 ArrayRef<const SCEVPredicate *> NoWrapPreds) const {
5676 if (AR1 == AR2)
5677 return true;
5678
5679 SCEVUnionPredicate NoWrapUnionPred(NoWrapPreds, SE);
5680 SCEVUnionPredicate AllPreds = Preds->getUnionWith(N: &NoWrapUnionPred, SE);
5681 auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
5682 if (Expr1 != Expr2 &&
5683 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr1, RHS: Expr2), SE) &&
5684 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr2, RHS: Expr1), SE))
5685 return false;
5686 return true;
5687 };
5688
5689 if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
5690 !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
5691 return false;
5692 return true;
5693}
5694
5695static SCEVFlags getNoWrapFlagsForGEP(GEPOperator *GEP, const SCEV *Accum,
5696 ScalarEvolution &SE) {
5697 SCEVFlags Flags = SCEV::FlagNone;
5698 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
5699 // If the increment has any nowrap flags, then we know the address
5700 // space cannot be wrapped around.
5701 if (NW != GEPNoWrapFlags::none())
5702 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNW);
5703 // If the GEP is nuw or nusw with non-negative offset, we know that
5704 // no unsigned wrap occurs. We cannot set the nsw flag as only the
5705 // offset is treated as signed, while the base is unsigned.
5706 if (NW.hasNoUnsignedWrap() ||
5707 (NW.hasNoUnsignedSignedWrap() && SE.isKnownNonNegative(S: Accum)))
5708 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
5709
5710 return Flags;
5711}
5712
5713/// A helper function for createAddRecFromPHI to handle simple cases.
5714///
5715/// This function tries to find an AddRec expression for the simplest (yet most
5716/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
5717/// If it fails, createAddRecFromPHI will use a more general, but slow,
5718/// technique for finding the AddRec expression.
5719const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
5720 Value *BEValueV,
5721 Value *StartValueV) {
5722 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5723 assert(L && L->getHeader() == PN->getParent());
5724 assert(BEValueV && StartValueV);
5725
5726 const SCEV *Accum = nullptr;
5727 SCEVFlags Flags = SCEV::FlagNone;
5728 if (auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CtxI: PN)) {
5729 if (BO->Opcode != Instruction::Add)
5730 return nullptr;
5731
5732 if (BO->LHS == PN && L->isLoopInvariant(V: BO->RHS))
5733 Accum = getSCEV(V: BO->RHS);
5734 else if (BO->RHS == PN && L->isLoopInvariant(V: BO->LHS))
5735 Accum = getSCEV(V: BO->LHS);
5736
5737 if (!Accum)
5738 return nullptr;
5739
5740 if (BO->IsNUW)
5741 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5742 if (BO->IsNSW)
5743 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5744 } else {
5745 // Handle pointer induction variable: PN = PHI(Start, gep PN,
5746 // LoopInvariant).
5747 auto *GEP = dyn_cast<GEPOperator>(Val: BEValueV);
5748 if (!GEP || GEP->getPointerOperand() != PN || GEP->getNumIndices() != 1)
5749 return nullptr;
5750 Value *Idx = *GEP->idx_begin();
5751 if (!L->isLoopInvariant(V: Idx))
5752 return nullptr;
5753
5754 Type *IntIdxTy = getEffectiveSCEVType(Ty: GEP->getType());
5755 Accum = getMulExpr(LHS: getTruncateOrSignExtend(V: getSCEV(V: Idx), Ty: IntIdxTy),
5756 RHS: getSizeOfExpr(IntTy: IntIdxTy, AllocTy: GEP->getSourceElementType()));
5757 Flags = getNoWrapFlagsForGEP(GEP, Accum, SE&: *this);
5758 }
5759
5760 const SCEV *StartVal = getSCEV(V: StartValueV);
5761 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5762 insertValueToMap(V: PN, S: PHISCEV);
5763
5764 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
5765 inferNoWrapViaConstantRanges(AR);
5766
5767 // We can add Flags to the post-inc expression only if we
5768 // know that it is *undefined behavior* for BEValueV to
5769 // overflow.
5770 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV)) {
5771 assert(isLoopInvariant(Accum, L) &&
5772 "Accum is defined outside L, but is not invariant?");
5773 if (isAddRecNeverPoison(I: BEInst, L))
5774 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5775 }
5776
5777 return PHISCEV;
5778}
5779
5780const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
5781 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5782 if (!L || L->getHeader() != PN->getParent())
5783 return nullptr;
5784
5785 // The loop may have multiple entrances or multiple exits; we can analyze
5786 // this phi as an addrec if it has a unique entry value and a unique
5787 // backedge value.
5788 Value *BEValueV = nullptr, *StartValueV = nullptr;
5789 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5790 Value *V = PN->getIncomingValue(i);
5791 if (L->contains(BB: PN->getIncomingBlock(i))) {
5792 if (!BEValueV) {
5793 BEValueV = V;
5794 } else if (BEValueV != V) {
5795 BEValueV = nullptr;
5796 break;
5797 }
5798 } else if (!StartValueV) {
5799 StartValueV = V;
5800 } else if (StartValueV != V) {
5801 StartValueV = nullptr;
5802 break;
5803 }
5804 }
5805 if (!BEValueV || !StartValueV)
5806 return nullptr;
5807
5808 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
5809 "PHI node already processed?");
5810
5811 // First, try to find AddRec expression without creating a fictituos symbolic
5812 // value for PN.
5813 if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5814 return S;
5815
5816 // Handle PHI node value symbolically.
5817 const SCEV *SymbolicName = getUnknown(V: PN);
5818 insertValueToMap(V: PN, S: SymbolicName);
5819
5820 // Using this symbolic name for the PHI, analyze the value coming around
5821 // the back-edge.
5822 const SCEV *BEValue = getSCEV(V: BEValueV);
5823
5824 // NOTE: If BEValue is loop invariant, we know that the PHI node just
5825 // has a special value for the first iteration of the loop.
5826
5827 // If the value coming around the backedge is an add with the symbolic
5828 // value we just inserted, then we found a simple induction variable!
5829 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: BEValue)) {
5830 // If there is a single occurrence of the symbolic value, replace it
5831 // with a recurrence.
5832 unsigned FoundIndex = Add->getNumOperands();
5833 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5834 if (Add->getOperand(i) == SymbolicName)
5835 if (FoundIndex == e) {
5836 FoundIndex = i;
5837 break;
5838 }
5839
5840 if (FoundIndex != Add->getNumOperands()) {
5841 // Create an add with everything but the specified operand.
5842 SmallVector<SCEVUse, 8> Ops;
5843 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5844 if (i != FoundIndex)
5845 Ops.push_back(Elt: SCEVBackedgeConditionFolder::rewrite(S: Add->getOperand(i),
5846 L, SE&: *this));
5847 const SCEV *Accum = getAddExpr(Ops);
5848
5849 // This is not a valid addrec if the step amount is varying each
5850 // loop iteration, but is not itself an addrec in this loop.
5851 if (isLoopInvariant(S: Accum, L) ||
5852 (isa<SCEVAddRecExpr>(Val: Accum) &&
5853 cast<SCEVAddRecExpr>(Val: Accum)->getLoop() == L)) {
5854 SCEVFlags Flags = SCEV::FlagNone;
5855
5856 if (auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CtxI: PN)) {
5857 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
5858 if (BO->IsNUW)
5859 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5860 if (BO->IsNSW)
5861 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5862 }
5863 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: BEValueV)) {
5864 if (GEP->getOperand(i_nocapture: 0) == PN)
5865 Flags = getNoWrapFlagsForGEP(GEP, Accum, SE&: *this);
5866
5867 // We cannot transfer nuw and nsw flags from subtraction
5868 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
5869 // for instance.
5870 }
5871
5872 const SCEV *StartVal = getSCEV(V: StartValueV);
5873 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5874
5875 // Okay, for the entire analysis of this edge we assumed the PHI
5876 // to be symbolic. We now need to go back and purge all of the
5877 // entries for the scalars that use the symbolic expression.
5878 forgetMemoizedResults(SCEVs: {SymbolicName});
5879 insertValueToMap(V: PN, S: PHISCEV);
5880
5881 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
5882 inferNoWrapViaConstantRanges(AR);
5883
5884 // We can add Flags to the post-inc expression only if we
5885 // know that it is *undefined behavior* for BEValueV to
5886 // overflow.
5887 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV))
5888 if (isLoopInvariant(S: Accum, L) && isAddRecNeverPoison(I: BEInst, L))
5889 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5890
5891 return PHISCEV;
5892 }
5893 }
5894 } else {
5895 // Otherwise, this could be a loop like this:
5896 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
5897 // In this case, j = {1,+,1} and BEValue is j.
5898 // Because the other in-value of i (0) fits the evolution of BEValue
5899 // i really is an addrec evolution.
5900 //
5901 // We can generalize this saying that i is the shifted value of BEValue
5902 // by one iteration:
5903 // PHI(f(0), f({1,+,1})) --> f({0,+,1})
5904
5905 // Do not allow refinement in rewriting of BEValue.
5906 const SCEV *Shifted = SCEVShiftRewriter::rewrite(S: BEValue, L, SE&: *this);
5907 const SCEV *Start = SCEVInitRewriter::rewrite(S: Shifted, L, SE&: *this, IgnoreOtherLoops: false);
5908 if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
5909 isGuaranteedNotToCauseUB(Op: Shifted) && ::impliesPoison(AssumedPoison: Shifted, S: Start)) {
5910 const SCEV *StartVal = getSCEV(V: StartValueV);
5911 if (Start == StartVal) {
5912 // Okay, for the entire analysis of this edge we assumed the PHI
5913 // to be symbolic. We now need to go back and purge all of the
5914 // entries for the scalars that use the symbolic expression.
5915 forgetMemoizedResults(SCEVs: {SymbolicName});
5916 insertValueToMap(V: PN, S: Shifted);
5917 return Shifted;
5918 }
5919 }
5920 }
5921
5922 // Remove the temporary PHI node SCEV that has been inserted while intending
5923 // to create an AddRecExpr for this PHI node. We can not keep this temporary
5924 // as it will prevent later (possibly simpler) SCEV expressions to be added
5925 // to the ValueExprMap.
5926 eraseValueFromMap(V: PN);
5927
5928 return nullptr;
5929}
5930
5931// Try to match a control flow sequence that branches out at BI and merges back
5932// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
5933// match.
5934static bool BrPHIToSelect(DominatorTree &DT, CondBrInst *BI, PHINode *Merge,
5935 Value *&C, Value *&LHS, Value *&RHS) {
5936 C = BI->getCondition();
5937
5938 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(i: 0));
5939 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(i: 1));
5940
5941 Use &LeftUse = Merge->getOperandUse(i: 0);
5942 Use &RightUse = Merge->getOperandUse(i: 1);
5943
5944 if (DT.dominates(BBE: LeftEdge, U: LeftUse) && DT.dominates(BBE: RightEdge, U: RightUse)) {
5945 LHS = LeftUse;
5946 RHS = RightUse;
5947 return true;
5948 }
5949
5950 if (DT.dominates(BBE: LeftEdge, U: RightUse) && DT.dominates(BBE: RightEdge, U: LeftUse)) {
5951 LHS = RightUse;
5952 RHS = LeftUse;
5953 return true;
5954 }
5955
5956 return false;
5957}
5958
5959static bool getOperandsForSelectLikePHI(DominatorTree &DT, PHINode *PN,
5960 Value *&Cond, Value *&LHS,
5961 Value *&RHS) {
5962 auto IsReachable =
5963 [&](BasicBlock *BB) { return DT.isReachableFromEntry(A: BB); };
5964 if (PN->getNumIncomingValues() == 2 && all_of(Range: PN->blocks(), P: IsReachable)) {
5965 // Try to match
5966 //
5967 // br %cond, label %left, label %right
5968 // left:
5969 // br label %merge
5970 // right:
5971 // br label %merge
5972 // merge:
5973 // V = phi [ %x, %left ], [ %y, %right ]
5974 //
5975 // as "select %cond, %x, %y"
5976
5977 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
5978 assert(IDom && "At least the entry block should dominate PN");
5979
5980 auto *BI = dyn_cast<CondBrInst>(Val: IDom->getTerminator());
5981 return BI && BrPHIToSelect(DT, BI, Merge: PN, C&: Cond, LHS, RHS);
5982 }
5983 return false;
5984}
5985
5986const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
5987 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
5988 if (getOperandsForSelectLikePHI(DT, PN, Cond, LHS, RHS) &&
5989 properlyDominates(S: getSCEV(V: LHS), BB: PN->getParent()) &&
5990 properlyDominates(S: getSCEV(V: RHS), BB: PN->getParent()))
5991 return createNodeForSelectOrPHI(V: PN, Cond, TrueVal: LHS, FalseVal: RHS);
5992
5993 return nullptr;
5994}
5995
5996static BinaryOperator *getCommonInstForPHI(PHINode *PN) {
5997 BinaryOperator *CommonInst = nullptr;
5998 // Check if instructions are identical.
5999 for (Value *Incoming : PN->incoming_values()) {
6000 auto *IncomingInst = dyn_cast<BinaryOperator>(Val: Incoming);
6001 if (!IncomingInst)
6002 return nullptr;
6003 if (CommonInst) {
6004 if (!CommonInst->isIdenticalToWhenDefined(I: IncomingInst))
6005 return nullptr; // Not identical, give up
6006 } else {
6007 // Remember binary operator
6008 CommonInst = IncomingInst;
6009 }
6010 }
6011 return CommonInst;
6012}
6013
6014/// Returns SCEV for the first operand of a phi if all phi operands have
6015/// identical opcodes and operands
6016/// eg.
6017/// a: %add = %a + %b
6018/// br %c
6019/// b: %add1 = %a + %b
6020/// br %c
6021/// c: %phi = phi [%add, a], [%add1, b]
6022/// scev(%phi) => scev(%add)
6023const SCEV *
6024ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
6025 BinaryOperator *CommonInst = getCommonInstForPHI(PN);
6026 if (!CommonInst)
6027 return nullptr;
6028
6029 // Check if SCEV exprs for instructions are identical.
6030 const SCEV *CommonSCEV = getSCEV(V: CommonInst);
6031 bool SCEVExprsIdentical =
6032 all_of(Range: drop_begin(RangeOrContainer: PN->incoming_values()),
6033 P: [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
6034 return SCEVExprsIdentical ? CommonSCEV : nullptr;
6035}
6036
6037const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
6038 if (const SCEV *S = createAddRecFromPHI(PN))
6039 return S;
6040
6041 // We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
6042 // phi node for X.
6043 if (Value *V = simplifyInstruction(
6044 I: PN, Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
6045 /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
6046 return getSCEV(V);
6047
6048 if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
6049 return S;
6050
6051 if (const SCEV *S = createNodeFromSelectLikePHI(PN))
6052 return S;
6053
6054 // If it's not a loop phi, we can't handle it yet.
6055 return getUnknown(V: PN);
6056}
6057
6058bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
6059 SCEVTypes RootKind) {
6060 struct FindClosure {
6061 const SCEV *OperandToFind;
6062 const SCEVTypes RootKind; // Must be a sequential min/max expression.
6063 const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind.
6064
6065 bool Found = false;
6066
6067 bool canRecurseInto(SCEVTypes Kind) const {
6068 // We can only recurse into the SCEV expression of the same effective type
6069 // as the type of our root SCEV expression, and into zero-extensions.
6070 return RootKind == Kind || NonSequentialRootKind == Kind ||
6071 scZeroExtend == Kind;
6072 };
6073
6074 FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind)
6075 : OperandToFind(OperandToFind), RootKind(RootKind),
6076 NonSequentialRootKind(
6077 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
6078 Ty: RootKind)) {}
6079
6080 bool follow(const SCEV *S) {
6081 Found = S == OperandToFind;
6082
6083 return !isDone() && canRecurseInto(Kind: S->getSCEVType());
6084 }
6085
6086 bool isDone() const { return Found; }
6087 };
6088
6089 FindClosure FC(OperandToFind, RootKind);
6090 visitAll(Root, Visitor&: FC);
6091 return FC.Found;
6092}
6093
6094std::optional<const SCEV *>
6095ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(Type *Ty,
6096 ICmpInst *Cond,
6097 Value *TrueVal,
6098 Value *FalseVal) {
6099 // Try to match some simple smax or umax patterns.
6100 auto *ICI = Cond;
6101
6102 Value *LHS = ICI->getOperand(i_nocapture: 0);
6103 Value *RHS = ICI->getOperand(i_nocapture: 1);
6104
6105 switch (ICI->getPredicate()) {
6106 case ICmpInst::ICMP_SLT:
6107 case ICmpInst::ICMP_SLE:
6108 case ICmpInst::ICMP_ULT:
6109 case ICmpInst::ICMP_ULE:
6110 std::swap(a&: LHS, b&: RHS);
6111 [[fallthrough]];
6112 case ICmpInst::ICMP_SGT:
6113 case ICmpInst::ICMP_SGE:
6114 case ICmpInst::ICMP_UGT:
6115 case ICmpInst::ICMP_UGE:
6116 // a > b ? a+x : b+x -> max(a, b)+x
6117 // a > b ? b+x : a+x -> min(a, b)+x
6118 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty)) {
6119 bool Signed = ICI->isSigned();
6120 const SCEV *LA = getSCEV(V: TrueVal);
6121 const SCEV *RA = getSCEV(V: FalseVal);
6122 const SCEV *LS = getSCEV(V: LHS);
6123 const SCEV *RS = getSCEV(V: RHS);
6124 if (LA->getType()->isPointerTy()) {
6125 // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
6126 // Need to make sure we can't produce weird expressions involving
6127 // negated pointers.
6128 if (LA == LS && RA == RS)
6129 return Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS);
6130 if (LA == RS && RA == LS)
6131 return Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS);
6132 }
6133 auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
6134 if (Op->getType()->isPointerTy()) {
6135 Op = getPtrToAddrExpr(Op);
6136 if (isa<SCEVCouldNotCompute>(Val: Op))
6137 return Op;
6138 }
6139 if (Signed)
6140 Op = getNoopOrSignExtend(V: Op, Ty);
6141 else
6142 Op = getNoopOrZeroExtend(V: Op, Ty);
6143 return Op;
6144 };
6145 LS = CoerceOperand(LS);
6146 RS = CoerceOperand(RS);
6147 if (isa<SCEVCouldNotCompute>(Val: LS) || isa<SCEVCouldNotCompute>(Val: RS))
6148 break;
6149 const SCEV *LDiff = getMinusSCEV(LHS: LA, RHS: LS);
6150 const SCEV *RDiff = getMinusSCEV(LHS: RA, RHS: RS);
6151 if (LDiff == RDiff)
6152 return getAddExpr(LHS: Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS),
6153 RHS: LDiff);
6154 LDiff = getMinusSCEV(LHS: LA, RHS: RS);
6155 RDiff = getMinusSCEV(LHS: RA, RHS: LS);
6156 if (LDiff == RDiff)
6157 return getAddExpr(LHS: Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS),
6158 RHS: LDiff);
6159 }
6160 break;
6161 case ICmpInst::ICMP_NE:
6162 // x != 0 ? x+y : C+y -> x == 0 ? C+y : x+y
6163 std::swap(a&: TrueVal, b&: FalseVal);
6164 [[fallthrough]];
6165 case ICmpInst::ICMP_EQ:
6166 // x == 0 ? C+y : x+y -> umax(x, C)+y iff C u<= 1
6167 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty) &&
6168 isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()) {
6169 const SCEV *X = getNoopOrZeroExtend(V: getSCEV(V: LHS), Ty);
6170 const SCEV *TrueValExpr = getSCEV(V: TrueVal); // C+y
6171 const SCEV *FalseValExpr = getSCEV(V: FalseVal); // x+y
6172 const SCEV *Y = getMinusSCEV(LHS: FalseValExpr, RHS: X); // y = (x+y)-x
6173 const SCEV *C = getMinusSCEV(LHS: TrueValExpr, RHS: Y); // C = (C+y)-y
6174 if (isa<SCEVConstant>(Val: C) && cast<SCEVConstant>(Val: C)->getAPInt().ule(RHS: 1))
6175 return getAddExpr(LHS: getUMaxExpr(LHS: X, RHS: C), RHS: Y);
6176 }
6177 // x == 0 ? 0 : umin (..., x, ...) -> umin_seq(x, umin (...))
6178 // x == 0 ? 0 : umin_seq(..., x, ...) -> umin_seq(x, umin_seq(...))
6179 // x == 0 ? 0 : umin (..., umin_seq(..., x, ...), ...)
6180 // -> umin_seq(x, umin (..., umin_seq(...), ...))
6181 if (isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero() &&
6182 isa<ConstantInt>(Val: TrueVal) && cast<ConstantInt>(Val: TrueVal)->isZero()) {
6183 const SCEV *X = getSCEV(V: LHS);
6184 while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: X))
6185 X = ZExt->getOperand();
6186 if (getTypeSizeInBits(Ty: X->getType()) <= getTypeSizeInBits(Ty)) {
6187 const SCEV *FalseValExpr = getSCEV(V: FalseVal);
6188 if (SCEVMinMaxExprContains(Root: FalseValExpr, OperandToFind: X, RootKind: scSequentialUMinExpr))
6189 return getUMinExpr(LHS: getNoopOrZeroExtend(V: X, Ty), RHS: FalseValExpr,
6190 /*Sequential=*/true);
6191 }
6192 }
6193 break;
6194 default:
6195 break;
6196 }
6197
6198 return std::nullopt;
6199}
6200
6201static std::optional<const SCEV *>
6202createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr,
6203 const SCEV *TrueExpr, const SCEV *FalseExpr) {
6204 assert(CondExpr->getType()->isIntegerTy(1) &&
6205 TrueExpr->getType() == FalseExpr->getType() &&
6206 TrueExpr->getType()->isIntegerTy(1) &&
6207 "Unexpected operands of a select.");
6208
6209 // i1 cond ? i1 x : i1 C --> C + (i1 cond ? (i1 x - i1 C) : i1 0)
6210 // --> C + (umin_seq cond, x - C)
6211 //
6212 // i1 cond ? i1 C : i1 x --> C + (i1 cond ? i1 0 : (i1 x - i1 C))
6213 // --> C + (i1 ~cond ? (i1 x - i1 C) : i1 0)
6214 // --> C + (umin_seq ~cond, x - C)
6215
6216 // FIXME: while we can't legally model the case where both of the hands
6217 // are fully variable, we only require that the *difference* is constant.
6218 if (!isa<SCEVConstant>(Val: TrueExpr) && !isa<SCEVConstant>(Val: FalseExpr))
6219 return std::nullopt;
6220
6221 const SCEV *X, *C;
6222 if (isa<SCEVConstant>(Val: TrueExpr)) {
6223 CondExpr = SE->getNotSCEV(V: CondExpr);
6224 X = FalseExpr;
6225 C = TrueExpr;
6226 } else {
6227 X = TrueExpr;
6228 C = FalseExpr;
6229 }
6230 return SE->getAddExpr(LHS: C, RHS: SE->getUMinExpr(LHS: CondExpr, RHS: SE->getMinusSCEV(LHS: X, RHS: C),
6231 /*Sequential=*/true));
6232}
6233
6234static std::optional<const SCEV *>
6235createNodeForSelectViaUMinSeq(ScalarEvolution *SE, Value *Cond, Value *TrueVal,
6236 Value *FalseVal) {
6237 if (!isa<ConstantInt>(Val: TrueVal) && !isa<ConstantInt>(Val: FalseVal))
6238 return std::nullopt;
6239
6240 const auto *SECond = SE->getSCEV(V: Cond);
6241 const auto *SETrue = SE->getSCEV(V: TrueVal);
6242 const auto *SEFalse = SE->getSCEV(V: FalseVal);
6243 return createNodeForSelectViaUMinSeq(SE, CondExpr: SECond, TrueExpr: SETrue, FalseExpr: SEFalse);
6244}
6245
6246const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
6247 Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) {
6248 assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?");
6249 assert(TrueVal->getType() == FalseVal->getType() &&
6250 V->getType() == TrueVal->getType() &&
6251 "Types of select hands and of the result must match.");
6252
6253 // For now, only deal with i1-typed `select`s.
6254 if (!V->getType()->isIntegerTy(BitWidth: 1))
6255 return getUnknown(V);
6256
6257 if (std::optional<const SCEV *> S =
6258 createNodeForSelectViaUMinSeq(SE: this, Cond, TrueVal, FalseVal))
6259 return *S;
6260
6261 return getUnknown(V);
6262}
6263
6264const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond,
6265 Value *TrueVal,
6266 Value *FalseVal) {
6267 // Handle "constant" branch or select. This can occur for instance when a
6268 // loop pass transforms an inner loop and moves on to process the outer loop.
6269 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond))
6270 return getSCEV(V: CI->isOne() ? TrueVal : FalseVal);
6271
6272 if (auto *I = dyn_cast<Instruction>(Val: V)) {
6273 if (auto *ICI = dyn_cast<ICmpInst>(Val: Cond)) {
6274 if (std::optional<const SCEV *> S =
6275 createNodeForSelectOrPHIInstWithICmpInstCond(Ty: I->getType(), Cond: ICI,
6276 TrueVal, FalseVal))
6277 return *S;
6278 }
6279 }
6280
6281 return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal);
6282}
6283
6284/// Expand GEP instructions into add and multiply operations. This allows them
6285/// to be analyzed by regular SCEV code.
6286const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
6287 assert(GEP->getSourceElementType()->isSized() &&
6288 "GEP source element type must be sized");
6289
6290 SmallVector<SCEVUse, 4> IndexExprs;
6291 for (Value *Index : GEP->indices())
6292 IndexExprs.push_back(Elt: getSCEV(V: Index));
6293 return getGEPExpr(GEP, IndexExprs);
6294}
6295
6296APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S,
6297 const Instruction *CtxI) {
6298 uint64_t BitWidth = getTypeSizeInBits(Ty: S->getType());
6299 auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
6300 return TrailingZeros >= BitWidth
6301 ? APInt::getZero(numBits: BitWidth)
6302 : APInt::getOneBitSet(numBits: BitWidth, BitNo: TrailingZeros);
6303 };
6304 auto GetGCDMultiple = [this, CtxI](const SCEVNAryExpr *N) {
6305 // The result is GCD of all operands results.
6306 APInt Res = getConstantMultiple(S: N->getOperand(i: 0), CtxI);
6307 for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
6308 Res = APIntOps::GreatestCommonDivisor(
6309 A: Res, B: getConstantMultiple(S: N->getOperand(i: I), CtxI));
6310 return Res;
6311 };
6312
6313 switch (S->getSCEVType()) {
6314 case scConstant:
6315 return cast<SCEVConstant>(Val: S)->getAPInt();
6316 case scPtrToAddr:
6317 return getConstantMultiple(S: cast<SCEVCastExpr>(Val: S)->getOperand());
6318 case scUDivExpr:
6319 case scVScale:
6320 return APInt(BitWidth, 1);
6321 case scTruncate: {
6322 // Only multiples that are a power of 2 will hold after truncation.
6323 const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(Val: S);
6324 uint32_t TZ = getMinTrailingZeros(S: T->getOperand(), CtxI);
6325 return GetShiftedByZeros(TZ);
6326 }
6327 case scZeroExtend: {
6328 const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(Val: S);
6329 return getConstantMultiple(S: Z->getOperand(), CtxI).zext(width: BitWidth);
6330 }
6331 case scSignExtend: {
6332 // Only multiples that are a power of 2 will hold after sext.
6333 const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(Val: S);
6334 uint32_t TZ = getMinTrailingZeros(S: E->getOperand(), CtxI);
6335 return GetShiftedByZeros(TZ);
6336 }
6337 case scMulExpr: {
6338 const SCEVMulExpr *M = cast<SCEVMulExpr>(Val: S);
6339 if (M->hasNoUnsignedWrap()) {
6340 // The result is the product of all operand results.
6341 APInt Res = getConstantMultiple(S: M->getOperand(i: 0), CtxI);
6342 for (const SCEV *Operand : M->operands().drop_front())
6343 Res = Res * getConstantMultiple(S: Operand, CtxI);
6344 return Res;
6345 }
6346
6347 // If there are no wrap guarentees, find the trailing zeros, which is the
6348 // sum of trailing zeros for all its operands.
6349 uint32_t TZ = 0;
6350 for (const SCEV *Operand : M->operands())
6351 TZ += getMinTrailingZeros(S: Operand, CtxI);
6352 return GetShiftedByZeros(TZ);
6353 }
6354 case scAddExpr:
6355 case scAddRecExpr: {
6356 const SCEVNAryExpr *N = cast<SCEVNAryExpr>(Val: S);
6357 if (N->hasNoUnsignedWrap())
6358 return GetGCDMultiple(N);
6359 // Find the trailing bits, which is the minimum of its operands.
6360 uint32_t TZ = getMinTrailingZeros(S: N->getOperand(i: 0), CtxI);
6361 for (const SCEV *Operand : N->operands().drop_front())
6362 TZ = std::min(a: TZ, b: getMinTrailingZeros(S: Operand, CtxI));
6363 return GetShiftedByZeros(TZ);
6364 }
6365 case scUMaxExpr:
6366 case scSMaxExpr:
6367 case scUMinExpr:
6368 case scSMinExpr:
6369 case scSequentialUMinExpr:
6370 return GetGCDMultiple(cast<SCEVNAryExpr>(Val: S));
6371 case scUnknown: {
6372 // Ask ValueTracking for known bits. SCEVUnknown only become available at
6373 // the point their underlying IR instruction has been defined. If CtxI was
6374 // not provided, use:
6375 // * the first instruction in the entry block if it is an argument
6376 // * the instruction itself otherwise.
6377 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6378 if (!CtxI) {
6379 if (isa<Argument>(Val: U->getValue()))
6380 CtxI = &*F.getEntryBlock().begin();
6381 else if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
6382 CtxI = I;
6383 }
6384 unsigned Known =
6385 computeKnownBits(V: U->getValue(),
6386 Q: SimplifyQuery(getDataLayout(), &DT, &AC, CtxI)
6387 .allowEphemerals(AllowEphemerals: true))
6388 .countMinTrailingZeros();
6389 return GetShiftedByZeros(Known);
6390 }
6391 case scCouldNotCompute:
6392 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6393 }
6394 llvm_unreachable("Unknown SCEV kind!");
6395}
6396
6397APInt ScalarEvolution::getConstantMultiple(const SCEV *S,
6398 const Instruction *CtxI) {
6399 // Skip looking up and updating the cache if there is a context instruction,
6400 // as the result will only be valid in the specified context.
6401 if (CtxI)
6402 return getConstantMultipleImpl(S, CtxI);
6403
6404 auto I = ConstantMultipleCache.find(Val: S);
6405 if (I != ConstantMultipleCache.end())
6406 return I->second;
6407
6408 APInt Result = getConstantMultipleImpl(S, CtxI);
6409 auto InsertPair = ConstantMultipleCache.insert(KV: {S, Result});
6410 assert(InsertPair.second && "Should insert a new key");
6411 return InsertPair.first->second;
6412}
6413
6414APInt ScalarEvolution::getNonZeroConstantMultiple(const SCEV *S) {
6415 APInt Multiple = getConstantMultiple(S);
6416 return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
6417}
6418
6419uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S,
6420 const Instruction *CtxI) {
6421 return std::min(a: getConstantMultiple(S, CtxI).countTrailingZeros(),
6422 b: (unsigned)getTypeSizeInBits(Ty: S->getType()));
6423}
6424
6425/// Helper method to assign a range to V from metadata present in the IR.
6426static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
6427 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
6428 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_range))
6429 return getConstantRangeFromMetadata(RangeMD: *MD);
6430 if (const auto *CB = dyn_cast<CallBase>(Val: V))
6431 if (std::optional<ConstantRange> Range = CB->getRange())
6432 return Range;
6433 }
6434 if (auto *A = dyn_cast<Argument>(Val: V))
6435 if (std::optional<ConstantRange> Range = A->getRange())
6436 return Range;
6437
6438 return std::nullopt;
6439}
6440
6441void ScalarEvolution::setNoWrapFlags(SCEVAddRecExpr *AddRec, SCEVFlags Flags) {
6442 SCEVFlags NWFlags = Flags & SCEV::FlagsNoWrapMask;
6443 if (AddRec->getNoWrapFlags(Mask: NWFlags) != NWFlags) {
6444 AddRec->setNoWrapFlags(NWFlags);
6445 UnsignedRanges.erase(Val: AddRec);
6446 SignedRanges.erase(Val: AddRec);
6447 ConstantMultipleCache.erase(Val: AddRec);
6448 }
6449}
6450
6451ConstantRange ScalarEvolution::
6452getRangeForUnknownRecurrence(const SCEVUnknown *U) {
6453 const DataLayout &DL = getDataLayout();
6454
6455 unsigned BitWidth = getTypeSizeInBits(Ty: U->getType());
6456 const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
6457
6458 // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
6459 // use information about the trip count to improve our available range. Note
6460 // that the trip count independent cases are already handled by known bits.
6461 // WARNING: The definition of recurrence used here is subtly different than
6462 // the one used by AddRec (and thus most of this file). Step is allowed to
6463 // be arbitrarily loop varying here, where AddRec allows only loop invariant
6464 // and other addrecs in the same loop (for non-affine addrecs). The code
6465 // below intentionally handles the case where step is not loop invariant.
6466 auto *P = dyn_cast<PHINode>(Val: U->getValue());
6467 if (!P)
6468 return FullSet;
6469
6470 // Make sure that no Phi input comes from an unreachable block. Otherwise,
6471 // even the values that are not available in these blocks may come from them,
6472 // and this leads to false-positive recurrence test.
6473 for (auto *Pred : predecessors(BB: P->getParent()))
6474 if (!DT.isReachableFromEntry(A: Pred))
6475 return FullSet;
6476
6477 BinaryOperator *BO;
6478 Value *Start, *Step;
6479 if (!matchSimpleRecurrence(P, BO, Start, Step))
6480 return FullSet;
6481
6482 // If we found a recurrence in reachable code, we must be in a loop. Note
6483 // that BO might be in some subloop of L, and that's completely okay.
6484 auto *L = LI.getLoopFor(BB: P->getParent());
6485 assert(L && L->getHeader() == P->getParent());
6486 if (!L->contains(BB: BO->getParent()))
6487 // NOTE: This bailout should be an assert instead. However, asserting
6488 // the condition here exposes a case where LoopFusion is querying SCEV
6489 // with malformed loop information during the midst of the transform.
6490 // There doesn't appear to be an obvious fix, so for the moment bailout
6491 // until the caller issue can be fixed. PR49566 tracks the bug.
6492 return FullSet;
6493
6494 // TODO: Extend to other opcodes such as mul, and div
6495 switch (BO->getOpcode()) {
6496 default:
6497 return FullSet;
6498 case Instruction::AShr:
6499 case Instruction::LShr:
6500 case Instruction::Shl:
6501 break;
6502 };
6503
6504 if (BO->getOperand(i_nocapture: 0) != P)
6505 // TODO: Handle the power function forms some day.
6506 return FullSet;
6507
6508 unsigned TC = getSmallConstantMaxTripCount(L);
6509 if (!TC || TC >= BitWidth)
6510 return FullSet;
6511
6512 auto KnownStart = computeKnownBits(V: Start, DL, AC: &AC, CtxI: nullptr, DT: &DT);
6513 auto KnownStep = computeKnownBits(V: Step, DL, AC: &AC, CtxI: nullptr, DT: &DT);
6514 assert(KnownStart.getBitWidth() == BitWidth &&
6515 KnownStep.getBitWidth() == BitWidth);
6516
6517 // Compute total shift amount, being careful of overflow and bitwidths.
6518 auto MaxShiftAmt = KnownStep.getMaxValue();
6519 APInt TCAP(BitWidth, TC-1);
6520 bool Overflow = false;
6521 auto TotalShift = MaxShiftAmt.umul_ov(RHS: TCAP, Overflow);
6522 if (Overflow)
6523 return FullSet;
6524
6525 switch (BO->getOpcode()) {
6526 default:
6527 llvm_unreachable("filtered out above");
6528 case Instruction::AShr: {
6529 // For each ashr, three cases:
6530 // shift = 0 => unchanged value
6531 // saturation => 0 or -1
6532 // other => a value closer to zero (of the same sign)
6533 // Thus, the end value is closer to zero than the start.
6534 auto KnownEnd = KnownBits::ashr(LHS: KnownStart,
6535 RHS: KnownBits::makeConstant(C: TotalShift));
6536 if (KnownStart.isNonNegative())
6537 // Analogous to lshr (simply not yet canonicalized)
6538 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6539 Upper: KnownStart.getMaxValue() + 1);
6540 if (KnownStart.isNegative())
6541 // End >=u Start && End <=s Start
6542 return ConstantRange::getNonEmpty(Lower: KnownStart.getMinValue(),
6543 Upper: KnownEnd.getMaxValue() + 1);
6544 break;
6545 }
6546 case Instruction::LShr: {
6547 // For each lshr, three cases:
6548 // shift = 0 => unchanged value
6549 // saturation => 0
6550 // other => a smaller positive number
6551 // Thus, the low end of the unsigned range is the last value produced.
6552 auto KnownEnd = KnownBits::lshr(LHS: KnownStart,
6553 RHS: KnownBits::makeConstant(C: TotalShift));
6554 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6555 Upper: KnownStart.getMaxValue() + 1);
6556 }
6557 case Instruction::Shl: {
6558 // Iff no bits are shifted out, value increases on every shift.
6559 auto KnownEnd = KnownBits::shl(LHS: KnownStart,
6560 RHS: KnownBits::makeConstant(C: TotalShift));
6561 if (TotalShift.ult(RHS: KnownStart.countMinLeadingZeros()))
6562 return ConstantRange(KnownStart.getMinValue(),
6563 KnownEnd.getMaxValue() + 1);
6564 break;
6565 }
6566 };
6567 return FullSet;
6568}
6569
6570// The goal of this function is to check if recursively visiting the operands
6571// of this PHI might lead to an infinite loop. If we do see such a loop,
6572// there's no good way to break it, so we avoid analyzing such cases.
6573//
6574// getRangeRef previously used a visited set to avoid infinite loops, but this
6575// caused other issues: the result was dependent on the order of getRangeRef
6576// calls, and the interaction with createSCEVIter could cause a stack overflow
6577// in some cases (see issue #148253).
6578//
6579// FIXME: The way this is implemented is overly conservative; this checks
6580// for a few obviously safe patterns, but anything that doesn't lead to
6581// recursion is fine.
6582static bool RangeRefPHIAllowedOperands(DominatorTree &DT, PHINode *PHI) {
6583 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
6584 if (getOperandsForSelectLikePHI(DT, PN: PHI, Cond, LHS, RHS))
6585 return true;
6586
6587 if (all_of(Range: PHI->operands(),
6588 P: [&](Value *Operand) { return DT.dominates(Def: Operand, User: PHI); }))
6589 return true;
6590
6591 return false;
6592}
6593
6594const ConstantRange &
6595ScalarEvolution::getRangeRefIter(const SCEV *S,
6596 ScalarEvolution::RangeSignHint SignHint) {
6597 DenseMap<const SCEV *, ConstantRange> &Cache =
6598 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6599 : SignedRanges;
6600 SmallVector<SCEVUse> WorkList;
6601 SmallPtrSet<const SCEV *, 8> Seen;
6602
6603 // Add Expr to the worklist, if Expr is either an N-ary expression or a
6604 // SCEVUnknown PHI node.
6605 auto AddToWorklist = [&WorkList, &Seen, &Cache](const SCEV *Expr) {
6606 if (!Seen.insert(Ptr: Expr).second)
6607 return;
6608 if (Cache.contains(Val: Expr))
6609 return;
6610 switch (Expr->getSCEVType()) {
6611 case scUnknown:
6612 if (!isa<PHINode>(Val: cast<SCEVUnknown>(Val: Expr)->getValue()))
6613 break;
6614 [[fallthrough]];
6615 case scConstant:
6616 case scVScale:
6617 case scTruncate:
6618 case scZeroExtend:
6619 case scSignExtend:
6620 case scPtrToAddr:
6621 case scAddExpr:
6622 case scMulExpr:
6623 case scUDivExpr:
6624 case scAddRecExpr:
6625 case scUMaxExpr:
6626 case scSMaxExpr:
6627 case scUMinExpr:
6628 case scSMinExpr:
6629 case scSequentialUMinExpr:
6630 WorkList.push_back(Elt: Expr);
6631 break;
6632 case scCouldNotCompute:
6633 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6634 }
6635 };
6636 AddToWorklist(S);
6637
6638 // Build worklist by queuing operands of N-ary expressions and phi nodes.
6639 for (unsigned I = 0; I != WorkList.size(); ++I) {
6640 const SCEV *P = WorkList[I];
6641 auto *UnknownS = dyn_cast<SCEVUnknown>(Val: P);
6642 // If it is not a `SCEVUnknown`, just recurse into operands.
6643 if (!UnknownS) {
6644 for (const SCEV *Op : P->operands())
6645 AddToWorklist(Op);
6646 continue;
6647 }
6648 // `SCEVUnknown`'s require special treatment.
6649 if (PHINode *P = dyn_cast<PHINode>(Val: UnknownS->getValue())) {
6650 if (!RangeRefPHIAllowedOperands(DT, PHI: P))
6651 continue;
6652 for (auto &Op : reverse(C: P->operands()))
6653 AddToWorklist(getSCEV(V: Op));
6654 }
6655 }
6656
6657 if (!WorkList.empty()) {
6658 // Use getRangeRef to compute ranges for items in the worklist in reverse
6659 // order. This will force ranges for earlier operands to be computed before
6660 // their users in most cases.
6661 for (const SCEV *P : reverse(C: drop_begin(RangeOrContainer&: WorkList))) {
6662 getRangeRef(S: P, Hint: SignHint);
6663 }
6664 }
6665
6666 return getRangeRef(S, Hint: SignHint, Depth: 0);
6667}
6668
6669const APInt *ScalarEvolution::getConstantAPIntOrNull(const SCEV *S) {
6670 if (const auto *C = dyn_cast<SCEVConstant>(Val: S))
6671 return &C->getAPInt();
6672 return nullptr;
6673}
6674
6675/// Determine the range for a particular SCEV. If SignHint is
6676/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
6677/// with a "cleaner" unsigned (resp. signed) representation.
6678const ConstantRange &ScalarEvolution::getRangeRef(
6679 const SCEV *S, ScalarEvolution::RangeSignHint SignHint, unsigned Depth) {
6680 DenseMap<const SCEV *, ConstantRange> &Cache =
6681 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6682 : SignedRanges;
6683 ConstantRange::PreferredRangeType RangeType =
6684 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? ConstantRange::Unsigned
6685 : ConstantRange::Signed;
6686
6687 // See if we've computed this range already.
6688 auto I = Cache.find(Val: S);
6689 if (I != Cache.end())
6690 return I->second;
6691
6692 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: S))
6693 return setRange(S: C, Hint: SignHint, CR: ConstantRange(C->getAPInt()));
6694
6695 // Switch to iteratively computing the range for S, if it is part of a deeply
6696 // nested expression.
6697 if (Depth > RangeIterThreshold)
6698 return getRangeRefIter(S, SignHint);
6699
6700 unsigned BitWidth = getTypeSizeInBits(Ty: S->getType());
6701 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
6702 using OBO = OverflowingBinaryOperator;
6703
6704 // If the value has known zeros, the maximum value will have those known zeros
6705 // as well.
6706 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
6707 APInt Multiple = getNonZeroConstantMultiple(S);
6708 APInt Remainder = APInt::getMaxValue(numBits: BitWidth).urem(RHS: Multiple);
6709 if (!Remainder.isZero())
6710 ConservativeResult =
6711 ConstantRange(APInt::getMinValue(numBits: BitWidth),
6712 APInt::getMaxValue(numBits: BitWidth) - Remainder + 1);
6713 }
6714 else {
6715 uint32_t TZ = getMinTrailingZeros(S);
6716 if (TZ != 0) {
6717 ConservativeResult = ConstantRange(
6718 APInt::getSignedMinValue(numBits: BitWidth),
6719 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: TZ).shl(shiftAmt: TZ) + 1);
6720 }
6721 }
6722
6723 switch (S->getSCEVType()) {
6724 case scConstant:
6725 llvm_unreachable("Already handled above.");
6726 case scVScale:
6727 return setRange(S, Hint: SignHint, CR: getVScaleRange(F: &F, BitWidth));
6728 case scTruncate: {
6729 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: S);
6730 ConstantRange X = getRangeRef(S: Trunc->getOperand(), SignHint, Depth: Depth + 1);
6731 return setRange(
6732 S: Trunc, Hint: SignHint,
6733 CR: ConservativeResult.intersectWith(CR: X.truncate(BitWidth), Type: RangeType));
6734 }
6735 case scZeroExtend: {
6736 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: S);
6737 ConstantRange X = getRangeRef(S: ZExt->getOperand(), SignHint, Depth: Depth + 1);
6738 return setRange(
6739 S: ZExt, Hint: SignHint,
6740 CR: ConservativeResult.intersectWith(CR: X.zeroExtend(BitWidth), Type: RangeType));
6741 }
6742 case scSignExtend: {
6743 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: S);
6744 ConstantRange X = getRangeRef(S: SExt->getOperand(), SignHint, Depth: Depth + 1);
6745 return setRange(
6746 S: SExt, Hint: SignHint,
6747 CR: ConservativeResult.intersectWith(CR: X.signExtend(BitWidth), Type: RangeType));
6748 }
6749 case scPtrToAddr: {
6750 const SCEVCastExpr *Cast = cast<SCEVCastExpr>(Val: S);
6751 ConstantRange X = getRangeRef(S: Cast->getOperand(), SignHint, Depth: Depth + 1);
6752 return setRange(S: Cast, Hint: SignHint, CR: X);
6753 }
6754 case scAddExpr: {
6755 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: S);
6756 // Check if this is a URem pattern: A - (A / B) * B, which is always < B.
6757 const SCEV *URemLHS = nullptr, *URemRHS = nullptr;
6758 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED &&
6759 match(S, P: m_scev_URem(LHS: m_SCEV(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE&: *this))) {
6760 ConstantRange LHSRange = getRangeRef(S: URemLHS, SignHint, Depth: Depth + 1);
6761 ConstantRange RHSRange = getRangeRef(S: URemRHS, SignHint, Depth: Depth + 1);
6762 ConservativeResult =
6763 ConservativeResult.intersectWith(CR: LHSRange.urem(Other: RHSRange), Type: RangeType);
6764 }
6765 ConstantRange X = getRangeRef(S: Add->getOperand(i: 0), SignHint, Depth: Depth + 1);
6766 unsigned WrapType = OBO::AnyWrap;
6767 if (Add->hasNoSignedWrap())
6768 WrapType |= OBO::NoSignedWrap;
6769 if (Add->hasNoUnsignedWrap())
6770 WrapType |= OBO::NoUnsignedWrap;
6771 for (const SCEV *Op : drop_begin(RangeOrContainer: Add->operands()))
6772 X = X.addWithNoWrap(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1), NoWrapKind: WrapType,
6773 RangeType);
6774 return setRange(S: Add, Hint: SignHint,
6775 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6776 }
6777 case scMulExpr: {
6778 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val: S);
6779 ConstantRange X = getRangeRef(S: Mul->getOperand(i: 0), SignHint, Depth: Depth + 1);
6780 for (const SCEV *Op : drop_begin(RangeOrContainer: Mul->operands()))
6781 X = X.multiply(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1));
6782 return setRange(S: Mul, Hint: SignHint,
6783 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6784 }
6785 case scUDivExpr: {
6786 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: S);
6787 ConstantRange X = getRangeRef(S: UDiv->getLHS(), SignHint, Depth: Depth + 1);
6788 ConstantRange Y = getRangeRef(S: UDiv->getRHS(), SignHint, Depth: Depth + 1);
6789 return setRange(S: UDiv, Hint: SignHint,
6790 CR: ConservativeResult.intersectWith(CR: X.udiv(Other: Y), Type: RangeType));
6791 }
6792 case scAddRecExpr: {
6793 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: S);
6794 // If there's no unsigned wrap, the value will never be less than its
6795 // initial value.
6796 if (AddRec->hasNoUnsignedWrap()) {
6797 APInt UnsignedMinValue = getUnsignedRangeMin(S: AddRec->getStart());
6798 if (!UnsignedMinValue.isZero())
6799 ConservativeResult = ConservativeResult.intersectWith(
6800 CR: ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), Type: RangeType);
6801 }
6802
6803 // If there's no signed wrap, and all the operands except initial value have
6804 // the same sign or zero, the value won't ever be:
6805 // 1: smaller than initial value if operands are non negative,
6806 // 2: bigger than initial value if operands are non positive.
6807 // For both cases, value can not cross signed min/max boundary.
6808 if (AddRec->hasNoSignedWrap()) {
6809 bool AllNonNeg = true;
6810 bool AllNonPos = true;
6811 for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
6812 if (!isKnownNonNegative(S: AddRec->getOperand(i)))
6813 AllNonNeg = false;
6814 if (!isKnownNonPositive(S: AddRec->getOperand(i)))
6815 AllNonPos = false;
6816 }
6817 if (AllNonNeg)
6818 ConservativeResult = ConservativeResult.intersectWith(
6819 CR: ConstantRange::getNonEmpty(Lower: getSignedRangeMin(S: AddRec->getStart()),
6820 Upper: APInt::getSignedMinValue(numBits: BitWidth)),
6821 Type: RangeType);
6822 else if (AllNonPos)
6823 ConservativeResult = ConservativeResult.intersectWith(
6824 CR: ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
6825 Upper: getSignedRangeMax(S: AddRec->getStart()) +
6826 1),
6827 Type: RangeType);
6828 }
6829
6830 // TODO: non-affine addrec
6831 if (AddRec->isAffine()) {
6832 const SCEV *MaxBEScev =
6833 getConstantMaxBackedgeTakenCount(L: AddRec->getLoop());
6834 if (!isa<SCEVCouldNotCompute>(Val: MaxBEScev)) {
6835 APInt MaxBECount = cast<SCEVConstant>(Val: MaxBEScev)->getAPInt();
6836
6837 // Adjust MaxBECount to the same bitwidth as AddRec. We can truncate if
6838 // MaxBECount's active bits are all <= AddRec's bit width.
6839 if (MaxBECount.getBitWidth() > BitWidth &&
6840 MaxBECount.getActiveBits() <= BitWidth)
6841 MaxBECount = MaxBECount.trunc(width: BitWidth);
6842 else if (MaxBECount.getBitWidth() < BitWidth)
6843 MaxBECount = MaxBECount.zext(width: BitWidth);
6844
6845 if (MaxBECount.getBitWidth() == BitWidth) {
6846 auto [RangeFromAffine, Flags] = getRangeForAffineAR(
6847 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
6848 ConservativeResult =
6849 ConservativeResult.intersectWith(CR: RangeFromAffine, Type: RangeType);
6850 const_cast<SCEVAddRecExpr *>(AddRec)->setNoWrapFlags(Flags);
6851
6852 auto RangeFromFactoring = getRangeViaFactoring(
6853 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
6854 ConservativeResult =
6855 ConservativeResult.intersectWith(CR: RangeFromFactoring, Type: RangeType);
6856 }
6857 }
6858
6859 // Now try symbolic BE count and more powerful methods.
6860 if (UseExpensiveRangeSharpening) {
6861 const SCEV *SymbolicMaxBECount =
6862 getSymbolicMaxBackedgeTakenCount(L: AddRec->getLoop());
6863 if (!isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount) &&
6864 getTypeSizeInBits(Ty: MaxBEScev->getType()) <= BitWidth &&
6865 AddRec->hasNoSelfWrap()) {
6866 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
6867 AddRec, MaxBECount: SymbolicMaxBECount, BitWidth, SignHint);
6868 ConservativeResult =
6869 ConservativeResult.intersectWith(CR: RangeFromAffineNew, Type: RangeType);
6870 }
6871 }
6872 }
6873
6874 return setRange(S: AddRec, Hint: SignHint, CR: std::move(ConservativeResult));
6875 }
6876 case scUMaxExpr:
6877 case scSMaxExpr:
6878 case scUMinExpr:
6879 case scSMinExpr:
6880 case scSequentialUMinExpr: {
6881 Intrinsic::ID ID;
6882 switch (S->getSCEVType()) {
6883 case scUMaxExpr:
6884 ID = Intrinsic::umax;
6885 break;
6886 case scSMaxExpr:
6887 ID = Intrinsic::smax;
6888 break;
6889 case scUMinExpr:
6890 case scSequentialUMinExpr:
6891 ID = Intrinsic::umin;
6892 break;
6893 case scSMinExpr:
6894 ID = Intrinsic::smin;
6895 break;
6896 default:
6897 llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.");
6898 }
6899
6900 const auto *NAry = cast<SCEVNAryExpr>(Val: S);
6901 ConstantRange X = getRangeRef(S: NAry->getOperand(i: 0), SignHint, Depth: Depth + 1);
6902 for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
6903 X = X.intrinsic(
6904 IntrinsicID: ID, Ops: {X, getRangeRef(S: NAry->getOperand(i), SignHint, Depth: Depth + 1)});
6905 return setRange(S, Hint: SignHint,
6906 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6907 }
6908 case scUnknown: {
6909 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6910 Value *V = U->getValue();
6911
6912 // Check if the IR explicitly contains !range metadata.
6913 std::optional<ConstantRange> MDRange = GetRangeFromMetadata(V);
6914 if (MDRange)
6915 ConservativeResult =
6916 ConservativeResult.intersectWith(CR: *MDRange, Type: RangeType);
6917
6918 // Use facts about recurrences in the underlying IR. Note that add
6919 // recurrences are AddRecExprs and thus don't hit this path. This
6920 // primarily handles shift recurrences.
6921 auto CR = getRangeForUnknownRecurrence(U);
6922 ConservativeResult = ConservativeResult.intersectWith(CR);
6923
6924 // See if ValueTracking can give us a useful range.
6925 const DataLayout &DL = getDataLayout();
6926 KnownBits Known = computeKnownBits(V, DL, AC: &AC, CtxI: nullptr, DT: &DT);
6927 if (Known.getBitWidth() != BitWidth)
6928 Known = Known.zextOrTrunc(BitWidth);
6929
6930 // ValueTracking may be able to compute a tighter result for the number of
6931 // sign bits than for the value of those sign bits.
6932 unsigned NS = ComputeNumSignBits(Op: V, DL, AC: &AC, CtxI: nullptr, DT: &DT);
6933 if (U->getType()->isPointerTy()) {
6934 // NS counts the sign bits of the whole pointer; drop those above the
6935 // index bits.
6936 unsigned PtrIdxDiff =
6937 DL.getPointerTypeSizeInBits(U->getType()) - BitWidth;
6938 NS = NS > PtrIdxDiff ? NS - PtrIdxDiff : 1;
6939 }
6940
6941 if (NS > 1) {
6942 // If we know any of the sign bits, we know all of the sign bits.
6943 if (!Known.Zero.getHiBits(numBits: NS).isZero())
6944 Known.Zero.setHighBits(NS);
6945 if (!Known.One.getHiBits(numBits: NS).isZero())
6946 Known.One.setHighBits(NS);
6947 }
6948
6949 if (Known.getMinValue() != Known.getMaxValue() + 1)
6950 ConservativeResult = ConservativeResult.intersectWith(
6951 CR: ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
6952 Type: RangeType);
6953 if (NS > 1)
6954 ConservativeResult = ConservativeResult.intersectWith(
6955 CR: ConstantRange(APInt::getSignedMinValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1),
6956 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1) + 1),
6957 Type: RangeType);
6958
6959 if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
6960 // Strengthen the range if the underlying IR value is a
6961 // global/alloca/heap allocation using the size of the object.
6962 bool CanBeNull;
6963 uint64_t DerefBytes = V->getPointerDereferenceableBytes(
6964 DL, CanBeNull, /*CanBeFreed=*/nullptr);
6965 if (DerefBytes > 1 && isUIntN(N: BitWidth, x: DerefBytes)) {
6966 // The highest address the object can start is DerefBytes bytes before
6967 // the end (unsigned max value). If this value is not a multiple of the
6968 // alignment, the last possible start value is the next lowest multiple
6969 // of the alignment. Note: The computations below cannot overflow,
6970 // because if they would there's no possible start address for the
6971 // object.
6972 APInt MaxVal =
6973 APInt::getMaxValue(numBits: BitWidth) - APInt(BitWidth, DerefBytes);
6974 uint64_t Align = U->getValue()->getPointerAlignment(DL).value();
6975 uint64_t Rem = MaxVal.urem(RHS: Align);
6976 MaxVal -= APInt(BitWidth, Rem);
6977 APInt MinVal = APInt::getZero(numBits: BitWidth);
6978 if (llvm::isKnownNonZero(V, Q: DL))
6979 MinVal = Align;
6980 ConservativeResult = ConservativeResult.intersectWith(
6981 CR: ConstantRange::getNonEmpty(Lower: MinVal, Upper: MaxVal + 1), Type: RangeType);
6982 }
6983 }
6984
6985 // A range of Phi is a subset of union of all ranges of its input.
6986 if (PHINode *Phi = dyn_cast<PHINode>(Val: V)) {
6987 // SCEVExpander sometimes creates SCEVUnknowns that are secretly
6988 // AddRecs; return the range for the corresponding AddRec.
6989 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: getSCEV(V)))
6990 return getRangeRef(S: AR, SignHint, Depth: Depth + 1);
6991
6992 // Make sure that we do not run over cycled Phis.
6993 if (RangeRefPHIAllowedOperands(DT, PHI: Phi)) {
6994 ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
6995
6996 for (const auto &Op : Phi->operands()) {
6997 auto OpRange = getRangeRef(S: getSCEV(V: Op), SignHint, Depth: Depth + 1);
6998 RangeFromOps = RangeFromOps.unionWith(CR: OpRange);
6999 // No point to continue if we already have a full set.
7000 if (RangeFromOps.isFullSet())
7001 break;
7002 }
7003 ConservativeResult =
7004 ConservativeResult.intersectWith(CR: RangeFromOps, Type: RangeType);
7005 }
7006 }
7007
7008 // vscale can't be equal to zero
7009 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V))
7010 if (II->getIntrinsicID() == Intrinsic::vscale) {
7011 ConstantRange Disallowed = APInt::getZero(numBits: BitWidth);
7012 ConservativeResult = ConservativeResult.difference(CR: Disallowed);
7013 }
7014
7015 return setRange(S: U, Hint: SignHint, CR: std::move(ConservativeResult));
7016 }
7017 case scCouldNotCompute:
7018 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
7019 }
7020
7021 return setRange(S, Hint: SignHint, CR: std::move(ConservativeResult));
7022}
7023
7024// Given a StartRange, Step and MaxBECount for an expression compute a range of
7025// values that the expression can take. Initially, the expression has a value
7026// from StartRange and then is changed by Step up to MaxBECount times. Signed
7027// argument defines if we treat Step as signed or unsigned. The second return
7028// value indicates that no wrapping occurred.
7029static std::pair<ConstantRange, bool>
7030getRangeForAffineARHelper(APInt Step, const ConstantRange &StartRange,
7031 const APInt &MaxBECount, bool Signed) {
7032 unsigned BitWidth = Step.getBitWidth();
7033 assert(BitWidth == StartRange.getBitWidth() &&
7034 BitWidth == MaxBECount.getBitWidth() && "mismatched bit widths");
7035 // If either Step or MaxBECount is 0, then the expression won't change, and we
7036 // just need to return the initial range.
7037 if (Step == 0 || MaxBECount == 0)
7038 return {StartRange, true};
7039
7040 // If we don't know anything about the initial value (i.e. StartRange is
7041 // FullRange), then we don't know anything about the final range either.
7042 // Return FullRange.
7043 if (StartRange.isFullSet())
7044 return {ConstantRange::getFull(BitWidth), false};
7045
7046 // If Step is signed and negative, then we use its absolute value, but we also
7047 // note that we're moving in the opposite direction.
7048 bool Descending = Signed && Step.isNegative();
7049
7050 if (Signed)
7051 // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
7052 // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
7053 // This equations hold true due to the well-defined wrap-around behavior of
7054 // APInt.
7055 Step = Step.abs();
7056
7057 // Check if Offset is more than full span of BitWidth. If it is, the
7058 // expression is guaranteed to overflow.
7059 if (APInt::getMaxValue(numBits: StartRange.getBitWidth()).udiv(RHS: Step).ult(RHS: MaxBECount))
7060 return {ConstantRange::getFull(BitWidth), false};
7061
7062 // Offset is by how much the expression can change. Checks above guarantee no
7063 // overflow here.
7064 APInt Offset = Step * MaxBECount;
7065
7066 // Minimum value of the final range will match the minimal value of StartRange
7067 // if the expression is increasing and will be decreased by Offset otherwise.
7068 // Maximum value of the final range will match the maximal value of StartRange
7069 // if the expression is decreasing and will be increased by Offset otherwise.
7070 APInt StartLower = StartRange.getLower();
7071 APInt StartUpper = StartRange.getUpper() - 1;
7072 bool Overflow;
7073 APInt MovedBoundary;
7074 if (Signed) {
7075 // This does not use sadd_ov, as we want to check overflow for a signed
7076 // start with an unsigned offset.
7077 if (Descending) {
7078 MovedBoundary = StartLower - std::move(Offset);
7079 Overflow = MovedBoundary.sgt(RHS: StartLower) || StartRange.isSignWrappedSet();
7080 } else {
7081 MovedBoundary = StartUpper + std::move(Offset);
7082 Overflow = MovedBoundary.slt(RHS: StartUpper) || StartRange.isSignWrappedSet();
7083 }
7084 } else {
7085 MovedBoundary = StartUpper.uadd_ov(RHS: std::move(Offset), Overflow);
7086 Overflow |= StartRange.isWrappedSet();
7087 }
7088
7089 // It's possible that the new minimum/maximum value will fall into the initial
7090 // range (due to wrap around). This means that the expression can take any
7091 // value in this bitwidth, and we have to return full range.
7092 if (StartRange.contains(Val: MovedBoundary))
7093 return {ConstantRange::getFull(BitWidth), false};
7094
7095 APInt NewLower =
7096 Descending ? std::move(MovedBoundary) : std::move(StartLower);
7097 APInt NewUpper =
7098 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
7099 NewUpper += 1;
7100
7101 // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
7102 return {ConstantRange::getNonEmpty(Lower: std::move(NewLower), Upper: std::move(NewUpper)),
7103 !Overflow};
7104}
7105
7106std::pair<ConstantRange, SCEVFlags>
7107ScalarEvolution::getRangeForAffineAR(const SCEV *Start, const SCEV *Step,
7108 const APInt &MaxBECount) {
7109 assert(getTypeSizeInBits(Start->getType()) ==
7110 getTypeSizeInBits(Step->getType()) &&
7111 getTypeSizeInBits(Start->getType()) == MaxBECount.getBitWidth() &&
7112 "mismatched bit widths");
7113
7114 // First, consider step signed.
7115 ConstantRange StartSRange = getSignedRange(S: Start);
7116 ConstantRange StepSRange = getSignedRange(S: Step);
7117
7118 // If Step can be both positive and negative, we need to find ranges for the
7119 // maximum absolute step values in both directions and union them.
7120 auto [SR1, NSW1] = getRangeForAffineARHelper(
7121 Step: StepSRange.getSignedMin(), StartRange: StartSRange, MaxBECount, /*Signed=*/true);
7122 auto [SR2, NSW2] = getRangeForAffineARHelper(Step: StepSRange.getSignedMax(),
7123 StartRange: StartSRange, MaxBECount,
7124 /*Signed=*/true);
7125 ConstantRange SR = SR1.unionWith(CR: SR2);
7126
7127 // Next, consider step unsigned.
7128 auto [UR, NUW] = getRangeForAffineARHelper(
7129 Step: getUnsignedRangeMax(S: Step), StartRange: getUnsignedRange(S: Start), MaxBECount,
7130 /*Signed=*/false);
7131
7132 SCEVFlags Flags = SCEV::FlagNone;
7133 if (NUW)
7134 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7135 if (NSW1 && NSW2)
7136 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7137
7138 // Finally, intersect signed and unsigned ranges.
7139 return {SR.intersectWith(CR: UR, Type: ConstantRange::Smallest), Flags};
7140}
7141
7142ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
7143 const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
7144 ScalarEvolution::RangeSignHint SignHint) {
7145 assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n");
7146 assert(AddRec->hasNoSelfWrap() &&
7147 "This only works for non-self-wrapping AddRecs!");
7148 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
7149 const SCEV *Step = AddRec->getStepRecurrence(SE&: *this);
7150 // Only deal with constant step to save compile time.
7151 if (!isa<SCEVConstant>(Val: Step))
7152 return ConstantRange::getFull(BitWidth);
7153 // Let's make sure that we can prove that we do not self-wrap during
7154 // MaxBECount iterations. We need this because MaxBECount is a maximum
7155 // iteration count estimate, and we might infer nw from some exit for which we
7156 // do not know max exit count (or any other side reasoning).
7157 // TODO: Turn into assert at some point.
7158 if (getTypeSizeInBits(Ty: MaxBECount->getType()) >
7159 getTypeSizeInBits(Ty: AddRec->getType()))
7160 return ConstantRange::getFull(BitWidth);
7161 MaxBECount = getNoopOrZeroExtend(V: MaxBECount, Ty: AddRec->getType());
7162 const SCEV *RangeWidth = getMinusOne(Ty: AddRec->getType());
7163 const SCEV *StepAbs = getUMinExpr(LHS: Step, RHS: getNegativeSCEV(V: Step));
7164 const SCEV *MaxItersWithoutWrap = getUDivExpr(LHS: RangeWidth, RHS: StepAbs);
7165 if (!isKnownPredicateViaConstantRanges(Pred: ICmpInst::ICMP_ULE, LHS: MaxBECount,
7166 RHS: MaxItersWithoutWrap))
7167 return ConstantRange::getFull(BitWidth);
7168
7169 ICmpInst::Predicate LEPred =
7170 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
7171 ICmpInst::Predicate GEPred =
7172 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
7173 const SCEV *End = AddRec->evaluateAtIteration(It: MaxBECount, SE&: *this);
7174
7175 // We know that there is no self-wrap. Let's take Start and End values and
7176 // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
7177 // the iteration. They either lie inside the range [Min(Start, End),
7178 // Max(Start, End)] or outside it:
7179 //
7180 // Case 1: RangeMin ... Start V1 ... VN End ... RangeMax;
7181 // Case 2: RangeMin Vk ... V1 Start ... End Vn ... Vk + 1 RangeMax;
7182 //
7183 // No self wrap flag guarantees that the intermediate values cannot be BOTH
7184 // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
7185 // knowledge, let's try to prove that we are dealing with Case 1. It is so if
7186 // Start <= End and step is positive, or Start >= End and step is negative.
7187 const SCEV *Start = applyLoopGuards(Expr: AddRec->getStart(), L: AddRec->getLoop());
7188 ConstantRange StartRange = getRangeRef(S: Start, SignHint);
7189 ConstantRange EndRange = getRangeRef(S: End, SignHint);
7190 ConstantRange RangeBetween = StartRange.unionWith(CR: EndRange);
7191 // If they already cover full iteration space, we will know nothing useful
7192 // even if we prove what we want to prove.
7193 if (RangeBetween.isFullSet())
7194 return RangeBetween;
7195 // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
7196 bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
7197 : RangeBetween.isWrappedSet();
7198 if (IsWrappedSet)
7199 return ConstantRange::getFull(BitWidth);
7200
7201 if (isKnownPositive(S: Step) &&
7202 isKnownPredicateViaConstantRanges(Pred: LEPred, LHS: Start, RHS: End))
7203 return RangeBetween;
7204 if (isKnownNegative(S: Step) &&
7205 isKnownPredicateViaConstantRanges(Pred: GEPred, LHS: Start, RHS: End))
7206 return RangeBetween;
7207 return ConstantRange::getFull(BitWidth);
7208}
7209
7210ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
7211 const SCEV *Step,
7212 const APInt &MaxBECount) {
7213 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
7214 // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
7215
7216 unsigned BitWidth = MaxBECount.getBitWidth();
7217 assert(getTypeSizeInBits(Start->getType()) == BitWidth &&
7218 getTypeSizeInBits(Step->getType()) == BitWidth &&
7219 "mismatched bit widths");
7220
7221 struct SelectPattern {
7222 Value *Condition = nullptr;
7223 APInt TrueValue;
7224 APInt FalseValue;
7225
7226 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
7227 const SCEV *S) {
7228 std::optional<unsigned> CastOp;
7229 APInt Offset(BitWidth, 0);
7230
7231 assert(SE.getTypeSizeInBits(S->getType()) == BitWidth &&
7232 "Should be!");
7233
7234 // Peel off a constant offset. In the future we could consider being
7235 // smarter here and handle {Start+Step,+,Step} too.
7236 const APInt *Off;
7237 if (match(S, P: m_scev_Add(Op0: m_scev_APInt(C&: Off), Op1: m_SCEV(V&: S))))
7238 Offset = *Off;
7239
7240 // Peel off a cast operation
7241 if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(Val: S)) {
7242 CastOp = SCast->getSCEVType();
7243 S = SCast->getOperand();
7244 }
7245
7246 using namespace llvm::PatternMatch;
7247
7248 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
7249 const APInt *TrueVal, *FalseVal;
7250 if (!SU ||
7251 !match(V: SU->getValue(), P: m_Select(C: m_Value(V&: Condition), L: m_APInt(Res&: TrueVal),
7252 R: m_APInt(Res&: FalseVal)))) {
7253 Condition = nullptr;
7254 return;
7255 }
7256
7257 TrueValue = *TrueVal;
7258 FalseValue = *FalseVal;
7259
7260 // Re-apply the cast we peeled off earlier
7261 if (CastOp)
7262 switch (*CastOp) {
7263 default:
7264 llvm_unreachable("Unknown SCEV cast type!");
7265
7266 case scTruncate:
7267 TrueValue = TrueValue.trunc(width: BitWidth);
7268 FalseValue = FalseValue.trunc(width: BitWidth);
7269 break;
7270 case scZeroExtend:
7271 TrueValue = TrueValue.zext(width: BitWidth);
7272 FalseValue = FalseValue.zext(width: BitWidth);
7273 break;
7274 case scSignExtend:
7275 TrueValue = TrueValue.sext(width: BitWidth);
7276 FalseValue = FalseValue.sext(width: BitWidth);
7277 break;
7278 }
7279
7280 // Re-apply the constant offset we peeled off earlier
7281 TrueValue += Offset;
7282 FalseValue += Offset;
7283 }
7284
7285 bool isRecognized() { return Condition != nullptr; }
7286 };
7287
7288 SelectPattern StartPattern(*this, BitWidth, Start);
7289 if (!StartPattern.isRecognized())
7290 return ConstantRange::getFull(BitWidth);
7291
7292 SelectPattern StepPattern(*this, BitWidth, Step);
7293 if (!StepPattern.isRecognized())
7294 return ConstantRange::getFull(BitWidth);
7295
7296 if (StartPattern.Condition != StepPattern.Condition) {
7297 // We don't handle this case today; but we could, by considering four
7298 // possibilities below instead of two. I'm not sure if there are cases where
7299 // that will help over what getRange already does, though.
7300 return ConstantRange::getFull(BitWidth);
7301 }
7302
7303 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
7304 // construct arbitrary general SCEV expressions here. This function is called
7305 // from deep in the call stack, and calling getSCEV (on a sext instruction,
7306 // say) can end up caching a suboptimal value.
7307
7308 // FIXME: without the explicit `this` receiver below, MSVC errors out with
7309 // C2352 and C2512 (otherwise it isn't needed).
7310
7311 const SCEV *TrueStart = this->getConstant(Val: StartPattern.TrueValue);
7312 const SCEV *TrueStep = this->getConstant(Val: StepPattern.TrueValue);
7313 const SCEV *FalseStart = this->getConstant(Val: StartPattern.FalseValue);
7314 const SCEV *FalseStep = this->getConstant(Val: StepPattern.FalseValue);
7315
7316 ConstantRange TrueRange =
7317 this->getRangeForAffineAR(Start: TrueStart, Step: TrueStep, MaxBECount).first;
7318 ConstantRange FalseRange =
7319 this->getRangeForAffineAR(Start: FalseStart, Step: FalseStep, MaxBECount).first;
7320
7321 return TrueRange.unionWith(CR: FalseRange);
7322}
7323
7324SCEVFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
7325 if (isa<ConstantExpr>(Val: V))
7326 return SCEV::FlagNone;
7327 const BinaryOperator *BinOp = cast<BinaryOperator>(Val: V);
7328
7329 // Return early if there are no flags to propagate to the SCEV.
7330 SCEVFlags Flags = SCEV::FlagNone;
7331 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: BinOp);
7332 PDI && PDI->isDisjoint()) {
7333 Flags = ScalarEvolution::setFlags(Flags: SCEV::FlagNUW, OnFlags: SCEV::FlagNSW);
7334 } else {
7335 if (BinOp->hasNoUnsignedWrap())
7336 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7337 if (BinOp->hasNoSignedWrap())
7338 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7339 }
7340 if (Flags == SCEV::FlagNone)
7341 return SCEV::FlagNone;
7342
7343 return isSCEVExprNeverPoison(I: BinOp) ? Flags : SCEV::FlagNone;
7344}
7345
7346const Instruction *
7347ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
7348 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
7349 return &*AddRec->getLoop()->getHeader()->begin();
7350 if (auto *U = dyn_cast<SCEVUnknown>(Val: S))
7351 if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
7352 return I;
7353 return nullptr;
7354}
7355
7356const Instruction *ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops,
7357 bool &Precise) {
7358 Precise = true;
7359 // Do a bounded search of the def relation of the requested SCEVs.
7360 SmallPtrSet<const SCEV *, 16> Visited;
7361 SmallVector<SCEVUse> Worklist;
7362 auto pushOp = [&](const SCEV *S) {
7363 if (!Visited.insert(Ptr: S).second)
7364 return;
7365 // Threshold of 30 here is arbitrary.
7366 if (Visited.size() > 30) {
7367 Precise = false;
7368 return;
7369 }
7370 Worklist.push_back(Elt: S);
7371 };
7372
7373 for (SCEVUse S : Ops)
7374 pushOp(S);
7375
7376 const Instruction *Bound = nullptr;
7377 while (!Worklist.empty()) {
7378 SCEVUse S = Worklist.pop_back_val();
7379 if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
7380 if (!Bound || DT.dominates(Def: Bound, User: DefI))
7381 Bound = DefI;
7382 } else {
7383 for (SCEVUse Op : S->operands())
7384 pushOp(Op);
7385 }
7386 }
7387 return Bound ? Bound : &*F.getEntryBlock().begin();
7388}
7389
7390const Instruction *
7391ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops) {
7392 bool Discard;
7393 return getDefiningScopeBound(Ops, Precise&: Discard);
7394}
7395
7396bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
7397 const Instruction *B) {
7398 if (A->getParent() == B->getParent() &&
7399 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7400 End: B->getIterator()))
7401 return true;
7402
7403 auto *BLoop = LI.getLoopFor(BB: B->getParent());
7404 if (BLoop && BLoop->getHeader() == B->getParent() &&
7405 BLoop->getLoopPreheader() == A->getParent() &&
7406 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7407 End: A->getParent()->end()) &&
7408 isGuaranteedToTransferExecutionToSuccessor(Begin: B->getParent()->begin(),
7409 End: B->getIterator()))
7410 return true;
7411 return false;
7412}
7413
7414bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
7415 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
7416 visitAll(Root: Op, Visitor&: PC);
7417 return PC.MaybePoison.empty();
7418}
7419
7420bool ScalarEvolution::isGuaranteedNotToCauseUB(const SCEV *Op) {
7421 return !SCEVExprContains(Root: Op, Pred: [this](const SCEV *S) {
7422 const SCEV *Op1;
7423 bool M = match(S, P: m_scev_UDiv(Op0: m_SCEV(), Op1: m_SCEV(V&: Op1)));
7424 // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
7425 // is a non-zero constant, we have to assume the UDiv may be UB.
7426 return M && (!isKnownNonZero(S: Op1) || !isGuaranteedNotToBePoison(Op: Op1));
7427 });
7428}
7429
7430bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
7431 // Only proceed if we can prove that I does not yield poison.
7432 if (!programUndefinedIfPoison(Inst: I))
7433 return false;
7434
7435 // At this point we know that if I is executed, then it does not wrap
7436 // according to at least one of NSW or NUW. If I is not executed, then we do
7437 // not know if the calculation that I represents would wrap. Multiple
7438 // instructions can map to the same SCEV. If we apply NSW or NUW from I to
7439 // the SCEV, we must guarantee no wrapping for that SCEV also when it is
7440 // derived from other instructions that map to the same SCEV. We cannot make
7441 // that guarantee for cases where I is not executed. So we need to find a
7442 // upper bound on the defining scope for the SCEV, and prove that I is
7443 // executed every time we enter that scope. When the bounding scope is a
7444 // loop (the common case), this is equivalent to proving I executes on every
7445 // iteration of that loop.
7446 SmallVector<SCEVUse> SCEVOps;
7447 for (const Use &Op : I->operands()) {
7448 // I could be an extractvalue from a call to an overflow intrinsic.
7449 // TODO: We can do better here in some cases.
7450 if (isSCEVable(Ty: Op->getType()))
7451 SCEVOps.push_back(Elt: getSCEV(V: Op));
7452 }
7453 auto *DefI = getDefiningScopeBound(Ops: SCEVOps);
7454 return isGuaranteedToTransferExecutionTo(A: DefI, B: I);
7455}
7456
7457bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
7458 // If we know that \c I can never be poison period, then that's enough.
7459 if (isSCEVExprNeverPoison(I))
7460 return true;
7461
7462 // If the loop only has one exit, then we know that, if the loop is entered,
7463 // any instruction dominating that exit will be executed. If any such
7464 // instruction would result in UB, the addrec cannot be poison.
7465 //
7466 // This is basically the same reasoning as in isSCEVExprNeverPoison(), but
7467 // also handles uses outside the loop header (they just need to dominate the
7468 // single exit).
7469
7470 auto *ExitingBB = L->getExitingBlock();
7471 if (!ExitingBB || !loopHasNoAbnormalExits(L))
7472 return false;
7473
7474 SmallPtrSet<const Value *, 16> KnownPoison;
7475 SmallVector<const Instruction *, 8> Worklist;
7476
7477 // We start by assuming \c I, the post-inc add recurrence, is poison. Only
7478 // things that are known to be poison under that assumption go on the
7479 // Worklist.
7480 KnownPoison.insert(Ptr: I);
7481 Worklist.push_back(Elt: I);
7482
7483 while (!Worklist.empty()) {
7484 const Instruction *Poison = Worklist.pop_back_val();
7485
7486 for (const Use &U : Poison->uses()) {
7487 const Instruction *PoisonUser = cast<Instruction>(Val: U.getUser());
7488 if (mustTriggerUB(I: PoisonUser, KnownPoison) &&
7489 DT.dominates(A: PoisonUser->getParent(), B: ExitingBB))
7490 return true;
7491
7492 if (propagatesPoison(PoisonOp: U) && L->contains(Inst: PoisonUser))
7493 if (KnownPoison.insert(Ptr: PoisonUser).second)
7494 Worklist.push_back(Elt: PoisonUser);
7495 }
7496 }
7497
7498 return false;
7499}
7500
7501ScalarEvolution::LoopProperties
7502ScalarEvolution::getLoopProperties(const Loop *L) {
7503 using LoopProperties = ScalarEvolution::LoopProperties;
7504
7505 auto Itr = LoopPropertiesCache.find(Val: L);
7506 if (Itr == LoopPropertiesCache.end()) {
7507 auto HasSideEffects = [](Instruction *I) {
7508 if (auto *SI = dyn_cast<StoreInst>(Val: I))
7509 return !SI->isSimple();
7510
7511 if (I->mayThrow())
7512 return true;
7513
7514 // Non-volatile memset / memcpy do not count as side-effect for forward
7515 // progress.
7516 if (isa<MemIntrinsic>(Val: I) && !I->isVolatile())
7517 return false;
7518
7519 return I->mayWriteToMemory();
7520 };
7521
7522 LoopProperties LP = {/* HasNoAbnormalExits */ true,
7523 /*HasNoSideEffects*/ true};
7524
7525 for (auto *BB : L->getBlocks())
7526 for (auto &I : *BB) {
7527 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7528 LP.HasNoAbnormalExits = false;
7529 if (HasSideEffects(&I))
7530 LP.HasNoSideEffects = false;
7531 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7532 break; // We're already as pessimistic as we can get.
7533 }
7534
7535 auto InsertPair = LoopPropertiesCache.insert(KV: {L, LP});
7536 assert(InsertPair.second && "We just checked!");
7537 Itr = InsertPair.first;
7538 }
7539
7540 return Itr->second;
7541}
7542
7543bool ScalarEvolution::loopIsFiniteByAssumption(const Loop *L) {
7544 // A mustprogress loop without side effects must be finite.
7545 // TODO: The check used here is very conservative. It's only *specific*
7546 // side effects which are well defined in infinite loops.
7547 return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L));
7548}
7549
7550const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
7551 // Worklist item with a Value and a bool indicating whether all operands have
7552 // been visited already.
7553 using PointerTy = PointerIntPair<Value *, 1, bool>;
7554 SmallVector<PointerTy> Stack;
7555
7556 Stack.emplace_back(Args&: V, Args: false);
7557 while (!Stack.empty()) {
7558 auto E = Stack.back();
7559 Value *CurV = E.getPointer();
7560
7561 if (getExistingSCEV(V: CurV)) {
7562 Stack.pop_back();
7563 continue;
7564 }
7565
7566 SmallVector<Value *> Ops;
7567 const SCEV *CreatedSCEV = nullptr;
7568 // If all operands have been visited already, create the SCEV.
7569 if (E.getInt()) {
7570 CreatedSCEV = createSCEV(V: CurV);
7571 } else {
7572 // Otherwise get the operands we need to create SCEV's for before creating
7573 // the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
7574 // just use it.
7575 CreatedSCEV = getOperandsToCreate(V: CurV, Ops);
7576 }
7577
7578 if (CreatedSCEV) {
7579 insertValueToMap(V: CurV, S: CreatedSCEV);
7580 Stack.pop_back();
7581 } else {
7582 Stack.back().setInt(true);
7583 // Queue its operands which need to be constructed.
7584 for (Value *Op : Ops)
7585 Stack.emplace_back(Args&: Op, Args: false);
7586 }
7587 }
7588
7589 return getExistingSCEV(V);
7590}
7591
7592const SCEV *
7593ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
7594 if (!isSCEVable(Ty: V->getType()))
7595 return getUnknown(V);
7596
7597 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7598 // Don't attempt to analyze instructions in blocks that aren't
7599 // reachable. Such instructions don't matter, and they aren't required
7600 // to obey basic rules for definitions dominating uses which this
7601 // analysis depends on.
7602 if (!DT.isReachableFromEntry(A: I->getParent()))
7603 return getUnknown(V: PoisonValue::get(T: V->getType()));
7604 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7605 return getConstant(V: CI);
7606 else if (isa<GlobalAlias>(Val: V))
7607 return getUnknown(V);
7608 else if (!isa<ConstantExpr>(Val: V))
7609 return getUnknown(V);
7610
7611 Operator *U = cast<Operator>(Val: V);
7612 if (auto BO =
7613 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CtxI: dyn_cast<Instruction>(Val: V))) {
7614 bool IsConstArg = isa<ConstantInt>(Val: BO->RHS);
7615 switch (BO->Opcode) {
7616 case Instruction::Add:
7617 case Instruction::Mul: {
7618 // For additions and multiplications, traverse add/mul chains for which we
7619 // can potentially create a single SCEV, to reduce the number of
7620 // get{Add,Mul}Expr calls.
7621 do {
7622 if (BO->Op) {
7623 if (BO->Op != V && getExistingSCEV(V: BO->Op)) {
7624 Ops.push_back(Elt: BO->Op);
7625 break;
7626 }
7627 }
7628 Ops.push_back(Elt: BO->RHS);
7629 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7630 CtxI: dyn_cast<Instruction>(Val: V));
7631 if (!NewBO ||
7632 (BO->Opcode == Instruction::Add &&
7633 (NewBO->Opcode != Instruction::Add &&
7634 NewBO->Opcode != Instruction::Sub)) ||
7635 (BO->Opcode == Instruction::Mul &&
7636 NewBO->Opcode != Instruction::Mul)) {
7637 Ops.push_back(Elt: BO->LHS);
7638 break;
7639 }
7640 // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
7641 // requires a SCEV for the LHS.
7642 if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
7643 auto *I = dyn_cast<Instruction>(Val: BO->Op);
7644 if (I && programUndefinedIfPoison(Inst: I)) {
7645 Ops.push_back(Elt: BO->LHS);
7646 break;
7647 }
7648 }
7649 BO = NewBO;
7650 } while (true);
7651 return nullptr;
7652 }
7653 case Instruction::Sub:
7654 case Instruction::UDiv:
7655 case Instruction::URem:
7656 break;
7657 case Instruction::AShr:
7658 case Instruction::Shl:
7659 case Instruction::Xor:
7660 if (!IsConstArg)
7661 return nullptr;
7662 break;
7663 case Instruction::And:
7664 case Instruction::Or:
7665 if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(BitWidth: 1))
7666 return nullptr;
7667 break;
7668 case Instruction::LShr:
7669 return getUnknown(V);
7670 default:
7671 llvm_unreachable("Unhandled binop");
7672 break;
7673 }
7674
7675 Ops.push_back(Elt: BO->LHS);
7676 Ops.push_back(Elt: BO->RHS);
7677 return nullptr;
7678 }
7679
7680 switch (U->getOpcode()) {
7681 case Instruction::Trunc:
7682 case Instruction::ZExt:
7683 case Instruction::SExt:
7684 case Instruction::PtrToAddr:
7685 case Instruction::PtrToInt:
7686 Ops.push_back(Elt: U->getOperand(i: 0));
7687 return nullptr;
7688
7689 case Instruction::BitCast:
7690 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType())) {
7691 Ops.push_back(Elt: U->getOperand(i: 0));
7692 return nullptr;
7693 }
7694 return getUnknown(V);
7695
7696 case Instruction::SDiv:
7697 case Instruction::SRem:
7698 Ops.push_back(Elt: U->getOperand(i: 0));
7699 Ops.push_back(Elt: U->getOperand(i: 1));
7700 return nullptr;
7701
7702 case Instruction::GetElementPtr:
7703 assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() &&
7704 "GEP source element type must be sized");
7705 llvm::append_range(C&: Ops, R: U->operands());
7706 return nullptr;
7707
7708 case Instruction::IntToPtr:
7709 return getUnknown(V);
7710
7711 case Instruction::PHI:
7712 // getNodeForPHI has four ways to turn a PHI into a SCEV; retrieve the
7713 // relevant nodes for each of them.
7714 //
7715 // The first is just to call simplifyInstruction, and get something back
7716 // that isn't a PHI.
7717 if (Value *V = simplifyInstruction(
7718 I: cast<PHINode>(Val: U),
7719 Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
7720 /*UseInstrInfo=*/true, /*CanUseUndef=*/false})) {
7721 assert(V);
7722 Ops.push_back(Elt: V);
7723 return nullptr;
7724 }
7725 // The second is createNodeForPHIWithIdenticalOperands: this looks for
7726 // operands which all perform the same operation, but haven't been
7727 // CSE'ed for whatever reason.
7728 if (BinaryOperator *BO = getCommonInstForPHI(PN: cast<PHINode>(Val: U))) {
7729 assert(BO);
7730 Ops.push_back(Elt: BO);
7731 return nullptr;
7732 }
7733 // The third is createNodeFromSelectLikePHI; this takes a PHI which
7734 // is equivalent to a select, and analyzes it like a select.
7735 {
7736 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
7737 if (getOperandsForSelectLikePHI(DT, PN: cast<PHINode>(Val: U), Cond, LHS, RHS)) {
7738 assert(Cond);
7739 assert(LHS);
7740 assert(RHS);
7741 if (auto *CondICmp = dyn_cast<ICmpInst>(Val: Cond)) {
7742 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 0));
7743 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 1));
7744 }
7745 Ops.push_back(Elt: Cond);
7746 Ops.push_back(Elt: LHS);
7747 Ops.push_back(Elt: RHS);
7748 return nullptr;
7749 }
7750 }
7751 // The fourth way is createAddRecFromPHI. It's complicated to handle here,
7752 // so just construct it recursively.
7753 //
7754 // In addition to getNodeForPHI, also construct nodes which might be needed
7755 // by getRangeRef.
7756 if (RangeRefPHIAllowedOperands(DT, PHI: cast<PHINode>(Val: U))) {
7757 for (Value *V : cast<PHINode>(Val: U)->operands())
7758 Ops.push_back(Elt: V);
7759 return nullptr;
7760 }
7761 return nullptr;
7762
7763 case Instruction::Select: {
7764 // Check if U is a select that can be simplified to a SCEVUnknown.
7765 auto CanSimplifyToUnknown = [this, U]() {
7766 if (U->getType()->isIntegerTy(BitWidth: 1) || isa<ConstantInt>(Val: U->getOperand(i: 0)))
7767 return false;
7768
7769 auto *ICI = dyn_cast<ICmpInst>(Val: U->getOperand(i: 0));
7770 if (!ICI)
7771 return false;
7772 Value *LHS = ICI->getOperand(i_nocapture: 0);
7773 Value *RHS = ICI->getOperand(i_nocapture: 1);
7774 if (ICI->getPredicate() == CmpInst::ICMP_EQ ||
7775 ICI->getPredicate() == CmpInst::ICMP_NE) {
7776 if (!(isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()))
7777 return true;
7778 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
7779 getTypeSizeInBits(Ty: U->getType()))
7780 return true;
7781 return false;
7782 };
7783 if (CanSimplifyToUnknown())
7784 return getUnknown(V: U);
7785
7786 llvm::append_range(C&: Ops, R: U->operands());
7787 return nullptr;
7788 break;
7789 }
7790 case Instruction::Call:
7791 case Instruction::Invoke:
7792 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand()) {
7793 Ops.push_back(Elt: RV);
7794 return nullptr;
7795 }
7796
7797 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
7798 switch (II->getIntrinsicID()) {
7799 case Intrinsic::abs:
7800 Ops.push_back(Elt: II->getArgOperand(i: 0));
7801 return nullptr;
7802 case Intrinsic::umax:
7803 case Intrinsic::umin:
7804 case Intrinsic::smax:
7805 case Intrinsic::smin:
7806 case Intrinsic::usub_sat:
7807 case Intrinsic::uadd_sat:
7808 Ops.push_back(Elt: II->getArgOperand(i: 0));
7809 Ops.push_back(Elt: II->getArgOperand(i: 1));
7810 return nullptr;
7811 case Intrinsic::start_loop_iterations:
7812 case Intrinsic::annotation:
7813 case Intrinsic::ptr_annotation:
7814 Ops.push_back(Elt: II->getArgOperand(i: 0));
7815 return nullptr;
7816 default:
7817 break;
7818 }
7819 }
7820 break;
7821 }
7822
7823 return nullptr;
7824}
7825
7826const SCEV *ScalarEvolution::createSCEV(Value *V) {
7827 if (!isSCEVable(Ty: V->getType()))
7828 return getUnknown(V);
7829
7830 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7831 // Don't attempt to analyze instructions in blocks that aren't
7832 // reachable. Such instructions don't matter, and they aren't required
7833 // to obey basic rules for definitions dominating uses which this
7834 // analysis depends on.
7835 if (!DT.isReachableFromEntry(A: I->getParent()))
7836 return getUnknown(V: PoisonValue::get(T: V->getType()));
7837 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7838 return getConstant(V: CI);
7839 else if (isa<GlobalAlias>(Val: V))
7840 return getUnknown(V);
7841 else if (!isa<ConstantExpr>(Val: V))
7842 return getUnknown(V);
7843
7844 const SCEV *LHS;
7845 const SCEV *RHS;
7846
7847 Operator *U = cast<Operator>(Val: V);
7848 if (auto BO =
7849 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CtxI: dyn_cast<Instruction>(Val: V))) {
7850 switch (BO->Opcode) {
7851 case Instruction::Add: {
7852 // The simple thing to do would be to just call getSCEV on both operands
7853 // and call getAddExpr with the result. However if we're looking at a
7854 // bunch of things all added together, this can be quite inefficient,
7855 // because it leads to N-1 getAddExpr calls for N ultimate operands.
7856 // Instead, gather up all the operands and make a single getAddExpr call.
7857 // LLVM IR canonical form means we need only traverse the left operands.
7858 SmallVector<SCEVUse, 4> AddOps;
7859 do {
7860 if (BO->Op) {
7861 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
7862 AddOps.push_back(Elt: OpSCEV);
7863 break;
7864 }
7865
7866 // If a NUW or NSW flag can be applied to the SCEV for this
7867 // addition, then compute the SCEV for this addition by itself
7868 // with a separate call to getAddExpr. We need to do that
7869 // instead of pushing the operands of the addition onto AddOps,
7870 // since the flags are only known to apply to this particular
7871 // addition - they may not apply to other additions that can be
7872 // formed with operands from AddOps.
7873 const SCEV *RHS = getSCEV(V: BO->RHS);
7874 SCEVFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
7875 if (Flags != SCEV::FlagNone) {
7876 const SCEV *LHS = getSCEV(V: BO->LHS);
7877 if (BO->Opcode == Instruction::Sub)
7878 AddOps.push_back(Elt: getMinusSCEV(LHS, RHS, Flags));
7879 else
7880 AddOps.push_back(Elt: getAddExpr(LHS, RHS, Flags));
7881 break;
7882 }
7883 }
7884
7885 if (BO->Opcode == Instruction::Sub)
7886 AddOps.push_back(Elt: getNegativeSCEV(V: getSCEV(V: BO->RHS)));
7887 else
7888 AddOps.push_back(Elt: getSCEV(V: BO->RHS));
7889
7890 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7891 CtxI: dyn_cast<Instruction>(Val: V));
7892 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
7893 NewBO->Opcode != Instruction::Sub)) {
7894 AddOps.push_back(Elt: getSCEV(V: BO->LHS));
7895 break;
7896 }
7897 BO = NewBO;
7898 } while (true);
7899
7900 return getAddExpr(Ops&: AddOps);
7901 }
7902
7903 case Instruction::Mul: {
7904 SmallVector<SCEVUse, 4> MulOps;
7905 do {
7906 if (BO->Op) {
7907 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
7908 MulOps.push_back(Elt: OpSCEV);
7909 break;
7910 }
7911
7912 SCEVFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
7913 if (Flags != SCEV::FlagNone) {
7914 LHS = getSCEV(V: BO->LHS);
7915 RHS = getSCEV(V: BO->RHS);
7916 MulOps.push_back(Elt: getMulExpr(LHS, RHS, Flags));
7917 break;
7918 }
7919 }
7920
7921 MulOps.push_back(Elt: getSCEV(V: BO->RHS));
7922 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7923 CtxI: dyn_cast<Instruction>(Val: V));
7924 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
7925 MulOps.push_back(Elt: getSCEV(V: BO->LHS));
7926 break;
7927 }
7928 BO = NewBO;
7929 } while (true);
7930
7931 return getMulExpr(Ops&: MulOps);
7932 }
7933 case Instruction::UDiv:
7934 LHS = getSCEV(V: BO->LHS);
7935 RHS = getSCEV(V: BO->RHS);
7936 return getUDivExpr(LHS, RHS);
7937 case Instruction::URem:
7938 LHS = getSCEV(V: BO->LHS);
7939 RHS = getSCEV(V: BO->RHS);
7940 return getURemExpr(LHS, RHS);
7941 case Instruction::Sub: {
7942 SCEVFlags Flags = SCEV::FlagNone;
7943 if (BO->Op)
7944 Flags = getNoWrapFlagsFromUB(V: BO->Op);
7945
7946 // Try to use ptrtoaddr for subtracts with at least one ptrtoint
7947 // operand. While we don't model ptrtoint directly in SCEV, the
7948 // difference between two pointer addresses is well-defined.
7949 Value *PtrLHS = nullptr, *PtrRHS = nullptr;
7950 bool HasPtrLHS = match(V: BO->LHS, P: m_PtrToInt(Op: m_Value(V&: PtrLHS)));
7951 bool HasPtrRHS = match(V: BO->RHS, P: m_PtrToInt(Op: m_Value(V&: PtrRHS)));
7952 if (HasPtrLHS || HasPtrRHS) {
7953 // Convert a ptrtoint operand (OrigOp) to ptrtoaddr of its pointer
7954 // PtrOp. When only one side is ptrtoint (BothPtr is false), skip
7955 // SCEVUnknown pointers since wrapping them in ptrtoaddr adds no
7956 // useful structure.
7957 auto GetOp = [&](bool HasPtr, Value *PtrOp, Value *OrigOp,
7958 bool BothPtr) -> const SCEV * {
7959 if (!HasPtr)
7960 return getSCEV(V: OrigOp);
7961 const SCEV *PtrSCEV = getSCEV(V: PtrOp);
7962 if (BothPtr || !isa<SCEVUnknown>(Val: PtrSCEV)) {
7963 const SCEV *Addr = getPtrToAddrExpr(Op: PtrSCEV);
7964 if (!isa<SCEVCouldNotCompute>(Val: Addr) &&
7965 getTypeSizeInBits(Ty: OrigOp->getType()) <=
7966 getTypeSizeInBits(Ty: Addr->getType()))
7967 return getTruncateOrNoop(V: Addr, Ty: OrigOp->getType());
7968 }
7969 return getSCEV(V: OrigOp);
7970 };
7971 const SCEV *L = GetOp(HasPtrLHS, PtrLHS, BO->LHS, HasPtrRHS);
7972 const SCEV *R = GetOp(HasPtrRHS, PtrRHS, BO->RHS, HasPtrLHS);
7973 return getMinusSCEV(LHS: L, RHS: R, Flags);
7974 }
7975
7976 LHS = getSCEV(V: BO->LHS);
7977 RHS = getSCEV(V: BO->RHS);
7978 return getMinusSCEV(LHS, RHS, Flags);
7979 }
7980 case Instruction::And:
7981 // For an expression like x&255 that merely masks off the high bits,
7982 // use zext(trunc(x)) as the SCEV expression.
7983 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
7984 if (CI->isZero())
7985 return getSCEV(V: BO->RHS);
7986 if (CI->isMinusOne())
7987 return getSCEV(V: BO->LHS);
7988 const APInt &A = CI->getValue();
7989
7990 // Instcombine's ShrinkDemandedConstant may strip bits out of
7991 // constants, obscuring what would otherwise be a low-bits mask.
7992 // Use computeKnownBits to compute what ShrinkDemandedConstant
7993 // knew about to reconstruct a low-bits mask value.
7994 unsigned LZ = A.countl_zero();
7995 unsigned TZ = A.countr_zero();
7996 unsigned BitWidth = A.getBitWidth();
7997 KnownBits Known(BitWidth);
7998 computeKnownBits(V: BO->LHS, Known, DL: getDataLayout(), AC: &AC, CtxI: nullptr, DT: &DT);
7999
8000 APInt EffectiveMask =
8001 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - LZ - TZ).shl(shiftAmt: TZ);
8002 if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
8003 const SCEV *MulCount = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ));
8004 const SCEV *LHS = getSCEV(V: BO->LHS);
8005 const SCEV *ShiftedLHS = nullptr;
8006 if (auto *LHSMul = dyn_cast<SCEVMulExpr>(Val: LHS)) {
8007 if (auto *OpC = dyn_cast<SCEVConstant>(Val: LHSMul->getOperand(i: 0))) {
8008 // For an expression like (x * 8) & 8, simplify the multiply.
8009 unsigned MulZeros = OpC->getAPInt().countr_zero();
8010 unsigned GCD = std::min(a: MulZeros, b: TZ);
8011 APInt DivAmt = APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ - GCD);
8012 SmallVector<SCEVUse, 4> MulOps;
8013 MulOps.push_back(Elt: getConstant(Val: OpC->getAPInt().ashr(ShiftAmt: GCD)));
8014 append_range(C&: MulOps, R: LHSMul->operands().drop_front());
8015 const SCEV *NewMul = getMulExpr(Ops&: MulOps, Flags: LHSMul->getNoWrapFlags());
8016 ShiftedLHS = getUDivExpr(LHS: NewMul, RHS: getConstant(Val: DivAmt));
8017 }
8018 }
8019 if (!ShiftedLHS)
8020 ShiftedLHS = getUDivExpr(LHS, RHS: MulCount);
8021 return getMulExpr(
8022 LHS: getZeroExtendExpr(
8023 Op: getTruncateExpr(Op: ShiftedLHS,
8024 Ty: IntegerType::get(C&: getContext(), NumBits: BitWidth - LZ - TZ)),
8025 Ty: BO->LHS->getType()),
8026 RHS: MulCount);
8027 }
8028 }
8029 // Binary `and` is a bit-wise `umin`.
8030 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
8031 LHS = getSCEV(V: BO->LHS);
8032 RHS = getSCEV(V: BO->RHS);
8033 return getUMinExpr(LHS, RHS);
8034 }
8035 break;
8036
8037 case Instruction::Or:
8038 // Binary `or` is a bit-wise `umax`.
8039 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
8040 LHS = getSCEV(V: BO->LHS);
8041 RHS = getSCEV(V: BO->RHS);
8042 return getUMaxExpr(LHS, RHS);
8043 }
8044 break;
8045
8046 case Instruction::Xor:
8047 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8048 // If the RHS of xor is -1, then this is a not operation.
8049 if (CI->isMinusOne())
8050 return getNotSCEV(V: getSCEV(V: BO->LHS));
8051
8052 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
8053 // This is a variant of the check for xor with -1, and it handles
8054 // the case where instcombine has trimmed non-demanded bits out
8055 // of an xor with -1.
8056 if (auto *LBO = dyn_cast<BinaryOperator>(Val: BO->LHS))
8057 if (ConstantInt *LCI = dyn_cast<ConstantInt>(Val: LBO->getOperand(i_nocapture: 1)))
8058 if (LBO->getOpcode() == Instruction::And &&
8059 LCI->getValue() == CI->getValue())
8060 if (const SCEVZeroExtendExpr *Z =
8061 dyn_cast<SCEVZeroExtendExpr>(Val: getSCEV(V: BO->LHS))) {
8062 Type *UTy = BO->LHS->getType();
8063 const SCEV *Z0 = Z->getOperand();
8064 Type *Z0Ty = Z0->getType();
8065 unsigned Z0TySize = getTypeSizeInBits(Ty: Z0Ty);
8066
8067 // If C is a low-bits mask, the zero extend is serving to
8068 // mask off the high bits. Complement the operand and
8069 // re-apply the zext.
8070 if (CI->getValue().isMask(numBits: Z0TySize))
8071 return getZeroExtendExpr(Op: getNotSCEV(V: Z0), Ty: UTy);
8072
8073 // If C is a single bit, it may be in the sign-bit position
8074 // before the zero-extend. In this case, represent the xor
8075 // using an add, which is equivalent, and re-apply the zext.
8076 APInt Trunc = CI->getValue().trunc(width: Z0TySize);
8077 if (Trunc.zext(width: getTypeSizeInBits(Ty: UTy)) == CI->getValue() &&
8078 Trunc.isSignMask())
8079 return getZeroExtendExpr(Op: getAddExpr(LHS: Z0, RHS: getConstant(Val: Trunc)),
8080 Ty: UTy);
8081 }
8082 }
8083 break;
8084
8085 case Instruction::Shl:
8086 // Turn shift left of a constant amount into a multiply.
8087 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8088 uint32_t BitWidth = cast<IntegerType>(Val: SA->getType())->getBitWidth();
8089
8090 // If the shift count is not less than the bitwidth, the result of
8091 // the shift is undefined. Don't try to analyze it, because the
8092 // resolution chosen here may differ from the resolution chosen in
8093 // other parts of the compiler.
8094 if (SA->getValue().uge(RHS: BitWidth))
8095 break;
8096
8097 // We can safely preserve the nuw flag in all cases. It's also safe to
8098 // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
8099 // requires special handling. It can be preserved as long as we're not
8100 // left shifting by bitwidth - 1.
8101 auto Flags = SCEV::FlagNone;
8102 if (BO->Op) {
8103 auto MulFlags = getNoWrapFlagsFromUB(V: BO->Op);
8104 if (any(Val: MulFlags & SCEV::FlagNSW) &&
8105 (any(Val: MulFlags & SCEV::FlagNUW) ||
8106 SA->getValue().ult(RHS: BitWidth - 1)))
8107 Flags = Flags | SCEV::FlagNSW;
8108 if (any(Val: MulFlags & SCEV::FlagNUW))
8109 Flags = Flags | SCEV::FlagNUW;
8110 }
8111
8112 ConstantInt *X = ConstantInt::get(
8113 Context&: getContext(), V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
8114 return getMulExpr(LHS: getSCEV(V: BO->LHS), RHS: getConstant(V: X), Flags);
8115 }
8116 break;
8117
8118 case Instruction::AShr:
8119 // AShr X, C, where C is a constant.
8120 ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS);
8121 if (!CI)
8122 break;
8123
8124 Type *OuterTy = BO->LHS->getType();
8125 uint64_t BitWidth = getTypeSizeInBits(Ty: OuterTy);
8126 // If the shift count is not less than the bitwidth, the result of
8127 // the shift is undefined. Don't try to analyze it, because the
8128 // resolution chosen here may differ from the resolution chosen in
8129 // other parts of the compiler.
8130 if (CI->getValue().uge(RHS: BitWidth))
8131 break;
8132
8133 if (CI->isZero())
8134 return getSCEV(V: BO->LHS); // shift by zero --> noop
8135
8136 uint64_t AShrAmt = CI->getZExtValue();
8137 Type *TruncTy = IntegerType::get(C&: getContext(), NumBits: BitWidth - AShrAmt);
8138
8139 Operator *L = dyn_cast<Operator>(Val: BO->LHS);
8140 const SCEV *AddTruncateExpr = nullptr;
8141 ConstantInt *ShlAmtCI = nullptr;
8142 const SCEV *AddConstant = nullptr;
8143
8144 if (L && L->getOpcode() == Instruction::Add) {
8145 // X = Shl A, n
8146 // Y = Add X, c
8147 // Z = AShr Y, m
8148 // n, c and m are constants.
8149
8150 Operator *LShift = dyn_cast<Operator>(Val: L->getOperand(i: 0));
8151 ConstantInt *AddOperandCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8152 if (LShift && LShift->getOpcode() == Instruction::Shl) {
8153 if (AddOperandCI) {
8154 const SCEV *ShlOp0SCEV = getSCEV(V: LShift->getOperand(i: 0));
8155 ShlAmtCI = dyn_cast<ConstantInt>(Val: LShift->getOperand(i: 1));
8156 // since we truncate to TruncTy, the AddConstant should be of the
8157 // same type, so create a new Constant with type same as TruncTy.
8158 // Also, the Add constant should be shifted right by AShr amount.
8159 APInt AddOperand = AddOperandCI->getValue().ashr(ShiftAmt: AShrAmt);
8160 AddConstant = getConstant(Val: AddOperand.trunc(width: BitWidth - AShrAmt));
8161 // we model the expression as sext(add(trunc(A), c << n)), since the
8162 // sext(trunc) part is already handled below, we create a
8163 // AddExpr(TruncExp) which will be used later.
8164 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8165 }
8166 }
8167 } else if (L && L->getOpcode() == Instruction::Shl) {
8168 // X = Shl A, n
8169 // Y = AShr X, m
8170 // Both n and m are constant.
8171
8172 const SCEV *ShlOp0SCEV = getSCEV(V: L->getOperand(i: 0));
8173 ShlAmtCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8174 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8175 }
8176
8177 if (AddTruncateExpr && ShlAmtCI) {
8178 // We can merge the two given cases into a single SCEV statement,
8179 // incase n = m, the mul expression will be 2^0, so it gets resolved to
8180 // a simpler case. The following code handles the two cases:
8181 //
8182 // 1) For a two-shift sext-inreg, i.e. n = m,
8183 // use sext(trunc(x)) as the SCEV expression.
8184 //
8185 // 2) When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
8186 // expression. We already checked that ShlAmt < BitWidth, so
8187 // the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
8188 // ShlAmt - AShrAmt < Amt.
8189 const APInt &ShlAmt = ShlAmtCI->getValue();
8190 if (ShlAmt.ult(RHS: BitWidth) && ShlAmt.uge(RHS: AShrAmt)) {
8191 APInt Mul = APInt::getOneBitSet(numBits: BitWidth - AShrAmt,
8192 BitNo: ShlAmtCI->getZExtValue() - AShrAmt);
8193 const SCEV *CompositeExpr =
8194 getMulExpr(LHS: AddTruncateExpr, RHS: getConstant(Val: Mul));
8195 if (L->getOpcode() != Instruction::Shl)
8196 CompositeExpr = getAddExpr(LHS: CompositeExpr, RHS: AddConstant);
8197
8198 return getSignExtendExpr(Op: CompositeExpr, Ty: OuterTy);
8199 }
8200 }
8201 break;
8202 }
8203 }
8204
8205 switch (U->getOpcode()) {
8206 case Instruction::Trunc:
8207 return getTruncateExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8208
8209 case Instruction::ZExt:
8210 return getZeroExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8211
8212 case Instruction::SExt:
8213 if (auto BO = MatchBinaryOp(V: U->getOperand(i: 0), DL: getDataLayout(), AC, DT,
8214 CtxI: dyn_cast<Instruction>(Val: V))) {
8215 // The NSW flag of a subtract does not always survive the conversion to
8216 // A + (-1)*B. By pushing sign extension onto its operands we are much
8217 // more likely to preserve NSW and allow later AddRec optimisations.
8218 //
8219 // NOTE: This is effectively duplicating this logic from getSignExtend:
8220 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
8221 // but by that point the NSW information has potentially been lost.
8222 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
8223 Type *Ty = U->getType();
8224 auto *V1 = getSignExtendExpr(Op: getSCEV(V: BO->LHS), Ty);
8225 auto *V2 = getSignExtendExpr(Op: getSCEV(V: BO->RHS), Ty);
8226 return getMinusSCEV(LHS: V1, RHS: V2, Flags: SCEV::FlagNSW);
8227 }
8228 }
8229 return getSignExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8230
8231 case Instruction::BitCast:
8232 // BitCasts are no-op casts so we just eliminate the cast.
8233 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType()))
8234 return getSCEV(V: U->getOperand(i: 0));
8235 break;
8236
8237 case Instruction::PtrToAddr: {
8238 const SCEV *IntOp = getPtrToAddrExpr(Op: getSCEV(V: U->getOperand(i: 0)));
8239 if (isa<SCEVCouldNotCompute>(Val: IntOp))
8240 return getUnknown(V);
8241 return IntOp;
8242 }
8243
8244 case Instruction::PtrToInt:
8245 // SCEV only models ptrtoaddr.
8246 return getUnknown(V);
8247
8248 case Instruction::IntToPtr:
8249 // Just don't deal with inttoptr casts.
8250 return getUnknown(V);
8251
8252 case Instruction::SDiv:
8253 // If both operands are non-negative, this is just an udiv.
8254 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8255 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8256 return getUDivExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8257 break;
8258
8259 case Instruction::SRem:
8260 // If both operands are non-negative, this is just an urem.
8261 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8262 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8263 return getURemExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8264 break;
8265
8266 case Instruction::GetElementPtr:
8267 return createNodeForGEP(GEP: cast<GEPOperator>(Val: U));
8268
8269 case Instruction::PHI:
8270 return createNodeForPHI(PN: cast<PHINode>(Val: U));
8271
8272 case Instruction::Select:
8273 return createNodeForSelectOrPHI(V: U, Cond: U->getOperand(i: 0), TrueVal: U->getOperand(i: 1),
8274 FalseVal: U->getOperand(i: 2));
8275
8276 case Instruction::Call:
8277 case Instruction::Invoke:
8278 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand())
8279 return getSCEV(V: RV);
8280
8281 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
8282 switch (II->getIntrinsicID()) {
8283 case Intrinsic::abs:
8284 return getAbsExpr(
8285 Op: getSCEV(V: II->getArgOperand(i: 0)),
8286 /*IsNSW=*/cast<ConstantInt>(Val: II->getArgOperand(i: 1))->isOne());
8287 case Intrinsic::umax:
8288 LHS = getSCEV(V: II->getArgOperand(i: 0));
8289 RHS = getSCEV(V: II->getArgOperand(i: 1));
8290 return getUMaxExpr(LHS, RHS);
8291 case Intrinsic::umin:
8292 LHS = getSCEV(V: II->getArgOperand(i: 0));
8293 RHS = getSCEV(V: II->getArgOperand(i: 1));
8294 return getUMinExpr(LHS, RHS);
8295 case Intrinsic::smax:
8296 LHS = getSCEV(V: II->getArgOperand(i: 0));
8297 RHS = getSCEV(V: II->getArgOperand(i: 1));
8298 return getSMaxExpr(LHS, RHS);
8299 case Intrinsic::smin:
8300 LHS = getSCEV(V: II->getArgOperand(i: 0));
8301 RHS = getSCEV(V: II->getArgOperand(i: 1));
8302 return getSMinExpr(LHS, RHS);
8303 case Intrinsic::usub_sat: {
8304 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8305 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8306 const SCEV *ClampedY = getUMinExpr(LHS: X, RHS: Y);
8307 return getMinusSCEV(LHS: X, RHS: ClampedY, Flags: SCEV::FlagNUW);
8308 }
8309 case Intrinsic::uadd_sat: {
8310 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8311 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8312 const SCEV *ClampedX = getUMinExpr(LHS: X, RHS: getNotSCEV(V: Y));
8313 return getAddExpr(LHS: ClampedX, RHS: Y, Flags: SCEV::FlagNUW);
8314 }
8315 case Intrinsic::start_loop_iterations:
8316 case Intrinsic::annotation:
8317 case Intrinsic::ptr_annotation:
8318 // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is
8319 // just eqivalent to the first operand for SCEV purposes.
8320 return getSCEV(V: II->getArgOperand(i: 0));
8321 case Intrinsic::vscale:
8322 return getVScale(Ty: II->getType());
8323 default:
8324 break;
8325 }
8326 }
8327 break;
8328 }
8329
8330 return getUnknown(V);
8331}
8332
8333//===----------------------------------------------------------------------===//
8334// Iteration Count Computation Code
8335//
8336
8337const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount) {
8338 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8339 return getCouldNotCompute();
8340
8341 auto *ExitCountType = ExitCount->getType();
8342 assert(ExitCountType->isIntegerTy());
8343 auto *EvalTy = Type::getIntNTy(C&: ExitCountType->getContext(),
8344 N: 1 + ExitCountType->getScalarSizeInBits());
8345 return getTripCountFromExitCount(ExitCount, EvalTy, L: nullptr);
8346}
8347
8348const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
8349 Type *EvalTy,
8350 const Loop *L) {
8351 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8352 return getCouldNotCompute();
8353
8354 unsigned ExitCountSize = getTypeSizeInBits(Ty: ExitCount->getType());
8355 unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
8356
8357 auto CanAddOneWithoutOverflow = [&]() {
8358 ConstantRange ExitCountRange =
8359 getRangeRef(S: ExitCount, SignHint: RangeSignHint::HINT_RANGE_UNSIGNED);
8360 if (!ExitCountRange.contains(Val: APInt::getMaxValue(numBits: ExitCountSize)))
8361 return true;
8362
8363 return L && isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: ExitCount,
8364 RHS: getMinusOne(Ty: ExitCount->getType()));
8365 };
8366
8367 // If we need to zero extend the backedge count, check if we can add one to
8368 // it prior to zero extending without overflow. Provided this is safe, it
8369 // allows better simplification of the +1.
8370 if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
8371 return getZeroExtendExpr(
8372 Op: getAddExpr(LHS: ExitCount, RHS: getOne(Ty: ExitCount->getType())), Ty: EvalTy);
8373
8374 // Get the total trip count from the count by adding 1. This may wrap.
8375 return getAddExpr(LHS: getTruncateOrZeroExtend(V: ExitCount, Ty: EvalTy), RHS: getOne(Ty: EvalTy));
8376}
8377
8378static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
8379 if (!ExitCount)
8380 return 0;
8381
8382 ConstantInt *ExitConst = ExitCount->getValue();
8383
8384 // Guard against huge trip counts.
8385 if (ExitConst->getValue().getActiveBits() > 32)
8386 return 0;
8387
8388 // In case of integer overflow, this returns 0, which is correct.
8389 return ((unsigned)ExitConst->getZExtValue()) + 1;
8390}
8391
8392unsigned ScalarEvolution::getSmallConstantTripCount(const Loop *L) {
8393 auto *ExitCount = dyn_cast<SCEVConstant>(Val: getBackedgeTakenCount(L, Kind: Exact));
8394 return getConstantTripCount(ExitCount);
8395}
8396
8397unsigned
8398ScalarEvolution::getSmallConstantTripCount(const Loop *L,
8399 const BasicBlock *ExitingBlock) {
8400 assert(ExitingBlock && "Must pass a non-null exiting block!");
8401 assert(L->isLoopExiting(ExitingBlock) &&
8402 "Exiting block must actually branch out of the loop!");
8403 const SCEVConstant *ExitCount =
8404 dyn_cast<SCEVConstant>(Val: getExitCount(L, ExitingBlock));
8405 return getConstantTripCount(ExitCount);
8406}
8407
8408unsigned ScalarEvolution::getSmallConstantMaxTripCount(
8409 const Loop *L, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8410
8411 const auto *MaxExitCount =
8412 Predicates ? getPredicatedConstantMaxBackedgeTakenCount(L, Predicates&: *Predicates)
8413 : getConstantMaxBackedgeTakenCount(L);
8414 return getConstantTripCount(ExitCount: dyn_cast<SCEVConstant>(Val: MaxExitCount));
8415}
8416
8417unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L) {
8418 SmallVector<BasicBlock *, 8> ExitingBlocks;
8419 L->getExitingBlocks(ExitingBlocks);
8420
8421 // An exit with an uncomputable exit count makes the result 1.
8422 if (ExitingBlocks.empty() ||
8423 any_of(Range&: ExitingBlocks, P: [this, L](BasicBlock *ExitingBB) {
8424 return isa<SCEVCouldNotCompute>(Val: getExitCount(L, ExitingBlock: ExitingBB));
8425 }))
8426 return 1;
8427
8428 LoopGuards Guards = LoopGuards::collect(L, SE&: *this);
8429 unsigned Res = 0;
8430 for (BasicBlock *ExitingBB : ExitingBlocks)
8431 Res = std::gcd(
8432 m: Res, n: getSmallConstantTripMultiple(ExitCount: getExitCount(L, ExitingBlock: ExitingBB), Guards));
8433 return Res;
8434}
8435
8436unsigned
8437ScalarEvolution::getSmallConstantTripMultiple(const SCEV *ExitCount,
8438 const LoopGuards &Guards) {
8439 assert(!isa<SCEVCouldNotCompute>(ExitCount) && "Must be computable!");
8440
8441 // Get the trip count
8442 const SCEV *TCExpr =
8443 getTripCountFromExitCount(ExitCount: applyLoopGuards(Expr: ExitCount, Guards));
8444
8445 APInt Multiple = getNonZeroConstantMultiple(S: TCExpr);
8446 // If a trip multiple is huge (>=2^32), the trip count is still divisible by
8447 // the greatest power of 2 divisor less than 2^32.
8448 return Multiple.getActiveBits() > 32
8449 ? 1U << std::min(a: 31U, b: Multiple.countTrailingZeros())
8450 : (unsigned)Multiple.getZExtValue();
8451}
8452
8453unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8454 const SCEV *ExitCount) {
8455 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8456 return 1;
8457
8458 return getSmallConstantTripMultiple(ExitCount, Guards: LoopGuards::collect(L, SE&: *this));
8459}
8460
8461/// Returns the largest constant divisor of the trip count of this loop as a
8462/// normal unsigned value, if possible. This means that the actual trip count is
8463/// always a multiple of the returned value (don't forget the trip count could
8464/// very well be zero as well!).
8465///
8466/// Returns 1 if the trip count is unknown or not guaranteed to be the
8467/// multiple of a constant (which is also the case if the trip count is simply
8468/// constant, use getSmallConstantTripCount for that case), Will also return 1
8469/// if the trip count is very large (>= 2^32).
8470///
8471/// As explained in the comments for getSmallConstantTripCount, this assumes
8472/// that control exits the loop via ExitingBlock.
8473unsigned
8474ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8475 const BasicBlock *ExitingBlock) {
8476 assert(ExitingBlock && "Must pass a non-null exiting block!");
8477 assert(L->isLoopExiting(ExitingBlock) &&
8478 "Exiting block must actually branch out of the loop!");
8479 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
8480 return getSmallConstantTripMultiple(L, ExitCount);
8481}
8482
8483const SCEV *ScalarEvolution::getExitCount(const Loop *L,
8484 const BasicBlock *ExitingBlock,
8485 ExitCountKind Kind) {
8486 switch (Kind) {
8487 case Exact:
8488 return getBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this);
8489 case SymbolicMaximum:
8490 return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this);
8491 case ConstantMaximum:
8492 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this);
8493 };
8494 llvm_unreachable("Invalid ExitCountKind!");
8495}
8496
8497const SCEV *ScalarEvolution::getPredicatedExitCount(
8498 const Loop *L, const BasicBlock *ExitingBlock,
8499 SmallVectorImpl<const SCEVPredicate *> *Predicates, ExitCountKind Kind) {
8500 switch (Kind) {
8501 case Exact:
8502 return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this,
8503 Predicates);
8504 case SymbolicMaximum:
8505 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this,
8506 Predicates);
8507 case ConstantMaximum:
8508 return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this,
8509 Predicates);
8510 };
8511 llvm_unreachable("Invalid ExitCountKind!");
8512}
8513
8514const SCEV *ScalarEvolution::getPredicatedBackedgeTakenCount(
8515 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8516 return getPredicatedBackedgeTakenInfo(L).getExact(L, SE: this, Predicates: &Preds);
8517}
8518
8519const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
8520 ExitCountKind Kind) {
8521 switch (Kind) {
8522 case Exact:
8523 return getBackedgeTakenInfo(L).getExact(L, SE: this);
8524 case ConstantMaximum:
8525 return getBackedgeTakenInfo(L).getConstantMax(SE: this);
8526 case SymbolicMaximum:
8527 return getBackedgeTakenInfo(L).getSymbolicMax(L, SE: this);
8528 };
8529 llvm_unreachable("Invalid ExitCountKind!");
8530}
8531
8532const SCEV *ScalarEvolution::getPredicatedSymbolicMaxBackedgeTakenCount(
8533 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8534 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, SE: this, Predicates: &Preds);
8535}
8536
8537const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
8538 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8539 return getPredicatedBackedgeTakenInfo(L).getConstantMax(SE: this, Predicates: &Preds);
8540}
8541
8542bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
8543 return getBackedgeTakenInfo(L).isConstantMaxOrZero(SE: this);
8544}
8545
8546/// Push PHI nodes in the header of the given loop onto the given Worklist.
8547static void PushLoopPHIs(const Loop *L,
8548 SmallVectorImpl<Instruction *> &Worklist,
8549 SmallPtrSetImpl<Instruction *> &Visited) {
8550 BasicBlock *Header = L->getHeader();
8551
8552 // Push all Loop-header PHIs onto the Worklist stack.
8553 for (PHINode &PN : Header->phis())
8554 if (Visited.insert(Ptr: &PN).second)
8555 Worklist.push_back(Elt: &PN);
8556}
8557
8558ScalarEvolution::BackedgeTakenInfo &
8559ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
8560 auto &BTI = getBackedgeTakenInfo(L);
8561 if (BTI.hasFullInfo())
8562 return BTI;
8563
8564 auto Pair = PredicatedBackedgeTakenCounts.try_emplace(Key: L);
8565
8566 if (!Pair.second)
8567 return Pair.first->second;
8568
8569 BackedgeTakenInfo Result =
8570 computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
8571
8572 return PredicatedBackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8573}
8574
8575ScalarEvolution::BackedgeTakenInfo &
8576ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
8577 // Initially insert an invalid entry for this loop. If the insertion
8578 // succeeds, proceed to actually compute a backedge-taken count and
8579 // update the value. The temporary CouldNotCompute value tells SCEV
8580 // code elsewhere that it shouldn't attempt to request a new
8581 // backedge-taken count, which could result in infinite recursion.
8582 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
8583 BackedgeTakenCounts.try_emplace(Key: L);
8584 if (!Pair.second)
8585 return Pair.first->second;
8586
8587 // computeBackedgeTakenCount may allocate memory for its result. Inserting it
8588 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
8589 // must be cleared in this scope.
8590 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
8591
8592 // Now that we know more about the trip count for this loop, forget any
8593 // existing SCEV values for PHI nodes in this loop since they are only
8594 // conservative estimates made without the benefit of trip count
8595 // information. This invalidation is not necessary for correctness, and is
8596 // only done to produce more precise results.
8597 if (Result.hasAnyInfo()) {
8598 // Invalidate any expression using an addrec in this loop.
8599 SmallVector<SCEVUse, 8> ToForget;
8600 auto LoopUsersIt = LoopUsers.find(Val: L);
8601 if (LoopUsersIt != LoopUsers.end())
8602 append_range(C&: ToForget, R&: LoopUsersIt->second);
8603 forgetMemoizedResults(SCEVs: ToForget);
8604
8605 // Invalidate constant-evolved loop header phis.
8606 for (PHINode &PN : L->getHeader()->phis())
8607 ConstantEvolutionLoopExitValue.erase(Val: &PN);
8608 }
8609
8610 // Re-lookup the insert position, since the call to
8611 // computeBackedgeTakenCount above could result in a
8612 // recusive call to getBackedgeTakenInfo (on a different
8613 // loop), which would invalidate the iterator computed
8614 // earlier.
8615 return BackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8616}
8617
8618void ScalarEvolution::forgetAllLoops() {
8619 // This method is intended to forget all info about loops. It should
8620 // invalidate caches as if the following happened:
8621 // - The trip counts of all loops have changed arbitrarily
8622 // - Every llvm::Value has been updated in place to produce a different
8623 // result.
8624 BackedgeTakenCounts.clear();
8625 PredicatedBackedgeTakenCounts.clear();
8626 BECountUsers.clear();
8627 LoopPropertiesCache.clear();
8628 ConstantEvolutionLoopExitValue.clear();
8629 ValueExprMap.clear();
8630 ValuesAtScopes.clear();
8631 ValuesAtScopesUsers.clear();
8632 LoopDispositions.clear();
8633 BlockDispositions.clear();
8634 UnsignedRanges.clear();
8635 SignedRanges.clear();
8636 ExprValueMap.clear();
8637 HasRecMap.clear();
8638 ConstantMultipleCache.clear();
8639 PredicatedSCEVRewrites.clear();
8640 FoldCache.clear();
8641 FoldCacheUser.clear();
8642}
8643void ScalarEvolution::visitAndClearUsers(
8644 SmallVectorImpl<Instruction *> &Worklist,
8645 SmallPtrSetImpl<Instruction *> &Visited,
8646 SmallVectorImpl<SCEVUse> &ToForget) {
8647 // Nothing can be invalidated if no value has a SCEV yet.
8648 if (ValueExprMap.empty()) {
8649 Worklist.clear();
8650 return;
8651 }
8652 while (!Worklist.empty()) {
8653 Instruction *I = Worklist.pop_back_val();
8654 if (!isSCEVable(Ty: I->getType()) && !isa<WithOverflowInst>(Val: I))
8655 continue;
8656
8657 ValueExprMapType::iterator It =
8658 ValueExprMap.find_as(Val: static_cast<Value *>(I));
8659 if (It != ValueExprMap.end()) {
8660 ToForget.push_back(Elt: It->second);
8661 eraseValueFromMap(V: It->first);
8662 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
8663 ConstantEvolutionLoopExitValue.erase(Val: PN);
8664 }
8665
8666 PushDefUseChildren(I, Worklist, Visited);
8667 }
8668}
8669
8670void ScalarEvolution::forgetLoop(const Loop *L) {
8671 SmallVector<const Loop *, 16> LoopWorklist(1, L);
8672 SmallVector<Instruction *, 32> Worklist;
8673 SmallPtrSet<Instruction *, 16> Visited;
8674 SmallVector<SCEVUse, 16> ToForget;
8675
8676 // Iterate over all the loops and sub-loops to drop SCEV information.
8677 while (!LoopWorklist.empty()) {
8678 auto *CurrL = LoopWorklist.pop_back_val();
8679
8680 // Drop any stored trip count value.
8681 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ false);
8682 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ true);
8683
8684 // Drop information about predicated SCEV rewrites for this loop.
8685 PredicatedSCEVRewrites.remove_if(
8686 Pred: [&](const auto &Entry) { return Entry.first.second == CurrL; });
8687
8688 auto LoopUsersItr = LoopUsers.find(Val: CurrL);
8689 if (LoopUsersItr != LoopUsers.end())
8690 llvm::append_range(C&: ToForget, R&: LoopUsersItr->second);
8691
8692 // Drop information about expressions based on loop-header PHIs.
8693 PushLoopPHIs(L: CurrL, Worklist, Visited);
8694 visitAndClearUsers(Worklist, Visited, ToForget);
8695
8696 LoopPropertiesCache.erase(Val: CurrL);
8697 // Forget all contained loops too, to avoid dangling entries in the
8698 // ValuesAtScopes map.
8699 LoopWorklist.append(in_start: CurrL->begin(), in_end: CurrL->end());
8700 }
8701 forgetMemoizedResults(SCEVs: ToForget);
8702}
8703
8704void ScalarEvolution::forgetTopmostLoop(const Loop *L) {
8705 forgetLoop(L: L->getOutermostLoop());
8706}
8707
8708void ScalarEvolution::forgetValue(Value *V) {
8709 Instruction *I = dyn_cast<Instruction>(Val: V);
8710 if (!I) return;
8711
8712 // Drop information about expressions based on loop-header PHIs.
8713 SmallVector<Instruction *, 16> Worklist;
8714 SmallPtrSet<Instruction *, 8> Visited;
8715 SmallVector<SCEVUse, 8> ToForget;
8716 Worklist.push_back(Elt: I);
8717 Visited.insert(Ptr: I);
8718 visitAndClearUsers(Worklist, Visited, ToForget);
8719
8720 forgetMemoizedResults(SCEVs: ToForget);
8721}
8722
8723void ScalarEvolution::forgetValues(ArrayRef<Value *> Values) {
8724 SmallVector<Instruction *, 16> Worklist;
8725 SmallPtrSet<Instruction *, 8> Visited;
8726 SmallVector<SCEVUse, 8> ToForget;
8727 for (Value *V : Values)
8728 if (auto *I = dyn_cast<Instruction>(Val: V))
8729 if (Visited.insert(Ptr: I).second)
8730 Worklist.push_back(Elt: I);
8731 visitAndClearUsers(Worklist, Visited, ToForget);
8732
8733 forgetMemoizedResults(SCEVs: ToForget);
8734}
8735
8736void ScalarEvolution::forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V) {
8737 // If SCEV looked through a trivial LCSSA phi node, we might have SCEV's
8738 // directly using a SCEVUnknown/SCEVAddRec defined in the loop. After an
8739 // extra predecessor is added, this is no longer valid. Find all Unknowns and
8740 // AddRecs defined in the loop and invalidate any SCEV's making use of them.
8741 auto InvalidateValue = [&](Value *Val) {
8742 if (!isSCEVable(Ty: Val->getType()))
8743 return;
8744 if (const SCEV *S = getExistingSCEV(V: Val)) {
8745 struct InvalidationRootCollector {
8746 Loop *L;
8747 SmallVector<SCEVUse, 8> Roots;
8748
8749 InvalidationRootCollector(Loop *L) : L(L) {}
8750
8751 bool follow(const SCEV *S) {
8752 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
8753 if (auto *I = dyn_cast<Instruction>(Val: SU->getValue()))
8754 if (L->contains(Inst: I))
8755 Roots.push_back(Elt: S);
8756 } else if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S)) {
8757 if (L->contains(L: AddRec->getLoop()))
8758 Roots.push_back(Elt: S);
8759 }
8760 return true;
8761 }
8762 bool isDone() const { return false; }
8763 };
8764
8765 InvalidationRootCollector C(L);
8766 visitAll(Root: S, Visitor&: C);
8767 forgetMemoizedResults(SCEVs: C.Roots);
8768 }
8769 };
8770
8771 InvalidateValue(V);
8772
8773 // If V has a non-SCEV-able type (e.g. {i64, i1} from a with.overflow
8774 // intrinsic), its users (e.g. extractvalue) may have stale SCEV
8775 // expressions referencing loop-internal values.
8776 if (!isSCEVable(Ty: V->getType()) &&
8777 any_of(Range: V->incoming_values(), P: IsaPred<WithOverflowInst>))
8778 for (User *U : V->users())
8779 InvalidateValue(U);
8780 // Also perform the normal invalidation.
8781 forgetValue(V);
8782}
8783
8784void ScalarEvolution::forgetLoopDispositions() { LoopDispositions.clear(); }
8785
8786void ScalarEvolution::forgetBlockAndLoopDispositions(Value *V) {
8787 // Unless a specific value is passed to invalidation, completely clear both
8788 // caches.
8789 if (!V) {
8790 BlockDispositions.clear();
8791 LoopDispositions.clear();
8792 return;
8793 }
8794
8795 if (!isSCEVable(Ty: V->getType()))
8796 return;
8797
8798 const SCEV *S = getExistingSCEV(V);
8799 if (!S)
8800 return;
8801
8802 // Invalidate the block and loop dispositions cached for S. Dispositions of
8803 // S's users may change if S's disposition changes (i.e. a user may change to
8804 // loop-invariant, if S changes to loop invariant), so also invalidate
8805 // dispositions of S's users recursively.
8806 SmallVector<SCEVUse, 8> Worklist = {S};
8807 SmallPtrSet<const SCEV *, 8> Seen = {S};
8808 while (!Worklist.empty()) {
8809 const SCEV *Curr = Worklist.pop_back_val();
8810 bool LoopDispoRemoved = LoopDispositions.erase(Val: Curr);
8811 bool BlockDispoRemoved = BlockDispositions.erase(Val: Curr);
8812 if (!LoopDispoRemoved && !BlockDispoRemoved)
8813 continue;
8814 auto Users = SCEVUsers.find(Val: Curr);
8815 if (Users != SCEVUsers.end())
8816 for (const auto *User : Users->second)
8817 if (Seen.insert(Ptr: User).second)
8818 Worklist.push_back(Elt: User);
8819 }
8820}
8821
8822/// Get the exact loop backedge taken count considering all loop exits. A
8823/// computable result can only be returned for loops with all exiting blocks
8824/// dominating the latch. howFarToZero assumes that the limit of each loop test
8825/// is never skipped. This is a valid assumption as long as the loop exits via
8826/// that test. For precise results, it is the caller's responsibility to specify
8827/// the relevant loop exiting block using getExact(ExitingBlock, SE).
8828const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
8829 const Loop *L, ScalarEvolution *SE,
8830 SmallVectorImpl<const SCEVPredicate *> *Preds) const {
8831 // If any exits were not computable, the loop is not computable.
8832 if (!isComplete() || ExitNotTaken.empty())
8833 return SE->getCouldNotCompute();
8834
8835 const BasicBlock *Latch = L->getLoopLatch();
8836 // All exiting blocks we have collected must dominate the only backedge.
8837 if (!Latch)
8838 return SE->getCouldNotCompute();
8839
8840 // All exiting blocks we have gathered dominate loop's latch, so exact trip
8841 // count is simply a minimum out of all these calculated exit counts.
8842 SmallVector<SCEVUse, 2> Ops;
8843 for (const auto &ENT : ExitNotTaken) {
8844 const SCEV *BECount = ENT.ExactNotTaken;
8845 assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!");
8846 assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&
8847 "We should only have known counts for exiting blocks that dominate "
8848 "latch!");
8849
8850 Ops.push_back(Elt: BECount);
8851
8852 if (Preds)
8853 append_range(C&: *Preds, R: ENT.Predicates);
8854
8855 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
8856 "Predicate should be always true!");
8857 }
8858
8859 // If an earlier exit exits on the first iteration (exit count zero), then
8860 // a later poison exit count should not propagate into the result. This are
8861 // exactly the semantics provided by umin_seq.
8862 return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true);
8863}
8864
8865const ScalarEvolution::ExitNotTakenInfo *
8866ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
8867 const BasicBlock *ExitingBlock,
8868 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8869 for (const auto &ENT : ExitNotTaken)
8870 if (ENT.ExitingBlock == ExitingBlock) {
8871 if (ENT.hasAlwaysTruePredicate())
8872 return &ENT;
8873 else if (Predicates) {
8874 append_range(C&: *Predicates, R: ENT.Predicates);
8875 return &ENT;
8876 }
8877 }
8878
8879 return nullptr;
8880}
8881
8882/// getConstantMax - Get the constant max backedge taken count for the loop.
8883const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
8884 ScalarEvolution *SE,
8885 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8886 if (!getConstantMax())
8887 return SE->getCouldNotCompute();
8888
8889 for (const auto &ENT : ExitNotTaken)
8890 if (!ENT.hasAlwaysTruePredicate()) {
8891 if (!Predicates)
8892 return SE->getCouldNotCompute();
8893 append_range(C&: *Predicates, R: ENT.Predicates);
8894 }
8895
8896 assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||
8897 isa<SCEVConstant>(getConstantMax())) &&
8898 "No point in having a non-constant max backedge taken count!");
8899 return getConstantMax();
8900}
8901
8902const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
8903 const Loop *L, ScalarEvolution *SE,
8904 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8905 if (!SymbolicMax) {
8906 // Form an expression for the maximum exit count possible for this loop. We
8907 // merge the max and exact information to approximate a version of
8908 // getConstantMaxBackedgeTakenCount which isn't restricted to just
8909 // constants.
8910 SmallVector<SCEVUse, 4> ExitCounts;
8911
8912 for (const auto &ENT : ExitNotTaken) {
8913 const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
8914 if (!isa<SCEVCouldNotCompute>(Val: ExitCount)) {
8915 assert(SE->DT.dominates(ENT.ExitingBlock, L->getLoopLatch()) &&
8916 "We should only have known counts for exiting blocks that "
8917 "dominate latch!");
8918 ExitCounts.push_back(Elt: ExitCount);
8919 if (Predicates)
8920 append_range(C&: *Predicates, R: ENT.Predicates);
8921
8922 assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
8923 "Predicate should be always true!");
8924 }
8925 }
8926 if (ExitCounts.empty())
8927 SymbolicMax = SE->getCouldNotCompute();
8928 else
8929 SymbolicMax =
8930 SE->getUMinFromMismatchedTypes(Ops&: ExitCounts, /*Sequential*/ true);
8931 }
8932 return SymbolicMax;
8933}
8934
8935bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
8936 ScalarEvolution *SE) const {
8937 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
8938 return !ENT.hasAlwaysTruePredicate();
8939 };
8940 return MaxOrZero && !any_of(Range: ExitNotTaken, P: PredicateNotAlwaysTrue);
8941}
8942
8943ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E)
8944 : ExitLimit(E, E, E, false) {}
8945
8946ScalarEvolution::ExitLimit::ExitLimit(
8947 const SCEV *E, const SCEV *ConstantMaxNotTaken,
8948 const SCEV *SymbolicMaxNotTaken, bool MaxOrZero,
8949 ArrayRef<ArrayRef<const SCEVPredicate *>> PredLists)
8950 : ExactNotTaken(E), ConstantMaxNotTaken(ConstantMaxNotTaken),
8951 SymbolicMaxNotTaken(SymbolicMaxNotTaken), MaxOrZero(MaxOrZero) {
8952 // If we prove the max count is zero, so is the symbolic bound. This happens
8953 // in practice due to differences in a) how context sensitive we've chosen
8954 // to be and b) how we reason about bounds implied by UB.
8955 if (ConstantMaxNotTaken->isZero()) {
8956 this->ExactNotTaken = E = ConstantMaxNotTaken;
8957 this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
8958 }
8959
8960 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
8961 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
8962 "Exact is not allowed to be less precise than Constant Max");
8963 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
8964 !isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken)) &&
8965 "Exact is not allowed to be less precise than Symbolic Max");
8966 assert((isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken) ||
8967 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
8968 "Symbolic Max is not allowed to be less precise than Constant Max");
8969 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
8970 isa<SCEVConstant>(ConstantMaxNotTaken)) &&
8971 "No point in having a non-constant max backedge taken count!");
8972 SmallPtrSet<const SCEVPredicate *, 4> SeenPreds;
8973 for (const auto PredList : PredLists)
8974 for (const auto *P : PredList) {
8975 if (SeenPreds.contains(Ptr: P))
8976 continue;
8977 assert(!isa<SCEVUnionPredicate>(P) && "Only add leaf predicates here!");
8978 SeenPreds.insert(Ptr: P);
8979 Predicates.push_back(Elt: P);
8980 }
8981 assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&
8982 "Backedge count should be int");
8983 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
8984 !ConstantMaxNotTaken->getType()->isPointerTy()) &&
8985 "Max backedge count should be int");
8986}
8987
8988ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E,
8989 const SCEV *ConstantMaxNotTaken,
8990 const SCEV *SymbolicMaxNotTaken,
8991 bool MaxOrZero,
8992 ArrayRef<const SCEVPredicate *> PredList)
8993 : ExitLimit(E, ConstantMaxNotTaken, SymbolicMaxNotTaken, MaxOrZero,
8994 ArrayRef({PredList})) {}
8995
8996/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
8997/// computable exit into a persistent ExitNotTakenInfo array.
8998ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
8999 ArrayRef<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo> ExitCounts,
9000 bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
9001 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
9002 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
9003
9004 ExitNotTaken.reserve(N: ExitCounts.size());
9005 std::transform(first: ExitCounts.begin(), last: ExitCounts.end(),
9006 result: std::back_inserter(x&: ExitNotTaken),
9007 unary_op: [&](const EdgeExitInfo &EEI) {
9008 BasicBlock *ExitBB = EEI.first;
9009 const ExitLimit &EL = EEI.second;
9010 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
9011 EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
9012 EL.Predicates);
9013 });
9014 assert((isa<SCEVCouldNotCompute>(ConstantMax) ||
9015 isa<SCEVConstant>(ConstantMax)) &&
9016 "No point in having a non-constant max backedge taken count!");
9017}
9018
9019/// Compute the number of times the backedge of the specified loop will execute.
9020ScalarEvolution::BackedgeTakenInfo
9021ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
9022 bool AllowPredicates) {
9023 SmallVector<BasicBlock *, 8> ExitingBlocks;
9024 L->getExitingBlocks(ExitingBlocks);
9025
9026 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
9027
9028 SmallVector<EdgeExitInfo, 4> ExitCounts;
9029 bool CouldComputeBECount = true;
9030 BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
9031 const SCEV *MustExitMaxBECount = nullptr;
9032 const SCEV *MayExitMaxBECount = nullptr;
9033 bool MustExitMaxOrZero = false;
9034 bool IsOnlyExit = ExitingBlocks.size() == 1;
9035
9036 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
9037 // and compute maxBECount.
9038 // Do a union of all the predicates here.
9039 for (BasicBlock *ExitBB : ExitingBlocks) {
9040 // We canonicalize untaken exits to br (constant), ignore them so that
9041 // proving an exit untaken doesn't negatively impact our ability to reason
9042 // about the loop as whole.
9043 if (auto *BI = dyn_cast<CondBrInst>(Val: ExitBB->getTerminator()))
9044 if (auto *CI = dyn_cast<ConstantInt>(Val: BI->getCondition())) {
9045 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9046 if (ExitIfTrue == CI->isZero())
9047 continue;
9048 }
9049
9050 ExitLimit EL = computeExitLimit(L, ExitingBlock: ExitBB, IsOnlyExit, AllowPredicates);
9051
9052 assert((AllowPredicates || EL.Predicates.empty()) &&
9053 "Predicated exit limit when predicates are not allowed!");
9054
9055 // 1. For each exit that can be computed, add an entry to ExitCounts.
9056 // CouldComputeBECount is true only if all exits can be computed.
9057 if (EL.ExactNotTaken != getCouldNotCompute())
9058 ++NumExitCountsComputed;
9059 else
9060 // We couldn't compute an exact value for this exit, so
9061 // we won't be able to compute an exact value for the loop.
9062 CouldComputeBECount = false;
9063 // Remember exit count if either exact or symbolic is known. Because
9064 // Exact always implies symbolic, only check symbolic.
9065 if (EL.SymbolicMaxNotTaken != getCouldNotCompute())
9066 ExitCounts.emplace_back(Args&: ExitBB, Args&: EL);
9067 else {
9068 assert(EL.ExactNotTaken == getCouldNotCompute() &&
9069 "Exact is known but symbolic isn't?");
9070 ++NumExitCountsNotComputed;
9071 }
9072
9073 // 2. Derive the loop's MaxBECount from each exit's max number of
9074 // non-exiting iterations. Partition the loop exits into two kinds:
9075 // LoopMustExits and LoopMayExits.
9076 //
9077 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
9078 // is a LoopMayExit. If any computable LoopMustExit is found, then
9079 // MaxBECount is the minimum EL.ConstantMaxNotTaken of computable
9080 // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
9081 // EL.ConstantMaxNotTaken, where CouldNotCompute is considered greater than
9082 // any
9083 // computable EL.ConstantMaxNotTaken.
9084 if (EL.ConstantMaxNotTaken != getCouldNotCompute() && Latch &&
9085 DT.dominates(A: ExitBB, B: Latch)) {
9086 if (!MustExitMaxBECount) {
9087 MustExitMaxBECount = EL.ConstantMaxNotTaken;
9088 MustExitMaxOrZero = EL.MaxOrZero;
9089 } else {
9090 MustExitMaxBECount = getUMinFromMismatchedTypes(LHS: MustExitMaxBECount,
9091 RHS: EL.ConstantMaxNotTaken);
9092 }
9093 } else if (MayExitMaxBECount != getCouldNotCompute()) {
9094 if (!MayExitMaxBECount || EL.ConstantMaxNotTaken == getCouldNotCompute())
9095 MayExitMaxBECount = EL.ConstantMaxNotTaken;
9096 else {
9097 MayExitMaxBECount = getUMaxFromMismatchedTypes(LHS: MayExitMaxBECount,
9098 RHS: EL.ConstantMaxNotTaken);
9099 }
9100 }
9101 }
9102 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
9103 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
9104 // The loop backedge will be taken the maximum or zero times if there's
9105 // a single exit that must be taken the maximum or zero times.
9106 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
9107
9108 // Remember which SCEVs are used in exit limits for invalidation purposes.
9109 // We only care about non-constant SCEVs here, so we can ignore
9110 // EL.ConstantMaxNotTaken
9111 // and MaxBECount, which must be SCEVConstant.
9112 for (const auto &Pair : ExitCounts) {
9113 if (!isa<SCEVConstant>(Val: Pair.second.ExactNotTaken))
9114 BECountUsers[Pair.second.ExactNotTaken].insert(Ptr: {L, AllowPredicates});
9115 if (!isa<SCEVConstant>(Val: Pair.second.SymbolicMaxNotTaken))
9116 BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
9117 Ptr: {L, AllowPredicates});
9118 }
9119 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
9120 MaxBECount, MaxOrZero);
9121}
9122
9123ScalarEvolution::ExitLimit
9124ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
9125 bool IsOnlyExit, bool AllowPredicates) {
9126 assert(L->contains(ExitingBlock) && "Exit count for non-loop block?");
9127 // If our exiting block does not dominate the latch, then its connection with
9128 // loop's exit limit may be far from trivial.
9129 const BasicBlock *Latch = L->getLoopLatch();
9130 if (!Latch || !DT.dominates(A: ExitingBlock, B: Latch))
9131 return getCouldNotCompute();
9132
9133 Instruction *Term = ExitingBlock->getTerminator();
9134 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Term)) {
9135 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9136 assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&
9137 "It should have one successor in loop and one exit block!");
9138 // Proceed to the next level to examine the exit condition expression.
9139 return computeExitLimitFromCond(L, ExitCond: BI->getCondition(), ExitIfTrue,
9140 /*ControlsOnlyExit=*/IsOnlyExit,
9141 AllowPredicates);
9142 }
9143
9144 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: Term)) {
9145 // For switch, make sure that there is a single exit from the loop.
9146 BasicBlock *Exit = nullptr;
9147 for (auto *SBB : successors(BB: ExitingBlock))
9148 if (!L->contains(BB: SBB)) {
9149 if (Exit) // Multiple exit successors.
9150 return getCouldNotCompute();
9151 Exit = SBB;
9152 }
9153 assert(Exit && "Exiting block must have at least one exit");
9154 return computeExitLimitFromSingleExitSwitch(
9155 L, Switch: SI, ExitingBB: Exit, /*ControlsOnlyExit=*/IsSubExpr: IsOnlyExit);
9156 }
9157
9158 return getCouldNotCompute();
9159}
9160
9161ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCond(
9162 const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9163 bool AllowPredicates) {
9164 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
9165 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
9166 ControlsOnlyExit, AllowPredicates);
9167}
9168
9169std::optional<ScalarEvolution::ExitLimit>
9170ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
9171 bool ExitIfTrue, bool ControlsOnlyExit,
9172 bool AllowPredicates) {
9173 (void)this->L;
9174 (void)this->ExitIfTrue;
9175 (void)this->AllowPredicates;
9176
9177 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9178 this->AllowPredicates == AllowPredicates &&
9179 "Variance in assumed invariant key components!");
9180 auto Itr = TripCountMap.find(Val: {ExitCond, ControlsOnlyExit});
9181 if (Itr == TripCountMap.end())
9182 return std::nullopt;
9183 return Itr->second;
9184}
9185
9186void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond,
9187 bool ExitIfTrue,
9188 bool ControlsOnlyExit,
9189 bool AllowPredicates,
9190 const ExitLimit &EL) {
9191 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9192 this->AllowPredicates == AllowPredicates &&
9193 "Variance in assumed invariant key components!");
9194
9195 auto InsertResult = TripCountMap.insert(KV: {{ExitCond, ControlsOnlyExit}, EL});
9196 assert(InsertResult.second && "Expected successful insertion!");
9197 (void)InsertResult;
9198 (void)ExitIfTrue;
9199}
9200
9201ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
9202 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9203 bool ControlsOnlyExit, bool AllowPredicates) {
9204
9205 if (auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit,
9206 AllowPredicates))
9207 return *MaybeEL;
9208
9209 ExitLimit EL = computeExitLimitFromCondImpl(
9210 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates);
9211 Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL);
9212 return EL;
9213}
9214
9215ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
9216 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9217 bool ControlsOnlyExit, bool AllowPredicates) {
9218 // Handle BinOp conditions (And, Or).
9219 if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
9220 Cache, L, ExitCond, ExitIfTrue, AllowPredicates))
9221 return *LimitFromBinOp;
9222
9223 // With an icmp, it may be feasible to compute an exact backedge-taken count.
9224 // Proceed to the next level to examine the icmp.
9225 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(Val: ExitCond)) {
9226 ExitLimit EL =
9227 computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue, IsSubExpr: ControlsOnlyExit);
9228 if (EL.hasFullInfo() || !AllowPredicates)
9229 return EL;
9230
9231 // Try again, but use SCEV predicates this time.
9232 return computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue,
9233 IsSubExpr: ControlsOnlyExit,
9234 /*AllowPredicates=*/true);
9235 }
9236
9237 // Check for a constant condition. These are normally stripped out by
9238 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
9239 // preserve the CFG and is temporarily leaving constant conditions
9240 // in place.
9241 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: ExitCond)) {
9242 if (ExitIfTrue == !CI->getZExtValue())
9243 // The backedge is always taken.
9244 return getCouldNotCompute();
9245 // The backedge is never taken.
9246 return getZero(Ty: CI->getType());
9247 }
9248
9249 // If we're exiting based on the overflow flag of an x.with.overflow intrinsic
9250 // with a constant step, we can form an equivalent icmp predicate and figure
9251 // out how many iterations will be taken before we exit.
9252 const WithOverflowInst *WO;
9253 const APInt *C;
9254 if (match(V: ExitCond, P: m_ExtractValue<1>(V: m_WithOverflowInst(I&: WO))) &&
9255 match(V: WO->getRHS(), P: m_APInt(Res&: C))) {
9256 ConstantRange NWR =
9257 ConstantRange::makeExactNoWrapRegion(BinOp: WO->getBinaryOp(), Other: *C,
9258 NoWrapKind: WO->getNoWrapKind());
9259 CmpInst::Predicate Pred;
9260 APInt NewRHSC, Offset;
9261 NWR.getEquivalentICmp(Pred, RHS&: NewRHSC, Offset);
9262 if (!ExitIfTrue)
9263 Pred = ICmpInst::getInversePredicate(pred: Pred);
9264 auto *LHS = getSCEV(V: WO->getLHS());
9265 if (Offset != 0)
9266 LHS = getAddExpr(LHS, RHS: getConstant(Val: Offset));
9267 auto EL = computeExitLimitFromICmp(L, Pred, LHS, RHS: getConstant(Val: NewRHSC),
9268 IsSubExpr: ControlsOnlyExit, AllowPredicates);
9269 if (EL.hasAnyInfo())
9270 return EL;
9271 }
9272
9273 // If it's not an integer or pointer comparison then compute it the hard way.
9274 return computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9275}
9276
9277std::optional<ScalarEvolution::ExitLimit>
9278ScalarEvolution::computeExitLimitFromCondFromBinOp(ExitLimitCacheTy &Cache,
9279 const Loop *L,
9280 Value *ExitCond,
9281 bool ExitIfTrue,
9282 bool AllowPredicates) {
9283 // Check if the controlling expression for this loop is an And or Or.
9284 Value *Op0, *Op1;
9285 bool IsAnd;
9286 if (match(V: ExitCond, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9287 IsAnd = true;
9288 else if (match(V: ExitCond, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9289 IsAnd = false;
9290 else
9291 return std::nullopt;
9292
9293 // A sub-condition of a non-trivial binop never solely controls the exit,
9294 // whether we exit always depends on both conditions.
9295 ExitLimit EL0 = computeExitLimitFromCondCached(
9296 Cache, L, ExitCond: Op0, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9297 ExitLimit EL1 = computeExitLimitFromCondCached(
9298 Cache, L, ExitCond: Op1, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9299
9300 // EitherMayExit is true in these two cases:
9301 // br (and Op0 Op1), loop, exit
9302 // br (or Op0 Op1), exit, loop
9303 bool EitherMayExit = IsAnd ^ ExitIfTrue;
9304
9305 const SCEV *BECount = getCouldNotCompute();
9306 const SCEV *ConstantMaxBECount = getCouldNotCompute();
9307 const SCEV *SymbolicMaxBECount = getCouldNotCompute();
9308 if (EitherMayExit) {
9309 bool UseSequentialUMin = !isa<BinaryOperator>(Val: ExitCond);
9310 // Both conditions must be same for the loop to continue executing.
9311 // Choose the less conservative count.
9312 if (EL0.ExactNotTaken != getCouldNotCompute() &&
9313 EL1.ExactNotTaken != getCouldNotCompute()) {
9314 BECount = getUMinFromMismatchedTypes(LHS: EL0.ExactNotTaken, RHS: EL1.ExactNotTaken,
9315 Sequential: UseSequentialUMin);
9316 }
9317 if (EL0.ConstantMaxNotTaken == getCouldNotCompute())
9318 ConstantMaxBECount = EL1.ConstantMaxNotTaken;
9319 else if (EL1.ConstantMaxNotTaken == getCouldNotCompute())
9320 ConstantMaxBECount = EL0.ConstantMaxNotTaken;
9321 else
9322 ConstantMaxBECount = getUMinFromMismatchedTypes(LHS: EL0.ConstantMaxNotTaken,
9323 RHS: EL1.ConstantMaxNotTaken);
9324 if (EL0.SymbolicMaxNotTaken == getCouldNotCompute())
9325 SymbolicMaxBECount = EL1.SymbolicMaxNotTaken;
9326 else if (EL1.SymbolicMaxNotTaken == getCouldNotCompute())
9327 SymbolicMaxBECount = EL0.SymbolicMaxNotTaken;
9328 else
9329 SymbolicMaxBECount = getUMinFromMismatchedTypes(
9330 LHS: EL0.SymbolicMaxNotTaken, RHS: EL1.SymbolicMaxNotTaken, Sequential: UseSequentialUMin);
9331 } else {
9332 // Both conditions must be same at the same time for the loop to exit.
9333 // For now, be conservative.
9334 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
9335 BECount = EL0.ExactNotTaken;
9336 }
9337
9338 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able
9339 // to be more aggressive when computing BECount than when computing
9340 // ConstantMaxBECount. In these cases it is possible for EL0.ExactNotTaken
9341 // and
9342 // EL1.ExactNotTaken to match, but for EL0.ConstantMaxNotTaken and
9343 // EL1.ConstantMaxNotTaken to not.
9344 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
9345 !isa<SCEVCouldNotCompute>(Val: BECount))
9346 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
9347 if (isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount))
9348 SymbolicMaxBECount =
9349 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
9350 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
9351 {ArrayRef(EL0.Predicates), ArrayRef(EL1.Predicates)});
9352}
9353
9354ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9355 const Loop *L, ICmpInst *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9356 bool AllowPredicates) {
9357 // If the condition was exit on true, convert the condition to exit on false
9358 CmpPredicate Pred;
9359 if (!ExitIfTrue)
9360 Pred = ExitCond->getCmpPredicate();
9361 else
9362 Pred = ExitCond->getInverseCmpPredicate();
9363 const ICmpInst::Predicate OriginalPred = Pred;
9364
9365 const SCEV *LHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 0));
9366 const SCEV *RHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 1));
9367
9368 ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, IsSubExpr: ControlsOnlyExit,
9369 AllowPredicates);
9370 if (EL.hasAnyInfo())
9371 return EL;
9372
9373 auto *ExhaustiveCount =
9374 computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9375
9376 if (!isa<SCEVCouldNotCompute>(Val: ExhaustiveCount))
9377 return ExhaustiveCount;
9378
9379 return computeShiftCompareExitLimit(LHS: ExitCond->getOperand(i_nocapture: 0),
9380 RHS: ExitCond->getOperand(i_nocapture: 1), L, Pred: OriginalPred);
9381}
9382ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9383 const Loop *L, CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS,
9384 bool ControlsOnlyExit, bool AllowPredicates) {
9385
9386 // Try to evaluate any dependencies out of the loop.
9387 LHS = getSCEVAtScope(S: LHS, L);
9388 RHS = getSCEVAtScope(S: RHS, L);
9389
9390 // At this point, we would like to compute how many iterations of the
9391 // loop the predicate will return true for these inputs.
9392 if (isLoopInvariant(S: LHS, L) && !isLoopInvariant(S: RHS, L)) {
9393 // If there is a loop-invariant, force it into the RHS.
9394 std::swap(a&: LHS, b&: RHS);
9395 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
9396 }
9397
9398 bool ControllingFiniteLoop = ControlsOnlyExit && loopHasNoAbnormalExits(L) &&
9399 loopIsFiniteByAssumption(L);
9400 // Simplify the operands before analyzing them.
9401 (void)SimplifyICmpOperands(Pred, LHS, RHS, /*Depth=*/0);
9402
9403 // If we have a comparison of a chrec against a constant, try to use value
9404 // ranges to answer this query.
9405 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS))
9406 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
9407 if (AddRec->getLoop() == L) {
9408 // Form the constant range.
9409 ConstantRange CompRange =
9410 ConstantRange::makeExactICmpRegion(Pred, Other: RHSC->getAPInt());
9411
9412 const SCEV *Ret = AddRec->getNumIterationsInRange(Range: CompRange, SE&: *this);
9413 if (!isa<SCEVCouldNotCompute>(Val: Ret)) return Ret;
9414 }
9415
9416 // If this loop must exit based on this condition (or execute undefined
9417 // behaviour), see if we can improve wrap flags. This is essentially
9418 // a must execute style proof.
9419 if (ControllingFiniteLoop && isLoopInvariant(S: RHS, L)) {
9420 // If we can prove the test sequence produced must repeat the same values
9421 // on self-wrap of the IV, then we can infer that IV doesn't self wrap
9422 // because if it did, we'd have an infinite (undefined) loop.
9423 // TODO: We can peel off any functions which are invertible *in L*. Loop
9424 // invariant terms are effectively constants for our purposes here.
9425 SCEVUse InnerLHS = LHS;
9426 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val&: LHS))
9427 InnerLHS = ZExt->getOperand();
9428 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: InnerLHS);
9429 AR && !AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() &&
9430 isKnownToBeAPowerOfTwo(S: AR->getStepRecurrence(SE&: *this), /*OrZero=*/true,
9431 /*OrNegative=*/true)) {
9432 auto Flags = AR->getNoWrapFlags();
9433 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
9434 SmallVector<SCEVUse> Operands{AR->operands()};
9435 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9436 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9437 }
9438
9439 // For a slt/ult condition with a positive step, can we prove nsw/nuw?
9440 // From no-self-wrap, this follows trivially from the fact that every
9441 // (un)signed-wrapped, but not self-wrapped value must be LT than the
9442 // last value before (un)signed wrap. Since we know that last value
9443 // didn't exit, nor will any smaller one.
9444 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT) {
9445 auto WrapType = Pred == ICmpInst::ICMP_SLT ? SCEV::FlagNSW : SCEV::FlagNUW;
9446 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS);
9447 AR && AR->getLoop() == L && AR->isAffine() &&
9448 !AR->getNoWrapFlags(Mask: WrapType) && AR->hasNoSelfWrap() &&
9449 isKnownPositive(S: AR->getStepRecurrence(SE&: *this))) {
9450 auto Flags = AR->getNoWrapFlags();
9451 Flags = setFlags(Flags, OnFlags: WrapType);
9452 SmallVector<SCEVUse> Operands{AR->operands()};
9453 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9454 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9455 }
9456 }
9457 }
9458
9459 switch (Pred) {
9460 case ICmpInst::ICMP_NE: { // while (X != Y)
9461 // Convert to: while (X-Y != 0)
9462 if (LHS->getType()->isPointerTy()) {
9463 LHS = getPtrToAddrExpr(Op: LHS);
9464 if (isa<SCEVCouldNotCompute>(Val: LHS))
9465 return LHS;
9466 }
9467 if (RHS->getType()->isPointerTy()) {
9468 RHS = getPtrToAddrExpr(Op: RHS);
9469 if (isa<SCEVCouldNotCompute>(Val: RHS))
9470 return RHS;
9471 }
9472 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit,
9473 AllowPredicates);
9474 if (EL.hasAnyInfo())
9475 return EL;
9476 break;
9477 }
9478 case ICmpInst::ICMP_EQ: { // while (X == Y)
9479 // Convert to: while (X-Y == 0)
9480 if (LHS->getType()->isPointerTy()) {
9481 LHS = getPtrToAddrExpr(Op: LHS);
9482 if (isa<SCEVCouldNotCompute>(Val: LHS))
9483 return LHS;
9484 }
9485 if (RHS->getType()->isPointerTy()) {
9486 RHS = getPtrToAddrExpr(Op: RHS);
9487 if (isa<SCEVCouldNotCompute>(Val: RHS))
9488 return RHS;
9489 }
9490 ExitLimit EL = howFarToNonZero(V: getMinusSCEV(LHS, RHS), L);
9491 if (EL.hasAnyInfo()) return EL;
9492 break;
9493 }
9494 case ICmpInst::ICMP_SLE:
9495 case ICmpInst::ICMP_ULE:
9496 // Since the loop is finite, an invariant RHS cannot include the boundary
9497 // value, otherwise it would loop forever.
9498 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9499 !isLoopInvariant(S: RHS, L)) {
9500 // Otherwise, perform the addition in a wider type, to avoid overflow.
9501 // If the LHS is an addrec with the appropriate nowrap flag, the
9502 // extension will be sunk into it and the exit count can be analyzed.
9503 auto *OldType = dyn_cast<IntegerType>(Val: LHS->getType());
9504 if (!OldType)
9505 break;
9506 // Prefer doubling the bitwidth over adding a single bit to make it more
9507 // likely that we use a legal type.
9508 auto *NewType =
9509 Type::getIntNTy(C&: OldType->getContext(), N: OldType->getBitWidth() * 2);
9510 if (ICmpInst::isSigned(Pred)) {
9511 LHS = getSignExtendExpr(Op: LHS, Ty: NewType);
9512 RHS = getSignExtendExpr(Op: RHS, Ty: NewType);
9513 } else {
9514 LHS = getZeroExtendExpr(Op: LHS, Ty: NewType);
9515 RHS = getZeroExtendExpr(Op: RHS, Ty: NewType);
9516 }
9517 }
9518 RHS = getAddExpr(LHS: getOne(Ty: RHS->getType()), RHS);
9519 [[fallthrough]];
9520 case ICmpInst::ICMP_SLT:
9521 case ICmpInst::ICMP_ULT: { // while (X < Y)
9522 bool IsSigned = ICmpInst::isSigned(Pred);
9523 ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, /*Invert=*/false,
9524 ControlsOnlyExit, AllowPredicates);
9525 if (EL.hasAnyInfo())
9526 return EL;
9527 break;
9528 }
9529 case ICmpInst::ICMP_SGE:
9530 case ICmpInst::ICMP_UGE:
9531 // Since the loop is finite, an invariant RHS cannot include the boundary
9532 // value, otherwise it would loop forever.
9533 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9534 !isLoopInvariant(S: RHS, L))
9535 break;
9536 RHS = getAddExpr(LHS: getMinusOne(Ty: RHS->getType()), RHS);
9537 [[fallthrough]];
9538 case ICmpInst::ICMP_SGT:
9539 case ICmpInst::ICMP_UGT: { // while (X > Y)
9540 // "X > Y" is analyzed as the equivalent "~X < ~Y".
9541 bool IsSigned = ICmpInst::isSigned(Pred);
9542 ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, /*Invert=*/true,
9543 ControlsOnlyExit, AllowPredicates);
9544 if (EL.hasAnyInfo())
9545 return EL;
9546 break;
9547 }
9548 default:
9549 break;
9550 }
9551
9552 return getCouldNotCompute();
9553}
9554
9555ScalarEvolution::ExitLimit
9556ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L,
9557 SwitchInst *Switch,
9558 BasicBlock *ExitingBlock,
9559 bool ControlsOnlyExit) {
9560 assert(!L->contains(ExitingBlock) && "Not an exiting block!");
9561
9562 // Give up if the exit is the default dest of a switch.
9563 if (Switch->getDefaultDest() == ExitingBlock)
9564 return getCouldNotCompute();
9565
9566 assert(L->contains(Switch->getDefaultDest()) &&
9567 "Default case must not exit the loop!");
9568 const SCEV *LHS = getSCEVAtScope(V: Switch->getCondition(), L);
9569 const SCEV *RHS = getConstant(V: Switch->findCaseDest(BB: ExitingBlock));
9570
9571 // while (X != Y) --> while (X-Y != 0)
9572 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit);
9573 if (EL.hasAnyInfo())
9574 return EL;
9575
9576 return getCouldNotCompute();
9577}
9578
9579static ConstantInt *
9580EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
9581 ScalarEvolution &SE) {
9582 const SCEV *InVal = SE.getConstant(V: C);
9583 const SCEV *Val = AddRec->evaluateAtIteration(It: InVal, SE);
9584 assert(isa<SCEVConstant>(Val) &&
9585 "Evaluation of SCEV at constant didn't fold correctly?");
9586 return cast<SCEVConstant>(Val)->getValue();
9587}
9588
9589ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
9590 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) {
9591 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: RHSV);
9592 if (!RHS)
9593 return getCouldNotCompute();
9594
9595 const BasicBlock *Latch = L->getLoopLatch();
9596 if (!Latch)
9597 return getCouldNotCompute();
9598
9599 const BasicBlock *Predecessor = L->getLoopPredecessor();
9600 if (!Predecessor)
9601 return getCouldNotCompute();
9602
9603 // Return true if V is of the form "LHS `shift_op` <positive constant>".
9604 // Return LHS in OutLHS, shift_op in OutOpCode, and the shift amount in
9605 // OutShiftAmt.
9606 auto MatchPositiveShift = [](Value *V, Value *&OutLHS,
9607 Instruction::BinaryOps &OutOpCode,
9608 unsigned &OutShiftAmt) {
9609 using namespace PatternMatch;
9610
9611 ConstantInt *ShiftAmt;
9612 if (match(V, P: m_LShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9613 OutOpCode = Instruction::LShr;
9614 else if (match(V, P: m_AShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9615 OutOpCode = Instruction::AShr;
9616 else if (match(V, P: m_Shl(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9617 OutOpCode = Instruction::Shl;
9618 else
9619 return false;
9620
9621 uint64_t Amt = ShiftAmt->getValue().getLimitedValue();
9622 if (Amt == 0 || Amt >= OutLHS->getType()->getScalarSizeInBits())
9623 return false;
9624 OutShiftAmt = Amt;
9625 return true;
9626 };
9627
9628 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in
9629 //
9630 // loop:
9631 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ]
9632 // %iv.shifted = lshr i32 %iv, <positive constant>
9633 //
9634 // Return true on a successful match. Return the corresponding PHI node (%iv
9635 // above) in PNOut, the opcode of the shift operation in OpCodeOut, and the
9636 // shift amount in ShiftAmtOut.
9637 auto MatchShiftRecurrence = [&](Value *V, PHINode *&PNOut,
9638 Instruction::BinaryOps &OpCodeOut,
9639 unsigned &ShiftAmtOut) {
9640 std::optional<Instruction::BinaryOps> PostShiftOpCode;
9641
9642 {
9643 Instruction::BinaryOps OpC;
9644 Value *V;
9645 unsigned Amt;
9646
9647 // If we encounter a shift instruction, "peel off" the shift operation,
9648 // and remember that we did so. Later when we inspect %iv's backedge
9649 // value, we will make sure that the backedge value uses the same
9650 // operation.
9651 //
9652 // Note: the peeled shift operation does not have to be the same
9653 // instruction as the one feeding into the PHI's backedge value. We only
9654 // really care about it being the same *kind* of shift instruction --
9655 // that's all that is required for our later inferences to hold.
9656 if (MatchPositiveShift(LHS, V, OpC, Amt)) {
9657 PostShiftOpCode = OpC;
9658 LHS = V;
9659 }
9660 }
9661
9662 PNOut = dyn_cast<PHINode>(Val: LHS);
9663 if (!PNOut || PNOut->getParent() != L->getHeader())
9664 return false;
9665
9666 Value *BEValue = PNOut->getIncomingValueForBlock(BB: Latch);
9667 Value *OpLHS;
9668
9669 return
9670 // The backedge value for the PHI node must be a shift by a positive
9671 // amount
9672 MatchPositiveShift(BEValue, OpLHS, OpCodeOut, ShiftAmtOut) &&
9673
9674 // of the PHI node itself
9675 OpLHS == PNOut &&
9676
9677 // and the kind of shift should be match the kind of shift we peeled
9678 // off, if any.
9679 (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut);
9680 };
9681
9682 PHINode *PN;
9683 Instruction::BinaryOps OpCode;
9684 unsigned ShiftAmt;
9685 if (!MatchShiftRecurrence(LHS, PN, OpCode, ShiftAmt))
9686 return getCouldNotCompute();
9687
9688 const DataLayout &DL = getDataLayout();
9689
9690 // The key rationale for this optimization is that for some kinds of shift
9691 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1
9692 // within a finite number of iterations. If the condition guarding the
9693 // backedge (in the sense that the backedge is taken if the condition is true)
9694 // is false for the value the shift recurrence stabilizes to, then we know
9695 // that the backedge is taken only a finite number of times.
9696
9697 ConstantInt *StableValue = nullptr;
9698 switch (OpCode) {
9699 default:
9700 llvm_unreachable("Impossible case!");
9701
9702 case Instruction::AShr: {
9703 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
9704 // bitwidth(K) iterations.
9705 Value *FirstValue = PN->getIncomingValueForBlock(BB: Predecessor);
9706 KnownBits Known = computeKnownBits(V: FirstValue, DL, AC: &AC,
9707 CtxI: Predecessor->getTerminator(), DT: &DT);
9708 auto *Ty = cast<IntegerType>(Val: RHS->getType());
9709 if (Known.isNonNegative())
9710 StableValue = ConstantInt::get(Ty, V: 0);
9711 else if (Known.isNegative())
9712 StableValue = ConstantInt::get(Ty, V: -1, IsSigned: true);
9713 else
9714 return getCouldNotCompute();
9715
9716 break;
9717 }
9718 case Instruction::LShr:
9719 case Instruction::Shl:
9720 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>}
9721 // stabilize to 0 in at most bitwidth(K) iterations.
9722 StableValue = ConstantInt::get(Ty: cast<IntegerType>(Val: RHS->getType()), V: 0);
9723 break;
9724 }
9725
9726 auto *Result =
9727 ConstantFoldCompareInstOperands(Predicate: Pred, LHS: StableValue, RHS, DL, TLI: &TLI);
9728 assert(Result->getType()->isIntegerTy(1) &&
9729 "Otherwise cannot be an operand to a branch instruction");
9730
9731 if (Result->isNullValue()) {
9732 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
9733 unsigned MaxBTC = BitWidth;
9734
9735 // For right-shift recurrences (lshr/ashr with non-negative start), we can
9736 // compute a tighter max backedge-taken count from the range of the start
9737 // value. After k shifts of ShiftAmt, value = start >> (k * ShiftAmt).
9738 // The value reaches 0 (the stable value) when k * ShiftAmt >=
9739 // activeBits(start), so max BTC = ceil(activeBits(maxStart) / ShiftAmt).
9740 if (OpCode == Instruction::LShr || OpCode == Instruction::AShr) {
9741 Value *StartValue = PN->getIncomingValueForBlock(BB: Predecessor);
9742 const SCEV *StartSCEV = getSCEV(V: StartValue);
9743 APInt MaxStart = getUnsignedRangeMax(S: StartSCEV);
9744 if (MaxStart.isStrictlyPositive()) {
9745 unsigned ActiveBits = MaxStart.getActiveBits();
9746 unsigned RangeBTC = divideCeil(Numerator: ActiveBits, Denominator: ShiftAmt);
9747 MaxBTC = std::min(a: MaxBTC, b: RangeBTC);
9748 }
9749 }
9750
9751 const SCEV *UpperBound =
9752 getConstant(Ty: getEffectiveSCEVType(Ty: RHS->getType()), V: MaxBTC);
9753 return ExitLimit(getCouldNotCompute(), UpperBound, UpperBound, false);
9754 }
9755
9756 return getCouldNotCompute();
9757}
9758
9759/// Return true if we can constant fold an instruction of the specified type,
9760/// assuming that all operands were constants.
9761static bool canConstantFold(const Instruction *I,
9762 const TargetLibraryInfo *TLI) {
9763 if (isa<BinaryOperator, UnaryOperator, GEPOperator, FreezeInst, CmpInst,
9764 SelectInst, CastInst, LoadInst, ExtractElementInst, InsertElementInst,
9765 ExtractValueInst, InsertValueInst>(Val: I))
9766 return true;
9767
9768 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
9769 if (const Function *F = CI->getCalledFunction())
9770 return canConstantFoldCallTo(Call: CI, F, TLI);
9771 return false;
9772}
9773
9774/// Determine whether this instruction can constant evolve within this loop
9775/// assuming its operands can all constant evolve.
9776static bool canConstantEvolve(Instruction *I, const Loop *L,
9777 const TargetLibraryInfo *TLI) {
9778 // An instruction outside of the loop can't be derived from a loop PHI.
9779 if (!L->contains(Inst: I)) return false;
9780
9781 if (isa<PHINode>(Val: I)) {
9782 // We don't currently keep track of the control flow needed to evaluate
9783 // PHIs, so we cannot handle PHIs inside of loops.
9784 return L->getHeader() == I->getParent();
9785 }
9786
9787 // If we won't be able to constant fold this expression even if the operands
9788 // are constants, bail early.
9789 return canConstantFold(I, TLI);
9790}
9791
9792/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
9793/// recursing through each instruction operand until reaching a loop header phi.
9794static PHINode *
9795getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
9796 DenseMap<Instruction *, PHINode *> &PHIMap,
9797 const TargetLibraryInfo *TLI, unsigned Depth) {
9798 if (Depth > MaxConstantEvolvingDepth)
9799 return nullptr;
9800
9801 // Otherwise, we can evaluate this instruction if all of its operands are
9802 // constant or derived from a PHI node themselves.
9803 PHINode *PHI = nullptr;
9804 for (Value *Op : UseInst->operands()) {
9805 if (isa<Constant>(Val: Op)) continue;
9806
9807 Instruction *OpInst = dyn_cast<Instruction>(Val: Op);
9808 if (!OpInst || !canConstantEvolve(I: OpInst, L, TLI))
9809 return nullptr;
9810
9811 PHINode *P = dyn_cast<PHINode>(Val: OpInst);
9812 if (!P)
9813 // If this operand is already visited, reuse the prior result.
9814 // We may have P != PHI if this is the deepest point at which the
9815 // inconsistent paths meet.
9816 P = PHIMap.lookup(Val: OpInst);
9817 if (!P) {
9818 // Recurse and memoize the results, whether a phi is found or not.
9819 // This recursive call invalidates pointers into PHIMap.
9820 P = getConstantEvolvingPHIOperands(UseInst: OpInst, L, PHIMap, TLI, Depth: Depth + 1);
9821 PHIMap[OpInst] = P;
9822 }
9823 if (!P)
9824 return nullptr; // Not evolving from PHI
9825 if (PHI && PHI != P)
9826 return nullptr; // Evolving from multiple different PHIs.
9827 PHI = P;
9828 }
9829 // This is a expression evolving from a constant PHI!
9830 return PHI;
9831}
9832
9833/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
9834/// in the loop that V is derived from. We allow arbitrary operations along the
9835/// way, but the operands of an operation must either be constants or a value
9836/// derived from a constant PHI. If this expression does not fit with these
9837/// constraints, return null.
9838static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L,
9839 const TargetLibraryInfo *TLI) {
9840 Instruction *I = dyn_cast<Instruction>(Val: V);
9841 if (!I || !canConstantEvolve(I, L, TLI))
9842 return nullptr;
9843
9844 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
9845 return PN;
9846
9847 // Record non-constant instructions contained by the loop.
9848 DenseMap<Instruction *, PHINode *> PHIMap;
9849 return getConstantEvolvingPHIOperands(UseInst: I, L, PHIMap, TLI, Depth: 0);
9850}
9851
9852/// EvaluateExpression - Given an expression that passes the
9853/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
9854/// in the loop has the value PHIVal. If we can't fold this expression for some
9855/// reason, return null.
9856static Constant *EvaluateExpression(Value *V, const Loop *L,
9857 DenseMap<Instruction *, Constant *> &Vals,
9858 const DataLayout &DL,
9859 const TargetLibraryInfo *TLI) {
9860 // Convenient constant check, but redundant for recursive calls.
9861 if (Constant *C = dyn_cast<Constant>(Val: V)) return C;
9862 Instruction *I = dyn_cast<Instruction>(Val: V);
9863 if (!I) return nullptr;
9864
9865 if (Constant *C = Vals.lookup(Val: I)) return C;
9866
9867 // An instruction inside the loop depends on a value outside the loop that we
9868 // weren't given a mapping for, or a value such as a call inside the loop.
9869 if (!canConstantEvolve(I, L, TLI))
9870 return nullptr;
9871
9872 // An unmapped PHI can be due to a branch or another loop inside this loop,
9873 // or due to this not being the initial iteration through a loop where we
9874 // couldn't compute the evolution of this particular PHI last time.
9875 if (isa<PHINode>(Val: I)) return nullptr;
9876
9877 std::vector<Constant*> Operands(I->getNumOperands());
9878
9879 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
9880 Instruction *Operand = dyn_cast<Instruction>(Val: I->getOperand(i));
9881 if (!Operand) {
9882 Operands[i] = dyn_cast<Constant>(Val: I->getOperand(i));
9883 if (!Operands[i]) return nullptr;
9884 continue;
9885 }
9886 Constant *C = EvaluateExpression(V: Operand, L, Vals, DL, TLI);
9887 Vals[Operand] = C;
9888 if (!C) return nullptr;
9889 Operands[i] = C;
9890 }
9891
9892 return ConstantFoldInstOperands(I, Ops: Operands, DL, TLI,
9893 /*AllowNonDeterministic=*/false);
9894}
9895
9896
9897// If every incoming value to PN except the one for BB is a specific Constant,
9898// return that, else return nullptr.
9899static Constant *getOtherIncomingValue(PHINode *PN, BasicBlock *BB) {
9900 Constant *IncomingVal = nullptr;
9901
9902 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
9903 if (PN->getIncomingBlock(i) == BB)
9904 continue;
9905
9906 auto *CurrentVal = dyn_cast<Constant>(Val: PN->getIncomingValue(i));
9907 if (!CurrentVal)
9908 return nullptr;
9909
9910 if (IncomingVal != CurrentVal) {
9911 if (IncomingVal)
9912 return nullptr;
9913 IncomingVal = CurrentVal;
9914 }
9915 }
9916
9917 return IncomingVal;
9918}
9919
9920/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
9921/// in the header of its containing loop, we know the loop executes a
9922/// constant number of times, and the PHI node is just a recurrence
9923/// involving constants, fold it.
9924Constant *
9925ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
9926 const APInt &BEs,
9927 const Loop *L) {
9928 auto [I, Inserted] = ConstantEvolutionLoopExitValue.try_emplace(Key: PN);
9929 if (!Inserted)
9930 return I->second;
9931
9932 if (BEs.ugt(RHS: MaxBruteForceIterations))
9933 return nullptr; // Not going to evaluate it.
9934
9935 Constant *&RetVal = I->second;
9936
9937 DenseMap<Instruction *, Constant *> CurrentIterVals;
9938 BasicBlock *Header = L->getHeader();
9939 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9940
9941 BasicBlock *Latch = L->getLoopLatch();
9942 if (!Latch)
9943 return nullptr;
9944
9945 for (PHINode &PHI : Header->phis()) {
9946 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
9947 CurrentIterVals[&PHI] = StartCST;
9948 }
9949 if (!CurrentIterVals.count(Val: PN))
9950 return RetVal = nullptr;
9951
9952 Value *BEValue = PN->getIncomingValueForBlock(BB: Latch);
9953
9954 // Execute the loop symbolically to determine the exit value.
9955 assert(BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) &&
9956 "BEs is <= MaxBruteForceIterations which is an 'unsigned'!");
9957
9958 unsigned NumIterations = BEs.getZExtValue(); // must be in range
9959 unsigned IterationNum = 0;
9960 const DataLayout &DL = getDataLayout();
9961 for (; ; ++IterationNum) {
9962 if (IterationNum == NumIterations)
9963 return RetVal = CurrentIterVals[PN]; // Got exit value!
9964
9965 // Compute the value of the PHIs for the next iteration.
9966 // EvaluateExpression adds non-phi values to the CurrentIterVals map.
9967 DenseMap<Instruction *, Constant *> NextIterVals;
9968 Constant *NextPHI =
9969 EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9970 if (!NextPHI)
9971 return nullptr; // Couldn't evaluate!
9972 NextIterVals[PN] = NextPHI;
9973
9974 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
9975
9976 // Also evaluate the other PHI nodes. However, we don't get to stop if we
9977 // cease to be able to evaluate one of them or if they stop evolving,
9978 // because that doesn't necessarily prevent us from computing PN.
9979 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
9980 for (const auto &I : CurrentIterVals) {
9981 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
9982 if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
9983 PHIsToCompute.emplace_back(Args&: PHI, Args: I.second);
9984 }
9985 // We use two distinct loops because EvaluateExpression may invalidate any
9986 // iterators into CurrentIterVals.
9987 for (const auto &I : PHIsToCompute) {
9988 PHINode *PHI = I.first;
9989 Constant *&NextPHI = NextIterVals[PHI];
9990 if (!NextPHI) { // Not already computed.
9991 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
9992 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9993 }
9994 if (NextPHI != I.second)
9995 StoppedEvolving = false;
9996 }
9997
9998 // If all entries in CurrentIterVals == NextIterVals then we can stop
9999 // iterating, the loop can't continue to change.
10000 if (StoppedEvolving)
10001 return RetVal = CurrentIterVals[PN];
10002
10003 CurrentIterVals.swap(RHS&: NextIterVals);
10004 }
10005}
10006
10007const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L,
10008 Value *Cond,
10009 bool ExitWhen) {
10010 PHINode *PN = getConstantEvolvingPHI(V: Cond, L, TLI: &TLI);
10011 if (!PN) return getCouldNotCompute();
10012
10013 // If the loop is canonicalized, the PHI will have exactly two entries.
10014 // That's the only form we support here.
10015 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
10016
10017 DenseMap<Instruction *, Constant *> CurrentIterVals;
10018 BasicBlock *Header = L->getHeader();
10019 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
10020
10021 BasicBlock *Latch = L->getLoopLatch();
10022 assert(Latch && "Should follow from NumIncomingValues == 2!");
10023
10024 for (PHINode &PHI : Header->phis()) {
10025 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
10026 CurrentIterVals[&PHI] = StartCST;
10027 }
10028 if (!CurrentIterVals.count(Val: PN))
10029 return getCouldNotCompute();
10030
10031 // Okay, we find a PHI node that defines the trip count of this loop. Execute
10032 // the loop symbolically to determine when the condition gets a value of
10033 // "ExitWhen".
10034 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis.
10035 const DataLayout &DL = getDataLayout();
10036 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
10037 auto *CondVal = dyn_cast_or_null<ConstantInt>(
10038 Val: EvaluateExpression(V: Cond, L, Vals&: CurrentIterVals, DL, TLI: &TLI));
10039
10040 // Couldn't symbolically evaluate.
10041 if (!CondVal) return getCouldNotCompute();
10042
10043 if (CondVal->getValue() == uint64_t(ExitWhen)) {
10044 ++NumBruteForceTripCountsComputed;
10045 return getConstant(Ty: Type::getInt32Ty(C&: getContext()), V: IterationNum);
10046 }
10047
10048 // Update all the PHI nodes for the next iteration.
10049 DenseMap<Instruction *, Constant *> NextIterVals;
10050
10051 // Create a list of which PHIs we need to compute. We want to do this before
10052 // calling EvaluateExpression on them because that may invalidate iterators
10053 // into CurrentIterVals.
10054 SmallVector<PHINode *, 8> PHIsToCompute;
10055 for (const auto &I : CurrentIterVals) {
10056 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
10057 if (!PHI || PHI->getParent() != Header) continue;
10058 PHIsToCompute.push_back(Elt: PHI);
10059 }
10060 for (PHINode *PHI : PHIsToCompute) {
10061 Constant *&NextPHI = NextIterVals[PHI];
10062 if (NextPHI) continue; // Already computed!
10063
10064 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
10065 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
10066 }
10067 CurrentIterVals.swap(RHS&: NextIterVals);
10068 }
10069
10070 // Too many iterations were needed to evaluate.
10071 return getCouldNotCompute();
10072}
10073
10074SCEVUse ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
10075 auto &Values = ValuesAtScopes[V];
10076 // Check to see if we've folded this expression at this loop before.
10077 for (auto &LS : Values)
10078 if (LS.first == L)
10079 return LS.second ? LS.second : SCEVUse(V);
10080
10081 Values.emplace_back(Args&: L, Args: nullptr);
10082
10083 // Otherwise compute it.
10084 SCEVUse C = computeSCEVAtScope(S: V, L);
10085 for (auto &LS : reverse(C&: ValuesAtScopes[V]))
10086 if (LS.first == L) {
10087 LS.second = C;
10088 // Record the dependency under the bare expression: invalidation walks
10089 // expressions, and any use flags on C do not change which expression
10090 // this is the value at scope of.
10091 if (!isa<SCEVConstant>(Val: C))
10092 ValuesAtScopesUsers[C.getPointer()].push_back(Elt: {L, V});
10093 break;
10094 }
10095 return C;
10096}
10097
10098SCEVUse ScalarEvolution::getSCEVAtExit(const SCEV *V, const Loop *L,
10099 const BasicBlock *ExitingBlock) {
10100 SCEVUse ExitValue = getSCEVAtScope(V, L: L->getParentLoop());
10101 if (!isLoopInvariant(S: ExitValue, L)) {
10102 // If we failed to evaluate it in the outer scope, try to evaluate an
10103 // addrec for the specific exit.
10104 // TODO: Generalize this to other expressions.
10105 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
10106 if (!isa<SCEVCouldNotCompute>(Val: ExitCount))
10107 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: V))
10108 if (AddRec->getLoop() == L)
10109 ExitValue = AddRec->evaluateAtIteration(It: ExitCount, SE&: *this);
10110 }
10111 return ExitValue;
10112}
10113
10114/// This builds up a Constant using the ConstantExpr interface. That way, we
10115/// will return Constants for objects which aren't represented by a
10116/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
10117/// Returns NULL if the SCEV isn't representable as a Constant.
10118static Constant *BuildConstantFromSCEV(const SCEV *V) {
10119 switch (V->getSCEVType()) {
10120 case scCouldNotCompute:
10121 case scAddRecExpr:
10122 case scVScale:
10123 return nullptr;
10124 case scConstant:
10125 return cast<SCEVConstant>(Val: V)->getValue();
10126 case scUnknown:
10127 return dyn_cast<Constant>(Val: cast<SCEVUnknown>(Val: V)->getValue());
10128 case scPtrToAddr: {
10129 const SCEVPtrToAddrExpr *P2I = cast<SCEVPtrToAddrExpr>(Val: V);
10130 if (Constant *CastOp = BuildConstantFromSCEV(V: P2I->getOperand()))
10131 return ConstantExpr::getPtrToAddr(C: CastOp, Ty: P2I->getType());
10132
10133 return nullptr;
10134 }
10135 case scTruncate: {
10136 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(Val: V);
10137 if (Constant *CastOp = BuildConstantFromSCEV(V: ST->getOperand()))
10138 return ConstantExpr::getTrunc(C: CastOp, Ty: ST->getType());
10139 return nullptr;
10140 }
10141 case scAddExpr: {
10142 const SCEVAddExpr *SA = cast<SCEVAddExpr>(Val: V);
10143 Constant *C = nullptr;
10144 for (const SCEV *Op : SA->operands()) {
10145 Constant *OpC = BuildConstantFromSCEV(V: Op);
10146 if (!OpC)
10147 return nullptr;
10148 if (!C) {
10149 C = OpC;
10150 continue;
10151 }
10152 assert(!C->getType()->isPointerTy() &&
10153 "Can only have one pointer, and it must be last");
10154 if (OpC->getType()->isPointerTy()) {
10155 // The offsets have been converted to bytes. We can add bytes using
10156 // an i8 GEP.
10157 C = ConstantExpr::getPtrAdd(Ptr: OpC, Offset: C);
10158 } else {
10159 C = ConstantExpr::getAdd(C1: C, C2: OpC);
10160 }
10161 }
10162 return C;
10163 }
10164 case scMulExpr:
10165 case scSignExtend:
10166 case scZeroExtend:
10167 case scUDivExpr:
10168 case scSMaxExpr:
10169 case scUMaxExpr:
10170 case scSMinExpr:
10171 case scUMinExpr:
10172 case scSequentialUMinExpr:
10173 return nullptr;
10174 }
10175 llvm_unreachable("Unknown SCEV kind!");
10176}
10177
10178const SCEV *ScalarEvolution::getWithOperands(const SCEV *S,
10179 SmallVectorImpl<SCEVUse> &NewOps) {
10180 switch (S->getSCEVType()) {
10181 case scTruncate:
10182 case scZeroExtend:
10183 case scSignExtend:
10184 case scPtrToAddr:
10185 return getCastExpr(Kind: S->getSCEVType(), Op: NewOps[0], Ty: S->getType());
10186 case scAddRecExpr: {
10187 auto *AddRec = cast<SCEVAddRecExpr>(Val: S);
10188 return getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), NWFlags: AddRec->getNoWrapFlags());
10189 }
10190 case scAddExpr:
10191 return getAddExpr(Ops&: NewOps, Flags: cast<SCEVAddExpr>(Val: S)->getNoWrapFlags());
10192 case scMulExpr:
10193 return getMulExpr(Ops&: NewOps, Flags: cast<SCEVMulExpr>(Val: S)->getNoWrapFlags());
10194 case scUDivExpr:
10195 return getUDivExpr(LHS: NewOps[0], RHS: NewOps[1]);
10196 case scUMaxExpr:
10197 case scSMaxExpr:
10198 case scUMinExpr:
10199 case scSMinExpr:
10200 return getMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10201 case scSequentialUMinExpr:
10202 return getSequentialMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10203 case scConstant:
10204 case scVScale:
10205 case scUnknown:
10206 return S;
10207 case scCouldNotCompute:
10208 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10209 }
10210 llvm_unreachable("Unknown SCEV kind!");
10211}
10212
10213SCEVUse ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
10214 switch (V->getSCEVType()) {
10215 case scConstant:
10216 case scVScale:
10217 return V;
10218 case scAddRecExpr: {
10219 // If this is a loop recurrence for a loop that does not contain L, then we
10220 // are dealing with the final value computed by the loop.
10221 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: V);
10222 // First, attempt to evaluate each operand.
10223 // Avoid performing the look-up in the common case where the specified
10224 // expression has no loop-variant portions.
10225 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
10226 SCEVUse OpAtScope = getSCEVAtScope(V: AddRec->getOperand(i), L);
10227 if (OpAtScope == AddRec->getOperand(i))
10228 continue;
10229
10230 // Okay, at least one of these operands is loop variant but might be
10231 // foldable. Build a new instance of the folded commutative expression.
10232 SmallVector<SCEVUse, 8> NewOps;
10233 NewOps.reserve(N: AddRec->getNumOperands());
10234 append_range(C&: NewOps, R: AddRec->operands().take_front(N: i));
10235 NewOps.push_back(Elt: OpAtScope);
10236 for (++i; i != e; ++i)
10237 NewOps.push_back(Elt: getSCEVAtScope(V: AddRec->getOperand(i), L));
10238
10239 const SCEV *FoldedRec = getAddRecExpr(
10240 Operands&: NewOps, L: AddRec->getLoop(), NWFlags: AddRec->getNoWrapFlags(Mask: SCEV::FlagNW));
10241 AddRec = dyn_cast<SCEVAddRecExpr>(Val: FoldedRec);
10242 // The addrec may be folded to a nonrecurrence, for example, if the
10243 // induction variable is multiplied by zero after constant folding. Go
10244 // ahead and return the folded value.
10245 if (!AddRec)
10246 return FoldedRec;
10247 break;
10248 }
10249
10250 // If the scope is outside the addrec's loop, evaluate it by using the
10251 // loop exit value of the addrec.
10252 if (!AddRec->getLoop()->contains(L)) {
10253 SCEVUse ExitValue = AddRec->getExitValue(SE&: *this);
10254 if (isa<SCEVCouldNotCompute>(Val: ExitValue))
10255 return AddRec;
10256 return ExitValue;
10257 }
10258
10259 return AddRec;
10260 }
10261 case scTruncate:
10262 case scZeroExtend:
10263 case scSignExtend:
10264 case scPtrToAddr:
10265 case scAddExpr:
10266 case scMulExpr:
10267 case scUDivExpr:
10268 case scUMaxExpr:
10269 case scSMaxExpr:
10270 case scUMinExpr:
10271 case scSMinExpr:
10272 case scSequentialUMinExpr: {
10273 ArrayRef<SCEVUse> Ops = V->operands();
10274 // Avoid performing the look-up in the common case where the specified
10275 // expression has no loop-variant portions.
10276 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
10277 SCEVUse OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10278 if (OpAtScope != Ops[i].getPointer()) {
10279 // Okay, at least one of these operands is loop variant but might be
10280 // foldable. Build a new instance of the folded commutative expression.
10281 SmallVector<SCEVUse, 8> NewOps;
10282 NewOps.reserve(N: Ops.size());
10283 append_range(C&: NewOps, R: Ops.take_front(N: i));
10284 NewOps.push_back(Elt: OpAtScope);
10285
10286 for (++i; i != e; ++i) {
10287 OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10288 NewOps.push_back(Elt: OpAtScope);
10289 }
10290
10291 return getWithOperands(S: V, NewOps);
10292 }
10293 }
10294 // If we got here, all operands are loop invariant.
10295 return V;
10296 }
10297 case scUnknown: {
10298 // If this instruction is evolved from a constant-evolving PHI, compute the
10299 // exit value from the loop without using SCEVs.
10300 const SCEVUnknown *SU = cast<SCEVUnknown>(Val: V);
10301 Instruction *I = dyn_cast<Instruction>(Val: SU->getValue());
10302 if (!I)
10303 return V; // This is some other type of SCEVUnknown, just return it.
10304
10305 if (PHINode *PN = dyn_cast<PHINode>(Val: I)) {
10306 const Loop *CurrLoop = this->LI[I->getParent()];
10307 // Looking for loop exit value.
10308 if (CurrLoop && CurrLoop->getParentLoop() == L &&
10309 PN->getParent() == CurrLoop->getHeader()) {
10310 // Okay, there is no closed form solution for the PHI node. Check
10311 // to see if the loop that contains it has a known backedge-taken
10312 // count. If so, we may be able to force computation of the exit
10313 // value.
10314 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: CurrLoop);
10315 // This trivial case can show up in some degenerate cases where
10316 // the incoming IR has not yet been fully simplified.
10317 if (BackedgeTakenCount->isZero()) {
10318 Value *InitValue = nullptr;
10319 bool MultipleInitValues = false;
10320 for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
10321 if (!CurrLoop->contains(BB: PN->getIncomingBlock(i))) {
10322 if (!InitValue)
10323 InitValue = PN->getIncomingValue(i);
10324 else if (InitValue != PN->getIncomingValue(i)) {
10325 MultipleInitValues = true;
10326 break;
10327 }
10328 }
10329 }
10330 if (!MultipleInitValues && InitValue)
10331 return getSCEV(V: InitValue);
10332 }
10333 // Do we have a loop invariant value flowing around the backedge
10334 // for a loop which must execute the backedge?
10335 if (!isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount) &&
10336 isKnownNonZero(S: BackedgeTakenCount) &&
10337 PN->getNumIncomingValues() == 2) {
10338
10339 unsigned InLoopPred =
10340 CurrLoop->contains(BB: PN->getIncomingBlock(i: 0)) ? 0 : 1;
10341 Value *BackedgeVal = PN->getIncomingValue(i: InLoopPred);
10342 if (CurrLoop->isLoopInvariant(V: BackedgeVal))
10343 return getSCEV(V: BackedgeVal);
10344 }
10345 if (auto *BTCC = dyn_cast<SCEVConstant>(Val: BackedgeTakenCount)) {
10346 // Okay, we know how many times the containing loop executes. If
10347 // this is a constant evolving PHI node, get the final value at
10348 // the specified iteration number.
10349 Constant *RV =
10350 getConstantEvolutionLoopExitValue(PN, BEs: BTCC->getAPInt(), L: CurrLoop);
10351 if (RV)
10352 return getSCEV(V: RV);
10353 }
10354 }
10355 }
10356
10357 // Okay, this is an expression that we cannot symbolically evaluate
10358 // into a SCEV. Check to see if it's possible to symbolically evaluate
10359 // the arguments into constants, and if so, try to constant propagate the
10360 // result. This is particularly useful for computing loop exit values.
10361 if (!canConstantFold(I, TLI: &TLI))
10362 return V; // This is some other type of SCEVUnknown, just return it.
10363
10364 SmallVector<Constant *, 4> Operands;
10365 Operands.reserve(N: I->getNumOperands());
10366 bool MadeImprovement = false;
10367 for (Value *Op : I->operands()) {
10368 if (Constant *C = dyn_cast<Constant>(Val: Op)) {
10369 Operands.push_back(Elt: C);
10370 continue;
10371 }
10372
10373 // If any of the operands is non-constant and if they are
10374 // non-integer and non-pointer, don't even try to analyze them
10375 // with scev techniques.
10376 if (!isSCEVable(Ty: Op->getType()))
10377 return V;
10378
10379 const SCEV *OrigV = getSCEV(V: Op);
10380 const SCEV *OpV = getSCEVAtScope(V: OrigV, L);
10381 MadeImprovement |= OrigV != OpV;
10382
10383 Constant *C = BuildConstantFromSCEV(V: OpV);
10384 if (!C)
10385 return V;
10386 assert(C->getType() == Op->getType() && "Type mismatch");
10387 Operands.push_back(Elt: C);
10388 }
10389
10390 // Check to see if getSCEVAtScope actually made an improvement.
10391 if (!MadeImprovement)
10392 return V; // This is some other type of SCEVUnknown, just return it.
10393
10394 Constant *C = nullptr;
10395 const DataLayout &DL = getDataLayout();
10396 C = ConstantFoldInstOperands(I, Ops: Operands, DL, TLI: &TLI,
10397 /*AllowNonDeterministic=*/false);
10398 if (!C)
10399 return V;
10400 return getSCEV(V: C);
10401 }
10402 case scCouldNotCompute:
10403 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10404 }
10405 llvm_unreachable("Unknown SCEV type!");
10406}
10407
10408SCEVUse ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
10409 return getSCEVAtScope(V: getSCEV(V), L);
10410}
10411
10412const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const {
10413 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: S))
10414 return stripInjectiveFunctions(S: ZExt->getOperand());
10415 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
10416 return stripInjectiveFunctions(S: SExt->getOperand());
10417 return S;
10418}
10419
10420/// Finds the minimum unsigned root of the following equation:
10421///
10422/// A * X = B (mod N)
10423///
10424/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
10425/// A and B isn't important.
10426///
10427/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
10428static const SCEV *
10429SolveLinEquationWithOverflow(const APInt &A, const SCEV *B,
10430 SmallVectorImpl<const SCEVPredicate *> *Predicates,
10431 ScalarEvolution &SE, const Loop *L) {
10432 uint32_t BW = A.getBitWidth();
10433 assert(BW == SE.getTypeSizeInBits(B->getType()));
10434 assert(A != 0 && "A must be non-zero.");
10435
10436 // 1. D = gcd(A, N)
10437 //
10438 // The gcd of A and N may have only one prime factor: 2. The number of
10439 // trailing zeros in A is its multiplicity
10440 uint32_t Mult2 = A.countr_zero();
10441 // D = 2^Mult2
10442
10443 // 2. Check if B is divisible by D.
10444 //
10445 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
10446 // is not less than multiplicity of this prime factor for D.
10447 unsigned MinTZ = SE.getMinTrailingZeros(S: B);
10448 // Try again with the terminator of the loop predecessor for context-specific
10449 // result, if MinTZ s too small.
10450 if (MinTZ < Mult2 && L->getLoopPredecessor())
10451 MinTZ = SE.getMinTrailingZeros(S: B, CtxI: L->getLoopPredecessor()->getTerminator());
10452 if (MinTZ < Mult2) {
10453 // Check if we can prove there's no remainder using URem.
10454 const SCEV *URem =
10455 SE.getURemExpr(LHS: B, RHS: SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2)));
10456 const SCEV *Zero = SE.getZero(Ty: B->getType());
10457 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: URem, RHS: Zero)) {
10458 // Try to add a predicate ensuring B is a multiple of 1 << Mult2.
10459 if (!Predicates)
10460 return SE.getCouldNotCompute();
10461
10462 // Avoid adding a predicate that is known to be false.
10463 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_NE, LHS: URem, RHS: Zero))
10464 return SE.getCouldNotCompute();
10465 Predicates->push_back(Elt: SE.getEqualPredicate(LHS: URem, RHS: Zero));
10466 }
10467 }
10468
10469 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
10470 // modulo (N / D).
10471 //
10472 // If D == 1, (N / D) == N == 2^BW, so we need one extra bit to represent
10473 // (N / D) in general. The inverse itself always fits into BW bits, though,
10474 // so we immediately truncate it.
10475 APInt AD = A.lshr(shiftAmt: Mult2).trunc(width: BW - Mult2); // AD = A / D
10476 APInt I = AD.multiplicativeInverse().zext(width: BW);
10477
10478 // 4. Compute the minimum unsigned root of the equation:
10479 // I * (B / D) mod (N / D)
10480 // To simplify the computation, we factor out the divide by D:
10481 // (I * B mod N) / D
10482 const SCEV *D = SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2));
10483 return SE.getUDivExactExpr(LHS: SE.getMulExpr(LHS: B, RHS: SE.getConstant(Val: I)), RHS: D);
10484}
10485
10486/// For a given quadratic addrec, generate coefficients of the corresponding
10487/// quadratic equation, multiplied by a common value to ensure that they are
10488/// integers.
10489/// The returned value is a tuple { A, B, C, M, BitWidth }, where
10490/// Ax^2 + Bx + C is the quadratic function, M is the value that A, B and C
10491/// were multiplied by, and BitWidth is the bit width of the original addrec
10492/// coefficients.
10493/// This function returns std::nullopt if the addrec coefficients are not
10494/// compile- time constants.
10495static std::optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
10496GetQuadraticEquation(const SCEVAddRecExpr *AddRec) {
10497 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
10498 const SCEVConstant *LC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 0));
10499 const SCEVConstant *MC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 1));
10500 const SCEVConstant *NC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 2));
10501 LLVM_DEBUG(dbgs() << __func__ << ": analyzing quadratic addrec: "
10502 << *AddRec << '\n');
10503
10504 // We currently can only solve this if the coefficients are constants.
10505 if (!LC || !MC || !NC) {
10506 LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n");
10507 return std::nullopt;
10508 }
10509
10510 APInt L = LC->getAPInt();
10511 APInt M = MC->getAPInt();
10512 APInt N = NC->getAPInt();
10513 assert(!N.isZero() && "This is not a quadratic addrec");
10514
10515 unsigned BitWidth = LC->getAPInt().getBitWidth();
10516 unsigned NewWidth = BitWidth + 1;
10517 LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "
10518 << BitWidth << '\n');
10519 // The sign-extension (as opposed to a zero-extension) here matches the
10520 // extension used in SolveQuadraticEquationWrap (with the same motivation).
10521 N = N.sext(width: NewWidth);
10522 M = M.sext(width: NewWidth);
10523 L = L.sext(width: NewWidth);
10524
10525 // The increments are M, M+N, M+2N, ..., so the accumulated values are
10526 // L+M, (L+M)+(M+N), (L+M)+(M+N)+(M+2N), ..., that is,
10527 // L+M, L+2M+N, L+3M+3N, ...
10528 // After n iterations the accumulated value Acc is L + nM + n(n-1)/2 N.
10529 //
10530 // The equation Acc = 0 is then
10531 // L + nM + n(n-1)/2 N = 0, or 2L + 2M n + n(n-1) N = 0.
10532 // In a quadratic form it becomes:
10533 // N n^2 + (2M-N) n + 2L = 0.
10534
10535 APInt A = N;
10536 APInt B = 2 * M - A;
10537 APInt C = 2 * L;
10538 APInt T = APInt(NewWidth, 2);
10539 LLVM_DEBUG(dbgs() << __func__ << ": equation " << A << "x^2 + " << B
10540 << "x + " << C << ", coeff bw: " << NewWidth
10541 << ", multiplied by " << T << '\n');
10542 return std::make_tuple(args&: A, args&: B, args&: C, args&: T, args&: BitWidth);
10543}
10544
10545/// Helper function to compare optional APInts:
10546/// (a) if X and Y both exist, return min(X, Y),
10547/// (b) if neither X nor Y exist, return std::nullopt,
10548/// (c) if exactly one of X and Y exists, return that value.
10549static std::optional<APInt> MinOptional(std::optional<APInt> X,
10550 std::optional<APInt> Y) {
10551 if (X && Y) {
10552 unsigned W = std::max(a: X->getBitWidth(), b: Y->getBitWidth());
10553 APInt XW = X->sext(width: W);
10554 APInt YW = Y->sext(width: W);
10555 return XW.slt(RHS: YW) ? *X : *Y;
10556 }
10557 if (!X && !Y)
10558 return std::nullopt;
10559 return X ? *X : *Y;
10560}
10561
10562/// Helper function to truncate an optional APInt to a given BitWidth.
10563/// When solving addrec-related equations, it is preferable to return a value
10564/// that has the same bit width as the original addrec's coefficients. If the
10565/// solution fits in the original bit width, truncate it (except for i1).
10566/// Returning a value of a different bit width may inhibit some optimizations.
10567///
10568/// In general, a solution to a quadratic equation generated from an addrec
10569/// may require BW+1 bits, where BW is the bit width of the addrec's
10570/// coefficients. The reason is that the coefficients of the quadratic
10571/// equation are BW+1 bits wide (to avoid truncation when converting from
10572/// the addrec to the equation).
10573static std::optional<APInt> TruncIfPossible(std::optional<APInt> X,
10574 unsigned BitWidth) {
10575 if (!X)
10576 return std::nullopt;
10577 unsigned W = X->getBitWidth();
10578 if (BitWidth > 1 && BitWidth < W && X->isIntN(N: BitWidth))
10579 return X->trunc(width: BitWidth);
10580 return X;
10581}
10582
10583/// Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n
10584/// iterations. The values L, M, N are assumed to be signed, and they
10585/// should all have the same bit widths.
10586/// Find the least n >= 0 such that c(n) = 0 in the arithmetic modulo 2^BW,
10587/// where BW is the bit width of the addrec's coefficients.
10588/// If the calculated value is a BW-bit integer (for BW > 1), it will be
10589/// returned as such, otherwise the bit width of the returned value may
10590/// be greater than BW.
10591///
10592/// This function returns std::nullopt if
10593/// (a) the addrec coefficients are not constant, or
10594/// (b) SolveQuadraticEquationWrap was unable to find a solution. For cases
10595/// like x^2 = 5, no integer solutions exist, in other cases an integer
10596/// solution may exist, but SolveQuadraticEquationWrap may fail to find it.
10597static std::optional<APInt>
10598SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
10599 APInt A, B, C, M;
10600 unsigned BitWidth;
10601 auto T = GetQuadraticEquation(AddRec);
10602 if (!T)
10603 return std::nullopt;
10604
10605 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10606 LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n");
10607 std::optional<APInt> X =
10608 APIntOps::SolveQuadraticEquationWrap(A, B, C, RangeWidth: BitWidth + 1);
10609 if (!X)
10610 return std::nullopt;
10611
10612 ConstantInt *CX = ConstantInt::get(Context&: SE.getContext(), V: *X);
10613 ConstantInt *V = EvaluateConstantChrecAtConstant(AddRec, C: CX, SE);
10614 if (!V->isZero())
10615 return std::nullopt;
10616
10617 return TruncIfPossible(X, BitWidth);
10618}
10619
10620/// Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n
10621/// iterations. The values M, N are assumed to be signed, and they
10622/// should all have the same bit widths.
10623/// Find the least n such that c(n) does not belong to the given range,
10624/// while c(n-1) does.
10625///
10626/// This function returns std::nullopt if
10627/// (a) the addrec coefficients are not constant, or
10628/// (b) SolveQuadraticEquationWrap was unable to find a solution for the
10629/// bounds of the range.
10630static std::optional<APInt>
10631SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec,
10632 const ConstantRange &Range, ScalarEvolution &SE) {
10633 assert(AddRec->getOperand(0)->isZero() &&
10634 "Starting value of addrec should be 0");
10635 LLVM_DEBUG(dbgs() << __func__ << ": solving boundary crossing for range "
10636 << Range << ", addrec " << *AddRec << '\n');
10637 // This case is handled in getNumIterationsInRange. Here we can assume that
10638 // we start in the range.
10639 assert(Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) &&
10640 "Addrec's initial value should be in range");
10641
10642 APInt A, B, C, M;
10643 unsigned BitWidth;
10644 auto T = GetQuadraticEquation(AddRec);
10645 if (!T)
10646 return std::nullopt;
10647
10648 // Be careful about the return value: there can be two reasons for not
10649 // returning an actual number. First, if no solutions to the equations
10650 // were found, and second, if the solutions don't leave the given range.
10651 // The first case means that the actual solution is "unknown", the second
10652 // means that it's known, but not valid. If the solution is unknown, we
10653 // cannot make any conclusions.
10654 // Return a pair: the optional solution and a flag indicating if the
10655 // solution was found.
10656 auto SolveForBoundary =
10657 [&](APInt Bound) -> std::pair<std::optional<APInt>, bool> {
10658 // Solve for signed overflow and unsigned overflow, pick the lower
10659 // solution.
10660 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: checking boundary "
10661 << Bound << " (before multiplying by " << M << ")\n");
10662 Bound *= M; // The quadratic equation multiplier.
10663
10664 std::optional<APInt> SO;
10665 if (BitWidth > 1) {
10666 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10667 "signed overflow\n");
10668 SO = APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth);
10669 }
10670 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10671 "unsigned overflow\n");
10672 std::optional<APInt> UO =
10673 APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth + 1);
10674
10675 auto LeavesRange = [&] (const APInt &X) {
10676 ConstantInt *C0 = ConstantInt::get(Context&: SE.getContext(), V: X);
10677 ConstantInt *V0 = EvaluateConstantChrecAtConstant(AddRec, C: C0, SE);
10678 if (Range.contains(Val: V0->getValue()))
10679 return false;
10680 // X should be at least 1, so X-1 is non-negative.
10681 ConstantInt *C1 = ConstantInt::get(Context&: SE.getContext(), V: X-1);
10682 ConstantInt *V1 = EvaluateConstantChrecAtConstant(AddRec, C: C1, SE);
10683 if (Range.contains(Val: V1->getValue()))
10684 return true;
10685 return false;
10686 };
10687
10688 // If SolveQuadraticEquationWrap returns std::nullopt, it means that there
10689 // can be a solution, but the function failed to find it. We cannot treat it
10690 // as "no solution".
10691 if (!SO || !UO)
10692 return {std::nullopt, false};
10693
10694 // Check the smaller value first to see if it leaves the range.
10695 // At this point, both SO and UO must have values.
10696 std::optional<APInt> Min = MinOptional(X: SO, Y: UO);
10697 if (LeavesRange(*Min))
10698 return { Min, true };
10699 std::optional<APInt> Max = Min == SO ? UO : SO;
10700 if (LeavesRange(*Max))
10701 return { Max, true };
10702
10703 // Solutions were found, but were eliminated, hence the "true".
10704 return {std::nullopt, true};
10705 };
10706
10707 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10708 // Lower bound is inclusive, subtract 1 to represent the exiting value.
10709 APInt Lower = Range.getLower().sext(width: A.getBitWidth()) - 1;
10710 APInt Upper = Range.getUpper().sext(width: A.getBitWidth());
10711 auto SL = SolveForBoundary(Lower);
10712 auto SU = SolveForBoundary(Upper);
10713 // If any of the solutions was unknown, no meaninigful conclusions can
10714 // be made.
10715 if (!SL.second || !SU.second)
10716 return std::nullopt;
10717
10718 // Claim: The correct solution is not some value between Min and Max.
10719 //
10720 // Justification: Assuming that Min and Max are different values, one of
10721 // them is when the first signed overflow happens, the other is when the
10722 // first unsigned overflow happens. Crossing the range boundary is only
10723 // possible via an overflow (treating 0 as a special case of it, modeling
10724 // an overflow as crossing k*2^W for some k).
10725 //
10726 // The interesting case here is when Min was eliminated as an invalid
10727 // solution, but Max was not. The argument is that if there was another
10728 // overflow between Min and Max, it would also have been eliminated if
10729 // it was considered.
10730 //
10731 // For a given boundary, it is possible to have two overflows of the same
10732 // type (signed/unsigned) without having the other type in between: this
10733 // can happen when the vertex of the parabola is between the iterations
10734 // corresponding to the overflows. This is only possible when the two
10735 // overflows cross k*2^W for the same k. In such case, if the second one
10736 // left the range (and was the first one to do so), the first overflow
10737 // would have to enter the range, which would mean that either we had left
10738 // the range before or that we started outside of it. Both of these cases
10739 // are contradictions.
10740 //
10741 // Claim: In the case where SolveForBoundary returns std::nullopt, the correct
10742 // solution is not some value between the Max for this boundary and the
10743 // Min of the other boundary.
10744 //
10745 // Justification: Assume that we had such Max_A and Min_B corresponding
10746 // to range boundaries A and B and such that Max_A < Min_B. If there was
10747 // a solution between Max_A and Min_B, it would have to be caused by an
10748 // overflow corresponding to either A or B. It cannot correspond to B,
10749 // since Min_B is the first occurrence of such an overflow. If it
10750 // corresponded to A, it would have to be either a signed or an unsigned
10751 // overflow that is larger than both eliminated overflows for A. But
10752 // between the eliminated overflows and this overflow, the values would
10753 // cover the entire value space, thus crossing the other boundary, which
10754 // is a contradiction.
10755
10756 return TruncIfPossible(X: MinOptional(X: SL.first, Y: SU.first), BitWidth);
10757}
10758
10759ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V,
10760 const Loop *L,
10761 bool ControlsOnlyExit,
10762 bool AllowPredicates) {
10763
10764 // This is only used for loops with a "x != y" exit test. The exit condition
10765 // is now expressed as a single expression, V = x-y. So the exit test is
10766 // effectively V != 0. We know and take advantage of the fact that this
10767 // expression only being used in a comparison by zero context.
10768
10769 SmallVector<const SCEVPredicate *> Predicates;
10770 // If the value is a constant
10771 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10772 // If the value is already zero, the branch will execute zero times.
10773 if (C->getValue()->isZero()) return C;
10774 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10775 }
10776
10777 const SCEVAddRecExpr *AddRec =
10778 dyn_cast<SCEVAddRecExpr>(Val: stripInjectiveFunctions(S: V));
10779
10780 if (!AddRec && AllowPredicates)
10781 // Try to make this an AddRec using runtime tests, in the first X
10782 // iterations of this loop, where X is the SCEV expression found by the
10783 // algorithm below.
10784 AddRec = convertSCEVToAddRecWithPredicates(S: V, L, Preds&: Predicates);
10785
10786 if (!AddRec || AddRec->getLoop() != L)
10787 return getCouldNotCompute();
10788
10789 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
10790 // the quadratic equation to solve it.
10791 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
10792 // We can only use this value if the chrec ends up with an exact zero
10793 // value at this index. When solving for "X*X != 5", for example, we
10794 // should not accept a root of 2.
10795 if (auto S = SolveQuadraticAddRecExact(AddRec, SE&: *this)) {
10796 const auto *R = cast<SCEVConstant>(Val: getConstant(Val: *S));
10797 return ExitLimit(R, R, R, false, Predicates);
10798 }
10799 return getCouldNotCompute();
10800 }
10801
10802 // Otherwise we can only handle this if it is affine.
10803 if (!AddRec->isAffine())
10804 return getCouldNotCompute();
10805
10806 // If this is an affine expression, the execution count of this branch is
10807 // the minimum unsigned root of the following equation:
10808 //
10809 // Start + Step*N = 0 (mod 2^BW)
10810 //
10811 // equivalent to:
10812 //
10813 // Step*N = -Start (mod 2^BW)
10814 //
10815 // where BW is the common bit width of Start and Step.
10816
10817 // Get the initial value for the loop.
10818 const SCEV *Start = getSCEVAtScope(V: AddRec->getStart(), L: L->getParentLoop());
10819 const SCEV *Step = getSCEVAtScope(V: AddRec->getOperand(i: 1), L: L->getParentLoop());
10820
10821 if (!isLoopInvariant(S: Step, L))
10822 return getCouldNotCompute();
10823
10824 LoopGuards Guards = LoopGuards::collect(L, SE&: *this);
10825 // Specialize step for this loop so we get context sensitive facts below.
10826 const SCEV *StepWLG = applyLoopGuards(Expr: Step, Guards);
10827
10828 // For positive steps (counting up until unsigned overflow):
10829 // N = -Start/Step (as unsigned)
10830 // For negative steps (counting down to zero):
10831 // N = Start/-Step
10832 // First compute the unsigned distance from zero in the direction of Step.
10833 bool CountDown = isKnownNegative(S: StepWLG);
10834 if (!CountDown && !isKnownNonNegative(S: StepWLG))
10835 return getCouldNotCompute();
10836
10837 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(V: Start);
10838 // Handle unitary steps, which cannot wraparound.
10839 // 1*N = -Start; -1*N = Start (mod 2^BW), so:
10840 // N = Distance (as unsigned)
10841
10842 if (match(S: Step, P: m_CombineOr(Ps: m_scev_One(), Ps: m_scev_AllOnes()))) {
10843 APInt MaxBECount = getUnsignedRangeMax(S: applyLoopGuards(Expr: Distance, Guards));
10844 MaxBECount = APIntOps::umin(A: MaxBECount, B: getUnsignedRangeMax(S: Distance));
10845
10846 // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated,
10847 // we end up with a loop whose backedge-taken count is n - 1. Detect this
10848 // case, and see if we can improve the bound.
10849 //
10850 // Explicitly handling this here is necessary because getUnsignedRange
10851 // isn't context-sensitive; it doesn't know that we only care about the
10852 // range inside the loop.
10853 const SCEV *Zero = getZero(Ty: Distance->getType());
10854 const SCEV *One = getOne(Ty: Distance->getType());
10855 const SCEV *DistancePlusOne = getAddExpr(LHS: Distance, RHS: One);
10856 if (isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: DistancePlusOne, RHS: Zero)) {
10857 // If Distance + 1 doesn't overflow, we can compute the maximum distance
10858 // as "unsigned_max(Distance + 1) - 1". Also apply the loop guards to
10859 // Distance + 1; the range of Distance itself may be a wrapped set even
10860 // when the guards bound Distance + 1 tightly.
10861 APInt Max = APIntOps::umin(
10862 A: getUnsignedRangeMax(S: applyLoopGuards(Expr: DistancePlusOne, Guards)),
10863 B: getUnsignedRangeMax(S: DistancePlusOne));
10864 MaxBECount = APIntOps::umin(A: MaxBECount, B: Max - 1);
10865 }
10866 return ExitLimit(Distance, getConstant(Val: MaxBECount), Distance, false,
10867 Predicates);
10868 }
10869
10870 // If the condition controls loop exit (the loop exits only if the expression
10871 // is true) and the addition is no-wrap we can use unsigned divide to
10872 // compute the backedge count. In this case, the step may not divide the
10873 // distance, but we don't care because if the condition is "missed" the loop
10874 // will have undefined behavior due to wrapping.
10875 if (ControlsOnlyExit && AddRec->hasNoSelfWrap() &&
10876 loopHasNoAbnormalExits(L: AddRec->getLoop())) {
10877
10878 // If the stride is zero and the start is non-zero, the loop must be
10879 // infinite. In C++, most loops are finite by assumption, in which case the
10880 // step being zero implies UB must execute if the loop is entered.
10881 if (!(loopIsFiniteByAssumption(L) && isKnownNonZero(S: Start)) &&
10882 !isKnownNonZero(S: StepWLG))
10883 return getCouldNotCompute();
10884
10885 const SCEV *Exact =
10886 getUDivExpr(LHS: Distance, RHS: CountDown ? getNegativeSCEV(V: Step) : Step);
10887 const SCEV *ConstantMax = getCouldNotCompute();
10888 if (Exact != getCouldNotCompute()) {
10889 APInt MaxInt = getUnsignedRangeMax(S: applyLoopGuards(Expr: Exact, Guards));
10890 ConstantMax =
10891 getConstant(Val: APIntOps::umin(A: MaxInt, B: getUnsignedRangeMax(S: Exact)));
10892 }
10893 const SCEV *SymbolicMax =
10894 isa<SCEVCouldNotCompute>(Val: Exact) ? ConstantMax : Exact;
10895 return ExitLimit(Exact, ConstantMax, SymbolicMax, false, Predicates);
10896 }
10897
10898 // Solve the general equation.
10899 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Val: Step);
10900 if (!StepC || StepC->getValue()->isZero())
10901 return getCouldNotCompute();
10902 const SCEV *E = SolveLinEquationWithOverflow(
10903 A: StepC->getAPInt(), B: getNegativeSCEV(V: Start),
10904 Predicates: AllowPredicates ? &Predicates : nullptr, SE&: *this, L);
10905
10906 const SCEV *M = E;
10907 if (E != getCouldNotCompute()) {
10908 APInt MaxWithGuards = getUnsignedRangeMax(S: applyLoopGuards(Expr: E, Guards));
10909 M = getConstant(Val: APIntOps::umin(A: MaxWithGuards, B: getUnsignedRangeMax(S: E)));
10910 }
10911 auto *S = isa<SCEVCouldNotCompute>(Val: E) ? M : E;
10912 return ExitLimit(E, M, S, false, Predicates);
10913}
10914
10915ScalarEvolution::ExitLimit
10916ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) {
10917 // Loops that look like: while (X == 0) are very strange indeed. We don't
10918 // handle them yet except for the trivial case. This could be expanded in the
10919 // future as needed.
10920
10921 // If the value is a constant, check to see if it is known to be non-zero
10922 // already. If so, the backedge will execute zero times.
10923 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10924 if (!C->getValue()->isZero())
10925 return getZero(Ty: C->getType());
10926 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10927 }
10928
10929 // We could implement others, but I really doubt anyone writes loops like
10930 // this, and if they did, they would already be constant folded.
10931 return getCouldNotCompute();
10932}
10933
10934std::pair<const BasicBlock *, const BasicBlock *>
10935ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(const BasicBlock *BB)
10936 const {
10937 // If the block has a unique predecessor, then there is no path from the
10938 // predecessor to the block that does not go through the direct edge
10939 // from the predecessor to the block.
10940 if (const BasicBlock *Pred = BB->getSinglePredecessor())
10941 return {Pred, BB};
10942
10943 // A loop's header is defined to be a block that dominates the loop.
10944 // If the header has a unique predecessor outside the loop, it must be
10945 // a block that has exactly one successor that can reach the loop.
10946 if (const Loop *L = LI.getLoopFor(BB))
10947 return {L->getLoopPredecessor(), L->getHeader()};
10948
10949 return {nullptr, BB};
10950}
10951
10952/// SCEV structural equivalence is usually sufficient for testing whether two
10953/// expressions are equal, however for the purposes of looking for a condition
10954/// guarding a loop, it can be useful to be a little more general, since a
10955/// front-end may have replicated the controlling expression.
10956static bool HasSameValue(const SCEV *A, const SCEV *B) {
10957 // Quick check to see if they are the same SCEV.
10958 if (A == B) return true;
10959
10960 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) {
10961 // Not all instructions that are "identical" compute the same value. For
10962 // instance, two distinct alloca instructions allocating the same type are
10963 // identical and do not read memory; but compute distinct values.
10964 return A->isIdenticalTo(I: B) && (isa<BinaryOperator>(Val: A) || isa<GetElementPtrInst>(Val: A));
10965 };
10966
10967 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
10968 // two different instructions with the same value. Check for this case.
10969 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(Val: A))
10970 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(Val: B))
10971 if (const Instruction *AI = dyn_cast<Instruction>(Val: AU->getValue()))
10972 if (const Instruction *BI = dyn_cast<Instruction>(Val: BU->getValue()))
10973 if (ComputesEqualValues(AI, BI))
10974 return true;
10975
10976 // Otherwise assume they may have a different value.
10977 return false;
10978}
10979
10980static bool MatchBinarySub(const SCEV *S, SCEVUse &LHS, SCEVUse &RHS) {
10981 const SCEV *Op0, *Op1;
10982 if (!match(S, P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))))
10983 return false;
10984 if (match(S: Op0, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
10985 LHS = Op1;
10986 return true;
10987 }
10988 if (match(S: Op1, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
10989 LHS = Op0;
10990 return true;
10991 }
10992 return false;
10993}
10994
10995bool ScalarEvolution::SimplifyICmpOperands(CmpPredicate &Pred, SCEVUse &LHS,
10996 SCEVUse &RHS, unsigned Depth) {
10997 bool Changed = false;
10998 // Simplifies ICMP to trivial true or false by turning it into '0 == 0' or
10999 // '0 != 0'.
11000 auto TrivialCase = [&](bool TriviallyTrue) {
11001 LHS = RHS = getConstant(V: ConstantInt::getFalse(Context&: getContext()));
11002 Pred = TriviallyTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
11003 return true;
11004 };
11005 // If we hit the max recursion limit bail out.
11006 if (Depth >= 3)
11007 return false;
11008
11009 const SCEV *NewLHS, *NewRHS;
11010 if (match(U: LHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
11011 match(U: RHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale()))) {
11012 const SCEVMulExpr *LMul = cast<SCEVMulExpr>(Val&: LHS);
11013 const SCEVMulExpr *RMul = cast<SCEVMulExpr>(Val&: RHS);
11014
11015 // (X * vscale) pred (Y * vscale) ==> X pred Y
11016 // when both multiples are NSW.
11017 // (X * vscale) uicmp/eq/ne (Y * vscale) ==> X uicmp/eq/ne Y
11018 // when both multiples are NUW.
11019 if ((LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap()) ||
11020 (LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap() &&
11021 !ICmpInst::isSigned(Pred))) {
11022 LHS = NewLHS;
11023 RHS = NewRHS;
11024 Changed = true;
11025 }
11026 }
11027
11028 // Canonicalize a constant to the right side.
11029 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS)) {
11030 // Check for both operands constant.
11031 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
11032 if (!ICmpInst::compare(LHS: LHSC->getAPInt(), RHS: RHSC->getAPInt(), Pred))
11033 return TrivialCase(false);
11034 return TrivialCase(true);
11035 }
11036 // Otherwise swap the operands to put the constant on the right.
11037 std::swap(a&: LHS, b&: RHS);
11038 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11039 Changed = true;
11040 }
11041
11042 // (K + A) pred (K + B) --> A pred B
11043 // For equality, no flags are needed.
11044 // For signed, both adds must be NSW. For unsigned, both must be NUW.
11045 {
11046 const SCEVConstant *C = nullptr;
11047 if (match(U: LHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: NewLHS))) &&
11048 match(U: RHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: NewRHS)))) {
11049 const auto *LAdd = cast<SCEVAddExpr>(Val&: LHS);
11050 const auto *RAdd = cast<SCEVAddExpr>(Val&: RHS);
11051 if (ICmpInst::isEquality(P: Pred) ||
11052 (ICmpInst::isSigned(Pred) && LAdd->hasNoSignedWrap() &&
11053 RAdd->hasNoSignedWrap()) ||
11054 (ICmpInst::isUnsigned(Pred) && LAdd->hasNoUnsignedWrap() &&
11055 RAdd->hasNoUnsignedWrap())) {
11056 LHS = NewLHS;
11057 RHS = NewRHS;
11058 Changed = true;
11059 }
11060 }
11061 }
11062
11063 // (C * A) pred (C * B) --> A pred B
11064 // For equality predicates, both muls must be NUW or both must be NSW
11065 // (either suffices to make multiplication by C injective; C == 0 is
11066 // impossible because SCEV folds 0 * X to 0).
11067 // For signed ordering, C must be positive and both muls must be NSW.
11068 // For unsigned ordering, both muls must be NUW.
11069 {
11070 const SCEVConstant *C = nullptr;
11071 if (match(U: LHS, P: m_scev_Mul(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: NewLHS))) &&
11072 match(U: RHS, P: m_scev_Mul(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: NewRHS)))) {
11073 const auto *LMul = cast<SCEVMulExpr>(Val&: LHS);
11074 const auto *RMul = cast<SCEVMulExpr>(Val&: RHS);
11075 bool BothNUW = LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap();
11076 bool BothNSW = LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap();
11077 if ((ICmpInst::isEquality(P: Pred) && (BothNUW || BothNSW)) ||
11078 (ICmpInst::isSigned(Pred) && BothNSW &&
11079 C->getAPInt().isStrictlyPositive()) ||
11080 (ICmpInst::isUnsigned(Pred) && BothNUW)) {
11081 LHS = NewLHS;
11082 RHS = NewRHS;
11083 Changed = true;
11084 }
11085 }
11086 }
11087
11088 // If we're comparing an addrec with a value which is loop-invariant in the
11089 // addrec's loop, put the addrec on the left. Also make a dominance check,
11090 // as both operands could be addrecs loop-invariant in each other's loop.
11091 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: RHS)) {
11092 const Loop *L = AR->getLoop();
11093 if (isLoopInvariant(S: LHS, L) && properlyDominates(S: LHS, BB: L->getHeader())) {
11094 std::swap(a&: LHS, b&: RHS);
11095 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11096 Changed = true;
11097 }
11098 }
11099
11100 // If there's a constant operand, canonicalize comparisons with boundary
11101 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
11102 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(Val&: RHS)) {
11103 const APInt &RA = RC->getAPInt();
11104
11105 bool SimplifiedByConstantRange = false;
11106
11107 if (!ICmpInst::isEquality(P: Pred)) {
11108 ConstantRange ExactCR = ConstantRange::makeExactICmpRegion(Pred, Other: RA);
11109 if (ExactCR.isFullSet())
11110 return TrivialCase(true);
11111 if (ExactCR.isEmptySet())
11112 return TrivialCase(false);
11113
11114 APInt NewRHS;
11115 CmpInst::Predicate NewPred;
11116 if (ExactCR.getEquivalentICmp(Pred&: NewPred, RHS&: NewRHS) &&
11117 ICmpInst::isEquality(P: NewPred)) {
11118 // We were able to convert an inequality to an equality.
11119 Pred = NewPred;
11120 RHS = getConstant(Val: NewRHS);
11121 Changed = SimplifiedByConstantRange = true;
11122 }
11123 }
11124
11125 if (!SimplifiedByConstantRange) {
11126 switch (Pred) {
11127 default:
11128 break;
11129 case ICmpInst::ICMP_EQ:
11130 case ICmpInst::ICMP_NE:
11131 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
11132 if (RA.isZero() && MatchBinarySub(S: LHS, LHS, RHS))
11133 Changed = true;
11134 break;
11135
11136 // The "Should have been caught earlier!" messages refer to the fact
11137 // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above
11138 // should have fired on the corresponding cases, and canonicalized the
11139 // check to trivial case.
11140
11141 case ICmpInst::ICMP_UGE:
11142 assert(!RA.isMinValue() && "Should have been caught earlier!");
11143 Pred = ICmpInst::ICMP_UGT;
11144 RHS = getConstant(Val: RA - 1);
11145 Changed = true;
11146 break;
11147 case ICmpInst::ICMP_ULE:
11148 assert(!RA.isMaxValue() && "Should have been caught earlier!");
11149 Pred = ICmpInst::ICMP_ULT;
11150 RHS = getConstant(Val: RA + 1);
11151 Changed = true;
11152 break;
11153 case ICmpInst::ICMP_SGE:
11154 assert(!RA.isMinSignedValue() && "Should have been caught earlier!");
11155 Pred = ICmpInst::ICMP_SGT;
11156 RHS = getConstant(Val: RA - 1);
11157 Changed = true;
11158 break;
11159 case ICmpInst::ICMP_SLE:
11160 assert(!RA.isMaxSignedValue() && "Should have been caught earlier!");
11161 Pred = ICmpInst::ICMP_SLT;
11162 RHS = getConstant(Val: RA + 1);
11163 Changed = true;
11164 break;
11165 }
11166 }
11167 }
11168
11169 // a /u b == 0 => a < b
11170 // a /u b != 0 => a >= b
11171 if (ICmpInst::isEquality(P: Pred) && RHS->isZero() &&
11172 match(U: LHS, P: m_scev_UDiv(Op0: m_SCEV(V&: LHS), Op1: m_SCEV(V&: RHS)))) {
11173 Pred = Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE;
11174 Changed = true;
11175 }
11176
11177 // Check for obvious equality.
11178 if (HasSameValue(A: LHS, B: RHS)) {
11179 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
11180 return TrivialCase(true);
11181 if (ICmpInst::isFalseWhenEqual(predicate: Pred))
11182 return TrivialCase(false);
11183 }
11184
11185 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
11186 // adding or subtracting 1 from one of the operands.
11187 switch (Pred) {
11188 case ICmpInst::ICMP_SLE:
11189 if (!getSignedRangeMax(S: RHS).isMaxSignedValue()) {
11190 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11191 Flags: SCEV::FlagNSW);
11192 Pred = ICmpInst::ICMP_SLT;
11193 Changed = true;
11194 } else if (!getSignedRangeMin(S: LHS).isMinSignedValue()) {
11195 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS,
11196 Flags: SCEV::FlagNSW);
11197 Pred = ICmpInst::ICMP_SLT;
11198 Changed = true;
11199 }
11200 break;
11201 case ICmpInst::ICMP_SGE:
11202 if (!getSignedRangeMin(S: RHS).isMinSignedValue()) {
11203 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS,
11204 Flags: SCEV::FlagNSW);
11205 Pred = ICmpInst::ICMP_SGT;
11206 Changed = true;
11207 } else if (!getSignedRangeMax(S: LHS).isMaxSignedValue()) {
11208 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11209 Flags: SCEV::FlagNSW);
11210 Pred = ICmpInst::ICMP_SGT;
11211 Changed = true;
11212 }
11213 break;
11214 case ICmpInst::ICMP_ULE:
11215 if (!getUnsignedRangeMax(S: RHS).isMaxValue()) {
11216 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11217 Flags: SCEV::FlagNUW);
11218 Pred = ICmpInst::ICMP_ULT;
11219 Changed = true;
11220 } else if (!getUnsignedRangeMin(S: LHS).isMinValue()) {
11221 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS);
11222 Pred = ICmpInst::ICMP_ULT;
11223 Changed = true;
11224 }
11225 break;
11226 case ICmpInst::ICMP_UGE:
11227 // If RHS is an op we can fold the -1, try that first.
11228 // Otherwise prefer LHS to preserve the nuw flag.
11229 if ((isa<SCEVConstant>(Val: RHS) ||
11230 (isa<SCEVAddExpr, SCEVAddRecExpr>(Val: RHS) &&
11231 isa<SCEVConstant>(Val: cast<SCEVNAryExpr>(Val&: RHS)->getOperand(i: 0)))) &&
11232 !getUnsignedRangeMin(S: RHS).isMinValue()) {
11233 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11234 Pred = ICmpInst::ICMP_UGT;
11235 Changed = true;
11236 } else if (!getUnsignedRangeMax(S: LHS).isMaxValue()) {
11237 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11238 Flags: SCEV::FlagNUW);
11239 Pred = ICmpInst::ICMP_UGT;
11240 Changed = true;
11241 } else if (!getUnsignedRangeMin(S: RHS).isMinValue()) {
11242 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11243 Pred = ICmpInst::ICMP_UGT;
11244 Changed = true;
11245 }
11246 break;
11247 default:
11248 break;
11249 }
11250
11251 // TODO: More simplifications are possible here.
11252
11253 // Recursively simplify until we either hit a recursion limit or nothing
11254 // changes.
11255 if (Changed)
11256 (void)SimplifyICmpOperands(Pred, LHS, RHS, Depth: Depth + 1);
11257
11258 return Changed;
11259}
11260
11261bool ScalarEvolution::isKnownNegative(const SCEV *S) {
11262 return getSignedRangeMax(S).isNegative();
11263}
11264
11265bool ScalarEvolution::isKnownPositive(const SCEV *S) {
11266 return getSignedRangeMin(S).isStrictlyPositive();
11267}
11268
11269bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
11270 return !getSignedRangeMin(S).isNegative();
11271}
11272
11273bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
11274 return !getSignedRangeMax(S).isStrictlyPositive();
11275}
11276
11277bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
11278 // Query push down for cases where the unsigned range is
11279 // less than sufficient.
11280 if (const auto *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
11281 return isKnownNonZero(S: SExt->getOperand(i: 0));
11282 return getUnsignedRangeMin(S) != 0;
11283}
11284
11285bool ScalarEvolution::isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero,
11286 bool OrNegative) {
11287 auto NonRecursive = [OrNegative](const SCEV *S) {
11288 if (auto *C = dyn_cast<SCEVConstant>(Val: S))
11289 return C->getAPInt().isPowerOf2() ||
11290 (OrNegative && C->getAPInt().isNegatedPowerOf2());
11291
11292 // vscale is a power-of-two.
11293 return isa<SCEVVScale>(Val: S);
11294 };
11295
11296 if (NonRecursive(S))
11297 return true;
11298
11299 auto *Mul = dyn_cast<SCEVMulExpr>(Val: S);
11300 if (!Mul)
11301 return false;
11302 return all_of(Range: Mul->operands(), P: NonRecursive) && (OrZero || isKnownNonZero(S));
11303}
11304
11305bool ScalarEvolution::isKnownMultipleOf(
11306 const SCEV *S, uint64_t M,
11307 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
11308 if (M == 0)
11309 return false;
11310 if (M == 1)
11311 return true;
11312
11313 // For a constant, check that "S % M == 0".
11314 if (auto *Cst = dyn_cast<SCEVConstant>(Val: S)) {
11315 APInt C = Cst->getAPInt();
11316 return C.urem(RHS: M) == 0;
11317 }
11318
11319 // Basic tests have failed.
11320 // Check "S % M == 0" at compile time and record runtime Assumptions.
11321 auto *STy = dyn_cast<IntegerType>(Val: S->getType());
11322 const SCEV *SmodM =
11323 getURemExpr(LHS: S, RHS: getConstant(V: ConstantInt::get(Ty: STy, V: M, IsSigned: false)));
11324 const SCEV *Zero = getZero(Ty: STy);
11325
11326 // Check whether "S % M == 0" is known at compile time.
11327 if (isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero))
11328 return true;
11329
11330 // Check whether "S % M != 0" is known at compile time.
11331 if (isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: SmodM, RHS: Zero))
11332 return false;
11333
11334 if (!Predicates)
11335 return false;
11336
11337 // Look through Add and AddRec expressions with nuw to improve the
11338 // precision of added predicates. S is a multiple of M if S starts with a
11339 // multiple of M and at every iteration step S only adds multiples of M.
11340 if (isa<SCEVAddExpr, SCEVAddRecExpr>(Val: S) &&
11341 cast<SCEVNAryExpr>(Val: S)->hasNoUnsignedWrap() &&
11342 all_of(Range: S->operands(),
11343 P: [&](SCEVUse Op) { return isKnownMultipleOf(S: Op, M, Predicates); }))
11344 return true;
11345
11346 // Similarly, look through Mul with nuw, where any operand being a
11347 // known-multiple is sufficient.
11348 if (auto *Mul = dyn_cast<SCEVMulExpr>(Val: S))
11349 if (Mul->hasNoUnsignedWrap() && any_of(Range: S->operands(), P: [&](SCEVUse Op) {
11350 return isKnownMultipleOf(S: Op, M, Predicates);
11351 }))
11352 return true;
11353
11354 // Similarly, look through MinMax, with no wrapping arithmetic to consider.
11355 if (isa<SCEVMinMaxExpr>(Val: S) && all_of(Range: S->operands(), P: [&](SCEVUse Op) {
11356 return isKnownMultipleOf(S: Op, M, Predicates);
11357 }))
11358 return true;
11359
11360 const SCEVPredicate *P = getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero);
11361
11362 // Detect redundant predicates.
11363 for (auto *A : *Predicates)
11364 if (A->implies(N: P, SE&: *this))
11365 return true;
11366
11367 // Only record non-redundant predicates.
11368 Predicates->push_back(Elt: P);
11369 return true;
11370}
11371
11372bool ScalarEvolution::haveSameSign(const SCEV *S1, const SCEV *S2) {
11373 return ((isKnownNonNegative(S: S1) && isKnownNonNegative(S: S2)) ||
11374 (isKnownNegative(S: S1) && isKnownNegative(S: S2)));
11375}
11376
11377std::pair<const SCEV *, const SCEV *>
11378ScalarEvolution::SplitIntoInitAndPostInc(const Loop *L, const SCEV *S) {
11379 // Compute SCEV on entry of loop L.
11380 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, SE&: *this);
11381 if (Start == getCouldNotCompute())
11382 return { Start, Start };
11383 // Compute post increment SCEV for loop L.
11384 const SCEV *PostInc = SCEVPostIncRewriter::rewrite(S, L, SE&: *this);
11385 assert(PostInc != getCouldNotCompute() && "Unexpected could not compute");
11386 return { Start, PostInc };
11387}
11388
11389bool ScalarEvolution::isKnownViaInduction(CmpPredicate Pred, SCEVUse LHS,
11390 SCEVUse RHS) {
11391 // First collect all loops.
11392 SmallPtrSet<const Loop *, 8> LoopsUsed;
11393 getUsedLoops(S: LHS, LoopsUsed);
11394 getUsedLoops(S: RHS, LoopsUsed);
11395
11396 if (LoopsUsed.empty())
11397 return false;
11398
11399 // Domination relationship must be a linear order on collected loops.
11400#ifndef NDEBUG
11401 for (const auto *L1 : LoopsUsed)
11402 for (const auto *L2 : LoopsUsed)
11403 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||
11404 DT.dominates(L2->getHeader(), L1->getHeader())) &&
11405 "Domination relationship is not a linear order");
11406#endif
11407
11408 const Loop *MDL =
11409 *llvm::max_element(Range&: LoopsUsed, C: [&](const Loop *L1, const Loop *L2) {
11410 return DT.properlyDominates(A: L1->getHeader(), B: L2->getHeader());
11411 });
11412
11413 // Get init and post increment value for LHS.
11414 auto SplitLHS = SplitIntoInitAndPostInc(L: MDL, S: LHS);
11415 // if LHS contains unknown non-invariant SCEV then bail out.
11416 if (SplitLHS.first == getCouldNotCompute())
11417 return false;
11418 assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
11419 // Get init and post increment value for RHS.
11420 auto SplitRHS = SplitIntoInitAndPostInc(L: MDL, S: RHS);
11421 // if RHS contains unknown non-invariant SCEV then bail out.
11422 if (SplitRHS.first == getCouldNotCompute())
11423 return false;
11424 assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
11425 // It is possible that init SCEV contains an invariant load but it does
11426 // not dominate MDL and is not available at MDL loop entry, so we should
11427 // check it here.
11428 if (!isAvailableAtLoopEntry(S: SplitLHS.first, L: MDL) ||
11429 !isAvailableAtLoopEntry(S: SplitRHS.first, L: MDL))
11430 return false;
11431
11432 // It seems backedge guard check is faster than entry one so in some cases
11433 // it can speed up whole estimation by short circuit
11434 return isLoopBackedgeGuardedByCond(L: MDL, Pred, LHS: SplitLHS.second,
11435 RHS: SplitRHS.second) &&
11436 isLoopEntryGuardedByCond(L: MDL, Pred, LHS: SplitLHS.first, RHS: SplitRHS.first);
11437}
11438
11439bool ScalarEvolution::isKnownPredicate(CmpPredicate Pred, SCEVUse LHS,
11440 SCEVUse RHS) {
11441 // Canonicalize the inputs first.
11442 (void)SimplifyICmpOperands(Pred, LHS, RHS);
11443
11444 return isKnownViaInduction(Pred, LHS, RHS) ||
11445 isKnownPredicateViaSplitting(Pred, LHS, RHS) ||
11446 isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
11447}
11448
11449std::optional<bool> ScalarEvolution::evaluatePredicate(CmpPredicate Pred,
11450 const SCEV *LHS,
11451 const SCEV *RHS) {
11452 if (isKnownPredicate(Pred, LHS, RHS))
11453 return true;
11454 if (isKnownPredicate(Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11455 return false;
11456 return std::nullopt;
11457}
11458
11459bool ScalarEvolution::isKnownPredicateAt(CmpPredicate Pred, const SCEV *LHS,
11460 const SCEV *RHS,
11461 const Instruction *CtxI) {
11462 // TODO: Analyze guards and assumes from Context's block.
11463 return isKnownPredicate(Pred, LHS, RHS) ||
11464 isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS);
11465}
11466
11467std::optional<bool>
11468ScalarEvolution::evaluatePredicateAt(CmpPredicate Pred, const SCEV *LHS,
11469 const SCEV *RHS, const Instruction *CtxI) {
11470 std::optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
11471 if (KnownWithoutContext)
11472 return KnownWithoutContext;
11473
11474 if (isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS))
11475 return true;
11476 if (isBasicBlockEntryGuardedByCond(
11477 BB: CtxI->getParent(), Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11478 return false;
11479 return std::nullopt;
11480}
11481
11482bool ScalarEvolution::isKnownOnEveryIteration(CmpPredicate Pred,
11483 const SCEVAddRecExpr *LHS,
11484 const SCEV *RHS) {
11485 const Loop *L = LHS->getLoop();
11486 return isLoopEntryGuardedByCond(L, Pred, LHS: LHS->getStart(), RHS) &&
11487 isLoopBackedgeGuardedByCond(L, Pred, LHS: LHS->getPostIncExpr(SE&: *this), RHS);
11488}
11489
11490std::optional<ScalarEvolution::MonotonicPredicateType>
11491ScalarEvolution::getMonotonicPredicateType(const SCEVAddRecExpr *LHS,
11492 ICmpInst::Predicate Pred) {
11493 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
11494
11495#ifndef NDEBUG
11496 // Verify an invariant: inverting the predicate should turn a monotonically
11497 // increasing change to a monotonically decreasing one, and vice versa.
11498 if (Result) {
11499 auto ResultSwapped =
11500 getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred));
11501
11502 assert(*ResultSwapped != *Result &&
11503 "monotonicity should flip as we flip the predicate");
11504 }
11505#endif
11506
11507 return Result;
11508}
11509
11510std::optional<ScalarEvolution::MonotonicPredicateType>
11511ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
11512 ICmpInst::Predicate Pred) {
11513 // A zero step value for LHS means the induction variable is essentially a
11514 // loop invariant value. We don't really depend on the predicate actually
11515 // flipping from false to true (for increasing predicates, and the other way
11516 // around for decreasing predicates), all we care about is that *if* the
11517 // predicate changes then it only changes from false to true.
11518 //
11519 // A zero step value in itself is not very useful, but there may be places
11520 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
11521 // as general as possible.
11522
11523 // Only handle LE/LT/GE/GT predicates.
11524 if (!ICmpInst::isRelational(P: Pred))
11525 return std::nullopt;
11526
11527 bool IsGreater = ICmpInst::isGE(P: Pred) || ICmpInst::isGT(P: Pred);
11528 assert((IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) &&
11529 "Should be greater or less!");
11530
11531 // Check that AR does not wrap.
11532 if (ICmpInst::isUnsigned(Pred)) {
11533 if (!LHS->hasNoUnsignedWrap())
11534 return std::nullopt;
11535 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11536 }
11537 assert(ICmpInst::isSigned(Pred) &&
11538 "Relational predicate is either signed or unsigned!");
11539 if (!LHS->hasNoSignedWrap())
11540 return std::nullopt;
11541
11542 const SCEV *Step = LHS->getStepRecurrence(SE&: *this);
11543
11544 if (isKnownNonNegative(S: Step))
11545 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11546
11547 if (isKnownNonPositive(S: Step))
11548 return !IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11549
11550 return std::nullopt;
11551}
11552
11553std::optional<ScalarEvolution::LoopInvariantPredicate>
11554ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
11555 const SCEV *RHS, const Loop *L,
11556 const Instruction *CtxI) {
11557 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11558 if (!isLoopInvariant(S: RHS, L)) {
11559 if (!isLoopInvariant(S: LHS, L))
11560 return std::nullopt;
11561
11562 std::swap(a&: LHS, b&: RHS);
11563 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11564 }
11565
11566 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11567 if (!ArLHS || ArLHS->getLoop() != L)
11568 return std::nullopt;
11569
11570 auto MonotonicType = getMonotonicPredicateType(LHS: ArLHS, Pred);
11571 if (!MonotonicType)
11572 return std::nullopt;
11573 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
11574 // true as the loop iterates, and the backedge is control dependent on
11575 // "ArLHS `Pred` RHS" == true then we can reason as follows:
11576 //
11577 // * if the predicate was false in the first iteration then the predicate
11578 // is never evaluated again, since the loop exits without taking the
11579 // backedge.
11580 // * if the predicate was true in the first iteration then it will
11581 // continue to be true for all future iterations since it is
11582 // monotonically increasing.
11583 //
11584 // For both the above possibilities, we can replace the loop varying
11585 // predicate with its value on the first iteration of the loop (which is
11586 // loop invariant).
11587 //
11588 // A similar reasoning applies for a monotonically decreasing predicate, by
11589 // replacing true with false and false with true in the above two bullets.
11590 bool Increasing = *MonotonicType == ScalarEvolution::MonotonicallyIncreasing;
11591 auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
11592
11593 if (isLoopBackedgeGuardedByCond(L, Pred: P, LHS, RHS))
11594 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11595 RHS);
11596
11597 if (!CtxI)
11598 return std::nullopt;
11599 // Try to prove via context.
11600 // TODO: Support other cases.
11601 switch (Pred) {
11602 default:
11603 break;
11604 case ICmpInst::ICMP_ULE:
11605 case ICmpInst::ICMP_ULT: {
11606 assert(ArLHS->hasNoUnsignedWrap() && "Is a requirement of monotonicity!");
11607 // Given preconditions
11608 // (1) ArLHS does not cross the border of positive and negative parts of
11609 // range because of:
11610 // - Positive step; (TODO: lift this limitation)
11611 // - nuw - does not cross zero boundary;
11612 // - nsw - does not cross SINT_MAX boundary;
11613 // (2) ArLHS <s RHS
11614 // (3) RHS >=s 0
11615 // we can replace the loop variant ArLHS <u RHS condition with loop
11616 // invariant Start(ArLHS) <u RHS.
11617 //
11618 // Because of (1) there are two options:
11619 // - ArLHS is always negative. It means that ArLHS <u RHS is always false;
11620 // - ArLHS is always non-negative. Because of (3) RHS is also non-negative.
11621 // It means that ArLHS <s RHS <=> ArLHS <u RHS.
11622 // Because of (2) ArLHS <u RHS is trivially true.
11623 // All together it means that ArLHS <u RHS <=> Start(ArLHS) >=s 0.
11624 // We can strengthen this to Start(ArLHS) <u RHS.
11625 auto SignFlippedPred = ICmpInst::getFlippedSignednessPredicate(Pred);
11626 if (ArLHS->hasNoSignedWrap() && ArLHS->isAffine() &&
11627 isKnownPositive(S: ArLHS->getStepRecurrence(SE&: *this)) &&
11628 isKnownNonNegative(S: RHS) &&
11629 isKnownPredicateAt(Pred: SignFlippedPred, LHS: ArLHS, RHS, CtxI))
11630 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11631 RHS);
11632 }
11633 }
11634
11635 return std::nullopt;
11636}
11637
11638std::optional<ScalarEvolution::LoopInvariantPredicate>
11639ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations(
11640 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11641 const Instruction *CtxI, const SCEV *MaxIter) {
11642 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11643 Pred, LHS, RHS, L, CtxI, MaxIter))
11644 return LIP;
11645 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: MaxIter))
11646 // Number of iterations expressed as UMIN isn't always great for expressing
11647 // the value on the last iteration. If the straightforward approach didn't
11648 // work, try the following trick: if the a predicate is invariant for X, it
11649 // is also invariant for umin(X, ...). So try to find something that works
11650 // among subexpressions of MaxIter expressed as umin.
11651 for (SCEVUse Op : UMin->operands())
11652 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11653 Pred, LHS, RHS, L, CtxI, MaxIter: Op))
11654 return LIP;
11655 return std::nullopt;
11656}
11657
11658std::optional<ScalarEvolution::LoopInvariantPredicate>
11659ScalarEvolution::getLoopInvariantExitCondDuringFirstIterationsImpl(
11660 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11661 const Instruction *CtxI, const SCEV *MaxIter) {
11662 // Try to prove the following set of facts:
11663 // - The predicate is monotonic in the iteration space.
11664 // - If the check does not fail on the 1st iteration:
11665 // - No overflow will happen during first MaxIter iterations;
11666 // - It will not fail on the MaxIter'th iteration.
11667 // If the check does fail on the 1st iteration, we leave the loop and no
11668 // other checks matter.
11669
11670 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11671 if (!isLoopInvariant(S: RHS, L)) {
11672 if (!isLoopInvariant(S: LHS, L))
11673 return std::nullopt;
11674
11675 std::swap(a&: LHS, b&: RHS);
11676 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11677 }
11678
11679 auto *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11680 if (!AR || AR->getLoop() != L)
11681 return std::nullopt;
11682
11683 // Even if both are valid, we need to consistently chose the unsigned or the
11684 // signed predicate below, not mixtures of both. For now, prefer the unsigned
11685 // predicate.
11686 Pred = Pred.dropSameSign();
11687
11688 // The predicate must be relational (i.e. <, <=, >=, >).
11689 if (!ICmpInst::isRelational(P: Pred))
11690 return std::nullopt;
11691
11692 // TODO: Support steps other than +/- 1.
11693 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
11694 auto *One = getOne(Ty: Step->getType());
11695 auto *MinusOne = getNegativeSCEV(V: One);
11696 if (Step != One && Step != MinusOne)
11697 return std::nullopt;
11698
11699 // Type mismatch here means that MaxIter is potentially larger than max
11700 // unsigned value in start type, which mean we cannot prove no wrap for the
11701 // indvar.
11702 if (AR->getType() != MaxIter->getType())
11703 return std::nullopt;
11704
11705 // Value of IV on suggested last iteration.
11706 const SCEV *Last = AR->evaluateAtIteration(It: MaxIter, SE&: *this);
11707 // Does it still meet the requirement?
11708 if (!isLoopBackedgeGuardedByCond(L, Pred, LHS: Last, RHS))
11709 return std::nullopt;
11710 // Because step is +/- 1 and MaxIter has same type as Start (i.e. it does
11711 // not exceed max unsigned value of this type), this effectively proves
11712 // that there is no wrap during the iteration. To prove that there is no
11713 // signed/unsigned wrap, we need to check that
11714 // Start <= Last for step = 1 or Start >= Last for step = -1.
11715 ICmpInst::Predicate NoOverflowPred =
11716 CmpInst::isSigned(Pred) ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
11717 if (Step == MinusOne)
11718 NoOverflowPred = ICmpInst::getSwappedPredicate(pred: NoOverflowPred);
11719 const SCEV *Start = AR->getStart();
11720 if (!isKnownPredicateAt(Pred: NoOverflowPred, LHS: Start, RHS: Last, CtxI))
11721 return std::nullopt;
11722
11723 // Everything is fine.
11724 return ScalarEvolution::LoopInvariantPredicate(Pred, Start, RHS);
11725}
11726
11727bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
11728 SCEVUse LHS,
11729 SCEVUse RHS) {
11730 if (HasSameValue(A: LHS, B: RHS))
11731 return ICmpInst::isTrueWhenEqual(predicate: Pred);
11732
11733 auto CheckRange = [&](bool IsSigned) {
11734 auto RangeLHS = IsSigned ? getSignedRange(S: LHS) : getUnsignedRange(S: LHS);
11735 auto RangeRHS = IsSigned ? getSignedRange(S: RHS) : getUnsignedRange(S: RHS);
11736 return RangeLHS.icmp(Pred, Other: RangeRHS);
11737 };
11738
11739 // The check at the top of the function catches the case where the values are
11740 // known to be equal.
11741 if (Pred == CmpInst::ICMP_EQ)
11742 return false;
11743
11744 if (Pred == CmpInst::ICMP_NE) {
11745 if (CheckRange(true) || CheckRange(false))
11746 return true;
11747 auto *Diff = getMinusSCEV(LHS, RHS);
11748 return !isa<SCEVCouldNotCompute>(Val: Diff) && isKnownNonZero(S: Diff);
11749 }
11750
11751 return CheckRange(CmpInst::isSigned(Pred));
11752}
11753
11754bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
11755 SCEVUse LHS, SCEVUse RHS) {
11756 // Match X to (A + C1)<ExpectedFlags> and Y to (A + C2)<ExpectedFlags>, where
11757 // C1 and C2 are constant integers. If either X or Y are not add expressions,
11758 // consider them as X + 0 and Y + 0 respectively. C1 and C2 are returned via
11759 // OutC1 and OutC2.
11760 auto MatchBinaryAddToConst = [this](SCEVUse X, SCEVUse Y, APInt &OutC1,
11761 APInt &OutC2, SCEVFlags ExpectedFlags) {
11762 SCEVUse XNonConstOp, XConstOp;
11763 SCEVUse YNonConstOp, YConstOp;
11764 SCEVFlags XFlagsPresent;
11765 SCEVFlags YFlagsPresent;
11766
11767 if (!splitBinaryAdd(Expr: X, L&: XConstOp, R&: XNonConstOp, Flags&: XFlagsPresent)) {
11768 XConstOp = getZero(Ty: X->getType());
11769 XNonConstOp = X;
11770 XFlagsPresent = ExpectedFlags;
11771 }
11772 if (!isa<SCEVConstant>(Val: XConstOp))
11773 return false;
11774
11775 if (!splitBinaryAdd(Expr: Y, L&: YConstOp, R&: YNonConstOp, Flags&: YFlagsPresent)) {
11776 YConstOp = getZero(Ty: Y->getType());
11777 YNonConstOp = Y;
11778 YFlagsPresent = ExpectedFlags;
11779 }
11780
11781 if (YNonConstOp != XNonConstOp)
11782 return false;
11783
11784 if (!isa<SCEVConstant>(Val: YConstOp))
11785 return false;
11786
11787 // When matching ADDs with NUW flags (and unsigned predicates), only the
11788 // second ADD (with the larger constant) requires NUW.
11789 if ((YFlagsPresent & ExpectedFlags) != ExpectedFlags)
11790 return false;
11791 if (ExpectedFlags != SCEV::FlagNUW &&
11792 (XFlagsPresent & ExpectedFlags) != ExpectedFlags) {
11793 return false;
11794 }
11795
11796 OutC1 = cast<SCEVConstant>(Val&: XConstOp)->getAPInt();
11797 OutC2 = cast<SCEVConstant>(Val&: YConstOp)->getAPInt();
11798
11799 return true;
11800 };
11801
11802 APInt C1;
11803 APInt C2;
11804
11805 switch (Pred) {
11806 default:
11807 break;
11808
11809 case ICmpInst::ICMP_SGE:
11810 std::swap(a&: LHS, b&: RHS);
11811 [[fallthrough]];
11812 case ICmpInst::ICMP_SLE:
11813 // (X + C1)<nsw> s<= (X + C2)<nsw> if C1 s<= C2.
11814 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.sle(RHS: C2))
11815 return true;
11816
11817 break;
11818
11819 case ICmpInst::ICMP_SGT:
11820 std::swap(a&: LHS, b&: RHS);
11821 [[fallthrough]];
11822 case ICmpInst::ICMP_SLT:
11823 // (X + C1)<nsw> s< (X + C2)<nsw> if C1 s< C2.
11824 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.slt(RHS: C2))
11825 return true;
11826
11827 break;
11828
11829 case ICmpInst::ICMP_UGE:
11830 std::swap(a&: LHS, b&: RHS);
11831 [[fallthrough]];
11832 case ICmpInst::ICMP_ULE:
11833 // (X + C1) u<= (X + C2)<nuw> for C1 u<= C2.
11834 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(RHS: C2))
11835 return true;
11836
11837 break;
11838
11839 case ICmpInst::ICMP_UGT:
11840 std::swap(a&: LHS, b&: RHS);
11841 [[fallthrough]];
11842 case ICmpInst::ICMP_ULT:
11843 // (X + C1) u< (X + C2)<nuw> if C1 u< C2.
11844 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(RHS: C2))
11845 return true;
11846 break;
11847 }
11848
11849 return false;
11850}
11851
11852bool ScalarEvolution::isKnownPredicateViaSplitting(CmpPredicate Pred,
11853 SCEVUse LHS, SCEVUse RHS) {
11854 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate)
11855 return false;
11856
11857 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on
11858 // the stack can result in exponential time complexity.
11859 SaveAndRestore Restore(ProvingSplitPredicate, true);
11860
11861 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L
11862 //
11863 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use
11864 // isKnownPredicate. isKnownPredicate is more powerful, but also more
11865 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the
11866 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to
11867 // use isKnownPredicate later if needed.
11868 return isKnownNonNegative(S: RHS) &&
11869 isKnownPredicate(Pred: CmpInst::ICMP_SGE, LHS, RHS: getZero(Ty: LHS->getType())) &&
11870 isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS, RHS);
11871}
11872
11873bool ScalarEvolution::isImpliedViaGuard(const BasicBlock *BB, CmpPredicate Pred,
11874 const SCEV *LHS, const SCEV *RHS) {
11875 // No need to even try if we know the module has no guards.
11876 if (!HasGuards)
11877 return false;
11878
11879 return any_of(Range: *BB, P: [&](const Instruction &I) {
11880 using namespace llvm::PatternMatch;
11881
11882 Value *Condition;
11883 return match(V: &I, P: m_Intrinsic<Intrinsic::experimental_guard>(
11884 Ops: m_Value(V&: Condition))) &&
11885 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse: false);
11886 });
11887}
11888
11889/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
11890/// protected by a conditional between LHS and RHS. This is used to
11891/// to eliminate casts.
11892bool ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
11893 CmpPredicate Pred,
11894 const SCEV *LHS,
11895 const SCEV *RHS) {
11896 // Interpret a null as meaning no loop, where there is obviously no guard
11897 // (interprocedural conditions notwithstanding). Do not bother about
11898 // unreachable loops.
11899 if (!L || !DT.isReachableFromEntry(A: L->getHeader()))
11900 return true;
11901
11902 if (VerifyIR)
11903 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) &&
11904 "This cannot be done on broken IR!");
11905
11906
11907 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11908 return true;
11909
11910 BasicBlock *Latch = L->getLoopLatch();
11911 if (!Latch)
11912 return false;
11913
11914 CondBrInst *LoopContinuePredicate =
11915 dyn_cast<CondBrInst>(Val: Latch->getTerminator());
11916 if (LoopContinuePredicate &&
11917 isImpliedCond(Pred, LHS, RHS, FoundCondValue: LoopContinuePredicate->getCondition(),
11918 Inverse: LoopContinuePredicate->getSuccessor(i: 0) != L->getHeader()))
11919 return true;
11920
11921 // We don't want more than one activation of the following loops on the stack
11922 // -- that can lead to O(n!) time complexity.
11923 if (WalkingBEDominatingConds)
11924 return false;
11925
11926 SaveAndRestore ClearOnExit(WalkingBEDominatingConds, true);
11927
11928 // See if we can exploit a trip count to prove the predicate.
11929 const auto &BETakenInfo = getBackedgeTakenInfo(L);
11930 const SCEV *LatchBECount = BETakenInfo.getExact(ExitingBlock: Latch, SE: this);
11931 if (LatchBECount != getCouldNotCompute()) {
11932 // We know that Latch branches back to the loop header exactly
11933 // LatchBECount times. This means the backdege condition at Latch is
11934 // equivalent to "{0,+,1} u< LatchBECount".
11935 Type *Ty = LatchBECount->getType();
11936 auto NoWrapFlags = SCEVFlags(SCEV::FlagNUW | SCEV::FlagNW);
11937 const SCEV *LoopCounter =
11938 getAddRecExpr(Start: getZero(Ty), Step: getOne(Ty), L, Flags: NoWrapFlags);
11939 if (isImpliedCond(Pred, LHS, RHS, FoundPred: ICmpInst::ICMP_ULT, FoundLHS: LoopCounter,
11940 FoundRHS: LatchBECount))
11941 return true;
11942 }
11943
11944 // Check conditions due to any @llvm.assume intrinsics.
11945 for (auto &AssumeVH : AC.assumptions()) {
11946 if (!AssumeVH)
11947 continue;
11948 auto *CI = cast<CallInst>(Val&: AssumeVH);
11949 if (!DT.dominates(Def: CI, User: Latch->getTerminator()))
11950 continue;
11951
11952 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: CI->getArgOperand(i: 0), Inverse: false))
11953 return true;
11954 }
11955
11956 if (isImpliedViaGuard(BB: Latch, Pred, LHS, RHS))
11957 return true;
11958
11959 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
11960 DTN != HeaderDTN; DTN = DTN->getIDom()) {
11961 assert(DTN && "should reach the loop header before reaching the root!");
11962
11963 BasicBlock *BB = DTN->getBlock();
11964 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
11965 return true;
11966
11967 BasicBlock *PBB = BB->getSinglePredecessor();
11968 if (!PBB)
11969 continue;
11970
11971 CondBrInst *ContBr = dyn_cast<CondBrInst>(Val: PBB->getTerminator());
11972 if (!ContBr || ContBr->getSuccessor(i: 0) == ContBr->getSuccessor(i: 1))
11973 continue;
11974
11975 // If we have an edge `E` within the loop body that dominates the only
11976 // latch, the condition guarding `E` also guards the backedge. This
11977 // reasoning works only for loops with a single latch.
11978 // We're constructively (and conservatively) enumerating edges within the
11979 // loop body that dominate the latch. The dominator tree better agree
11980 // with us on this:
11981 assert(DT.dominates(BasicBlockEdge(PBB, BB), Latch) && "should be!");
11982 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: ContBr->getCondition(),
11983 Inverse: BB != ContBr->getSuccessor(i: 0)))
11984 return true;
11985 }
11986
11987 return false;
11988}
11989
11990bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
11991 CmpPredicate Pred,
11992 const SCEV *LHS,
11993 const SCEV *RHS) {
11994 // Do not bother proving facts for unreachable code.
11995 if (!DT.isReachableFromEntry(A: BB))
11996 return true;
11997 if (VerifyIR)
11998 assert(!verifyFunction(*BB->getParent(), &dbgs()) &&
11999 "This cannot be done on broken IR!");
12000
12001 // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
12002 // the facts (a >= b && a != b) separately. A typical situation is when the
12003 // non-strict comparison is known from ranges and non-equality is known from
12004 // dominating predicates. If we are proving strict comparison, we always try
12005 // to prove non-equality and non-strict comparison separately.
12006 CmpPredicate NonStrictPredicate = ICmpInst::getNonStrictCmpPredicate(Pred);
12007 const bool ProvingStrictComparison =
12008 Pred != NonStrictPredicate.dropSameSign();
12009 bool ProvedNonStrictComparison = false;
12010 bool ProvedNonEquality = false;
12011
12012 auto SplitAndProve = [&](std::function<bool(CmpPredicate)> Fn) -> bool {
12013 if (!ProvedNonStrictComparison)
12014 ProvedNonStrictComparison = Fn(NonStrictPredicate);
12015 if (!ProvedNonEquality)
12016 ProvedNonEquality = Fn(ICmpInst::ICMP_NE);
12017 if (ProvedNonStrictComparison && ProvedNonEquality)
12018 return true;
12019 return false;
12020 };
12021
12022 if (ProvingStrictComparison) {
12023 auto ProofFn = [&](CmpPredicate P) {
12024 return isKnownViaNonRecursiveReasoning(Pred: P, LHS, RHS);
12025 };
12026 if (SplitAndProve(ProofFn))
12027 return true;
12028 }
12029
12030 // Try to prove (Pred, LHS, RHS) using isImpliedCond.
12031 auto ProveViaCond = [&](const Value *Condition, bool Inverse) {
12032 const Instruction *CtxI = &BB->front();
12033 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI))
12034 return true;
12035 if (ProvingStrictComparison) {
12036 auto ProofFn = [&](CmpPredicate P) {
12037 return isImpliedCond(Pred: P, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI);
12038 };
12039 if (SplitAndProve(ProofFn))
12040 return true;
12041 }
12042 return false;
12043 };
12044
12045 // Starting at the block's predecessor, climb up the predecessor chain, as long
12046 // as there are predecessors that can be found that have unique successors
12047 // leading to the original block.
12048 const Loop *ContainingLoop = LI.getLoopFor(BB);
12049 const BasicBlock *PredBB;
12050 if (ContainingLoop && ContainingLoop->getHeader() == BB)
12051 PredBB = ContainingLoop->getLoopPredecessor();
12052 else
12053 PredBB = BB->getSinglePredecessor();
12054 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
12055 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
12056 const CondBrInst *BlockEntryPredicate =
12057 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
12058 if (!BlockEntryPredicate)
12059 continue;
12060
12061 if (ProveViaCond(BlockEntryPredicate->getCondition(),
12062 BlockEntryPredicate->getSuccessor(i: 0) != Pair.second))
12063 return true;
12064 }
12065
12066 // Check conditions due to any @llvm.assume intrinsics.
12067 for (auto &AssumeVH : AC.assumptions()) {
12068 if (!AssumeVH)
12069 continue;
12070 auto *CI = cast<CallInst>(Val&: AssumeVH);
12071 if (!DT.dominates(Def: CI, BB))
12072 continue;
12073
12074 if (ProveViaCond(CI->getArgOperand(i: 0), false))
12075 return true;
12076 }
12077
12078 // Check conditions due to any @llvm.experimental.guard intrinsics.
12079 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
12080 M: F.getParent(), id: Intrinsic::experimental_guard);
12081 if (GuardDecl)
12082 for (const auto *GU : GuardDecl->users())
12083 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
12084 if (Guard->getFunction() == BB->getParent() && DT.dominates(Def: Guard, BB))
12085 if (ProveViaCond(Guard->getArgOperand(i: 0), false))
12086 return true;
12087 return false;
12088}
12089
12090bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred,
12091 const SCEV *LHS,
12092 const SCEV *RHS) {
12093 // Interpret a null as meaning no loop, where there is obviously no guard
12094 // (interprocedural conditions notwithstanding).
12095 if (!L)
12096 return false;
12097
12098 // Both LHS and RHS must be available at loop entry.
12099 assert(isAvailableAtLoopEntry(LHS, L) &&
12100 "LHS is not available at Loop Entry");
12101 assert(isAvailableAtLoopEntry(RHS, L) &&
12102 "RHS is not available at Loop Entry");
12103
12104 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
12105 return true;
12106
12107 return isBasicBlockEntryGuardedByCond(BB: L->getHeader(), Pred, LHS, RHS);
12108}
12109
12110bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12111 const SCEV *RHS,
12112 const Value *FoundCondValue, bool Inverse,
12113 const Instruction *CtxI) {
12114 // False conditions implies anything. Do not bother analyzing it further.
12115 if (FoundCondValue ==
12116 ConstantInt::getBool(Context&: FoundCondValue->getContext(), V: Inverse))
12117 return true;
12118
12119 if (!PendingLoopPredicates.insert(Ptr: FoundCondValue).second)
12120 return false;
12121
12122 llvm::scope_exit ClearOnExit(
12123 [&]() { PendingLoopPredicates.erase(Ptr: FoundCondValue); });
12124
12125 // Recursively handle And and Or conditions.
12126 const Value *Op0, *Op1;
12127 if (match(V: FoundCondValue, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12128 if (!Inverse)
12129 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12130 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12131 } else if (match(V: FoundCondValue, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12132 if (Inverse)
12133 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12134 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12135 }
12136
12137 const ICmpInst *ICI = dyn_cast<ICmpInst>(Val: FoundCondValue);
12138 if (!ICI) return false;
12139
12140 // Now that we found a conditional branch that dominates the loop or controls
12141 // the loop latch. Check to see if it is the comparison we are looking for.
12142 CmpPredicate FoundPred;
12143 if (Inverse)
12144 FoundPred = ICI->getInverseCmpPredicate();
12145 else
12146 FoundPred = ICI->getCmpPredicate();
12147
12148 const SCEV *FoundLHS = getSCEV(V: ICI->getOperand(i_nocapture: 0));
12149 const SCEV *FoundRHS = getSCEV(V: ICI->getOperand(i_nocapture: 1));
12150
12151 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, Context: CtxI);
12152}
12153
12154bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12155 const SCEV *RHS, CmpPredicate FoundPred,
12156 const SCEV *FoundLHS, const SCEV *FoundRHS,
12157 const Instruction *CtxI) {
12158 // Balance the types.
12159 if (getTypeSizeInBits(Ty: LHS->getType()) <
12160 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12161 // For unsigned and equality predicates, try to prove that both found
12162 // operands fit into narrow unsigned range. If so, try to prove facts in
12163 // narrow types.
12164 if (!CmpInst::isSigned(Pred: FoundPred) && !FoundLHS->getType()->isPointerTy() &&
12165 !FoundRHS->getType()->isPointerTy()) {
12166 auto *NarrowType = LHS->getType();
12167 auto *WideType = FoundLHS->getType();
12168 auto BitWidth = getTypeSizeInBits(Ty: NarrowType);
12169 const SCEV *MaxValue = getZeroExtendExpr(
12170 Op: getConstant(Val: APInt::getMaxValue(numBits: BitWidth)), Ty: WideType);
12171 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundLHS,
12172 RHS: MaxValue) &&
12173 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundRHS,
12174 RHS: MaxValue)) {
12175 const SCEV *TruncFoundLHS = getTruncateExpr(Op: FoundLHS, Ty: NarrowType);
12176 const SCEV *TruncFoundRHS = getTruncateExpr(Op: FoundRHS, Ty: NarrowType);
12177 // We cannot preserve samesign after truncation.
12178 if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred: FoundPred.dropSameSign(),
12179 FoundLHS: TruncFoundLHS, FoundRHS: TruncFoundRHS, CtxI))
12180 return true;
12181 }
12182 }
12183
12184 if (LHS->getType()->isPointerTy() || RHS->getType()->isPointerTy())
12185 return false;
12186 if (CmpInst::isSigned(Pred)) {
12187 LHS = getSignExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12188 RHS = getSignExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12189 } else {
12190 LHS = getZeroExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12191 RHS = getZeroExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12192 }
12193 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
12194 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12195 if (FoundLHS->getType()->isPointerTy() || FoundRHS->getType()->isPointerTy())
12196 return false;
12197 if (CmpInst::isSigned(Pred: FoundPred)) {
12198 FoundLHS = getSignExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12199 FoundRHS = getSignExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12200 } else {
12201 FoundLHS = getZeroExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12202 FoundRHS = getZeroExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12203 }
12204 }
12205 return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS,
12206 FoundRHS, CtxI);
12207}
12208
12209bool ScalarEvolution::isImpliedCondBalancedTypes(
12210 CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS, CmpPredicate FoundPred,
12211 SCEVUse FoundLHS, SCEVUse FoundRHS, const Instruction *CtxI) {
12212 assert(getTypeSizeInBits(LHS->getType()) ==
12213 getTypeSizeInBits(FoundLHS->getType()) &&
12214 "Types should be balanced!");
12215 // Canonicalize the query to match the way instcombine will have
12216 // canonicalized the comparison.
12217 if (SimplifyICmpOperands(Pred, LHS, RHS))
12218 if (LHS == RHS)
12219 return CmpInst::isTrueWhenEqual(predicate: Pred);
12220 if (SimplifyICmpOperands(Pred&: FoundPred, LHS&: FoundLHS, RHS&: FoundRHS))
12221 if (FoundLHS == FoundRHS)
12222 return CmpInst::isFalseWhenEqual(predicate: FoundPred);
12223
12224 // Check to see if we can make the LHS or RHS match.
12225 if (LHS == FoundRHS || RHS == FoundLHS) {
12226 if (isa<SCEVConstant>(Val: RHS)) {
12227 std::swap(a&: FoundLHS, b&: FoundRHS);
12228 FoundPred = ICmpInst::getSwappedCmpPredicate(Pred: FoundPred);
12229 } else {
12230 std::swap(a&: LHS, b&: RHS);
12231 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12232 }
12233 }
12234
12235 // Check whether the found predicate is the same as the desired predicate.
12236 if (auto P = CmpPredicate::getMatching(A: FoundPred, B: Pred))
12237 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12238
12239 // Check whether swapping the found predicate makes it the same as the
12240 // desired predicate.
12241 if (auto P = CmpPredicate::getMatching(
12242 A: ICmpInst::getSwappedCmpPredicate(Pred: FoundPred), B: Pred)) {
12243 // We can write the implication
12244 // 0. LHS Pred RHS <- FoundLHS SwapPred FoundRHS
12245 // using one of the following ways:
12246 // 1. LHS Pred RHS <- FoundRHS Pred FoundLHS
12247 // 2. RHS SwapPred LHS <- FoundLHS SwapPred FoundRHS
12248 // Both require swapping the operands of one condition. Don't do this if it
12249 // would break canonical constant/addrec ordering.
12250 if (!isa<SCEVConstant>(Val: RHS) && !isa<SCEVAddRecExpr>(Val: LHS))
12251 return isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P), LHS: RHS,
12252 RHS: LHS, FoundLHS, FoundRHS, Context: CtxI);
12253 if (!isa<SCEVConstant>(Val: FoundRHS) && !isa<SCEVAddRecExpr>(Val: FoundLHS))
12254 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: FoundRHS, FoundRHS: FoundLHS, Context: CtxI);
12255
12256 return false;
12257 }
12258
12259 auto IsSignFlippedPredicate = [](CmpInst::Predicate P1,
12260 CmpInst::Predicate P2) {
12261 assert(P1 != P2 && "Handled earlier!");
12262 return CmpInst::isRelational(P: P2) &&
12263 P1 == ICmpInst::getFlippedSignednessPredicate(Pred: P2);
12264 };
12265 if (IsSignFlippedPredicate(Pred, FoundPred)) {
12266 // Unsigned comparison is the same as signed comparison when both the
12267 // operands are non-negative or negative.
12268 if (haveSameSign(S1: FoundLHS, S2: FoundRHS))
12269 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12270 // Create local copies that we can freely swap and canonicalize our
12271 // conditions to "le/lt".
12272 CmpPredicate CanonicalPred = Pred, CanonicalFoundPred = FoundPred;
12273 const SCEV *CanonicalLHS = LHS, *CanonicalRHS = RHS,
12274 *CanonicalFoundLHS = FoundLHS, *CanonicalFoundRHS = FoundRHS;
12275 if (ICmpInst::isGT(P: CanonicalPred) || ICmpInst::isGE(P: CanonicalPred)) {
12276 CanonicalPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalPred);
12277 CanonicalFoundPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalFoundPred);
12278 std::swap(a&: CanonicalLHS, b&: CanonicalRHS);
12279 std::swap(a&: CanonicalFoundLHS, b&: CanonicalFoundRHS);
12280 }
12281 assert((ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) &&
12282 "Must be!");
12283 assert((ICmpInst::isLT(CanonicalFoundPred) ||
12284 ICmpInst::isLE(CanonicalFoundPred)) &&
12285 "Must be!");
12286 if (ICmpInst::isSigned(Pred: CanonicalPred) && isKnownNonNegative(S: CanonicalRHS))
12287 // Use implication:
12288 // x <u y && y >=s 0 --> x <s y.
12289 // If we can prove the left part, the right part is also proven.
12290 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
12291 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
12292 FoundRHS: CanonicalFoundRHS);
12293 if (ICmpInst::isUnsigned(Pred: CanonicalPred) && isKnownNegative(S: CanonicalRHS))
12294 // Use implication:
12295 // x <s y && y <s 0 --> x <u y.
12296 // If we can prove the left part, the right part is also proven.
12297 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
12298 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
12299 FoundRHS: CanonicalFoundRHS);
12300 }
12301
12302 // Check if we can make progress by sharpening ranges.
12303 if (FoundPred == ICmpInst::ICMP_NE &&
12304 (isa<SCEVConstant>(Val: FoundLHS) || isa<SCEVConstant>(Val: FoundRHS))) {
12305
12306 const SCEVConstant *C = nullptr;
12307 const SCEV *V = nullptr;
12308
12309 if (isa<SCEVConstant>(Val: FoundLHS)) {
12310 C = cast<SCEVConstant>(Val&: FoundLHS);
12311 V = FoundRHS;
12312 } else {
12313 C = cast<SCEVConstant>(Val&: FoundRHS);
12314 V = FoundLHS;
12315 }
12316
12317 // The guarding predicate tells us that C != V. If the known range
12318 // of V is [C, t), we can sharpen the range to [C + 1, t). The
12319 // range we consider has to correspond to same signedness as the
12320 // predicate we're interested in folding.
12321
12322 APInt Min = ICmpInst::isSigned(Pred) ?
12323 getSignedRangeMin(S: V) : getUnsignedRangeMin(S: V);
12324
12325 if (Min == C->getAPInt()) {
12326 // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
12327 // This is true even if (Min + 1) wraps around -- in case of
12328 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
12329
12330 APInt SharperMin = Min + 1;
12331
12332 switch (Pred) {
12333 case ICmpInst::ICMP_SGE:
12334 case ICmpInst::ICMP_UGE:
12335 // We know V `Pred` SharperMin. If this implies LHS `Pred`
12336 // RHS, we're done.
12337 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin),
12338 Context: CtxI))
12339 return true;
12340 [[fallthrough]];
12341
12342 case ICmpInst::ICMP_SGT:
12343 case ICmpInst::ICMP_UGT:
12344 // We know from the range information that (V `Pred` Min ||
12345 // V == Min). We know from the guarding condition that !(V
12346 // == Min). This gives us
12347 //
12348 // V `Pred` Min || V == Min && !(V == Min)
12349 // => V `Pred` Min
12350 //
12351 // If V `Pred` Min implies LHS `Pred` RHS, we're done.
12352
12353 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12354 return true;
12355 break;
12356
12357 // `LHS < RHS` and `LHS <= RHS` are handled in the same way as `RHS > LHS` and `RHS >= LHS` respectively.
12358 case ICmpInst::ICMP_SLE:
12359 case ICmpInst::ICMP_ULE:
12360 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12361 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin), Context: CtxI))
12362 return true;
12363 [[fallthrough]];
12364
12365 case ICmpInst::ICMP_SLT:
12366 case ICmpInst::ICMP_ULT:
12367 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12368 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12369 return true;
12370 break;
12371
12372 default:
12373 // No change
12374 break;
12375 }
12376 }
12377 }
12378
12379 // Check whether the actual condition is beyond sufficient.
12380 if (FoundPred == ICmpInst::ICMP_EQ)
12381 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
12382 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12383 return true;
12384 if (Pred == ICmpInst::ICMP_NE)
12385 if (!ICmpInst::isTrueWhenEqual(predicate: FoundPred))
12386 if (isImpliedCondOperands(Pred: FoundPred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12387 return true;
12388
12389 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS))
12390 return true;
12391
12392 // Otherwise assume the worst.
12393 return false;
12394}
12395
12396bool ScalarEvolution::splitBinaryAdd(SCEVUse Expr, SCEVUse &L, SCEVUse &R,
12397 SCEVFlags &Flags) {
12398 if (!match(U: Expr, P: m_scev_Add(Op0: m_SCEV(V&: L), Op1: m_SCEV(V&: R))))
12399 return false;
12400
12401 Flags = cast<SCEVAddExpr>(Val&: Expr)->getNoWrapFlags();
12402 return true;
12403}
12404
12405std::optional<APInt>
12406ScalarEvolution::computeConstantDifference(const SCEV *More, const SCEV *Less) {
12407 // We avoid subtracting expressions here because this function is usually
12408 // fairly deep in the call stack (i.e. is called many times).
12409
12410 unsigned BW = getTypeSizeInBits(Ty: More->getType());
12411 APInt Diff(BW, 0);
12412 APInt DiffMul(BW, 1);
12413 // Try various simplifications to reduce the difference to a constant. Limit
12414 // the number of allowed simplifications to keep compile-time low.
12415 for (unsigned I = 0; I < 8; ++I) {
12416 if (More == Less)
12417 return Diff;
12418
12419 // Reduce addrecs with identical steps to their start value.
12420 if (isa<SCEVAddRecExpr>(Val: Less) && isa<SCEVAddRecExpr>(Val: More)) {
12421 const auto *LAR = cast<SCEVAddRecExpr>(Val: Less);
12422 const auto *MAR = cast<SCEVAddRecExpr>(Val: More);
12423
12424 if (LAR->getLoop() != MAR->getLoop())
12425 return std::nullopt;
12426
12427 // We look at affine expressions only; not for correctness but to keep
12428 // getStepRecurrence cheap.
12429 if (!LAR->isAffine() || !MAR->isAffine())
12430 return std::nullopt;
12431
12432 if (LAR->getStepRecurrence(SE&: *this) != MAR->getStepRecurrence(SE&: *this))
12433 return std::nullopt;
12434
12435 Less = LAR->getStart();
12436 More = MAR->getStart();
12437 continue;
12438 }
12439
12440 // Try to match a common constant multiply.
12441 auto MatchConstMul =
12442 [](const SCEV *S) -> std::optional<std::pair<const SCEV *, APInt>> {
12443 const APInt *C;
12444 const SCEV *Op;
12445 if (match(S, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEV(V&: Op))))
12446 return {{Op, *C}};
12447 return std::nullopt;
12448 };
12449 if (auto MatchedMore = MatchConstMul(More)) {
12450 if (auto MatchedLess = MatchConstMul(Less)) {
12451 if (MatchedMore->second == MatchedLess->second) {
12452 More = MatchedMore->first;
12453 Less = MatchedLess->first;
12454 DiffMul *= MatchedMore->second;
12455 continue;
12456 }
12457 }
12458 }
12459
12460 // Try to cancel out common factors in two add expressions.
12461 SmallDenseMap<const SCEV *, int, 8> Multiplicity;
12462 auto Add = [&](const SCEV *S, int Mul) {
12463 if (auto *C = dyn_cast<SCEVConstant>(Val: S)) {
12464 if (Mul == 1) {
12465 Diff += C->getAPInt() * DiffMul;
12466 } else {
12467 assert(Mul == -1);
12468 Diff -= C->getAPInt() * DiffMul;
12469 }
12470 } else
12471 Multiplicity[S] += Mul;
12472 };
12473 auto Decompose = [&](const SCEV *S, int Mul) {
12474 if (isa<SCEVAddExpr>(Val: S)) {
12475 for (const SCEV *Op : S->operands())
12476 Add(Op, Mul);
12477 } else
12478 Add(S, Mul);
12479 };
12480 Decompose(More, 1);
12481 Decompose(Less, -1);
12482
12483 // Check whether all the non-constants cancel out, or reduce to new
12484 // More/Less values.
12485 const SCEV *NewMore = nullptr, *NewLess = nullptr;
12486 for (const auto &[S, Mul] : Multiplicity) {
12487 if (Mul == 0)
12488 continue;
12489 if (Mul == 1) {
12490 if (NewMore)
12491 return std::nullopt;
12492 NewMore = S;
12493 } else if (Mul == -1) {
12494 if (NewLess)
12495 return std::nullopt;
12496 NewLess = S;
12497 } else
12498 return std::nullopt;
12499 }
12500
12501 // Values stayed the same, no point in trying further.
12502 if (NewMore == More || NewLess == Less)
12503 return std::nullopt;
12504
12505 More = NewMore;
12506 Less = NewLess;
12507
12508 // Reduced to constant.
12509 if (!More && !Less)
12510 return Diff;
12511
12512 // Left with variable on only one side, bail out.
12513 if (!More || !Less)
12514 return std::nullopt;
12515 }
12516
12517 // Did not reduce to constant.
12518 return std::nullopt;
12519}
12520
12521bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
12522 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12523 const SCEV *FoundRHS, const Instruction *CtxI) {
12524 // Try to recognize the following pattern:
12525 //
12526 // FoundRHS = ...
12527 // ...
12528 // loop:
12529 // FoundLHS = {Start,+,W}
12530 // context_bb: // Basic block from the same loop
12531 // known(Pred, FoundLHS, FoundRHS)
12532 //
12533 // If some predicate is known in the context of a loop, it is also known on
12534 // each iteration of this loop, including the first iteration. Therefore, in
12535 // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to
12536 // prove the original pred using this fact.
12537 if (!CtxI)
12538 return false;
12539 const BasicBlock *ContextBB = CtxI->getParent();
12540 // Make sure AR varies in the context block.
12541 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS)) {
12542 const Loop *L = AR->getLoop();
12543 const auto *Latch = L->getLoopLatch();
12544 // Make sure that context belongs to the loop and executes on 1st iteration
12545 // (if it ever executes at all).
12546 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12547 return false;
12548 if (!isAvailableAtLoopEntry(S: FoundRHS, L: AR->getLoop()))
12549 return false;
12550 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: AR->getStart(), FoundRHS);
12551 }
12552
12553 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundRHS)) {
12554 const Loop *L = AR->getLoop();
12555 const auto *Latch = L->getLoopLatch();
12556 // Make sure that context belongs to the loop and executes on 1st iteration
12557 // (if it ever executes at all).
12558 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12559 return false;
12560 if (!isAvailableAtLoopEntry(S: FoundLHS, L: AR->getLoop()))
12561 return false;
12562 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS: AR->getStart());
12563 }
12564
12565 return false;
12566}
12567
12568bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(CmpPredicate Pred,
12569 const SCEV *LHS,
12570 const SCEV *RHS,
12571 const SCEV *FoundLHS,
12572 const SCEV *FoundRHS) {
12573 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT)
12574 return false;
12575
12576 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12577 if (!AddRecLHS)
12578 return false;
12579
12580 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12581 if (!AddRecFoundLHS)
12582 return false;
12583
12584 // We'd like to let SCEV reason about control dependencies, so we constrain
12585 // both the inequalities to be about add recurrences on the same loop. This
12586 // way we can use isLoopEntryGuardedByCond later.
12587
12588 const Loop *L = AddRecFoundLHS->getLoop();
12589 if (L != AddRecLHS->getLoop())
12590 return false;
12591
12592 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1)
12593 //
12594 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C)
12595 // ... (2)
12596 //
12597 // Informal proof for (2), assuming (1) [*]:
12598 //
12599 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**]
12600 //
12601 // Then
12602 //
12603 // FoundLHS s< FoundRHS s< INT_MIN - C
12604 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ]
12605 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ]
12606 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s<
12607 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ]
12608 // <=> FoundLHS + C s< FoundRHS + C
12609 //
12610 // [*]: (1) can be proved by ruling out overflow.
12611 //
12612 // [**]: This can be proved by analyzing all the four possibilities:
12613 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and
12614 // (A s>= 0, B s>= 0).
12615 //
12616 // Note:
12617 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C"
12618 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS
12619 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS
12620 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is
12621 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS +
12622 // C)".
12623
12624 std::optional<APInt> LDiff = computeConstantDifference(More: LHS, Less: FoundLHS);
12625 if (!LDiff)
12626 return false;
12627 std::optional<APInt> RDiff = computeConstantDifference(More: RHS, Less: FoundRHS);
12628 if (!RDiff || *LDiff != *RDiff)
12629 return false;
12630
12631 if (LDiff->isMinValue())
12632 return true;
12633
12634 APInt FoundRHSLimit;
12635
12636 if (Pred == CmpInst::ICMP_ULT) {
12637 FoundRHSLimit = -(*RDiff);
12638 } else {
12639 assert(Pred == CmpInst::ICMP_SLT && "Checked above!");
12640 FoundRHSLimit = APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: RHS->getType())) - *RDiff;
12641 }
12642
12643 // Try to prove (1) or (2), as needed.
12644 return isAvailableAtLoopEntry(S: FoundRHS, L) &&
12645 isLoopEntryGuardedByCond(L, Pred, LHS: FoundRHS,
12646 RHS: getConstant(Val: FoundRHSLimit));
12647}
12648
12649bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred, const SCEV *LHS,
12650 const SCEV *RHS, const SCEV *FoundLHS,
12651 const SCEV *FoundRHS, unsigned Depth) {
12652 const PHINode *LPhi = nullptr, *RPhi = nullptr;
12653
12654 llvm::scope_exit ClearOnExit([&]() {
12655 if (LPhi) {
12656 bool Erased = PendingMerges.erase(Ptr: LPhi);
12657 assert(Erased && "Failed to erase LPhi!");
12658 (void)Erased;
12659 }
12660 if (RPhi) {
12661 bool Erased = PendingMerges.erase(Ptr: RPhi);
12662 assert(Erased && "Failed to erase RPhi!");
12663 (void)Erased;
12664 }
12665 });
12666
12667 // Find respective Phis and check that they are not being pending.
12668 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(Val: LHS))
12669 if (auto *Phi = dyn_cast<PHINode>(Val: LU->getValue())) {
12670 if (!PendingMerges.insert(Ptr: Phi).second)
12671 return false;
12672 LPhi = Phi;
12673 }
12674 if (const SCEVUnknown *RU = dyn_cast<SCEVUnknown>(Val: RHS))
12675 if (auto *Phi = dyn_cast<PHINode>(Val: RU->getValue())) {
12676 // If we detect a loop of Phi nodes being processed by this method, for
12677 // example:
12678 //
12679 // %a = phi i32 [ %some1, %preheader ], [ %b, %latch ]
12680 // %b = phi i32 [ %some2, %preheader ], [ %a, %latch ]
12681 //
12682 // we don't want to deal with a case that complex, so return conservative
12683 // answer false.
12684 if (!PendingMerges.insert(Ptr: Phi).second)
12685 return false;
12686 RPhi = Phi;
12687 }
12688
12689 // If none of LHS, RHS is a Phi, nothing to do here.
12690 if (!LPhi && !RPhi)
12691 return false;
12692
12693 // If there is a SCEVUnknown Phi we are interested in, make it left.
12694 if (!LPhi) {
12695 std::swap(a&: LHS, b&: RHS);
12696 std::swap(a&: FoundLHS, b&: FoundRHS);
12697 std::swap(a&: LPhi, b&: RPhi);
12698 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12699 }
12700
12701 assert(LPhi && "LPhi should definitely be a SCEVUnknown Phi!");
12702 const BasicBlock *LBB = LPhi->getParent();
12703 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(Val: RHS);
12704
12705 auto ProvedEasily = [&](const SCEV *S1, const SCEV *S2) {
12706 return isKnownViaNonRecursiveReasoning(Pred, LHS: S1, RHS: S2) ||
12707 isImpliedCondOperandsViaRanges(Pred, LHS: S1, RHS: S2, FoundPred: Pred, FoundLHS, FoundRHS) ||
12708 isImpliedViaOperations(Pred, LHS: S1, RHS: S2, FoundLHS, FoundRHS, Depth);
12709 };
12710
12711 if (RPhi && RPhi->getParent() == LBB) {
12712 // Case one: RHS is also a SCEVUnknown Phi from the same basic block.
12713 // If we compare two Phis from the same block, and for each entry block
12714 // the predicate is true for incoming values from this block, then the
12715 // predicate is also true for the Phis.
12716 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12717 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12718 const SCEV *R = getSCEV(V: RPhi->getIncomingValueForBlock(BB: IncBB));
12719 if (!ProvedEasily(L, R))
12720 return false;
12721 }
12722 } else if (RAR && RAR->getLoop()->getHeader() == LBB) {
12723 // Case two: RHS is also a Phi from the same basic block, and it is an
12724 // AddRec. It means that there is a loop which has both AddRec and Unknown
12725 // PHIs, for it we can compare incoming values of AddRec from above the loop
12726 // and latch with their respective incoming values of LPhi.
12727 // TODO: Generalize to handle loops with many inputs in a header.
12728 if (LPhi->getNumIncomingValues() != 2) return false;
12729
12730 auto *RLoop = RAR->getLoop();
12731 auto *Predecessor = RLoop->getLoopPredecessor();
12732 assert(Predecessor && "Loop with AddRec with no predecessor?");
12733 const SCEV *L1 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Predecessor));
12734 if (!ProvedEasily(L1, RAR->getStart()))
12735 return false;
12736 auto *Latch = RLoop->getLoopLatch();
12737 assert(Latch && "Loop with AddRec with no latch?");
12738 const SCEV *L2 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Latch));
12739 if (!ProvedEasily(L2, RAR->getPostIncExpr(SE&: *this)))
12740 return false;
12741 } else {
12742 // In all other cases go over inputs of LHS and compare each of them to RHS,
12743 // the predicate is true for (LHS, RHS) if it is true for all such pairs.
12744 // At this point RHS is either a non-Phi, or it is a Phi from some block
12745 // different from LBB.
12746 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12747 // Check that RHS is available in this block.
12748 if (!dominates(S: RHS, BB: IncBB))
12749 return false;
12750 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12751 // Make sure L does not refer to a value from a potentially previous
12752 // iteration of a loop.
12753 if (!properlyDominates(S: L, BB: LBB))
12754 return false;
12755 // Addrecs are considered to properly dominate their loop, so are missed
12756 // by the previous check. Discard any values that have computable
12757 // evolution in this loop.
12758 if (auto *Loop = LI.getLoopFor(BB: LBB))
12759 if (hasComputableLoopEvolution(S: L, L: Loop))
12760 return false;
12761 if (!ProvedEasily(L, RHS))
12762 return false;
12763 }
12764 }
12765 return true;
12766}
12767
12768bool ScalarEvolution::isImpliedCondOperandsViaShift(CmpPredicate Pred,
12769 const SCEV *LHS,
12770 const SCEV *RHS,
12771 const SCEV *FoundLHS,
12772 const SCEV *FoundRHS) {
12773 // We want to imply LHS < RHS from LHS < (RHS >> shiftvalue). First, make
12774 // sure that we are dealing with same LHS.
12775 if (RHS == FoundRHS) {
12776 std::swap(a&: LHS, b&: RHS);
12777 std::swap(a&: FoundLHS, b&: FoundRHS);
12778 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12779 }
12780 if (LHS != FoundLHS)
12781 return false;
12782
12783 auto *SUFoundRHS = dyn_cast<SCEVUnknown>(Val: FoundRHS);
12784 if (!SUFoundRHS)
12785 return false;
12786
12787 Value *Shiftee, *ShiftValue;
12788
12789 using namespace PatternMatch;
12790 if (match(V: SUFoundRHS->getValue(),
12791 P: m_LShr(L: m_Value(V&: Shiftee), R: m_Value(V&: ShiftValue)))) {
12792 auto *ShifteeS = getSCEV(V: Shiftee);
12793 // Prove one of the following:
12794 // LHS <u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <u RHS
12795 // LHS <=u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <=u RHS
12796 // LHS <s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12797 // ---> LHS <s RHS
12798 // LHS <=s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12799 // ---> LHS <=s RHS
12800 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
12801 return isKnownPredicate(Pred: ICmpInst::ICMP_ULE, LHS: ShifteeS, RHS);
12802 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
12803 if (isKnownNonNegative(S: ShifteeS))
12804 return isKnownPredicate(Pred: ICmpInst::ICMP_SLE, LHS: ShifteeS, RHS);
12805 }
12806
12807 return false;
12808}
12809
12810bool ScalarEvolution::isImpliedCondOperandsViaMatchingDiff(
12811 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12812 const SCEV *FoundRHS) {
12813 // Only valid for equality predicates: (A == B) implies (C == D) when
12814 // the SCEV difference A - B equals C - D (they check the same
12815 // underlying relationship at every iteration).
12816 if (!ICmpInst::isEquality(P: Pred))
12817 return false;
12818
12819 // Restrict to cases involving loop recurrences - that's where this
12820 // pattern arises (correlated IV comparisons). This avoids calling
12821 // getMinusSCEV on arbitrary non-loop expressions.
12822 if ((!isa<SCEVAddRecExpr>(Val: LHS) && !isa<SCEVAddRecExpr>(Val: RHS)) ||
12823 (!isa<SCEVAddRecExpr>(Val: FoundLHS) && !isa<SCEVAddRecExpr>(Val: FoundRHS)))
12824 return false;
12825
12826 // AddRecs from different loops can never produce matching differences.
12827 const SCEVAddRecExpr *QueryAddRec = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12828 if (!QueryAddRec)
12829 QueryAddRec = cast<SCEVAddRecExpr>(Val: RHS);
12830 const SCEVAddRecExpr *FoundAddRec = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12831 if (!FoundAddRec)
12832 FoundAddRec = cast<SCEVAddRecExpr>(Val: FoundRHS);
12833 if (QueryAddRec->getLoop() != FoundAddRec->getLoop())
12834 return false;
12835
12836 // If the strides differ, the differences can never match.
12837 if (QueryAddRec->getStepRecurrence(SE&: *this) !=
12838 FoundAddRec->getStepRecurrence(SE&: *this))
12839 return false;
12840
12841 // Compute differences. For pointer-typed operands sharing the same base,
12842 // getMinusSCEV strips the common base and returns an integer SCEV.
12843 // For example, {base,+,8} - (base+8*n) = {-8n,+,8}
12844 const SCEV *FoundDiff = getMinusSCEV(LHS: FoundLHS, RHS: FoundRHS);
12845 if (isa<SCEVCouldNotCompute>(Val: FoundDiff))
12846 return false;
12847
12848 const SCEV *Diff = getMinusSCEV(LHS, RHS);
12849 if (isa<SCEVCouldNotCompute>(Val: Diff))
12850 return false;
12851
12852 return Diff == FoundDiff;
12853}
12854
12855bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred, const SCEV *LHS,
12856 const SCEV *RHS,
12857 const SCEV *FoundLHS,
12858 const SCEV *FoundRHS,
12859 const Instruction *CtxI) {
12860 return isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred: Pred, FoundLHS,
12861 FoundRHS) ||
12862 isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS,
12863 FoundRHS) ||
12864 isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS) ||
12865 isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
12866 CtxI) ||
12867 isImpliedCondOperandsViaMatchingDiff(Pred, LHS, RHS, FoundLHS,
12868 FoundRHS) ||
12869 isImpliedCondOperandsHelper(Pred, LHS, RHS, FoundLHS, FoundRHS);
12870}
12871
12872/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
12873template <typename MinMaxExprType>
12874static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
12875 const SCEV *Candidate) {
12876 const MinMaxExprType *MinMaxExpr = dyn_cast<MinMaxExprType>(MaybeMinMaxExpr);
12877 if (!MinMaxExpr)
12878 return false;
12879
12880 return is_contained(MinMaxExpr->operands(), Candidate);
12881}
12882
12883static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE,
12884 CmpPredicate Pred, const SCEV *LHS,
12885 const SCEV *RHS) {
12886 // If both sides are affine addrecs for the same loop, with equal
12887 // steps, and we know the recurrences don't wrap, then we only
12888 // need to check the predicate on the starting values.
12889
12890 if (!ICmpInst::isRelational(P: Pred))
12891 return false;
12892
12893 const SCEV *LStart, *RStart, *Step;
12894 const Loop *L;
12895 if (!match(S: LHS,
12896 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: LStart), Op1: m_SCEV(V&: Step), L: m_Loop(L))) ||
12897 !match(S: RHS, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: RStart), Op1: m_scev_Specific(S: Step),
12898 L: m_SpecificLoop(L))))
12899 return false;
12900 const SCEVAddRecExpr *LAR = cast<SCEVAddRecExpr>(Val: LHS);
12901 const SCEVAddRecExpr *RAR = cast<SCEVAddRecExpr>(Val: RHS);
12902 SCEVFlags NW = ICmpInst::isSigned(Pred) ? SCEV::FlagNSW : SCEV::FlagNUW;
12903 if (!LAR->getNoWrapFlags(Mask: NW) || !RAR->getNoWrapFlags(Mask: NW))
12904 return false;
12905
12906 return SE.isKnownPredicate(Pred, LHS: LStart, RHS: RStart);
12907}
12908
12909/// Is LHS `Pred` RHS true because one of them is an AddRec that is known not to
12910/// go below its own start value?
12911static bool IsKnownPredicateViaAddRecMonotonicity(ScalarEvolution &SE,
12912 CmpPredicate Pred,
12913 const SCEV *LHS,
12914 const SCEV *RHS) {
12915 // Normalize to (AddRec Pred Start).
12916 if (!isa<SCEVAddRecExpr>(Val: LHS) && isa<SCEVAddRecExpr>(Val: RHS)) {
12917 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12918 std::swap(a&: LHS, b&: RHS);
12919 }
12920
12921 // The recurrence is equal to Start in the first iteration, so only the
12922 // non-strict predicate holds.
12923 if (Pred != ICmpInst::ICMP_UGE && Pred != ICmpInst::ICMP_SGE)
12924 return false;
12925
12926 const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12927 if (!AR || AR->getStart() != RHS)
12928 return false;
12929
12930 return SE.getMonotonicPredicateType(LHS: AR, Pred) ==
12931 ScalarEvolution::MonotonicallyIncreasing;
12932}
12933
12934/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
12935/// expression?
12936static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, CmpPredicate Pred,
12937 const SCEV *LHS, const SCEV *RHS) {
12938 switch (Pred) {
12939 default:
12940 return false;
12941
12942 case ICmpInst::ICMP_SGE:
12943 std::swap(a&: LHS, b&: RHS);
12944 [[fallthrough]];
12945 case ICmpInst::ICMP_SLE:
12946 return
12947 // min(A, ...) <= A
12948 IsMinMaxConsistingOf<SCEVSMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12949 // A <= max(A, ...)
12950 IsMinMaxConsistingOf<SCEVSMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12951
12952 case ICmpInst::ICMP_UGE:
12953 std::swap(a&: LHS, b&: RHS);
12954 [[fallthrough]];
12955 case ICmpInst::ICMP_ULE:
12956 return
12957 // min(A, ...) <= A
12958 // FIXME: what about umin_seq?
12959 IsMinMaxConsistingOf<SCEVUMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12960 // A <= max(A, ...)
12961 IsMinMaxConsistingOf<SCEVUMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12962
12963 case ICmpInst::ICMP_UGT:
12964 std::swap(a&: LHS, b&: RHS);
12965 [[fallthrough]];
12966 case ICmpInst::ICMP_ULT:
12967 // umin(Ops) u<= each Op, so proving Op u< RHS for any Op proves
12968 // umin(Ops) u< RHS.
12969 //
12970 // Use computeConstantDifference instead of the more powerful
12971 // isKnownPredicate to keep this check cheap: isKnownPredicateViaMinOrMax
12972 // is called from isKnownViaNonRecursiveReasoning, so recursing into
12973 // the full predicate prover would be expensive.
12974 if (const auto *Min = dyn_cast<SCEVUMinExpr>(Val: LHS)) {
12975 for (SCEVUse Op : Min->operands()) {
12976 std::optional<APInt> Diff = SE.computeConstantDifference(More: RHS, Less: Op);
12977 // When Op and RHS share a common base differing by a
12978 // constant offset D (RHS - Op = D), Op u< RHS holds iff D != 0 and
12979 // RHS >= D (unsigned), i.e. the subtraction doesn't underflow.
12980 if (Diff && !Diff->isZero() && SE.getUnsignedRangeMin(S: RHS).uge(RHS: *Diff))
12981 return true;
12982 }
12983 }
12984 return false;
12985 }
12986
12987 llvm_unreachable("covered switch fell through?!");
12988}
12989
12990bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
12991 const SCEV *RHS,
12992 const SCEV *FoundLHS,
12993 const SCEV *FoundRHS,
12994 unsigned Depth) {
12995 assert(getTypeSizeInBits(LHS->getType()) ==
12996 getTypeSizeInBits(RHS->getType()) &&
12997 "LHS and RHS have different sizes?");
12998 assert(getTypeSizeInBits(FoundLHS->getType()) ==
12999 getTypeSizeInBits(FoundRHS->getType()) &&
13000 "FoundLHS and FoundRHS have different sizes?");
13001 // We want to avoid hurting the compile time with analysis of too big trees.
13002 if (Depth > MaxSCEVOperationsImplicationDepth)
13003 return false;
13004
13005 // We only want to work with GT comparison so far.
13006 if (ICmpInst::isLT(P: Pred)) {
13007 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
13008 std::swap(a&: LHS, b&: RHS);
13009 std::swap(a&: FoundLHS, b&: FoundRHS);
13010 }
13011
13012 CmpInst::Predicate P = Pred.getPreferredSignedPredicate();
13013
13014 // For unsigned, try to reduce it to corresponding signed comparison.
13015 if (P == ICmpInst::ICMP_UGT)
13016 // We can replace unsigned predicate with its signed counterpart if all
13017 // involved values are non-negative.
13018 // TODO: We could have better support for unsigned.
13019 if (isKnownNonNegative(S: FoundLHS) && isKnownNonNegative(S: FoundRHS)) {
13020 // Knowing that both FoundLHS and FoundRHS are non-negative, and knowing
13021 // FoundLHS >u FoundRHS, we also know that FoundLHS >s FoundRHS. Let us
13022 // use this fact to prove that LHS and RHS are non-negative.
13023 const SCEV *MinusOne = getMinusOne(Ty: LHS->getType());
13024 if (isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS, RHS: MinusOne, FoundLHS,
13025 FoundRHS) &&
13026 isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS: RHS, RHS: MinusOne, FoundLHS,
13027 FoundRHS))
13028 P = ICmpInst::ICMP_SGT;
13029 }
13030
13031 if (P != ICmpInst::ICMP_SGT)
13032 return false;
13033
13034 auto GetOpFromSExt = [&](const SCEV *S) -> const SCEV * {
13035 if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(Val: S))
13036 return Ext->getOperand();
13037 // TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
13038 // the constant in some cases.
13039 return S;
13040 };
13041
13042 // Acquire values from extensions.
13043 auto *OrigLHS = LHS;
13044 auto *OrigFoundLHS = FoundLHS;
13045 LHS = GetOpFromSExt(LHS);
13046 FoundLHS = GetOpFromSExt(FoundLHS);
13047
13048 // Is the SGT predicate can be proved trivially or using the found context.
13049 auto IsSGTViaContext = [&](const SCEV *S1, const SCEV *S2) {
13050 return isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2) ||
13051 isImpliedViaOperations(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2, FoundLHS: OrigFoundLHS,
13052 FoundRHS, Depth: Depth + 1);
13053 };
13054
13055 if (auto *LHSAddExpr = dyn_cast<SCEVAddExpr>(Val: LHS)) {
13056 // We want to avoid creation of any new non-constant SCEV. Since we are
13057 // going to compare the operands to RHS, we should be certain that we don't
13058 // need any size extensions for this. So let's decline all cases when the
13059 // sizes of types of LHS and RHS do not match.
13060 // TODO: Maybe try to get RHS from sext to catch more cases?
13061 if (getTypeSizeInBits(Ty: LHS->getType()) != getTypeSizeInBits(Ty: RHS->getType()))
13062 return false;
13063
13064 // Should not overflow.
13065 if (!LHSAddExpr->hasNoSignedWrap())
13066 return false;
13067
13068 SCEVUse LL = LHSAddExpr->getOperand(i: 0);
13069 SCEVUse LR = LHSAddExpr->getOperand(i: 1);
13070 auto *MinusOne = getMinusOne(Ty: RHS->getType());
13071
13072 // Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
13073 auto IsSumGreaterThanRHS = [&](const SCEV *S1, const SCEV *S2) {
13074 return IsSGTViaContext(S1, MinusOne) && IsSGTViaContext(S2, RHS);
13075 };
13076 // Try to prove the following rule:
13077 // (LHS = LL + LR) && (LL >= 0) && (LR > RHS) => (LHS > RHS).
13078 // (LHS = LL + LR) && (LR >= 0) && (LL > RHS) => (LHS > RHS).
13079 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
13080 return true;
13081 } else if (auto *LHSUnknownExpr = dyn_cast<SCEVUnknown>(Val: LHS)) {
13082 Value *LL, *LR;
13083 // FIXME: Once we have SDiv implemented, we can get rid of this matching.
13084
13085 using namespace llvm::PatternMatch;
13086
13087 if (match(V: LHSUnknownExpr->getValue(), P: m_SDiv(L: m_Value(V&: LL), R: m_Value(V&: LR)))) {
13088 // Rules for division.
13089 // We are going to perform some comparisons with Denominator and its
13090 // derivative expressions. In general case, creating a SCEV for it may
13091 // lead to a complex analysis of the entire graph, and in particular it
13092 // can request trip count recalculation for the same loop. This would
13093 // cache as SCEVCouldNotCompute to avoid the infinite recursion. To avoid
13094 // this, we only want to create SCEVs that are constants in this section.
13095 // So we bail if Denominator is not a constant.
13096 if (!isa<ConstantInt>(Val: LR))
13097 return false;
13098
13099 auto *Denominator = cast<SCEVConstant>(Val: getSCEV(V: LR));
13100
13101 // We want to make sure that LHS = FoundLHS / Denominator. If it is so,
13102 // then a SCEV for the numerator already exists and matches with FoundLHS.
13103 auto *Numerator = getExistingSCEV(V: LL);
13104 if (!Numerator || Numerator->getType() != FoundLHS->getType())
13105 return false;
13106
13107 // Make sure that the numerator matches with FoundLHS and the denominator
13108 // is positive.
13109 if (!HasSameValue(A: Numerator, B: FoundLHS) || !isKnownPositive(S: Denominator))
13110 return false;
13111
13112 auto *DTy = Denominator->getType();
13113 auto *FRHSTy = FoundRHS->getType();
13114 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
13115 // One of types is a pointer and another one is not. We cannot extend
13116 // them properly to a wider type, so let us just reject this case.
13117 // TODO: Usage of getEffectiveSCEVType for DTy, FRHSTy etc should help
13118 // to avoid this check.
13119 return false;
13120
13121 // Given that:
13122 // FoundLHS > FoundRHS, LHS = FoundLHS / Denominator, Denominator > 0.
13123 auto *WTy = getWiderType(T1: DTy, T2: FRHSTy);
13124 auto *DenominatorExt = getNoopOrSignExtend(V: Denominator, Ty: WTy);
13125 auto *FoundRHSExt = getNoopOrSignExtend(V: FoundRHS, Ty: WTy);
13126
13127 // Try to prove the following rule:
13128 // (FoundRHS > Denominator - 2) && (RHS <= 0) => (LHS > RHS).
13129 // For example, given that FoundLHS > 2. It means that FoundLHS is at
13130 // least 3. If we divide it by Denominator < 4, we will have at least 1.
13131 auto *DenomMinusTwo = getMinusSCEV(LHS: DenominatorExt, RHS: getConstant(Ty: WTy, V: 2));
13132 if (isKnownNonPositive(S: RHS) &&
13133 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
13134 return true;
13135
13136 // Try to prove the following rule:
13137 // (FoundRHS > -1 - Denominator) && (RHS < 0) => (LHS > RHS).
13138 // For example, given that FoundLHS > -3. Then FoundLHS is at least -2.
13139 // If we divide it by Denominator > 2, then:
13140 // 1. If FoundLHS is negative, then the result is 0.
13141 // 2. If FoundLHS is non-negative, then the result is non-negative.
13142 // Anyways, the result is non-negative.
13143 auto *MinusOne = getMinusOne(Ty: WTy);
13144 auto *NegDenomMinusOne = getMinusSCEV(LHS: MinusOne, RHS: DenominatorExt);
13145 if (isKnownNegative(S: RHS) &&
13146 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
13147 return true;
13148 }
13149 }
13150
13151 // If our expression contained SCEVUnknown Phis, and we split it down and now
13152 // need to prove something for them, try to prove the predicate for every
13153 // possible incoming values of those Phis.
13154 if (isImpliedViaMerge(Pred, LHS: OrigLHS, RHS, FoundLHS: OrigFoundLHS, FoundRHS, Depth: Depth + 1))
13155 return true;
13156
13157 return false;
13158}
13159
13160static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS,
13161 const SCEV *RHS) {
13162 // zext x u<= sext x, sext x s<= zext x
13163 const SCEV *Op;
13164 switch (Pred) {
13165 case ICmpInst::ICMP_SGE:
13166 std::swap(a&: LHS, b&: RHS);
13167 [[fallthrough]];
13168 case ICmpInst::ICMP_SLE: {
13169 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
13170 return match(S: LHS, P: m_scev_SExt(Op0: m_SCEV(V&: Op))) &&
13171 match(S: RHS, P: m_scev_ZExt(Op0: m_scev_Specific(S: Op)));
13172 }
13173 case ICmpInst::ICMP_UGE:
13174 std::swap(a&: LHS, b&: RHS);
13175 [[fallthrough]];
13176 case ICmpInst::ICMP_ULE: {
13177 // If operand >=u 0 then ZExt == SExt. If operand <u 0 then ZExt <u SExt.
13178 return match(S: LHS, P: m_scev_ZExt(Op0: m_SCEV(V&: Op))) &&
13179 match(S: RHS, P: m_scev_SExt(Op0: m_scev_Specific(S: Op)));
13180 }
13181 default:
13182 return false;
13183 };
13184 llvm_unreachable("unhandled case");
13185}
13186
13187bool ScalarEvolution::isKnownViaNonRecursiveReasoning(CmpPredicate Pred,
13188 SCEVUse LHS,
13189 SCEVUse RHS) {
13190 return isKnownPredicateExtendIdiom(Pred, LHS, RHS) ||
13191 isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
13192 IsKnownPredicateViaMinOrMax(SE&: *this, Pred, LHS, RHS) ||
13193 IsKnownPredicateViaAddRecStart(SE&: *this, Pred, LHS, RHS) ||
13194 IsKnownPredicateViaAddRecMonotonicity(SE&: *this, Pred, LHS, RHS) ||
13195 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
13196}
13197
13198bool ScalarEvolution::isImpliedCondOperandsHelper(CmpPredicate Pred,
13199 const SCEV *LHS,
13200 const SCEV *RHS,
13201 const SCEV *FoundLHS,
13202 const SCEV *FoundRHS) {
13203 switch (Pred) {
13204 default:
13205 llvm_unreachable("Unexpected CmpPredicate value!");
13206 case ICmpInst::ICMP_EQ:
13207 case ICmpInst::ICMP_NE:
13208 if (HasSameValue(A: LHS, B: FoundLHS) && HasSameValue(A: RHS, B: FoundRHS))
13209 return true;
13210 break;
13211 case ICmpInst::ICMP_SLT:
13212 case ICmpInst::ICMP_SLE:
13213 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS, RHS: FoundLHS) &&
13214 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS: RHS, RHS: FoundRHS))
13215 return true;
13216 break;
13217 case ICmpInst::ICMP_SGT:
13218 case ICmpInst::ICMP_SGE:
13219 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS, RHS: FoundLHS) &&
13220 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS: RHS, RHS: FoundRHS))
13221 return true;
13222 break;
13223 case ICmpInst::ICMP_ULT:
13224 case ICmpInst::ICMP_ULE:
13225 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS, RHS: FoundLHS) &&
13226 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS: RHS, RHS: FoundRHS))
13227 return true;
13228 break;
13229 case ICmpInst::ICMP_UGT:
13230 case ICmpInst::ICMP_UGE:
13231 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS, RHS: FoundLHS) &&
13232 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: RHS, RHS: FoundRHS))
13233 return true;
13234 break;
13235 }
13236
13237 // Maybe it can be proved via operations?
13238 if (isImpliedViaOperations(Pred, LHS, RHS, FoundLHS, FoundRHS))
13239 return true;
13240
13241 return false;
13242}
13243
13244bool ScalarEvolution::isImpliedCondOperandsViaRanges(
13245 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
13246 const SCEV *FoundLHS, const SCEV *FoundRHS) {
13247 if (!isa<SCEVConstant>(Val: RHS) || !isa<SCEVConstant>(Val: FoundRHS))
13248 // The restriction on `FoundRHS` be lifted easily -- it exists only to
13249 // reduce the compile time impact of this optimization.
13250 return false;
13251
13252 std::optional<APInt> Addend = computeConstantDifference(More: LHS, Less: FoundLHS);
13253 if (!Addend)
13254 return false;
13255
13256 const APInt &ConstFoundRHS = cast<SCEVConstant>(Val: FoundRHS)->getAPInt();
13257
13258 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the
13259 // antecedent "`FoundLHS` `FoundPred` `FoundRHS`".
13260 ConstantRange FoundLHSRange =
13261 ConstantRange::makeExactICmpRegion(Pred: FoundPred, Other: ConstFoundRHS);
13262
13263 // Since `LHS` is `FoundLHS` + `Addend`, we can compute a range for `LHS`:
13264 ConstantRange LHSRange = FoundLHSRange.add(Other: ConstantRange(*Addend));
13265
13266 // We can also compute the range of values for `LHS` that satisfy the
13267 // consequent, "`LHS` `Pred` `RHS`":
13268 const APInt &ConstRHS = cast<SCEVConstant>(Val: RHS)->getAPInt();
13269 // The antecedent implies the consequent if every value of `LHS` that
13270 // satisfies the antecedent also satisfies the consequent.
13271 return LHSRange.icmp(Pred, Other: ConstRHS);
13272}
13273
13274bool ScalarEvolution::canIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
13275 bool IsSigned, bool Invert) {
13276 assert(isKnownPositive(Stride) && "Positive stride expected!");
13277
13278 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
13279 const SCEV *One = getOne(Ty: Stride->getType());
13280
13281 if (IsSigned) {
13282 APInt MaxRHS = getRangeMax(S: RHS, /*IsSigned=*/true, Invert);
13283 APInt MaxValue = APInt::getSignedMaxValue(numBits: BitWidth);
13284 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13285
13286 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
13287 return (std::move(MaxValue) - MaxStrideMinusOne).slt(RHS: MaxRHS);
13288 }
13289
13290 APInt MaxRHS = getRangeMax(S: RHS, /*IsSigned=*/false, Invert);
13291 APInt MaxValue = APInt::getMaxValue(numBits: BitWidth);
13292 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13293
13294 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
13295 return (std::move(MaxValue) - MaxStrideMinusOne).ult(RHS: MaxRHS);
13296}
13297
13298const SCEV *ScalarEvolution::getUDivCeilSCEV(const SCEV *N, const SCEV *D) {
13299 // umin(N, 1) + floor((N - umin(N, 1)) / D)
13300 // This is equivalent to "1 + floor((N - 1) / D)" for N != 0. The umin
13301 // expression fixes the case of N=0.
13302 const SCEV *MinNOne = getUMinExpr(LHS: N, RHS: getOne(Ty: N->getType()));
13303 const SCEV *NMinusOne = getMinusSCEV(LHS: N, RHS: MinNOne);
13304 return getAddExpr(LHS: MinNOne, RHS: getUDivExpr(LHS: NMinusOne, RHS: D));
13305}
13306
13307const SCEV *
13308ScalarEvolution::computeMaxBECountForLT(const SCEV *Start, const SCEV *Stride,
13309 const SCEV *End, unsigned BitWidth,
13310 bool IsSigned, bool Invert) {
13311 // The logic in this function assumes we can represent a positive stride.
13312 // If we can't, the backedge-taken count must be zero.
13313 if (IsSigned && BitWidth == 1)
13314 return getZero(Ty: Stride->getType());
13315
13316 // This code below only been closely audited for negative strides in the
13317 // unsigned comparison case, it may be correct for signed comparison, but
13318 // that needs to be established.
13319 if (IsSigned && isKnownNegative(S: Stride))
13320 return getCouldNotCompute();
13321
13322 // Calculate the maximum backedge count based on the range of values
13323 // permitted by Start, End, and Stride. If Invert is true, both Start and End
13324 // need inverting. Stride was already negated by the caller.
13325 APInt MinStart = getRangeMin(S: Start, IsSigned, Invert);
13326
13327 APInt MinStride =
13328 IsSigned ? getSignedRangeMin(S: Stride) : getUnsignedRangeMin(S: Stride);
13329
13330 // We assume either the stride is positive, or the backedge-taken count
13331 // is zero. So force StrideForMaxBECount to be at least one.
13332 APInt One(BitWidth, 1);
13333 APInt StrideForMaxBECount = IsSigned ? APIntOps::smax(A: One, B: MinStride)
13334 : APIntOps::umax(A: One, B: MinStride);
13335
13336 APInt MaxValue = IsSigned ? APInt::getSignedMaxValue(numBits: BitWidth)
13337 : APInt::getMaxValue(numBits: BitWidth);
13338 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
13339
13340 // Although End can be a MAX expression we estimate MaxEnd considering only
13341 // the case End = RHS of the loop termination condition. This is safe because
13342 // in the other case (End - Start) is zero, leading to a zero maximum backedge
13343 // taken count.
13344 APInt MaxEnd = getRangeMax(S: End, IsSigned, Invert);
13345 MaxEnd =
13346 IsSigned ? APIntOps::smin(A: MaxEnd, B: Limit) : APIntOps::umin(A: MaxEnd, B: Limit);
13347
13348 // MaxBECount = ceil((max(MaxEnd, MinStart) - MinStart) / Stride)
13349 MaxEnd = IsSigned ? APIntOps::smax(A: MaxEnd, B: MinStart)
13350 : APIntOps::umax(A: MaxEnd, B: MinStart);
13351
13352 APInt Delta = MaxEnd - MinStart;
13353
13354 // Try to refine Delta in case End - Start (or Start - End if Invert) gives a
13355 // tighter bound after folding.
13356 const SCEV *DeltaExpr =
13357 Invert ? getMinusSCEV(LHS: Start, RHS: End) : getMinusSCEV(LHS: End, RHS: Start);
13358 Delta = APIntOps::umin(A: Delta, B: getUnsignedRangeMax(S: DeltaExpr));
13359
13360 return getUDivCeilSCEV(N: getConstant(Val: Delta), D: getConstant(Val: StrideForMaxBECount));
13361}
13362
13363ScalarEvolution::ExitLimit
13364ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
13365 const Loop *L, bool IsSigned, bool Invert,
13366 bool ControlsOnlyExit, bool AllowPredicates) {
13367 SmallVector<const SCEVPredicate *> Predicates;
13368
13369 // Loop guards for L, collected on demand.
13370 std::optional<LoopGuards> CachedGuards;
13371 auto getGuards = [&]() -> const LoopGuards & {
13372 if (!CachedGuards)
13373 CachedGuards.emplace(args: LoopGuards::collect(L, SE&: *this));
13374 return *CachedGuards;
13375 };
13376
13377 // FIXME: Extend the non-invariant RHS analysis to greater-than comparisons.
13378 if (Invert && !isLoopInvariant(S: RHS, L))
13379 return getCouldNotCompute();
13380
13381 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13382 bool PredicatedIV = false;
13383 // FIXME: Generalize the NUW inference below to decreasing IVs.
13384 if (!IV && !Invert) {
13385 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: LHS)) {
13386 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: ZExt->getOperand());
13387 if (AR && AR->getLoop() == L && AR->isAffine()) {
13388 auto canProveNUW = [&]() {
13389 // We can use the comparison to infer no-wrap flags only if it fully
13390 // controls the loop exit.
13391 if (!ControlsOnlyExit)
13392 return false;
13393
13394 if (!isLoopInvariant(S: RHS, L))
13395 return false;
13396
13397 if (!isKnownNonZero(S: AR->getStepRecurrence(SE&: *this)))
13398 // We need the sequence defined by AR to strictly increase in the
13399 // unsigned integer domain for the logic below to hold.
13400 return false;
13401
13402 const unsigned InnerBitWidth = getTypeSizeInBits(Ty: AR->getType());
13403 const unsigned OuterBitWidth = getTypeSizeInBits(Ty: RHS->getType());
13404 // If RHS <=u Limit, then there must exist a value V in the sequence
13405 // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and
13406 // V <=u UINT_MAX. Thus, we must exit the loop before unsigned
13407 // overflow occurs. This limit also implies that a signed comparison
13408 // (in the wide bitwidth) is equivalent to an unsigned comparison as
13409 // the high bits on both sides must be zero.
13410 APInt StrideMax = getUnsignedRangeMax(S: AR->getStepRecurrence(SE&: *this));
13411 APInt Limit = APInt::getMaxValue(numBits: InnerBitWidth) - (StrideMax - 1);
13412 Limit = Limit.zext(width: OuterBitWidth);
13413 return getUnsignedRangeMax(S: applyLoopGuards(Expr: RHS, Guards: getGuards()))
13414 .ule(RHS: Limit);
13415 };
13416 auto Flags = AR->getNoWrapFlags();
13417 if (!hasFlags(Flags, TestFlags: SCEV::FlagNUW) && canProveNUW())
13418 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
13419
13420 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
13421 if (AR->hasNoUnsignedWrap()) {
13422 // Emulate what getZeroExtendExpr would have done during construction
13423 // if we'd been able to infer the fact just above at that time.
13424 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
13425 Type *Ty = ZExt->getType();
13426 const SCEV *S = getAddRecExpr(
13427 Start: getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: 0),
13428 Step: getZeroExtendExpr(Op: Step, Ty, Depth: 0), L, Flags: AR->getNoWrapFlags());
13429 IV = dyn_cast<SCEVAddRecExpr>(Val: S);
13430 }
13431 }
13432 }
13433 }
13434
13435 if (!IV && AllowPredicates) {
13436 // Try to make this an AddRec using runtime tests, in the first X
13437 // iterations of this loop, where X is the SCEV expression found by the
13438 // algorithm below.
13439 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13440 PredicatedIV = true;
13441 }
13442
13443 // Avoid weird loops
13444 if (!IV || IV->getLoop() != L || !IV->isAffine())
13445 return getCouldNotCompute();
13446
13447 // A precondition of this method is that the condition being analyzed
13448 // reaches an exiting branch which dominates the latch. Given that, we can
13449 // assume that an increment which violates the nowrap specification and
13450 // produces poison must cause undefined behavior when the resulting poison
13451 // value is branched upon and thus we can conclude that the backedge is
13452 // taken no more often than would be required to produce that poison value.
13453 // Note that a well defined loop can exit on the iteration which violates
13454 // the nowrap specification if there is another exit (either explicit or
13455 // implicit/exceptional) which causes the loop to execute before the
13456 // exiting instruction we're analyzing would trigger UB.
13457 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13458 bool NoWrap = ControlsOnlyExit && any(Val: IV->getNoWrapFlags(Mask: WrapType));
13459 // Reverse the ordering for greater-than comparisons.
13460 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13461 if (Invert)
13462 Cond = ICmpInst::getSwappedPredicate(pred: Cond);
13463
13464 // The step of ~IV is the negated step of IV.
13465 const SCEV *Stride = IV->getStepRecurrence(SE&: *this);
13466 if (Invert)
13467 Stride = getNegativeSCEV(V: Stride);
13468 const SCEV *GuardedStride = Stride;
13469
13470 // Whether the IV may reach the maximum (or minimum if inverted) value
13471 // before the exit is taken.
13472 bool IVMayOverflow = true;
13473
13474 bool PositiveStride = isKnownPositive(S: Stride);
13475 // A dominating guard may prove the stride positive.
13476 if (!PositiveStride) {
13477 const SCEV *LoopGuardedStride = applyLoopGuards(Expr: Stride, Guards: getGuards());
13478 if (isKnownPositive(S: LoopGuardedStride)) {
13479 GuardedStride = LoopGuardedStride;
13480 PositiveStride = true;
13481 // Encode the context-sensitive stride > 0 fact into the expression
13482 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13483 }
13484 }
13485
13486 // Avoid negative or zero stride values.
13487 if (!PositiveStride) {
13488 // FIXME: Generalize the unknown-stride analysis to decreasing IVs.
13489 if (Invert)
13490 return getCouldNotCompute();
13491
13492 // We can compute the correct backedge taken count for loops with unknown
13493 // strides if we can prove that the loop is not an infinite loop with side
13494 // effects. Here's the loop structure we are trying to handle -
13495 //
13496 // i = start
13497 // do {
13498 // A[i] = i;
13499 // i += s;
13500 // } while (i < end);
13501 //
13502 // The backedge taken count for such loops is evaluated as -
13503 // (max(end, start + stride) - start - 1) /u stride
13504 //
13505 // The additional preconditions that we need to check to prove correctness
13506 // of the above formula is as follows -
13507 //
13508 // a) IV is either nuw or nsw depending upon signedness (indicated by the
13509 // NoWrap flag).
13510 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13511 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13512 // no side effects within the loop) or a predicate is added to ensure
13513 // stride is positive.
13514 // c) loop has a single static exit (with no abnormal exits)
13515 //
13516 // Precondition a) implies that if the stride is negative, this is a single
13517 // trip loop. The backedge taken count formula reduces to zero in this case.
13518 //
13519 // Precondition b) and c) combine to imply that if rhs is invariant in L,
13520 // then a zero stride means the backedge can't be taken without executing
13521 // undefined behavior.
13522 //
13523 // The positive stride case is the same as isKnownPositive(Stride) returning
13524 // true (original behavior of the function).
13525 //
13526 if (PredicatedIV || !NoWrap || !loopHasNoAbnormalExits(L))
13527 return getCouldNotCompute();
13528
13529 if (!loopIsFiniteByAssumption(L)) {
13530 // If the loop may be infinite, add a predicate ensuring Stride is
13531 // positive, to guarantee forward progress.
13532 if (!AllowPredicates || !isLoopInvariant(S: Stride, L))
13533 return getCouldNotCompute();
13534
13535 const SCEV *Zero = getZero(Ty: Stride->getType());
13536 const SCEVPredicate *P =
13537 getComparePredicate(Pred: ICmpInst::ICMP_SGT, LHS: Stride, RHS: Zero);
13538 Predicates.push_back(Elt: P);
13539 // When the predicate holds (Stride > 0), umax(Stride, 1) == Stride,
13540 // so the result is unchanged. To prevent div by zero.
13541 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13542 } else if (!isKnownNonZero(S: Stride)) {
13543 // If we have a step of zero, and RHS isn't invariant in L, we don't know
13544 // if it might eventually be greater than start and if so, on which
13545 // iteration. We can't even produce a useful upper bound.
13546 if (!isLoopInvariant(S: RHS, L))
13547 return getCouldNotCompute();
13548
13549 // We allow a potentially zero stride, but we need to divide by stride
13550 // below. Since the loop can't be infinite and this check must control
13551 // the sole exit, we can infer the exit must be taken on the first
13552 // iteration (e.g. backedge count = 0) if the stride is zero. Given that,
13553 // we know the numerator in the divides below must be zero, so we can
13554 // pick an arbitrary non-zero value for the denominator (e.g. stride)
13555 // and produce the right result.
13556 // FIXME: Handle the case where Stride is poison?
13557 auto wouldZeroStrideBeUB = [&]() {
13558 // Proof by contradiction. Suppose the stride were zero. If we can
13559 // prove that the backedge *is* taken on the first iteration, then since
13560 // we know this condition controls the sole exit, we must have an
13561 // infinite loop. We can't have a (well defined) infinite loop per
13562 // check just above.
13563 // Note: The (Start - Stride) term is used to get the start' term from
13564 // (start' + stride,+,stride). Remember that we only care about the
13565 // result of this expression when stride == 0 at runtime.
13566 auto *StartIfZero = getMinusSCEV(LHS: IV->getStart(), RHS: Stride);
13567 return isLoopEntryGuardedByCond(L, Pred: Cond, LHS: StartIfZero, RHS);
13568 };
13569 if (!wouldZeroStrideBeUB()) {
13570 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13571 }
13572 }
13573 } else {
13574 // Avoid proven overflow cases: this will ensure that the backedge taken
13575 // count will not generate any unsigned overflow.
13576 IVMayOverflow = canIVOverflowOnLT(RHS, Stride: GuardedStride, IsSigned, Invert);
13577 if (IVMayOverflow && !NoWrap)
13578 return getCouldNotCompute();
13579 }
13580
13581 // On all paths just preceeding, we established the following invariant:
13582 // IV can be assumed not to overflow up to and including the exiting
13583 // iteration. We proved this in one of two ways:
13584 // 1) We can show overflow doesn't occur before the exiting iteration
13585 // 1a) canIVOverflowOnLT, and b) step of one
13586 // 2) We can show that if overflow occurs, the loop must execute UB
13587 // before any possible exit.
13588 // Note that we have not yet proved RHS invariant (in general).
13589
13590 const SCEV *Start = IV->getStart();
13591
13592 // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond.
13593 // If we convert to integers, isLoopEntryGuardedByCond will miss some cases.
13594 // Use integer-typed versions for actual computation; we can't subtract
13595 // pointers in general.
13596 const SCEV *OrigStart = Start;
13597 const SCEV *OrigRHS = RHS;
13598 if (Start->getType()->isPointerTy()) {
13599 Start = getPtrToAddrExpr(Op: Start);
13600 if (isa<SCEVCouldNotCompute>(Val: Start))
13601 return Start;
13602 }
13603 if (RHS->getType()->isPointerTy()) {
13604 RHS = getPtrToAddrExpr(Op: RHS);
13605 if (isa<SCEVCouldNotCompute>(Val: RHS))
13606 return RHS;
13607 }
13608
13609 const SCEV *End = nullptr, *BECount = getCouldNotCompute(),
13610 *BECountIfBackedgeTaken = getCouldNotCompute();
13611 if (!isLoopInvariant(S: RHS, L)) {
13612 assert(!Invert && "RHS must be loop-invariant for Invert");
13613 const auto *RHSAddRec = dyn_cast<SCEVAddRecExpr>(Val: RHS);
13614 if (PositiveStride && RHSAddRec != nullptr && RHSAddRec->getLoop() == L &&
13615 any(Val: RHSAddRec->getNoWrapFlags())) {
13616 // The structure of loop we are trying to calculate backedge count of:
13617 //
13618 // left = left_start
13619 // right = right_start
13620 //
13621 // while(left < right){
13622 // ... do something here ...
13623 // left += s1; // stride of left is s1 (s1 > 0)
13624 // right += s2; // stride of right is s2 (s2 < 0)
13625 // }
13626 //
13627
13628 const SCEV *RHSStart = RHSAddRec->getStart();
13629 const SCEV *RHSStride = RHSAddRec->getStepRecurrence(SE&: *this);
13630
13631 // If Stride - RHSStride is positive and does not overflow, we can write
13632 // backedge count as ->
13633 // ceil((End - Start) /u (Stride - RHSStride))
13634 // Where, End = max(RHSStart, Start)
13635
13636 // Check if RHSStride < 0 and Stride - RHSStride will not overflow.
13637 if (isKnownNegative(S: RHSStride) &&
13638 willNotOverflow(BinOp: Instruction::Sub, /*Signed=*/true, LHS: Stride,
13639 RHS: RHSStride)) {
13640
13641 const SCEV *Denominator = getMinusSCEV(LHS: Stride, RHS: RHSStride);
13642 if (isKnownPositive(S: Denominator)) {
13643 End = IsSigned ? getSMaxExpr(LHS: RHSStart, RHS: Start)
13644 : getUMaxExpr(LHS: RHSStart, RHS: Start);
13645
13646 // We can do this because End >= Start, as End = max(RHSStart, Start)
13647 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13648
13649 BECount = getUDivCeilSCEV(N: Delta, D: Denominator);
13650 BECountIfBackedgeTaken =
13651 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHSStart, RHS: Start), D: Denominator);
13652 }
13653 }
13654 }
13655 } else {
13656 // Let End = max(RHS,Start). We use the expression (End-Start)/Stride to
13657 // describe the backedge count: if the backedge is taken at least once then
13658 // End is RHS, and if not End is Start so we get a backedge count of zero.
13659 // Inverted, End is min(RHS, Start).
13660 //
13661 // AddingStrideMinusOneMayOverflow has the following preconditions:
13662 //
13663 // 1. Start <= End, signed if IsSigned (inverted: End <= Start)
13664 // 2. The index variable doesn't overflow.
13665 //
13666 // Therefore, we know N exists such that
13667 // (Start + Stride * N) >= End, and computing "(Start + Stride * N)"
13668 // doesn't overflow.
13669 //
13670 // Using this information, try to prove whether the addition in
13671 // "(End - Start) + (Stride - 1)" has unsigned overflow.
13672 //
13673 // If the IV cannot overflow, RHS is at least Stride - 1 below the maximum
13674 // value, so the distance End - Start is at most UMAX - (Stride - 1) and
13675 // the (Stride - 1) addition below cannot overflow.
13676 const SCEV *One = getOne(Ty: Stride->getType());
13677 bool AddingStrideMinusOneMayOverflow = IVMayOverflow && [&] {
13678 if (isKnownToBeAPowerOfTwo(S: Stride)) {
13679 // Suppose Stride is a power of two, and Start/End are unsigned
13680 // integers. Let UMAX be the largest representable unsigned
13681 // integer.
13682 //
13683 // By the preconditions of this function, we know
13684 // "(Start + Stride * N) >= End", and this doesn't overflow.
13685 // As a formula:
13686 //
13687 // End <= (Start + Stride * N) <= UMAX
13688 //
13689 // Subtracting Start from all the terms:
13690 //
13691 // End - Start <= Stride * N <= UMAX - Start
13692 //
13693 // Since Start is unsigned, UMAX - Start <= UMAX. Therefore:
13694 //
13695 // End - Start <= Stride * N <= UMAX
13696 //
13697 // Stride * N is a multiple of Stride. Therefore,
13698 //
13699 // End - Start <= Stride * N <= UMAX - (UMAX mod Stride)
13700 //
13701 // Since Stride is a power of two, UMAX + 1 is divisible by
13702 // Stride. Therefore, UMAX mod Stride == Stride - 1. So we can
13703 // write:
13704 //
13705 // End - Start <= Stride * N <= UMAX - Stride - 1
13706 //
13707 // Dropping the middle term:
13708 //
13709 // End - Start <= UMAX - Stride - 1
13710 //
13711 // Adding Stride - 1 to both sides:
13712 //
13713 // (End - Start) + (Stride - 1) <= UMAX
13714 //
13715 // In other words, the addition doesn't have unsigned overflow.
13716 //
13717 // A similar proof works if we treat Start/End as signed values.
13718 // Just rewrite steps before "End - Start <= Stride * N <= UMAX"
13719 // to use signed max instead of unsigned max. Note that we're
13720 // trying to prove a lack of unsigned overflow in either case.
13721 // Inverted: "Start - End <= Stride * N <= Start - MIN <= UMAX", same.
13722 return false;
13723 }
13724 if (!Invert && (Start == Stride || Start == getMinusSCEV(LHS: Stride, RHS: One))) {
13725 // If Start is equal to Stride, (End - Start) + (Stride - 1) == End
13726 // - 1. If !IsSigned, 0 <u Stride == Start <=u End; so 0 <u End - 1
13727 // <u End. If IsSigned, 0 <s Stride == Start <=s End; so 0 <s End -
13728 // 1 <s End.
13729 //
13730 // If Start is equal to Stride - 1, (End - Start) + Stride - 1 ==
13731 // End.
13732 //
13733 // Both need Start to be the smaller value, so neither applies inverted.
13734 return false;
13735 }
13736 return true;
13737 }();
13738
13739 // If inverted, the analyzed values are complements: "~V - Offset" is "~(V +
13740 // Offset)" and "~To - ~From" is "From - To".
13741 auto StepBack = [&](const SCEV *V, const SCEV *Offset) -> const SCEV * {
13742 if (Invert)
13743 return getAddExpr(LHS: V, RHS: Offset);
13744 return getMinusSCEV(LHS: V, RHS: Offset);
13745 };
13746 auto Distance = [&](const SCEV *From, const SCEV *To) {
13747 return Invert ? getMinusSCEV(LHS: From, RHS: To) : getMinusSCEV(LHS: To, RHS: From);
13748 };
13749
13750 const SCEV *OrigPrevStart = StepBack(OrigStart, Stride);
13751 assert(isAvailableAtLoopEntry(OrigPrevStart, L) && "Must be!");
13752 assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!");
13753 assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!");
13754 // Can we prove Start - Stride < RHS, and either Start - Stride < Start or
13755 // (via !AddingStrideMinusOneMayOverflow) that (RHS - Start) + (Stride - 1)
13756 // does not overflow?
13757 if ((!AddingStrideMinusOneMayOverflow ||
13758 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigPrevStart, RHS: OrigStart)) &&
13759 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigPrevStart, RHS: OrigRHS)) {
13760 // In this case, we can use a refined formula for computing backedge
13761 // taken count. The general formula remains:
13762 // "End-Start /uceiling Stride"
13763 // We want to use the alternate formula:
13764 // "((RHS - 1) - (Start - Stride)) /u Stride"
13765 // Let's do a quick case analysis to show these are equivalent under
13766 // our preconditions. When inverted, the proof uses complemented Start,
13767 // RHS and End; Stride remains positive.
13768 // * For RHS <= Start (End is Start), the backedge-taken count must be
13769 // zero. Together with the precondition "Start - Stride < RHS", we have
13770 // "Start - Stride < RHS <= Start". Subtracting Start - Stride from
13771 // all sides we get "0 < RHS - (Start - Stride) <= Stride".
13772 // Subtracting 1 we get "0 <= (RHS - 1) - (Start - Stride) < Stride".
13773 // So dividing that by Stride gives zero.
13774 //
13775 // * For RHS > Start (End is RHS), the backedge count must be
13776 // "RHS-Start /uceil Stride", so it is sufficient to show that the
13777 // numerator "((RHS - 1) - (Start - Stride))" does not overflow.
13778 //
13779 // If "Start - Stride < Start" holds, we have
13780 // "RHS > Start > Start - Stride". As such
13781 // "RHS - (Start - Stride) - 1" does not overflow, which is the
13782 // reassociated numerator.
13783 //
13784 // Otherwise !AddingStrideMinusOneMayOverflow guarantees that
13785 // "(End - Start) + (Stride - 1)" does not overflow unsigned. Here
13786 // "End" is "RHS", as "RHS > Start", so this is the reassociated
13787 // numerator. Neither sub-term wraps unsigned: "RHS - Start"
13788 // due to "RHS > Start", and "Stride - 1", as Stride is non-zero.
13789 const SCEV *Numerator =
13790 getMinusSCEV(LHS: Distance(StepBack(Start, Stride), RHS), RHS: One);
13791 BECount = getUDivExpr(LHS: Numerator, RHS: Stride);
13792 }
13793
13794 if (isa<SCEVCouldNotCompute>(Val: BECount)) {
13795 auto canProveRHSIsAtOrBeyondStart = [&]() {
13796 // Inverted, the claim is "Start >= RHS". Reverse the comparisons below
13797 // by swapping their operands rather than their predicates:
13798 // isLoopEntryGuardedByCond is sensitive to operand order and loses the
13799 // proof if the IV bound moves to the other side.
13800 auto SwapIfInverted = [&](const SCEV *A, const SCEV *B) {
13801 return Invert ? std::pair(B, A) : std::pair(A, B);
13802 };
13803
13804 auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13805 const SCEV *GuardedRHS = applyLoopGuards(Expr: OrigRHS, Guards: getGuards());
13806 const SCEV *GuardedStart = applyLoopGuards(Expr: OrigStart, Guards: getGuards());
13807 if (Invert)
13808 std::swap(a&: GuardedRHS, b&: GuardedStart);
13809
13810 auto [GELHS, GERHS] = SwapIfInverted(OrigRHS, OrigStart);
13811 if (isLoopEntryGuardedByCond(L, Pred: CondGE, LHS: GELHS, RHS: GERHS) ||
13812 isKnownPredicate(Pred: CondGE, LHS: GuardedRHS, RHS: GuardedStart))
13813 return true;
13814
13815 // (RHS > Start - 1) implies RHS >= Start.
13816 // * "RHS >= Start" is trivially equivalent to "RHS > Start - 1" if
13817 // "Start - 1" doesn't overflow.
13818 // * For signed comparison, if Start - 1 does overflow, it's equal
13819 // to INT_MAX, and "RHS >s INT_MAX" is trivially false.
13820 // * For unsigned comparison, if Start - 1 does overflow, it's equal
13821 // to UINT_MAX, and "RHS >u UINT_MAX" is trivially false.
13822 //
13823 // FIXME: Should isLoopEntryGuardedByCond do this for us?
13824 auto CondGT = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13825 auto [GTLHS, GTRHS] = SwapIfInverted(OrigRHS, StepBack(OrigStart, One));
13826 return isLoopEntryGuardedByCond(L, Pred: CondGT, LHS: GTLHS, RHS: GTRHS);
13827 };
13828
13829 // If we know that RHS >= Start in the context of loop, then we know
13830 // that max(RHS, Start) = RHS at this point.
13831 if (canProveRHSIsAtOrBeyondStart()) {
13832 End = RHS;
13833 } else {
13834 // If RHS < Start, the backedge will be taken zero times. So in
13835 // general, we can write the backedge-taken count as:
13836 //
13837 // RHS >= Start ? ceil(RHS - Start) / Stride : 0
13838 //
13839 // We convert it to the following to make it more convenient for SCEV:
13840 //
13841 // ceil(max(RHS, Start) - Start) / Stride
13842 //
13843 // Inverted, this is ceil(Start - min(RHS, Start)) / Stride.
13844 if (Invert)
13845 End = IsSigned ? getSMinExpr(LHS: RHS, RHS: Start) : getUMinExpr(LHS: RHS, RHS: Start);
13846 else
13847 End = IsSigned ? getSMaxExpr(LHS: RHS, RHS: Start) : getUMaxExpr(LHS: RHS, RHS: Start);
13848
13849 // See what would happen if we assume the backedge is taken. This is
13850 // used to compute MaxBECount.
13851 BECountIfBackedgeTaken = getUDivCeilSCEV(N: Distance(Start, RHS), D: Stride);
13852 }
13853
13854 const SCEV *Delta = Distance(Start, End);
13855 if (!AddingStrideMinusOneMayOverflow) {
13856 // floor((D + (S - 1)) / S)
13857 // We prefer this formulation if it's legal because it's fewer
13858 // operations.
13859 BECount =
13860 getUDivExpr(LHS: getAddExpr(LHS: Delta, RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13861 } else {
13862 BECount = getUDivCeilSCEV(N: Delta, D: Stride);
13863 }
13864 }
13865 }
13866
13867 const SCEV *ConstantMaxBECount;
13868 bool MaxOrZero = false;
13869 if (isa<SCEVConstant>(Val: BECount)) {
13870 ConstantMaxBECount = BECount;
13871 } else {
13872 ConstantMaxBECount = computeMaxBECountForLT(
13873 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned,
13874 Invert);
13875 // If we know exactly how many times the backedge will be taken if it's
13876 // taken at least once, then the backedge count will either be that or
13877 // zero. If that count exceeds the range-based bound, the backedge can
13878 // never be taken.
13879 const APInt *IfTaken, *RangeMax;
13880 if (match(S: BECountIfBackedgeTaken, P: m_scev_APInt(C&: IfTaken))) {
13881 if (match(S: ConstantMaxBECount, P: m_scev_APInt(C&: RangeMax)) &&
13882 IfTaken->ugt(RHS: *RangeMax)) {
13883 ConstantMaxBECount = getZero(Ty: BECountIfBackedgeTaken->getType());
13884 } else {
13885 ConstantMaxBECount = BECountIfBackedgeTaken;
13886 MaxOrZero = true;
13887 }
13888 }
13889 }
13890
13891 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
13892 !isa<SCEVCouldNotCompute>(Val: BECount))
13893 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
13894
13895 const SCEV *SymbolicMaxBECount =
13896 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13897 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, MaxOrZero,
13898 Predicates);
13899}
13900
13901const SCEV *SCEVAddRecExpr::getNumIterationsInRange(const ConstantRange &Range,
13902 ScalarEvolution &SE) const {
13903 if (Range.isFullSet()) // Infinite loop.
13904 return SE.getCouldNotCompute();
13905
13906 // If the start is a non-zero constant, shift the range to simplify things.
13907 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: getStart()))
13908 if (!SC->getValue()->isZero()) {
13909 SmallVector<SCEVUse, 4> Operands(operands());
13910 Operands[0] = SE.getZero(Ty: SC->getType());
13911 const SCEV *Shifted = SE.getAddRecExpr(Operands, L: getLoop(),
13912 NWFlags: getNoWrapFlags(Mask: FlagNW));
13913 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Val: Shifted))
13914 return ShiftedAddRec->getNumIterationsInRange(
13915 Range: Range.subtract(CI: SC->getAPInt()), SE);
13916 // This is strange and shouldn't happen.
13917 return SE.getCouldNotCompute();
13918 }
13919
13920 // The only time we can solve this is when we have all constant indices.
13921 // Otherwise, we cannot determine the overflow conditions.
13922 if (!all_of(Range: operands(), P: IsaPred<SCEVConstant>))
13923 return SE.getCouldNotCompute();
13924
13925 // Okay at this point we know that all elements of the chrec are constants and
13926 // that the start element is zero.
13927
13928 // First check to see if the range contains zero. If not, the first
13929 // iteration exits.
13930 unsigned BitWidth = SE.getTypeSizeInBits(Ty: getType());
13931 if (!Range.contains(Val: APInt(BitWidth, 0)))
13932 return SE.getZero(Ty: getType());
13933
13934 if (isAffine()) {
13935 // If this is an affine expression then we have this situation:
13936 // Solve {0,+,A} in Range === Ax in Range
13937
13938 // We know that zero is in the range. If A is positive then we know that
13939 // the upper value of the range must be the first possible exit value.
13940 // If A is negative then the lower of the range is the last possible loop
13941 // value. Also note that we already checked for a full range.
13942 APInt A = cast<SCEVConstant>(Val: getOperand(i: 1))->getAPInt();
13943 APInt End = A.sge(RHS: 1) ? (Range.getUpper() - 1) : Range.getLower();
13944
13945 // The exit value should be (End+A)/A.
13946 APInt ExitVal = (End + A).udiv(RHS: A);
13947 ConstantInt *ExitValue = ConstantInt::get(Context&: SE.getContext(), V: ExitVal);
13948
13949 // Evaluate at the exit value. If we really did fall out of the valid
13950 // range, then we computed our trip count, otherwise wrap around or other
13951 // things must have happened.
13952 ConstantInt *Val = EvaluateConstantChrecAtConstant(AddRec: this, C: ExitValue, SE);
13953 if (Range.contains(Val: Val->getValue()))
13954 return SE.getCouldNotCompute(); // Something strange happened
13955
13956 // Ensure that the previous value is in the range.
13957 assert(Range.contains(
13958 EvaluateConstantChrecAtConstant(this,
13959 ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) &&
13960 "Linear scev computation is off in a bad way!");
13961 return SE.getConstant(V: ExitValue);
13962 }
13963
13964 if (isQuadratic()) {
13965 if (auto S = SolveQuadraticAddRecRange(AddRec: this, Range, SE))
13966 return SE.getConstant(Val: *S);
13967 }
13968
13969 return SE.getCouldNotCompute();
13970}
13971
13972const SCEVAddRecExpr *
13973SCEVAddRecExpr::getPostIncExpr(ScalarEvolution &SE) const {
13974 assert(getNumOperands() > 1 && "AddRec with zero step?");
13975 // There is a temptation to just call getAddExpr(this, getStepRecurrence(SE)),
13976 // but in this case we cannot guarantee that the value returned will be an
13977 // AddRec because SCEV does not have a fixed point where it stops
13978 // simplification: it is legal to return ({rec1} + {rec2}). For example, it
13979 // may happen if we reach arithmetic depth limit while simplifying. So we
13980 // construct the returned value explicitly.
13981 SmallVector<SCEVUse, 3> Ops;
13982 // If this is {A,+,B,+,C,...,+,N}, then its step is {B,+,C,+,...,+,N}, and
13983 // (this + Step) is {A+B,+,B+C,+...,+,N}.
13984 for (unsigned i = 0, e = getNumOperands() - 1; i < e; ++i)
13985 Ops.push_back(Elt: SE.getAddExpr(LHS: getOperand(i), RHS: getOperand(i: i + 1)));
13986 // We know that the last operand is not a constant zero (otherwise it would
13987 // have been popped out earlier). This guarantees us that if the result has
13988 // the same last operand, then it will also not be popped out, meaning that
13989 // the returned value will be an AddRec.
13990 const SCEV *Last = getOperand(i: getNumOperands() - 1);
13991 assert(!Last->isZero() && "Recurrency with zero step?");
13992 Ops.push_back(Elt: Last);
13993 return cast<SCEVAddRecExpr>(Val: SE.getAddRecExpr(Operands&: Ops, L: getLoop(), NWFlags: SCEV::FlagNone));
13994}
13995
13996// Return true when S contains at least an undef value.
13997bool ScalarEvolution::containsUndefs(const SCEV *S) const {
13998 return SCEVExprContains(
13999 Root: S, Pred: [](const SCEV *S) { return match(S, P: m_scev_UndefOrPoison()); });
14000}
14001
14002// Return true when S contains a value that is a nullptr.
14003bool ScalarEvolution::containsErasedValue(const SCEV *S) const {
14004 return SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
14005 if (const auto *SU = dyn_cast<SCEVUnknown>(Val: S))
14006 return SU->getValue() == nullptr;
14007 return false;
14008 });
14009}
14010
14011/// Return the size of an element read or written by Inst.
14012const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) {
14013 if (!isa<LoadInst, StoreInst>(Val: Inst))
14014 return nullptr;
14015 Type *ETy = getEffectiveSCEVType(Ty: getLoadStorePointerOperand(V: Inst)->getType());
14016 return getSizeOfExpr(IntTy: ETy, AllocTy: getLoadStoreType(I: Inst));
14017}
14018
14019//===----------------------------------------------------------------------===//
14020// SCEVCallbackVH Class Implementation
14021//===----------------------------------------------------------------------===//
14022
14023void ScalarEvolution::SCEVCallbackVH::deleted() {
14024 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
14025 if (PHINode *PN = dyn_cast<PHINode>(Val: getValPtr()))
14026 SE->ConstantEvolutionLoopExitValue.erase(Val: PN);
14027 SE->eraseValueFromMap(V: getValPtr());
14028 // this now dangles!
14029}
14030
14031void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
14032 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
14033
14034 // Forget all the expressions associated with users of the old value,
14035 // so that future queries will recompute the expressions using the new
14036 // value.
14037 SE->forgetValue(V: getValPtr());
14038 // this now dangles!
14039}
14040
14041ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
14042 : CallbackVH(V), SE(se) {}
14043
14044//===----------------------------------------------------------------------===//
14045// ScalarEvolution Class Implementation
14046//===----------------------------------------------------------------------===//
14047
14048ScalarEvolution::ScalarEvolution(Function &F, TargetLibraryInfo &TLI,
14049 AssumptionCache &AC, DominatorTree &DT,
14050 LoopInfo &LI)
14051 : F(F), DL(F.getDataLayout()), TLI(TLI), AC(AC), DT(DT), LI(LI),
14052 CouldNotCompute(new SCEVCouldNotCompute()), ValuesAtScopes(64),
14053 LoopDispositions(64), BlockDispositions(64) {
14054 // To use guards for proving predicates, we need to scan every instruction in
14055 // relevant basic blocks, and not just terminators. Doing this is a waste of
14056 // time if the IR does not actually contain any calls to
14057 // @llvm.experimental.guard, so do a quick check and remember this beforehand.
14058 //
14059 // This pessimizes the case where a pass that preserves ScalarEvolution wants
14060 // to _add_ guards to the module when there weren't any before, and wants
14061 // ScalarEvolution to optimize based on those guards. For now we prefer to be
14062 // efficient in lieu of being smart in that rather obscure case.
14063
14064 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
14065 M: F.getParent(), id: Intrinsic::experimental_guard);
14066 HasGuards = GuardDecl && !GuardDecl->use_empty();
14067}
14068
14069ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg)
14070 : F(Arg.F), DL(Arg.DL), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC),
14071 DT(Arg.DT), LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)),
14072 ValueExprMap(std::move(Arg.ValueExprMap)),
14073 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
14074 PendingMerges(std::move(Arg.PendingMerges)),
14075 ConstantMultipleCache(std::move(Arg.ConstantMultipleCache)),
14076 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
14077 PredicatedBackedgeTakenCounts(
14078 std::move(Arg.PredicatedBackedgeTakenCounts)),
14079 BECountUsers(std::move(Arg.BECountUsers)),
14080 ConstantEvolutionLoopExitValue(
14081 std::move(Arg.ConstantEvolutionLoopExitValue)),
14082 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
14083 ValuesAtScopesUsers(std::move(Arg.ValuesAtScopesUsers)),
14084 LoopDispositions(std::move(Arg.LoopDispositions)),
14085 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
14086 BlockDispositions(std::move(Arg.BlockDispositions)),
14087 SCEVUsers(std::move(Arg.SCEVUsers)),
14088 UnsignedRanges(std::move(Arg.UnsignedRanges)),
14089 SignedRanges(std::move(Arg.SignedRanges)),
14090 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
14091 UniquePreds(std::move(Arg.UniquePreds)),
14092 SCEVAllocator(std::move(Arg.SCEVAllocator)),
14093 ConstantSCEVs(std::move(Arg.ConstantSCEVs)),
14094 LoopUsers(std::move(Arg.LoopUsers)),
14095 PredicatedSCEVRewrites(std::move(Arg.PredicatedSCEVRewrites)),
14096 FirstUnknown(Arg.FirstUnknown) {
14097 Arg.FirstUnknown = nullptr;
14098}
14099
14100ScalarEvolution::~ScalarEvolution() {
14101 // Iterate through all the SCEVUnknown instances and call their
14102 // destructors, so that they release their references to their values.
14103 for (SCEVUnknown *U = FirstUnknown; U;) {
14104 SCEVUnknown *Tmp = U;
14105 U = U->Next;
14106 Tmp->~SCEVUnknown();
14107 }
14108 FirstUnknown = nullptr;
14109
14110 ExprValueMap.clear();
14111 ValueExprMap.clear();
14112 HasRecMap.clear();
14113 BackedgeTakenCounts.clear();
14114 PredicatedBackedgeTakenCounts.clear();
14115
14116 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage");
14117 assert(PendingMerges.empty() && "isImpliedViaMerge garbage");
14118 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!");
14119 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!");
14120}
14121
14122bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) {
14123 return !isa<SCEVCouldNotCompute>(Val: getBackedgeTakenCount(L));
14124}
14125
14126/// When printing a top-level SCEV for trip counts, it's helpful to include
14127/// a type for constants which are otherwise hard to disambiguate.
14128static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV* S) {
14129 if (isa<SCEVConstant>(Val: S))
14130 OS << *S->getType() << " ";
14131 OS << *S;
14132}
14133
14134static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE,
14135 const Loop *L) {
14136 // Print all inner loops first
14137 for (Loop *I : *L)
14138 PrintLoopInfo(OS, SE, L: I);
14139
14140 OS << "Loop ";
14141 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14142 OS << ": ";
14143
14144 SmallVector<BasicBlock *, 8> ExitingBlocks;
14145 L->getExitingBlocks(ExitingBlocks);
14146 if (ExitingBlocks.size() != 1)
14147 OS << "<multiple exits> ";
14148
14149 auto *BTC = SE->getBackedgeTakenCount(L);
14150 if (!isa<SCEVCouldNotCompute>(Val: BTC)) {
14151 OS << "backedge-taken count is ";
14152 PrintSCEVWithTypeHint(OS, S: BTC);
14153 } else
14154 OS << "Unpredictable backedge-taken count.";
14155 OS << "\n";
14156
14157 if (ExitingBlocks.size() > 1)
14158 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14159 OS << " exit count for " << ExitingBlock->getName() << ": ";
14160 const SCEV *EC = SE->getExitCount(L, ExitingBlock);
14161 PrintSCEVWithTypeHint(OS, S: EC);
14162 if (isa<SCEVCouldNotCompute>(Val: EC)) {
14163 // Retry with predicates.
14164 SmallVector<const SCEVPredicate *> Predicates;
14165 EC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates);
14166 if (!isa<SCEVCouldNotCompute>(Val: EC)) {
14167 OS << "\n predicated exit count for " << ExitingBlock->getName()
14168 << ": ";
14169 PrintSCEVWithTypeHint(OS, S: EC);
14170 OS << "\n Predicates:\n";
14171 for (const auto *P : Predicates)
14172 P->print(OS, Depth: 4);
14173 }
14174 }
14175 OS << "\n";
14176 }
14177
14178 OS << "Loop ";
14179 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14180 OS << ": ";
14181
14182 auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L);
14183 if (!isa<SCEVCouldNotCompute>(Val: ConstantBTC)) {
14184 OS << "constant max backedge-taken count is ";
14185 PrintSCEVWithTypeHint(OS, S: ConstantBTC);
14186 if (SE->isBackedgeTakenCountMaxOrZero(L))
14187 OS << ", actual taken count either this or zero.";
14188 } else {
14189 OS << "Unpredictable constant max backedge-taken count. ";
14190 }
14191
14192 OS << "\n"
14193 "Loop ";
14194 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14195 OS << ": ";
14196
14197 auto *SymbolicBTC = SE->getSymbolicMaxBackedgeTakenCount(L);
14198 if (!isa<SCEVCouldNotCompute>(Val: SymbolicBTC)) {
14199 OS << "symbolic max backedge-taken count is ";
14200 PrintSCEVWithTypeHint(OS, S: SymbolicBTC);
14201 if (SE->isBackedgeTakenCountMaxOrZero(L))
14202 OS << ", actual taken count either this or zero.";
14203 } else {
14204 OS << "Unpredictable symbolic max backedge-taken count. ";
14205 }
14206 OS << "\n";
14207
14208 if (ExitingBlocks.size() > 1)
14209 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14210 OS << " symbolic max exit count for " << ExitingBlock->getName() << ": ";
14211 auto *ExitBTC = SE->getExitCount(L, ExitingBlock,
14212 Kind: ScalarEvolution::SymbolicMaximum);
14213 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14214 if (isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14215 // Retry with predicates.
14216 SmallVector<const SCEVPredicate *> Predicates;
14217 ExitBTC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates,
14218 Kind: ScalarEvolution::SymbolicMaximum);
14219 if (!isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14220 OS << "\n predicated symbolic max exit count for "
14221 << ExitingBlock->getName() << ": ";
14222 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14223 OS << "\n Predicates:\n";
14224 for (const auto *P : Predicates)
14225 P->print(OS, Depth: 4);
14226 }
14227 }
14228 OS << "\n";
14229 }
14230
14231 SmallVector<const SCEVPredicate *, 4> Preds;
14232 auto *PBT = SE->getPredicatedBackedgeTakenCount(L, Preds);
14233 if (PBT != BTC) {
14234 OS << "Loop ";
14235 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14236 OS << ": ";
14237 if (!isa<SCEVCouldNotCompute>(Val: PBT)) {
14238 OS << "Predicated backedge-taken count is ";
14239 PrintSCEVWithTypeHint(OS, S: PBT);
14240 } else
14241 OS << "Unpredictable predicated backedge-taken count.";
14242 OS << "\n";
14243 OS << " Predicates:\n";
14244 for (const auto *P : Preds)
14245 P->print(OS, Depth: 4);
14246 }
14247 Preds.clear();
14248
14249 auto *PredConstantMax =
14250 SE->getPredicatedConstantMaxBackedgeTakenCount(L, Preds);
14251 if (PredConstantMax != ConstantBTC) {
14252 OS << "Loop ";
14253 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14254 OS << ": ";
14255 if (!isa<SCEVCouldNotCompute>(Val: PredConstantMax)) {
14256 OS << "Predicated constant max backedge-taken count is ";
14257 PrintSCEVWithTypeHint(OS, S: PredConstantMax);
14258 } else
14259 OS << "Unpredictable predicated constant max backedge-taken count.";
14260 OS << "\n";
14261 OS << " Predicates:\n";
14262 for (const auto *P : Preds)
14263 P->print(OS, Depth: 4);
14264 }
14265 Preds.clear();
14266
14267 auto *PredSymbolicMax =
14268 SE->getPredicatedSymbolicMaxBackedgeTakenCount(L, Preds);
14269 if (SymbolicBTC != PredSymbolicMax) {
14270 OS << "Loop ";
14271 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14272 OS << ": ";
14273 if (!isa<SCEVCouldNotCompute>(Val: PredSymbolicMax)) {
14274 OS << "Predicated symbolic max backedge-taken count is ";
14275 PrintSCEVWithTypeHint(OS, S: PredSymbolicMax);
14276 } else
14277 OS << "Unpredictable predicated symbolic max backedge-taken count.";
14278 OS << "\n";
14279 OS << " Predicates:\n";
14280 for (const auto *P : Preds)
14281 P->print(OS, Depth: 4);
14282 }
14283
14284 if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
14285 OS << "Loop ";
14286 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14287 OS << ": ";
14288 OS << "Trip multiple is " << SE->getSmallConstantTripMultiple(L) << "\n";
14289 }
14290}
14291
14292namespace llvm {
14293// Note: these overloaded operators need to be in the llvm namespace for them
14294// to be resolved correctly. If we put them outside the llvm namespace, the
14295//
14296// OS << ": " << SE.getLoopDisposition(SV, InnerL);
14297//
14298// code below "breaks" and start printing raw enum values as opposed to the
14299// string values.
14300static raw_ostream &operator<<(raw_ostream &OS,
14301 ScalarEvolution::LoopDisposition LD) {
14302 switch (LD) {
14303 case ScalarEvolution::LoopVariant:
14304 OS << "Variant";
14305 break;
14306 case ScalarEvolution::LoopInvariant:
14307 OS << "Invariant";
14308 break;
14309 case ScalarEvolution::LoopUniform:
14310 OS << "Uniform";
14311 break;
14312 case ScalarEvolution::LoopComputable:
14313 OS << "Computable";
14314 break;
14315 }
14316 return OS;
14317}
14318
14319static raw_ostream &operator<<(raw_ostream &OS,
14320 llvm::ScalarEvolution::BlockDisposition BD) {
14321 switch (BD) {
14322 case ScalarEvolution::DoesNotDominateBlock:
14323 OS << "DoesNotDominate";
14324 break;
14325 case ScalarEvolution::DominatesBlock:
14326 OS << "Dominates";
14327 break;
14328 case ScalarEvolution::ProperlyDominatesBlock:
14329 OS << "ProperlyDominates";
14330 break;
14331 }
14332 return OS;
14333}
14334} // namespace llvm
14335
14336void ScalarEvolution::print(raw_ostream &OS) const {
14337 // ScalarEvolution's implementation of the print method is to print
14338 // out SCEV values of all instructions that are interesting. Doing
14339 // this potentially causes it to create new SCEV objects though,
14340 // which technically conflicts with the const qualifier. This isn't
14341 // observable from outside the class though, so casting away the
14342 // const isn't dangerous.
14343 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14344
14345 if (ClassifyExpressions) {
14346 OS << "Classifying expressions for: ";
14347 F.printAsOperand(O&: OS, /*PrintType=*/false);
14348 OS << "\n";
14349 for (Instruction &I : instructions(F))
14350 if (isSCEVable(Ty: I.getType()) && !isa<CmpInst>(Val: I)) {
14351 OS << I << '\n';
14352 OS << " --> ";
14353 const SCEV *SV = SE.getSCEV(V: &I);
14354 SV->print(OS);
14355 if (!isa<SCEVCouldNotCompute>(Val: SV)) {
14356 OS << " U: ";
14357 SE.getUnsignedRange(S: SV).print(OS);
14358 OS << " S: ";
14359 SE.getSignedRange(S: SV).print(OS);
14360 }
14361
14362 const Loop *L = LI.getLoopFor(BB: I.getParent());
14363
14364 SCEVUse AtUse = SE.getSCEVAtScope(V: SV, L);
14365 if (AtUse != SV) {
14366 OS << " --> ";
14367 OS << AtUse;
14368 if (!isa<SCEVCouldNotCompute>(Val: AtUse)) {
14369 OS << " U: ";
14370 SE.getUnsignedRange(S: AtUse).print(OS);
14371 OS << " S: ";
14372 SE.getSignedRange(S: AtUse).print(OS);
14373 }
14374 }
14375
14376 if (L) {
14377 OS << "\t\t" "Exits: ";
14378 SCEVUse ExitValue = SE.getSCEVAtScope(V: SV, L: L->getParentLoop());
14379 if (!SE.isLoopInvariant(S: ExitValue, L)) {
14380 OS << "<<Unknown>>";
14381 } else {
14382 OS << ExitValue;
14383 }
14384
14385 ListSeparator LS(", ", "\t\tLoopDispositions: { ");
14386 for (const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
14387 OS << LS;
14388 Iter->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14389 OS << ": " << SE.getLoopDisposition(S: SV, L: Iter);
14390 }
14391
14392 for (const auto *InnerL : depth_first(G: L)) {
14393 if (InnerL == L)
14394 continue;
14395 OS << LS;
14396 InnerL->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14397 OS << ": " << SE.getLoopDisposition(S: SV, L: InnerL);
14398 }
14399
14400 OS << " }";
14401 }
14402
14403 OS << "\n";
14404 }
14405 }
14406
14407 OS << "Determining loop execution counts for: ";
14408 F.printAsOperand(O&: OS, /*PrintType=*/false);
14409 OS << "\n";
14410 for (Loop *I : LI)
14411 PrintLoopInfo(OS, SE: &SE, L: I);
14412}
14413
14414ScalarEvolution::LoopDisposition
14415ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) {
14416 auto &Values = LoopDispositions[S];
14417 for (auto &V : Values) {
14418 if (V.getPointer() == L)
14419 return V.getInt();
14420 }
14421 Values.emplace_back(Args&: L, Args: LoopVariant);
14422 LoopDisposition D = computeLoopDisposition(S, L);
14423 auto &Values2 = LoopDispositions[S];
14424 for (auto &V : llvm::reverse(C&: Values2)) {
14425 if (V.getPointer() == L) {
14426 V.setInt(D);
14427 break;
14428 }
14429 }
14430 return D;
14431}
14432
14433ScalarEvolution::LoopDisposition
14434ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
14435 switch (S->getSCEVType()) {
14436 case scConstant:
14437 case scVScale:
14438 return LoopInvariant;
14439 case scAddRecExpr: {
14440 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14441
14442 // If L is the addrec's loop, it's computable.
14443 if (AR->getLoop() == L)
14444 return LoopComputable;
14445
14446 // Add recurrences are never invariant in the function-body (null loop).
14447 if (!L)
14448 return LoopVariant;
14449
14450 // Everything that is not defined at loop entry is variant.
14451 if (DT.dominates(A: L->getHeader(), B: AR->getLoop()->getHeader())) {
14452 if (L->contains(L: AR->getLoop()) &&
14453 llvm::all_of(Range: AR->operands(),
14454 P: [&](const SCEV *Op) { return isLoopUniform(S: Op, L); }))
14455 return LoopUniform;
14456
14457 return LoopVariant;
14458 }
14459 assert(!L->contains(AR->getLoop()) && "Containing loop's header does not"
14460 " dominate the contained loop's header?");
14461
14462 // This recurrence is invariant w.r.t. L if AR's loop contains L.
14463 if (AR->getLoop()->contains(L))
14464 return LoopInvariant;
14465
14466 // This recurrence is variant w.r.t. L if any of its operands
14467 // are variant.
14468 for (SCEVUse Op : AR->operands())
14469 if (!isLoopInvariant(S: Op, L))
14470 return LoopVariant;
14471
14472 // Otherwise it's loop-invariant.
14473 return LoopInvariant;
14474 }
14475 case scTruncate:
14476 case scZeroExtend:
14477 case scSignExtend:
14478 case scPtrToAddr:
14479 case scAddExpr:
14480 case scMulExpr:
14481 case scUDivExpr:
14482 case scUMaxExpr:
14483 case scSMaxExpr:
14484 case scUMinExpr:
14485 case scSMinExpr:
14486 case scSequentialUMinExpr: {
14487 bool HasVarying = false;
14488 bool HasUniform = false;
14489 for (SCEVUse Op : S->operands()) {
14490 LoopDisposition D = getLoopDisposition(S: Op, L);
14491 if (D == LoopVariant)
14492 return LoopVariant;
14493 if (D == LoopComputable)
14494 HasVarying = true;
14495 if (D == LoopUniform)
14496 HasUniform = true;
14497 }
14498 return HasVarying ? (HasUniform ? LoopVariant : LoopComputable)
14499 : (HasUniform ? LoopUniform : LoopInvariant);
14500 }
14501 case scUnknown:
14502 // All non-instruction values are loop invariant. All instructions are loop
14503 // invariant if they are not contained in the specified loop.
14504 // Instructions are never considered invariant in the function body
14505 // (null loop) because they are defined within the "loop".
14506 if (auto *I = dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue()))
14507 return (L && !L->contains(Inst: I)) ? LoopInvariant : LoopVariant;
14508 return LoopInvariant;
14509 case scCouldNotCompute:
14510 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14511 }
14512 llvm_unreachable("Unknown SCEV kind!");
14513}
14514
14515bool ScalarEvolution::isLoopUniform(const SCEV *S, const Loop *L) {
14516 LoopDisposition D = getLoopDisposition(S, L);
14517 return D == LoopUniform || D == LoopInvariant;
14518}
14519
14520bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) {
14521 return getLoopDisposition(S, L) == LoopInvariant;
14522}
14523
14524bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) {
14525 return getLoopDisposition(S, L) == LoopComputable;
14526}
14527
14528ScalarEvolution::BlockDisposition
14529ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14530 auto &Values = BlockDispositions[S];
14531 for (auto &V : Values) {
14532 if (V.getPointer() == BB)
14533 return V.getInt();
14534 }
14535 Values.emplace_back(Args&: BB, Args: DoesNotDominateBlock);
14536 BlockDisposition D = computeBlockDisposition(S, BB);
14537 auto &Values2 = BlockDispositions[S];
14538 for (auto &V : llvm::reverse(C&: Values2)) {
14539 if (V.getPointer() == BB) {
14540 V.setInt(D);
14541 break;
14542 }
14543 }
14544 return D;
14545}
14546
14547ScalarEvolution::BlockDisposition
14548ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14549 switch (S->getSCEVType()) {
14550 case scConstant:
14551 case scVScale:
14552 return ProperlyDominatesBlock;
14553 case scAddRecExpr: {
14554 // This uses a "dominates" query instead of "properly dominates" query
14555 // to test for proper dominance too, because the instruction which
14556 // produces the addrec's value is a PHI, and a PHI effectively properly
14557 // dominates its entire containing block.
14558 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14559 if (!DT.dominates(A: AR->getLoop()->getHeader(), B: BB))
14560 return DoesNotDominateBlock;
14561
14562 // Fall through into SCEVNAryExpr handling.
14563 [[fallthrough]];
14564 }
14565 case scTruncate:
14566 case scZeroExtend:
14567 case scSignExtend:
14568 case scPtrToAddr:
14569 case scAddExpr:
14570 case scMulExpr:
14571 case scUDivExpr:
14572 case scUMaxExpr:
14573 case scSMaxExpr:
14574 case scUMinExpr:
14575 case scSMinExpr:
14576 case scSequentialUMinExpr: {
14577 bool Proper = true;
14578 for (const SCEV *NAryOp : S->operands()) {
14579 BlockDisposition D = getBlockDisposition(S: NAryOp, BB);
14580 if (D == DoesNotDominateBlock)
14581 return DoesNotDominateBlock;
14582 if (D == DominatesBlock)
14583 Proper = false;
14584 }
14585 return Proper ? ProperlyDominatesBlock : DominatesBlock;
14586 }
14587 case scUnknown:
14588 if (Instruction *I =
14589 dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue())) {
14590 if (I->getParent() == BB)
14591 return DominatesBlock;
14592 if (DT.properlyDominates(A: I->getParent(), B: BB))
14593 return ProperlyDominatesBlock;
14594 return DoesNotDominateBlock;
14595 }
14596 return ProperlyDominatesBlock;
14597 case scCouldNotCompute:
14598 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14599 }
14600 llvm_unreachable("Unknown SCEV kind!");
14601}
14602
14603bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
14604 return getBlockDisposition(S, BB) >= DominatesBlock;
14605}
14606
14607bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) {
14608 return getBlockDisposition(S, BB) == ProperlyDominatesBlock;
14609}
14610
14611void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
14612 bool Predicated) {
14613 auto &BECounts =
14614 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14615 auto It = BECounts.find(Val: L);
14616 if (It != BECounts.end()) {
14617 for (const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
14618 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14619 if (!isa<SCEVConstant>(Val: S)) {
14620 auto UserIt = BECountUsers.find(Val: S);
14621 assert(UserIt != BECountUsers.end());
14622 UserIt->second.erase(Ptr: {L, Predicated});
14623 }
14624 }
14625 }
14626 BECounts.erase(I: It);
14627 }
14628}
14629
14630void ScalarEvolution::forgetMemoizedResults(ArrayRef<SCEVUse> SCEVs) {
14631 SmallPtrSet<const SCEV *, 8> ToForget(llvm::from_range, SCEVs);
14632 SmallVector<SCEVUse, 8> Worklist(ToForget.begin(), ToForget.end());
14633
14634 while (!Worklist.empty()) {
14635 const SCEV *Curr = Worklist.pop_back_val();
14636 auto Users = SCEVUsers.find(Val: Curr);
14637 if (Users != SCEVUsers.end())
14638 for (const auto *User : Users->second)
14639 if (ToForget.insert(Ptr: User).second)
14640 Worklist.push_back(Elt: User);
14641 }
14642
14643 for (const auto *S : ToForget)
14644 forgetMemoizedResultsImpl(S);
14645
14646 PredicatedSCEVRewrites.remove_if(
14647 Pred: [&](const auto &Entry) { return ToForget.count(Ptr: Entry.first.first); });
14648}
14649
14650void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) {
14651 LoopDispositions.erase(Val: S);
14652 BlockDispositions.erase(Val: S);
14653 UnsignedRanges.erase(Val: S);
14654 SignedRanges.erase(Val: S);
14655 HasRecMap.erase(Val: S);
14656 ConstantMultipleCache.erase(Val: S);
14657
14658 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
14659 UnsignedWrapViaInductionTried.erase(Ptr: AR);
14660 SignedWrapViaInductionTried.erase(Ptr: AR);
14661 }
14662
14663 auto ExprIt = ExprValueMap.find(Val: S);
14664 if (ExprIt != ExprValueMap.end()) {
14665 for (Value *V : ExprIt->second) {
14666 auto ValueIt = ValueExprMap.find_as(Val: V);
14667 if (ValueIt != ValueExprMap.end())
14668 ValueExprMap.erase(I: ValueIt);
14669 }
14670 ExprValueMap.erase(I: ExprIt);
14671 }
14672
14673 auto ScopeIt = ValuesAtScopes.find(Val: S);
14674 if (ScopeIt != ValuesAtScopes.end()) {
14675 for (const auto &Pair : ScopeIt->second)
14676 if (!isa_and_nonnull<SCEVConstant>(Val: Pair.second))
14677 llvm::erase(C&: ValuesAtScopesUsers[Pair.second.getPointer()],
14678 V: std::make_pair(x: Pair.first, y&: S));
14679 ValuesAtScopes.erase(I: ScopeIt);
14680 }
14681
14682 auto ScopeUserIt = ValuesAtScopesUsers.find(Val: S);
14683 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
14684 for (const auto &Pair : ScopeUserIt->second)
14685 // The recorded value at scope is a use of S, which may carry no-wrap
14686 // flags that are not part of this key.
14687 llvm::erase_if(C&: ValuesAtScopes[Pair.second], P: [&](const auto &LS) {
14688 return LS.first == Pair.first && LS.second.getPointer() == S;
14689 });
14690 ValuesAtScopesUsers.erase(I: ScopeUserIt);
14691 }
14692
14693 auto BEUsersIt = BECountUsers.find(Val: S);
14694 if (BEUsersIt != BECountUsers.end()) {
14695 // Work on a copy, as forgetBackedgeTakenCounts() will modify the original.
14696 auto Copy = BEUsersIt->second;
14697 for (const auto &Pair : Copy)
14698 forgetBackedgeTakenCounts(L: Pair.getPointer(), Predicated: Pair.getInt());
14699 BECountUsers.erase(I: BEUsersIt);
14700 }
14701
14702 auto FoldUser = FoldCacheUser.find(Val: S);
14703 if (FoldUser != FoldCacheUser.end())
14704 for (auto &KV : FoldUser->second)
14705 FoldCache.erase(Val: KV);
14706 FoldCacheUser.erase(Val: S);
14707}
14708
14709void
14710ScalarEvolution::getUsedLoops(const SCEV *S,
14711 SmallPtrSetImpl<const Loop *> &LoopsUsed) {
14712 struct FindUsedLoops {
14713 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
14714 : LoopsUsed(LoopsUsed) {}
14715 SmallPtrSetImpl<const Loop *> &LoopsUsed;
14716 bool follow(const SCEV *S) {
14717 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S))
14718 LoopsUsed.insert(Ptr: AR->getLoop());
14719 return true;
14720 }
14721
14722 bool isDone() const { return false; }
14723 };
14724
14725 FindUsedLoops F(LoopsUsed);
14726 SCEVTraversal<FindUsedLoops>(F).visitAll(Root: S);
14727}
14728
14729void ScalarEvolution::getReachableBlocks(
14730 SmallPtrSetImpl<BasicBlock *> &Reachable, Function &F) {
14731 SmallVector<BasicBlock *> Worklist;
14732 Worklist.push_back(Elt: &F.getEntryBlock());
14733 while (!Worklist.empty()) {
14734 BasicBlock *BB = Worklist.pop_back_val();
14735 if (!Reachable.insert(Ptr: BB).second)
14736 continue;
14737
14738 Value *Cond;
14739 BasicBlock *TrueBB, *FalseBB;
14740 if (match(V: BB->getTerminator(), P: m_Br(C: m_Value(V&: Cond), T: m_BasicBlock(V&: TrueBB),
14741 F: m_BasicBlock(V&: FalseBB)))) {
14742 if (auto *C = dyn_cast<ConstantInt>(Val: Cond)) {
14743 Worklist.push_back(Elt: C->isOne() ? TrueBB : FalseBB);
14744 continue;
14745 }
14746
14747 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
14748 const SCEV *L = getSCEV(V: Cmp->getOperand(i_nocapture: 0));
14749 const SCEV *R = getSCEV(V: Cmp->getOperand(i_nocapture: 1));
14750 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getCmpPredicate(), LHS: L, RHS: R)) {
14751 Worklist.push_back(Elt: TrueBB);
14752 continue;
14753 }
14754 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getInverseCmpPredicate(), LHS: L,
14755 RHS: R)) {
14756 Worklist.push_back(Elt: FalseBB);
14757 continue;
14758 }
14759 }
14760 }
14761
14762 append_range(C&: Worklist, R: successors(BB));
14763 }
14764}
14765
14766void ScalarEvolution::verify() const {
14767 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14768 ScalarEvolution SE2(F, TLI, AC, DT, LI);
14769
14770 SmallVector<Loop *, 8> LoopStack(LI.begin(), LI.end());
14771
14772 // Map's SCEV expressions from one ScalarEvolution "universe" to another.
14773 struct SCEVMapper : public SCEVRewriteVisitor<SCEVMapper> {
14774 SCEVMapper(ScalarEvolution &SE) : SCEVRewriteVisitor<SCEVMapper>(SE) {}
14775
14776 const SCEV *visitConstant(const SCEVConstant *Constant) {
14777 return SE.getConstant(Val: Constant->getAPInt());
14778 }
14779
14780 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14781 return SE.getUnknown(V: Expr->getValue());
14782 }
14783
14784 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
14785 return SE.getCouldNotCompute();
14786 }
14787 };
14788
14789 SCEVMapper SCM(SE2);
14790 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
14791 SE2.getReachableBlocks(Reachable&: ReachableBlocks, F);
14792
14793 auto GetDelta = [&](const SCEV *Old, const SCEV *New) -> const SCEV * {
14794 if (containsUndefs(S: Old) || containsUndefs(S: New)) {
14795 // SCEV treats "undef" as an unknown but consistent value (i.e. it does
14796 // not propagate undef aggressively). This means we can (and do) fail
14797 // verification in cases where a transform makes a value go from "undef"
14798 // to "undef+1" (say). The transform is fine, since in both cases the
14799 // result is "undef", but SCEV thinks the value increased by 1.
14800 return nullptr;
14801 }
14802
14803 // Unless VerifySCEVStrict is set, we only compare constant deltas.
14804 const SCEV *Delta = SE2.getMinusSCEV(LHS: Old, RHS: New);
14805 if (!VerifySCEVStrict && !isa<SCEVConstant>(Val: Delta))
14806 return nullptr;
14807
14808 return Delta;
14809 };
14810
14811 while (!LoopStack.empty()) {
14812 auto *L = LoopStack.pop_back_val();
14813 llvm::append_range(C&: LoopStack, R&: *L);
14814
14815 // Only verify BECounts in reachable loops. For an unreachable loop,
14816 // any BECount is legal.
14817 if (!ReachableBlocks.contains(Ptr: L->getHeader()))
14818 continue;
14819
14820 // Only verify cached BECounts. Computing new BECounts may change the
14821 // results of subsequent SCEV uses.
14822 auto It = BackedgeTakenCounts.find(Val: L);
14823 if (It == BackedgeTakenCounts.end())
14824 continue;
14825
14826 auto *CurBECount =
14827 SCM.visit(S: It->second.getExact(L, SE: const_cast<ScalarEvolution *>(this)));
14828 auto *NewBECount = SE2.getBackedgeTakenCount(L);
14829
14830 if (CurBECount == SE2.getCouldNotCompute() ||
14831 NewBECount == SE2.getCouldNotCompute()) {
14832 // NB! This situation is legal, but is very suspicious -- whatever pass
14833 // change the loop to make a trip count go from could not compute to
14834 // computable or vice-versa *should have* invalidated SCEV. However, we
14835 // choose not to assert here (for now) since we don't want false
14836 // positives.
14837 continue;
14838 }
14839
14840 if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) >
14841 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14842 NewBECount = SE2.getZeroExtendExpr(Op: NewBECount, Ty: CurBECount->getType());
14843 else if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) <
14844 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14845 CurBECount = SE2.getZeroExtendExpr(Op: CurBECount, Ty: NewBECount->getType());
14846
14847 const SCEV *Delta = GetDelta(CurBECount, NewBECount);
14848 if (Delta && !Delta->isZero()) {
14849 dbgs() << "Trip Count for " << *L << " Changed!\n";
14850 dbgs() << "Old: " << *CurBECount << "\n";
14851 dbgs() << "New: " << *NewBECount << "\n";
14852 dbgs() << "Delta: " << *Delta << "\n";
14853 std::abort();
14854 }
14855 }
14856
14857 // Collect all valid loops currently in LoopInfo.
14858 SmallPtrSet<Loop *, 32> ValidLoops;
14859 SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end());
14860 while (!Worklist.empty()) {
14861 Loop *L = Worklist.pop_back_val();
14862 if (ValidLoops.insert(Ptr: L).second)
14863 Worklist.append(in_start: L->begin(), in_end: L->end());
14864 }
14865 for (const auto &KV : ValueExprMap) {
14866#ifndef NDEBUG
14867 // Check for SCEV expressions referencing invalid/deleted loops.
14868 if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
14869 assert(ValidLoops.contains(AR->getLoop()) &&
14870 "AddRec references invalid loop");
14871 }
14872#endif
14873
14874 // Check that the value is also part of the reverse map.
14875 auto It = ExprValueMap.find(Val: KV.second);
14876 if (It == ExprValueMap.end() || !It->second.contains(key: KV.first)) {
14877 dbgs() << "Value " << *KV.first
14878 << " is in ValueExprMap but not in ExprValueMap\n";
14879 std::abort();
14880 }
14881
14882 if (auto *I = dyn_cast<Instruction>(Val: &*KV.first)) {
14883 if (!ReachableBlocks.contains(Ptr: I->getParent()))
14884 continue;
14885 const SCEV *OldSCEV = SCM.visit(S: KV.second);
14886 const SCEV *NewSCEV = SE2.getSCEV(V: I);
14887 const SCEV *Delta = GetDelta(OldSCEV, NewSCEV);
14888 if (Delta && !Delta->isZero()) {
14889 dbgs() << "SCEV for value " << *I << " changed!\n"
14890 << "Old: " << *OldSCEV << "\n"
14891 << "New: " << *NewSCEV << "\n"
14892 << "Delta: " << *Delta << "\n";
14893 std::abort();
14894 }
14895 }
14896 }
14897
14898 for (const auto &KV : ExprValueMap) {
14899 for (Value *V : KV.second) {
14900 const SCEV *S = ValueExprMap.lookup(Val: V);
14901 if (!S) {
14902 dbgs() << "Value " << *V
14903 << " is in ExprValueMap but not in ValueExprMap\n";
14904 std::abort();
14905 }
14906 if (S != KV.first) {
14907 dbgs() << "Value " << *V << " mapped to " << *S << " rather than "
14908 << *KV.first << "\n";
14909 std::abort();
14910 }
14911 }
14912 }
14913
14914 // Verify integrity of SCEV users.
14915 for (const auto &S : UniqueSCEVs) {
14916 for (SCEVUse Op : S.operands()) {
14917 // We do not store dependencies of constants.
14918 if (isa<SCEVConstant>(Val: Op))
14919 continue;
14920 auto It = SCEVUsers.find(Val: Op);
14921 if (It != SCEVUsers.end() && It->second.count(Ptr: &S))
14922 continue;
14923 dbgs() << "Use of operand " << *Op << " by user " << S
14924 << " is not being tracked!\n";
14925 std::abort();
14926 }
14927 }
14928
14929 // Verify integrity of ValuesAtScopes users.
14930 for (const auto &ValueAndVec : ValuesAtScopes) {
14931 const SCEV *Value = ValueAndVec.first;
14932 for (const auto &LoopAndValueAtScope : ValueAndVec.second) {
14933 const Loop *L = LoopAndValueAtScope.first;
14934 SCEVUse ValueAtScope = LoopAndValueAtScope.second;
14935 if (!isa<SCEVConstant>(Val: ValueAtScope)) {
14936 auto It = ValuesAtScopesUsers.find(Val: ValueAtScope.getPointer());
14937 if (It != ValuesAtScopesUsers.end() &&
14938 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: Value)))
14939 continue;
14940 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14941 << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
14942 std::abort();
14943 }
14944 }
14945 }
14946
14947 for (const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
14948 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
14949 for (const auto &LoopAndValue : ValueAtScopeAndVec.second) {
14950 const Loop *L = LoopAndValue.first;
14951 const SCEV *Value = LoopAndValue.second;
14952 assert(!isa<SCEVConstant>(Value));
14953 auto It = ValuesAtScopes.find(Val: Value);
14954 // The recorded value at scope may carry no-wrap flags that are not part
14955 // of the key it is recorded under.
14956 if (It != ValuesAtScopes.end() && any_of(Range: It->second, P: [&](const auto &LS) {
14957 return LS.first == L && LS.second.getPointer() == ValueAtScope;
14958 }))
14959 continue;
14960 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14961 << *ValueAtScope << " missing in ValuesAtScopes\n";
14962 std::abort();
14963 }
14964 }
14965
14966 // Verify integrity of BECountUsers.
14967 auto VerifyBECountUsers = [&](bool Predicated) {
14968 auto &BECounts =
14969 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14970 for (const auto &LoopAndBEInfo : BECounts) {
14971 for (const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
14972 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14973 if (!isa<SCEVConstant>(Val: S)) {
14974 auto UserIt = BECountUsers.find(Val: S);
14975 if (UserIt != BECountUsers.end() &&
14976 UserIt->second.contains(Ptr: { LoopAndBEInfo.first, Predicated }))
14977 continue;
14978 dbgs() << "Value " << *S << " for loop " << *LoopAndBEInfo.first
14979 << " missing from BECountUsers\n";
14980 std::abort();
14981 }
14982 }
14983 }
14984 }
14985 };
14986 VerifyBECountUsers(/* Predicated */ false);
14987 VerifyBECountUsers(/* Predicated */ true);
14988
14989 // Verify intergity of loop disposition cache.
14990 for (auto &[S, Values] : LoopDispositions) {
14991 for (auto [Loop, CachedDisposition] : Values) {
14992 const auto RecomputedDisposition = SE2.getLoopDisposition(S, L: Loop);
14993 if (CachedDisposition != RecomputedDisposition) {
14994 dbgs() << "Cached disposition of " << *S << " for loop " << *Loop
14995 << " is incorrect: cached " << CachedDisposition << ", actual "
14996 << RecomputedDisposition << "\n";
14997 std::abort();
14998 }
14999 }
15000 }
15001
15002 // Verify integrity of the block disposition cache.
15003 for (auto &[S, Values] : BlockDispositions) {
15004 for (auto [BB, CachedDisposition] : Values) {
15005 const auto RecomputedDisposition = SE2.getBlockDisposition(S, BB);
15006 if (CachedDisposition != RecomputedDisposition) {
15007 dbgs() << "Cached disposition of " << *S << " for block %"
15008 << BB->getName() << " is incorrect: cached " << CachedDisposition
15009 << ", actual " << RecomputedDisposition << "\n";
15010 std::abort();
15011 }
15012 }
15013 }
15014
15015 // Verify FoldCache/FoldCacheUser caches.
15016 for (auto [FoldID, Expr] : FoldCache) {
15017 auto I = FoldCacheUser.find(Val: Expr);
15018 if (I == FoldCacheUser.end()) {
15019 dbgs() << "Missing entry in FoldCacheUser for cached expression " << *Expr
15020 << "!\n";
15021 std::abort();
15022 }
15023 if (!is_contained(Range: I->second, Element: FoldID)) {
15024 dbgs() << "Missing FoldID in cached users of " << *Expr << "!\n";
15025 std::abort();
15026 }
15027 }
15028 for (auto [Expr, IDs] : FoldCacheUser) {
15029 for (auto &FoldID : IDs) {
15030 const SCEV *S = FoldCache.lookup(Val: FoldID);
15031 if (!S) {
15032 dbgs() << "Missing entry in FoldCache for expression " << *Expr
15033 << "!\n";
15034 std::abort();
15035 }
15036 if (S != Expr) {
15037 dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: " << *S
15038 << " != " << *Expr << "!\n";
15039 std::abort();
15040 }
15041 }
15042 }
15043
15044 // Verify that ConstantMultipleCache computations are correct. We check that
15045 // cached multiples and recomputed multiples are multiples of each other to
15046 // verify correctness. It is possible that a recomputed multiple is different
15047 // from the cached multiple due to strengthened no wrap flags or changes in
15048 // KnownBits computations.
15049 for (auto [S, Multiple] : ConstantMultipleCache) {
15050 APInt RecomputedMultiple = SE2.getConstantMultiple(S);
15051 if ((Multiple != 0 && RecomputedMultiple != 0 &&
15052 Multiple.urem(RHS: RecomputedMultiple) != 0 &&
15053 RecomputedMultiple.urem(RHS: Multiple) != 0)) {
15054 dbgs() << "Incorrect cached computation in ConstantMultipleCache for "
15055 << *S << " : Computed " << RecomputedMultiple
15056 << " but cache contains " << Multiple << "!\n";
15057 std::abort();
15058 }
15059 }
15060}
15061
15062bool ScalarEvolution::invalidate(
15063 Function &F, const PreservedAnalyses &PA,
15064 FunctionAnalysisManager::Invalidator &Inv) {
15065 // Invalidate the ScalarEvolution object whenever it isn't preserved or one
15066 // of its dependencies is invalidated.
15067 auto PAC = PA.getChecker<ScalarEvolutionAnalysis>();
15068 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) ||
15069 Inv.invalidate<AssumptionAnalysis>(IR&: F, PA) ||
15070 Inv.invalidate<DominatorTreeAnalysis>(IR&: F, PA) ||
15071 Inv.invalidate<LoopAnalysis>(IR&: F, PA);
15072}
15073
15074AnalysisKey ScalarEvolutionAnalysis::Key;
15075
15076ScalarEvolution ScalarEvolutionAnalysis::run(Function &F,
15077 FunctionAnalysisManager &AM) {
15078 auto &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
15079 auto &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
15080 auto &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
15081 auto &LI = AM.getResult<LoopAnalysis>(IR&: F);
15082 return ScalarEvolution(F, TLI, AC, DT, LI);
15083}
15084
15085PreservedAnalyses
15086ScalarEvolutionVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
15087 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).verify();
15088 return PreservedAnalyses::all();
15089}
15090
15091PreservedAnalyses
15092ScalarEvolutionPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
15093 // For compatibility with opt's -analyze feature under legacy pass manager
15094 // which was not ported to NPM. This keeps tests using
15095 // update_analyze_test_checks.py working.
15096 OS << "Printing analysis 'Scalar Evolution Analysis' for function '"
15097 << F.getName() << "':\n";
15098 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).print(OS);
15099 return PreservedAnalyses::all();
15100}
15101
15102INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass, "scalar-evolution",
15103 "Scalar Evolution Analysis", false, true)
15104INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
15105INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
15106INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
15107INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
15108INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution",
15109 "Scalar Evolution Analysis", false, true)
15110
15111char ScalarEvolutionWrapperPass::ID = 0;
15112
15113ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) {}
15114
15115bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) {
15116 SE.reset(p: new ScalarEvolution(
15117 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F),
15118 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
15119 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
15120 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
15121 return false;
15122}
15123
15124void ScalarEvolutionWrapperPass::releaseMemory() { SE.reset(); }
15125
15126void ScalarEvolutionWrapperPass::print(raw_ostream &OS, const Module *) const {
15127 SE->print(OS);
15128}
15129
15130void ScalarEvolutionWrapperPass::verifyAnalysis() const {
15131 if (!VerifySCEV)
15132 return;
15133
15134 SE->verify();
15135}
15136
15137void ScalarEvolutionWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
15138 AU.setPreservesAll();
15139 AU.addRequiredTransitive<AssumptionCacheTracker>();
15140 AU.addRequiredTransitive<LoopInfoWrapperPass>();
15141 AU.addRequiredTransitive<DominatorTreeWrapperPass>();
15142 AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>();
15143}
15144
15145const SCEVPredicate *ScalarEvolution::getEqualPredicate(const SCEV *LHS,
15146 const SCEV *RHS) {
15147 return getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS, RHS);
15148}
15149
15150const SCEVPredicate *
15151ScalarEvolution::getComparePredicate(const ICmpInst::Predicate Pred,
15152 const SCEV *LHS, const SCEV *RHS) {
15153 FoldingSetNodeID ID;
15154 assert(LHS->getType() == RHS->getType() &&
15155 "Type mismatch between LHS and RHS");
15156 // Unique this node based on the arguments
15157 ID.AddInteger(I: SCEVPredicate::P_Compare);
15158 ID.AddInteger(I: Pred);
15159 ID.AddPointer(Ptr: LHS);
15160 ID.AddPointer(Ptr: RHS);
15161 FoldingSetInsertToken Token;
15162 if (const auto *S = UniquePreds.lookup(ID, Token))
15163 return S;
15164 SCEVComparePredicate *Eq = new (SCEVAllocator)
15165 SCEVComparePredicate(ID.Intern(Allocator&: SCEVAllocator), Pred, LHS, RHS);
15166 UniquePreds.insert(N: Eq, Token);
15167 return Eq;
15168}
15169
15170const SCEVPredicate *ScalarEvolution::getWrapPredicate(
15171 const SCEVAddRecExpr *AR,
15172 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15173 FoldingSetNodeID ID;
15174 // Unique this node based on the arguments
15175 ID.AddInteger(I: SCEVPredicate::P_Wrap);
15176 ID.AddPointer(Ptr: AR);
15177 ID.AddInteger(I: AddedFlags);
15178 FoldingSetInsertToken Token;
15179 if (const auto *S = UniquePreds.lookup(ID, Token))
15180 return S;
15181 auto *OF = new (SCEVAllocator)
15182 SCEVWrapPredicate(ID.Intern(Allocator&: SCEVAllocator), AR, AddedFlags);
15183 UniquePreds.insert(N: OF, Token);
15184 return OF;
15185}
15186
15187namespace {
15188
15189class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
15190public:
15191
15192 /// Rewrites \p S in the context of a loop L and the SCEV predication
15193 /// infrastructure.
15194 ///
15195 /// If \p Pred is non-null, the SCEV expression is rewritten to respect the
15196 /// equivalences present in \p Pred.
15197 ///
15198 /// If \p NewPreds is non-null, rewrite is free to add further predicates to
15199 /// \p NewPreds such that the result will be an AddRecExpr.
15200 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
15201 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15202 const SCEVPredicate *Pred) {
15203 SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred);
15204 return Rewriter.visit(S);
15205 }
15206
15207 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
15208 if (Pred) {
15209 if (auto *U = dyn_cast<SCEVUnionPredicate>(Val: Pred)) {
15210 for (const auto *Pred : U->getPredicates())
15211 if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred))
15212 if (IPred->getLHS() == Expr &&
15213 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15214 return IPred->getRHS();
15215 } else if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred)) {
15216 if (IPred->getLHS() == Expr &&
15217 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15218 return IPred->getRHS();
15219 }
15220 }
15221 return convertToAddRecWithPreds(Expr);
15222 }
15223
15224 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
15225 const SCEV *Operand = visit(S: Expr->getOperand());
15226 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15227 if (AR && AR->getLoop() == L && AR->isAffine()) {
15228 // This couldn't be folded because the operand didn't have the nuw
15229 // flag. Add the nusw flag as an assumption that we could make.
15230 const SCEV *Step = AR->getStepRecurrence(SE);
15231 Type *Ty = Expr->getType();
15232 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNUSW))
15233 return SE.getAddRecExpr(Start: SE.getZeroExtendExpr(Op: AR->getStart(), Ty),
15234 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15235 Flags: AR->getNoWrapFlags());
15236 }
15237 return SE.getZeroExtendExpr(Op: Operand, Ty: Expr->getType());
15238 }
15239
15240 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
15241 const SCEV *Operand = visit(S: Expr->getOperand());
15242 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15243 if (AR && AR->getLoop() == L && AR->isAffine()) {
15244 // This couldn't be folded because the operand didn't have the nsw
15245 // flag. Add the nssw flag as an assumption that we could make.
15246 const SCEV *Step = AR->getStepRecurrence(SE);
15247 Type *Ty = Expr->getType();
15248 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNSSW))
15249 return SE.getAddRecExpr(Start: SE.getSignExtendExpr(Op: AR->getStart(), Ty),
15250 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15251 Flags: AR->getNoWrapFlags());
15252 }
15253 return SE.getSignExtendExpr(Op: Operand, Ty: Expr->getType());
15254 }
15255
15256private:
15257 explicit SCEVPredicateRewriter(
15258 const Loop *L, ScalarEvolution &SE,
15259 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15260 const SCEVPredicate *Pred)
15261 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {}
15262
15263 bool addOverflowAssumption(const SCEVPredicate *P) {
15264 if (!NewPreds) {
15265 // Check if we've already made this assumption.
15266 return Pred && Pred->implies(N: P, SE);
15267 }
15268 NewPreds->push_back(Elt: P);
15269 return true;
15270 }
15271
15272 bool addOverflowAssumption(const SCEVAddRecExpr *AR,
15273 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15274 auto *A = SE.getWrapPredicate(AR, AddedFlags);
15275 return addOverflowAssumption(P: A);
15276 }
15277
15278 // If \p Expr represents a PHINode, we try to see if it can be represented
15279 // as an AddRec, possibly under a predicate (PHISCEVPred). If it is possible
15280 // to add this predicate as a runtime overflow check, we return the AddRec.
15281 // If \p Expr does not meet these conditions (is not a PHI node, or we
15282 // couldn't create an AddRec for it, or couldn't add the predicate), we just
15283 // return \p Expr.
15284 const SCEV *convertToAddRecWithPreds(const SCEVUnknown *Expr) {
15285 if (!isa<PHINode>(Val: Expr->getValue()))
15286 return Expr;
15287 std::optional<
15288 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
15289 PredicatedRewrite = SE.createAddRecFromPHIWithCasts(SymbolicPHI: Expr);
15290 if (!PredicatedRewrite)
15291 return Expr;
15292 for (const auto *P : PredicatedRewrite->second){
15293 // Wrap predicates from outer loops are not supported.
15294 if (auto *WP = dyn_cast<const SCEVWrapPredicate>(Val: P)) {
15295 if (L != WP->getExpr()->getLoop())
15296 return Expr;
15297 }
15298 if (!addOverflowAssumption(P))
15299 return Expr;
15300 }
15301 return PredicatedRewrite->first;
15302 }
15303
15304 SmallVectorImpl<const SCEVPredicate *> *NewPreds;
15305 const SCEVPredicate *Pred;
15306 const Loop *L;
15307};
15308
15309} // end anonymous namespace
15310
15311const SCEV *
15312ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L,
15313 const SCEVPredicate &Preds) {
15314 return SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: nullptr, Pred: &Preds);
15315}
15316
15317const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
15318 const SCEV *S, const Loop *L,
15319 SmallVectorImpl<const SCEVPredicate *> &Preds) {
15320 SmallVector<const SCEVPredicate *> TransformPreds;
15321 S = SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: &TransformPreds, Pred: nullptr);
15322 auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S);
15323
15324 if (!AddRec)
15325 return nullptr;
15326
15327 // Check if any of the transformed predicates is known to be false. In that
15328 // case, it doesn't make sense to convert to a predicated AddRec, as the
15329 // versioned loop will never execute.
15330 for (const SCEVPredicate *Pred : TransformPreds) {
15331 auto *WrapPred = dyn_cast<SCEVWrapPredicate>(Val: Pred);
15332 if (!WrapPred || WrapPred->getFlags() != SCEVWrapPredicate::IncrementNSSW)
15333 continue;
15334
15335 const SCEVAddRecExpr *AddRecToCheck = WrapPred->getExpr();
15336 const SCEV *ExitCount = getBackedgeTakenCount(L: AddRecToCheck->getLoop());
15337 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
15338 continue;
15339
15340 const SCEV *Step = AddRecToCheck->getStepRecurrence(SE&: *this);
15341 if (!Step->isOne())
15342 continue;
15343
15344 ExitCount = getTruncateOrSignExtend(V: ExitCount, Ty: Step->getType());
15345 const SCEV *Add = getAddExpr(LHS: AddRecToCheck->getStart(), RHS: ExitCount);
15346 if (isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS: Add, RHS: AddRecToCheck->getStart()))
15347 return nullptr;
15348 }
15349
15350 // Since the transformation was successful, we can now transfer the SCEV
15351 // predicates.
15352 Preds.append(in_start: TransformPreds.begin(), in_end: TransformPreds.end());
15353
15354 return AddRec;
15355}
15356
15357/// SCEV predicates
15358SCEVPredicate::SCEVPredicate(const FoldingSetNodeIDRef ID,
15359 SCEVPredicateKind Kind)
15360 : FastID(ID), Kind(Kind) {}
15361
15362SCEVComparePredicate::SCEVComparePredicate(const FoldingSetNodeIDRef ID,
15363 const ICmpInst::Predicate Pred,
15364 const SCEV *LHS, const SCEV *RHS)
15365 : SCEVPredicate(ID, P_Compare), Pred(Pred), LHS(LHS), RHS(RHS) {
15366 assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match");
15367 assert(LHS != RHS && "LHS and RHS are the same SCEV");
15368}
15369
15370bool SCEVComparePredicate::implies(const SCEVPredicate *N,
15371 ScalarEvolution &SE) const {
15372 const auto *Op = dyn_cast<SCEVComparePredicate>(Val: N);
15373
15374 if (!Op)
15375 return false;
15376
15377 if (Pred != ICmpInst::ICMP_EQ)
15378 return false;
15379
15380 return Op->LHS == LHS && Op->RHS == RHS;
15381}
15382
15383bool SCEVComparePredicate::isAlwaysTrue() const { return false; }
15384
15385void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
15386 if (Pred == ICmpInst::ICMP_EQ)
15387 OS.indent(NumSpaces: Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n";
15388 else
15389 OS.indent(NumSpaces: Depth) << "Compare predicate: " << *LHS << " " << Pred << ") "
15390 << *RHS << "\n";
15391
15392}
15393
15394SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
15395 const SCEVAddRecExpr *AR,
15396 IncrementWrapFlags Flags)
15397 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {}
15398
15399const SCEVAddRecExpr *SCEVWrapPredicate::getExpr() const { return AR; }
15400
15401bool SCEVWrapPredicate::implies(const SCEVPredicate *N,
15402 ScalarEvolution &SE) const {
15403 const auto *Op = dyn_cast<SCEVWrapPredicate>(Val: N);
15404 if (!Op || setFlags(Flags, OnFlags: Op->Flags) != Flags)
15405 return false;
15406
15407 if (Op->AR == AR)
15408 return true;
15409
15410 if (Flags != SCEVWrapPredicate::IncrementNSSW &&
15411 Flags != SCEVWrapPredicate::IncrementNUSW)
15412 return false;
15413
15414 const SCEV *Start = AR->getStart();
15415 const SCEV *OpStart = Op->AR->getStart();
15416 if (Start->getType()->isPointerTy() != OpStart->getType()->isPointerTy())
15417 return false;
15418
15419 // Reject pointers to different address spaces.
15420 if (Start->getType()->isPointerTy() && Start->getType() != OpStart->getType())
15421 return false;
15422
15423 // NUSW/NSSW on a wider-type AddRec does not imply the same on a
15424 // narrower-type AddRec.
15425 if (SE.getTypeSizeInBits(Ty: AR->getType()) >
15426 SE.getTypeSizeInBits(Ty: Op->AR->getType()))
15427 return false;
15428
15429 const SCEV *Step = AR->getStepRecurrence(SE);
15430 const SCEV *OpStep = Op->AR->getStepRecurrence(SE);
15431 if (!SE.isKnownPositive(S: Step) || !SE.isKnownPositive(S: OpStep))
15432 return false;
15433
15434 // If both steps are positive, this implies N, if N's start and step are
15435 // ULE/SLE (for NSUW/NSSW) than this'.
15436 Type *WiderTy = SE.getWiderType(T1: Step->getType(), T2: OpStep->getType());
15437 Step = SE.getNoopOrZeroExtend(V: Step, Ty: WiderTy);
15438 OpStep = SE.getNoopOrZeroExtend(V: OpStep, Ty: WiderTy);
15439
15440 bool IsNUW = Flags == SCEVWrapPredicate::IncrementNUSW;
15441 OpStart = IsNUW ? SE.getNoopOrZeroExtend(V: OpStart, Ty: WiderTy)
15442 : SE.getNoopOrSignExtend(V: OpStart, Ty: WiderTy);
15443 Start = IsNUW ? SE.getNoopOrZeroExtend(V: Start, Ty: WiderTy)
15444 : SE.getNoopOrSignExtend(V: Start, Ty: WiderTy);
15445 CmpInst::Predicate Pred = IsNUW ? CmpInst::ICMP_ULE : CmpInst::ICMP_SLE;
15446 return SE.isKnownPredicate(Pred, LHS: OpStep, RHS: Step) &&
15447 SE.isKnownPredicate(Pred, LHS: OpStart, RHS: Start);
15448}
15449
15450bool SCEVWrapPredicate::isAlwaysTrue() const {
15451 SCEVFlags ScevFlags = AR->getNoWrapFlags();
15452 IncrementWrapFlags IFlags = Flags;
15453
15454 if (ScalarEvolution::setFlags(Flags: ScevFlags, OnFlags: SCEV::FlagNSW) == ScevFlags)
15455 IFlags = clearFlags(Flags: IFlags, OffFlags: IncrementNSSW);
15456
15457 return IFlags == IncrementAnyWrap;
15458}
15459
15460void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const {
15461 OS.indent(NumSpaces: Depth) << *getExpr() << " Added Flags: ";
15462 if (SCEVWrapPredicate::IncrementNUSW & getFlags())
15463 OS << "<nusw>";
15464 if (SCEVWrapPredicate::IncrementNSSW & getFlags())
15465 OS << "<nssw>";
15466 OS << "\n";
15467}
15468
15469/// Union predicates don't get cached so create a dummy set ID for it.
15470SCEVUnionPredicate::SCEVUnionPredicate(ArrayRef<const SCEVPredicate *> Preds,
15471 ScalarEvolution &SE)
15472 : SCEVPredicate(FoldingSetNodeIDRef(), P_Union) {
15473 for (const auto *P : Preds)
15474 add(N: P, SE);
15475}
15476
15477bool SCEVUnionPredicate::isAlwaysTrue() const {
15478 return all_of(Range: Preds,
15479 P: [](const SCEVPredicate *I) { return I->isAlwaysTrue(); });
15480}
15481
15482bool SCEVUnionPredicate::implies(const SCEVPredicate *N,
15483 ScalarEvolution &SE) const {
15484 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N))
15485 return all_of(Range: Set->Preds, P: [this, &SE](const SCEVPredicate *I) {
15486 return this->implies(N: I, SE);
15487 });
15488
15489 if (any_of(Range: Preds,
15490 P: [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); }))
15491 return true;
15492
15493 // A wrap predicate may be implied by a wrap predicate in Preds after applying
15494 // equal predicates.
15495 const auto *NWrap = dyn_cast<SCEVWrapPredicate>(Val: N);
15496 if (!NWrap)
15497 return false;
15498 const Loop *L = NWrap->getExpr()->getLoop();
15499 return any_of(Range: Preds, P: [&](const SCEVPredicate *I) {
15500 const auto *IWrap = dyn_cast<SCEVWrapPredicate>(Val: I);
15501 if (!IWrap)
15502 return false;
15503 const auto *RewrittenAR = dyn_cast<SCEVAddRecExpr>(
15504 Val: SE.rewriteUsingPredicate(S: IWrap->getExpr(), L, Preds: *this));
15505 return RewrittenAR &&
15506 SE.getWrapPredicate(AR: RewrittenAR, AddedFlags: IWrap->getFlags())->implies(N, SE);
15507 });
15508}
15509
15510void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const {
15511 for (const auto *Pred : Preds)
15512 Pred->print(OS, Depth);
15513}
15514
15515void SCEVUnionPredicate::add(const SCEVPredicate *N, ScalarEvolution &SE) {
15516 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N)) {
15517 for (const auto *Pred : Set->Preds)
15518 add(N: Pred, SE);
15519 return;
15520 }
15521
15522 // Implication checks are quadratic in the number of predicates. Stop doing
15523 // them if there are many predicates, as they should be too expensive to use
15524 // anyway at that point.
15525 bool CheckImplies = Preds.size() < 16;
15526
15527 // Only add predicate if it is not already implied by this union predicate.
15528 if (CheckImplies && implies(N, SE))
15529 return;
15530
15531 // Build a new vector containing the current predicates, except the ones that
15532 // are implied by the new predicate N.
15533 SmallVector<const SCEVPredicate *> PrunedPreds;
15534 for (auto *P : Preds) {
15535 if (CheckImplies && N->implies(N: P, SE))
15536 continue;
15537 PrunedPreds.push_back(Elt: P);
15538 }
15539 Preds = std::move(PrunedPreds);
15540 Preds.push_back(Elt: N);
15541}
15542
15543PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE,
15544 Loop &L)
15545 : SE(SE), L(L) {
15546 SmallVector<const SCEVPredicate*, 4> Empty;
15547 Preds = std::make_unique<SCEVUnionPredicate>(args&: Empty, args&: SE);
15548}
15549
15550void ScalarEvolution::registerUser(const SCEV *User, ArrayRef<SCEVUse> Ops) {
15551 for (const SCEV *Op : Ops)
15552 // We do not expect that forgetting cached data for SCEVConstants will ever
15553 // open any prospects for sharpening or introduce any correctness issues,
15554 // so we don't bother storing their dependencies.
15555 if (!isa<SCEVConstant>(Val: Op))
15556 SCEVUsers[Op].insert(Ptr: User);
15557}
15558
15559const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) {
15560 const SCEV *Expr = SE.getSCEV(V);
15561 return getPredicatedSCEV(Expr);
15562}
15563
15564const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
15565 RewriteEntry &Entry = RewriteMap[Expr];
15566
15567 // If we already have an entry and the version matches, return it.
15568 if (Entry.second && Generation == Entry.first)
15569 return Entry.second;
15570
15571 // We found an entry but it's stale. Rewrite the stale entry
15572 // according to the current predicate.
15573 if (Entry.second)
15574 Expr = Entry.second;
15575
15576 const SCEV *NewSCEV = SE.rewriteUsingPredicate(S: Expr, L: &L, Preds: *Preds);
15577 Entry = {Generation, NewSCEV};
15578
15579 return NewSCEV;
15580}
15581
15582const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
15583 if (!BackedgeCount) {
15584 SmallVector<const SCEVPredicate *, 4> Preds;
15585 BackedgeCount = SE.getPredicatedBackedgeTakenCount(L: &L, Preds);
15586 for (const auto *P : Preds)
15587 addPredicate(Pred: *P);
15588 }
15589 return BackedgeCount;
15590}
15591
15592const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
15593 if (!SymbolicMaxBackedgeCount) {
15594 SmallVector<const SCEVPredicate *, 4> Preds;
15595 SymbolicMaxBackedgeCount =
15596 SE.getPredicatedSymbolicMaxBackedgeTakenCount(L: &L, Preds);
15597 for (const auto *P : Preds)
15598 addPredicate(Pred: *P);
15599 }
15600 return SymbolicMaxBackedgeCount;
15601}
15602
15603unsigned PredicatedScalarEvolution::getSmallConstantMaxTripCount() {
15604 if (!SmallConstantMaxTripCount) {
15605 SmallVector<const SCEVPredicate *, 4> Preds;
15606 SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(L: &L, Predicates: &Preds);
15607 for (const auto *P : Preds)
15608 addPredicate(Pred: *P);
15609 }
15610 return *SmallConstantMaxTripCount;
15611}
15612
15613void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
15614 if (Preds->implies(N: &Pred, SE))
15615 return;
15616
15617 SmallVector<const SCEVPredicate *, 4> NewPreds(Preds->getPredicates());
15618 NewPreds.push_back(Elt: &Pred);
15619 Preds = std::make_unique<SCEVUnionPredicate>(args&: NewPreds, args&: SE);
15620 updateGeneration();
15621}
15622
15623void PredicatedScalarEvolution::addPredicates(
15624 ArrayRef<const SCEVPredicate *> Preds) {
15625 for (const SCEVPredicate *P : Preds)
15626 addPredicate(Pred: *P);
15627}
15628
15629const SCEVPredicate &PredicatedScalarEvolution::getPredicate() const {
15630 return *Preds;
15631}
15632
15633void PredicatedScalarEvolution::updateGeneration() {
15634 // If the generation number wrapped recompute everything.
15635 if (++Generation == 0) {
15636 for (auto &II : RewriteMap) {
15637 const SCEV *Rewritten = II.second.second;
15638 II.second = {Generation, SE.rewriteUsingPredicate(S: Rewritten, L: &L, Preds: *Preds)};
15639 }
15640 }
15641}
15642
15643const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(
15644 Value *V, SmallVectorImpl<const SCEVPredicate *> *ExtraPreds) {
15645 const SCEV *Expr = this->getSCEV(V);
15646 SmallVector<const SCEVPredicate *, 4> NewPreds;
15647 auto *New = SE.convertSCEVToAddRecWithPredicates(S: Expr, L: &L, Preds&: NewPreds);
15648
15649 if (!New)
15650 return nullptr;
15651
15652 if (ExtraPreds) {
15653 ExtraPreds->append(RHS: NewPreds);
15654 return New;
15655 }
15656
15657 addPredicates(Preds: NewPreds);
15658
15659 RewriteMap[SE.getSCEV(V)] = {Generation, New};
15660 return New;
15661}
15662
15663PredicatedScalarEvolution::PredicatedScalarEvolution(
15664 const PredicatedScalarEvolution &Init)
15665 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L),
15666 Preds(std::make_unique<SCEVUnionPredicate>(args: Init.Preds->getPredicates(),
15667 args&: SE)),
15668 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {}
15669
15670void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const {
15671 // For each block.
15672 for (auto *BB : L.getBlocks())
15673 for (auto &I : *BB) {
15674 if (!SE.isSCEVable(Ty: I.getType()))
15675 continue;
15676
15677 auto *Expr = SE.getSCEV(V: &I);
15678 auto II = RewriteMap.find(Val: Expr);
15679
15680 if (II == RewriteMap.end())
15681 continue;
15682
15683 // Don't print things that are not interesting.
15684 if (II->second.second == Expr)
15685 continue;
15686
15687 OS.indent(NumSpaces: Depth) << "[PSE]" << I << ":\n";
15688 OS.indent(NumSpaces: Depth + 2) << *Expr << "\n";
15689 OS.indent(NumSpaces: Depth + 2) << "--> " << *II->second.second << "\n";
15690 }
15691}
15692
15693ScalarEvolution::LoopGuards
15694ScalarEvolution::LoopGuards::collect(const Loop *L, ScalarEvolution &SE) {
15695 BasicBlock *Header = L->getHeader();
15696 BasicBlock *Pred = L->getLoopPredecessor();
15697 LoopGuards Guards(SE);
15698 if (!Pred)
15699 return Guards;
15700 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
15701 collectFromBlock(SE, Guards, Block: Header, Pred, VisitedBlocks);
15702 return Guards;
15703}
15704
15705void ScalarEvolution::LoopGuards::collectFromPHI(
15706 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15707 const PHINode &Phi, SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks,
15708 SmallDenseMap<const BasicBlock *, LoopGuards> &IncomingGuards,
15709 unsigned Depth) {
15710 if (!SE.isSCEVable(Ty: Phi.getType()))
15711 return;
15712
15713 using MinMaxPattern = std::pair<const SCEVConstant *, SCEVTypes>;
15714 auto GetMinMaxConst = [&](unsigned IncomingIdx) -> MinMaxPattern {
15715 const BasicBlock *InBlock = Phi.getIncomingBlock(i: IncomingIdx);
15716 if (!VisitedBlocks.insert(Ptr: InBlock).second)
15717 return {nullptr, scCouldNotCompute};
15718
15719 // Avoid analyzing unreachable blocks so that we don't get trapped
15720 // traversing cycles with ill-formed dominance or infinite cycles
15721 if (!SE.DT.isReachableFromEntry(A: InBlock))
15722 return {nullptr, scCouldNotCompute};
15723
15724 auto [G, Inserted] = IncomingGuards.try_emplace(Key: InBlock, Args: LoopGuards(SE));
15725 if (Inserted)
15726 collectFromBlock(SE, Guards&: G->second, Block: Phi.getParent(), Pred: InBlock, VisitedBlocks,
15727 Depth: Depth + 1);
15728 auto &RewriteMap = G->second.RewriteMap;
15729 if (RewriteMap.empty())
15730 return {nullptr, scCouldNotCompute};
15731 auto S = RewriteMap.find(Val: SE.getSCEV(V: Phi.getIncomingValue(i: IncomingIdx)));
15732 if (S == RewriteMap.end())
15733 return {nullptr, scCouldNotCompute};
15734 auto *SM = dyn_cast_if_present<SCEVMinMaxExpr>(Val: S->second);
15735 if (!SM)
15736 return {nullptr, scCouldNotCompute};
15737 if (const SCEVConstant *C0 = dyn_cast<SCEVConstant>(Val: SM->getOperand(i: 0)))
15738 return {C0, SM->getSCEVType()};
15739 return {nullptr, scCouldNotCompute};
15740 };
15741 auto MergeMinMaxConst = [](MinMaxPattern P1,
15742 MinMaxPattern P2) -> MinMaxPattern {
15743 auto [C1, T1] = P1;
15744 auto [C2, T2] = P2;
15745 if (!C1 || !C2 || T1 != T2)
15746 return {nullptr, scCouldNotCompute};
15747 switch (T1) {
15748 case scUMaxExpr:
15749 return {C1->getAPInt().ult(RHS: C2->getAPInt()) ? C1 : C2, T1};
15750 case scSMaxExpr:
15751 return {C1->getAPInt().slt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15752 case scUMinExpr:
15753 return {C1->getAPInt().ugt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15754 case scSMinExpr:
15755 return {C1->getAPInt().sgt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15756 default:
15757 llvm_unreachable("Trying to merge non-MinMaxExpr SCEVs.");
15758 }
15759 };
15760 auto P = GetMinMaxConst(0);
15761 for (unsigned int In = 1; In < Phi.getNumIncomingValues(); In++) {
15762 if (!P.first)
15763 break;
15764 P = MergeMinMaxConst(P, GetMinMaxConst(In));
15765 }
15766 if (P.first) {
15767 const SCEV *LHS = SE.getSCEV(V: const_cast<PHINode *>(&Phi));
15768 SmallVector<SCEVUse, 2> Ops({P.first, LHS});
15769 const SCEV *RHS = SE.getMinMaxExpr(Kind: P.second, Ops);
15770 Guards.RewriteMap.insert(KV: {LHS, RHS});
15771 }
15772}
15773
15774// Return a new SCEV that modifies \p Expr to the closest number divides by
15775// \p Divisor and less or equal than Expr. For now, only handle constant
15776// Expr.
15777static const SCEV *getPreviousSCEVDivisibleByDivisor(const SCEV *Expr,
15778 const APInt &DivisorVal,
15779 ScalarEvolution &SE) {
15780 const APInt *ExprVal;
15781 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15782 DivisorVal.isNonPositive())
15783 return Expr;
15784 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15785 // return the SCEV: Expr - Expr % Divisor
15786 return SE.getConstant(Val: *ExprVal - Rem);
15787}
15788
15789// Return a new SCEV that modifies \p Expr to the closest number divides by
15790// \p Divisor and greater or equal than Expr. For now, only handle constant
15791// Expr.
15792static const SCEV *getNextSCEVDivisibleByDivisor(const SCEV *Expr,
15793 const APInt &DivisorVal,
15794 ScalarEvolution &SE) {
15795 const APInt *ExprVal;
15796 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15797 DivisorVal.isNonPositive())
15798 return Expr;
15799 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15800 if (Rem.isZero())
15801 return Expr;
15802 // return the SCEV: Expr + Divisor - Expr % Divisor
15803 return SE.getConstant(Val: *ExprVal + DivisorVal - Rem);
15804}
15805
15806static bool collectDivisibilityInformation(
15807 ICmpInst::Predicate Predicate, const SCEV *LHS, const SCEV *RHS,
15808 DenseMap<const SCEV *, const SCEV *> &DivInfo,
15809 DenseMap<const SCEV *, APInt> &Multiples, ScalarEvolution &SE) {
15810 // If we have LHS == 0, check if LHS is computing a property of some unknown
15811 // SCEV %v which we can rewrite %v to express explicitly.
15812 if (Predicate != CmpInst::ICMP_EQ || !match(S: RHS, P: m_scev_Zero()))
15813 return false;
15814 // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to
15815 // explicitly express that.
15816 const SCEVUnknown *URemLHS = nullptr;
15817 const SCEV *URemRHS = nullptr;
15818 if (!match(S: LHS, P: m_scev_URem(LHS: m_SCEVUnknown(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE)))
15819 return false;
15820
15821 const SCEV *Multiple =
15822 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: URemLHS, RHS: URemRHS), RHS: URemRHS);
15823 DivInfo[URemLHS] = Multiple;
15824 if (auto *C = dyn_cast<SCEVConstant>(Val: URemRHS))
15825 Multiples[URemLHS] = C->getAPInt();
15826 return true;
15827}
15828
15829// Check if the condition is a divisibility guard (A % B == 0).
15830static bool isDivisibilityGuard(const SCEV *LHS, const SCEV *RHS,
15831 ScalarEvolution &SE) {
15832 const SCEV *X, *Y;
15833 return match(S: LHS, P: m_scev_URem(LHS: m_SCEV(V&: X), RHS: m_SCEV(V&: Y), SE)) && RHS->isZero();
15834}
15835
15836// Apply divisibility by \p Divisor on MinMaxExpr with constant values,
15837// recursively. This is done by aligning up/down the constant value to the
15838// Divisor.
15839static const SCEV *applyDivisibilityOnMinMaxExpr(const SCEV *MinMaxExpr,
15840 APInt Divisor,
15841 ScalarEvolution &SE) {
15842 // Return true if \p Expr is a MinMax SCEV expression with a non-negative
15843 // constant operand. If so, return in \p SCTy the SCEV type and in \p RHS
15844 // the non-constant operand and in \p LHS the constant operand.
15845 auto IsMinMaxSCEVWithNonNegativeConstant =
15846 [&](const SCEV *Expr, SCEVTypes &SCTy, const SCEV *&LHS,
15847 const SCEV *&RHS) {
15848 if (auto *MinMax = dyn_cast<SCEVMinMaxExpr>(Val: Expr)) {
15849 if (MinMax->getNumOperands() != 2)
15850 return false;
15851 if (auto *C = dyn_cast<SCEVConstant>(Val: MinMax->getOperand(i: 0))) {
15852 if (C->getAPInt().isNegative())
15853 return false;
15854 SCTy = MinMax->getSCEVType();
15855 LHS = MinMax->getOperand(i: 0);
15856 RHS = MinMax->getOperand(i: 1);
15857 return true;
15858 }
15859 }
15860 return false;
15861 };
15862
15863 const SCEV *MinMaxLHS = nullptr, *MinMaxRHS = nullptr;
15864 SCEVTypes SCTy;
15865 if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS,
15866 MinMaxRHS))
15867 return MinMaxExpr;
15868 auto IsMin = isa<SCEVSMinExpr>(Val: MinMaxExpr) || isa<SCEVUMinExpr>(Val: MinMaxExpr);
15869 assert(SE.isKnownNonNegative(MinMaxLHS) && "Expected non-negative operand!");
15870 auto *DivisibleExpr =
15871 IsMin ? getPreviousSCEVDivisibleByDivisor(Expr: MinMaxLHS, DivisorVal: Divisor, SE)
15872 : getNextSCEVDivisibleByDivisor(Expr: MinMaxLHS, DivisorVal: Divisor, SE);
15873 SmallVector<SCEVUse> Ops = {
15874 applyDivisibilityOnMinMaxExpr(MinMaxExpr: MinMaxRHS, Divisor, SE), DivisibleExpr};
15875 return SE.getMinMaxExpr(Kind: SCTy, Ops);
15876}
15877
15878void ScalarEvolution::LoopGuards::collectFromBlock(
15879 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15880 const BasicBlock *Block, const BasicBlock *Pred,
15881 SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks, unsigned Depth) {
15882
15883 assert(SE.DT.isReachableFromEntry(Block) && SE.DT.isReachableFromEntry(Pred));
15884
15885 SmallVector<SCEVUse> ExprsToRewrite;
15886 auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS,
15887 const SCEV *RHS,
15888 DenseMap<const SCEV *, const SCEV *> &RewriteMap,
15889 const LoopGuards &DivGuards) {
15890 // WARNING: It is generally unsound to apply any wrap flags to the proposed
15891 // replacement SCEV which isn't directly implied by the structure of that
15892 // SCEV. In particular, using contextual facts to imply flags is *NOT*
15893 // legal. See the scoping rules for flags in the header to understand why.
15894
15895 // Puts rewrite rule \p From -> \p To into the rewrite map. Also if \p From
15896 // and \p FromRewritten are the same (i.e. there has been no rewrite
15897 // registered for \p From), then puts this value in the list of rewritten
15898 // expressions.
15899 auto AddRewrite = [&](const SCEV *From, const SCEV *FromRewritten,
15900 const SCEV *To) {
15901 if (From == FromRewritten)
15902 ExprsToRewrite.push_back(Elt: From);
15903 RewriteMap[From] = To;
15904 };
15905
15906 // Checks whether \p S has already been rewritten. In that case returns the
15907 // existing rewrite because we want to chain further rewrites onto the
15908 // already rewritten value. Otherwise returns \p S.
15909 auto GetMaybeRewritten = [&](const SCEV *S) {
15910 return RewriteMap.lookup_or(Val: S, Default&: S);
15911 };
15912
15913 // Check for a condition of the form (-C1 + X < C2). InstCombine will
15914 // create this form when combining two checks of the form (X u< C2 + C1) and
15915 // (X >=u C1).
15916 auto MatchRangeCheckIdiom = [&](ICmpInst::Predicate Pred,
15917 const SCEV *MatchLHS,
15918 const SCEV *MatchRHS) {
15919 const SCEVConstant *C1;
15920 const SCEVUnknown *LHSUnknown;
15921 auto *C2 = dyn_cast<SCEVConstant>(Val: MatchRHS);
15922 if (!match(S: MatchLHS,
15923 P: m_scev_Add(Op0: m_SCEVConstant(V&: C1), Op1: m_SCEVUnknown(V&: LHSUnknown))) ||
15924 !C2)
15925 return false;
15926
15927 auto ExactRegion =
15928 ConstantRange::makeExactICmpRegion(Pred, Other: C2->getAPInt())
15929 .sub(Other: C1->getAPInt());
15930
15931 // Tighten the raw range with what we already know about LHSUnknown
15932 // from prior guards recorded in RewriteMap, or from SCEV's own range
15933 // analysis.
15934 const SCEV *RewrittenLHS = GetMaybeRewritten(LHSUnknown);
15935 ExactRegion = ExactRegion.intersectWith(CR: SE.getUnsignedRange(S: RewrittenLHS),
15936 Type: ConstantRange::Unsigned);
15937
15938 // Bail if the guard is inconsistent with prior facts, or if the range
15939 // is still not a monotonic non-wrapping interval after tightening.
15940 if (ExactRegion.isEmptySet() || ExactRegion.isWrappedSet() ||
15941 ExactRegion.isFullSet())
15942 return false;
15943
15944 const SCEV *RegionMin = SE.getConstant(Val: ExactRegion.getUnsignedMin());
15945 const SCEV *RegionMax = SE.getConstant(Val: ExactRegion.getUnsignedMax());
15946 const SCEV *ClampedLHS =
15947 SE.getUMaxExpr(LHS: RegionMin, RHS: SE.getUMinExpr(LHS: RewrittenLHS, RHS: RegionMax));
15948 AddRewrite(LHSUnknown, RewrittenLHS, ClampedLHS);
15949 return true;
15950 };
15951 if (MatchRangeCheckIdiom(Predicate, LHS, RHS))
15952 return;
15953
15954 // Do not apply information for constants or if RHS contains an AddRec.
15955 if (isa<SCEVConstant>(Val: LHS) || SE.containsAddRecurrence(S: RHS))
15956 return;
15957
15958 // If RHS is SCEVUnknown, make sure the information is applied to it.
15959 if (!isa<SCEVUnknown>(Val: LHS) && isa<SCEVUnknown>(Val: RHS)) {
15960 std::swap(a&: LHS, b&: RHS);
15961 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
15962 }
15963
15964 const SCEV *RewrittenLHS = GetMaybeRewritten(LHS);
15965 // Apply divisibility information when computing the constant multiple.
15966 const APInt &DividesBy =
15967 SE.getConstantMultiple(S: DivGuards.rewrite(Expr: RewrittenLHS));
15968
15969 // Collect rewrites for LHS and its transitive operands based on the
15970 // condition.
15971 // For min/max expressions, also apply the guard to its operands:
15972 // 'min(a, b) >= c' -> '(a >= c) and (b >= c)',
15973 // 'min(a, b) > c' -> '(a > c) and (b > c)',
15974 // 'max(a, b) <= c' -> '(a <= c) and (b <= c)',
15975 // 'max(a, b) < c' -> '(a < c) and (b < c)'.
15976
15977 // We cannot express strict predicates in SCEV, so instead we replace them
15978 // with non-strict ones against plus or minus one of RHS depending on the
15979 // predicate.
15980 const SCEV *One = SE.getOne(Ty: RHS->getType());
15981 switch (Predicate) {
15982 case CmpInst::ICMP_ULT:
15983 if (RHS->getType()->isPointerTy())
15984 return;
15985 RHS = SE.getUMaxExpr(LHS: RHS, RHS: One);
15986 [[fallthrough]];
15987 case CmpInst::ICMP_SLT: {
15988 RHS = SE.getMinusSCEV(LHS: RHS, RHS: One);
15989 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
15990 break;
15991 }
15992 case CmpInst::ICMP_UGT:
15993 case CmpInst::ICMP_SGT:
15994 RHS = SE.getAddExpr(LHS: RHS, RHS: One);
15995 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
15996 break;
15997 case CmpInst::ICMP_ULE:
15998 case CmpInst::ICMP_SLE:
15999 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16000 break;
16001 case CmpInst::ICMP_UGE:
16002 case CmpInst::ICMP_SGE:
16003 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16004 break;
16005 default:
16006 break;
16007 }
16008
16009 SmallVector<SCEVUse, 16> Worklist(1, LHS);
16010 SmallPtrSet<const SCEV *, 16> Visited;
16011
16012 auto EnqueueOperands = [&Worklist](const SCEVNAryExpr *S) {
16013 append_range(C&: Worklist, R: S->operands());
16014 };
16015
16016 while (!Worklist.empty()) {
16017 const SCEV *From = Worklist.pop_back_val();
16018 if (isa<SCEVConstant>(Val: From))
16019 continue;
16020 if (!Visited.insert(Ptr: From).second)
16021 continue;
16022 const SCEV *FromRewritten = GetMaybeRewritten(From);
16023 const SCEV *To = nullptr;
16024
16025 switch (Predicate) {
16026 case CmpInst::ICMP_ULT:
16027 case CmpInst::ICMP_ULE:
16028 To = SE.getUMinExpr(LHS: FromRewritten, RHS);
16029 if (auto *UMax = dyn_cast<SCEVUMaxExpr>(Val: FromRewritten))
16030 EnqueueOperands(UMax);
16031 break;
16032 case CmpInst::ICMP_SLT:
16033 case CmpInst::ICMP_SLE:
16034 To = SE.getSMinExpr(LHS: FromRewritten, RHS);
16035 if (auto *SMax = dyn_cast<SCEVSMaxExpr>(Val: FromRewritten))
16036 EnqueueOperands(SMax);
16037 break;
16038 case CmpInst::ICMP_UGT:
16039 case CmpInst::ICMP_UGE:
16040 To = SE.getUMaxExpr(LHS: FromRewritten, RHS);
16041 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: FromRewritten))
16042 EnqueueOperands(UMin);
16043 break;
16044 case CmpInst::ICMP_SGT:
16045 case CmpInst::ICMP_SGE:
16046 To = SE.getSMaxExpr(LHS: FromRewritten, RHS);
16047 if (auto *SMin = dyn_cast<SCEVSMinExpr>(Val: FromRewritten))
16048 EnqueueOperands(SMin);
16049 break;
16050 case CmpInst::ICMP_EQ:
16051 if (isa<SCEVConstant>(Val: RHS))
16052 To = RHS;
16053 break;
16054 case CmpInst::ICMP_NE:
16055 if (match(S: RHS, P: m_scev_Zero())) {
16056 const SCEV *OneAlignedUp =
16057 getNextSCEVDivisibleByDivisor(Expr: One, DivisorVal: DividesBy, SE);
16058 To = SE.getUMaxExpr(LHS: FromRewritten, RHS: OneAlignedUp);
16059 } else {
16060 // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS),
16061 // but creating the subtraction eagerly is expensive. Track the
16062 // inequalities in a separate map, and materialize the rewrite lazily
16063 // when encountering a suitable subtraction while re-writing.
16064 if (LHS->getType()->isPointerTy()) {
16065 LHS = SE.getPtrToAddrExpr(Op: LHS);
16066 RHS = SE.getPtrToAddrExpr(Op: RHS);
16067 if (isa<SCEVCouldNotCompute>(Val: LHS) || isa<SCEVCouldNotCompute>(Val: RHS))
16068 break;
16069 }
16070 const SCEVConstant *C;
16071 const SCEV *A, *B;
16072 if (match(S: RHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: A))) &&
16073 match(S: LHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: B)))) {
16074 RHS = A;
16075 LHS = B;
16076 }
16077 if (LHS > RHS)
16078 std::swap(a&: LHS, b&: RHS);
16079 Guards.NotEqual.insert(V: {LHS, RHS});
16080 continue;
16081 }
16082 break;
16083 default:
16084 break;
16085 }
16086
16087 if (To)
16088 AddRewrite(From, FromRewritten, To);
16089 }
16090 };
16091
16092 SmallVector<PointerIntPair<Value *, 1, bool>> Terms;
16093 // First, collect information from assumptions dominating the loop.
16094 for (auto &AssumeVH : SE.AC.assumptions()) {
16095 if (!AssumeVH)
16096 continue;
16097 auto *AssumeI = cast<CallInst>(Val&: AssumeVH);
16098 if (!SE.DT.dominates(Def: AssumeI, BB: Block))
16099 continue;
16100 Terms.emplace_back(Args: AssumeI->getOperand(i_nocapture: 0), Args: true);
16101 }
16102
16103 // Second, collect information from llvm.experimental.guards dominating the loop.
16104 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
16105 M: SE.F.getParent(), id: Intrinsic::experimental_guard);
16106 if (GuardDecl)
16107 for (const auto *GU : GuardDecl->users())
16108 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
16109 if (Guard->getFunction() == Block->getParent() &&
16110 SE.DT.dominates(Def: Guard, BB: Block))
16111 Terms.emplace_back(Args: Guard->getArgOperand(i: 0), Args: true);
16112
16113 // Third, collect conditions from dominating branches. Starting at the loop
16114 // predecessor, climb up the predecessor chain, as long as there are
16115 // predecessors that can be found that have unique successors leading to the
16116 // original header.
16117 // TODO: share this logic with isLoopEntryGuardedByCond.
16118 unsigned NumCollectedConditions = 0;
16119 VisitedBlocks.insert(Ptr: Block);
16120 std::pair<const BasicBlock *, const BasicBlock *> Pair(Pred, Block);
16121 for (; Pair.first;
16122 Pair = SE.getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
16123 VisitedBlocks.insert(Ptr: Pair.second);
16124 const CondBrInst *LoopEntryPredicate =
16125 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
16126 if (!LoopEntryPredicate)
16127 continue;
16128
16129 Terms.emplace_back(Args: LoopEntryPredicate->getCondition(),
16130 Args: LoopEntryPredicate->getSuccessor(i: 0) == Pair.second);
16131 NumCollectedConditions++;
16132
16133 // If we are recursively collecting guards stop after 2
16134 // conditions to limit compile-time impact for now.
16135 if (Depth > 0 && NumCollectedConditions == 2)
16136 break;
16137 }
16138 // Finally, if we stopped climbing the predecessor chain because
16139 // there wasn't a unique one to continue, try to collect conditions
16140 // for PHINodes by recursively following all of their incoming
16141 // blocks and try to merge the found conditions to build a new one
16142 // for the Phi.
16143 if (Pair.second->hasNPredecessorsOrMore(N: 2) &&
16144 Depth < MaxLoopGuardCollectionDepth) {
16145 SmallDenseMap<const BasicBlock *, LoopGuards> IncomingGuards;
16146 for (auto &Phi : Pair.second->phis())
16147 collectFromPHI(SE, Guards, Phi, VisitedBlocks, IncomingGuards, Depth);
16148 }
16149
16150 // Now apply the information from the collected conditions to
16151 // Guards.RewriteMap. Conditions are processed in reverse order, so the
16152 // earliest conditions is processed first, except guards with divisibility
16153 // information, which are moved to the back. This ensures the SCEVs with the
16154 // shortest dependency chains are constructed first.
16155 SmallVector<std::tuple<CmpInst::Predicate, const SCEV *, const SCEV *>>
16156 GuardsToProcess;
16157 for (auto [Term, EnterIfTrue] : reverse(C&: Terms)) {
16158 SmallVector<Value *, 8> Worklist;
16159 SmallPtrSet<Value *, 8> Visited;
16160 Worklist.push_back(Elt: Term);
16161 while (!Worklist.empty()) {
16162 Value *Cond = Worklist.pop_back_val();
16163 if (!Visited.insert(Ptr: Cond).second)
16164 continue;
16165
16166 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
16167 auto Predicate =
16168 EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate();
16169 const auto *LHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 0));
16170 const auto *RHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 1));
16171 // If LHS is a constant, apply information to the other expression.
16172 // TODO: If LHS is not a constant, check if using CompareSCEVComplexity
16173 // can improve results.
16174 if (isa<SCEVConstant>(Val: LHS)) {
16175 std::swap(a&: LHS, b&: RHS);
16176 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
16177 }
16178 GuardsToProcess.emplace_back(Args&: Predicate, Args&: LHS, Args&: RHS);
16179 continue;
16180 }
16181
16182 Value *L, *R;
16183 if (EnterIfTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: L), R: m_Value(V&: R)))
16184 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: L), R: m_Value(V&: R)))) {
16185 Worklist.push_back(Elt: L);
16186 Worklist.push_back(Elt: R);
16187 }
16188 }
16189 }
16190
16191 // Process divisibility guards in reverse order to populate DivGuards early.
16192 DenseMap<const SCEV *, APInt> Multiples;
16193 LoopGuards DivGuards(SE);
16194 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess) {
16195 if (!isDivisibilityGuard(LHS, RHS, SE))
16196 continue;
16197 collectDivisibilityInformation(Predicate, LHS, RHS, DivInfo&: DivGuards.RewriteMap,
16198 Multiples, SE);
16199 }
16200
16201 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess)
16202 CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap, DivGuards);
16203
16204 // Apply divisibility information last. This ensures it is applied to the
16205 // outermost expression after other rewrites for the given value.
16206 for (const auto &[K, Divisor] : Multiples) {
16207 const SCEV *DivisorSCEV = SE.getConstant(Val: Divisor);
16208 Guards.RewriteMap[K] =
16209 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: applyDivisibilityOnMinMaxExpr(
16210 MinMaxExpr: Guards.rewrite(Expr: K), Divisor, SE),
16211 RHS: DivisorSCEV),
16212 RHS: DivisorSCEV);
16213 ExprsToRewrite.push_back(Elt: K);
16214 }
16215
16216 // Let the rewriter preserve NUW/NSW flags if the unsigned/signed ranges of
16217 // the replacement expressions are contained in the ranges of the replaced
16218 // expressions.
16219 Guards.PreserveNUW = true;
16220 Guards.PreserveNSW = true;
16221 for (const SCEV *Expr : ExprsToRewrite) {
16222 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16223 Guards.PreserveNUW &=
16224 SE.getUnsignedRange(S: Expr).contains(CR: SE.getUnsignedRange(S: RewriteTo));
16225 Guards.PreserveNSW &=
16226 SE.getSignedRange(S: Expr).contains(CR: SE.getSignedRange(S: RewriteTo));
16227 }
16228
16229 // Now that all rewrite information is collect, rewrite the collected
16230 // expressions with the information in the map. This applies information to
16231 // sub-expressions.
16232 if (ExprsToRewrite.size() > 1) {
16233 for (const SCEV *Expr : ExprsToRewrite) {
16234 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16235 Guards.RewriteMap.erase(Val: Expr);
16236 Guards.RewriteMap.insert(KV: {Expr, Guards.rewrite(Expr: RewriteTo)});
16237 }
16238 }
16239}
16240
16241const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
16242 /// A rewriter to replace SCEV expressions in Map with the corresponding entry
16243 /// in the map. It skips AddRecExpr because we cannot guarantee that the
16244 /// replacement is loop invariant in the loop of the AddRec.
16245 class SCEVLoopGuardRewriter
16246 : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> {
16247 const DenseMap<const SCEV *, const SCEV *> &Map;
16248 const SmallDenseSet<std::pair<const SCEV *, const SCEV *>> &NotEqual;
16249
16250 SCEVFlags FlagMask = SCEV::FlagNone;
16251
16252 public:
16253 SCEVLoopGuardRewriter(ScalarEvolution &SE,
16254 const ScalarEvolution::LoopGuards &Guards)
16255 : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap),
16256 NotEqual(Guards.NotEqual) {
16257 if (Guards.PreserveNUW)
16258 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNUW);
16259 if (Guards.PreserveNSW)
16260 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNSW);
16261 }
16262
16263 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
16264
16265 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
16266 return Map.lookup_or(Val: Expr, Default&: Expr);
16267 }
16268
16269 const SCEV *visitPtrToAddrExpr(const SCEVPtrToAddrExpr *Expr) {
16270 if (const SCEV *S = Map.lookup(Val: Expr))
16271 return S;
16272 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitPtrToAddrExpr(
16273 Expr);
16274 }
16275
16276 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
16277 if (const SCEV *S = Map.lookup(Val: Expr))
16278 return S;
16279
16280 // If we didn't find the extact ZExt expr in the map, check if there's
16281 // an entry for a smaller ZExt we can use instead.
16282 Type *Ty = Expr->getType();
16283 const SCEV *Op = Expr->getOperand(i: 0);
16284 unsigned Bitwidth = Ty->getScalarSizeInBits() / 2;
16285 while (Bitwidth % 8 == 0 && Bitwidth >= 8 &&
16286 Bitwidth > Op->getType()->getScalarSizeInBits()) {
16287 Type *NarrowTy = IntegerType::get(C&: SE.getContext(), NumBits: Bitwidth);
16288 auto *NarrowExt = SE.getZeroExtendExpr(Op, Ty: NarrowTy);
16289 if (const SCEV *S = Map.lookup(Val: NarrowExt))
16290 return SE.getZeroExtendExpr(Op: S, Ty);
16291 Bitwidth = Bitwidth / 2;
16292 }
16293
16294 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitZeroExtendExpr(
16295 Expr);
16296 }
16297
16298 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
16299 if (const SCEV *S = Map.lookup(Val: Expr))
16300 return S;
16301 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSignExtendExpr(
16302 Expr);
16303 }
16304
16305 const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) {
16306 if (const SCEV *S = Map.lookup(Val: Expr))
16307 return S;
16308 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitUMinExpr(Expr);
16309 }
16310
16311 const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) {
16312 if (const SCEV *S = Map.lookup(Val: Expr))
16313 return S;
16314 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSMinExpr(Expr);
16315 }
16316
16317 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
16318 if (const SCEV *S = Map.lookup(Val: Expr))
16319 return S;
16320
16321 // Helper to check if S is a subtraction (A - B) where A != B, and if so,
16322 // return UMax(S, 1).
16323 auto RewriteSubtraction = [&](const SCEV *S) -> const SCEV * {
16324 SCEVUse LHS, RHS;
16325 if (MatchBinarySub(S, LHS, RHS)) {
16326 if (LHS > RHS)
16327 std::swap(a&: LHS, b&: RHS);
16328 if (NotEqual.contains(V: {LHS, RHS})) {
16329 const SCEV *OneAlignedUp = getNextSCEVDivisibleByDivisor(
16330 Expr: SE.getOne(Ty: S->getType()), DivisorVal: SE.getConstantMultiple(S), SE);
16331 return SE.getUMaxExpr(LHS: OneAlignedUp, RHS: S);
16332 }
16333 }
16334 return nullptr;
16335 };
16336
16337 // Check if Expr itself is a subtraction pattern with guard info.
16338 if (const SCEV *Rewritten = RewriteSubtraction(Expr))
16339 return Rewritten;
16340
16341 // Trip count expressions sometimes consist of adding 3 operands, i.e.
16342 // (Const + A + B). There may be guard info for A + B, and if so, apply
16343 // it.
16344 // TODO: Could more generally apply guards to Add sub-expressions.
16345 if (isa<SCEVConstant>(Val: Expr->getOperand(i: 0))) {
16346 if (Expr->getNumOperands() == 3) {
16347 const SCEV *Add =
16348 SE.getAddExpr(LHS: Expr->getOperand(i: 1), RHS: Expr->getOperand(i: 2));
16349 if (const SCEV *Rewritten = RewriteSubtraction(Add))
16350 return SE.getAddExpr(
16351 LHS: Expr->getOperand(i: 0), RHS: Rewritten,
16352 Flags: ScalarEvolution::maskFlags(Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16353 if (const SCEV *S = Map.lookup(Val: Add))
16354 return SE.getAddExpr(LHS: Expr->getOperand(i: 0), RHS: S);
16355 }
16356
16357 // For expressions of the form (Const + A), check if we have guard info
16358 // for (Const + 1 + A), and rewrite to ((Const + 1 + A) - 1). This makes
16359 // sure we don't lose information when rewriting expressions based on
16360 // back-edge taken counts in some cases.
16361 if (Expr->getNumOperands() == 2) {
16362 const SCEV *S = nullptr;
16363 // Handle (-1 + 1 + A) without constructing SCEVs.
16364 if (match(U: Expr->getOperand(i: 0), P: m_scev_AllOnes())) {
16365 S = Map.lookup(Val: Expr->getOperand(i: 1));
16366 } else {
16367 const SCEV *NewC =
16368 SE.getAddExpr(LHS: Expr->getOperand(i: 0), RHS: SE.getOne(Ty: Expr->getType()));
16369 S = Map.lookup(Val: SE.getAddExpr(LHS: NewC, RHS: Expr->getOperand(i: 1)));
16370 }
16371 if (S)
16372 return SE.getAddExpr(LHS: S, RHS: SE.getMinusOne(Ty: Expr->getType()));
16373 }
16374 }
16375 SmallVector<SCEVUse, 2> Operands;
16376 bool Changed = false;
16377 for (SCEVUse Op : Expr->operands()) {
16378 Operands.push_back(
16379 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
16380 Changed |= Op != Operands.back();
16381 }
16382 // We are only replacing operands with equivalent values, so transfer the
16383 // flags from the original expression.
16384 return !Changed ? Expr
16385 : SE.getAddExpr(Ops&: Operands,
16386 Flags: ScalarEvolution::maskFlags(
16387 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16388 }
16389
16390 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
16391 SmallVector<SCEVUse, 2> Operands;
16392 bool Changed = false;
16393 for (SCEVUse Op : Expr->operands()) {
16394 Operands.push_back(
16395 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
16396 Changed |= Op != Operands.back();
16397 }
16398 // We are only replacing operands with equivalent values, so transfer the
16399 // flags from the original expression.
16400 return !Changed ? Expr
16401 : SE.getMulExpr(Ops&: Operands,
16402 Flags: ScalarEvolution::maskFlags(
16403 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16404 }
16405 };
16406
16407 if (RewriteMap.empty() && NotEqual.empty())
16408 return Expr;
16409
16410 SCEVLoopGuardRewriter Rewriter(SE, *this);
16411 return Rewriter.visit(S: Expr);
16412}
16413
16414const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
16415 return applyLoopGuards(Expr, Guards: LoopGuards::collect(L, SE&: *this));
16416}
16417
16418const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr,
16419 const LoopGuards &Guards) {
16420 return Guards.rewrite(Expr);
16421}
16422