1//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the scalar evolution analysis
10// engine, which is used primarily to analyze expressions involving induction
11// variables in loops.
12//
13// There are several aspects to this library. First is the representation of
14// scalar expressions, which are represented as subclasses of the SCEV class.
15// These classes are used to represent certain types of subexpressions that we
16// can handle. We only create one SCEV of a particular shape, so
17// pointer-comparisons for equality are legal.
18//
19// One important aspect of the SCEV objects is that they are never cyclic, even
20// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
21// the PHI node is one of the idioms that we can represent (e.g., a polynomial
22// recurrence) then we represent it directly as a recurrence node, otherwise we
23// represent it as a SCEVUnknown node.
24//
25// In addition to being able to represent expressions of various types, we also
26// have folders that are used to build the *canonical* representation for a
27// particular expression. These folders are capable of using a variety of
28// rewrite rules to simplify the expressions.
29//
30// Once the folders are defined, we can implement the more interesting
31// higher-level code, such as the code that recognizes PHI nodes of various
32// types, computes the execution count of a loop, etc.
33//
34// TODO: We should use these routines and value representations to implement
35// dependence analysis!
36//
37//===----------------------------------------------------------------------===//
38//
39// There are several good references for the techniques used in this analysis.
40//
41// Chains of recurrences -- a method to expedite the evaluation
42// of closed-form functions
43// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
44//
45// On computational properties of chains of recurrences
46// Eugene V. Zima
47//
48// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
49// Robert A. van Engelen
50//
51// Efficient Symbolic Analysis for Optimizing Compilers
52// Robert A. van Engelen
53//
54// Using the chains of recurrences algebra for data dependence testing and
55// induction variable substitution
56// MS Thesis, Johnie Birch
57//
58//===----------------------------------------------------------------------===//
59
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/ADT/APInt.h"
62#include "llvm/ADT/ArrayRef.h"
63#include "llvm/ADT/DenseMap.h"
64#include "llvm/ADT/DepthFirstIterator.h"
65#include "llvm/ADT/FoldingSet.h"
66#include "llvm/ADT/STLExtras.h"
67#include "llvm/ADT/ScopeExit.h"
68#include "llvm/ADT/Sequence.h"
69#include "llvm/ADT/SmallPtrSet.h"
70#include "llvm/ADT/SmallVector.h"
71#include "llvm/ADT/Statistic.h"
72#include "llvm/ADT/StringExtras.h"
73#include "llvm/ADT/StringRef.h"
74#include "llvm/Analysis/AssumptionCache.h"
75#include "llvm/Analysis/ConstantFolding.h"
76#include "llvm/Analysis/InstructionSimplify.h"
77#include "llvm/Analysis/LoopInfo.h"
78#include "llvm/Analysis/MemoryBuiltins.h"
79#include "llvm/Analysis/ScalarEvolutionExpressions.h"
80#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
81#include "llvm/Analysis/TargetLibraryInfo.h"
82#include "llvm/Analysis/ValueTracking.h"
83#include "llvm/Config/llvm-config.h"
84#include "llvm/IR/Argument.h"
85#include "llvm/IR/BasicBlock.h"
86#include "llvm/IR/CFG.h"
87#include "llvm/IR/Constant.h"
88#include "llvm/IR/ConstantRange.h"
89#include "llvm/IR/Constants.h"
90#include "llvm/IR/DataLayout.h"
91#include "llvm/IR/DerivedTypes.h"
92#include "llvm/IR/Dominators.h"
93#include "llvm/IR/Function.h"
94#include "llvm/IR/GlobalAlias.h"
95#include "llvm/IR/GlobalValue.h"
96#include "llvm/IR/InstIterator.h"
97#include "llvm/IR/InstrTypes.h"
98#include "llvm/IR/Instruction.h"
99#include "llvm/IR/Instructions.h"
100#include "llvm/IR/IntrinsicInst.h"
101#include "llvm/IR/Intrinsics.h"
102#include "llvm/IR/LLVMContext.h"
103#include "llvm/IR/Operator.h"
104#include "llvm/IR/PatternMatch.h"
105#include "llvm/IR/Type.h"
106#include "llvm/IR/Use.h"
107#include "llvm/IR/User.h"
108#include "llvm/IR/Value.h"
109#include "llvm/IR/Verifier.h"
110#include "llvm/InitializePasses.h"
111#include "llvm/Pass.h"
112#include "llvm/Support/Casting.h"
113#include "llvm/Support/CommandLine.h"
114#include "llvm/Support/Compiler.h"
115#include "llvm/Support/Debug.h"
116#include "llvm/Support/ErrorHandling.h"
117#include "llvm/Support/InterleavedRange.h"
118#include "llvm/Support/KnownBits.h"
119#include "llvm/Support/SaveAndRestore.h"
120#include "llvm/Support/raw_ostream.h"
121#include <algorithm>
122#include <cassert>
123#include <climits>
124#include <cstdint>
125#include <cstdlib>
126#include <map>
127#include <memory>
128#include <numeric>
129#include <optional>
130#include <tuple>
131#include <utility>
132#include <vector>
133
134using namespace llvm;
135using namespace PatternMatch;
136using namespace SCEVPatternMatch;
137
138#define DEBUG_TYPE "scalar-evolution"
139
140STATISTIC(NumExitCountsComputed,
141 "Number of loop exits with predictable exit counts");
142STATISTIC(NumExitCountsNotComputed,
143 "Number of loop exits without predictable exit counts");
144STATISTIC(NumBruteForceTripCountsComputed,
145 "Number of loops with trip counts computed by force");
146
147#ifdef EXPENSIVE_CHECKS
148bool llvm::VerifySCEV = true;
149#else
150bool llvm::VerifySCEV = false;
151#endif
152
153static cl::opt<unsigned>
154 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
155 cl::desc("Maximum number of iterations SCEV will "
156 "symbolically execute a constant "
157 "derived loop"),
158 cl::init(Val: 100));
159
160static cl::opt<bool, true> VerifySCEVOpt(
161 "verify-scev", cl::Hidden, cl::location(L&: VerifySCEV),
162 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
163static cl::opt<bool> VerifySCEVStrict(
164 "verify-scev-strict", cl::Hidden,
165 cl::desc("Enable stricter verification with -verify-scev is passed"));
166
167static cl::opt<bool> VerifyIR(
168 "scev-verify-ir", cl::Hidden,
169 cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"),
170 cl::init(Val: false));
171
172static cl::opt<unsigned> MulOpsInlineThreshold(
173 "scev-mulops-inline-threshold", cl::Hidden,
174 cl::desc("Threshold for inlining multiplication operands into a SCEV"),
175 cl::init(Val: 32));
176
177static cl::opt<unsigned> AddOpsInlineThreshold(
178 "scev-addops-inline-threshold", cl::Hidden,
179 cl::desc("Threshold for inlining addition operands into a SCEV"),
180 cl::init(Val: 500));
181
182static cl::opt<unsigned> MaxSCEVCompareDepth(
183 "scalar-evolution-max-scev-compare-depth", cl::Hidden,
184 cl::desc("Maximum depth of recursive SCEV complexity comparisons"),
185 cl::init(Val: 32));
186
187static cl::opt<unsigned> MaxSCEVOperationsImplicationDepth(
188 "scalar-evolution-max-scev-operations-implication-depth", cl::Hidden,
189 cl::desc("Maximum depth of recursive SCEV operations implication analysis"),
190 cl::init(Val: 2));
191
192static cl::opt<unsigned> MaxValueCompareDepth(
193 "scalar-evolution-max-value-compare-depth", cl::Hidden,
194 cl::desc("Maximum depth of recursive value complexity comparisons"),
195 cl::init(Val: 2));
196
197static cl::opt<unsigned>
198 MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden,
199 cl::desc("Maximum depth of recursive arithmetics"),
200 cl::init(Val: 32));
201
202static cl::opt<unsigned> MaxConstantEvolvingDepth(
203 "scalar-evolution-max-constant-evolving-depth", cl::Hidden,
204 cl::desc("Maximum depth of recursive constant evolving"), cl::init(Val: 32));
205
206static cl::opt<unsigned>
207 MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden,
208 cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"),
209 cl::init(Val: 8));
210
211static cl::opt<unsigned>
212 MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden,
213 cl::desc("Max coefficients in AddRec during evolving"),
214 cl::init(Val: 8));
215
216static cl::opt<unsigned>
217 HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden,
218 cl::desc("Size of the expression which is considered huge"),
219 cl::init(Val: 4096));
220
221static cl::opt<unsigned> RangeIterThreshold(
222 "scev-range-iter-threshold", cl::Hidden,
223 cl::desc("Threshold for switching to iteratively computing SCEV ranges"),
224 cl::init(Val: 32));
225
226static cl::opt<unsigned> MaxLoopGuardCollectionDepth(
227 "scalar-evolution-max-loop-guard-collection-depth", cl::Hidden,
228 cl::desc("Maximum depth for recursive loop guard collection"), cl::init(Val: 1));
229
230static cl::opt<bool>
231ClassifyExpressions("scalar-evolution-classify-expressions",
232 cl::Hidden, cl::init(Val: true),
233 cl::desc("When printing analysis, include information on every instruction"));
234
235static cl::opt<bool> UseExpensiveRangeSharpening(
236 "scalar-evolution-use-expensive-range-sharpening", cl::Hidden,
237 cl::init(Val: false),
238 cl::desc("Use more powerful methods of sharpening expression ranges. May "
239 "be costly in terms of compile time"));
240
241static cl::opt<unsigned> MaxPhiSCCAnalysisSize(
242 "scalar-evolution-max-scc-analysis-depth", cl::Hidden,
243 cl::desc("Maximum amount of nodes to process while searching SCEVUnknown "
244 "Phi strongly connected components"),
245 cl::init(Val: 8));
246
247static cl::opt<bool>
248 EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden,
249 cl::desc("Handle <= and >= in finite loops"),
250 cl::init(Val: true));
251
252static cl::opt<bool> UseContextForNoWrapFlagInference(
253 "scalar-evolution-use-context-for-no-wrap-flag-strenghening", cl::Hidden,
254 cl::desc("Infer nuw/nsw flags using context where suitable"),
255 cl::init(Val: true));
256
257//===----------------------------------------------------------------------===//
258// SCEV class definitions
259//===----------------------------------------------------------------------===//
260
261void SCEV::computeAndSetCanonical(ScalarEvolution &SE) {
262 // Leaf nodes are always their own canonical.
263 switch (getSCEVType()) {
264 case scConstant:
265 case scVScale:
266 case scUnknown:
267 CanonicalSCEV = this;
268 return;
269 default:
270 break;
271 }
272
273 // For all other expressions, check whether any immediate operand has a
274 // different canonical. Since operands are always created before their parent,
275 // their canonical pointers are already set — no recursion needed.
276 bool Changed = false;
277 SmallVector<SCEVUse, 4> CanonOps;
278 for (SCEVUse Op : operands()) {
279 CanonOps.push_back(Elt: Op->getCanonical());
280 Changed |= CanonOps.back() != Op.getPointer();
281 }
282
283 if (!Changed) {
284 CanonicalSCEV = this;
285 return;
286 }
287
288 auto *NAry = dyn_cast<SCEVNAryExpr>(Val: this);
289 SCEV::NoWrapFlags Flags = NAry ? NAry->getNoWrapFlags() : SCEV::FlagAnyWrap;
290 switch (getSCEVType()) {
291 case scPtrToAddr:
292 CanonicalSCEV = SE.getPtrToAddrExpr(Op: CanonOps[0]);
293 return;
294 case scPtrToInt:
295 CanonicalSCEV = SE.getPtrToIntExpr(Op: CanonOps[0], Ty: getType());
296 return;
297 case scTruncate:
298 CanonicalSCEV = SE.getTruncateExpr(Op: CanonOps[0], Ty: getType());
299 return;
300 case scZeroExtend:
301 CanonicalSCEV = SE.getZeroExtendExpr(Op: CanonOps[0], Ty: getType());
302 return;
303 case scSignExtend:
304 CanonicalSCEV = SE.getSignExtendExpr(Op: CanonOps[0], Ty: getType());
305 return;
306 case scUDivExpr:
307 CanonicalSCEV = SE.getUDivExpr(LHS: CanonOps[0], RHS: CanonOps[1]);
308 return;
309 case scAddExpr:
310 CanonicalSCEV = SE.getAddExpr(Ops&: CanonOps, Flags);
311 return;
312 case scMulExpr:
313 CanonicalSCEV = SE.getMulExpr(Ops&: CanonOps, Flags);
314 return;
315 case scAddRecExpr:
316 CanonicalSCEV = SE.getAddRecExpr(
317 Operands&: CanonOps, L: cast<SCEVAddRecExpr>(Val: this)->getLoop(), Flags);
318 return;
319 case scSMaxExpr:
320 CanonicalSCEV = SE.getSMaxExpr(Operands&: CanonOps);
321 return;
322 case scUMaxExpr:
323 CanonicalSCEV = SE.getUMaxExpr(Operands&: CanonOps);
324 return;
325 case scSMinExpr:
326 CanonicalSCEV = SE.getSMinExpr(Operands&: CanonOps);
327 return;
328 case scUMinExpr:
329 CanonicalSCEV = SE.getUMinExpr(Operands&: CanonOps);
330 return;
331 case scSequentialUMinExpr:
332 CanonicalSCEV = SE.getUMinExpr(Operands&: CanonOps, /*Sequential=*/true);
333 return;
334 default:
335 llvm_unreachable("Unknown SCEV type");
336 }
337}
338
339//===----------------------------------------------------------------------===//
340// Implementation of the SCEV class.
341//
342
343#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
344LLVM_DUMP_METHOD void SCEV::dump() const {
345 print(dbgs());
346 dbgs() << '\n';
347}
348#endif
349
350void SCEV::print(raw_ostream &OS) const {
351 switch (getSCEVType()) {
352 case scConstant:
353 cast<SCEVConstant>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
354 return;
355 case scVScale:
356 OS << "vscale";
357 return;
358 case scPtrToAddr:
359 case scPtrToInt: {
360 const SCEVCastExpr *PtrCast = cast<SCEVCastExpr>(Val: this);
361 const SCEV *Op = PtrCast->getOperand();
362 StringRef OpS = getSCEVType() == scPtrToAddr ? "addr" : "int";
363 OS << "(ptrto" << OpS << " " << *Op->getType() << " " << *Op << " to "
364 << *PtrCast->getType() << ")";
365 return;
366 }
367 case scTruncate: {
368 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: this);
369 const SCEV *Op = Trunc->getOperand();
370 OS << "(trunc " << *Op->getType() << " " << *Op << " to "
371 << *Trunc->getType() << ")";
372 return;
373 }
374 case scZeroExtend: {
375 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: this);
376 const SCEV *Op = ZExt->getOperand();
377 OS << "(zext " << *Op->getType() << " " << *Op << " to "
378 << *ZExt->getType() << ")";
379 return;
380 }
381 case scSignExtend: {
382 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: this);
383 const SCEV *Op = SExt->getOperand();
384 OS << "(sext " << *Op->getType() << " " << *Op << " to "
385 << *SExt->getType() << ")";
386 return;
387 }
388 case scAddRecExpr: {
389 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: this);
390 OS << "{" << *AR->getOperand(i: 0);
391 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
392 OS << ",+," << *AR->getOperand(i);
393 OS << "}<";
394 if (AR->hasNoUnsignedWrap())
395 OS << "nuw><";
396 if (AR->hasNoSignedWrap())
397 OS << "nsw><";
398 if (AR->hasNoSelfWrap() && !AR->hasNoUnsignedWrap() &&
399 !AR->hasNoSignedWrap())
400 OS << "nw><";
401 AR->getLoop()->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
402 OS << ">";
403 return;
404 }
405 case scAddExpr:
406 case scMulExpr:
407 case scUMaxExpr:
408 case scSMaxExpr:
409 case scUMinExpr:
410 case scSMinExpr:
411 case scSequentialUMinExpr: {
412 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(Val: this);
413 const char *OpStr = nullptr;
414 switch (NAry->getSCEVType()) {
415 case scAddExpr: OpStr = " + "; break;
416 case scMulExpr: OpStr = " * "; break;
417 case scUMaxExpr: OpStr = " umax "; break;
418 case scSMaxExpr: OpStr = " smax "; break;
419 case scUMinExpr:
420 OpStr = " umin ";
421 break;
422 case scSMinExpr:
423 OpStr = " smin ";
424 break;
425 case scSequentialUMinExpr:
426 OpStr = " umin_seq ";
427 break;
428 default:
429 llvm_unreachable("There are no other nary expression types.");
430 }
431 OS << "("
432 << llvm::interleaved(R: llvm::make_pointee_range(Range: NAry->operands()), Separator: OpStr)
433 << ")";
434 switch (NAry->getSCEVType()) {
435 case scAddExpr:
436 case scMulExpr:
437 if (NAry->hasNoUnsignedWrap())
438 OS << "<nuw>";
439 if (NAry->hasNoSignedWrap())
440 OS << "<nsw>";
441 break;
442 default:
443 // Nothing to print for other nary expressions.
444 break;
445 }
446 return;
447 }
448 case scUDivExpr: {
449 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: this);
450 OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")";
451 return;
452 }
453 case scUnknown:
454 cast<SCEVUnknown>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
455 return;
456 case scCouldNotCompute:
457 OS << "***COULDNOTCOMPUTE***";
458 return;
459 }
460 llvm_unreachable("Unknown SCEV kind!");
461}
462
463Type *SCEV::getType() const {
464 switch (getSCEVType()) {
465 case scConstant:
466 return cast<SCEVConstant>(Val: this)->getType();
467 case scVScale:
468 return cast<SCEVVScale>(Val: this)->getType();
469 case scPtrToAddr:
470 case scPtrToInt:
471 case scTruncate:
472 case scZeroExtend:
473 case scSignExtend:
474 return cast<SCEVCastExpr>(Val: this)->getType();
475 case scAddRecExpr:
476 return cast<SCEVAddRecExpr>(Val: this)->getType();
477 case scMulExpr:
478 return cast<SCEVMulExpr>(Val: this)->getType();
479 case scUMaxExpr:
480 case scSMaxExpr:
481 case scUMinExpr:
482 case scSMinExpr:
483 return cast<SCEVMinMaxExpr>(Val: this)->getType();
484 case scSequentialUMinExpr:
485 return cast<SCEVSequentialMinMaxExpr>(Val: this)->getType();
486 case scAddExpr:
487 return cast<SCEVAddExpr>(Val: this)->getType();
488 case scUDivExpr:
489 return cast<SCEVUDivExpr>(Val: this)->getType();
490 case scUnknown:
491 return cast<SCEVUnknown>(Val: this)->getType();
492 case scCouldNotCompute:
493 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
494 }
495 llvm_unreachable("Unknown SCEV kind!");
496}
497
498ArrayRef<SCEVUse> SCEV::operands() const {
499 switch (getSCEVType()) {
500 case scConstant:
501 case scVScale:
502 case scUnknown:
503 return {};
504 case scPtrToAddr:
505 case scPtrToInt:
506 case scTruncate:
507 case scZeroExtend:
508 case scSignExtend:
509 return cast<SCEVCastExpr>(Val: this)->operands();
510 case scAddRecExpr:
511 case scAddExpr:
512 case scMulExpr:
513 case scUMaxExpr:
514 case scSMaxExpr:
515 case scUMinExpr:
516 case scSMinExpr:
517 case scSequentialUMinExpr:
518 return cast<SCEVNAryExpr>(Val: this)->operands();
519 case scUDivExpr:
520 return cast<SCEVUDivExpr>(Val: this)->operands();
521 case scCouldNotCompute:
522 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
523 }
524 llvm_unreachable("Unknown SCEV kind!");
525}
526
527bool SCEV::isZero() const { return match(S: this, P: m_scev_Zero()); }
528
529bool SCEV::isOne() const { return match(S: this, P: m_scev_One()); }
530
531bool SCEV::isAllOnesValue() const { return match(S: this, P: m_scev_AllOnes()); }
532
533bool SCEV::isNonConstantNegative() const {
534 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: this);
535 if (!Mul) return false;
536
537 // If there is a constant factor, it will be first.
538 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: 0));
539 if (!SC) return false;
540
541 // Return true if the value is negative, this matches things like (-42 * V).
542 return SC->getAPInt().isNegative();
543}
544
545SCEVCouldNotCompute::SCEVCouldNotCompute() :
546 SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0) {}
547
548bool SCEVCouldNotCompute::classof(const SCEV *S) {
549 return S->getSCEVType() == scCouldNotCompute;
550}
551
552const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
553 FoldingSetNodeID ID;
554 ID.AddInteger(I: scConstant);
555 ID.AddPointer(Ptr: V);
556 void *IP = nullptr;
557 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
558 SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(Allocator&: SCEVAllocator), V);
559 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
560 S->computeAndSetCanonical(SE&: *this);
561 return S;
562}
563
564const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
565 return getConstant(V: ConstantInt::get(Context&: getContext(), V: Val));
566}
567
568const SCEV *
569ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
570 IntegerType *ITy = cast<IntegerType>(Val: getEffectiveSCEVType(Ty));
571 // TODO: Avoid implicit trunc?
572 // See https://github.com/llvm/llvm-project/issues/112510.
573 return getConstant(
574 V: ConstantInt::get(Ty: ITy, V, IsSigned: isSigned, /*ImplicitTrunc=*/true));
575}
576
577const SCEV *ScalarEvolution::getVScale(Type *Ty) {
578 FoldingSetNodeID ID;
579 ID.AddInteger(I: scVScale);
580 ID.AddPointer(Ptr: Ty);
581 void *IP = nullptr;
582 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
583 return S;
584 SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(Allocator&: SCEVAllocator), Ty);
585 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
586 S->computeAndSetCanonical(SE&: *this);
587 return S;
588}
589
590const SCEV *ScalarEvolution::getElementCount(Type *Ty, ElementCount EC,
591 SCEV::NoWrapFlags Flags) {
592 const SCEV *Res = getConstant(Ty, V: EC.getKnownMinValue());
593 if (EC.isScalable())
594 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty), Flags);
595 return Res;
596}
597
598SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
599 SCEVUse op, Type *ty)
600 : SCEV(ID, SCEVTy, computeExpressionSize(Args: op)), Op(op), Ty(ty) {}
601
602SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID,
603 const SCEV *Op, Type *ITy)
604 : SCEVCastExpr(ID, scPtrToAddr, Op, ITy) {
605 assert(getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() &&
606 "Must be a non-bit-width-changing pointer-to-integer cast!");
607}
608
609SCEVPtrToIntExpr::SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, SCEVUse Op,
610 Type *ITy)
611 : SCEVCastExpr(ID, scPtrToInt, Op, ITy) {
612 assert(getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() &&
613 "Must be a non-bit-width-changing pointer-to-integer cast!");
614}
615
616SCEVIntegralCastExpr::SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID,
617 SCEVTypes SCEVTy, SCEVUse op,
618 Type *ty)
619 : SCEVCastExpr(ID, SCEVTy, op, ty) {}
620
621SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
622 Type *ty)
623 : SCEVIntegralCastExpr(ID, scTruncate, op, ty) {
624 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
625 "Cannot truncate non-integer value!");
626}
627
628SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
629 Type *ty)
630 : SCEVIntegralCastExpr(ID, scZeroExtend, op, ty) {
631 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
632 "Cannot zero extend non-integer value!");
633}
634
635SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
636 Type *ty)
637 : SCEVIntegralCastExpr(ID, scSignExtend, op, ty) {
638 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
639 "Cannot sign extend non-integer value!");
640}
641
642void SCEVUnknown::deleted() {
643 // Clear this SCEVUnknown from various maps.
644 SE->forgetMemoizedResults(SCEVs: {this});
645
646 // Remove this SCEVUnknown from the uniquing map.
647 SE->UniqueSCEVs.RemoveNode(N: this);
648
649 // Release the value.
650 setValPtr(nullptr);
651}
652
653void SCEVUnknown::allUsesReplacedWith(Value *New) {
654 // Clear this SCEVUnknown from various maps.
655 SE->forgetMemoizedResults(SCEVs: {this});
656
657 // Remove this SCEVUnknown from the uniquing map.
658 SE->UniqueSCEVs.RemoveNode(N: this);
659
660 // Replace the value pointer in case someone is still using this SCEVUnknown.
661 setValPtr(New);
662}
663
664//===----------------------------------------------------------------------===//
665// SCEV Utilities
666//===----------------------------------------------------------------------===//
667
668/// Compare the two values \p LV and \p RV in terms of their "complexity" where
669/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
670/// operands in SCEV expressions.
671static int CompareValueComplexity(const LoopInfo *const LI, Value *LV,
672 Value *RV, unsigned Depth) {
673 if (Depth > MaxValueCompareDepth)
674 return 0;
675
676 // Order pointer values after integer values. This helps SCEVExpander form
677 // GEPs.
678 bool LIsPointer = LV->getType()->isPointerTy(),
679 RIsPointer = RV->getType()->isPointerTy();
680 if (LIsPointer != RIsPointer)
681 return (int)LIsPointer - (int)RIsPointer;
682
683 // Compare getValueID values.
684 unsigned LID = LV->getValueID(), RID = RV->getValueID();
685 if (LID != RID)
686 return (int)LID - (int)RID;
687
688 // Sort arguments by their position.
689 if (const auto *LA = dyn_cast<Argument>(Val: LV)) {
690 const auto *RA = cast<Argument>(Val: RV);
691 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
692 return (int)LArgNo - (int)RArgNo;
693 }
694
695 if (const auto *LGV = dyn_cast<GlobalValue>(Val: LV)) {
696 const auto *RGV = cast<GlobalValue>(Val: RV);
697
698 if (auto L = LGV->getLinkage() - RGV->getLinkage())
699 return L;
700
701 const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
702 auto LT = GV->getLinkage();
703 return !(GlobalValue::isPrivateLinkage(Linkage: LT) ||
704 GlobalValue::isInternalLinkage(Linkage: LT));
705 };
706
707 // Use the names to distinguish the two values, but only if the
708 // names are semantically important.
709 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
710 return LGV->getName().compare(RHS: RGV->getName());
711 }
712
713 // For instructions, compare their loop depth, and their operand count. This
714 // is pretty loose.
715 if (const auto *LInst = dyn_cast<Instruction>(Val: LV)) {
716 const auto *RInst = cast<Instruction>(Val: RV);
717
718 // Compare loop depths.
719 const BasicBlock *LParent = LInst->getParent(),
720 *RParent = RInst->getParent();
721 if (LParent != RParent) {
722 unsigned LDepth = LI->getLoopDepth(BB: LParent),
723 RDepth = LI->getLoopDepth(BB: RParent);
724 if (LDepth != RDepth)
725 return (int)LDepth - (int)RDepth;
726 }
727
728 // Compare the number of operands.
729 unsigned LNumOps = LInst->getNumOperands(),
730 RNumOps = RInst->getNumOperands();
731 if (LNumOps != RNumOps)
732 return (int)LNumOps - (int)RNumOps;
733
734 for (unsigned Idx : seq(Size: LNumOps)) {
735 int Result = CompareValueComplexity(LI, LV: LInst->getOperand(i: Idx),
736 RV: RInst->getOperand(i: Idx), Depth: Depth + 1);
737 if (Result != 0)
738 return Result;
739 }
740 }
741
742 return 0;
743}
744
745// Return negative, zero, or positive, if LHS is less than, equal to, or greater
746// than RHS, respectively. A three-way result allows recursive comparisons to be
747// more efficient.
748// If the max analysis depth was reached, return std::nullopt, assuming we do
749// not know if they are equivalent for sure.
750static std::optional<int>
751CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
752 const SCEV *RHS, DominatorTree &DT, unsigned Depth = 0) {
753 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
754 if (LHS == RHS)
755 return 0;
756
757 // Primarily, sort the SCEVs by their getSCEVType().
758 SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
759 if (LType != RType)
760 return (int)LType - (int)RType;
761
762 if (Depth > MaxSCEVCompareDepth)
763 return std::nullopt;
764
765 // Aside from the getSCEVType() ordering, the particular ordering
766 // isn't very important except that it's beneficial to be consistent,
767 // so that (a + b) and (b + a) don't end up as different expressions.
768 switch (LType) {
769 case scUnknown: {
770 const SCEVUnknown *LU = cast<SCEVUnknown>(Val: LHS);
771 const SCEVUnknown *RU = cast<SCEVUnknown>(Val: RHS);
772
773 int X =
774 CompareValueComplexity(LI, LV: LU->getValue(), RV: RU->getValue(), Depth: Depth + 1);
775 return X;
776 }
777
778 case scConstant: {
779 const SCEVConstant *LC = cast<SCEVConstant>(Val: LHS);
780 const SCEVConstant *RC = cast<SCEVConstant>(Val: RHS);
781
782 // Compare constant values.
783 const APInt &LA = LC->getAPInt();
784 const APInt &RA = RC->getAPInt();
785 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
786 if (LBitWidth != RBitWidth)
787 return (int)LBitWidth - (int)RBitWidth;
788 return LA.ult(RHS: RA) ? -1 : 1;
789 }
790
791 case scVScale: {
792 const auto *LTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: LHS)->getType());
793 const auto *RTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: RHS)->getType());
794 return LTy->getBitWidth() - RTy->getBitWidth();
795 }
796
797 case scAddRecExpr: {
798 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(Val: LHS);
799 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(Val: RHS);
800
801 // There is always a dominance between two recs that are used by one SCEV,
802 // so we can safely sort recs by loop header dominance. We require such
803 // order in getAddExpr.
804 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
805 if (LLoop != RLoop) {
806 const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
807 assert(LHead != RHead && "Two loops share the same header?");
808 if (DT.dominates(A: LHead, B: RHead))
809 return 1;
810 assert(DT.dominates(RHead, LHead) &&
811 "No dominance between recurrences used by one SCEV?");
812 return -1;
813 }
814
815 [[fallthrough]];
816 }
817
818 case scTruncate:
819 case scZeroExtend:
820 case scSignExtend:
821 case scPtrToAddr:
822 case scPtrToInt:
823 case scAddExpr:
824 case scMulExpr:
825 case scUDivExpr:
826 case scSMaxExpr:
827 case scUMaxExpr:
828 case scSMinExpr:
829 case scUMinExpr:
830 case scSequentialUMinExpr: {
831 ArrayRef<SCEVUse> LOps = LHS->operands();
832 ArrayRef<SCEVUse> ROps = RHS->operands();
833
834 // Lexicographically compare n-ary-like expressions.
835 unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
836 if (LNumOps != RNumOps)
837 return (int)LNumOps - (int)RNumOps;
838
839 for (unsigned i = 0; i != LNumOps; ++i) {
840 auto X = CompareSCEVComplexity(LI, LHS: LOps[i].getPointer(),
841 RHS: ROps[i].getPointer(), DT, Depth: Depth + 1);
842 if (X != 0)
843 return X;
844 }
845 return 0;
846 }
847
848 case scCouldNotCompute:
849 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
850 }
851 llvm_unreachable("Unknown SCEV kind!");
852}
853
854/// Given a list of SCEV objects, order them by their complexity, and group
855/// objects of the same complexity together by value. When this routine is
856/// finished, we know that any duplicates in the vector are consecutive and that
857/// complexity is monotonically increasing.
858///
859/// Note that we go take special precautions to ensure that we get deterministic
860/// results from this routine. In other words, we don't want the results of
861/// this to depend on where the addresses of various SCEV objects happened to
862/// land in memory.
863static void GroupByComplexity(SmallVectorImpl<SCEVUse> &Ops, LoopInfo *LI,
864 DominatorTree &DT) {
865 if (Ops.size() < 2) return; // Noop
866
867 // Whether LHS has provably less complexity than RHS.
868 auto IsLessComplex = [&](SCEVUse LHS, SCEVUse RHS) {
869 auto Complexity = CompareSCEVComplexity(LI, LHS, RHS, DT);
870 return Complexity && *Complexity < 0;
871 };
872 if (Ops.size() == 2) {
873 // This is the common case, which also happens to be trivially simple.
874 // Special case it.
875 SCEVUse &LHS = Ops[0], &RHS = Ops[1];
876 if (IsLessComplex(RHS, LHS))
877 std::swap(a&: LHS, b&: RHS);
878 return;
879 }
880
881 // Do the rough sort by complexity.
882 llvm::stable_sort(
883 Range&: Ops, C: [&](SCEVUse LHS, SCEVUse RHS) { return IsLessComplex(LHS, RHS); });
884
885 // Now that we are sorted by complexity, group elements of the same
886 // complexity. Note that this is, at worst, N^2, but the vector is likely to
887 // be extremely short in practice. Note that we take this approach because we
888 // do not want to depend on the addresses of the objects we are grouping.
889 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
890 const SCEV *S = Ops[i];
891 unsigned Complexity = S->getSCEVType();
892
893 // If there are any objects of the same complexity and same value as this
894 // one, group them.
895 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
896 if (Ops[j] == S) { // Found a duplicate.
897 // Move it to immediately after i'th element.
898 std::swap(a&: Ops[i+1], b&: Ops[j]);
899 ++i; // no need to rescan it.
900 if (i == e-2) return; // Done!
901 }
902 }
903 }
904}
905
906/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
907/// least HugeExprThreshold nodes).
908static bool hasHugeExpression(ArrayRef<SCEVUse> Ops) {
909 return any_of(Range&: Ops, P: [](const SCEV *S) {
910 return S->getExpressionSize() >= HugeExprThreshold;
911 });
912}
913
914/// Performs a number of common optimizations on the passed \p Ops. If the
915/// whole expression reduces down to a single operand, it will be returned.
916///
917/// The following optimizations are performed:
918/// * Fold constants using the \p Fold function.
919/// * Remove identity constants satisfying \p IsIdentity.
920/// * If a constant satisfies \p IsAbsorber, return it.
921/// * Sort operands by complexity.
922template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
923static const SCEV *
924constantFoldAndGroupOps(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT,
925 SmallVectorImpl<SCEVUse> &Ops, FoldT Fold,
926 IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
927 const SCEVConstant *Folded = nullptr;
928 for (unsigned Idx = 0; Idx < Ops.size();) {
929 const SCEV *Op = Ops[Idx];
930 if (const auto *C = dyn_cast<SCEVConstant>(Val: Op)) {
931 if (!Folded)
932 Folded = C;
933 else
934 Folded = cast<SCEVConstant>(
935 SE.getConstant(Fold(Folded->getAPInt(), C->getAPInt())));
936 Ops.erase(CI: Ops.begin() + Idx);
937 continue;
938 }
939 ++Idx;
940 }
941
942 if (Ops.empty()) {
943 assert(Folded && "Must have folded value");
944 return Folded;
945 }
946
947 if (Folded && IsAbsorber(Folded->getAPInt()))
948 return Folded;
949
950 GroupByComplexity(Ops, LI: &LI, DT);
951 if (Folded && !IsIdentity(Folded->getAPInt()))
952 Ops.insert(I: Ops.begin(), Elt: Folded);
953
954 return Ops.size() == 1 ? Ops[0] : nullptr;
955}
956
957//===----------------------------------------------------------------------===//
958// Simple SCEV method implementations
959//===----------------------------------------------------------------------===//
960
961/// Compute BC(It, K). The result has width W. Assume, K > 0.
962static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
963 ScalarEvolution &SE,
964 Type *ResultTy) {
965 // Handle the simplest case efficiently.
966 if (K == 1)
967 return SE.getTruncateOrZeroExtend(V: It, Ty: ResultTy);
968
969 // We are using the following formula for BC(It, K):
970 //
971 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
972 //
973 // Suppose, W is the bitwidth of the return value. We must be prepared for
974 // overflow. Hence, we must assure that the result of our computation is
975 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
976 // safe in modular arithmetic.
977 //
978 // However, this code doesn't use exactly that formula; the formula it uses
979 // is something like the following, where T is the number of factors of 2 in
980 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
981 // exponentiation:
982 //
983 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
984 //
985 // This formula is trivially equivalent to the previous formula. However,
986 // this formula can be implemented much more efficiently. The trick is that
987 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
988 // arithmetic. To do exact division in modular arithmetic, all we have
989 // to do is multiply by the inverse. Therefore, this step can be done at
990 // width W.
991 //
992 // The next issue is how to safely do the division by 2^T. The way this
993 // is done is by doing the multiplication step at a width of at least W + T
994 // bits. This way, the bottom W+T bits of the product are accurate. Then,
995 // when we perform the division by 2^T (which is equivalent to a right shift
996 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
997 // truncated out after the division by 2^T.
998 //
999 // In comparison to just directly using the first formula, this technique
1000 // is much more efficient; using the first formula requires W * K bits,
1001 // but this formula less than W + K bits. Also, the first formula requires
1002 // a division step, whereas this formula only requires multiplies and shifts.
1003 //
1004 // It doesn't matter whether the subtraction step is done in the calculation
1005 // width or the input iteration count's width; if the subtraction overflows,
1006 // the result must be zero anyway. We prefer here to do it in the width of
1007 // the induction variable because it helps a lot for certain cases; CodeGen
1008 // isn't smart enough to ignore the overflow, which leads to much less
1009 // efficient code if the width of the subtraction is wider than the native
1010 // register width.
1011 //
1012 // (It's possible to not widen at all by pulling out factors of 2 before
1013 // the multiplication; for example, K=2 can be calculated as
1014 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
1015 // extra arithmetic, so it's not an obvious win, and it gets
1016 // much more complicated for K > 3.)
1017
1018 // Protection from insane SCEVs; this bound is conservative,
1019 // but it probably doesn't matter.
1020 if (K > 1000)
1021 return SE.getCouldNotCompute();
1022
1023 unsigned W = SE.getTypeSizeInBits(Ty: ResultTy);
1024
1025 // Calculate K! / 2^T and T; we divide out the factors of two before
1026 // multiplying for calculating K! / 2^T to avoid overflow.
1027 // Other overflow doesn't matter because we only care about the bottom
1028 // W bits of the result.
1029 APInt OddFactorial(W, 1);
1030 unsigned T = 1;
1031 for (unsigned i = 3; i <= K; ++i) {
1032 unsigned TwoFactors = countr_zero(Val: i);
1033 T += TwoFactors;
1034 OddFactorial *= (i >> TwoFactors);
1035 }
1036
1037 // We need at least W + T bits for the multiplication step
1038 unsigned CalculationBits = W + T;
1039
1040 // Calculate 2^T, at width T+W.
1041 APInt DivFactor = APInt::getOneBitSet(numBits: CalculationBits, BitNo: T);
1042
1043 // Calculate the multiplicative inverse of K! / 2^T;
1044 // this multiplication factor will perform the exact division by
1045 // K! / 2^T.
1046 APInt MultiplyFactor = OddFactorial.multiplicativeInverse();
1047
1048 // Calculate the product, at width T+W
1049 IntegerType *CalculationTy = IntegerType::get(C&: SE.getContext(),
1050 NumBits: CalculationBits);
1051 const SCEV *Dividend = SE.getTruncateOrZeroExtend(V: It, Ty: CalculationTy);
1052 for (unsigned i = 1; i != K; ++i) {
1053 const SCEV *S = SE.getMinusSCEV(LHS: It, RHS: SE.getConstant(Ty: It->getType(), V: i));
1054 Dividend = SE.getMulExpr(LHS: Dividend,
1055 RHS: SE.getTruncateOrZeroExtend(V: S, Ty: CalculationTy));
1056 }
1057
1058 // Divide by 2^T
1059 const SCEV *DivResult = SE.getUDivExpr(LHS: Dividend, RHS: SE.getConstant(Val: DivFactor));
1060
1061 // Truncate the result, and divide by K! / 2^T.
1062
1063 return SE.getMulExpr(LHS: SE.getConstant(Val: MultiplyFactor),
1064 RHS: SE.getTruncateOrZeroExtend(V: DivResult, Ty: ResultTy));
1065}
1066
1067/// Return the value of this chain of recurrences at the specified iteration
1068/// number. We can evaluate this recurrence by multiplying each element in the
1069/// chain by the binomial coefficient corresponding to it. In other words, we
1070/// can evaluate {A,+,B,+,C,+,D} as:
1071///
1072/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
1073///
1074/// where BC(It, k) stands for binomial coefficient.
1075const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
1076 ScalarEvolution &SE) const {
1077 return evaluateAtIteration(Operands: operands(), It, SE);
1078}
1079
1080const SCEV *SCEVAddRecExpr::evaluateAtIteration(ArrayRef<SCEVUse> Operands,
1081 const SCEV *It,
1082 ScalarEvolution &SE) {
1083 assert(Operands.size() > 0);
1084 const SCEV *Result = Operands[0].getPointer();
1085 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1086 // The computation is correct in the face of overflow provided that the
1087 // multiplication is performed _after_ the evaluation of the binomial
1088 // coefficient.
1089 const SCEV *Coeff = BinomialCoefficient(It, K: i, SE, ResultTy: Result->getType());
1090 if (isa<SCEVCouldNotCompute>(Val: Coeff))
1091 return Coeff;
1092
1093 Result =
1094 SE.getAddExpr(LHS: Result, RHS: SE.getMulExpr(LHS: Operands[i].getPointer(), RHS: Coeff));
1095 }
1096 return Result;
1097}
1098
1099//===----------------------------------------------------------------------===//
1100// SCEV Expression folder implementations
1101//===----------------------------------------------------------------------===//
1102
1103/// The SCEVCastSinkingRewriter takes a scalar evolution expression,
1104/// which computes a pointer-typed value, and rewrites the whole expression
1105/// tree so that *all* the computations are done on integers, and the only
1106/// pointer-typed operands in the expression are SCEVUnknown.
1107/// The CreatePtrCast callback is invoked to create the actual conversion
1108/// (ptrtoint or ptrtoaddr) at the SCEVUnknown leaves.
1109class SCEVCastSinkingRewriter
1110 : public SCEVRewriteVisitor<SCEVCastSinkingRewriter> {
1111 using Base = SCEVRewriteVisitor<SCEVCastSinkingRewriter>;
1112 using ConversionFn = function_ref<const SCEV *(const SCEVUnknown *)>;
1113 Type *TargetTy;
1114 ConversionFn CreatePtrCast;
1115
1116public:
1117 SCEVCastSinkingRewriter(ScalarEvolution &SE, Type *TargetTy,
1118 ConversionFn CreatePtrCast)
1119 : Base(SE), TargetTy(TargetTy), CreatePtrCast(std::move(CreatePtrCast)) {}
1120
1121 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE,
1122 Type *TargetTy, ConversionFn CreatePtrCast) {
1123 SCEVCastSinkingRewriter Rewriter(SE, TargetTy, std::move(CreatePtrCast));
1124 return Rewriter.visit(S: Scev);
1125 }
1126
1127 const SCEV *visit(const SCEV *S) {
1128 Type *STy = S->getType();
1129 // If the expression is not pointer-typed, just keep it as-is.
1130 if (!STy->isPointerTy())
1131 return S;
1132 // Else, recursively sink the cast down into it.
1133 return Base::visit(S);
1134 }
1135
1136 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
1137 // Preserve wrap flags on rewritten SCEVAddExpr, which the default
1138 // implementation drops.
1139 SmallVector<SCEVUse, 2> Operands;
1140 bool Changed = false;
1141 for (SCEVUse Op : Expr->operands()) {
1142 Operands.push_back(Elt: visit(S: Op.getPointer()));
1143 Changed |= Op.getPointer() != Operands.back();
1144 }
1145 return !Changed ? Expr : SE.getAddExpr(Ops&: Operands, Flags: Expr->getNoWrapFlags());
1146 }
1147
1148 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
1149 SmallVector<SCEVUse, 2> Operands;
1150 bool Changed = false;
1151 for (SCEVUse Op : Expr->operands()) {
1152 Operands.push_back(Elt: visit(S: Op.getPointer()));
1153 Changed |= Op.getPointer() != Operands.back();
1154 }
1155 return !Changed ? Expr : SE.getMulExpr(Ops&: Operands, Flags: Expr->getNoWrapFlags());
1156 }
1157
1158 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
1159 assert(Expr->getType()->isPointerTy() &&
1160 "Should only reach pointer-typed SCEVUnknown's.");
1161 // Perform some basic constant folding. If the operand of the cast is a
1162 // null pointer, don't create a cast SCEV expression (that will be left
1163 // as-is), but produce a zero constant.
1164 if (isa<ConstantPointerNull>(Val: Expr->getValue()))
1165 return SE.getZero(Ty: TargetTy);
1166 return CreatePtrCast(Expr);
1167 }
1168};
1169
1170const SCEV *ScalarEvolution::getLosslessPtrToIntExpr(const SCEV *Op) {
1171 assert(Op->getType()->isPointerTy() && "Op must be a pointer");
1172
1173 // It isn't legal for optimizations to construct new ptrtoint expressions
1174 // for non-integral pointers.
1175 if (getDataLayout().isNonIntegralPointerType(Ty: Op->getType()))
1176 return getCouldNotCompute();
1177
1178 Type *IntPtrTy = getDataLayout().getIntPtrType(Op->getType());
1179
1180 // We can only trivially model ptrtoint if SCEV's effective (integer) type
1181 // is sufficiently wide to represent all possible pointer values.
1182 // We could theoretically teach SCEV to truncate wider pointers, but
1183 // that isn't implemented for now.
1184 if (getDataLayout().getTypeSizeInBits(Ty: getEffectiveSCEVType(Ty: Op->getType())) !=
1185 getDataLayout().getTypeSizeInBits(Ty: IntPtrTy))
1186 return getCouldNotCompute();
1187
1188 // Use the rewriter to sink the cast down to SCEVUnknown leaves.
1189 const SCEV *IntOp = SCEVCastSinkingRewriter::rewrite(
1190 Scev: Op, SE&: *this, TargetTy: IntPtrTy, CreatePtrCast: [this, IntPtrTy](const SCEVUnknown *U) {
1191 FoldingSetNodeID ID;
1192 ID.AddInteger(I: scPtrToInt);
1193 ID.AddPointer(Ptr: U);
1194 void *IP = nullptr;
1195 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
1196 return S;
1197 SCEV *S = new (SCEVAllocator)
1198 SCEVPtrToIntExpr(ID.Intern(Allocator&: SCEVAllocator), U, IntPtrTy);
1199 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1200 S->computeAndSetCanonical(SE&: *this);
1201 registerUser(User: S, Ops: U);
1202 return static_cast<const SCEV *>(S);
1203 });
1204 assert(IntOp->getType()->isIntegerTy() &&
1205 "We must have succeeded in sinking the cast, "
1206 "and ending up with an integer-typed expression!");
1207 return IntOp;
1208}
1209
1210const SCEV *ScalarEvolution::getPtrToAddrExpr(const SCEV *Op) {
1211 assert(Op->getType()->isPointerTy() && "Op must be a pointer");
1212
1213 // Treat pointers with unstable representation conservatively, since the
1214 // address bits may change.
1215 if (DL.hasUnstableRepresentation(Ty: Op->getType()))
1216 return getCouldNotCompute();
1217
1218 Type *Ty = DL.getAddressType(PtrTy: Op->getType());
1219
1220 // Use the rewriter to sink the cast down to SCEVUnknown leaves.
1221 // The rewriter handles null pointer constant folding.
1222 const SCEV *IntOp = SCEVCastSinkingRewriter::rewrite(
1223 Scev: Op, SE&: *this, TargetTy: Ty, CreatePtrCast: [this, Ty](const SCEVUnknown *U) {
1224 FoldingSetNodeID ID;
1225 ID.AddInteger(I: scPtrToAddr);
1226 ID.AddPointer(Ptr: U);
1227 ID.AddPointer(Ptr: Ty);
1228 void *IP = nullptr;
1229 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
1230 return S;
1231 SCEV *S = new (SCEVAllocator)
1232 SCEVPtrToAddrExpr(ID.Intern(Allocator&: SCEVAllocator), U, Ty);
1233 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1234 S->computeAndSetCanonical(SE&: *this);
1235 registerUser(User: S, Ops: U);
1236 return static_cast<const SCEV *>(S);
1237 });
1238 assert(IntOp->getType()->isIntegerTy() &&
1239 "We must have succeeded in sinking the cast, "
1240 "and ending up with an integer-typed expression!");
1241 return IntOp;
1242}
1243
1244const SCEV *ScalarEvolution::getPtrToIntExpr(const SCEV *Op, Type *Ty) {
1245 assert(Ty->isIntegerTy() && "Target type must be an integer type!");
1246
1247 const SCEV *IntOp = getLosslessPtrToIntExpr(Op);
1248 if (isa<SCEVCouldNotCompute>(Val: IntOp))
1249 return IntOp;
1250
1251 return getTruncateOrZeroExtend(V: IntOp, Ty);
1252}
1253
1254const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, Type *Ty,
1255 unsigned Depth) {
1256 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
1257 "This is not a truncating conversion!");
1258 assert(isSCEVable(Ty) &&
1259 "This is not a conversion to a SCEVable type!");
1260 assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!");
1261 Ty = getEffectiveSCEVType(Ty);
1262
1263 FoldingSetNodeID ID;
1264 ID.AddInteger(I: scTruncate);
1265 ID.AddPointer(Ptr: Op);
1266 ID.AddPointer(Ptr: Ty);
1267 void *IP = nullptr;
1268 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
1269
1270 // Fold if the operand is constant.
1271 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
1272 return getConstant(
1273 V: cast<ConstantInt>(Val: ConstantExpr::getTrunc(C: SC->getValue(), Ty)));
1274
1275 // trunc(trunc(x)) --> trunc(x)
1276 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val: Op))
1277 return getTruncateExpr(Op: ST->getOperand(), Ty, Depth: Depth + 1);
1278
1279 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
1280 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val: Op))
1281 return getTruncateOrSignExtend(V: SS->getOperand(), Ty, Depth: Depth + 1);
1282
1283 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
1284 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val: Op))
1285 return getTruncateOrZeroExtend(V: SZ->getOperand(), Ty, Depth: Depth + 1);
1286
1287 if (Depth > MaxCastDepth) {
1288 SCEV *S =
1289 new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator), Op, Ty);
1290 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1291 S->computeAndSetCanonical(SE&: *this);
1292 registerUser(User: S, Ops: Op);
1293 return S;
1294 }
1295
1296 // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
1297 // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
1298 // if after transforming we have at most one truncate, not counting truncates
1299 // that replace other casts.
1300 if (isa<SCEVAddExpr>(Val: Op) || isa<SCEVMulExpr>(Val: Op)) {
1301 auto *CommOp = cast<SCEVCommutativeExpr>(Val: Op);
1302 SmallVector<SCEVUse, 4> Operands;
1303 unsigned numTruncs = 0;
1304 for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1305 ++i) {
1306 const SCEV *S = getTruncateExpr(Op: CommOp->getOperand(i), Ty, Depth: Depth + 1);
1307 if (!isa<SCEVIntegralCastExpr>(Val: CommOp->getOperand(i)) &&
1308 isa<SCEVTruncateExpr>(Val: S))
1309 numTruncs++;
1310 Operands.push_back(Elt: S);
1311 }
1312 if (numTruncs < 2) {
1313 if (isa<SCEVAddExpr>(Val: Op))
1314 return getAddExpr(Ops&: Operands);
1315 if (isa<SCEVMulExpr>(Val: Op))
1316 return getMulExpr(Ops&: Operands);
1317 llvm_unreachable("Unexpected SCEV type for Op.");
1318 }
1319 // Although we checked in the beginning that ID is not in the cache, it is
1320 // possible that during recursion and different modification ID was inserted
1321 // into the cache. So if we find it, just return it.
1322 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
1323 return S;
1324 }
1325
1326 // If the input value is a chrec scev, truncate the chrec's operands.
1327 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val: Op)) {
1328 SmallVector<SCEVUse, 4> Operands;
1329 for (const SCEV *Op : AddRec->operands())
1330 Operands.push_back(Elt: getTruncateExpr(Op, Ty, Depth: Depth + 1));
1331 return getAddRecExpr(Operands, L: AddRec->getLoop(), Flags: SCEV::FlagAnyWrap);
1332 }
1333
1334 // Return zero if truncating to known zeros.
1335 uint32_t MinTrailingZeros = getMinTrailingZeros(S: Op);
1336 if (MinTrailingZeros >= getTypeSizeInBits(Ty))
1337 return getZero(Ty);
1338
1339 // The cast wasn't folded; create an explicit cast node. We can reuse
1340 // the existing insert position since if we get here, we won't have
1341 // made any changes which would invalidate it.
1342 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator),
1343 Op, Ty);
1344 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1345 S->computeAndSetCanonical(SE&: *this);
1346 registerUser(User: S, Ops: Op);
1347 return S;
1348}
1349
1350// Get the limit of a recurrence such that incrementing by Step cannot cause
1351// signed overflow as long as the value of the recurrence within the
1352// loop does not exceed this limit before incrementing.
1353static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
1354 ICmpInst::Predicate *Pred,
1355 ScalarEvolution *SE) {
1356 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1357 if (SE->isKnownPositive(S: Step)) {
1358 *Pred = ICmpInst::ICMP_SLT;
1359 return SE->getConstant(Val: APInt::getSignedMinValue(numBits: BitWidth) -
1360 SE->getSignedRangeMax(S: Step));
1361 }
1362 if (SE->isKnownNegative(S: Step)) {
1363 *Pred = ICmpInst::ICMP_SGT;
1364 return SE->getConstant(Val: APInt::getSignedMaxValue(numBits: BitWidth) -
1365 SE->getSignedRangeMin(S: Step));
1366 }
1367 return nullptr;
1368}
1369
1370// Get the limit of a recurrence such that incrementing by Step cannot cause
1371// unsigned overflow as long as the value of the recurrence within the loop does
1372// not exceed this limit before incrementing.
1373static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step,
1374 ICmpInst::Predicate *Pred,
1375 ScalarEvolution *SE) {
1376 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1377 *Pred = ICmpInst::ICMP_ULT;
1378
1379 return SE->getConstant(Val: APInt::getMinValue(numBits: BitWidth) -
1380 SE->getUnsignedRangeMax(S: Step));
1381}
1382
1383namespace {
1384
1385struct ExtendOpTraitsBase {
1386 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(const SCEV *, Type *,
1387 unsigned);
1388};
1389
1390// Used to make code generic over signed and unsigned overflow.
1391template <typename ExtendOp> struct ExtendOpTraits {
1392 // Members present:
1393 //
1394 // static const SCEV::NoWrapFlags WrapType;
1395 //
1396 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
1397 //
1398 // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1399 // ICmpInst::Predicate *Pred,
1400 // ScalarEvolution *SE);
1401};
1402
1403template <>
1404struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
1405 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW;
1406
1407 static const GetExtendExprTy GetExtendExpr;
1408
1409 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1410 ICmpInst::Predicate *Pred,
1411 ScalarEvolution *SE) {
1412 return getSignedOverflowLimitForStep(Step, Pred, SE);
1413 }
1414};
1415
1416const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1417 SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr;
1418
1419template <>
1420struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
1421 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW;
1422
1423 static const GetExtendExprTy GetExtendExpr;
1424
1425 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1426 ICmpInst::Predicate *Pred,
1427 ScalarEvolution *SE) {
1428 return getUnsignedOverflowLimitForStep(Step, Pred, SE);
1429 }
1430};
1431
1432const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1433 SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr;
1434
1435} // end anonymous namespace
1436
1437// The recurrence AR has been shown to have no signed/unsigned wrap or something
1438// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
1439// easily prove NSW/NUW for its preincrement or postincrement sibling. This
1440// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
1441// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
1442// expression "Step + sext/zext(PreIncAR)" is congruent with
1443// "sext/zext(PostIncAR)"
1444template <typename ExtendOpTy>
1445static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty,
1446 ScalarEvolution *SE, unsigned Depth) {
1447 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1448 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1449
1450 const Loop *L = AR->getLoop();
1451 const SCEV *Start = AR->getStart();
1452 const SCEV *Step = AR->getStepRecurrence(SE&: *SE);
1453
1454 // Check for a simple looking step prior to loop entry.
1455 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Val: Start);
1456 if (!SA)
1457 return nullptr;
1458
1459 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
1460 // subtraction is expensive. For this purpose, perform a quick and dirty
1461 // difference, by checking for Step in the operand list. Note, that
1462 // SA might have repeated ops, like %a + %a + ..., so only remove one.
1463 SmallVector<SCEVUse, 4> DiffOps(SA->operands());
1464 for (auto It = DiffOps.begin(); It != DiffOps.end(); ++It)
1465 if (*It == Step) {
1466 DiffOps.erase(CI: It);
1467 break;
1468 }
1469
1470 if (DiffOps.size() == SA->getNumOperands())
1471 return nullptr;
1472
1473 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
1474 // `Step`:
1475
1476 // 1. NSW/NUW flags on the step increment.
1477 auto PreStartFlags =
1478 ScalarEvolution::maskFlags(Flags: SA->getNoWrapFlags(), Mask: SCEV::FlagNUW);
1479 const SCEV *PreStart = SE->getAddExpr(Ops&: DiffOps, Flags: PreStartFlags);
1480 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
1481 Val: SE->getAddRecExpr(Start: PreStart, Step, L, Flags: SCEV::FlagAnyWrap));
1482
1483 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
1484 // "S+X does not sign/unsign-overflow".
1485 //
1486
1487 const SCEV *BECount = SE->getBackedgeTakenCount(L);
1488 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType)) &&
1489 !isa<SCEVCouldNotCompute>(Val: BECount) && SE->isKnownPositive(S: BECount))
1490 return PreStart;
1491
1492 // 2. Direct overflow check on the step operation's expression.
1493 unsigned BitWidth = SE->getTypeSizeInBits(Ty: AR->getType());
1494 Type *WideTy = IntegerType::get(C&: SE->getContext(), NumBits: BitWidth * 2);
1495 const SCEV *OperandExtendedStart =
1496 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
1497 (SE->*GetExtendExpr)(Step, WideTy, Depth));
1498 if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
1499 if (PreAR && any(AR->getNoWrapFlags(Mask: WrapType))) {
1500 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
1501 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
1502 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
1503 SE->setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(PreAR), Flags: WrapType);
1504 }
1505 return PreStart;
1506 }
1507
1508 // 3. Loop precondition.
1509 ICmpInst::Predicate Pred;
1510 const SCEV *OverflowLimit =
1511 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1512
1513 if (OverflowLimit &&
1514 SE->isLoopEntryGuardedByCond(L, Pred, LHS: PreStart, RHS: OverflowLimit))
1515 return PreStart;
1516
1517 return nullptr;
1518}
1519
1520// Get the normalized zero or sign extended expression for this AddRec's Start.
1521template <typename ExtendOpTy>
1522static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
1523 ScalarEvolution *SE,
1524 unsigned Depth) {
1525 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1526
1527 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, Ty, SE, Depth);
1528 if (!PreStart)
1529 return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
1530
1531 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(SE&: *SE), Ty,
1532 Depth),
1533 (SE->*GetExtendExpr)(PreStart, Ty, Depth));
1534}
1535
1536// Try to prove away overflow by looking at "nearby" add recurrences. A
1537// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
1538// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
1539//
1540// Formally:
1541//
1542// {S,+,X} == {S-T,+,X} + T
1543// => Ext({S,+,X}) == Ext({S-T,+,X} + T)
1544//
1545// If ({S-T,+,X} + T) does not overflow ... (1)
1546//
1547// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
1548//
1549// If {S-T,+,X} does not overflow ... (2)
1550//
1551// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
1552// == {Ext(S-T)+Ext(T),+,Ext(X)}
1553//
1554// If (S-T)+T does not overflow ... (3)
1555//
1556// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
1557// == {Ext(S),+,Ext(X)} == LHS
1558//
1559// Thus, if (1), (2) and (3) are true for some T, then
1560// Ext({S,+,X}) == {Ext(S),+,Ext(X)}
1561//
1562// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
1563// does not overflow" restricted to the 0th iteration. Therefore we only need
1564// to check for (1) and (2).
1565//
1566// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
1567// is `Delta` (defined below).
1568template <typename ExtendOpTy>
1569bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
1570 const SCEV *Step,
1571 const Loop *L) {
1572 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1573
1574 // We restrict `Start` to a constant to prevent SCEV from spending too much
1575 // time here. It is correct (but more expensive) to continue with a
1576 // non-constant `Start` and do a general SCEV subtraction to compute
1577 // `PreStart` below.
1578 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Val: Start);
1579 if (!StartC)
1580 return false;
1581
1582 APInt StartAI = StartC->getAPInt();
1583
1584 for (unsigned Delta : {-2, -1, 1, 2}) {
1585 const SCEV *PreStart = getConstant(Val: StartAI - Delta);
1586
1587 FoldingSetNodeID ID;
1588 ID.AddInteger(I: scAddRecExpr);
1589 ID.AddPointer(Ptr: PreStart);
1590 ID.AddPointer(Ptr: Step);
1591 ID.AddPointer(Ptr: L);
1592 void *IP = nullptr;
1593 const auto *PreAR =
1594 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
1595
1596 // Give up if we don't already have the add recurrence we need because
1597 // actually constructing an add recurrence is relatively expensive.
1598 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType))) { // proves (2)
1599 const SCEV *DeltaS = getConstant(Ty: StartC->getType(), V: Delta);
1600 ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
1601 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1602 DeltaS, &Pred, this);
1603 if (Limit && isKnownPredicate(Pred, LHS: PreAR, RHS: Limit)) // proves (1)
1604 return true;
1605 }
1606 }
1607
1608 return false;
1609}
1610
1611// Finds an integer D for an expression (C + x + y + ...) such that the top
1612// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
1613// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
1614// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
1615// the (C + x + y + ...) expression is \p WholeAddExpr.
1616static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1617 const SCEVConstant *ConstantTerm,
1618 const SCEVAddExpr *WholeAddExpr) {
1619 const APInt &C = ConstantTerm->getAPInt();
1620 const unsigned BitWidth = C.getBitWidth();
1621 // Find number of trailing zeros of (x + y + ...) w/o the C first:
1622 uint32_t TZ = BitWidth;
1623 for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
1624 TZ = std::min(a: TZ, b: SE.getMinTrailingZeros(S: WholeAddExpr->getOperand(i: I)));
1625 if (TZ) {
1626 // Set D to be as many least significant bits of C as possible while still
1627 // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
1628 return TZ < BitWidth ? C.trunc(width: TZ).zext(width: BitWidth) : C;
1629 }
1630 return APInt(BitWidth, 0);
1631}
1632
1633// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
1634// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
1635// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
1636// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
1637static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1638 const APInt &ConstantStart,
1639 const SCEV *Step) {
1640 const unsigned BitWidth = ConstantStart.getBitWidth();
1641 const uint32_t TZ = SE.getMinTrailingZeros(S: Step);
1642 if (TZ)
1643 return TZ < BitWidth ? ConstantStart.trunc(width: TZ).zext(width: BitWidth)
1644 : ConstantStart;
1645 return APInt(BitWidth, 0);
1646}
1647
1648static void insertFoldCacheEntry(
1649 const ScalarEvolution::FoldID &ID, const SCEV *S,
1650 DenseMap<ScalarEvolution::FoldID, const SCEV *> &FoldCache,
1651 DenseMap<const SCEV *, SmallVector<ScalarEvolution::FoldID, 2>>
1652 &FoldCacheUser) {
1653 auto I = FoldCache.insert(KV: {ID, S});
1654 if (!I.second) {
1655 // Remove FoldCacheUser entry for ID when replacing an existing FoldCache
1656 // entry.
1657 auto &UserIDs = FoldCacheUser[I.first->second];
1658 assert(count(UserIDs, ID) == 1 && "unexpected duplicates in UserIDs");
1659 for (unsigned I = 0; I != UserIDs.size(); ++I)
1660 if (UserIDs[I] == ID) {
1661 std::swap(a&: UserIDs[I], b&: UserIDs.back());
1662 break;
1663 }
1664 UserIDs.pop_back();
1665 I.first->second = S;
1666 }
1667 FoldCacheUser[S].push_back(Elt: ID);
1668}
1669
1670const SCEV *
1671ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) {
1672 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1673 "This is not an extending conversion!");
1674 assert(isSCEVable(Ty) &&
1675 "This is not a conversion to a SCEVable type!");
1676 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1677 Ty = getEffectiveSCEVType(Ty);
1678
1679 FoldID ID(scZeroExtend, Op, Ty);
1680 if (const SCEV *S = FoldCache.lookup(Val: ID))
1681 return S;
1682
1683 const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
1684 if (!isa<SCEVZeroExtendExpr>(Val: S))
1685 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1686 return S;
1687}
1688
1689const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
1690 unsigned Depth) {
1691 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1692 "This is not an extending conversion!");
1693 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1694 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1695
1696 // Fold if the operand is constant.
1697 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
1698 return getConstant(Val: SC->getAPInt().zext(width: getTypeSizeInBits(Ty)));
1699
1700 // zext(zext(x)) --> zext(x)
1701 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val: Op))
1702 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1703
1704 // If the operand is an affine AddRec with the no-unsigned-wrap flag, the
1705 // zero-extension distributes over the recurrence.
1706 const SCEV *Start, *Step;
1707 const Loop *L;
1708 if (Depth <= MaxCastDepth &&
1709 match(S: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1710 const auto *AR = cast<SCEVAddRecExpr>(Val: Op);
1711 if (AR->hasNoUnsignedWrap()) {
1712 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1713 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1714 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1715 }
1716 }
1717
1718 // Before doing any expensive analysis, check to see if we've already
1719 // computed a SCEV for this Op and Ty.
1720 FoldingSetNodeID ID;
1721 ID.AddInteger(I: scZeroExtend);
1722 ID.AddPointer(Ptr: Op);
1723 ID.AddPointer(Ptr: Ty);
1724 void *IP = nullptr;
1725 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
1726 if (Depth > MaxCastDepth) {
1727 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1728 Op, Ty);
1729 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1730 S->computeAndSetCanonical(SE&: *this);
1731 registerUser(User: S, Ops: Op);
1732 return S;
1733 }
1734
1735 // zext(trunc(x)) --> zext(x) or x or trunc(x)
1736 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
1737 // It's possible the bits taken off by the truncate were all zero bits. If
1738 // so, we should be able to simplify this further.
1739 const SCEV *X = ST->getOperand();
1740 ConstantRange CR = getUnsignedRange(S: X);
1741 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1742 unsigned NewBits = getTypeSizeInBits(Ty);
1743 if (CR.truncate(BitWidth: TruncBits).zeroExtend(BitWidth: NewBits).contains(
1744 CR: CR.zextOrTrunc(BitWidth: NewBits)))
1745 return getTruncateOrZeroExtend(V: X, Ty, Depth);
1746 }
1747
1748 // If the input value is a chrec scev, and we can prove that the value
1749 // did not overflow the old, smaller, value, we can zero extend all of the
1750 // operands (often constants). This allows analysis of something like
1751 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
1752 if (match(S: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1753 const auto *AR = cast<SCEVAddRecExpr>(Val: Op);
1754 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1755
1756 // The no-unsigned-wrap case is handled before the uniquing lookup above.
1757
1758 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1759 // Note that this serves two purposes: It filters out loops that are
1760 // simply not analyzable, and it covers the case where this code is
1761 // being called from within backedge-taken count analysis, such that
1762 // attempting to ask for the backedge-taken count would likely result
1763 // in infinite recursion. In the later case, the analysis code will
1764 // cope with a conservative value, and it will take care to purge
1765 // that value once it has finished.
1766 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1767 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
1768 // Manually compute the final value for AR, checking for overflow.
1769
1770 // Check whether the backedge-taken count can be losslessly casted to
1771 // the addrec's type. The count is always unsigned.
1772 const SCEV *CastedMaxBECount =
1773 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
1774 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1775 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
1776 if (MaxBECount == RecastedMaxBECount) {
1777 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
1778 // Check whether Start+Step*MaxBECount has no unsigned overflow.
1779 const SCEV *ZMul =
1780 getMulExpr(LHS: CastedMaxBECount, RHS: Step, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1781 const SCEV *ZAdd = getZeroExtendExpr(
1782 Op: getAddExpr(LHS: Start, RHS: ZMul, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1), Ty: WideTy,
1783 Depth: Depth + 1);
1784 const SCEV *WideStart = getZeroExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
1785 const SCEV *WideMaxBECount =
1786 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
1787 const SCEV *OperandExtendedAdd =
1788 getAddExpr(LHS: WideStart,
1789 RHS: getMulExpr(LHS: WideMaxBECount,
1790 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1791 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
1792 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1793 if (ZAdd == OperandExtendedAdd) {
1794 // Cache knowledge of AR NUW, which is propagated to this AddRec.
1795 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1796 // Return the expression with the addrec on the outside.
1797 Start =
1798 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1799 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1800 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1801 }
1802 // Similar to above, only this time treat the step value as signed.
1803 // This covers loops that count down.
1804 OperandExtendedAdd =
1805 getAddExpr(LHS: WideStart,
1806 RHS: getMulExpr(LHS: WideMaxBECount,
1807 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1808 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
1809 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1810 if (ZAdd == OperandExtendedAdd) {
1811 // Cache knowledge of AR NW, which is propagated to this AddRec.
1812 // Negative step causes unsigned wrap, but it still can't self-wrap.
1813 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1814 // Return the expression with the addrec on the outside.
1815 Start =
1816 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1817 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1818 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1819 }
1820 }
1821 }
1822
1823 // Normally, in the cases we can prove no-overflow via a
1824 // backedge guarding condition, we can also compute a backedge
1825 // taken count for the loop. The exceptions are assumptions and
1826 // guards present in the loop -- SCEV is not great at exploiting
1827 // these to compute max backedge taken counts, but can still use
1828 // these to prove lack of overflow. Use this fact to avoid
1829 // doing extra work that may not pay off.
1830 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount) || HasGuards ||
1831 !AC.assumptions().empty()) {
1832
1833 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1834 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
1835 if (AR->hasNoUnsignedWrap()) {
1836 // Same as nuw case above - duplicated here to avoid a compile time
1837 // issue. It's not clear that the order of checks does matter, but
1838 // it's one of two issue possible causes for a change which was
1839 // reverted. Be conservative for the moment.
1840 Start =
1841 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1842 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1843 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1844 }
1845
1846 // For a negative step, we can extend the operands iff doing so only
1847 // traverses values in the range zext([0,UINT_MAX]).
1848 if (isKnownNegative(S: Step)) {
1849 const SCEV *N =
1850 getConstant(Val: APInt::getMaxValue(numBits: BitWidth) - getSignedRangeMin(S: Step));
1851 if (isLoopBackedgeGuardedByCond(L, Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N) ||
1852 isKnownOnEveryIteration(Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N)) {
1853 // Cache knowledge of AR NW, which is propagated to this
1854 // AddRec. Negative step causes unsigned wrap, but it
1855 // still can't self-wrap.
1856 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1857 // Return the expression with the addrec on the outside.
1858 Start =
1859 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1860 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1861 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1862 }
1863 }
1864 }
1865
1866 // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
1867 // if D + (C - D + Step * n) could be proven to not unsigned wrap
1868 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1869 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
1870 const APInt &C = SC->getAPInt();
1871 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
1872 if (D != 0) {
1873 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1874 const SCEV *SResidual =
1875 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
1876 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1877 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: SCEV::FlagNSW | SCEV::FlagNUW,
1878 Depth: Depth + 1);
1879 }
1880 }
1881
1882 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1883 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1884 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1885 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1886 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1887 }
1888 }
1889
1890 // zext(A % B) --> zext(A) % zext(B)
1891 {
1892 const SCEV *LHS;
1893 const SCEV *RHS;
1894 if (match(S: Op, P: m_scev_URem(LHS: m_SCEV(V&: LHS), RHS: m_SCEV(V&: RHS), SE&: *this)))
1895 return getURemExpr(LHS: getZeroExtendExpr(Op: LHS, Ty, Depth: Depth + 1),
1896 RHS: getZeroExtendExpr(Op: RHS, Ty, Depth: Depth + 1));
1897 }
1898
1899 // zext(A / B) --> zext(A) / zext(B).
1900 if (auto *Div = dyn_cast<SCEVUDivExpr>(Val: Op))
1901 return getUDivExpr(LHS: getZeroExtendExpr(Op: Div->getLHS(), Ty, Depth: Depth + 1),
1902 RHS: getZeroExtendExpr(Op: Div->getRHS(), Ty, Depth: Depth + 1));
1903
1904 if (auto *SA = dyn_cast<SCEVAddExpr>(Val: Op)) {
1905 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
1906 if (SA->hasNoUnsignedWrap()) {
1907 // If the addition does not unsign overflow then we can, by definition,
1908 // commute the zero extension with the addition operation.
1909 SmallVector<SCEVUse, 4> Ops;
1910 for (SCEVUse Op : SA->operands())
1911 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1912 return getAddExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1913 }
1914
1915 const APInt *C, *C2;
1916 // zext (C + A)<nsw> -> (sext(C) + sext(A))<nsw> if zext (C + A)<nsw> >=s 0.
1917 // Currently the non-negative check is done manually, as isKnownNonNegative
1918 // is too expensive.
1919 if (SA->hasNoSignedWrap() &&
1920 match(V: SA, P: m_scev_Add(Op0: m_scev_APInt(C),
1921 Op1: m_scev_SMax(Op0: m_scev_APInt(C&: C2), Op1: m_SCEV()))) &&
1922 C->isNegative() && !C->isMinSignedValue() && C2->sge(RHS: C->abs())) {
1923 assert(isKnownNonNegative(SA) && "incorrectly determined non-negative");
1924 return getAddExpr(LHS: getSignExtendExpr(Op: SA->getOperand(i: 0), Ty, Depth: Depth + 1),
1925 RHS: getSignExtendExpr(Op: SA->getOperand(i: 1), Ty, Depth: Depth + 1),
1926 Flags: SCEV::FlagNSW, Depth: Depth + 1);
1927 }
1928
1929 // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
1930 // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
1931 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1932 //
1933 // Often address arithmetics contain expressions like
1934 // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
1935 // This transformation is useful while proving that such expressions are
1936 // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
1937 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1938 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1939 if (D != 0) {
1940 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1941 const SCEV *SResidual =
1942 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagAnyWrap, Depth);
1943 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1944 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1945 Depth: Depth + 1);
1946 }
1947 }
1948 }
1949
1950 if (auto *SM = dyn_cast<SCEVMulExpr>(Val: Op)) {
1951 // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
1952 if (SM->hasNoUnsignedWrap()) {
1953 // If the multiply does not unsign overflow then we can, by definition,
1954 // commute the zero extension with the multiply operation.
1955 SmallVector<SCEVUse, 4> Ops;
1956 for (SCEVUse Op : SM->operands())
1957 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1958 return getMulExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1959 }
1960
1961 // zext(2^K * (trunc X to iN)) to iM ->
1962 // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
1963 //
1964 // Proof:
1965 //
1966 // zext(2^K * (trunc X to iN)) to iM
1967 // = zext((trunc X to iN) << K) to iM
1968 // = zext((trunc X to i{N-K}) << K)<nuw> to iM
1969 // (because shl removes the top K bits)
1970 // = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
1971 // = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
1972 //
1973 const APInt *C;
1974 const SCEV *TruncRHS;
1975 if (match(V: SM,
1976 P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_scev_Trunc(Op0: m_SCEV(V&: TruncRHS)))) &&
1977 C->isPowerOf2()) {
1978 int NewTruncBits =
1979 getTypeSizeInBits(Ty: SM->getOperand(i: 1)->getType()) - C->logBase2();
1980 Type *NewTruncTy = IntegerType::get(C&: getContext(), NumBits: NewTruncBits);
1981 return getMulExpr(
1982 LHS: getZeroExtendExpr(Op: SM->getOperand(i: 0), Ty),
1983 RHS: getZeroExtendExpr(Op: getTruncateExpr(Op: TruncRHS, Ty: NewTruncTy), Ty),
1984 Flags: SCEV::FlagNUW, Depth: Depth + 1);
1985 }
1986 }
1987
1988 // zext(umin(x, y)) -> umin(zext(x), zext(y))
1989 // zext(umax(x, y)) -> umax(zext(x), zext(y))
1990 if (isa<SCEVUMinExpr>(Val: Op) || isa<SCEVUMaxExpr>(Val: Op)) {
1991 auto *MinMax = cast<SCEVMinMaxExpr>(Val: Op);
1992 SmallVector<SCEVUse, 4> Operands;
1993 for (SCEVUse Operand : MinMax->operands())
1994 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1995 if (isa<SCEVUMinExpr>(Val: MinMax))
1996 return getUMinExpr(Operands);
1997 return getUMaxExpr(Operands);
1998 }
1999
2000 // zext(umin_seq(x, y)) -> umin_seq(zext(x), zext(y))
2001 if (auto *MinMax = dyn_cast<SCEVSequentialMinMaxExpr>(Val: Op)) {
2002 assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
2003 SmallVector<SCEVUse, 4> Operands;
2004 for (SCEVUse Operand : MinMax->operands())
2005 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
2006 return getUMinExpr(Operands, /*Sequential*/ true);
2007 }
2008
2009 // The cast wasn't folded; create an explicit cast node.
2010 // Recompute the insert position, as it may have been invalidated.
2011 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
2012 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2013 Op, Ty);
2014 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
2015 S->computeAndSetCanonical(SE&: *this);
2016 registerUser(User: S, Ops: Op);
2017 return S;
2018}
2019
2020const SCEV *
2021ScalarEvolution::getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) {
2022 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2023 "This is not an extending conversion!");
2024 assert(isSCEVable(Ty) &&
2025 "This is not a conversion to a SCEVable type!");
2026 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
2027 Ty = getEffectiveSCEVType(Ty);
2028
2029 FoldID ID(scSignExtend, Op, Ty);
2030 if (const SCEV *S = FoldCache.lookup(Val: ID))
2031 return S;
2032
2033 const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
2034 if (!isa<SCEVSignExtendExpr>(Val: S))
2035 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
2036 return S;
2037}
2038
2039const SCEV *ScalarEvolution::getSignExtendExprImpl(const SCEV *Op, Type *Ty,
2040 unsigned Depth) {
2041 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2042 "This is not an extending conversion!");
2043 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
2044 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
2045 Ty = getEffectiveSCEVType(Ty);
2046
2047 // Fold if the operand is constant.
2048 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
2049 return getConstant(Val: SC->getAPInt().sext(width: getTypeSizeInBits(Ty)));
2050
2051 // sext(sext(x)) --> sext(x)
2052 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val: Op))
2053 return getSignExtendExpr(Op: SS->getOperand(), Ty, Depth: Depth + 1);
2054
2055 // sext(zext(x)) --> zext(x)
2056 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val: Op))
2057 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
2058
2059 // If the operand is an affine AddRec with the no-signed-wrap flag, the
2060 // sign-extension distributes over the recurrence.
2061 const SCEV *Start, *Step;
2062 const Loop *L;
2063 if (Depth <= MaxCastDepth &&
2064 match(S: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
2065 const auto *AR = cast<SCEVAddRecExpr>(Val: Op);
2066 if (AR->hasNoSignedWrap()) {
2067 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2068 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2069 return getAddRecExpr(Start, Step, L, Flags: SCEV::FlagNSW);
2070 }
2071 }
2072
2073 // Before doing any expensive analysis, check to see if we've already
2074 // computed a SCEV for this Op and Ty.
2075 FoldingSetNodeID ID;
2076 ID.AddInteger(I: scSignExtend);
2077 ID.AddPointer(Ptr: Op);
2078 ID.AddPointer(Ptr: Ty);
2079 void *IP = nullptr;
2080 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
2081 // Limit recursion depth.
2082 if (Depth > MaxCastDepth) {
2083 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2084 Op, Ty);
2085 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
2086 S->computeAndSetCanonical(SE&: *this);
2087 registerUser(User: S, Ops: Op);
2088 return S;
2089 }
2090
2091 // sext(trunc(x)) --> sext(x) or x or trunc(x)
2092 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2093 // It's possible the bits taken off by the truncate were all sign bits. If
2094 // so, we should be able to simplify this further.
2095 const SCEV *X = ST->getOperand();
2096 ConstantRange CR = getSignedRange(S: X);
2097 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
2098 unsigned NewBits = getTypeSizeInBits(Ty);
2099 if (CR.truncate(BitWidth: TruncBits).signExtend(BitWidth: NewBits).contains(
2100 CR: CR.sextOrTrunc(BitWidth: NewBits)))
2101 return getTruncateOrSignExtend(V: X, Ty, Depth);
2102 }
2103
2104 if (auto *SA = dyn_cast<SCEVAddExpr>(Val: Op)) {
2105 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
2106 if (SA->hasNoSignedWrap()) {
2107 // If the addition does not sign overflow then we can, by definition,
2108 // commute the sign extension with the addition operation.
2109 SmallVector<SCEVUse, 4> Ops;
2110 for (SCEVUse Op : SA->operands())
2111 Ops.push_back(Elt: getSignExtendExpr(Op, Ty, Depth: Depth + 1));
2112 return getAddExpr(Ops, Flags: SCEV::FlagNSW, Depth: Depth + 1);
2113 }
2114
2115 // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
2116 // if D + (C - D + x + y + ...) could be proven to not signed wrap
2117 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
2118 //
2119 // For instance, this will bring two seemingly different expressions:
2120 // 1 + sext(5 + 20 * %x + 24 * %y) and
2121 // sext(6 + 20 * %x + 24 * %y)
2122 // to the same form:
2123 // 2 + sext(4 + 20 * %x + 24 * %y)
2124 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
2125 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
2126 if (D != 0) {
2127 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
2128 const SCEV *SResidual =
2129 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagAnyWrap, Depth);
2130 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
2131 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
2132 Depth: Depth + 1);
2133 }
2134 }
2135 }
2136 // If the input value is a chrec scev, and we can prove that the value
2137 // did not overflow the old, smaller, value, we can sign extend all of the
2138 // operands (often constants). This allows analysis of something like
2139 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
2140 if (match(S: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
2141 const auto *AR = cast<SCEVAddRecExpr>(Val: Op);
2142 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
2143
2144 // The no-signed-wrap case is handled before the uniquing lookup above.
2145
2146 // Check whether the backedge-taken count is SCEVCouldNotCompute.
2147 // Note that this serves two purposes: It filters out loops that are
2148 // simply not analyzable, and it covers the case where this code is
2149 // being called from within backedge-taken count analysis, such that
2150 // attempting to ask for the backedge-taken count would likely result
2151 // in infinite recursion. In the later case, the analysis code will
2152 // cope with a conservative value, and it will take care to purge
2153 // that value once it has finished.
2154 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
2155 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
2156 // Manually compute the final value for AR, checking for
2157 // overflow.
2158
2159 // Check whether the backedge-taken count can be losslessly casted to
2160 // the addrec's type. The count is always unsigned.
2161 const SCEV *CastedMaxBECount =
2162 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
2163 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
2164 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
2165 if (MaxBECount == RecastedMaxBECount) {
2166 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
2167 // Check whether Start+Step*MaxBECount has no signed overflow.
2168 const SCEV *SMul =
2169 getMulExpr(LHS: CastedMaxBECount, RHS: Step, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2170 const SCEV *SAdd = getSignExtendExpr(
2171 Op: getAddExpr(LHS: Start, RHS: SMul, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1), Ty: WideTy,
2172 Depth: Depth + 1);
2173 const SCEV *WideStart = getSignExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
2174 const SCEV *WideMaxBECount =
2175 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
2176 const SCEV *OperandExtendedAdd =
2177 getAddExpr(LHS: WideStart,
2178 RHS: getMulExpr(LHS: WideMaxBECount,
2179 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2180 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2181 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2182 if (SAdd == OperandExtendedAdd) {
2183 // Cache knowledge of AR NSW, which is propagated to this AddRec.
2184 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2185 // Return the expression with the addrec on the outside.
2186 Start =
2187 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2188 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2189 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2190 }
2191 // Similar to above, only this time treat the step value as unsigned.
2192 // This covers loops that count up with an unsigned step.
2193 OperandExtendedAdd =
2194 getAddExpr(LHS: WideStart,
2195 RHS: getMulExpr(LHS: WideMaxBECount,
2196 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2197 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2198 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2199 if (SAdd == OperandExtendedAdd) {
2200 // If AR wraps around then
2201 //
2202 // abs(Step) * MaxBECount > unsigned-max(AR->getType())
2203 // => SAdd != OperandExtendedAdd
2204 //
2205 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
2206 // (SAdd == OperandExtendedAdd => AR is NW)
2207
2208 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
2209
2210 // Return the expression with the addrec on the outside.
2211 Start =
2212 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2213 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2214 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2215 }
2216 }
2217 }
2218
2219 auto NewFlags = proveNoSignedWrapViaInduction(AR);
2220 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
2221 if (AR->hasNoSignedWrap()) {
2222 // Same as nsw case above - duplicated here to avoid a compile time
2223 // issue. It's not clear that the order of checks does matter, but
2224 // it's one of two issue possible causes for a change which was
2225 // reverted. Be conservative for the moment.
2226 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2227 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2228 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2229 }
2230
2231 // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
2232 // if D + (C - D + Step * n) could be proven to not signed wrap
2233 // where D maximizes the number of trailing zeros of (C - D + Step * n)
2234 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
2235 const APInt &C = SC->getAPInt();
2236 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
2237 if (D != 0) {
2238 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
2239 const SCEV *SResidual =
2240 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
2241 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
2242 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
2243 Depth: Depth + 1);
2244 }
2245 }
2246
2247 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
2248 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2249 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2250 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2251 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2252 }
2253 }
2254
2255 // If the input value is provably positive and we could not simplify
2256 // away the sext build a zext instead.
2257 if (isKnownNonNegative(S: Op))
2258 return getZeroExtendExpr(Op, Ty, Depth: Depth + 1);
2259
2260 // sext(smin(x, y)) -> smin(sext(x), sext(y))
2261 // sext(smax(x, y)) -> smax(sext(x), sext(y))
2262 if (isa<SCEVSMinExpr>(Val: Op) || isa<SCEVSMaxExpr>(Val: Op)) {
2263 auto *MinMax = cast<SCEVMinMaxExpr>(Val: Op);
2264 SmallVector<SCEVUse, 4> Operands;
2265 for (SCEVUse Operand : MinMax->operands())
2266 Operands.push_back(Elt: getSignExtendExpr(Op: Operand, Ty));
2267 if (isa<SCEVSMinExpr>(Val: MinMax))
2268 return getSMinExpr(Operands);
2269 return getSMaxExpr(Operands);
2270 }
2271
2272 // The cast wasn't folded; create an explicit cast node.
2273 // Recompute the insert position, as it may have been invalidated.
2274 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
2275 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2276 Op, Ty);
2277 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
2278 S->computeAndSetCanonical(SE&: *this);
2279 registerUser(User: S, Ops: Op);
2280 return S;
2281}
2282
2283const SCEV *ScalarEvolution::getCastExpr(SCEVTypes Kind, const SCEV *Op,
2284 Type *Ty) {
2285 switch (Kind) {
2286 case scTruncate:
2287 return getTruncateExpr(Op, Ty);
2288 case scZeroExtend:
2289 return getZeroExtendExpr(Op, Ty);
2290 case scSignExtend:
2291 return getSignExtendExpr(Op, Ty);
2292 case scPtrToInt:
2293 return getPtrToIntExpr(Op, Ty);
2294 default:
2295 llvm_unreachable("Not a SCEV cast expression!");
2296 }
2297}
2298
2299/// getAnyExtendExpr - Return a SCEV for the given operand extended with
2300/// unspecified bits out to the given type.
2301const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
2302 Type *Ty) {
2303 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2304 "This is not an extending conversion!");
2305 assert(isSCEVable(Ty) &&
2306 "This is not a conversion to a SCEVable type!");
2307 Ty = getEffectiveSCEVType(Ty);
2308
2309 // Sign-extend negative constants.
2310 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
2311 if (SC->getAPInt().isNegative())
2312 return getSignExtendExpr(Op, Ty);
2313
2314 // Peel off a truncate cast.
2315 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2316 const SCEV *NewOp = T->getOperand();
2317 if (getTypeSizeInBits(Ty: NewOp->getType()) < getTypeSizeInBits(Ty))
2318 return getAnyExtendExpr(Op: NewOp, Ty);
2319 return getTruncateOrNoop(V: NewOp, Ty);
2320 }
2321
2322 // Next try a zext cast. If the cast is folded, use it.
2323 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
2324 if (!isa<SCEVZeroExtendExpr>(Val: ZExt))
2325 return ZExt;
2326
2327 // Next try a sext cast. If the cast is folded, use it.
2328 const SCEV *SExt = getSignExtendExpr(Op, Ty);
2329 if (!isa<SCEVSignExtendExpr>(Val: SExt))
2330 return SExt;
2331
2332 // Force the cast to be folded into the operands of an addrec.
2333 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Op)) {
2334 SmallVector<SCEVUse, 4> Ops;
2335 for (const SCEV *Op : AR->operands())
2336 Ops.push_back(Elt: getAnyExtendExpr(Op, Ty));
2337 return getAddRecExpr(Operands&: Ops, L: AR->getLoop(), Flags: SCEV::FlagNW);
2338 }
2339
2340 // If the expression is obviously signed, use the sext cast value.
2341 if (isa<SCEVSMaxExpr>(Val: Op))
2342 return SExt;
2343
2344 // Absent any other information, use the zext cast value.
2345 return ZExt;
2346}
2347
2348/// Process the given Ops list, which is a list of operands to be added under
2349/// the given scale, update the given map. This is a helper function for
2350/// getAddRecExpr. As an example of what it does, given a sequence of operands
2351/// that would form an add expression like this:
2352///
2353/// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
2354///
2355/// where A and B are constants, update the map with these values:
2356///
2357/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
2358///
2359/// and add 13 + A*B*29 to AccumulatedConstant.
2360/// This will allow getAddRecExpr to produce this:
2361///
2362/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
2363///
2364/// This form often exposes folding opportunities that are hidden in
2365/// the original operand list.
2366///
2367/// Return true iff it appears that any interesting folding opportunities
2368/// may be exposed. This helps getAddRecExpr short-circuit extra work in
2369/// the common case where no interesting opportunities are present, and
2370/// is also used as a check to avoid infinite recursion.
2371static bool CollectAddOperandsWithScales(SmallDenseMap<SCEVUse, APInt, 16> &M,
2372 SmallVectorImpl<SCEVUse> &NewOps,
2373 APInt &AccumulatedConstant,
2374 ArrayRef<SCEVUse> Ops,
2375 const APInt &Scale,
2376 ScalarEvolution &SE) {
2377 bool Interesting = false;
2378
2379 // Iterate over the add operands. They are sorted, with constants first.
2380 unsigned i = 0;
2381 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Ops[i])) {
2382 ++i;
2383 // Pull a buried constant out to the outside.
2384 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
2385 Interesting = true;
2386 AccumulatedConstant += Scale * C->getAPInt();
2387 }
2388
2389 // Next comes everything else. We're especially interested in multiplies
2390 // here, but they're in the middle, so just visit the rest with one loop.
2391 for (; i != Ops.size(); ++i) {
2392 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: Ops[i]);
2393 if (Mul && isa<SCEVConstant>(Val: Mul->getOperand(i: 0))) {
2394 APInt NewScale =
2395 Scale * cast<SCEVConstant>(Val: Mul->getOperand(i: 0))->getAPInt();
2396 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Val: Mul->getOperand(i: 1))) {
2397 // A multiplication of a constant with another add; recurse.
2398 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: Mul->getOperand(i: 1));
2399 Interesting |= CollectAddOperandsWithScales(
2400 M, NewOps, AccumulatedConstant, Ops: Add->operands(), Scale: NewScale, SE);
2401 } else {
2402 // A multiplication of a constant with some other value. Update
2403 // the map.
2404 SmallVector<SCEVUse, 4> MulOps(drop_begin(RangeOrContainer: Mul->operands()));
2405 const SCEV *Key = SE.getMulExpr(Ops&: MulOps);
2406 auto Pair = M.insert(KV: {Key, NewScale});
2407 if (Pair.second) {
2408 NewOps.push_back(Elt: Pair.first->first);
2409 } else {
2410 Pair.first->second += NewScale;
2411 // The map already had an entry for this value, which may indicate
2412 // a folding opportunity.
2413 Interesting = true;
2414 }
2415 }
2416 } else {
2417 // An ordinary operand. Update the map.
2418 auto Pair = M.insert(KV: {Ops[i], Scale});
2419 if (Pair.second) {
2420 NewOps.push_back(Elt: Pair.first->first);
2421 } else {
2422 Pair.first->second += Scale;
2423 // The map already had an entry for this value, which may indicate
2424 // a folding opportunity.
2425 Interesting = true;
2426 }
2427 }
2428 }
2429
2430 return Interesting;
2431}
2432
2433bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed,
2434 const SCEV *LHS, const SCEV *RHS,
2435 const Instruction *CtxI) {
2436 const SCEV *(ScalarEvolution::*Operation)(SCEVUse, SCEVUse, SCEV::NoWrapFlags,
2437 unsigned);
2438 switch (BinOp) {
2439 default:
2440 llvm_unreachable("Unsupported binary op");
2441 case Instruction::Add:
2442 Operation = &ScalarEvolution::getAddExpr;
2443 break;
2444 case Instruction::Sub:
2445 Operation = &ScalarEvolution::getMinusSCEV;
2446 break;
2447 case Instruction::Mul:
2448 Operation = &ScalarEvolution::getMulExpr;
2449 break;
2450 }
2451
2452 const SCEV *(ScalarEvolution::*Extension)(const SCEV *, Type *, unsigned) =
2453 Signed ? &ScalarEvolution::getSignExtendExpr
2454 : &ScalarEvolution::getZeroExtendExpr;
2455
2456 // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
2457 auto *NarrowTy = cast<IntegerType>(Val: LHS->getType());
2458 auto *WideTy =
2459 IntegerType::get(C&: NarrowTy->getContext(), NumBits: NarrowTy->getBitWidth() * 2);
2460
2461 const SCEV *A = (this->*Extension)(
2462 (this->*Operation)(LHS, RHS, SCEV::FlagAnyWrap, 0), WideTy, 0);
2463 const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
2464 const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
2465 const SCEV *B = (this->*Operation)(LHSB, RHSB, SCEV::FlagAnyWrap, 0);
2466 if (A == B)
2467 return true;
2468 // Can we use context to prove the fact we need?
2469 if (!CtxI)
2470 return false;
2471 // TODO: Support mul.
2472 if (BinOp == Instruction::Mul)
2473 return false;
2474 auto *RHSC = dyn_cast<SCEVConstant>(Val: RHS);
2475 // TODO: Lift this limitation.
2476 if (!RHSC)
2477 return false;
2478 APInt C = RHSC->getAPInt();
2479 unsigned NumBits = C.getBitWidth();
2480 bool IsSub = (BinOp == Instruction::Sub);
2481 bool IsNegativeConst = (Signed && C.isNegative());
2482 // Compute the direction and magnitude by which we need to check overflow.
2483 bool OverflowDown = IsSub ^ IsNegativeConst;
2484 APInt Magnitude = C;
2485 if (IsNegativeConst) {
2486 if (C == APInt::getSignedMinValue(numBits: NumBits))
2487 // TODO: SINT_MIN on inversion gives the same negative value, we don't
2488 // want to deal with that.
2489 return false;
2490 Magnitude = -C;
2491 }
2492
2493 ICmpInst::Predicate Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2494 if (OverflowDown) {
2495 // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
2496 APInt Min = Signed ? APInt::getSignedMinValue(numBits: NumBits)
2497 : APInt::getMinValue(numBits: NumBits);
2498 APInt Limit = Min + Magnitude;
2499 return isKnownPredicateAt(Pred, LHS: getConstant(Val: Limit), RHS: LHS, CtxI);
2500 } else {
2501 // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
2502 APInt Max = Signed ? APInt::getSignedMaxValue(numBits: NumBits)
2503 : APInt::getMaxValue(numBits: NumBits);
2504 APInt Limit = Max - Magnitude;
2505 return isKnownPredicateAt(Pred, LHS, RHS: getConstant(Val: Limit), CtxI);
2506 }
2507}
2508
2509std::optional<SCEV::NoWrapFlags>
2510ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp(
2511 const OverflowingBinaryOperator *OBO) {
2512 // It cannot be done any better.
2513 if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
2514 return std::nullopt;
2515
2516 SCEV::NoWrapFlags Flags = SCEV::NoWrapFlags::FlagAnyWrap;
2517
2518 if (OBO->hasNoUnsignedWrap())
2519 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2520 if (OBO->hasNoSignedWrap())
2521 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2522
2523 bool Deduced = false;
2524
2525 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)OBO->getOpcode();
2526 const SCEV *LHS = getSCEV(V: OBO->getOperand(i_nocapture: 0));
2527 const SCEV *RHS = getSCEV(V: OBO->getOperand(i_nocapture: 1));
2528
2529 bool CanUseNSW = true;
2530 const APInt *ShiftAmt;
2531 // Treat `shl %a, C` as `mul %a, 1 << C`.
2532 if (match(V: OBO, P: m_Shl(L: m_Value(), R: m_APInt(Res&: ShiftAmt)))) {
2533 unsigned BitWidth = ShiftAmt->getBitWidth();
2534 if (ShiftAmt->uge(RHS: BitWidth))
2535 return std::nullopt;
2536 // NSW only transfers if the shift amount is < BitWidth - 1, as INT_MIN * -1
2537 // overflows.
2538 CanUseNSW = ShiftAmt->ult(RHS: BitWidth - 1);
2539 Opcode = Instruction::Mul;
2540 RHS = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: ShiftAmt->getZExtValue()));
2541 } else if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
2542 Opcode != Instruction::Mul) {
2543 return std::nullopt;
2544 }
2545
2546 const Instruction *CtxI =
2547 UseContextForNoWrapFlagInference ? dyn_cast<Instruction>(Val: OBO) : nullptr;
2548 if (!OBO->hasNoUnsignedWrap() &&
2549 willNotOverflow(BinOp: Opcode, /* Signed */ false, LHS, RHS, CtxI)) {
2550 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2551 Deduced = true;
2552 }
2553
2554 if (CanUseNSW && !OBO->hasNoSignedWrap() &&
2555 willNotOverflow(BinOp: Opcode, /* Signed */ true, LHS, RHS, CtxI)) {
2556 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2557 Deduced = true;
2558 }
2559
2560 if (Deduced)
2561 return Flags;
2562 return std::nullopt;
2563}
2564
2565// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
2566// `OldFlags' as can't-wrap behavior. Infer a more aggressive set of
2567// can't-overflow flags for the operation if possible.
2568static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE,
2569 SCEVTypes Type,
2570 ArrayRef<SCEVUse> Ops,
2571 SCEV::NoWrapFlags Flags) {
2572 using namespace std::placeholders;
2573
2574 using OBO = OverflowingBinaryOperator;
2575
2576 bool CanAnalyze =
2577 Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr;
2578 (void)CanAnalyze;
2579 assert(CanAnalyze && "don't call from other places!");
2580
2581 SCEV::NoWrapFlags SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
2582 SCEV::NoWrapFlags SignOrUnsignWrap =
2583 ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2584
2585 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
2586 auto IsKnownNonNegative = [&](SCEVUse U) {
2587 return SE->isKnownNonNegative(S: U);
2588 };
2589
2590 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Range&: Ops, P: IsKnownNonNegative))
2591 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SignOrUnsignMask);
2592
2593 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2594
2595 if (SignOrUnsignWrap != SignOrUnsignMask &&
2596 (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
2597 isa<SCEVConstant>(Val: Ops[0])) {
2598
2599 auto Opcode = [&] {
2600 switch (Type) {
2601 case scAddExpr:
2602 return Instruction::Add;
2603 case scMulExpr:
2604 return Instruction::Mul;
2605 default:
2606 llvm_unreachable("Unexpected SCEV op.");
2607 }
2608 }();
2609
2610 const APInt &C = cast<SCEVConstant>(Val: Ops[0])->getAPInt();
2611
2612 // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
2613 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
2614 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2615 BinOp: Opcode, Other: C, NoWrapKind: OBO::NoSignedWrap);
2616 if (NSWRegion.contains(CR: SE->getSignedRange(S: Ops[1])))
2617 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2618 }
2619
2620 // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
2621 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
2622 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2623 BinOp: Opcode, Other: C, NoWrapKind: OBO::NoUnsignedWrap);
2624 if (NUWRegion.contains(CR: SE->getUnsignedRange(S: Ops[1])))
2625 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2626 }
2627 }
2628
2629 // <0,+,nonnegative><nw> is also nuw
2630 // TODO: Add corresponding nsw case
2631 if (Type == scAddRecExpr && ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNW) &&
2632 !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) && Ops.size() == 2 &&
2633 Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
2634 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2635
2636 // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
2637 if (Type == scMulExpr && !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) &&
2638 Ops.size() == 2) {
2639 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[0]))
2640 if (UDiv->getOperand(i: 1) == Ops[1])
2641 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2642 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[1]))
2643 if (UDiv->getOperand(i: 1) == Ops[0])
2644 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2645 }
2646
2647 return Flags;
2648}
2649
2650bool ScalarEvolution::isAvailableAtLoopEntry(const SCEV *S, const Loop *L) {
2651 return isLoopInvariant(S, L) && properlyDominates(S, BB: L->getHeader());
2652}
2653
2654/// Get a canonical add expression, or something simpler if possible.
2655const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<SCEVUse> &Ops,
2656 SCEV::NoWrapFlags OrigFlags,
2657 unsigned Depth) {
2658 assert(!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2659 "only nuw or nsw allowed");
2660 assert(!Ops.empty() && "Cannot get empty add!");
2661 if (Ops.size() == 1) return Ops[0];
2662#ifndef NDEBUG
2663 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
2664 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
2665 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
2666 "SCEVAddExpr operand types don't match!");
2667 unsigned NumPtrs = count_if(
2668 Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
2669 assert(NumPtrs <= 1 && "add has at most one pointer operand");
2670#endif
2671
2672 const SCEV *Folded = constantFoldAndGroupOps(
2673 SE&: *this, LI, DT, Ops,
2674 Fold: [](const APInt &C1, const APInt &C2) { return C1 + C2; },
2675 IsIdentity: [](const APInt &C) { return C.isZero(); }, // identity
2676 IsAbsorber: [](const APInt &C) { return false; }); // absorber
2677 if (Folded)
2678 return Folded;
2679
2680 unsigned Idx = isa<SCEVConstant>(Val: Ops[0]) ? 1 : 0;
2681
2682 // Delay expensive flag strengthening until necessary.
2683 auto ComputeFlags = [this, OrigFlags](ArrayRef<SCEVUse> Ops) {
2684 return StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops, Flags: OrigFlags);
2685 };
2686
2687 // Limit recursion calls depth.
2688 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
2689 return getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops));
2690
2691 if (SCEV *S = findExistingSCEVInCache(SCEVType: scAddExpr, Ops)) {
2692 // Don't strengthen flags if we have no new information.
2693 SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
2694 if (Add->getNoWrapFlags(Mask: OrigFlags) != OrigFlags)
2695 Add->setNoWrapFlags(ComputeFlags(Ops));
2696 return S;
2697 }
2698
2699 // Okay, check to see if the same value occurs in the operand list more than
2700 // once. If so, merge them together into an multiply expression. Since we
2701 // sorted the list, these values are required to be adjacent.
2702 Type *Ty = Ops[0]->getType();
2703 bool FoundMatch = false;
2704 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
2705 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2
2706 // Scan ahead to count how many equal operands there are.
2707 unsigned Count = 2;
2708 while (i+Count != e && Ops[i+Count] == Ops[i])
2709 ++Count;
2710 // Merge the values into a multiply.
2711 SCEVUse Scale = getConstant(Ty, V: Count);
2712 const SCEV *Mul = getMulExpr(LHS: Scale, RHS: Ops[i], Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2713 if (Ops.size() == Count)
2714 return Mul;
2715 Ops[i] = Mul;
2716 Ops.erase(CS: Ops.begin()+i+1, CE: Ops.begin()+i+Count);
2717 --i; e -= Count - 1;
2718 FoundMatch = true;
2719 }
2720 if (FoundMatch)
2721 return getAddExpr(Ops, OrigFlags, Depth: Depth + 1);
2722
2723 // Check for truncates. If all the operands are truncated from the same
2724 // type, see if factoring out the truncate would permit the result to be
2725 // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
2726 // if the contents of the resulting outer trunc fold to something simple.
2727 auto FindTruncSrcType = [&]() -> Type * {
2728 // We're ultimately looking to fold an addrec of truncs and muls of only
2729 // constants and truncs, so if we find any other types of SCEV
2730 // as operands of the addrec then we bail and return nullptr here.
2731 // Otherwise, we return the type of the operand of a trunc that we find.
2732 if (auto *T = dyn_cast<SCEVTruncateExpr>(Val&: Ops[Idx]))
2733 return T->getOperand()->getType();
2734 if (const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
2735 SCEVUse LastOp = Mul->getOperand(i: Mul->getNumOperands() - 1);
2736 if (const auto *T = dyn_cast<SCEVTruncateExpr>(Val&: LastOp))
2737 return T->getOperand()->getType();
2738 }
2739 return nullptr;
2740 };
2741 if (auto *SrcType = FindTruncSrcType()) {
2742 SmallVector<SCEVUse, 8> LargeOps;
2743 bool Ok = true;
2744 // Check all the operands to see if they can be represented in the
2745 // source type of the truncate.
2746 for (const SCEV *Op : Ops) {
2747 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2748 if (T->getOperand()->getType() != SrcType) {
2749 Ok = false;
2750 break;
2751 }
2752 LargeOps.push_back(Elt: T->getOperand());
2753 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Op)) {
2754 LargeOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2755 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val: Op)) {
2756 SmallVector<SCEVUse, 8> LargeMulOps;
2757 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2758 if (const SCEVTruncateExpr *T =
2759 dyn_cast<SCEVTruncateExpr>(Val: M->getOperand(i: j))) {
2760 if (T->getOperand()->getType() != SrcType) {
2761 Ok = false;
2762 break;
2763 }
2764 LargeMulOps.push_back(Elt: T->getOperand());
2765 } else if (const auto *C = dyn_cast<SCEVConstant>(Val: M->getOperand(i: j))) {
2766 LargeMulOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2767 } else {
2768 Ok = false;
2769 break;
2770 }
2771 }
2772 if (Ok)
2773 LargeOps.push_back(Elt: getMulExpr(Ops&: LargeMulOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2774 } else {
2775 Ok = false;
2776 break;
2777 }
2778 }
2779 if (Ok) {
2780 // Evaluate the expression in the larger type.
2781 const SCEV *Fold = getAddExpr(Ops&: LargeOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2782 // If it folds to something simple, use it. Otherwise, don't.
2783 if (isa<SCEVConstant>(Val: Fold) || isa<SCEVUnknown>(Val: Fold))
2784 return getTruncateExpr(Op: Fold, Ty);
2785 }
2786 }
2787
2788 if (Ops.size() == 2) {
2789 // Check if we have an expression of the form ((X + C1) - C2), where C1 and
2790 // C2 can be folded in a way that allows retaining wrapping flags of (X +
2791 // C1).
2792 const SCEV *A = Ops[0];
2793 const SCEV *B = Ops[1];
2794 auto *AddExpr = dyn_cast<SCEVAddExpr>(Val: B);
2795 auto *C = dyn_cast<SCEVConstant>(Val: A);
2796 if (AddExpr && C && isa<SCEVConstant>(Val: AddExpr->getOperand(i: 0))) {
2797 auto C1 = cast<SCEVConstant>(Val: AddExpr->getOperand(i: 0))->getAPInt();
2798 auto C2 = C->getAPInt();
2799 SCEV::NoWrapFlags PreservedFlags = SCEV::FlagAnyWrap;
2800
2801 APInt ConstAdd = C1 + C2;
2802 auto AddFlags = AddExpr->getNoWrapFlags();
2803 // Adding a smaller constant is NUW if the original AddExpr was NUW.
2804 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNUW) &&
2805 ConstAdd.ule(RHS: C1)) {
2806 PreservedFlags =
2807 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNUW);
2808 }
2809
2810 // Adding a constant with the same sign and small magnitude is NSW, if the
2811 // original AddExpr was NSW.
2812 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNSW) &&
2813 C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
2814 ConstAdd.abs().ule(RHS: C1.abs())) {
2815 PreservedFlags =
2816 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNSW);
2817 }
2818
2819 if (PreservedFlags != SCEV::FlagAnyWrap) {
2820 SmallVector<SCEVUse, 4> NewOps(AddExpr->operands());
2821 NewOps[0] = getConstant(Val: ConstAdd);
2822 return getAddExpr(Ops&: NewOps, OrigFlags: PreservedFlags);
2823 }
2824 }
2825
2826 // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext
2827 // (B), if trunc (A) + -A + B does not unsigned-wrap.
2828 const SCEVAddExpr *InnerAdd;
2829 if (match(S: B, P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
2830 const SCEV *NarrowA = getTruncateExpr(Op: A, Ty: InnerAdd->getType());
2831 if (NarrowA == getNegativeSCEV(V: InnerAdd->getOperand(i: 0)) &&
2832 getZeroExtendExpr(Op: NarrowA, Ty: B->getType()) == A &&
2833 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops: {NarrowA, InnerAdd},
2834 Flags: SCEV::FlagAnyWrap),
2835 TestFlags: SCEV::FlagNUW)) {
2836 return getZeroExtendExpr(Op: getAddExpr(LHS: NarrowA, RHS: InnerAdd), Ty: B->getType());
2837 }
2838 }
2839 }
2840
2841 // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
2842 const SCEV *Y;
2843 if (Ops.size() == 2 &&
2844 match(U: Ops[0],
2845 P: m_scev_Mul(Op0: m_scev_AllOnes(),
2846 Op1: m_scev_URem(LHS: m_scev_Specific(S: Ops[1]), RHS: m_SCEV(V&: Y), SE&: *this))))
2847 return getMulExpr(LHS: Y, RHS: getUDivExpr(LHS: Ops[1], RHS: Y));
2848
2849 // Skip past any other cast SCEVs.
2850 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
2851 ++Idx;
2852
2853 // If there are add operands they would be next.
2854 if (Idx < Ops.size()) {
2855 bool DeletedAdd = false;
2856 // If the original flags and all inlined SCEVAddExprs are NUW, use the
2857 // common NUW flag for expression after inlining. Other flags cannot be
2858 // preserved, because they may depend on the original order of operations.
2859 SCEV::NoWrapFlags CommonFlags = maskFlags(Flags: OrigFlags, Mask: SCEV::FlagNUW);
2860 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[Idx])) {
2861 if (Ops.size() > AddOpsInlineThreshold ||
2862 Add->getNumOperands() > AddOpsInlineThreshold)
2863 break;
2864 // If we have an add, expand the add operands onto the end of the operands
2865 // list.
2866 Ops.erase(CI: Ops.begin()+Idx);
2867 append_range(C&: Ops, R: Add->operands());
2868 DeletedAdd = true;
2869 CommonFlags = maskFlags(Flags: CommonFlags, Mask: Add->getNoWrapFlags());
2870 }
2871
2872 // If we deleted at least one add, we added operands to the end of the list,
2873 // and they are not necessarily sorted. Recurse to resort and resimplify
2874 // any operands we just acquired.
2875 if (DeletedAdd)
2876 return getAddExpr(Ops, OrigFlags: CommonFlags, Depth: Depth + 1);
2877 }
2878
2879 // Skip over the add expression until we get to a multiply.
2880 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
2881 ++Idx;
2882
2883 // Check to see if there are any folding opportunities present with
2884 // operands multiplied by constant values.
2885 if (Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx])) {
2886 uint64_t BitWidth = getTypeSizeInBits(Ty);
2887 SmallDenseMap<SCEVUse, APInt, 16> M;
2888 SmallVector<SCEVUse, 8> NewOps;
2889 APInt AccumulatedConstant(BitWidth, 0);
2890 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2891 Ops, Scale: APInt(BitWidth, 1), SE&: *this)) {
2892 struct APIntCompare {
2893 bool operator()(const APInt &LHS, const APInt &RHS) const {
2894 return LHS.ult(RHS);
2895 }
2896 };
2897
2898 // Some interesting folding opportunity is present, so its worthwhile to
2899 // re-generate the operands list. Group the operands by constant scale,
2900 // to avoid multiplying by the same constant scale multiple times.
2901 std::map<APInt, SmallVector<SCEVUse, 4>, APIntCompare> MulOpLists;
2902 for (const SCEV *NewOp : NewOps)
2903 MulOpLists[M.find(Val: NewOp)->second].push_back(Elt: NewOp);
2904 // Re-generate the operands list.
2905 Ops.clear();
2906 if (AccumulatedConstant != 0)
2907 Ops.push_back(Elt: getConstant(Val: AccumulatedConstant));
2908 for (auto &MulOp : MulOpLists) {
2909 if (MulOp.first == 1) {
2910 Ops.push_back(Elt: getAddExpr(Ops&: MulOp.second, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2911 } else if (MulOp.first != 0) {
2912 Ops.push_back(Elt: getMulExpr(
2913 LHS: getConstant(Val: MulOp.first),
2914 RHS: getAddExpr(Ops&: MulOp.second, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2915 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2916 }
2917 }
2918 if (Ops.empty())
2919 return getZero(Ty);
2920 if (Ops.size() == 1)
2921 return Ops[0];
2922 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2923 }
2924 }
2925
2926 // Given a SCEVMulExpr and an operand index, return the product of all
2927 // operands except the one at OpIdx.
2928 auto StripFactor = [&](const SCEVMulExpr *M, unsigned OpIdx) -> SCEVUse {
2929 if (M->getNumOperands() == 2)
2930 return M->getOperand(i: OpIdx == 0);
2931 SmallVector<SCEVUse, 4> Remaining(M->operands().take_front(N: OpIdx));
2932 append_range(C&: Remaining, R: M->operands().drop_front(N: OpIdx + 1));
2933 return getMulExpr(Ops&: Remaining, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2934 };
2935
2936 // If we are adding something to a multiply expression, make sure the
2937 // something is not already an operand of the multiply. If so, merge it into
2938 // the multiply.
2939 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx]); ++Idx) {
2940 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val&: Ops[Idx]);
2941 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
2942 // Scan all terms to find every occurrence of common factor MulOpSCEV
2943 // and fold them in one shot:
2944 // A1*X + A2*X + ... + An*X --> X * (A1 + A2 + ... + An)
2945 const SCEV *MulOpSCEV = Mul->getOperand(i: MulOp);
2946 if (isa<SCEVConstant>(Val: MulOpSCEV))
2947 continue;
2948
2949 // Cofactors: 1 for bare addends matching MulOpSCEV, or the
2950 // remaining product for multiply terms containing MulOpSCEV.
2951 SmallVector<SCEVUse, 4> Cofactors;
2952 SmallVector<unsigned, 4> DeadIndices;
2953 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) {
2954 if (MulOpSCEV == Ops[AddOp]) {
2955 // W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
2956 Cofactors.push_back(Elt: getOne(Ty));
2957 DeadIndices.push_back(Elt: AddOp);
2958 continue;
2959 }
2960
2961 if (AddOp <= Idx || !isa<SCEVMulExpr>(Val: Ops[AddOp]))
2962 continue;
2963
2964 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Val&: Ops[AddOp]);
2965 for (unsigned OMulOp = 0, OE = OtherMul->getNumOperands(); OMulOp != OE;
2966 ++OMulOp) {
2967 if (OtherMul->getOperand(i: OMulOp) == MulOpSCEV) {
2968 // (A*B*C) + (A*D*E) --> A * (B*C + D*E)
2969 Cofactors.push_back(Elt: StripFactor(OtherMul, OMulOp));
2970 DeadIndices.push_back(Elt: AddOp);
2971 break;
2972 }
2973 }
2974 }
2975
2976 // Fold all collected cofactors with the anchor multiply's cofactor:
2977 // MulOpSCEV * (Cofactor_1 + ... + Cofactor_n + AnchorCofactor)
2978 if (!Cofactors.empty()) {
2979 Cofactors.push_back(Elt: StripFactor(Mul, MulOp));
2980
2981 SCEVUse InnerSum = getAddExpr(Ops&: Cofactors, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2982 SCEVUse OuterMul =
2983 getMulExpr(LHS: MulOpSCEV, RHS: InnerSum, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2984
2985 // DeadIndices does not include Idx (the anchor), hence +1.
2986 if (Ops.size() == DeadIndices.size() + 1)
2987 return OuterMul;
2988
2989 // Erase Ops[Idx] first, then erase DeadIndices in reverse order.
2990 // The -1 adjustment accounts for the shift from removing Idx;
2991 // reverse order means each erasure only shifts later positions,
2992 // which have already been processed.
2993 Ops.erase(CI: Ops.begin() + Idx);
2994 for (unsigned Dead : reverse(C&: DeadIndices))
2995 Ops.erase(CI: Ops.begin() + (Dead > Idx ? Dead - 1 : Dead));
2996
2997 Ops.push_back(Elt: OuterMul);
2998 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2999 }
3000 }
3001 }
3002
3003 // If there are any add recurrences in the operands list, see if any other
3004 // added values are loop invariant. If so, we can fold them into the
3005 // recurrence.
3006 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3007 ++Idx;
3008
3009 // Scan over all recurrences, trying to fold loop invariants into them.
3010 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
3011 // Scan all of the other operands to this add and add them to the vector if
3012 // they are loop invariant w.r.t. the recurrence.
3013 SmallVector<SCEVUse, 8> LIOps;
3014 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
3015 const Loop *AddRecLoop = AddRec->getLoop();
3016 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3017 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRecLoop)) {
3018 LIOps.push_back(Elt: Ops[i]);
3019 Ops.erase(CI: Ops.begin()+i);
3020 --i; --e;
3021 }
3022
3023 // If we found some loop invariants, fold them into the recurrence.
3024 if (!LIOps.empty()) {
3025 // Compute nowrap flags for the addition of the loop-invariant ops and
3026 // the addrec. Temporarily push it as an operand for that purpose. These
3027 // flags are valid in the scope of the addrec only.
3028 LIOps.push_back(Elt: AddRec);
3029 SCEV::NoWrapFlags Flags = ComputeFlags(LIOps);
3030 LIOps.pop_back();
3031
3032 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
3033 LIOps.push_back(Elt: AddRec->getStart());
3034
3035 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
3036
3037 // It is not in general safe to propagate flags valid on an add within
3038 // the addrec scope to one outside it. We must prove that the inner
3039 // scope is guaranteed to execute if the outer one does to be able to
3040 // safely propagate. We know the program is undefined if poison is
3041 // produced on the inner scoped addrec. We also know that *for this use*
3042 // the outer scoped add can't overflow (because of the flags we just
3043 // computed for the inner scoped add) without the program being undefined.
3044 // Proving that entry to the outer scope neccesitates entry to the inner
3045 // scope, thus proves the program undefined if the flags would be violated
3046 // in the outer scope.
3047 SCEV::NoWrapFlags AddFlags = Flags;
3048 if (AddFlags != SCEV::FlagAnyWrap) {
3049 auto *DefI = getDefiningScopeBound(Ops: LIOps);
3050 auto *ReachI = &*AddRecLoop->getHeader()->begin();
3051 if (!isGuaranteedToTransferExecutionTo(A: DefI, B: ReachI))
3052 AddFlags = SCEV::FlagAnyWrap;
3053 }
3054 AddRecOps[0] = getAddExpr(Ops&: LIOps, OrigFlags: AddFlags, Depth: Depth + 1);
3055
3056 // Build the new addrec. Propagate the NUW and NSW flags if both the
3057 // outer add and the inner addrec are guaranteed to have no overflow.
3058 // Always propagate NW.
3059 Flags = AddRec->getNoWrapFlags(Mask: setFlags(Flags, OnFlags: SCEV::FlagNW));
3060 const SCEV *NewRec = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags);
3061
3062 // If all of the other operands were loop invariant, we are done.
3063 if (Ops.size() == 1) return NewRec;
3064
3065 // Otherwise, add the folded AddRec by the non-invariant parts.
3066 for (unsigned i = 0;; ++i)
3067 if (Ops[i] == AddRec) {
3068 Ops[i] = NewRec;
3069 break;
3070 }
3071 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3072 }
3073
3074 // Okay, if there weren't any loop invariants to be folded, check to see if
3075 // there are multiple AddRec's with the same loop induction variable being
3076 // added together. If so, we can fold them.
3077 for (unsigned OtherIdx = Idx+1;
3078 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3079 ++OtherIdx) {
3080 // We expect the AddRecExpr's to be sorted in reverse dominance order,
3081 // so that the 1st found AddRecExpr is dominated by all others.
3082 assert(DT.dominates(
3083 cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),
3084 AddRec->getLoop()->getHeader()) &&
3085 "AddRecExprs are not sorted in reverse dominance order?");
3086 if (AddRecLoop == cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx])->getLoop()) {
3087 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L>
3088 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
3089 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3090 ++OtherIdx) {
3091 const auto *OtherAddRec = cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
3092 if (OtherAddRec->getLoop() == AddRecLoop) {
3093 for (unsigned i = 0, e = OtherAddRec->getNumOperands();
3094 i != e; ++i) {
3095 if (i >= AddRecOps.size()) {
3096 append_range(C&: AddRecOps, R: OtherAddRec->operands().drop_front(N: i));
3097 break;
3098 }
3099 AddRecOps[i] =
3100 getAddExpr(LHS: AddRecOps[i], RHS: OtherAddRec->getOperand(i),
3101 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3102 }
3103 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
3104 }
3105 }
3106 // Step size has changed, so we cannot guarantee no self-wraparound.
3107 Ops[Idx] = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags: SCEV::FlagAnyWrap);
3108 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3109 }
3110 }
3111
3112 // Otherwise couldn't fold anything into this recurrence. Move onto the
3113 // next one.
3114 }
3115
3116 // Okay, it looks like we really DO need an add expr. Check to see if we
3117 // already have one, otherwise create a new one.
3118 return getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops));
3119}
3120
3121const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
3122 SCEV::NoWrapFlags Flags) {
3123 FoldingSetNodeID ID;
3124 ID.AddInteger(I: scAddExpr);
3125 for (const SCEV *Op : Ops)
3126 ID.AddPointer(Ptr: Op);
3127 void *IP = nullptr;
3128 SCEVAddExpr *S =
3129 static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
3130 if (!S) {
3131 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3132 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3133 S = new (SCEVAllocator)
3134 SCEVAddExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size());
3135 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3136 S->computeAndSetCanonical(SE&: *this);
3137 registerUser(User: S, Ops);
3138 }
3139 S->setNoWrapFlags(Flags);
3140 return S;
3141}
3142
3143const SCEV *ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<SCEVUse> Ops,
3144 const Loop *L,
3145 SCEV::NoWrapFlags Flags) {
3146 FoldingSetNodeID ID;
3147 ID.AddInteger(I: scAddRecExpr);
3148 for (const SCEV *Op : Ops)
3149 ID.AddPointer(Ptr: Op);
3150 ID.AddPointer(Ptr: L);
3151 void *IP = nullptr;
3152 SCEVAddRecExpr *S =
3153 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
3154 if (!S) {
3155 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3156 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3157 S = new (SCEVAllocator)
3158 SCEVAddRecExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size(), L);
3159 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3160 S->computeAndSetCanonical(SE&: *this);
3161 LoopUsers[L].push_back(Elt: S);
3162 registerUser(User: S, Ops);
3163 }
3164 setNoWrapFlags(AddRec: S, Flags);
3165 return S;
3166}
3167
3168const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
3169 SCEV::NoWrapFlags Flags) {
3170 FoldingSetNodeID ID;
3171 ID.AddInteger(I: scMulExpr);
3172 for (const SCEV *Op : Ops)
3173 ID.AddPointer(Ptr: Op);
3174 void *IP = nullptr;
3175 SCEVMulExpr *S =
3176 static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
3177 if (!S) {
3178 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3179 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3180 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(Allocator&: SCEVAllocator),
3181 O, Ops.size());
3182 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3183 S->computeAndSetCanonical(SE&: *this);
3184 registerUser(User: S, Ops);
3185 }
3186 S->setNoWrapFlags(Flags);
3187 return S;
3188}
3189
3190static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
3191 uint64_t k = i*j;
3192 if (j > 1 && k / j != i) Overflow = true;
3193 return k;
3194}
3195
3196/// Compute the result of "n choose k", the binomial coefficient. If an
3197/// intermediate computation overflows, Overflow will be set and the return will
3198/// be garbage. Overflow is not cleared on absence of overflow.
3199static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
3200 // We use the multiplicative formula:
3201 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
3202 // At each iteration, we take the n-th term of the numeral and divide by the
3203 // (k-n)th term of the denominator. This division will always produce an
3204 // integral result, and helps reduce the chance of overflow in the
3205 // intermediate computations. However, we can still overflow even when the
3206 // final result would fit.
3207
3208 if (n == 0 || n == k) return 1;
3209 if (k > n) return 0;
3210
3211 if (k > n/2)
3212 k = n-k;
3213
3214 uint64_t r = 1;
3215 for (uint64_t i = 1; i <= k; ++i) {
3216 r = umul_ov(i: r, j: n-(i-1), Overflow);
3217 r /= i;
3218 }
3219 return r;
3220}
3221
3222/// Determine if any of the operands in this SCEV are a constant or if
3223/// any of the add or multiply expressions in this SCEV contain a constant.
3224static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
3225 struct FindConstantInAddMulChain {
3226 bool FoundConstant = false;
3227
3228 bool follow(const SCEV *S) {
3229 FoundConstant |= isa<SCEVConstant>(Val: S);
3230 return isa<SCEVAddExpr>(Val: S) || isa<SCEVMulExpr>(Val: S);
3231 }
3232
3233 bool isDone() const {
3234 return FoundConstant;
3235 }
3236 };
3237
3238 FindConstantInAddMulChain F;
3239 SCEVTraversal<FindConstantInAddMulChain> ST(F);
3240 ST.visitAll(Root: StartExpr);
3241 return F.FoundConstant;
3242}
3243
3244/// Get a canonical multiply expression, or something simpler if possible.
3245const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
3246 SCEV::NoWrapFlags OrigFlags,
3247 unsigned Depth) {
3248 assert(OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3249 "only nuw or nsw allowed");
3250 assert(!Ops.empty() && "Cannot get empty mul!");
3251 if (Ops.size() == 1) return Ops[0];
3252#ifndef NDEBUG
3253 Type *ETy = Ops[0]->getType();
3254 assert(!ETy->isPointerTy());
3255 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
3256 assert(Ops[i]->getType() == ETy &&
3257 "SCEVMulExpr operand types don't match!");
3258#endif
3259
3260 const SCEV *Folded = constantFoldAndGroupOps(
3261 SE&: *this, LI, DT, Ops,
3262 Fold: [](const APInt &C1, const APInt &C2) { return C1 * C2; },
3263 IsIdentity: [](const APInt &C) { return C.isOne(); }, // identity
3264 IsAbsorber: [](const APInt &C) { return C.isZero(); }); // absorber
3265 if (Folded)
3266 return Folded;
3267
3268 // Delay expensive flag strengthening until necessary.
3269 auto ComputeFlags = [this, OrigFlags](const ArrayRef<SCEVUse> Ops) {
3270 return StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops, Flags: OrigFlags);
3271 };
3272
3273 // Limit recursion calls depth.
3274 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
3275 return getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops));
3276
3277 if (SCEV *S = findExistingSCEVInCache(SCEVType: scMulExpr, Ops)) {
3278 // Don't strengthen flags if we have no new information.
3279 SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
3280 if (Mul->getNoWrapFlags(Mask: OrigFlags) != OrigFlags)
3281 Mul->setNoWrapFlags(ComputeFlags(Ops));
3282 return S;
3283 }
3284
3285 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: Ops[0])) {
3286 if (Ops.size() == 2) {
3287 // C1*(C2+V) -> C1*C2 + C1*V
3288 // If any of Add's ops are Adds or Muls with a constant, apply this
3289 // transformation as well.
3290 //
3291 // TODO: There are some cases where this transformation is not
3292 // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of
3293 // this transformation should be narrowed down.
3294 const SCEV *Op0, *Op1;
3295 if (match(U: Ops[1], P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))) &&
3296 containsConstantInAddMulChain(StartExpr: Ops[1])) {
3297 const SCEV *LHS = getMulExpr(LHS: LHSC, RHS: Op0, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3298 const SCEV *RHS = getMulExpr(LHS: LHSC, RHS: Op1, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3299 return getAddExpr(LHS, RHS, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3300 }
3301
3302 if (Ops[0]->isAllOnesValue()) {
3303 // If we have a mul by -1 of an add, try distributing the -1 among the
3304 // add operands.
3305 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[1])) {
3306 SmallVector<SCEVUse, 4> NewOps;
3307 bool AnyFolded = false;
3308 for (const SCEV *AddOp : Add->operands()) {
3309 const SCEV *Mul = getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddOp),
3310 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3311 if (!isa<SCEVMulExpr>(Val: Mul)) AnyFolded = true;
3312 NewOps.push_back(Elt: Mul);
3313 }
3314 if (AnyFolded)
3315 return getAddExpr(Ops&: NewOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3316 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: Ops[1])) {
3317 // Negation preserves a recurrence's no self-wrap property.
3318 SmallVector<SCEVUse, 4> Operands;
3319 for (const SCEV *AddRecOp : AddRec->operands())
3320 Operands.push_back(Elt: getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddRecOp),
3321 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3322 // Let M be the minimum representable signed value. AddRec with nsw
3323 // multiplied by -1 can have signed overflow if and only if it takes a
3324 // value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
3325 // maximum signed value. In all other cases signed overflow is
3326 // impossible.
3327 auto FlagsMask = SCEV::FlagNW;
3328 if (AddRec->hasNoSignedWrap()) {
3329 auto MinInt =
3330 APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: AddRec->getType()));
3331 if (getSignedRangeMin(S: AddRec) != MinInt)
3332 FlagsMask = setFlags(Flags: FlagsMask, OnFlags: SCEV::FlagNSW);
3333 }
3334 return getAddRecExpr(Operands, L: AddRec->getLoop(),
3335 Flags: AddRec->getNoWrapFlags(Mask: FlagsMask));
3336 }
3337 }
3338
3339 // Try to push the constant operand into a ZExt: C * zext (A + B) ->
3340 // zext (C*A + C*B) if trunc (C) * (A + B) does not unsigned-wrap.
3341 const SCEVAddExpr *InnerAdd;
3342 if (match(U: Ops[1], P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
3343 const SCEV *NarrowC = getTruncateExpr(Op: LHSC, Ty: InnerAdd->getType());
3344 if (isa<SCEVConstant>(Val: InnerAdd->getOperand(i: 0)) &&
3345 getZeroExtendExpr(Op: NarrowC, Ty: Ops[1]->getType()) == LHSC &&
3346 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops: {NarrowC, InnerAdd},
3347 Flags: SCEV::FlagAnyWrap),
3348 TestFlags: SCEV::FlagNUW)) {
3349 auto *Res = getMulExpr(LHS: NarrowC, RHS: InnerAdd, Flags: SCEV::FlagNUW, Depth: Depth + 1);
3350 return getZeroExtendExpr(Op: Res, Ty: Ops[1]->getType(), Depth: Depth + 1);
3351 };
3352 }
3353
3354 // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2,
3355 // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple
3356 // of C1, fold to (D /u (C2 /u C1)).
3357 const SCEV *D;
3358 APInt C1V = LHSC->getAPInt();
3359 // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1
3360 // as -1 * 1, as it won't enable additional folds.
3361 if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes())
3362 C1V = C1V.abs();
3363 const SCEVConstant *C2;
3364 if (C1V.isPowerOf2() &&
3365 match(U: Ops[1], P: m_scev_UDiv(Op0: m_SCEV(V&: D), Op1: m_SCEVConstant(V&: C2))) &&
3366 C2->getAPInt().isPowerOf2() &&
3367 C1V.logBase2() <= getMinTrailingZeros(S: D)) {
3368 const SCEV *NewMul = nullptr;
3369 if (C1V.uge(RHS: C2->getAPInt())) {
3370 NewMul = getMulExpr(LHS: getUDivExpr(LHS: getConstant(Val: C1V), RHS: C2), RHS: D);
3371 } else if (C2->getAPInt().logBase2() <= getMinTrailingZeros(S: D)) {
3372 assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier");
3373 NewMul = getUDivExpr(LHS: D, RHS: getUDivExpr(LHS: C2, RHS: getConstant(Val: C1V)));
3374 }
3375 if (NewMul)
3376 return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(V: NewMul);
3377 }
3378 }
3379 }
3380
3381 // Skip over the add expression until we get to a multiply.
3382 unsigned Idx = 0;
3383 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
3384 ++Idx;
3385
3386 // If there are mul operands inline them all into this expression.
3387 if (Idx < Ops.size()) {
3388 bool DeletedMul = false;
3389 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
3390 if (Ops.size() > MulOpsInlineThreshold)
3391 break;
3392 // If we have an mul, expand the mul operands onto the end of the
3393 // operands list.
3394 Ops.erase(CI: Ops.begin()+Idx);
3395 append_range(C&: Ops, R: Mul->operands());
3396 DeletedMul = true;
3397 }
3398
3399 // If we deleted at least one mul, we added operands to the end of the
3400 // list, and they are not necessarily sorted. Recurse to resort and
3401 // resimplify any operands we just acquired.
3402 if (DeletedMul)
3403 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3404 }
3405
3406 // If there are any add recurrences in the operands list, see if any other
3407 // added values are loop invariant. If so, we can fold them into the
3408 // recurrence.
3409 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3410 ++Idx;
3411
3412 // Scan over all recurrences, trying to fold loop invariants into them.
3413 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
3414 // Scan all of the other operands to this mul and add them to the vector
3415 // if they are loop invariant w.r.t. the recurrence.
3416 SmallVector<SCEVUse, 8> LIOps;
3417 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
3418 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3419 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRec->getLoop())) {
3420 LIOps.push_back(Elt: Ops[i]);
3421 Ops.erase(CI: Ops.begin()+i);
3422 --i; --e;
3423 }
3424
3425 // If we found some loop invariants, fold them into the recurrence.
3426 if (!LIOps.empty()) {
3427 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
3428 SmallVector<SCEVUse, 4> NewOps;
3429 NewOps.reserve(N: AddRec->getNumOperands());
3430 const SCEV *Scale = getMulExpr(Ops&: LIOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3431
3432 // If both the mul and addrec are nuw, we can preserve nuw.
3433 // If both the mul and addrec are nsw, we can only preserve nsw if either
3434 // a) they are also nuw, or
3435 // b) all multiplications of addrec operands with scale are nsw.
3436 SCEV::NoWrapFlags Flags =
3437 AddRec->getNoWrapFlags(Mask: ComputeFlags({Scale, AddRec}));
3438
3439 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
3440 NewOps.push_back(Elt: getMulExpr(LHS: Scale, RHS: AddRec->getOperand(i),
3441 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3442
3443 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW) && !hasFlags(Flags, TestFlags: SCEV::FlagNUW)) {
3444 ConstantRange NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
3445 BinOp: Instruction::Mul, Other: getSignedRange(S: Scale),
3446 NoWrapKind: OverflowingBinaryOperator::NoSignedWrap);
3447 if (!NSWRegion.contains(CR: getSignedRange(S: AddRec->getOperand(i))))
3448 Flags = clearFlags(Flags, OffFlags: SCEV::FlagNSW);
3449 }
3450 }
3451
3452 const SCEV *NewRec = getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags);
3453
3454 // If all of the other operands were loop invariant, we are done.
3455 if (Ops.size() == 1) return NewRec;
3456
3457 // Otherwise, multiply the folded AddRec by the non-invariant parts.
3458 for (unsigned i = 0;; ++i)
3459 if (Ops[i] == AddRec) {
3460 Ops[i] = NewRec;
3461 break;
3462 }
3463 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3464 }
3465
3466 // Okay, if there weren't any loop invariants to be folded, check to see
3467 // if there are multiple AddRec's with the same loop induction variable
3468 // being multiplied together. If so, we can fold them.
3469
3470 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
3471 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
3472 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
3473 // ]]],+,...up to x=2n}.
3474 // Note that the arguments to choose() are always integers with values
3475 // known at compile time, never SCEV objects.
3476 //
3477 // The implementation avoids pointless extra computations when the two
3478 // addrec's are of different length (mathematically, it's equivalent to
3479 // an infinite stream of zeros on the right).
3480 bool OpsModified = false;
3481 for (unsigned OtherIdx = Idx+1;
3482 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3483 ++OtherIdx) {
3484 const SCEVAddRecExpr *OtherAddRec =
3485 dyn_cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
3486 if (!OtherAddRec || OtherAddRec->getLoop() != AddRec->getLoop())
3487 continue;
3488
3489 // Limit max number of arguments to avoid creation of unreasonably big
3490 // SCEVAddRecs with very complex operands.
3491 if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
3492 MaxAddRecSize || hasHugeExpression(Ops: {AddRec, OtherAddRec}))
3493 continue;
3494
3495 bool Overflow = false;
3496 Type *Ty = AddRec->getType();
3497 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
3498 SmallVector<SCEVUse, 7> AddRecOps;
3499 for (int x = 0, xe = AddRec->getNumOperands() +
3500 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
3501 SmallVector<SCEVUse, 7> SumOps;
3502 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3503 uint64_t Coeff1 = Choose(n: x, k: 2*x - y, Overflow);
3504 for (int z = std::max(a: y-x, b: y-(int)AddRec->getNumOperands()+1),
3505 ze = std::min(a: x+1, b: (int)OtherAddRec->getNumOperands());
3506 z < ze && !Overflow; ++z) {
3507 uint64_t Coeff2 = Choose(n: 2*x - y, k: x-z, Overflow);
3508 uint64_t Coeff;
3509 if (LargerThan64Bits)
3510 Coeff = umul_ov(i: Coeff1, j: Coeff2, Overflow);
3511 else
3512 Coeff = Coeff1*Coeff2;
3513 const SCEV *CoeffTerm = getConstant(Ty, V: Coeff);
3514 const SCEV *Term1 = AddRec->getOperand(i: y-z);
3515 const SCEV *Term2 = OtherAddRec->getOperand(i: z);
3516 SumOps.push_back(Elt: getMulExpr(Op0: CoeffTerm, Op1: Term1, Op2: Term2,
3517 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3518 }
3519 }
3520 if (SumOps.empty())
3521 SumOps.push_back(Elt: getZero(Ty));
3522 AddRecOps.push_back(Elt: getAddExpr(Ops&: SumOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3523 }
3524 if (!Overflow) {
3525 const SCEV *NewAddRec = getAddRecExpr(Operands&: AddRecOps, L: AddRec->getLoop(),
3526 Flags: SCEV::FlagAnyWrap);
3527 if (Ops.size() == 2) return NewAddRec;
3528 Ops[Idx] = NewAddRec;
3529 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
3530 OpsModified = true;
3531 AddRec = dyn_cast<SCEVAddRecExpr>(Val: NewAddRec);
3532 if (!AddRec)
3533 break;
3534 }
3535 }
3536 if (OpsModified)
3537 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3538
3539 // Otherwise couldn't fold anything into this recurrence. Move onto the
3540 // next one.
3541 }
3542
3543 // Okay, it looks like we really DO need an mul expr. Check to see if we
3544 // already have one, otherwise create a new one.
3545 return getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops));
3546}
3547
3548/// Represents an unsigned remainder expression based on unsigned division.
3549const SCEV *ScalarEvolution::getURemExpr(SCEVUse LHS, SCEVUse RHS) {
3550 assert(getEffectiveSCEVType(LHS->getType()) ==
3551 getEffectiveSCEVType(RHS->getType()) &&
3552 "SCEVURemExpr operand types don't match!");
3553
3554 // Short-circuit easy cases
3555 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3556 // If constant is one, the result is trivial
3557 if (RHSC->getValue()->isOne())
3558 return getZero(Ty: LHS->getType()); // X urem 1 --> 0
3559
3560 // If constant is a power of two, fold into a zext(trunc(LHS)).
3561 if (RHSC->getAPInt().isPowerOf2()) {
3562 Type *FullTy = LHS->getType();
3563 Type *TruncTy =
3564 IntegerType::get(C&: getContext(), NumBits: RHSC->getAPInt().logBase2());
3565 return getZeroExtendExpr(Op: getTruncateExpr(Op: LHS, Ty: TruncTy), Ty: FullTy);
3566 }
3567 }
3568
3569 // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
3570 const SCEV *UDiv = getUDivExpr(LHS, RHS);
3571 const SCEV *Mult = getMulExpr(LHS: UDiv, RHS, Flags: SCEV::FlagNUW);
3572 return getMinusSCEV(LHS, RHS: Mult, Flags: SCEV::FlagNUW);
3573}
3574
3575/// Get a canonical unsigned division expression, or something simpler if
3576/// possible.
3577const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
3578 assert(!LHS->getType()->isPointerTy() &&
3579 "SCEVUDivExpr operand can't be pointer!");
3580 assert(LHS->getType() == RHS->getType() &&
3581 "SCEVUDivExpr operand types don't match!");
3582
3583 FoldingSetNodeID ID;
3584 ID.AddInteger(I: scUDivExpr);
3585 ID.AddPointer(Ptr: LHS);
3586 ID.AddPointer(Ptr: RHS);
3587 void *IP = nullptr;
3588 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
3589 return S;
3590
3591 // 0 udiv Y == 0
3592 if (match(U: LHS, P: m_scev_Zero()))
3593 return LHS;
3594
3595 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3596 if (RHSC->getValue()->isOne())
3597 return LHS; // X udiv 1 --> x
3598 // If the denominator is zero, the result of the udiv is undefined. Don't
3599 // try to analyze it, because the resolution chosen here may differ from
3600 // the resolution chosen in other parts of the compiler.
3601 if (!RHSC->getValue()->isZero()) {
3602 // Determine if the division can be folded into the operands of
3603 // its operands.
3604 // TODO: Generalize this to non-constants by using known-bits information.
3605 Type *Ty = LHS->getType();
3606 unsigned LZ = RHSC->getAPInt().countl_zero();
3607 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
3608 // For non-power-of-two values, effectively round the value up to the
3609 // nearest power of two.
3610 if (!RHSC->getAPInt().isPowerOf2())
3611 ++MaxShiftAmt;
3612 IntegerType *ExtTy =
3613 IntegerType::get(C&: getContext(), NumBits: getTypeSizeInBits(Ty) + MaxShiftAmt);
3614 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
3615 if (const SCEVConstant *Step =
3616 dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE&: *this))) {
3617 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
3618 const APInt &StepInt = Step->getAPInt();
3619 const APInt &DivInt = RHSC->getAPInt();
3620 if (!StepInt.urem(RHS: DivInt) &&
3621 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3622 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3623 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy),
3624 L: AR->getLoop(), Flags: SCEV::FlagAnyWrap)) {
3625 SmallVector<SCEVUse, 4> Operands;
3626 for (const SCEV *Op : AR->operands())
3627 Operands.push_back(Elt: getUDivExpr(LHS: Op, RHS));
3628 return getAddRecExpr(Operands, L: AR->getLoop(), Flags: SCEV::FlagNW);
3629 }
3630 /// Get a canonical UDivExpr for a recurrence.
3631 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
3632 const APInt *StartRem;
3633 if (!DivInt.urem(RHS: StepInt) && match(S: getURemExpr(LHS: AR->getStart(), RHS: Step),
3634 P: m_scev_APInt(C&: StartRem))) {
3635 bool NoWrap =
3636 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3637 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3638 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy), L: AR->getLoop(),
3639 Flags: SCEV::FlagAnyWrap);
3640
3641 // With N <= C and both N, C as powers-of-2, the transformation
3642 // {X,+,N}/C => {(X - X%N),+,N}/C preserves division results even
3643 // if wrapping occurs, as the division results remain equivalent for
3644 // all offsets in [[(X - X%N), X).
3645 bool CanFoldWithWrap = StepInt.ule(RHS: DivInt) && // N <= C
3646 StepInt.isPowerOf2() && DivInt.isPowerOf2();
3647 // Only fold if the subtraction can be folded in the start
3648 // expression.
3649 const SCEV *NewStart =
3650 getMinusSCEV(LHS: AR->getStart(), RHS: getConstant(Val: *StartRem));
3651 if (*StartRem != 0 && (NoWrap || CanFoldWithWrap) &&
3652 !isa<SCEVAddExpr>(Val: NewStart)) {
3653 const SCEV *NewLHS =
3654 getAddRecExpr(Start: NewStart, Step, L: AR->getLoop(),
3655 Flags: NoWrap ? SCEV::FlagNW : SCEV::FlagAnyWrap);
3656 if (LHS != NewLHS) {
3657 LHS = NewLHS;
3658
3659 // Reset the ID to include the new LHS, and check if it is
3660 // already cached.
3661 ID.clear();
3662 ID.AddInteger(I: scUDivExpr);
3663 ID.AddPointer(Ptr: LHS);
3664 ID.AddPointer(Ptr: RHS);
3665 IP = nullptr;
3666 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
3667 return S;
3668 }
3669 }
3670 }
3671 }
3672 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
3673 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val&: LHS)) {
3674 SmallVector<SCEVUse, 4> Operands;
3675 for (const SCEV *Op : M->operands())
3676 Operands.push_back(Elt: getZeroExtendExpr(Op, Ty: ExtTy));
3677 if (getZeroExtendExpr(Op: M, Ty: ExtTy) == getMulExpr(Ops&: Operands)) {
3678 // Find an operand that's safely divisible.
3679 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3680 const SCEV *Op = M->getOperand(i);
3681 const SCEV *Div = getUDivExpr(LHS: Op, RHS: RHSC);
3682 if (!isa<SCEVUDivExpr>(Val: Div) && getMulExpr(LHS: Div, RHS: RHSC) == Op) {
3683 Operands = SmallVector<SCEVUse, 4>(M->operands());
3684 Operands[i] = Div;
3685 return getMulExpr(Ops&: Operands);
3686 }
3687 }
3688
3689 // Even if it's not divisible, try to remove a common factor.
3690 if (const auto *LHSC = dyn_cast<SCEVConstant>(Val: M->getOperand(i: 0))) {
3691 APInt Factor = APIntOps::GreatestCommonDivisor(A: LHSC->getAPInt(),
3692 B: RHSC->getAPInt());
3693 if (!Factor.isIntN(N: 1)) {
3694 SmallVector<SCEVUse, 2> NewOperands;
3695 NewOperands.push_back(Elt: getConstant(Val: LHSC->getAPInt().udiv(RHS: Factor)));
3696 append_range(C&: NewOperands, R: M->operands().drop_front());
3697 const SCEV *NewMul = getMulExpr(Ops&: NewOperands);
3698 return getUDivExpr(LHS: NewMul,
3699 RHS: getConstant(Val: RHSC->getAPInt().udiv(RHS: Factor)));
3700 }
3701 }
3702 }
3703 }
3704
3705 // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
3706 if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(Val&: LHS)) {
3707 if (auto *DivisorConstant =
3708 dyn_cast<SCEVConstant>(Val: OtherDiv->getRHS())) {
3709 bool Overflow = false;
3710 APInt NewRHS =
3711 DivisorConstant->getAPInt().umul_ov(RHS: RHSC->getAPInt(), Overflow);
3712 if (Overflow) {
3713 return getConstant(Ty: RHSC->getType(), V: 0, isSigned: false);
3714 }
3715 return getUDivExpr(LHS: OtherDiv->getLHS(), RHS: getConstant(Val: NewRHS));
3716 }
3717 }
3718
3719 // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
3720 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(Val&: LHS)) {
3721 SmallVector<SCEVUse, 4> Operands;
3722 for (const SCEV *Op : A->operands())
3723 Operands.push_back(Elt: getZeroExtendExpr(Op, Ty: ExtTy));
3724 if (getZeroExtendExpr(Op: A, Ty: ExtTy) == getAddExpr(Ops&: Operands)) {
3725 Operands.clear();
3726 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
3727 const SCEV *Op = getUDivExpr(LHS: A->getOperand(i), RHS);
3728 if (isa<SCEVUDivExpr>(Val: Op) ||
3729 getMulExpr(LHS: Op, RHS) != A->getOperand(i))
3730 break;
3731 Operands.push_back(Elt: Op);
3732 }
3733 if (Operands.size() == A->getNumOperands())
3734 return getAddExpr(Ops&: Operands);
3735 }
3736 }
3737
3738 // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
3739 // This is an idiom for rounding A up to the next multiple of N, where A
3740 // is aready known to be a multiple of M. In this case, instcombine can
3741 // see that some low bits of the added constant are unused, so can clear
3742 // them, but we want to canonicalise to set the low bits. This makes the
3743 // pattern easier to match, without needing to check for known bits in
3744 // A*M.
3745 const APInt &N = RHSC->getAPInt();
3746 const APInt *NMinusM, *M;
3747 const SCEV *A;
3748 if (match(U: LHS, P: m_scev_Add(Op0: m_scev_APInt(C&: NMinusM),
3749 Op1: m_scev_Mul(Op0: m_scev_APInt(C&: M), Op1: m_SCEV(V&: A))))) {
3750 if (N.isPowerOf2() && M->isPowerOf2() && M->ult(RHS: N) &&
3751 *NMinusM == N - *M) {
3752 return getUDivExpr(
3753 LHS: getAddExpr(LHS: getConstant(Val: N - 1), RHS: getMulExpr(LHS: getConstant(Val: *M), RHS: A)),
3754 RHS);
3755 }
3756 }
3757
3758 // Fold if both operands are constant.
3759 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS))
3760 return getConstant(Val: LHSC->getAPInt().udiv(RHS: RHSC->getAPInt()));
3761 }
3762 }
3763
3764 // ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
3765 const APInt *NegC, *C;
3766 if (match(U: LHS,
3767 P: m_scev_Add(Op0: m_scev_APInt(C&: NegC),
3768 Op1: m_scev_SMax(Op0: m_scev_APInt(C), Op1: m_scev_Specific(S: RHS)))) &&
3769 NegC->isNegative() && !NegC->isMinSignedValue() && *C == -*NegC)
3770 return getZero(Ty: LHS->getType());
3771
3772 // (%a * %b)<nuw> / %b -> %a
3773 const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: LHS);
3774 if (Mul && Mul->hasNoUnsignedWrap()) {
3775 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
3776 if (Mul->getOperand(i) == RHS) {
3777 SmallVector<SCEVUse, 2> Operands;
3778 append_range(C&: Operands, R: Mul->operands().take_front(N: i));
3779 append_range(C&: Operands, R: Mul->operands().drop_front(N: i + 1));
3780 return getMulExpr(Ops&: Operands);
3781 }
3782 }
3783 }
3784
3785 // TODO: Generalize to handle any common factors.
3786 // udiv (mul nuw a, vscale), (mul nuw b, vscale) --> udiv a, b
3787 const SCEV *NewLHS, *NewRHS;
3788 if (match(U: LHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
3789 match(U: RHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale())))
3790 return getUDivExpr(LHS: NewLHS, RHS: NewRHS);
3791
3792 // The Insertion Point (IP) might be invalid by now (due to UniqueSCEVs
3793 // changes). Make sure we get a new one.
3794 IP = nullptr;
3795 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
3796 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(Allocator&: SCEVAllocator),
3797 LHS, RHS);
3798 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3799 S->computeAndSetCanonical(SE&: *this);
3800 registerUser(User: S, Ops: ArrayRef<SCEVUse>({LHS, RHS}));
3801 return S;
3802}
3803
3804APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) {
3805 APInt A = C1->getAPInt().abs();
3806 APInt B = C2->getAPInt().abs();
3807 uint32_t ABW = A.getBitWidth();
3808 uint32_t BBW = B.getBitWidth();
3809
3810 if (ABW > BBW)
3811 B = B.zext(width: ABW);
3812 else if (ABW < BBW)
3813 A = A.zext(width: BBW);
3814
3815 return APIntOps::GreatestCommonDivisor(A: std::move(A), B: std::move(B));
3816}
3817
3818/// Get a canonical unsigned division expression, or something simpler if
3819/// possible. There is no representation for an exact udiv in SCEV IR, but we
3820/// can attempt to optimize it prior to construction.
3821const SCEV *ScalarEvolution::getUDivExactExpr(SCEVUse LHS, SCEVUse RHS) {
3822 // Currently there is no exact specific logic.
3823
3824 return getUDivExpr(LHS, RHS);
3825}
3826
3827/// Get an add recurrence expression for the specified loop. Simplify the
3828/// expression as much as possible.
3829const SCEV *ScalarEvolution::getAddRecExpr(SCEVUse Start, SCEVUse Step,
3830 const Loop *L,
3831 SCEV::NoWrapFlags Flags) {
3832 SmallVector<SCEVUse, 4> Operands;
3833 Operands.push_back(Elt: Start);
3834 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Val&: Step))
3835 if (StepChrec->getLoop() == L) {
3836 append_range(C&: Operands, R: StepChrec->operands());
3837 return getAddRecExpr(Operands, L, Flags: maskFlags(Flags, Mask: SCEV::FlagNW));
3838 }
3839
3840 Operands.push_back(Elt: Step);
3841 return getAddRecExpr(Operands, L, Flags);
3842}
3843
3844/// Get an add recurrence expression for the specified loop. Simplify the
3845/// expression as much as possible.
3846const SCEV *ScalarEvolution::getAddRecExpr(SmallVectorImpl<SCEVUse> &Operands,
3847 const Loop *L,
3848 SCEV::NoWrapFlags Flags) {
3849 if (Operands.size() == 1) return Operands[0];
3850#ifndef NDEBUG
3851 Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
3852 for (const SCEV *Op : llvm::drop_begin(Operands)) {
3853 assert(getEffectiveSCEVType(Op->getType()) == ETy &&
3854 "SCEVAddRecExpr operand types don't match!");
3855 assert(!Op->getType()->isPointerTy() && "Step must be integer");
3856 }
3857 for (const SCEV *Op : Operands)
3858 assert(isAvailableAtLoopEntry(Op, L) &&
3859 "SCEVAddRecExpr operand is not available at loop entry!");
3860#endif
3861
3862 if (Operands.back()->isZero()) {
3863 Operands.pop_back();
3864 return getAddRecExpr(Operands, L, Flags: SCEV::FlagAnyWrap); // {X,+,0} --> X
3865 }
3866
3867 // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
3868 // use that information to infer NUW and NSW flags. However, computing a
3869 // BE count requires calling getAddRecExpr, so we may not yet have a
3870 // meaningful BE count at this point (and if we don't, we'd be stuck
3871 // with a SCEVCouldNotCompute as the cached BE count).
3872
3873 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
3874
3875 // Canonicalize nested AddRecs in by nesting them in order of loop depth.
3876 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Val&: Operands[0])) {
3877 const Loop *NestedLoop = NestedAR->getLoop();
3878 if (L->contains(L: NestedLoop)
3879 ? (L->getLoopDepth() < NestedLoop->getLoopDepth())
3880 : (!NestedLoop->contains(L) &&
3881 DT.dominates(A: L->getHeader(), B: NestedLoop->getHeader()))) {
3882 SmallVector<SCEVUse, 4> NestedOperands(NestedAR->operands());
3883 Operands[0] = NestedAR->getStart();
3884 // AddRecs require their operands be loop-invariant with respect to their
3885 // loops. Don't perform this transformation if it would break this
3886 // requirement.
3887 bool AllInvariant = all_of(
3888 Range&: Operands, P: [&](const SCEV *Op) { return isLoopInvariant(S: Op, L); });
3889
3890 if (AllInvariant) {
3891 // Create a recurrence for the outer loop with the same step size.
3892 //
3893 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
3894 // inner recurrence has the same property.
3895 SCEV::NoWrapFlags OuterFlags =
3896 maskFlags(Flags, Mask: SCEV::FlagNW | NestedAR->getNoWrapFlags());
3897
3898 NestedOperands[0] = getAddRecExpr(Operands, L, Flags: OuterFlags);
3899 AllInvariant = all_of(Range&: NestedOperands, P: [&](const SCEV *Op) {
3900 return isLoopInvariant(S: Op, L: NestedLoop);
3901 });
3902
3903 if (AllInvariant) {
3904 // Ok, both add recurrences are valid after the transformation.
3905 //
3906 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
3907 // the outer recurrence has the same property.
3908 SCEV::NoWrapFlags InnerFlags =
3909 maskFlags(Flags: NestedAR->getNoWrapFlags(), Mask: SCEV::FlagNW | Flags);
3910 return getAddRecExpr(Operands&: NestedOperands, L: NestedLoop, Flags: InnerFlags);
3911 }
3912 }
3913 // Reset Operands to its original state.
3914 Operands[0] = NestedAR;
3915 }
3916 }
3917
3918 // Okay, it looks like we really DO need an addrec expr. Check to see if we
3919 // already have one, otherwise create a new one.
3920 return getOrCreateAddRecExpr(Ops: Operands, L, Flags);
3921}
3922
3923const SCEV *ScalarEvolution::getGEPExpr(GEPOperator *GEP,
3924 ArrayRef<SCEVUse> IndexExprs) {
3925 const SCEV *BaseExpr = getSCEV(V: GEP->getPointerOperand());
3926 // getSCEV(Base)->getType() has the same address space as Base->getType()
3927 // because SCEV::getType() preserves the address space.
3928 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
3929 if (NW != GEPNoWrapFlags::none()) {
3930 // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
3931 // but to do that, we have to ensure that said flag is valid in the entire
3932 // defined scope of the SCEV.
3933 // TODO: non-instructions have global scope. We might be able to prove
3934 // some global scope cases
3935 auto *GEPI = dyn_cast<Instruction>(Val: GEP);
3936 if (!GEPI || !isSCEVExprNeverPoison(I: GEPI))
3937 NW = GEPNoWrapFlags::none();
3938 }
3939
3940 return getGEPExpr(BaseExpr, IndexExprs, SrcElementTy: GEP->getSourceElementType(), NW);
3941}
3942
3943const SCEV *ScalarEvolution::getGEPExpr(SCEVUse BaseExpr,
3944 ArrayRef<SCEVUse> IndexExprs,
3945 Type *SrcElementTy, GEPNoWrapFlags NW) {
3946 SCEV::NoWrapFlags OffsetWrap = SCEV::FlagAnyWrap;
3947 if (NW.hasNoUnsignedSignedWrap())
3948 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNSW);
3949 if (NW.hasNoUnsignedWrap())
3950 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNUW);
3951
3952 Type *CurTy = BaseExpr->getType();
3953 Type *IntIdxTy = getEffectiveSCEVType(Ty: BaseExpr->getType());
3954 bool FirstIter = true;
3955 SmallVector<SCEVUse, 4> Offsets;
3956 for (SCEVUse IndexExpr : IndexExprs) {
3957 // Compute the (potentially symbolic) offset in bytes for this index.
3958 if (StructType *STy = dyn_cast<StructType>(Val: CurTy)) {
3959 // For a struct, add the member offset.
3960 ConstantInt *Index = cast<SCEVConstant>(Val&: IndexExpr)->getValue();
3961 unsigned FieldNo = Index->getZExtValue();
3962 const SCEV *FieldOffset = getOffsetOfExpr(IntTy: IntIdxTy, STy, FieldNo);
3963 Offsets.push_back(Elt: FieldOffset);
3964
3965 // Update CurTy to the type of the field at Index.
3966 CurTy = STy->getTypeAtIndex(V: Index);
3967 } else {
3968 // Update CurTy to its element type.
3969 if (FirstIter) {
3970 assert(isa<PointerType>(CurTy) &&
3971 "The first index of a GEP indexes a pointer");
3972 CurTy = SrcElementTy;
3973 FirstIter = false;
3974 } else {
3975 CurTy = GetElementPtrInst::getTypeAtIndex(Ty: CurTy, Idx: (uint64_t)0);
3976 }
3977 // For an array, add the element offset, explicitly scaled.
3978 const SCEV *ElementSize = getSizeOfExpr(IntTy: IntIdxTy, AllocTy: CurTy);
3979 // Getelementptr indices are signed.
3980 IndexExpr = getTruncateOrSignExtend(V: IndexExpr, Ty: IntIdxTy);
3981
3982 // Multiply the index by the element size to compute the element offset.
3983 const SCEV *LocalOffset = getMulExpr(LHS: IndexExpr, RHS: ElementSize, Flags: OffsetWrap);
3984 Offsets.push_back(Elt: LocalOffset);
3985 }
3986 }
3987
3988 // Handle degenerate case of GEP without offsets.
3989 if (Offsets.empty())
3990 return BaseExpr;
3991
3992 // Add the offsets together, assuming nsw if inbounds.
3993 const SCEV *Offset = getAddExpr(Ops&: Offsets, OrigFlags: OffsetWrap);
3994 // Add the base address and the offset. We cannot use the nsw flag, as the
3995 // base address is unsigned. However, if we know that the offset is
3996 // non-negative, we can use nuw.
3997 bool NUW = NW.hasNoUnsignedWrap() ||
3998 (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Offset));
3999 SCEV::NoWrapFlags BaseWrap = NUW ? SCEV::FlagNUW : SCEV::FlagAnyWrap;
4000 auto *GEPExpr = getAddExpr(LHS: BaseExpr, RHS: Offset, Flags: BaseWrap);
4001 assert(BaseExpr->getType() == GEPExpr->getType() &&
4002 "GEP should not change type mid-flight.");
4003 return GEPExpr;
4004}
4005
4006SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
4007 ArrayRef<const SCEV *> Ops) {
4008 FoldingSetNodeID ID;
4009 ID.AddInteger(I: SCEVType);
4010 for (const SCEV *Op : Ops)
4011 ID.AddPointer(Ptr: Op);
4012 void *IP = nullptr;
4013 return UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
4014}
4015
4016SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
4017 ArrayRef<SCEVUse> Ops) {
4018 FoldingSetNodeID ID;
4019 ID.AddInteger(I: SCEVType);
4020 for (const SCEV *Op : Ops)
4021 ID.AddPointer(Ptr: Op);
4022 void *IP = nullptr;
4023 return UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
4024}
4025
4026const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
4027 SCEV::NoWrapFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
4028 return getSMaxExpr(LHS: Op, RHS: getNegativeSCEV(V: Op, Flags));
4029}
4030
4031const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
4032 SmallVectorImpl<SCEVUse> &Ops) {
4033 assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
4034 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4035 if (Ops.size() == 1) return Ops[0];
4036#ifndef NDEBUG
4037 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4038 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4039 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4040 "Operand types don't match!");
4041 assert(Ops[0]->getType()->isPointerTy() ==
4042 Ops[i]->getType()->isPointerTy() &&
4043 "min/max should be consistently pointerish");
4044 }
4045#endif
4046
4047 bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
4048 bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
4049
4050 const SCEV *Folded = constantFoldAndGroupOps(
4051 SE&: *this, LI, DT, Ops,
4052 Fold: [&](const APInt &C1, const APInt &C2) {
4053 switch (Kind) {
4054 case scSMaxExpr:
4055 return APIntOps::smax(A: C1, B: C2);
4056 case scSMinExpr:
4057 return APIntOps::smin(A: C1, B: C2);
4058 case scUMaxExpr:
4059 return APIntOps::umax(A: C1, B: C2);
4060 case scUMinExpr:
4061 return APIntOps::umin(A: C1, B: C2);
4062 default:
4063 llvm_unreachable("Unknown SCEV min/max opcode");
4064 }
4065 },
4066 IsIdentity: [&](const APInt &C) {
4067 // identity
4068 if (IsMax)
4069 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
4070 else
4071 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
4072 },
4073 IsAbsorber: [&](const APInt &C) {
4074 // absorber
4075 if (IsMax)
4076 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
4077 else
4078 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
4079 });
4080 if (Folded)
4081 return Folded;
4082
4083 // Check if we have created the same expression before.
4084 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops)) {
4085 return S;
4086 }
4087
4088 // Find the first operation of the same kind
4089 unsigned Idx = 0;
4090 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
4091 ++Idx;
4092
4093 // Check to see if one of the operands is of the same kind. If so, expand its
4094 // operands onto our operand list, and recurse to simplify.
4095 if (Idx < Ops.size()) {
4096 bool DeletedAny = false;
4097 while (Ops[Idx]->getSCEVType() == Kind) {
4098 const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Val&: Ops[Idx]);
4099 Ops.erase(CI: Ops.begin()+Idx);
4100 append_range(C&: Ops, R: SMME->operands());
4101 DeletedAny = true;
4102 }
4103
4104 if (DeletedAny)
4105 return getMinMaxExpr(Kind, Ops);
4106 }
4107
4108 // Okay, check to see if the same value occurs in the operand list twice. If
4109 // so, delete one. Since we sorted the list, these values are required to
4110 // be adjacent.
4111 llvm::CmpInst::Predicate GEPred =
4112 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
4113 llvm::CmpInst::Predicate LEPred =
4114 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
4115 llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
4116 llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
4117 for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
4118 if (Ops[i] == Ops[i + 1] ||
4119 isKnownViaNonRecursiveReasoning(Pred: FirstPred, LHS: Ops[i], RHS: Ops[i + 1])) {
4120 // X op Y op Y --> X op Y
4121 // X op Y --> X, if we know X, Y are ordered appropriately
4122 Ops.erase(CS: Ops.begin() + i + 1, CE: Ops.begin() + i + 2);
4123 --i;
4124 --e;
4125 } else if (isKnownViaNonRecursiveReasoning(Pred: SecondPred, LHS: Ops[i],
4126 RHS: Ops[i + 1])) {
4127 // X op Y --> Y, if we know X, Y are ordered appropriately
4128 Ops.erase(CS: Ops.begin() + i, CE: Ops.begin() + i + 1);
4129 --i;
4130 --e;
4131 }
4132 }
4133
4134 if (Ops.size() == 1) return Ops[0];
4135
4136 assert(!Ops.empty() && "Reduced smax down to nothing!");
4137
4138 // Okay, it looks like we really DO need an expr. Check to see if we
4139 // already have one, otherwise create a new one.
4140 FoldingSetNodeID ID;
4141 ID.AddInteger(I: Kind);
4142 for (const SCEV *Op : Ops)
4143 ID.AddPointer(Ptr: Op);
4144 void *IP = nullptr;
4145 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
4146 if (ExistingSCEV)
4147 return ExistingSCEV;
4148 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
4149 llvm::uninitialized_copy(Src&: Ops, Dst: O);
4150 SCEV *S = new (SCEVAllocator)
4151 SCEVMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
4152
4153 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
4154 S->computeAndSetCanonical(SE&: *this);
4155 registerUser(User: S, Ops);
4156 return S;
4157}
4158
4159namespace {
4160
4161class SCEVSequentialMinMaxDeduplicatingVisitor final
4162 : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
4163 std::optional<const SCEV *>> {
4164 using RetVal = std::optional<const SCEV *>;
4165 using Base = SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor, RetVal>;
4166
4167 ScalarEvolution &SE;
4168 const SCEVTypes RootKind; // Must be a sequential min/max expression.
4169 const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
4170 SmallPtrSet<const SCEV *, 16> SeenOps;
4171
4172 bool canRecurseInto(SCEVTypes Kind) const {
4173 // We can only recurse into the SCEV expression of the same effective type
4174 // as the type of our root SCEV expression.
4175 return RootKind == Kind || NonSequentialRootKind == Kind;
4176 };
4177
4178 RetVal visitAnyMinMaxExpr(const SCEV *S) {
4179 assert((isa<SCEVMinMaxExpr>(S) || isa<SCEVSequentialMinMaxExpr>(S)) &&
4180 "Only for min/max expressions.");
4181 SCEVTypes Kind = S->getSCEVType();
4182
4183 if (!canRecurseInto(Kind))
4184 return S;
4185
4186 auto *NAry = cast<SCEVNAryExpr>(Val: S);
4187 SmallVector<SCEVUse> NewOps;
4188 bool Changed = visit(Kind, OrigOps: NAry->operands(), NewOps);
4189
4190 if (!Changed)
4191 return S;
4192 if (NewOps.empty())
4193 return std::nullopt;
4194
4195 return isa<SCEVSequentialMinMaxExpr>(Val: S)
4196 ? SE.getSequentialMinMaxExpr(Kind, Operands&: NewOps)
4197 : SE.getMinMaxExpr(Kind, Ops&: NewOps);
4198 }
4199
4200 RetVal visit(const SCEV *S) {
4201 // Has the whole operand been seen already?
4202 if (!SeenOps.insert(Ptr: S).second)
4203 return std::nullopt;
4204 return Base::visit(S);
4205 }
4206
4207public:
4208 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
4209 SCEVTypes RootKind)
4210 : SE(SE), RootKind(RootKind),
4211 NonSequentialRootKind(
4212 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
4213 Ty: RootKind)) {}
4214
4215 bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<SCEVUse> OrigOps,
4216 SmallVectorImpl<SCEVUse> &NewOps) {
4217 bool Changed = false;
4218 SmallVector<SCEVUse> Ops;
4219 Ops.reserve(N: OrigOps.size());
4220
4221 for (const SCEV *Op : OrigOps) {
4222 RetVal NewOp = visit(S: Op);
4223 if (NewOp != Op)
4224 Changed = true;
4225 if (NewOp)
4226 Ops.emplace_back(Args&: *NewOp);
4227 }
4228
4229 if (Changed)
4230 NewOps = std::move(Ops);
4231 return Changed;
4232 }
4233
4234 RetVal visitConstant(const SCEVConstant *Constant) { return Constant; }
4235
4236 RetVal visitVScale(const SCEVVScale *VScale) { return VScale; }
4237
4238 RetVal visitPtrToAddrExpr(const SCEVPtrToAddrExpr *Expr) { return Expr; }
4239
4240 RetVal visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { return Expr; }
4241
4242 RetVal visitTruncateExpr(const SCEVTruncateExpr *Expr) { return Expr; }
4243
4244 RetVal visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { return Expr; }
4245
4246 RetVal visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { return Expr; }
4247
4248 RetVal visitAddExpr(const SCEVAddExpr *Expr) { return Expr; }
4249
4250 RetVal visitMulExpr(const SCEVMulExpr *Expr) { return Expr; }
4251
4252 RetVal visitUDivExpr(const SCEVUDivExpr *Expr) { return Expr; }
4253
4254 RetVal visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
4255
4256 RetVal visitSMaxExpr(const SCEVSMaxExpr *Expr) {
4257 return visitAnyMinMaxExpr(S: Expr);
4258 }
4259
4260 RetVal visitUMaxExpr(const SCEVUMaxExpr *Expr) {
4261 return visitAnyMinMaxExpr(S: Expr);
4262 }
4263
4264 RetVal visitSMinExpr(const SCEVSMinExpr *Expr) {
4265 return visitAnyMinMaxExpr(S: Expr);
4266 }
4267
4268 RetVal visitUMinExpr(const SCEVUMinExpr *Expr) {
4269 return visitAnyMinMaxExpr(S: Expr);
4270 }
4271
4272 RetVal visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Expr) {
4273 return visitAnyMinMaxExpr(S: Expr);
4274 }
4275
4276 RetVal visitUnknown(const SCEVUnknown *Expr) { return Expr; }
4277
4278 RetVal visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { return Expr; }
4279};
4280
4281} // namespace
4282
4283static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind) {
4284 switch (Kind) {
4285 case scConstant:
4286 case scVScale:
4287 case scTruncate:
4288 case scZeroExtend:
4289 case scSignExtend:
4290 case scPtrToAddr:
4291 case scPtrToInt:
4292 case scAddExpr:
4293 case scMulExpr:
4294 case scUDivExpr:
4295 case scAddRecExpr:
4296 case scUMaxExpr:
4297 case scSMaxExpr:
4298 case scUMinExpr:
4299 case scSMinExpr:
4300 case scUnknown:
4301 // If any operand is poison, the whole expression is poison.
4302 return true;
4303 case scSequentialUMinExpr:
4304 // FIXME: if the *first* operand is poison, the whole expression is poison.
4305 return false; // Pessimistically, say that it does not propagate poison.
4306 case scCouldNotCompute:
4307 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
4308 }
4309 llvm_unreachable("Unknown SCEV kind!");
4310}
4311
4312namespace {
4313// The only way poison may be introduced in a SCEV expression is from a
4314// poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown,
4315// not SCEVConstant). Notably, nowrap flags in SCEV nodes can *not*
4316// introduce poison -- they encode guaranteed, non-speculated knowledge.
4317//
4318// Additionally, all SCEV nodes propagate poison from inputs to outputs,
4319// with the notable exception of umin_seq, where only poison from the first
4320// operand is (unconditionally) propagated.
4321struct SCEVPoisonCollector {
4322 bool LookThroughMaybePoisonBlocking;
4323 SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
4324 SCEVPoisonCollector(bool LookThroughMaybePoisonBlocking)
4325 : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
4326
4327 bool follow(const SCEV *S) {
4328 if (!LookThroughMaybePoisonBlocking &&
4329 !scevUnconditionallyPropagatesPoisonFromOperands(Kind: S->getSCEVType()))
4330 return false;
4331
4332 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
4333 if (!isGuaranteedNotToBePoison(V: SU->getValue()))
4334 MaybePoison.insert(Ptr: SU);
4335 }
4336 return true;
4337 }
4338 bool isDone() const { return false; }
4339};
4340} // namespace
4341
4342/// Return true if V is poison given that AssumedPoison is already poison.
4343static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) {
4344 // First collect all SCEVs that might result in AssumedPoison to be poison.
4345 // We need to look through potentially poison-blocking operations here,
4346 // because we want to find all SCEVs that *might* result in poison, not only
4347 // those that are *required* to.
4348 SCEVPoisonCollector PC1(/* LookThroughMaybePoisonBlocking */ true);
4349 visitAll(Root: AssumedPoison, Visitor&: PC1);
4350
4351 // AssumedPoison is never poison. As the assumption is false, the implication
4352 // is true. Don't bother walking the other SCEV in this case.
4353 if (PC1.MaybePoison.empty())
4354 return true;
4355
4356 // Collect all SCEVs in S that, if poison, *will* result in S being poison
4357 // as well. We cannot look through potentially poison-blocking operations
4358 // here, as their arguments only *may* make the result poison.
4359 SCEVPoisonCollector PC2(/* LookThroughMaybePoisonBlocking */ false);
4360 visitAll(Root: S, Visitor&: PC2);
4361
4362 // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison,
4363 // it will also make S poison by being part of PC2.MaybePoison.
4364 return llvm::set_is_subset(S1: PC1.MaybePoison, S2: PC2.MaybePoison);
4365}
4366
4367void ScalarEvolution::getPoisonGeneratingValues(
4368 SmallPtrSetImpl<const Value *> &Result, const SCEV *S) {
4369 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ false);
4370 visitAll(Root: S, Visitor&: PC);
4371 for (const SCEVUnknown *SU : PC.MaybePoison)
4372 Result.insert(Ptr: SU->getValue());
4373}
4374
4375bool ScalarEvolution::canReuseInstruction(
4376 const SCEV *S, Instruction *I,
4377 SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts) {
4378 // If the instruction cannot be poison, it's always safe to reuse.
4379 if (programUndefinedIfPoison(Inst: I))
4380 return true;
4381
4382 // Otherwise, it is possible that I is more poisonous that S. Collect the
4383 // poison-contributors of S, and then check whether I has any additional
4384 // poison-contributors. Poison that is contributed through poison-generating
4385 // flags is handled by dropping those flags instead.
4386 SmallPtrSet<const Value *, 8> PoisonVals;
4387 getPoisonGeneratingValues(Result&: PoisonVals, S);
4388
4389 SmallVector<Value *> Worklist;
4390 SmallPtrSet<Value *, 8> Visited;
4391 Worklist.push_back(Elt: I);
4392 while (!Worklist.empty()) {
4393 Value *V = Worklist.pop_back_val();
4394 if (!Visited.insert(Ptr: V).second)
4395 continue;
4396
4397 // Avoid walking large instruction graphs.
4398 if (Visited.size() > 16)
4399 return false;
4400
4401 // Either the value can't be poison, or the S would also be poison if it
4402 // is.
4403 if (PoisonVals.contains(Ptr: V) || ::isGuaranteedNotToBePoison(V))
4404 continue;
4405
4406 auto *I = dyn_cast<Instruction>(Val: V);
4407 if (!I)
4408 return false;
4409
4410 // Disjoint or instructions are interpreted as adds by SCEV. However, we
4411 // can't replace an arbitrary add with disjoint or, even if we drop the
4412 // flag. We would need to convert the or into an add.
4413 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: I))
4414 if (PDI->isDisjoint())
4415 return false;
4416
4417 // FIXME: Ignore vscale, even though it technically could be poison. Do this
4418 // because SCEV currently assumes it can't be poison. Remove this special
4419 // case once we proper model when vscale can be poison.
4420 if (auto *II = dyn_cast<IntrinsicInst>(Val: I);
4421 II && II->getIntrinsicID() == Intrinsic::vscale)
4422 continue;
4423
4424 if (canCreatePoison(Op: cast<Operator>(Val: I), /*ConsiderFlagsAndMetadata*/ false))
4425 return false;
4426
4427 // If the instruction can't create poison, we can recurse to its operands.
4428 if (I->hasPoisonGeneratingAnnotations())
4429 DropPoisonGeneratingInsts.push_back(Elt: I);
4430
4431 llvm::append_range(C&: Worklist, R: I->operands());
4432 }
4433 return true;
4434}
4435
4436const SCEV *
4437ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
4438 SmallVectorImpl<SCEVUse> &Ops) {
4439 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
4440 "Not a SCEVSequentialMinMaxExpr!");
4441 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4442 if (Ops.size() == 1)
4443 return Ops[0];
4444#ifndef NDEBUG
4445 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4446 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4447 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4448 "Operand types don't match!");
4449 assert(Ops[0]->getType()->isPointerTy() ==
4450 Ops[i]->getType()->isPointerTy() &&
4451 "min/max should be consistently pointerish");
4452 }
4453#endif
4454
4455 // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
4456 // so we can *NOT* do any kind of sorting of the expressions!
4457
4458 // Check if we have created the same expression before.
4459 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops))
4460 return S;
4461
4462 // FIXME: there are *some* simplifications that we can do here.
4463
4464 // Keep only the first instance of an operand.
4465 {
4466 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
4467 bool Changed = Deduplicator.visit(Kind, OrigOps: Ops, NewOps&: Ops);
4468 if (Changed)
4469 return getSequentialMinMaxExpr(Kind, Ops);
4470 }
4471
4472 // Check to see if one of the operands is of the same kind. If so, expand its
4473 // operands onto our operand list, and recurse to simplify.
4474 {
4475 unsigned Idx = 0;
4476 bool DeletedAny = false;
4477 while (Idx < Ops.size()) {
4478 if (Ops[Idx]->getSCEVType() != Kind) {
4479 ++Idx;
4480 continue;
4481 }
4482 const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Val&: Ops[Idx]);
4483 Ops.erase(CI: Ops.begin() + Idx);
4484 Ops.insert(I: Ops.begin() + Idx, From: SMME->operands().begin(),
4485 To: SMME->operands().end());
4486 DeletedAny = true;
4487 }
4488
4489 if (DeletedAny)
4490 return getSequentialMinMaxExpr(Kind, Ops);
4491 }
4492
4493 const SCEV *SaturationPoint;
4494 ICmpInst::Predicate Pred;
4495 switch (Kind) {
4496 case scSequentialUMinExpr:
4497 SaturationPoint = getZero(Ty: Ops[0]->getType());
4498 Pred = ICmpInst::ICMP_ULE;
4499 break;
4500 default:
4501 llvm_unreachable("Not a sequential min/max type.");
4502 }
4503
4504 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4505 if (!isGuaranteedNotToCauseUB(Op: Ops[i]))
4506 continue;
4507 // We can replace %x umin_seq %y with %x umin %y if either:
4508 // * %y being poison implies %x is also poison.
4509 // * %x cannot be the saturating value (e.g. zero for umin).
4510 if (::impliesPoison(AssumedPoison: Ops[i], S: Ops[i - 1]) ||
4511 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_NE, LHS: Ops[i - 1],
4512 RHS: SaturationPoint)) {
4513 SmallVector<SCEVUse, 2> SeqOps = {Ops[i - 1], Ops[i]};
4514 Ops[i - 1] = getMinMaxExpr(
4515 Kind: SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Ty: Kind),
4516 Ops&: SeqOps);
4517 Ops.erase(CI: Ops.begin() + i);
4518 return getSequentialMinMaxExpr(Kind, Ops);
4519 }
4520 // Fold %x umin_seq %y to %x if %x ule %y.
4521 // TODO: We might be able to prove the predicate for a later operand.
4522 if (isKnownViaNonRecursiveReasoning(Pred, LHS: Ops[i - 1], RHS: Ops[i])) {
4523 Ops.erase(CI: Ops.begin() + i);
4524 return getSequentialMinMaxExpr(Kind, Ops);
4525 }
4526 }
4527
4528 // Okay, it looks like we really DO need an expr. Check to see if we
4529 // already have one, otherwise create a new one.
4530 FoldingSetNodeID ID;
4531 ID.AddInteger(I: Kind);
4532 for (const SCEV *Op : Ops)
4533 ID.AddPointer(Ptr: Op);
4534 void *IP = nullptr;
4535 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
4536 if (ExistingSCEV)
4537 return ExistingSCEV;
4538
4539 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
4540 llvm::uninitialized_copy(Src&: Ops, Dst: O);
4541 SCEV *S = new (SCEVAllocator)
4542 SCEVSequentialMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
4543
4544 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
4545 S->computeAndSetCanonical(SE&: *this);
4546 registerUser(User: S, Ops);
4547 return S;
4548}
4549
4550const SCEV *ScalarEvolution::getSMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4551 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4552 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4553}
4554
4555const SCEV *ScalarEvolution::getSMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4556 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4557}
4558
4559const SCEV *ScalarEvolution::getUMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4560 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4561 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4562}
4563
4564const SCEV *ScalarEvolution::getUMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4565 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4566}
4567
4568const SCEV *ScalarEvolution::getSMinExpr(SCEVUse LHS, SCEVUse RHS) {
4569 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4570 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4571}
4572
4573const SCEV *ScalarEvolution::getSMinExpr(SmallVectorImpl<SCEVUse> &Ops) {
4574 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4575}
4576
4577const SCEV *ScalarEvolution::getUMinExpr(SCEVUse LHS, SCEVUse RHS,
4578 bool Sequential) {
4579 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4580 return getUMinExpr(Operands&: Ops, Sequential);
4581}
4582
4583const SCEV *ScalarEvolution::getUMinExpr(SmallVectorImpl<SCEVUse> &Ops,
4584 bool Sequential) {
4585 return Sequential ? getSequentialMinMaxExpr(Kind: scSequentialUMinExpr, Ops)
4586 : getMinMaxExpr(Kind: scUMinExpr, Ops);
4587}
4588
4589const SCEV *
4590ScalarEvolution::getSizeOfExpr(Type *IntTy, TypeSize Size) {
4591 const SCEV *Res = getConstant(Ty: IntTy, V: Size.getKnownMinValue());
4592 if (Size.isScalable())
4593 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty: IntTy));
4594 return Res;
4595}
4596
4597const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
4598 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeAllocSize(Ty: AllocTy));
4599}
4600
4601const SCEV *ScalarEvolution::getStoreSizeOfExpr(Type *IntTy, Type *StoreTy) {
4602 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeStoreSize(Ty: StoreTy));
4603}
4604
4605const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
4606 StructType *STy,
4607 unsigned FieldNo) {
4608 // We can bypass creating a target-independent constant expression and then
4609 // folding it back into a ConstantInt. This is just a compile-time
4610 // optimization.
4611 const StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
4612 assert(!SL->getSizeInBits().isScalable() &&
4613 "Cannot get offset for structure containing scalable vector types");
4614 return getConstant(Ty: IntTy, V: SL->getElementOffset(Idx: FieldNo));
4615}
4616
4617const SCEV *ScalarEvolution::getUnknown(Value *V) {
4618 // Don't attempt to do anything other than create a SCEVUnknown object
4619 // here. createSCEV only calls getUnknown after checking for all other
4620 // interesting possibilities, and any other code that calls getUnknown
4621 // is doing so in order to hide a value from SCEV canonicalization.
4622
4623 FoldingSetNodeID ID;
4624 ID.AddInteger(I: scUnknown);
4625 ID.AddPointer(Ptr: V);
4626 void *IP = nullptr;
4627 if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) {
4628 assert(cast<SCEVUnknown>(S)->getValue() == V &&
4629 "Stale SCEVUnknown in uniquing map!");
4630 return S;
4631 }
4632 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(Allocator&: SCEVAllocator), V, this,
4633 FirstUnknown);
4634 FirstUnknown = cast<SCEVUnknown>(Val: S);
4635 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
4636 S->computeAndSetCanonical(SE&: *this);
4637 return S;
4638}
4639
4640//===----------------------------------------------------------------------===//
4641// Basic SCEV Analysis and PHI Idiom Recognition Code
4642//
4643
4644/// Test if values of the given type are analyzable within the SCEV
4645/// framework. This primarily includes integer types, and it can optionally
4646/// include pointer types if the ScalarEvolution class has access to
4647/// target-specific information.
4648bool ScalarEvolution::isSCEVable(Type *Ty) const {
4649 // Integers and pointers are always SCEVable.
4650 return Ty->isIntOrPtrTy();
4651}
4652
4653/// Return the size in bits of the specified type, for which isSCEVable must
4654/// return true.
4655uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
4656 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4657 if (Ty->isPointerTy())
4658 return getDataLayout().getIndexTypeSizeInBits(Ty);
4659 return getDataLayout().getTypeSizeInBits(Ty);
4660}
4661
4662/// Return a type with the same bitwidth as the given type and which represents
4663/// how SCEV will treat the given type, for which isSCEVable must return
4664/// true. For pointer types, this is the pointer index sized integer type.
4665Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
4666 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4667
4668 if (Ty->isIntegerTy())
4669 return Ty;
4670
4671 // The only other support type is pointer.
4672 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
4673 return getDataLayout().getIndexType(PtrTy: Ty);
4674}
4675
4676Type *ScalarEvolution::getWiderType(Type *T1, Type *T2) const {
4677 return getTypeSizeInBits(Ty: T1) >= getTypeSizeInBits(Ty: T2) ? T1 : T2;
4678}
4679
4680bool ScalarEvolution::instructionCouldExistWithOperands(const SCEV *A,
4681 const SCEV *B) {
4682 /// For a valid use point to exist, the defining scope of one operand
4683 /// must dominate the other.
4684 bool PreciseA, PreciseB;
4685 auto *ScopeA = getDefiningScopeBound(Ops: {A}, Precise&: PreciseA);
4686 auto *ScopeB = getDefiningScopeBound(Ops: {B}, Precise&: PreciseB);
4687 if (!PreciseA || !PreciseB)
4688 // Can't tell.
4689 return false;
4690 return (ScopeA == ScopeB) || DT.dominates(Def: ScopeA, User: ScopeB) ||
4691 DT.dominates(Def: ScopeB, User: ScopeA);
4692}
4693
4694const SCEV *ScalarEvolution::getCouldNotCompute() {
4695 return CouldNotCompute.get();
4696}
4697
4698bool ScalarEvolution::checkValidity(const SCEV *S) const {
4699 bool ContainsNulls = SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
4700 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
4701 return SU && SU->getValue() == nullptr;
4702 });
4703
4704 return !ContainsNulls;
4705}
4706
4707bool ScalarEvolution::containsAddRecurrence(const SCEV *S) {
4708 HasRecMapType::iterator I = HasRecMap.find(Val: S);
4709 if (I != HasRecMap.end())
4710 return I->second;
4711
4712 bool FoundAddRec =
4713 SCEVExprContains(Root: S, Pred: [](const SCEV *S) { return isa<SCEVAddRecExpr>(Val: S); });
4714 HasRecMap.insert(KV: {S, FoundAddRec});
4715 return FoundAddRec;
4716}
4717
4718/// Return the ValueOffsetPair set for \p S. \p S can be represented
4719/// by the value and offset from any ValueOffsetPair in the set.
4720ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) {
4721 ExprValueMapType::iterator SI = ExprValueMap.find_as(Val: S);
4722 if (SI == ExprValueMap.end())
4723 return {};
4724 return SI->second.getArrayRef();
4725}
4726
4727/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
4728/// cannot be used separately. eraseValueFromMap should be used to remove
4729/// V from ValueExprMap and ExprValueMap at the same time.
4730void ScalarEvolution::eraseValueFromMap(Value *V) {
4731 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4732 if (I != ValueExprMap.end()) {
4733 auto EVIt = ExprValueMap.find(Val: I->second);
4734 bool Removed = EVIt->second.remove(X: V);
4735 (void) Removed;
4736 assert(Removed && "Value not in ExprValueMap?");
4737 ValueExprMap.erase(I);
4738 }
4739}
4740
4741void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
4742 // A recursive query may have already computed the SCEV. It should be
4743 // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
4744 // inferred nowrap flags.
4745 auto It = ValueExprMap.find_as(Val: V);
4746 if (It == ValueExprMap.end()) {
4747 ValueExprMap.insert(KV: {SCEVCallbackVH(V, this), S});
4748 ExprValueMap[S].insert(X: V);
4749 }
4750}
4751
4752/// Return an existing SCEV if it exists, otherwise analyze the expression and
4753/// create a new one.
4754const SCEV *ScalarEvolution::getSCEV(Value *V) {
4755 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4756
4757 if (const SCEV *S = getExistingSCEV(V))
4758 return S;
4759 return createSCEVIter(V);
4760}
4761
4762const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
4763 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4764
4765 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4766 if (I != ValueExprMap.end()) {
4767 const SCEV *S = I->second;
4768 assert(checkValidity(S) &&
4769 "existing SCEV has not been properly invalidated");
4770 return S;
4771 }
4772 return nullptr;
4773}
4774
4775/// Return a SCEV corresponding to -V = -1*V
4776const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V,
4777 SCEV::NoWrapFlags Flags) {
4778 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4779 return getConstant(
4780 V: cast<ConstantInt>(Val: ConstantExpr::getNeg(C: VC->getValue())));
4781
4782 Type *Ty = V->getType();
4783 Ty = getEffectiveSCEVType(Ty);
4784 return getMulExpr(LHS: V, RHS: getMinusOne(Ty), Flags);
4785}
4786
4787/// If Expr computes ~A, return A else return nullptr
4788static const SCEV *MatchNotExpr(const SCEV *Expr) {
4789 const SCEV *MulOp;
4790 if (match(S: Expr, P: m_scev_Add(Op0: m_scev_AllOnes(),
4791 Op1: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: MulOp)))))
4792 return MulOp;
4793 return nullptr;
4794}
4795
4796/// Return a SCEV corresponding to ~V = -1-V
4797const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
4798 assert(!V->getType()->isPointerTy() && "Can't negate pointer");
4799
4800 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4801 return getConstant(
4802 V: cast<ConstantInt>(Val: ConstantExpr::getNot(C: VC->getValue())));
4803
4804 // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
4805 if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(Val: V)) {
4806 auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
4807 SmallVector<SCEVUse, 2> MatchedOperands;
4808 for (const SCEV *Operand : MME->operands()) {
4809 const SCEV *Matched = MatchNotExpr(Expr: Operand);
4810 if (!Matched)
4811 return (const SCEV *)nullptr;
4812 MatchedOperands.push_back(Elt: Matched);
4813 }
4814 return getMinMaxExpr(Kind: SCEVMinMaxExpr::negate(T: MME->getSCEVType()),
4815 Ops&: MatchedOperands);
4816 };
4817 if (const SCEV *Replaced = MatchMinMaxNegation(MME))
4818 return Replaced;
4819 }
4820
4821 Type *Ty = V->getType();
4822 Ty = getEffectiveSCEVType(Ty);
4823 return getMinusSCEV(LHS: getMinusOne(Ty), RHS: V);
4824}
4825
4826const SCEV *ScalarEvolution::removePointerBase(const SCEV *P) {
4827 assert(P->getType()->isPointerTy());
4828
4829 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: P)) {
4830 // The base of an AddRec is the first operand.
4831 SmallVector<SCEVUse> Ops{AddRec->operands()};
4832 Ops[0] = removePointerBase(P: Ops[0]);
4833 // Don't try to transfer nowrap flags for now. We could in some cases
4834 // (for example, if pointer operand of the AddRec is a SCEVUnknown).
4835 return getAddRecExpr(Operands&: Ops, L: AddRec->getLoop(), Flags: SCEV::FlagAnyWrap);
4836 }
4837 if (auto *Add = dyn_cast<SCEVAddExpr>(Val: P)) {
4838 // The base of an Add is the pointer operand.
4839 SmallVector<SCEVUse> Ops{Add->operands()};
4840 SCEVUse *PtrOp = nullptr;
4841 for (SCEVUse &AddOp : Ops) {
4842 if (AddOp->getType()->isPointerTy()) {
4843 assert(!PtrOp && "Cannot have multiple pointer ops");
4844 PtrOp = &AddOp;
4845 }
4846 }
4847 *PtrOp = removePointerBase(P: *PtrOp);
4848 // Don't try to transfer nowrap flags for now. We could in some cases
4849 // (for example, if the pointer operand of the Add is a SCEVUnknown).
4850 return getAddExpr(Ops);
4851 }
4852 // Any other expression must be a pointer base.
4853 return getZero(Ty: P->getType());
4854}
4855
4856const SCEV *ScalarEvolution::getMinusSCEV(SCEVUse LHS, SCEVUse RHS,
4857 SCEV::NoWrapFlags Flags,
4858 unsigned Depth) {
4859 // Fast path: X - X --> 0.
4860 if (LHS == RHS)
4861 return getZero(Ty: LHS->getType());
4862
4863 // If we subtract two pointers with different pointer bases, bail.
4864 // Eventually, we're going to add an assertion to getMulExpr that we
4865 // can't multiply by a pointer.
4866 if (RHS->getType()->isPointerTy()) {
4867 if (!LHS->getType()->isPointerTy() ||
4868 getPointerBase(V: LHS) != getPointerBase(V: RHS))
4869 return getCouldNotCompute();
4870 LHS = removePointerBase(P: LHS);
4871 RHS = removePointerBase(P: RHS);
4872 }
4873
4874 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
4875 // makes it so that we cannot make much use of NUW.
4876 auto AddFlags = SCEV::FlagAnyWrap;
4877 const bool RHSIsNotMinSigned =
4878 !getSignedRangeMin(S: RHS).isMinSignedValue();
4879 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW)) {
4880 // Let M be the minimum representable signed value. Then (-1)*RHS
4881 // signed-wraps if and only if RHS is M. That can happen even for
4882 // a NSW subtraction because e.g. (-1)*M signed-wraps even though
4883 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
4884 // (-1)*RHS, we need to prove that RHS != M.
4885 //
4886 // If LHS is non-negative and we know that LHS - RHS does not
4887 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
4888 // either by proving that RHS > M or that LHS >= 0.
4889 if (RHSIsNotMinSigned || isKnownNonNegative(S: LHS)) {
4890 AddFlags = SCEV::FlagNSW;
4891 }
4892 }
4893
4894 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
4895 // RHS is NSW and LHS >= 0.
4896 //
4897 // The difficulty here is that the NSW flag may have been proven
4898 // relative to a loop that is to be found in a recurrence in LHS and
4899 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
4900 // larger scope than intended.
4901 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
4902
4903 return getAddExpr(LHS, RHS: getNegativeSCEV(V: RHS, Flags: NegFlags), Flags: AddFlags, Depth);
4904}
4905
4906const SCEV *ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty,
4907 unsigned Depth) {
4908 Type *SrcTy = V->getType();
4909 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4910 "Cannot truncate or zero extend with non-integer arguments!");
4911 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4912 return V; // No conversion
4913 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4914 return getTruncateExpr(Op: V, Ty, Depth);
4915 return getZeroExtendExpr(Op: V, Ty, Depth);
4916}
4917
4918const SCEV *ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, Type *Ty,
4919 unsigned Depth) {
4920 Type *SrcTy = V->getType();
4921 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4922 "Cannot truncate or zero extend with non-integer arguments!");
4923 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4924 return V; // No conversion
4925 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4926 return getTruncateExpr(Op: V, Ty, Depth);
4927 return getSignExtendExpr(Op: V, Ty, Depth);
4928}
4929
4930const SCEV *
4931ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
4932 Type *SrcTy = V->getType();
4933 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4934 "Cannot noop or zero extend with non-integer arguments!");
4935 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4936 "getNoopOrZeroExtend cannot truncate!");
4937 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4938 return V; // No conversion
4939 return getZeroExtendExpr(Op: V, Ty);
4940}
4941
4942const SCEV *
4943ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
4944 Type *SrcTy = V->getType();
4945 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4946 "Cannot noop or sign extend with non-integer arguments!");
4947 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4948 "getNoopOrSignExtend cannot truncate!");
4949 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4950 return V; // No conversion
4951 return getSignExtendExpr(Op: V, Ty);
4952}
4953
4954const SCEV *
4955ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
4956 Type *SrcTy = V->getType();
4957 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4958 "Cannot noop or any extend with non-integer arguments!");
4959 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4960 "getNoopOrAnyExtend cannot truncate!");
4961 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4962 return V; // No conversion
4963 return getAnyExtendExpr(Op: V, Ty);
4964}
4965
4966const SCEV *
4967ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
4968 Type *SrcTy = V->getType();
4969 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4970 "Cannot truncate or noop with non-integer arguments!");
4971 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
4972 "getTruncateOrNoop cannot extend!");
4973 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4974 return V; // No conversion
4975 return getTruncateExpr(Op: V, Ty);
4976}
4977
4978const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
4979 const SCEV *RHS) {
4980 const SCEV *PromotedLHS = LHS;
4981 const SCEV *PromotedRHS = RHS;
4982
4983 if (getTypeSizeInBits(Ty: LHS->getType()) > getTypeSizeInBits(Ty: RHS->getType()))
4984 PromotedRHS = getZeroExtendExpr(Op: RHS, Ty: LHS->getType());
4985 else
4986 PromotedLHS = getNoopOrZeroExtend(V: LHS, Ty: RHS->getType());
4987
4988 return getUMaxExpr(LHS: PromotedLHS, RHS: PromotedRHS);
4989}
4990
4991const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
4992 const SCEV *RHS,
4993 bool Sequential) {
4994 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4995 return getUMinFromMismatchedTypes(Ops, Sequential);
4996}
4997
4998const SCEV *
4999ScalarEvolution::getUMinFromMismatchedTypes(SmallVectorImpl<SCEVUse> &Ops,
5000 bool Sequential) {
5001 assert(!Ops.empty() && "At least one operand must be!");
5002 // Trivial case.
5003 if (Ops.size() == 1)
5004 return Ops[0];
5005
5006 // Find the max type first.
5007 Type *MaxType = nullptr;
5008 for (SCEVUse S : Ops)
5009 if (MaxType)
5010 MaxType = getWiderType(T1: MaxType, T2: S->getType());
5011 else
5012 MaxType = S->getType();
5013 assert(MaxType && "Failed to find maximum type!");
5014
5015 // Extend all ops to max type.
5016 SmallVector<SCEVUse, 2> PromotedOps;
5017 for (SCEVUse S : Ops)
5018 PromotedOps.push_back(Elt: getNoopOrZeroExtend(V: S, Ty: MaxType));
5019
5020 // Generate umin.
5021 return getUMinExpr(Ops&: PromotedOps, Sequential);
5022}
5023
5024const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
5025 // A pointer operand may evaluate to a nonpointer expression, such as null.
5026 if (!V->getType()->isPointerTy())
5027 return V;
5028
5029 while (true) {
5030 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: V)) {
5031 V = AddRec->getStart();
5032 } else if (auto *Add = dyn_cast<SCEVAddExpr>(Val: V)) {
5033 const SCEV *PtrOp = nullptr;
5034 for (const SCEV *AddOp : Add->operands()) {
5035 if (AddOp->getType()->isPointerTy()) {
5036 assert(!PtrOp && "Cannot have multiple pointer ops");
5037 PtrOp = AddOp;
5038 }
5039 }
5040 assert(PtrOp && "Must have pointer op");
5041 V = PtrOp;
5042 } else // Not something we can look further into.
5043 return V;
5044 }
5045}
5046
5047/// Push users of the given Instruction onto the given Worklist.
5048static void PushDefUseChildren(Instruction *I,
5049 SmallVectorImpl<Instruction *> &Worklist,
5050 SmallPtrSetImpl<Instruction *> &Visited) {
5051 // Push the def-use children onto the Worklist stack.
5052 for (User *U : I->users()) {
5053 auto *UserInsn = cast<Instruction>(Val: U);
5054 if (Visited.insert(Ptr: UserInsn).second)
5055 Worklist.push_back(Elt: UserInsn);
5056 }
5057}
5058
5059namespace {
5060
5061/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
5062/// expression in case its Loop is L. If it is not L then
5063/// if IgnoreOtherLoops is true then use AddRec itself
5064/// otherwise rewrite cannot be done.
5065/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
5066class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
5067public:
5068 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
5069 bool IgnoreOtherLoops = true) {
5070 SCEVInitRewriter Rewriter(L, SE);
5071 const SCEV *Result = Rewriter.visit(S);
5072 if (Rewriter.hasSeenLoopVariantSCEVUnknown())
5073 return SE.getCouldNotCompute();
5074 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
5075 ? SE.getCouldNotCompute()
5076 : Result;
5077 }
5078
5079 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5080 if (!SE.isLoopInvariant(S: Expr, L))
5081 SeenLoopVariantSCEVUnknown = true;
5082 return Expr;
5083 }
5084
5085 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5086 // Only re-write AddRecExprs for this loop.
5087 if (Expr->getLoop() == L)
5088 return Expr->getStart();
5089 SeenOtherLoops = true;
5090 return Expr;
5091 }
5092
5093 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
5094
5095 bool hasSeenOtherLoops() { return SeenOtherLoops; }
5096
5097private:
5098 explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
5099 : SCEVRewriteVisitor(SE), L(L) {}
5100
5101 const Loop *L;
5102 bool SeenLoopVariantSCEVUnknown = false;
5103 bool SeenOtherLoops = false;
5104};
5105
5106/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
5107/// increment expression in case its Loop is L. If it is not L then
5108/// use AddRec itself.
5109/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
5110class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
5111public:
5112 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
5113 SCEVPostIncRewriter Rewriter(L, SE);
5114 const SCEV *Result = Rewriter.visit(S);
5115 return Rewriter.hasSeenLoopVariantSCEVUnknown()
5116 ? SE.getCouldNotCompute()
5117 : Result;
5118 }
5119
5120 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5121 if (!SE.isLoopInvariant(S: Expr, L))
5122 SeenLoopVariantSCEVUnknown = true;
5123 return Expr;
5124 }
5125
5126 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5127 // Only re-write AddRecExprs for this loop.
5128 if (Expr->getLoop() == L)
5129 return Expr->getPostIncExpr(SE);
5130 SeenOtherLoops = true;
5131 return Expr;
5132 }
5133
5134 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
5135
5136 bool hasSeenOtherLoops() { return SeenOtherLoops; }
5137
5138private:
5139 explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
5140 : SCEVRewriteVisitor(SE), L(L) {}
5141
5142 const Loop *L;
5143 bool SeenLoopVariantSCEVUnknown = false;
5144 bool SeenOtherLoops = false;
5145};
5146
5147/// This class evaluates the compare condition by matching it against the
5148/// condition of loop latch. If there is a match we assume a true value
5149/// for the condition while building SCEV nodes.
5150class SCEVBackedgeConditionFolder
5151 : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
5152public:
5153 static const SCEV *rewrite(const SCEV *S, const Loop *L,
5154 ScalarEvolution &SE) {
5155 bool IsPosBECond = false;
5156 Value *BECond = nullptr;
5157 if (BasicBlock *Latch = L->getLoopLatch()) {
5158 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Latch->getTerminator())) {
5159 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
5160 "Both outgoing branches should not target same header!");
5161 BECond = BI->getCondition();
5162 IsPosBECond = BI->getSuccessor(i: 0) == L->getHeader();
5163 } else {
5164 return S;
5165 }
5166 }
5167 SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
5168 return Rewriter.visit(S);
5169 }
5170
5171 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5172 const SCEV *Result = Expr;
5173 bool InvariantF = SE.isLoopInvariant(S: Expr, L);
5174
5175 if (!InvariantF) {
5176 Instruction *I = cast<Instruction>(Val: Expr->getValue());
5177 switch (I->getOpcode()) {
5178 case Instruction::Select: {
5179 SelectInst *SI = cast<SelectInst>(Val: I);
5180 std::optional<const SCEV *> Res =
5181 compareWithBackedgeCondition(IC: SI->getCondition());
5182 if (Res) {
5183 bool IsOne = cast<SCEVConstant>(Val: *Res)->getValue()->isOne();
5184 Result = SE.getSCEV(V: IsOne ? SI->getTrueValue() : SI->getFalseValue());
5185 }
5186 break;
5187 }
5188 default: {
5189 std::optional<const SCEV *> Res = compareWithBackedgeCondition(IC: I);
5190 if (Res)
5191 Result = *Res;
5192 break;
5193 }
5194 }
5195 }
5196 return Result;
5197 }
5198
5199private:
5200 explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
5201 bool IsPosBECond, ScalarEvolution &SE)
5202 : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
5203 IsPositiveBECond(IsPosBECond) {}
5204
5205 std::optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
5206
5207 const Loop *L;
5208 /// Loop back condition.
5209 Value *BackedgeCond = nullptr;
5210 /// Set to true if loop back is on positive branch condition.
5211 bool IsPositiveBECond;
5212};
5213
5214std::optional<const SCEV *>
5215SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
5216
5217 // If value matches the backedge condition for loop latch,
5218 // then return a constant evolution node based on loopback
5219 // branch taken.
5220 if (BackedgeCond == IC)
5221 return IsPositiveBECond ? SE.getOne(Ty: Type::getInt1Ty(C&: SE.getContext()))
5222 : SE.getZero(Ty: Type::getInt1Ty(C&: SE.getContext()));
5223 return std::nullopt;
5224}
5225
5226class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
5227public:
5228 static const SCEV *rewrite(const SCEV *S, const Loop *L,
5229 ScalarEvolution &SE) {
5230 SCEVShiftRewriter Rewriter(L, SE);
5231 const SCEV *Result = Rewriter.visit(S);
5232 return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
5233 }
5234
5235 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5236 // Only allow AddRecExprs for this loop.
5237 if (!SE.isLoopInvariant(S: Expr, L))
5238 Valid = false;
5239 return Expr;
5240 }
5241
5242 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5243 if (Expr->getLoop() == L && Expr->isAffine())
5244 return SE.getMinusSCEV(LHS: Expr, RHS: Expr->getStepRecurrence(SE));
5245 Valid = false;
5246 return Expr;
5247 }
5248
5249 bool isValid() { return Valid; }
5250
5251private:
5252 explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
5253 : SCEVRewriteVisitor(SE), L(L) {}
5254
5255 const Loop *L;
5256 bool Valid = true;
5257};
5258
5259} // end anonymous namespace
5260
5261void ScalarEvolution::inferNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
5262 if (!AR->isAffine())
5263 return;
5264
5265 // Force computation of ranges, which will also perform range-based flag
5266 // inference.
5267 if (!AR->hasNoSignedWrap())
5268 (void)getSignedRange(S: AR);
5269
5270 if (!AR->hasNoUnsignedWrap())
5271 (void)getUnsignedRange(S: AR);
5272
5273 if (!AR->hasNoSelfWrap()) {
5274 const SCEV *BECount = getConstantMaxBackedgeTakenCount(L: AR->getLoop());
5275 if (const SCEVConstant *BECountMax = dyn_cast<SCEVConstant>(Val: BECount)) {
5276 ConstantRange StepCR = getSignedRange(S: AR->getStepRecurrence(SE&: *this));
5277 const APInt &BECountAP = BECountMax->getAPInt();
5278 unsigned NoOverflowBitWidth =
5279 BECountAP.getActiveBits() + StepCR.getMinSignedBits();
5280 if (NoOverflowBitWidth <= getTypeSizeInBits(Ty: AR->getType()))
5281 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
5282 }
5283 }
5284}
5285
5286SCEV::NoWrapFlags
5287ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5288 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
5289
5290 if (AR->hasNoSignedWrap())
5291 return Result;
5292
5293 if (!AR->isAffine())
5294 return Result;
5295
5296 // This function can be expensive, only try to prove NSW once per AddRec.
5297 if (!SignedWrapViaInductionTried.insert(Ptr: AR).second)
5298 return Result;
5299
5300 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5301 const Loop *L = AR->getLoop();
5302
5303 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5304 // Note that this serves two purposes: It filters out loops that are
5305 // simply not analyzable, and it covers the case where this code is
5306 // being called from within backedge-taken count analysis, such that
5307 // attempting to ask for the backedge-taken count would likely result
5308 // in infinite recursion. In the later case, the analysis code will
5309 // cope with a conservative value, and it will take care to purge
5310 // that value once it has finished.
5311 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5312
5313 // Normally, in the cases we can prove no-overflow via a
5314 // backedge guarding condition, we can also compute a backedge
5315 // taken count for the loop. The exceptions are assumptions and
5316 // guards present in the loop -- SCEV is not great at exploiting
5317 // these to compute max backedge taken counts, but can still use
5318 // these to prove lack of overflow. Use this fact to avoid
5319 // doing extra work that may not pay off.
5320
5321 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5322 AC.assumptions().empty())
5323 return Result;
5324
5325 // If the backedge is guarded by a comparison with the pre-inc value the
5326 // addrec is safe. Also, if the entry is guarded by a comparison with the
5327 // start value and the backedge is guarded by a comparison with the post-inc
5328 // value, the addrec is safe.
5329 ICmpInst::Predicate Pred;
5330 const SCEV *OverflowLimit =
5331 getSignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
5332 if (OverflowLimit &&
5333 (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
5334 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))) {
5335 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNSW);
5336 }
5337 return Result;
5338}
5339SCEV::NoWrapFlags
5340ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5341 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
5342
5343 if (AR->hasNoUnsignedWrap())
5344 return Result;
5345
5346 if (!AR->isAffine())
5347 return Result;
5348
5349 // This function can be expensive, only try to prove NUW once per AddRec.
5350 if (!UnsignedWrapViaInductionTried.insert(Ptr: AR).second)
5351 return Result;
5352
5353 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5354 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
5355 const Loop *L = AR->getLoop();
5356
5357 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5358 // Note that this serves two purposes: It filters out loops that are
5359 // simply not analyzable, and it covers the case where this code is
5360 // being called from within backedge-taken count analysis, such that
5361 // attempting to ask for the backedge-taken count would likely result
5362 // in infinite recursion. In the later case, the analysis code will
5363 // cope with a conservative value, and it will take care to purge
5364 // that value once it has finished.
5365 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5366
5367 // Normally, in the cases we can prove no-overflow via a
5368 // backedge guarding condition, we can also compute a backedge
5369 // taken count for the loop. The exceptions are assumptions and
5370 // guards present in the loop -- SCEV is not great at exploiting
5371 // these to compute max backedge taken counts, but can still use
5372 // these to prove lack of overflow. Use this fact to avoid
5373 // doing extra work that may not pay off.
5374
5375 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5376 AC.assumptions().empty())
5377 return Result;
5378
5379 // If the backedge is guarded by a comparison with the pre-inc value the
5380 // addrec is safe. Also, if the entry is guarded by a comparison with the
5381 // start value and the backedge is guarded by a comparison with the post-inc
5382 // value, the addrec is safe.
5383 if (isKnownPositive(S: Step)) {
5384 const SCEV *N = getConstant(Val: APInt::getMinValue(numBits: BitWidth) -
5385 getUnsignedRangeMax(S: Step));
5386 if (isLoopBackedgeGuardedByCond(L, Pred: ICmpInst::ICMP_ULT, LHS: AR, RHS: N) ||
5387 isKnownOnEveryIteration(Pred: ICmpInst::ICMP_ULT, LHS: AR, RHS: N)) {
5388 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNUW);
5389 }
5390 }
5391
5392 return Result;
5393}
5394
5395namespace {
5396
5397/// Represents an abstract binary operation. This may exist as a
5398/// normal instruction or constant expression, or may have been
5399/// derived from an expression tree.
5400struct BinaryOp {
5401 unsigned Opcode;
5402 Value *LHS;
5403 Value *RHS;
5404 bool IsNSW = false;
5405 bool IsNUW = false;
5406
5407 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
5408 /// constant expression.
5409 Operator *Op = nullptr;
5410
5411 explicit BinaryOp(Operator *Op)
5412 : Opcode(Op->getOpcode()), LHS(Op->getOperand(i: 0)), RHS(Op->getOperand(i: 1)),
5413 Op(Op) {
5414 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: Op)) {
5415 IsNSW = OBO->hasNoSignedWrap();
5416 IsNUW = OBO->hasNoUnsignedWrap();
5417 }
5418 }
5419
5420 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
5421 bool IsNUW = false)
5422 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5423};
5424
5425} // end anonymous namespace
5426
5427/// Try to map \p V into a BinaryOp, and return \c std::nullopt on failure.
5428static std::optional<BinaryOp> MatchBinaryOp(Value *V, const DataLayout &DL,
5429 AssumptionCache &AC,
5430 const DominatorTree &DT,
5431 const Instruction *CxtI) {
5432 auto *Op = dyn_cast<Operator>(Val: V);
5433 if (!Op)
5434 return std::nullopt;
5435
5436 // Implementation detail: all the cleverness here should happen without
5437 // creating new SCEV expressions -- our caller knowns tricks to avoid creating
5438 // SCEV expressions when possible, and we should not break that.
5439
5440 switch (Op->getOpcode()) {
5441 case Instruction::Add:
5442 case Instruction::Sub:
5443 case Instruction::Mul:
5444 case Instruction::UDiv:
5445 case Instruction::URem:
5446 case Instruction::And:
5447 case Instruction::AShr:
5448 case Instruction::Shl:
5449 return BinaryOp(Op);
5450
5451 case Instruction::Or: {
5452 // Convert or disjoint into add nuw nsw.
5453 if (cast<PossiblyDisjointInst>(Val: Op)->isDisjoint()) {
5454 BinaryOp BinOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1),
5455 /*IsNSW=*/true, /*IsNUW=*/true);
5456 // Keep the reference to the original instruction so that we can later
5457 // check whether it can produce poison value or not.
5458 BinOp.Op = Op;
5459 return BinOp;
5460 }
5461 return BinaryOp(Op);
5462 }
5463
5464 case Instruction::Xor:
5465 if (auto *RHSC = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1)))
5466 // If the RHS of the xor is a signmask, then this is just an add.
5467 // Instcombine turns add of signmask into xor as a strength reduction step.
5468 if (RHSC->getValue().isSignMask())
5469 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5470 // Binary `xor` is a bit-wise `add`.
5471 if (V->getType()->isIntegerTy(BitWidth: 1))
5472 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5473 return BinaryOp(Op);
5474
5475 case Instruction::LShr:
5476 // Turn logical shift right of a constant into a unsigned divide.
5477 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1))) {
5478 uint32_t BitWidth = cast<IntegerType>(Val: Op->getType())->getBitWidth();
5479
5480 // If the shift count is not less than the bitwidth, the result of
5481 // the shift is undefined. Don't try to analyze it, because the
5482 // resolution chosen here may differ from the resolution chosen in
5483 // other parts of the compiler.
5484 if (SA->getValue().ult(RHS: BitWidth)) {
5485 Constant *X =
5486 ConstantInt::get(Context&: SA->getContext(),
5487 V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
5488 return BinaryOp(Instruction::UDiv, Op->getOperand(i: 0), X);
5489 }
5490 }
5491 return BinaryOp(Op);
5492
5493 case Instruction::ExtractValue: {
5494 auto *EVI = cast<ExtractValueInst>(Val: Op);
5495 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5496 break;
5497
5498 auto *WO = dyn_cast<WithOverflowInst>(Val: EVI->getAggregateOperand());
5499 if (!WO)
5500 break;
5501
5502 Instruction::BinaryOps BinOp = WO->getBinaryOp();
5503 bool Signed = WO->isSigned();
5504 // TODO: Should add nuw/nsw flags for mul as well.
5505 if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
5506 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5507
5508 // Now that we know that all uses of the arithmetic-result component of
5509 // CI are guarded by the overflow check, we can go ahead and pretend
5510 // that the arithmetic is non-overflowing.
5511 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5512 /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
5513 }
5514
5515 default:
5516 break;
5517 }
5518
5519 // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
5520 // semantics as a Sub, return a binary sub expression.
5521 if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
5522 if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5523 return BinaryOp(Instruction::Sub, II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1));
5524
5525 return std::nullopt;
5526}
5527
5528/// Helper function to createAddRecFromPHIWithCasts. We have a phi
5529/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
5530/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
5531/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
5532/// follows one of the following patterns:
5533/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5534/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5535/// If the SCEV expression of \p Op conforms with one of the expected patterns
5536/// we return the type of the truncation operation, and indicate whether the
5537/// truncated type should be treated as signed/unsigned by setting
5538/// \p Signed to true/false, respectively.
5539static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
5540 bool &Signed, ScalarEvolution &SE) {
5541 // The case where Op == SymbolicPHI (that is, with no type conversions on
5542 // the way) is handled by the regular add recurrence creating logic and
5543 // would have already been triggered in createAddRecForPHI. Reaching it here
5544 // means that createAddRecFromPHI had failed for this PHI before (e.g.,
5545 // because one of the other operands of the SCEVAddExpr updating this PHI is
5546 // not invariant).
5547 //
5548 // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
5549 // this case predicates that allow us to prove that Op == SymbolicPHI will
5550 // be added.
5551 if (Op == SymbolicPHI)
5552 return nullptr;
5553
5554 unsigned SourceBits = SE.getTypeSizeInBits(Ty: SymbolicPHI->getType());
5555 unsigned NewBits = SE.getTypeSizeInBits(Ty: Op->getType());
5556 if (SourceBits != NewBits)
5557 return nullptr;
5558
5559 if (match(S: Op, P: m_scev_SExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5560 Signed = true;
5561 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5562 }
5563 if (match(S: Op, P: m_scev_ZExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5564 Signed = false;
5565 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5566 }
5567 return nullptr;
5568}
5569
5570static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
5571 if (!PN->getType()->isIntegerTy())
5572 return nullptr;
5573 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5574 if (!L || L->getHeader() != PN->getParent())
5575 return nullptr;
5576 return L;
5577}
5578
5579// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
5580// computation that updates the phi follows the following pattern:
5581// (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
5582// which correspond to a phi->trunc->sext/zext->add->phi update chain.
5583// If so, try to see if it can be rewritten as an AddRecExpr under some
5584// Predicates. If successful, return them as a pair. Also cache the results
5585// of the analysis.
5586//
5587// Example usage scenario:
5588// Say the Rewriter is called for the following SCEV:
5589// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5590// where:
5591// %X = phi i64 (%Start, %BEValue)
5592// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
5593// and call this function with %SymbolicPHI = %X.
5594//
5595// The analysis will find that the value coming around the backedge has
5596// the following SCEV:
5597// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5598// Upon concluding that this matches the desired pattern, the function
5599// will return the pair {NewAddRec, SmallPredsVec} where:
5600// NewAddRec = {%Start,+,%Step}
5601// SmallPredsVec = {P1, P2, P3} as follows:
5602// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
5603// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
5604// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
5605// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
5606// under the predicates {P1,P2,P3}.
5607// This predicated rewrite will be cached in PredicatedSCEVRewrites:
5608// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
5609//
5610// TODO's:
5611//
5612// 1) Extend the Induction descriptor to also support inductions that involve
5613// casts: When needed (namely, when we are called in the context of the
5614// vectorizer induction analysis), a Set of cast instructions will be
5615// populated by this method, and provided back to isInductionPHI. This is
5616// needed to allow the vectorizer to properly record them to be ignored by
5617// the cost model and to avoid vectorizing them (otherwise these casts,
5618// which are redundant under the runtime overflow checks, will be
5619// vectorized, which can be costly).
5620//
5621// 2) Support additional induction/PHISCEV patterns: We also want to support
5622// inductions where the sext-trunc / zext-trunc operations (partly) occur
5623// after the induction update operation (the induction increment):
5624//
5625// (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
5626// which correspond to a phi->add->trunc->sext/zext->phi update chain.
5627//
5628// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
5629// which correspond to a phi->trunc->add->sext/zext->phi update chain.
5630//
5631// 3) Outline common code with createAddRecFromPHI to avoid duplication.
5632std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5633ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
5634 SmallVector<const SCEVPredicate *, 3> Predicates;
5635
5636 // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
5637 // return an AddRec expression under some predicate.
5638
5639 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5640 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5641 assert(L && "Expecting an integer loop header phi");
5642
5643 // The loop may have multiple entrances or multiple exits; we can analyze
5644 // this phi as an addrec if it has a unique entry value and a unique
5645 // backedge value.
5646 Value *BEValueV = nullptr, *StartValueV = nullptr;
5647 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5648 Value *V = PN->getIncomingValue(i);
5649 if (L->contains(BB: PN->getIncomingBlock(i))) {
5650 if (!BEValueV) {
5651 BEValueV = V;
5652 } else if (BEValueV != V) {
5653 BEValueV = nullptr;
5654 break;
5655 }
5656 } else if (!StartValueV) {
5657 StartValueV = V;
5658 } else if (StartValueV != V) {
5659 StartValueV = nullptr;
5660 break;
5661 }
5662 }
5663 if (!BEValueV || !StartValueV)
5664 return std::nullopt;
5665
5666 const SCEV *BEValue = getSCEV(V: BEValueV);
5667
5668 // If the value coming around the backedge is an add with the symbolic
5669 // value we just inserted, possibly with casts that we can ignore under
5670 // an appropriate runtime guard, then we found a simple induction variable!
5671 const auto *Add = dyn_cast<SCEVAddExpr>(Val: BEValue);
5672 if (!Add)
5673 return std::nullopt;
5674
5675 // If there is a single occurrence of the symbolic value, possibly
5676 // casted, replace it with a recurrence.
5677 unsigned FoundIndex = Add->getNumOperands();
5678 Type *TruncTy = nullptr;
5679 bool Signed;
5680 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5681 if ((TruncTy =
5682 isSimpleCastedPHI(Op: Add->getOperand(i), SymbolicPHI, Signed, SE&: *this)))
5683 if (FoundIndex == e) {
5684 FoundIndex = i;
5685 break;
5686 }
5687
5688 if (FoundIndex == Add->getNumOperands())
5689 return std::nullopt;
5690
5691 // Create an add with everything but the specified operand.
5692 SmallVector<SCEVUse, 8> Ops;
5693 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5694 if (i != FoundIndex)
5695 Ops.push_back(Elt: Add->getOperand(i));
5696 const SCEV *Accum = getAddExpr(Ops);
5697
5698 // The runtime checks will not be valid if the step amount is
5699 // varying inside the loop.
5700 if (!isLoopInvariant(S: Accum, L))
5701 return std::nullopt;
5702
5703 // *** Part2: Create the predicates
5704
5705 // Analysis was successful: we have a phi-with-cast pattern for which we
5706 // can return an AddRec expression under the following predicates:
5707 //
5708 // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
5709 // fits within the truncated type (does not overflow) for i = 0 to n-1.
5710 // P2: An Equal predicate that guarantees that
5711 // Start = (Ext ix (Trunc iy (Start) to ix) to iy)
5712 // P3: An Equal predicate that guarantees that
5713 // Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
5714 //
5715 // As we next prove, the above predicates guarantee that:
5716 // Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
5717 //
5718 //
5719 // More formally, we want to prove that:
5720 // Expr(i+1) = Start + (i+1) * Accum
5721 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5722 //
5723 // Given that:
5724 // 1) Expr(0) = Start
5725 // 2) Expr(1) = Start + Accum
5726 // = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
5727 // 3) Induction hypothesis (step i):
5728 // Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
5729 //
5730 // Proof:
5731 // Expr(i+1) =
5732 // = Start + (i+1)*Accum
5733 // = (Start + i*Accum) + Accum
5734 // = Expr(i) + Accum
5735 // = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
5736 // :: from step i
5737 //
5738 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
5739 //
5740 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
5741 // + (Ext ix (Trunc iy (Accum) to ix) to iy)
5742 // + Accum :: from P3
5743 //
5744 // = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
5745 // + Accum :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
5746 //
5747 // = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
5748 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5749 //
5750 // By induction, the same applies to all iterations 1<=i<n:
5751 //
5752
5753 // Create a truncated addrec for which we will add a no overflow check (P1).
5754 const SCEV *StartVal = getSCEV(V: StartValueV);
5755 const SCEV *PHISCEV =
5756 getAddRecExpr(Start: getTruncateExpr(Op: StartVal, Ty: TruncTy),
5757 Step: getTruncateExpr(Op: Accum, Ty: TruncTy), L, Flags: SCEV::FlagAnyWrap);
5758
5759 // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
5760 // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
5761 // will be constant.
5762 //
5763 // If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
5764 // add P1.
5765 if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV)) {
5766 SCEVWrapPredicate::IncrementWrapFlags AddedFlags =
5767 Signed ? SCEVWrapPredicate::IncrementNSSW
5768 : SCEVWrapPredicate::IncrementNUSW;
5769 const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
5770 Predicates.push_back(Elt: AddRecPred);
5771 }
5772
5773 // Create the Equal Predicates P2,P3:
5774
5775 // It is possible that the predicates P2 and/or P3 are computable at
5776 // compile time due to StartVal and/or Accum being constants.
5777 // If either one is, then we can check that now and escape if either P2
5778 // or P3 is false.
5779
5780 // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
5781 // for each of StartVal and Accum
5782 auto getExtendedExpr = [&](const SCEV *Expr,
5783 bool CreateSignExtend) -> const SCEV * {
5784 assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant");
5785 const SCEV *TruncatedExpr = getTruncateExpr(Op: Expr, Ty: TruncTy);
5786 const SCEV *ExtendedExpr =
5787 CreateSignExtend ? getSignExtendExpr(Op: TruncatedExpr, Ty: Expr->getType())
5788 : getZeroExtendExpr(Op: TruncatedExpr, Ty: Expr->getType());
5789 return ExtendedExpr;
5790 };
5791
5792 // Given:
5793 // ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
5794 // = getExtendedExpr(Expr)
5795 // Determine whether the predicate P: Expr == ExtendedExpr
5796 // is known to be false at compile time
5797 auto PredIsKnownFalse = [&](const SCEV *Expr,
5798 const SCEV *ExtendedExpr) -> bool {
5799 return Expr != ExtendedExpr &&
5800 isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: Expr, RHS: ExtendedExpr);
5801 };
5802
5803 const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
5804 if (PredIsKnownFalse(StartVal, StartExtended)) {
5805 LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";);
5806 return std::nullopt;
5807 }
5808
5809 // The Step is always Signed (because the overflow checks are either
5810 // NSSW or NUSW)
5811 const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
5812 if (PredIsKnownFalse(Accum, AccumExtended)) {
5813 LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";);
5814 return std::nullopt;
5815 }
5816
5817 auto AppendPredicate = [&](const SCEV *Expr,
5818 const SCEV *ExtendedExpr) -> void {
5819 if (Expr != ExtendedExpr &&
5820 !isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: Expr, RHS: ExtendedExpr)) {
5821 const SCEVPredicate *Pred = getEqualPredicate(LHS: Expr, RHS: ExtendedExpr);
5822 LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred);
5823 Predicates.push_back(Elt: Pred);
5824 }
5825 };
5826
5827 AppendPredicate(StartVal, StartExtended);
5828 AppendPredicate(Accum, AccumExtended);
5829
5830 // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
5831 // which the casts had been folded away. The caller can rewrite SymbolicPHI
5832 // into NewAR if it will also add the runtime overflow checks specified in
5833 // Predicates.
5834 auto *NewAR = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags: SCEV::FlagAnyWrap);
5835
5836 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5837 std::make_pair(x&: NewAR, y&: Predicates);
5838 // Remember the result of the analysis for this SCEV at this locayyytion.
5839 PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
5840 return PredRewrite;
5841}
5842
5843std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5844ScalarEvolution::createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI) {
5845 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5846 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5847 if (!L)
5848 return std::nullopt;
5849
5850 // Check to see if we already analyzed this PHI.
5851 auto I = PredicatedSCEVRewrites.find(Val: {SymbolicPHI, L});
5852 if (I != PredicatedSCEVRewrites.end()) {
5853 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5854 I->second;
5855 // Analysis was done before and failed to create an AddRec:
5856 if (Rewrite.first == SymbolicPHI)
5857 return std::nullopt;
5858 // Analysis was done before and succeeded to create an AddRec under
5859 // a predicate:
5860 assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec");
5861 assert(!(Rewrite.second).empty() && "Expected to find Predicates");
5862 return Rewrite;
5863 }
5864
5865 std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5866 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5867
5868 // Record in the cache that the analysis failed
5869 if (!Rewrite) {
5870 SmallVector<const SCEVPredicate *, 3> Predicates;
5871 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5872 return std::nullopt;
5873 }
5874
5875 return Rewrite;
5876}
5877
5878// FIXME: This utility is currently required because the Rewriter currently
5879// does not rewrite this expression:
5880// {0, +, (sext ix (trunc iy to ix) to iy)}
5881// into {0, +, %step},
5882// even when the following Equal predicate exists:
5883// "%step == (sext ix (trunc iy to ix) to iy)".
5884bool PredicatedScalarEvolution::areAddRecsEqualWithPreds(
5885 const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2,
5886 ArrayRef<const SCEVPredicate *> NoWrapPreds) const {
5887 if (AR1 == AR2)
5888 return true;
5889
5890 SCEVUnionPredicate NoWrapUnionPred(NoWrapPreds, SE);
5891 SCEVUnionPredicate AllPreds = Preds->getUnionWith(N: &NoWrapUnionPred, SE);
5892 auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
5893 if (Expr1 != Expr2 &&
5894 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr1, RHS: Expr2), SE) &&
5895 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr2, RHS: Expr1), SE))
5896 return false;
5897 return true;
5898 };
5899
5900 if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
5901 !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
5902 return false;
5903 return true;
5904}
5905
5906/// A helper function for createAddRecFromPHI to handle simple cases.
5907///
5908/// This function tries to find an AddRec expression for the simplest (yet most
5909/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
5910/// If it fails, createAddRecFromPHI will use a more general, but slow,
5911/// technique for finding the AddRec expression.
5912const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
5913 Value *BEValueV,
5914 Value *StartValueV) {
5915 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5916 assert(L && L->getHeader() == PN->getParent());
5917 assert(BEValueV && StartValueV);
5918
5919 auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CxtI: PN);
5920 if (!BO)
5921 return nullptr;
5922
5923 if (BO->Opcode != Instruction::Add)
5924 return nullptr;
5925
5926 const SCEV *Accum = nullptr;
5927 if (BO->LHS == PN && L->isLoopInvariant(V: BO->RHS))
5928 Accum = getSCEV(V: BO->RHS);
5929 else if (BO->RHS == PN && L->isLoopInvariant(V: BO->LHS))
5930 Accum = getSCEV(V: BO->LHS);
5931
5932 if (!Accum)
5933 return nullptr;
5934
5935 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5936 if (BO->IsNUW)
5937 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5938 if (BO->IsNSW)
5939 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5940
5941 const SCEV *StartVal = getSCEV(V: StartValueV);
5942 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5943 insertValueToMap(V: PN, S: PHISCEV);
5944
5945 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
5946 inferNoWrapViaConstantRanges(AR);
5947
5948 // We can add Flags to the post-inc expression only if we
5949 // know that it is *undefined behavior* for BEValueV to
5950 // overflow.
5951 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV)) {
5952 assert(isLoopInvariant(Accum, L) &&
5953 "Accum is defined outside L, but is not invariant?");
5954 if (isAddRecNeverPoison(I: BEInst, L))
5955 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5956 }
5957
5958 return PHISCEV;
5959}
5960
5961const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
5962 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5963 if (!L || L->getHeader() != PN->getParent())
5964 return nullptr;
5965
5966 // The loop may have multiple entrances or multiple exits; we can analyze
5967 // this phi as an addrec if it has a unique entry value and a unique
5968 // backedge value.
5969 Value *BEValueV = nullptr, *StartValueV = nullptr;
5970 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5971 Value *V = PN->getIncomingValue(i);
5972 if (L->contains(BB: PN->getIncomingBlock(i))) {
5973 if (!BEValueV) {
5974 BEValueV = V;
5975 } else if (BEValueV != V) {
5976 BEValueV = nullptr;
5977 break;
5978 }
5979 } else if (!StartValueV) {
5980 StartValueV = V;
5981 } else if (StartValueV != V) {
5982 StartValueV = nullptr;
5983 break;
5984 }
5985 }
5986 if (!BEValueV || !StartValueV)
5987 return nullptr;
5988
5989 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
5990 "PHI node already processed?");
5991
5992 // First, try to find AddRec expression without creating a fictituos symbolic
5993 // value for PN.
5994 if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5995 return S;
5996
5997 // Handle PHI node value symbolically.
5998 const SCEV *SymbolicName = getUnknown(V: PN);
5999 insertValueToMap(V: PN, S: SymbolicName);
6000
6001 // Using this symbolic name for the PHI, analyze the value coming around
6002 // the back-edge.
6003 const SCEV *BEValue = getSCEV(V: BEValueV);
6004
6005 // NOTE: If BEValue is loop invariant, we know that the PHI node just
6006 // has a special value for the first iteration of the loop.
6007
6008 // If the value coming around the backedge is an add with the symbolic
6009 // value we just inserted, then we found a simple induction variable!
6010 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: BEValue)) {
6011 // If there is a single occurrence of the symbolic value, replace it
6012 // with a recurrence.
6013 unsigned FoundIndex = Add->getNumOperands();
6014 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
6015 if (Add->getOperand(i) == SymbolicName)
6016 if (FoundIndex == e) {
6017 FoundIndex = i;
6018 break;
6019 }
6020
6021 if (FoundIndex != Add->getNumOperands()) {
6022 // Create an add with everything but the specified operand.
6023 SmallVector<SCEVUse, 8> Ops;
6024 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
6025 if (i != FoundIndex)
6026 Ops.push_back(Elt: SCEVBackedgeConditionFolder::rewrite(S: Add->getOperand(i),
6027 L, SE&: *this));
6028 const SCEV *Accum = getAddExpr(Ops);
6029
6030 // This is not a valid addrec if the step amount is varying each
6031 // loop iteration, but is not itself an addrec in this loop.
6032 if (isLoopInvariant(S: Accum, L) ||
6033 (isa<SCEVAddRecExpr>(Val: Accum) &&
6034 cast<SCEVAddRecExpr>(Val: Accum)->getLoop() == L)) {
6035 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
6036
6037 if (auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CxtI: PN)) {
6038 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
6039 if (BO->IsNUW)
6040 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
6041 if (BO->IsNSW)
6042 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
6043 }
6044 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: BEValueV)) {
6045 if (GEP->getOperand(i_nocapture: 0) == PN) {
6046 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
6047 // If the increment has any nowrap flags, then we know the address
6048 // space cannot be wrapped around.
6049 if (NW != GEPNoWrapFlags::none())
6050 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
6051 // If the GEP is nuw or nusw with non-negative offset, we know that
6052 // no unsigned wrap occurs. We cannot set the nsw flag as only the
6053 // offset is treated as signed, while the base is unsigned.
6054 if (NW.hasNoUnsignedWrap() ||
6055 (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Accum)))
6056 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
6057 }
6058
6059 // We cannot transfer nuw and nsw flags from subtraction
6060 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
6061 // for instance.
6062 }
6063
6064 const SCEV *StartVal = getSCEV(V: StartValueV);
6065 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
6066
6067 // Okay, for the entire analysis of this edge we assumed the PHI
6068 // to be symbolic. We now need to go back and purge all of the
6069 // entries for the scalars that use the symbolic expression.
6070 forgetMemoizedResults(SCEVs: {SymbolicName});
6071 insertValueToMap(V: PN, S: PHISCEV);
6072
6073 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
6074 inferNoWrapViaConstantRanges(AR);
6075
6076 // We can add Flags to the post-inc expression only if we
6077 // know that it is *undefined behavior* for BEValueV to
6078 // overflow.
6079 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV))
6080 if (isLoopInvariant(S: Accum, L) && isAddRecNeverPoison(I: BEInst, L))
6081 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
6082
6083 return PHISCEV;
6084 }
6085 }
6086 } else {
6087 // Otherwise, this could be a loop like this:
6088 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
6089 // In this case, j = {1,+,1} and BEValue is j.
6090 // Because the other in-value of i (0) fits the evolution of BEValue
6091 // i really is an addrec evolution.
6092 //
6093 // We can generalize this saying that i is the shifted value of BEValue
6094 // by one iteration:
6095 // PHI(f(0), f({1,+,1})) --> f({0,+,1})
6096
6097 // Do not allow refinement in rewriting of BEValue.
6098 const SCEV *Shifted = SCEVShiftRewriter::rewrite(S: BEValue, L, SE&: *this);
6099 const SCEV *Start = SCEVInitRewriter::rewrite(S: Shifted, L, SE&: *this, IgnoreOtherLoops: false);
6100 if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
6101 isGuaranteedNotToCauseUB(Op: Shifted) && ::impliesPoison(AssumedPoison: Shifted, S: Start)) {
6102 const SCEV *StartVal = getSCEV(V: StartValueV);
6103 if (Start == StartVal) {
6104 // Okay, for the entire analysis of this edge we assumed the PHI
6105 // to be symbolic. We now need to go back and purge all of the
6106 // entries for the scalars that use the symbolic expression.
6107 forgetMemoizedResults(SCEVs: {SymbolicName});
6108 insertValueToMap(V: PN, S: Shifted);
6109 return Shifted;
6110 }
6111 }
6112 }
6113
6114 // Remove the temporary PHI node SCEV that has been inserted while intending
6115 // to create an AddRecExpr for this PHI node. We can not keep this temporary
6116 // as it will prevent later (possibly simpler) SCEV expressions to be added
6117 // to the ValueExprMap.
6118 eraseValueFromMap(V: PN);
6119
6120 return nullptr;
6121}
6122
6123// Try to match a control flow sequence that branches out at BI and merges back
6124// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
6125// match.
6126static bool BrPHIToSelect(DominatorTree &DT, CondBrInst *BI, PHINode *Merge,
6127 Value *&C, Value *&LHS, Value *&RHS) {
6128 C = BI->getCondition();
6129
6130 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(i: 0));
6131 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(i: 1));
6132
6133 Use &LeftUse = Merge->getOperandUse(i: 0);
6134 Use &RightUse = Merge->getOperandUse(i: 1);
6135
6136 if (DT.dominates(BBE: LeftEdge, U: LeftUse) && DT.dominates(BBE: RightEdge, U: RightUse)) {
6137 LHS = LeftUse;
6138 RHS = RightUse;
6139 return true;
6140 }
6141
6142 if (DT.dominates(BBE: LeftEdge, U: RightUse) && DT.dominates(BBE: RightEdge, U: LeftUse)) {
6143 LHS = RightUse;
6144 RHS = LeftUse;
6145 return true;
6146 }
6147
6148 return false;
6149}
6150
6151static bool getOperandsForSelectLikePHI(DominatorTree &DT, PHINode *PN,
6152 Value *&Cond, Value *&LHS,
6153 Value *&RHS) {
6154 auto IsReachable =
6155 [&](BasicBlock *BB) { return DT.isReachableFromEntry(A: BB); };
6156 if (PN->getNumIncomingValues() == 2 && all_of(Range: PN->blocks(), P: IsReachable)) {
6157 // Try to match
6158 //
6159 // br %cond, label %left, label %right
6160 // left:
6161 // br label %merge
6162 // right:
6163 // br label %merge
6164 // merge:
6165 // V = phi [ %x, %left ], [ %y, %right ]
6166 //
6167 // as "select %cond, %x, %y"
6168
6169 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
6170 assert(IDom && "At least the entry block should dominate PN");
6171
6172 auto *BI = dyn_cast<CondBrInst>(Val: IDom->getTerminator());
6173 return BI && BrPHIToSelect(DT, BI, Merge: PN, C&: Cond, LHS, RHS);
6174 }
6175 return false;
6176}
6177
6178const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
6179 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
6180 if (getOperandsForSelectLikePHI(DT, PN, Cond, LHS, RHS) &&
6181 properlyDominates(S: getSCEV(V: LHS), BB: PN->getParent()) &&
6182 properlyDominates(S: getSCEV(V: RHS), BB: PN->getParent()))
6183 return createNodeForSelectOrPHI(V: PN, Cond, TrueVal: LHS, FalseVal: RHS);
6184
6185 return nullptr;
6186}
6187
6188static BinaryOperator *getCommonInstForPHI(PHINode *PN) {
6189 BinaryOperator *CommonInst = nullptr;
6190 // Check if instructions are identical.
6191 for (Value *Incoming : PN->incoming_values()) {
6192 auto *IncomingInst = dyn_cast<BinaryOperator>(Val: Incoming);
6193 if (!IncomingInst)
6194 return nullptr;
6195 if (CommonInst) {
6196 if (!CommonInst->isIdenticalToWhenDefined(I: IncomingInst))
6197 return nullptr; // Not identical, give up
6198 } else {
6199 // Remember binary operator
6200 CommonInst = IncomingInst;
6201 }
6202 }
6203 return CommonInst;
6204}
6205
6206/// Returns SCEV for the first operand of a phi if all phi operands have
6207/// identical opcodes and operands
6208/// eg.
6209/// a: %add = %a + %b
6210/// br %c
6211/// b: %add1 = %a + %b
6212/// br %c
6213/// c: %phi = phi [%add, a], [%add1, b]
6214/// scev(%phi) => scev(%add)
6215const SCEV *
6216ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
6217 BinaryOperator *CommonInst = getCommonInstForPHI(PN);
6218 if (!CommonInst)
6219 return nullptr;
6220
6221 // Check if SCEV exprs for instructions are identical.
6222 const SCEV *CommonSCEV = getSCEV(V: CommonInst);
6223 bool SCEVExprsIdentical =
6224 all_of(Range: drop_begin(RangeOrContainer: PN->incoming_values()),
6225 P: [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
6226 return SCEVExprsIdentical ? CommonSCEV : nullptr;
6227}
6228
6229const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
6230 if (const SCEV *S = createAddRecFromPHI(PN))
6231 return S;
6232
6233 // We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
6234 // phi node for X.
6235 if (Value *V = simplifyInstruction(
6236 I: PN, Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
6237 /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
6238 return getSCEV(V);
6239
6240 if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
6241 return S;
6242
6243 if (const SCEV *S = createNodeFromSelectLikePHI(PN))
6244 return S;
6245
6246 // If it's not a loop phi, we can't handle it yet.
6247 return getUnknown(V: PN);
6248}
6249
6250bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
6251 SCEVTypes RootKind) {
6252 struct FindClosure {
6253 const SCEV *OperandToFind;
6254 const SCEVTypes RootKind; // Must be a sequential min/max expression.
6255 const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind.
6256
6257 bool Found = false;
6258
6259 bool canRecurseInto(SCEVTypes Kind) const {
6260 // We can only recurse into the SCEV expression of the same effective type
6261 // as the type of our root SCEV expression, and into zero-extensions.
6262 return RootKind == Kind || NonSequentialRootKind == Kind ||
6263 scZeroExtend == Kind;
6264 };
6265
6266 FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind)
6267 : OperandToFind(OperandToFind), RootKind(RootKind),
6268 NonSequentialRootKind(
6269 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
6270 Ty: RootKind)) {}
6271
6272 bool follow(const SCEV *S) {
6273 Found = S == OperandToFind;
6274
6275 return !isDone() && canRecurseInto(Kind: S->getSCEVType());
6276 }
6277
6278 bool isDone() const { return Found; }
6279 };
6280
6281 FindClosure FC(OperandToFind, RootKind);
6282 visitAll(Root, Visitor&: FC);
6283 return FC.Found;
6284}
6285
6286std::optional<const SCEV *>
6287ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(Type *Ty,
6288 ICmpInst *Cond,
6289 Value *TrueVal,
6290 Value *FalseVal) {
6291 // Try to match some simple smax or umax patterns.
6292 auto *ICI = Cond;
6293
6294 Value *LHS = ICI->getOperand(i_nocapture: 0);
6295 Value *RHS = ICI->getOperand(i_nocapture: 1);
6296
6297 switch (ICI->getPredicate()) {
6298 case ICmpInst::ICMP_SLT:
6299 case ICmpInst::ICMP_SLE:
6300 case ICmpInst::ICMP_ULT:
6301 case ICmpInst::ICMP_ULE:
6302 std::swap(a&: LHS, b&: RHS);
6303 [[fallthrough]];
6304 case ICmpInst::ICMP_SGT:
6305 case ICmpInst::ICMP_SGE:
6306 case ICmpInst::ICMP_UGT:
6307 case ICmpInst::ICMP_UGE:
6308 // a > b ? a+x : b+x -> max(a, b)+x
6309 // a > b ? b+x : a+x -> min(a, b)+x
6310 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty)) {
6311 bool Signed = ICI->isSigned();
6312 const SCEV *LA = getSCEV(V: TrueVal);
6313 const SCEV *RA = getSCEV(V: FalseVal);
6314 const SCEV *LS = getSCEV(V: LHS);
6315 const SCEV *RS = getSCEV(V: RHS);
6316 if (LA->getType()->isPointerTy()) {
6317 // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
6318 // Need to make sure we can't produce weird expressions involving
6319 // negated pointers.
6320 if (LA == LS && RA == RS)
6321 return Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS);
6322 if (LA == RS && RA == LS)
6323 return Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS);
6324 }
6325 auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
6326 if (Op->getType()->isPointerTy()) {
6327 Op = getLosslessPtrToIntExpr(Op);
6328 if (isa<SCEVCouldNotCompute>(Val: Op))
6329 return Op;
6330 }
6331 if (Signed)
6332 Op = getNoopOrSignExtend(V: Op, Ty);
6333 else
6334 Op = getNoopOrZeroExtend(V: Op, Ty);
6335 return Op;
6336 };
6337 LS = CoerceOperand(LS);
6338 RS = CoerceOperand(RS);
6339 if (isa<SCEVCouldNotCompute>(Val: LS) || isa<SCEVCouldNotCompute>(Val: RS))
6340 break;
6341 const SCEV *LDiff = getMinusSCEV(LHS: LA, RHS: LS);
6342 const SCEV *RDiff = getMinusSCEV(LHS: RA, RHS: RS);
6343 if (LDiff == RDiff)
6344 return getAddExpr(LHS: Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS),
6345 RHS: LDiff);
6346 LDiff = getMinusSCEV(LHS: LA, RHS: RS);
6347 RDiff = getMinusSCEV(LHS: RA, RHS: LS);
6348 if (LDiff == RDiff)
6349 return getAddExpr(LHS: Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS),
6350 RHS: LDiff);
6351 }
6352 break;
6353 case ICmpInst::ICMP_NE:
6354 // x != 0 ? x+y : C+y -> x == 0 ? C+y : x+y
6355 std::swap(a&: TrueVal, b&: FalseVal);
6356 [[fallthrough]];
6357 case ICmpInst::ICMP_EQ:
6358 // x == 0 ? C+y : x+y -> umax(x, C)+y iff C u<= 1
6359 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty) &&
6360 isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()) {
6361 const SCEV *X = getNoopOrZeroExtend(V: getSCEV(V: LHS), Ty);
6362 const SCEV *TrueValExpr = getSCEV(V: TrueVal); // C+y
6363 const SCEV *FalseValExpr = getSCEV(V: FalseVal); // x+y
6364 const SCEV *Y = getMinusSCEV(LHS: FalseValExpr, RHS: X); // y = (x+y)-x
6365 const SCEV *C = getMinusSCEV(LHS: TrueValExpr, RHS: Y); // C = (C+y)-y
6366 if (isa<SCEVConstant>(Val: C) && cast<SCEVConstant>(Val: C)->getAPInt().ule(RHS: 1))
6367 return getAddExpr(LHS: getUMaxExpr(LHS: X, RHS: C), RHS: Y);
6368 }
6369 // x == 0 ? 0 : umin (..., x, ...) -> umin_seq(x, umin (...))
6370 // x == 0 ? 0 : umin_seq(..., x, ...) -> umin_seq(x, umin_seq(...))
6371 // x == 0 ? 0 : umin (..., umin_seq(..., x, ...), ...)
6372 // -> umin_seq(x, umin (..., umin_seq(...), ...))
6373 if (isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero() &&
6374 isa<ConstantInt>(Val: TrueVal) && cast<ConstantInt>(Val: TrueVal)->isZero()) {
6375 const SCEV *X = getSCEV(V: LHS);
6376 while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: X))
6377 X = ZExt->getOperand();
6378 if (getTypeSizeInBits(Ty: X->getType()) <= getTypeSizeInBits(Ty)) {
6379 const SCEV *FalseValExpr = getSCEV(V: FalseVal);
6380 if (SCEVMinMaxExprContains(Root: FalseValExpr, OperandToFind: X, RootKind: scSequentialUMinExpr))
6381 return getUMinExpr(LHS: getNoopOrZeroExtend(V: X, Ty), RHS: FalseValExpr,
6382 /*Sequential=*/true);
6383 }
6384 }
6385 break;
6386 default:
6387 break;
6388 }
6389
6390 return std::nullopt;
6391}
6392
6393static std::optional<const SCEV *>
6394createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr,
6395 const SCEV *TrueExpr, const SCEV *FalseExpr) {
6396 assert(CondExpr->getType()->isIntegerTy(1) &&
6397 TrueExpr->getType() == FalseExpr->getType() &&
6398 TrueExpr->getType()->isIntegerTy(1) &&
6399 "Unexpected operands of a select.");
6400
6401 // i1 cond ? i1 x : i1 C --> C + (i1 cond ? (i1 x - i1 C) : i1 0)
6402 // --> C + (umin_seq cond, x - C)
6403 //
6404 // i1 cond ? i1 C : i1 x --> C + (i1 cond ? i1 0 : (i1 x - i1 C))
6405 // --> C + (i1 ~cond ? (i1 x - i1 C) : i1 0)
6406 // --> C + (umin_seq ~cond, x - C)
6407
6408 // FIXME: while we can't legally model the case where both of the hands
6409 // are fully variable, we only require that the *difference* is constant.
6410 if (!isa<SCEVConstant>(Val: TrueExpr) && !isa<SCEVConstant>(Val: FalseExpr))
6411 return std::nullopt;
6412
6413 const SCEV *X, *C;
6414 if (isa<SCEVConstant>(Val: TrueExpr)) {
6415 CondExpr = SE->getNotSCEV(V: CondExpr);
6416 X = FalseExpr;
6417 C = TrueExpr;
6418 } else {
6419 X = TrueExpr;
6420 C = FalseExpr;
6421 }
6422 return SE->getAddExpr(LHS: C, RHS: SE->getUMinExpr(LHS: CondExpr, RHS: SE->getMinusSCEV(LHS: X, RHS: C),
6423 /*Sequential=*/true));
6424}
6425
6426static std::optional<const SCEV *>
6427createNodeForSelectViaUMinSeq(ScalarEvolution *SE, Value *Cond, Value *TrueVal,
6428 Value *FalseVal) {
6429 if (!isa<ConstantInt>(Val: TrueVal) && !isa<ConstantInt>(Val: FalseVal))
6430 return std::nullopt;
6431
6432 const auto *SECond = SE->getSCEV(V: Cond);
6433 const auto *SETrue = SE->getSCEV(V: TrueVal);
6434 const auto *SEFalse = SE->getSCEV(V: FalseVal);
6435 return createNodeForSelectViaUMinSeq(SE, CondExpr: SECond, TrueExpr: SETrue, FalseExpr: SEFalse);
6436}
6437
6438const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
6439 Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) {
6440 assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?");
6441 assert(TrueVal->getType() == FalseVal->getType() &&
6442 V->getType() == TrueVal->getType() &&
6443 "Types of select hands and of the result must match.");
6444
6445 // For now, only deal with i1-typed `select`s.
6446 if (!V->getType()->isIntegerTy(BitWidth: 1))
6447 return getUnknown(V);
6448
6449 if (std::optional<const SCEV *> S =
6450 createNodeForSelectViaUMinSeq(SE: this, Cond, TrueVal, FalseVal))
6451 return *S;
6452
6453 return getUnknown(V);
6454}
6455
6456const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond,
6457 Value *TrueVal,
6458 Value *FalseVal) {
6459 // Handle "constant" branch or select. This can occur for instance when a
6460 // loop pass transforms an inner loop and moves on to process the outer loop.
6461 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond))
6462 return getSCEV(V: CI->isOne() ? TrueVal : FalseVal);
6463
6464 if (auto *I = dyn_cast<Instruction>(Val: V)) {
6465 if (auto *ICI = dyn_cast<ICmpInst>(Val: Cond)) {
6466 if (std::optional<const SCEV *> S =
6467 createNodeForSelectOrPHIInstWithICmpInstCond(Ty: I->getType(), Cond: ICI,
6468 TrueVal, FalseVal))
6469 return *S;
6470 }
6471 }
6472
6473 return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal);
6474}
6475
6476/// Expand GEP instructions into add and multiply operations. This allows them
6477/// to be analyzed by regular SCEV code.
6478const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
6479 assert(GEP->getSourceElementType()->isSized() &&
6480 "GEP source element type must be sized");
6481
6482 SmallVector<SCEVUse, 4> IndexExprs;
6483 for (Value *Index : GEP->indices())
6484 IndexExprs.push_back(Elt: getSCEV(V: Index));
6485 return getGEPExpr(GEP, IndexExprs);
6486}
6487
6488APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S,
6489 const Instruction *CtxI) {
6490 uint64_t BitWidth = getTypeSizeInBits(Ty: S->getType());
6491 auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
6492 return TrailingZeros >= BitWidth
6493 ? APInt::getZero(numBits: BitWidth)
6494 : APInt::getOneBitSet(numBits: BitWidth, BitNo: TrailingZeros);
6495 };
6496 auto GetGCDMultiple = [this, CtxI](const SCEVNAryExpr *N) {
6497 // The result is GCD of all operands results.
6498 APInt Res = getConstantMultiple(S: N->getOperand(i: 0), CtxI);
6499 for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
6500 Res = APIntOps::GreatestCommonDivisor(
6501 A: Res, B: getConstantMultiple(S: N->getOperand(i: I), CtxI));
6502 return Res;
6503 };
6504
6505 switch (S->getSCEVType()) {
6506 case scConstant:
6507 return cast<SCEVConstant>(Val: S)->getAPInt();
6508 case scPtrToAddr:
6509 case scPtrToInt:
6510 return getConstantMultiple(S: cast<SCEVCastExpr>(Val: S)->getOperand());
6511 case scUDivExpr:
6512 case scVScale:
6513 return APInt(BitWidth, 1);
6514 case scTruncate: {
6515 // Only multiples that are a power of 2 will hold after truncation.
6516 const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(Val: S);
6517 uint32_t TZ = getMinTrailingZeros(S: T->getOperand(), CtxI);
6518 return GetShiftedByZeros(TZ);
6519 }
6520 case scZeroExtend: {
6521 const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(Val: S);
6522 return getConstantMultiple(S: Z->getOperand(), CtxI).zext(width: BitWidth);
6523 }
6524 case scSignExtend: {
6525 // Only multiples that are a power of 2 will hold after sext.
6526 const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(Val: S);
6527 uint32_t TZ = getMinTrailingZeros(S: E->getOperand(), CtxI);
6528 return GetShiftedByZeros(TZ);
6529 }
6530 case scMulExpr: {
6531 const SCEVMulExpr *M = cast<SCEVMulExpr>(Val: S);
6532 if (M->hasNoUnsignedWrap()) {
6533 // The result is the product of all operand results.
6534 APInt Res = getConstantMultiple(S: M->getOperand(i: 0), CtxI);
6535 for (const SCEV *Operand : M->operands().drop_front())
6536 Res = Res * getConstantMultiple(S: Operand, CtxI);
6537 return Res;
6538 }
6539
6540 // If there are no wrap guarentees, find the trailing zeros, which is the
6541 // sum of trailing zeros for all its operands.
6542 uint32_t TZ = 0;
6543 for (const SCEV *Operand : M->operands())
6544 TZ += getMinTrailingZeros(S: Operand, CtxI);
6545 return GetShiftedByZeros(TZ);
6546 }
6547 case scAddExpr:
6548 case scAddRecExpr: {
6549 const SCEVNAryExpr *N = cast<SCEVNAryExpr>(Val: S);
6550 if (N->hasNoUnsignedWrap())
6551 return GetGCDMultiple(N);
6552 // Find the trailing bits, which is the minimum of its operands.
6553 uint32_t TZ = getMinTrailingZeros(S: N->getOperand(i: 0), CtxI);
6554 for (const SCEV *Operand : N->operands().drop_front())
6555 TZ = std::min(a: TZ, b: getMinTrailingZeros(S: Operand, CtxI));
6556 return GetShiftedByZeros(TZ);
6557 }
6558 case scUMaxExpr:
6559 case scSMaxExpr:
6560 case scUMinExpr:
6561 case scSMinExpr:
6562 case scSequentialUMinExpr:
6563 return GetGCDMultiple(cast<SCEVNAryExpr>(Val: S));
6564 case scUnknown: {
6565 // Ask ValueTracking for known bits. SCEVUnknown only become available at
6566 // the point their underlying IR instruction has been defined. If CtxI was
6567 // not provided, use:
6568 // * the first instruction in the entry block if it is an argument
6569 // * the instruction itself otherwise.
6570 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6571 if (!CtxI) {
6572 if (isa<Argument>(Val: U->getValue()))
6573 CtxI = &*F.getEntryBlock().begin();
6574 else if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
6575 CtxI = I;
6576 }
6577 unsigned Known =
6578 computeKnownBits(V: U->getValue(),
6579 Q: SimplifyQuery(getDataLayout(), &DT, &AC, CtxI)
6580 .allowEphemerals(AllowEphemerals: true))
6581 .countMinTrailingZeros();
6582 return GetShiftedByZeros(Known);
6583 }
6584 case scCouldNotCompute:
6585 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6586 }
6587 llvm_unreachable("Unknown SCEV kind!");
6588}
6589
6590APInt ScalarEvolution::getConstantMultiple(const SCEV *S,
6591 const Instruction *CtxI) {
6592 // Skip looking up and updating the cache if there is a context instruction,
6593 // as the result will only be valid in the specified context.
6594 if (CtxI)
6595 return getConstantMultipleImpl(S, CtxI);
6596
6597 auto I = ConstantMultipleCache.find(Val: S);
6598 if (I != ConstantMultipleCache.end())
6599 return I->second;
6600
6601 APInt Result = getConstantMultipleImpl(S, CtxI);
6602 auto InsertPair = ConstantMultipleCache.insert(KV: {S, Result});
6603 assert(InsertPair.second && "Should insert a new key");
6604 return InsertPair.first->second;
6605}
6606
6607APInt ScalarEvolution::getNonZeroConstantMultiple(const SCEV *S) {
6608 APInt Multiple = getConstantMultiple(S);
6609 return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
6610}
6611
6612uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S,
6613 const Instruction *CtxI) {
6614 return std::min(a: getConstantMultiple(S, CtxI).countTrailingZeros(),
6615 b: (unsigned)getTypeSizeInBits(Ty: S->getType()));
6616}
6617
6618/// Helper method to assign a range to V from metadata present in the IR.
6619static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
6620 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
6621 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_range))
6622 return getConstantRangeFromMetadata(RangeMD: *MD);
6623 if (const auto *CB = dyn_cast<CallBase>(Val: V))
6624 if (std::optional<ConstantRange> Range = CB->getRange())
6625 return Range;
6626 }
6627 if (auto *A = dyn_cast<Argument>(Val: V))
6628 if (std::optional<ConstantRange> Range = A->getRange())
6629 return Range;
6630
6631 return std::nullopt;
6632}
6633
6634void ScalarEvolution::setNoWrapFlags(SCEVAddRecExpr *AddRec,
6635 SCEV::NoWrapFlags Flags) {
6636 if (AddRec->getNoWrapFlags(Mask: Flags) != Flags) {
6637 AddRec->setNoWrapFlags(Flags);
6638 UnsignedRanges.erase(Val: AddRec);
6639 SignedRanges.erase(Val: AddRec);
6640 ConstantMultipleCache.erase(Val: AddRec);
6641 }
6642}
6643
6644ConstantRange ScalarEvolution::
6645getRangeForUnknownRecurrence(const SCEVUnknown *U) {
6646 const DataLayout &DL = getDataLayout();
6647
6648 unsigned BitWidth = getTypeSizeInBits(Ty: U->getType());
6649 const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
6650
6651 // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
6652 // use information about the trip count to improve our available range. Note
6653 // that the trip count independent cases are already handled by known bits.
6654 // WARNING: The definition of recurrence used here is subtly different than
6655 // the one used by AddRec (and thus most of this file). Step is allowed to
6656 // be arbitrarily loop varying here, where AddRec allows only loop invariant
6657 // and other addrecs in the same loop (for non-affine addrecs). The code
6658 // below intentionally handles the case where step is not loop invariant.
6659 auto *P = dyn_cast<PHINode>(Val: U->getValue());
6660 if (!P)
6661 return FullSet;
6662
6663 // Make sure that no Phi input comes from an unreachable block. Otherwise,
6664 // even the values that are not available in these blocks may come from them,
6665 // and this leads to false-positive recurrence test.
6666 for (auto *Pred : predecessors(BB: P->getParent()))
6667 if (!DT.isReachableFromEntry(A: Pred))
6668 return FullSet;
6669
6670 BinaryOperator *BO;
6671 Value *Start, *Step;
6672 if (!matchSimpleRecurrence(P, BO, Start, Step))
6673 return FullSet;
6674
6675 // If we found a recurrence in reachable code, we must be in a loop. Note
6676 // that BO might be in some subloop of L, and that's completely okay.
6677 auto *L = LI.getLoopFor(BB: P->getParent());
6678 assert(L && L->getHeader() == P->getParent());
6679 if (!L->contains(BB: BO->getParent()))
6680 // NOTE: This bailout should be an assert instead. However, asserting
6681 // the condition here exposes a case where LoopFusion is querying SCEV
6682 // with malformed loop information during the midst of the transform.
6683 // There doesn't appear to be an obvious fix, so for the moment bailout
6684 // until the caller issue can be fixed. PR49566 tracks the bug.
6685 return FullSet;
6686
6687 // TODO: Extend to other opcodes such as mul, and div
6688 switch (BO->getOpcode()) {
6689 default:
6690 return FullSet;
6691 case Instruction::AShr:
6692 case Instruction::LShr:
6693 case Instruction::Shl:
6694 break;
6695 };
6696
6697 if (BO->getOperand(i_nocapture: 0) != P)
6698 // TODO: Handle the power function forms some day.
6699 return FullSet;
6700
6701 unsigned TC = getSmallConstantMaxTripCount(L);
6702 if (!TC || TC >= BitWidth)
6703 return FullSet;
6704
6705 auto KnownStart = computeKnownBits(V: Start, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6706 auto KnownStep = computeKnownBits(V: Step, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6707 assert(KnownStart.getBitWidth() == BitWidth &&
6708 KnownStep.getBitWidth() == BitWidth);
6709
6710 // Compute total shift amount, being careful of overflow and bitwidths.
6711 auto MaxShiftAmt = KnownStep.getMaxValue();
6712 APInt TCAP(BitWidth, TC-1);
6713 bool Overflow = false;
6714 auto TotalShift = MaxShiftAmt.umul_ov(RHS: TCAP, Overflow);
6715 if (Overflow)
6716 return FullSet;
6717
6718 switch (BO->getOpcode()) {
6719 default:
6720 llvm_unreachable("filtered out above");
6721 case Instruction::AShr: {
6722 // For each ashr, three cases:
6723 // shift = 0 => unchanged value
6724 // saturation => 0 or -1
6725 // other => a value closer to zero (of the same sign)
6726 // Thus, the end value is closer to zero than the start.
6727 auto KnownEnd = KnownBits::ashr(LHS: KnownStart,
6728 RHS: KnownBits::makeConstant(C: TotalShift));
6729 if (KnownStart.isNonNegative())
6730 // Analogous to lshr (simply not yet canonicalized)
6731 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6732 Upper: KnownStart.getMaxValue() + 1);
6733 if (KnownStart.isNegative())
6734 // End >=u Start && End <=s Start
6735 return ConstantRange::getNonEmpty(Lower: KnownStart.getMinValue(),
6736 Upper: KnownEnd.getMaxValue() + 1);
6737 break;
6738 }
6739 case Instruction::LShr: {
6740 // For each lshr, three cases:
6741 // shift = 0 => unchanged value
6742 // saturation => 0
6743 // other => a smaller positive number
6744 // Thus, the low end of the unsigned range is the last value produced.
6745 auto KnownEnd = KnownBits::lshr(LHS: KnownStart,
6746 RHS: KnownBits::makeConstant(C: TotalShift));
6747 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6748 Upper: KnownStart.getMaxValue() + 1);
6749 }
6750 case Instruction::Shl: {
6751 // Iff no bits are shifted out, value increases on every shift.
6752 auto KnownEnd = KnownBits::shl(LHS: KnownStart,
6753 RHS: KnownBits::makeConstant(C: TotalShift));
6754 if (TotalShift.ult(RHS: KnownStart.countMinLeadingZeros()))
6755 return ConstantRange(KnownStart.getMinValue(),
6756 KnownEnd.getMaxValue() + 1);
6757 break;
6758 }
6759 };
6760 return FullSet;
6761}
6762
6763// The goal of this function is to check if recursively visiting the operands
6764// of this PHI might lead to an infinite loop. If we do see such a loop,
6765// there's no good way to break it, so we avoid analyzing such cases.
6766//
6767// getRangeRef previously used a visited set to avoid infinite loops, but this
6768// caused other issues: the result was dependent on the order of getRangeRef
6769// calls, and the interaction with createSCEVIter could cause a stack overflow
6770// in some cases (see issue #148253).
6771//
6772// FIXME: The way this is implemented is overly conservative; this checks
6773// for a few obviously safe patterns, but anything that doesn't lead to
6774// recursion is fine.
6775static bool RangeRefPHIAllowedOperands(DominatorTree &DT, PHINode *PHI) {
6776 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
6777 if (getOperandsForSelectLikePHI(DT, PN: PHI, Cond, LHS, RHS))
6778 return true;
6779
6780 if (all_of(Range: PHI->operands(),
6781 P: [&](Value *Operand) { return DT.dominates(Def: Operand, User: PHI); }))
6782 return true;
6783
6784 return false;
6785}
6786
6787const ConstantRange &
6788ScalarEvolution::getRangeRefIter(const SCEV *S,
6789 ScalarEvolution::RangeSignHint SignHint) {
6790 DenseMap<const SCEV *, ConstantRange> &Cache =
6791 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6792 : SignedRanges;
6793 SmallVector<SCEVUse> WorkList;
6794 SmallPtrSet<const SCEV *, 8> Seen;
6795
6796 // Add Expr to the worklist, if Expr is either an N-ary expression or a
6797 // SCEVUnknown PHI node.
6798 auto AddToWorklist = [&WorkList, &Seen, &Cache](const SCEV *Expr) {
6799 if (!Seen.insert(Ptr: Expr).second)
6800 return;
6801 if (Cache.contains(Val: Expr))
6802 return;
6803 switch (Expr->getSCEVType()) {
6804 case scUnknown:
6805 if (!isa<PHINode>(Val: cast<SCEVUnknown>(Val: Expr)->getValue()))
6806 break;
6807 [[fallthrough]];
6808 case scConstant:
6809 case scVScale:
6810 case scTruncate:
6811 case scZeroExtend:
6812 case scSignExtend:
6813 case scPtrToAddr:
6814 case scPtrToInt:
6815 case scAddExpr:
6816 case scMulExpr:
6817 case scUDivExpr:
6818 case scAddRecExpr:
6819 case scUMaxExpr:
6820 case scSMaxExpr:
6821 case scUMinExpr:
6822 case scSMinExpr:
6823 case scSequentialUMinExpr:
6824 WorkList.push_back(Elt: Expr);
6825 break;
6826 case scCouldNotCompute:
6827 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6828 }
6829 };
6830 AddToWorklist(S);
6831
6832 // Build worklist by queuing operands of N-ary expressions and phi nodes.
6833 for (unsigned I = 0; I != WorkList.size(); ++I) {
6834 const SCEV *P = WorkList[I];
6835 auto *UnknownS = dyn_cast<SCEVUnknown>(Val: P);
6836 // If it is not a `SCEVUnknown`, just recurse into operands.
6837 if (!UnknownS) {
6838 for (const SCEV *Op : P->operands())
6839 AddToWorklist(Op);
6840 continue;
6841 }
6842 // `SCEVUnknown`'s require special treatment.
6843 if (PHINode *P = dyn_cast<PHINode>(Val: UnknownS->getValue())) {
6844 if (!RangeRefPHIAllowedOperands(DT, PHI: P))
6845 continue;
6846 for (auto &Op : reverse(C: P->operands()))
6847 AddToWorklist(getSCEV(V: Op));
6848 }
6849 }
6850
6851 if (!WorkList.empty()) {
6852 // Use getRangeRef to compute ranges for items in the worklist in reverse
6853 // order. This will force ranges for earlier operands to be computed before
6854 // their users in most cases.
6855 for (const SCEV *P : reverse(C: drop_begin(RangeOrContainer&: WorkList))) {
6856 getRangeRef(S: P, Hint: SignHint);
6857 }
6858 }
6859
6860 return getRangeRef(S, Hint: SignHint, Depth: 0);
6861}
6862
6863/// Determine the range for a particular SCEV. If SignHint is
6864/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
6865/// with a "cleaner" unsigned (resp. signed) representation.
6866const ConstantRange &ScalarEvolution::getRangeRef(
6867 const SCEV *S, ScalarEvolution::RangeSignHint SignHint, unsigned Depth) {
6868 DenseMap<const SCEV *, ConstantRange> &Cache =
6869 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6870 : SignedRanges;
6871 ConstantRange::PreferredRangeType RangeType =
6872 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? ConstantRange::Unsigned
6873 : ConstantRange::Signed;
6874
6875 // See if we've computed this range already.
6876 auto I = Cache.find(Val: S);
6877 if (I != Cache.end())
6878 return I->second;
6879
6880 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: S))
6881 return setRange(S: C, Hint: SignHint, CR: ConstantRange(C->getAPInt()));
6882
6883 // Switch to iteratively computing the range for S, if it is part of a deeply
6884 // nested expression.
6885 if (Depth > RangeIterThreshold)
6886 return getRangeRefIter(S, SignHint);
6887
6888 unsigned BitWidth = getTypeSizeInBits(Ty: S->getType());
6889 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
6890 using OBO = OverflowingBinaryOperator;
6891
6892 // If the value has known zeros, the maximum value will have those known zeros
6893 // as well.
6894 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
6895 APInt Multiple = getNonZeroConstantMultiple(S);
6896 APInt Remainder = APInt::getMaxValue(numBits: BitWidth).urem(RHS: Multiple);
6897 if (!Remainder.isZero())
6898 ConservativeResult =
6899 ConstantRange(APInt::getMinValue(numBits: BitWidth),
6900 APInt::getMaxValue(numBits: BitWidth) - Remainder + 1);
6901 }
6902 else {
6903 uint32_t TZ = getMinTrailingZeros(S);
6904 if (TZ != 0) {
6905 ConservativeResult = ConstantRange(
6906 APInt::getSignedMinValue(numBits: BitWidth),
6907 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: TZ).shl(shiftAmt: TZ) + 1);
6908 }
6909 }
6910
6911 switch (S->getSCEVType()) {
6912 case scConstant:
6913 llvm_unreachable("Already handled above.");
6914 case scVScale:
6915 return setRange(S, Hint: SignHint, CR: getVScaleRange(F: &F, BitWidth));
6916 case scTruncate: {
6917 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: S);
6918 ConstantRange X = getRangeRef(S: Trunc->getOperand(), SignHint, Depth: Depth + 1);
6919 return setRange(
6920 S: Trunc, Hint: SignHint,
6921 CR: ConservativeResult.intersectWith(CR: X.truncate(BitWidth), Type: RangeType));
6922 }
6923 case scZeroExtend: {
6924 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: S);
6925 ConstantRange X = getRangeRef(S: ZExt->getOperand(), SignHint, Depth: Depth + 1);
6926 return setRange(
6927 S: ZExt, Hint: SignHint,
6928 CR: ConservativeResult.intersectWith(CR: X.zeroExtend(BitWidth), Type: RangeType));
6929 }
6930 case scSignExtend: {
6931 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: S);
6932 ConstantRange X = getRangeRef(S: SExt->getOperand(), SignHint, Depth: Depth + 1);
6933 return setRange(
6934 S: SExt, Hint: SignHint,
6935 CR: ConservativeResult.intersectWith(CR: X.signExtend(BitWidth), Type: RangeType));
6936 }
6937 case scPtrToAddr:
6938 case scPtrToInt: {
6939 const SCEVCastExpr *Cast = cast<SCEVCastExpr>(Val: S);
6940 ConstantRange X = getRangeRef(S: Cast->getOperand(), SignHint, Depth: Depth + 1);
6941 return setRange(S: Cast, Hint: SignHint, CR: X);
6942 }
6943 case scAddExpr: {
6944 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: S);
6945 // Check if this is a URem pattern: A - (A / B) * B, which is always < B.
6946 const SCEV *URemLHS = nullptr, *URemRHS = nullptr;
6947 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED &&
6948 match(S, P: m_scev_URem(LHS: m_SCEV(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE&: *this))) {
6949 ConstantRange LHSRange = getRangeRef(S: URemLHS, SignHint, Depth: Depth + 1);
6950 ConstantRange RHSRange = getRangeRef(S: URemRHS, SignHint, Depth: Depth + 1);
6951 ConservativeResult =
6952 ConservativeResult.intersectWith(CR: LHSRange.urem(Other: RHSRange), Type: RangeType);
6953 }
6954 ConstantRange X = getRangeRef(S: Add->getOperand(i: 0), SignHint, Depth: Depth + 1);
6955 unsigned WrapType = OBO::AnyWrap;
6956 if (Add->hasNoSignedWrap())
6957 WrapType |= OBO::NoSignedWrap;
6958 if (Add->hasNoUnsignedWrap())
6959 WrapType |= OBO::NoUnsignedWrap;
6960 for (const SCEV *Op : drop_begin(RangeOrContainer: Add->operands()))
6961 X = X.addWithNoWrap(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1), NoWrapKind: WrapType,
6962 RangeType);
6963 return setRange(S: Add, Hint: SignHint,
6964 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6965 }
6966 case scMulExpr: {
6967 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val: S);
6968 ConstantRange X = getRangeRef(S: Mul->getOperand(i: 0), SignHint, Depth: Depth + 1);
6969 for (const SCEV *Op : drop_begin(RangeOrContainer: Mul->operands()))
6970 X = X.multiply(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1));
6971 return setRange(S: Mul, Hint: SignHint,
6972 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6973 }
6974 case scUDivExpr: {
6975 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: S);
6976 ConstantRange X = getRangeRef(S: UDiv->getLHS(), SignHint, Depth: Depth + 1);
6977 ConstantRange Y = getRangeRef(S: UDiv->getRHS(), SignHint, Depth: Depth + 1);
6978 return setRange(S: UDiv, Hint: SignHint,
6979 CR: ConservativeResult.intersectWith(CR: X.udiv(Other: Y), Type: RangeType));
6980 }
6981 case scAddRecExpr: {
6982 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: S);
6983 // If there's no unsigned wrap, the value will never be less than its
6984 // initial value.
6985 if (AddRec->hasNoUnsignedWrap()) {
6986 APInt UnsignedMinValue = getUnsignedRangeMin(S: AddRec->getStart());
6987 if (!UnsignedMinValue.isZero())
6988 ConservativeResult = ConservativeResult.intersectWith(
6989 CR: ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), Type: RangeType);
6990 }
6991
6992 // If there's no signed wrap, and all the operands except initial value have
6993 // the same sign or zero, the value won't ever be:
6994 // 1: smaller than initial value if operands are non negative,
6995 // 2: bigger than initial value if operands are non positive.
6996 // For both cases, value can not cross signed min/max boundary.
6997 if (AddRec->hasNoSignedWrap()) {
6998 bool AllNonNeg = true;
6999 bool AllNonPos = true;
7000 for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
7001 if (!isKnownNonNegative(S: AddRec->getOperand(i)))
7002 AllNonNeg = false;
7003 if (!isKnownNonPositive(S: AddRec->getOperand(i)))
7004 AllNonPos = false;
7005 }
7006 if (AllNonNeg)
7007 ConservativeResult = ConservativeResult.intersectWith(
7008 CR: ConstantRange::getNonEmpty(Lower: getSignedRangeMin(S: AddRec->getStart()),
7009 Upper: APInt::getSignedMinValue(numBits: BitWidth)),
7010 Type: RangeType);
7011 else if (AllNonPos)
7012 ConservativeResult = ConservativeResult.intersectWith(
7013 CR: ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
7014 Upper: getSignedRangeMax(S: AddRec->getStart()) +
7015 1),
7016 Type: RangeType);
7017 }
7018
7019 // TODO: non-affine addrec
7020 if (AddRec->isAffine()) {
7021 const SCEV *MaxBEScev =
7022 getConstantMaxBackedgeTakenCount(L: AddRec->getLoop());
7023 if (!isa<SCEVCouldNotCompute>(Val: MaxBEScev)) {
7024 APInt MaxBECount = cast<SCEVConstant>(Val: MaxBEScev)->getAPInt();
7025
7026 // Adjust MaxBECount to the same bitwidth as AddRec. We can truncate if
7027 // MaxBECount's active bits are all <= AddRec's bit width.
7028 if (MaxBECount.getBitWidth() > BitWidth &&
7029 MaxBECount.getActiveBits() <= BitWidth)
7030 MaxBECount = MaxBECount.trunc(width: BitWidth);
7031 else if (MaxBECount.getBitWidth() < BitWidth)
7032 MaxBECount = MaxBECount.zext(width: BitWidth);
7033
7034 if (MaxBECount.getBitWidth() == BitWidth) {
7035 auto [RangeFromAffine, Flags] = getRangeForAffineAR(
7036 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
7037 ConservativeResult =
7038 ConservativeResult.intersectWith(CR: RangeFromAffine, Type: RangeType);
7039 const_cast<SCEVAddRecExpr *>(AddRec)->setNoWrapFlags(Flags);
7040
7041 auto RangeFromFactoring = getRangeViaFactoring(
7042 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
7043 ConservativeResult =
7044 ConservativeResult.intersectWith(CR: RangeFromFactoring, Type: RangeType);
7045 }
7046 }
7047
7048 // Now try symbolic BE count and more powerful methods.
7049 if (UseExpensiveRangeSharpening) {
7050 const SCEV *SymbolicMaxBECount =
7051 getSymbolicMaxBackedgeTakenCount(L: AddRec->getLoop());
7052 if (!isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount) &&
7053 getTypeSizeInBits(Ty: MaxBEScev->getType()) <= BitWidth &&
7054 AddRec->hasNoSelfWrap()) {
7055 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
7056 AddRec, MaxBECount: SymbolicMaxBECount, BitWidth, SignHint);
7057 ConservativeResult =
7058 ConservativeResult.intersectWith(CR: RangeFromAffineNew, Type: RangeType);
7059 }
7060 }
7061 }
7062
7063 return setRange(S: AddRec, Hint: SignHint, CR: std::move(ConservativeResult));
7064 }
7065 case scUMaxExpr:
7066 case scSMaxExpr:
7067 case scUMinExpr:
7068 case scSMinExpr:
7069 case scSequentialUMinExpr: {
7070 Intrinsic::ID ID;
7071 switch (S->getSCEVType()) {
7072 case scUMaxExpr:
7073 ID = Intrinsic::umax;
7074 break;
7075 case scSMaxExpr:
7076 ID = Intrinsic::smax;
7077 break;
7078 case scUMinExpr:
7079 case scSequentialUMinExpr:
7080 ID = Intrinsic::umin;
7081 break;
7082 case scSMinExpr:
7083 ID = Intrinsic::smin;
7084 break;
7085 default:
7086 llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.");
7087 }
7088
7089 const auto *NAry = cast<SCEVNAryExpr>(Val: S);
7090 ConstantRange X = getRangeRef(S: NAry->getOperand(i: 0), SignHint, Depth: Depth + 1);
7091 for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
7092 X = X.intrinsic(
7093 IntrinsicID: ID, Ops: {X, getRangeRef(S: NAry->getOperand(i), SignHint, Depth: Depth + 1)});
7094 return setRange(S, Hint: SignHint,
7095 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
7096 }
7097 case scUnknown: {
7098 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
7099 Value *V = U->getValue();
7100
7101 // Check if the IR explicitly contains !range metadata.
7102 std::optional<ConstantRange> MDRange = GetRangeFromMetadata(V);
7103 if (MDRange)
7104 ConservativeResult =
7105 ConservativeResult.intersectWith(CR: *MDRange, Type: RangeType);
7106
7107 // Use facts about recurrences in the underlying IR. Note that add
7108 // recurrences are AddRecExprs and thus don't hit this path. This
7109 // primarily handles shift recurrences.
7110 auto CR = getRangeForUnknownRecurrence(U);
7111 ConservativeResult = ConservativeResult.intersectWith(CR);
7112
7113 // See if ValueTracking can give us a useful range.
7114 const DataLayout &DL = getDataLayout();
7115 KnownBits Known = computeKnownBits(V, DL, AC: &AC, CxtI: nullptr, DT: &DT);
7116 if (Known.getBitWidth() != BitWidth)
7117 Known = Known.zextOrTrunc(BitWidth);
7118
7119 // ValueTracking may be able to compute a tighter result for the number of
7120 // sign bits than for the value of those sign bits.
7121 unsigned NS = ComputeNumSignBits(Op: V, DL, AC: &AC, CxtI: nullptr, DT: &DT);
7122 if (U->getType()->isPointerTy()) {
7123 // If the pointer size is larger than the index size type, this can cause
7124 // NS to be larger than BitWidth. So compensate for this.
7125 unsigned ptrSize = DL.getPointerTypeSizeInBits(U->getType());
7126 int ptrIdxDiff = ptrSize - BitWidth;
7127 if (ptrIdxDiff > 0 && ptrSize > BitWidth && NS > (unsigned)ptrIdxDiff)
7128 NS -= ptrIdxDiff;
7129 }
7130
7131 if (NS > 1) {
7132 // If we know any of the sign bits, we know all of the sign bits.
7133 if (!Known.Zero.getHiBits(numBits: NS).isZero())
7134 Known.Zero.setHighBits(NS);
7135 if (!Known.One.getHiBits(numBits: NS).isZero())
7136 Known.One.setHighBits(NS);
7137 }
7138
7139 if (Known.getMinValue() != Known.getMaxValue() + 1)
7140 ConservativeResult = ConservativeResult.intersectWith(
7141 CR: ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
7142 Type: RangeType);
7143 if (NS > 1)
7144 ConservativeResult = ConservativeResult.intersectWith(
7145 CR: ConstantRange(APInt::getSignedMinValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1),
7146 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1) + 1),
7147 Type: RangeType);
7148
7149 if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
7150 // Strengthen the range if the underlying IR value is a
7151 // global/alloca/heap allocation using the size of the object.
7152 bool CanBeNull;
7153 uint64_t DerefBytes = V->getPointerDereferenceableBytes(
7154 DL, CanBeNull, /*CanBeFreed=*/nullptr);
7155 if (DerefBytes > 1 && isUIntN(N: BitWidth, x: DerefBytes)) {
7156 // The highest address the object can start is DerefBytes bytes before
7157 // the end (unsigned max value). If this value is not a multiple of the
7158 // alignment, the last possible start value is the next lowest multiple
7159 // of the alignment. Note: The computations below cannot overflow,
7160 // because if they would there's no possible start address for the
7161 // object.
7162 APInt MaxVal =
7163 APInt::getMaxValue(numBits: BitWidth) - APInt(BitWidth, DerefBytes);
7164 uint64_t Align = U->getValue()->getPointerAlignment(DL).value();
7165 uint64_t Rem = MaxVal.urem(RHS: Align);
7166 MaxVal -= APInt(BitWidth, Rem);
7167 APInt MinVal = APInt::getZero(numBits: BitWidth);
7168 if (llvm::isKnownNonZero(V, Q: DL))
7169 MinVal = Align;
7170 ConservativeResult = ConservativeResult.intersectWith(
7171 CR: ConstantRange::getNonEmpty(Lower: MinVal, Upper: MaxVal + 1), Type: RangeType);
7172 }
7173 }
7174
7175 // A range of Phi is a subset of union of all ranges of its input.
7176 if (PHINode *Phi = dyn_cast<PHINode>(Val: V)) {
7177 // SCEVExpander sometimes creates SCEVUnknowns that are secretly
7178 // AddRecs; return the range for the corresponding AddRec.
7179 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: getSCEV(V)))
7180 return getRangeRef(S: AR, SignHint, Depth: Depth + 1);
7181
7182 // Make sure that we do not run over cycled Phis.
7183 if (RangeRefPHIAllowedOperands(DT, PHI: Phi)) {
7184 ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
7185
7186 for (const auto &Op : Phi->operands()) {
7187 auto OpRange = getRangeRef(S: getSCEV(V: Op), SignHint, Depth: Depth + 1);
7188 RangeFromOps = RangeFromOps.unionWith(CR: OpRange);
7189 // No point to continue if we already have a full set.
7190 if (RangeFromOps.isFullSet())
7191 break;
7192 }
7193 ConservativeResult =
7194 ConservativeResult.intersectWith(CR: RangeFromOps, Type: RangeType);
7195 }
7196 }
7197
7198 // vscale can't be equal to zero
7199 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V))
7200 if (II->getIntrinsicID() == Intrinsic::vscale) {
7201 ConstantRange Disallowed = APInt::getZero(numBits: BitWidth);
7202 ConservativeResult = ConservativeResult.difference(CR: Disallowed);
7203 }
7204
7205 return setRange(S: U, Hint: SignHint, CR: std::move(ConservativeResult));
7206 }
7207 case scCouldNotCompute:
7208 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
7209 }
7210
7211 return setRange(S, Hint: SignHint, CR: std::move(ConservativeResult));
7212}
7213
7214// Given a StartRange, Step and MaxBECount for an expression compute a range of
7215// values that the expression can take. Initially, the expression has a value
7216// from StartRange and then is changed by Step up to MaxBECount times. Signed
7217// argument defines if we treat Step as signed or unsigned. The second return
7218// value indicates that no wrapping occurred.
7219static std::pair<ConstantRange, bool>
7220getRangeForAffineARHelper(APInt Step, const ConstantRange &StartRange,
7221 const APInt &MaxBECount, bool Signed) {
7222 unsigned BitWidth = Step.getBitWidth();
7223 assert(BitWidth == StartRange.getBitWidth() &&
7224 BitWidth == MaxBECount.getBitWidth() && "mismatched bit widths");
7225 // If either Step or MaxBECount is 0, then the expression won't change, and we
7226 // just need to return the initial range.
7227 if (Step == 0 || MaxBECount == 0)
7228 return {StartRange, true};
7229
7230 // If we don't know anything about the initial value (i.e. StartRange is
7231 // FullRange), then we don't know anything about the final range either.
7232 // Return FullRange.
7233 if (StartRange.isFullSet())
7234 return {ConstantRange::getFull(BitWidth), false};
7235
7236 // If Step is signed and negative, then we use its absolute value, but we also
7237 // note that we're moving in the opposite direction.
7238 bool Descending = Signed && Step.isNegative();
7239
7240 if (Signed)
7241 // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
7242 // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
7243 // This equations hold true due to the well-defined wrap-around behavior of
7244 // APInt.
7245 Step = Step.abs();
7246
7247 // Check if Offset is more than full span of BitWidth. If it is, the
7248 // expression is guaranteed to overflow.
7249 if (APInt::getMaxValue(numBits: StartRange.getBitWidth()).udiv(RHS: Step).ult(RHS: MaxBECount))
7250 return {ConstantRange::getFull(BitWidth), false};
7251
7252 // Offset is by how much the expression can change. Checks above guarantee no
7253 // overflow here.
7254 APInt Offset = Step * MaxBECount;
7255
7256 // Minimum value of the final range will match the minimal value of StartRange
7257 // if the expression is increasing and will be decreased by Offset otherwise.
7258 // Maximum value of the final range will match the maximal value of StartRange
7259 // if the expression is decreasing and will be increased by Offset otherwise.
7260 APInt StartLower = StartRange.getLower();
7261 APInt StartUpper = StartRange.getUpper() - 1;
7262 bool Overflow;
7263 APInt MovedBoundary;
7264 if (Signed) {
7265 // This does not use sadd_ov, as we want to check overflow for a signed
7266 // start with an unsigned offset.
7267 if (Descending) {
7268 MovedBoundary = StartLower - std::move(Offset);
7269 Overflow = MovedBoundary.sgt(RHS: StartLower) || StartRange.isSignWrappedSet();
7270 } else {
7271 MovedBoundary = StartUpper + std::move(Offset);
7272 Overflow = MovedBoundary.slt(RHS: StartUpper) || StartRange.isSignWrappedSet();
7273 }
7274 } else {
7275 MovedBoundary = StartUpper.uadd_ov(RHS: std::move(Offset), Overflow);
7276 Overflow |= StartRange.isWrappedSet();
7277 }
7278
7279 // It's possible that the new minimum/maximum value will fall into the initial
7280 // range (due to wrap around). This means that the expression can take any
7281 // value in this bitwidth, and we have to return full range.
7282 if (StartRange.contains(Val: MovedBoundary))
7283 return {ConstantRange::getFull(BitWidth), false};
7284
7285 APInt NewLower =
7286 Descending ? std::move(MovedBoundary) : std::move(StartLower);
7287 APInt NewUpper =
7288 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
7289 NewUpper += 1;
7290
7291 // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
7292 return {ConstantRange::getNonEmpty(Lower: std::move(NewLower), Upper: std::move(NewUpper)),
7293 !Overflow};
7294}
7295
7296std::pair<ConstantRange, SCEV::NoWrapFlags>
7297ScalarEvolution::getRangeForAffineAR(const SCEV *Start, const SCEV *Step,
7298 const APInt &MaxBECount) {
7299 assert(getTypeSizeInBits(Start->getType()) ==
7300 getTypeSizeInBits(Step->getType()) &&
7301 getTypeSizeInBits(Start->getType()) == MaxBECount.getBitWidth() &&
7302 "mismatched bit widths");
7303
7304 // First, consider step signed.
7305 ConstantRange StartSRange = getSignedRange(S: Start);
7306 ConstantRange StepSRange = getSignedRange(S: Step);
7307
7308 // If Step can be both positive and negative, we need to find ranges for the
7309 // maximum absolute step values in both directions and union them.
7310 auto [SR1, NSW1] = getRangeForAffineARHelper(
7311 Step: StepSRange.getSignedMin(), StartRange: StartSRange, MaxBECount, /*Signed=*/true);
7312 auto [SR2, NSW2] = getRangeForAffineARHelper(Step: StepSRange.getSignedMax(),
7313 StartRange: StartSRange, MaxBECount,
7314 /*Signed=*/true);
7315 ConstantRange SR = SR1.unionWith(CR: SR2);
7316
7317 // Next, consider step unsigned.
7318 auto [UR, NUW] = getRangeForAffineARHelper(
7319 Step: getUnsignedRangeMax(S: Step), StartRange: getUnsignedRange(S: Start), MaxBECount,
7320 /*Signed=*/false);
7321
7322 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7323 if (NUW)
7324 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7325 if (NSW1 && NSW2)
7326 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7327
7328 // Finally, intersect signed and unsigned ranges.
7329 return {SR.intersectWith(CR: UR, Type: ConstantRange::Smallest), Flags};
7330}
7331
7332ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
7333 const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
7334 ScalarEvolution::RangeSignHint SignHint) {
7335 assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n");
7336 assert(AddRec->hasNoSelfWrap() &&
7337 "This only works for non-self-wrapping AddRecs!");
7338 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
7339 const SCEV *Step = AddRec->getStepRecurrence(SE&: *this);
7340 // Only deal with constant step to save compile time.
7341 if (!isa<SCEVConstant>(Val: Step))
7342 return ConstantRange::getFull(BitWidth);
7343 // Let's make sure that we can prove that we do not self-wrap during
7344 // MaxBECount iterations. We need this because MaxBECount is a maximum
7345 // iteration count estimate, and we might infer nw from some exit for which we
7346 // do not know max exit count (or any other side reasoning).
7347 // TODO: Turn into assert at some point.
7348 if (getTypeSizeInBits(Ty: MaxBECount->getType()) >
7349 getTypeSizeInBits(Ty: AddRec->getType()))
7350 return ConstantRange::getFull(BitWidth);
7351 MaxBECount = getNoopOrZeroExtend(V: MaxBECount, Ty: AddRec->getType());
7352 const SCEV *RangeWidth = getMinusOne(Ty: AddRec->getType());
7353 const SCEV *StepAbs = getUMinExpr(LHS: Step, RHS: getNegativeSCEV(V: Step));
7354 const SCEV *MaxItersWithoutWrap = getUDivExpr(LHS: RangeWidth, RHS: StepAbs);
7355 if (!isKnownPredicateViaConstantRanges(Pred: ICmpInst::ICMP_ULE, LHS: MaxBECount,
7356 RHS: MaxItersWithoutWrap))
7357 return ConstantRange::getFull(BitWidth);
7358
7359 ICmpInst::Predicate LEPred =
7360 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
7361 ICmpInst::Predicate GEPred =
7362 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
7363 const SCEV *End = AddRec->evaluateAtIteration(It: MaxBECount, SE&: *this);
7364
7365 // We know that there is no self-wrap. Let's take Start and End values and
7366 // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
7367 // the iteration. They either lie inside the range [Min(Start, End),
7368 // Max(Start, End)] or outside it:
7369 //
7370 // Case 1: RangeMin ... Start V1 ... VN End ... RangeMax;
7371 // Case 2: RangeMin Vk ... V1 Start ... End Vn ... Vk + 1 RangeMax;
7372 //
7373 // No self wrap flag guarantees that the intermediate values cannot be BOTH
7374 // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
7375 // knowledge, let's try to prove that we are dealing with Case 1. It is so if
7376 // Start <= End and step is positive, or Start >= End and step is negative.
7377 const SCEV *Start = applyLoopGuards(Expr: AddRec->getStart(), L: AddRec->getLoop());
7378 ConstantRange StartRange = getRangeRef(S: Start, SignHint);
7379 ConstantRange EndRange = getRangeRef(S: End, SignHint);
7380 ConstantRange RangeBetween = StartRange.unionWith(CR: EndRange);
7381 // If they already cover full iteration space, we will know nothing useful
7382 // even if we prove what we want to prove.
7383 if (RangeBetween.isFullSet())
7384 return RangeBetween;
7385 // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
7386 bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
7387 : RangeBetween.isWrappedSet();
7388 if (IsWrappedSet)
7389 return ConstantRange::getFull(BitWidth);
7390
7391 if (isKnownPositive(S: Step) &&
7392 isKnownPredicateViaConstantRanges(Pred: LEPred, LHS: Start, RHS: End))
7393 return RangeBetween;
7394 if (isKnownNegative(S: Step) &&
7395 isKnownPredicateViaConstantRanges(Pred: GEPred, LHS: Start, RHS: End))
7396 return RangeBetween;
7397 return ConstantRange::getFull(BitWidth);
7398}
7399
7400ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
7401 const SCEV *Step,
7402 const APInt &MaxBECount) {
7403 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
7404 // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
7405
7406 unsigned BitWidth = MaxBECount.getBitWidth();
7407 assert(getTypeSizeInBits(Start->getType()) == BitWidth &&
7408 getTypeSizeInBits(Step->getType()) == BitWidth &&
7409 "mismatched bit widths");
7410
7411 struct SelectPattern {
7412 Value *Condition = nullptr;
7413 APInt TrueValue;
7414 APInt FalseValue;
7415
7416 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
7417 const SCEV *S) {
7418 std::optional<unsigned> CastOp;
7419 APInt Offset(BitWidth, 0);
7420
7421 assert(SE.getTypeSizeInBits(S->getType()) == BitWidth &&
7422 "Should be!");
7423
7424 // Peel off a constant offset. In the future we could consider being
7425 // smarter here and handle {Start+Step,+,Step} too.
7426 const APInt *Off;
7427 if (match(S, P: m_scev_Add(Op0: m_scev_APInt(C&: Off), Op1: m_SCEV(V&: S))))
7428 Offset = *Off;
7429
7430 // Peel off a cast operation
7431 if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(Val: S)) {
7432 CastOp = SCast->getSCEVType();
7433 S = SCast->getOperand();
7434 }
7435
7436 using namespace llvm::PatternMatch;
7437
7438 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
7439 const APInt *TrueVal, *FalseVal;
7440 if (!SU ||
7441 !match(V: SU->getValue(), P: m_Select(C: m_Value(V&: Condition), L: m_APInt(Res&: TrueVal),
7442 R: m_APInt(Res&: FalseVal)))) {
7443 Condition = nullptr;
7444 return;
7445 }
7446
7447 TrueValue = *TrueVal;
7448 FalseValue = *FalseVal;
7449
7450 // Re-apply the cast we peeled off earlier
7451 if (CastOp)
7452 switch (*CastOp) {
7453 default:
7454 llvm_unreachable("Unknown SCEV cast type!");
7455
7456 case scTruncate:
7457 TrueValue = TrueValue.trunc(width: BitWidth);
7458 FalseValue = FalseValue.trunc(width: BitWidth);
7459 break;
7460 case scZeroExtend:
7461 TrueValue = TrueValue.zext(width: BitWidth);
7462 FalseValue = FalseValue.zext(width: BitWidth);
7463 break;
7464 case scSignExtend:
7465 TrueValue = TrueValue.sext(width: BitWidth);
7466 FalseValue = FalseValue.sext(width: BitWidth);
7467 break;
7468 }
7469
7470 // Re-apply the constant offset we peeled off earlier
7471 TrueValue += Offset;
7472 FalseValue += Offset;
7473 }
7474
7475 bool isRecognized() { return Condition != nullptr; }
7476 };
7477
7478 SelectPattern StartPattern(*this, BitWidth, Start);
7479 if (!StartPattern.isRecognized())
7480 return ConstantRange::getFull(BitWidth);
7481
7482 SelectPattern StepPattern(*this, BitWidth, Step);
7483 if (!StepPattern.isRecognized())
7484 return ConstantRange::getFull(BitWidth);
7485
7486 if (StartPattern.Condition != StepPattern.Condition) {
7487 // We don't handle this case today; but we could, by considering four
7488 // possibilities below instead of two. I'm not sure if there are cases where
7489 // that will help over what getRange already does, though.
7490 return ConstantRange::getFull(BitWidth);
7491 }
7492
7493 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
7494 // construct arbitrary general SCEV expressions here. This function is called
7495 // from deep in the call stack, and calling getSCEV (on a sext instruction,
7496 // say) can end up caching a suboptimal value.
7497
7498 // FIXME: without the explicit `this` receiver below, MSVC errors out with
7499 // C2352 and C2512 (otherwise it isn't needed).
7500
7501 const SCEV *TrueStart = this->getConstant(Val: StartPattern.TrueValue);
7502 const SCEV *TrueStep = this->getConstant(Val: StepPattern.TrueValue);
7503 const SCEV *FalseStart = this->getConstant(Val: StartPattern.FalseValue);
7504 const SCEV *FalseStep = this->getConstant(Val: StepPattern.FalseValue);
7505
7506 ConstantRange TrueRange =
7507 this->getRangeForAffineAR(Start: TrueStart, Step: TrueStep, MaxBECount).first;
7508 ConstantRange FalseRange =
7509 this->getRangeForAffineAR(Start: FalseStart, Step: FalseStep, MaxBECount).first;
7510
7511 return TrueRange.unionWith(CR: FalseRange);
7512}
7513
7514SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
7515 if (isa<ConstantExpr>(Val: V)) return SCEV::FlagAnyWrap;
7516 const BinaryOperator *BinOp = cast<BinaryOperator>(Val: V);
7517
7518 // Return early if there are no flags to propagate to the SCEV.
7519 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7520 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: BinOp);
7521 PDI && PDI->isDisjoint()) {
7522 Flags = ScalarEvolution::setFlags(Flags: SCEV::FlagNUW, OnFlags: SCEV::FlagNSW);
7523 } else {
7524 if (BinOp->hasNoUnsignedWrap())
7525 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7526 if (BinOp->hasNoSignedWrap())
7527 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7528 }
7529 if (Flags == SCEV::FlagAnyWrap)
7530 return SCEV::FlagAnyWrap;
7531
7532 return isSCEVExprNeverPoison(I: BinOp) ? Flags : SCEV::FlagAnyWrap;
7533}
7534
7535const Instruction *
7536ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
7537 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
7538 return &*AddRec->getLoop()->getHeader()->begin();
7539 if (auto *U = dyn_cast<SCEVUnknown>(Val: S))
7540 if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
7541 return I;
7542 return nullptr;
7543}
7544
7545const Instruction *ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops,
7546 bool &Precise) {
7547 Precise = true;
7548 // Do a bounded search of the def relation of the requested SCEVs.
7549 SmallPtrSet<const SCEV *, 16> Visited;
7550 SmallVector<SCEVUse> Worklist;
7551 auto pushOp = [&](const SCEV *S) {
7552 if (!Visited.insert(Ptr: S).second)
7553 return;
7554 // Threshold of 30 here is arbitrary.
7555 if (Visited.size() > 30) {
7556 Precise = false;
7557 return;
7558 }
7559 Worklist.push_back(Elt: S);
7560 };
7561
7562 for (SCEVUse S : Ops)
7563 pushOp(S);
7564
7565 const Instruction *Bound = nullptr;
7566 while (!Worklist.empty()) {
7567 SCEVUse S = Worklist.pop_back_val();
7568 if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
7569 if (!Bound || DT.dominates(Def: Bound, User: DefI))
7570 Bound = DefI;
7571 } else {
7572 for (SCEVUse Op : S->operands())
7573 pushOp(Op);
7574 }
7575 }
7576 return Bound ? Bound : &*F.getEntryBlock().begin();
7577}
7578
7579const Instruction *
7580ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops) {
7581 bool Discard;
7582 return getDefiningScopeBound(Ops, Precise&: Discard);
7583}
7584
7585bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
7586 const Instruction *B) {
7587 if (A->getParent() == B->getParent() &&
7588 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7589 End: B->getIterator()))
7590 return true;
7591
7592 auto *BLoop = LI.getLoopFor(BB: B->getParent());
7593 if (BLoop && BLoop->getHeader() == B->getParent() &&
7594 BLoop->getLoopPreheader() == A->getParent() &&
7595 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7596 End: A->getParent()->end()) &&
7597 isGuaranteedToTransferExecutionToSuccessor(Begin: B->getParent()->begin(),
7598 End: B->getIterator()))
7599 return true;
7600 return false;
7601}
7602
7603bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
7604 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
7605 visitAll(Root: Op, Visitor&: PC);
7606 return PC.MaybePoison.empty();
7607}
7608
7609bool ScalarEvolution::isGuaranteedNotToCauseUB(const SCEV *Op) {
7610 return !SCEVExprContains(Root: Op, Pred: [this](const SCEV *S) {
7611 const SCEV *Op1;
7612 bool M = match(S, P: m_scev_UDiv(Op0: m_SCEV(), Op1: m_SCEV(V&: Op1)));
7613 // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
7614 // is a non-zero constant, we have to assume the UDiv may be UB.
7615 return M && (!isKnownNonZero(S: Op1) || !isGuaranteedNotToBePoison(Op: Op1));
7616 });
7617}
7618
7619bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
7620 // Only proceed if we can prove that I does not yield poison.
7621 if (!programUndefinedIfPoison(Inst: I))
7622 return false;
7623
7624 // At this point we know that if I is executed, then it does not wrap
7625 // according to at least one of NSW or NUW. If I is not executed, then we do
7626 // not know if the calculation that I represents would wrap. Multiple
7627 // instructions can map to the same SCEV. If we apply NSW or NUW from I to
7628 // the SCEV, we must guarantee no wrapping for that SCEV also when it is
7629 // derived from other instructions that map to the same SCEV. We cannot make
7630 // that guarantee for cases where I is not executed. So we need to find a
7631 // upper bound on the defining scope for the SCEV, and prove that I is
7632 // executed every time we enter that scope. When the bounding scope is a
7633 // loop (the common case), this is equivalent to proving I executes on every
7634 // iteration of that loop.
7635 SmallVector<SCEVUse> SCEVOps;
7636 for (const Use &Op : I->operands()) {
7637 // I could be an extractvalue from a call to an overflow intrinsic.
7638 // TODO: We can do better here in some cases.
7639 if (isSCEVable(Ty: Op->getType()))
7640 SCEVOps.push_back(Elt: getSCEV(V: Op));
7641 }
7642 auto *DefI = getDefiningScopeBound(Ops: SCEVOps);
7643 return isGuaranteedToTransferExecutionTo(A: DefI, B: I);
7644}
7645
7646bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
7647 // If we know that \c I can never be poison period, then that's enough.
7648 if (isSCEVExprNeverPoison(I))
7649 return true;
7650
7651 // If the loop only has one exit, then we know that, if the loop is entered,
7652 // any instruction dominating that exit will be executed. If any such
7653 // instruction would result in UB, the addrec cannot be poison.
7654 //
7655 // This is basically the same reasoning as in isSCEVExprNeverPoison(), but
7656 // also handles uses outside the loop header (they just need to dominate the
7657 // single exit).
7658
7659 auto *ExitingBB = L->getExitingBlock();
7660 if (!ExitingBB || !loopHasNoAbnormalExits(L))
7661 return false;
7662
7663 SmallPtrSet<const Value *, 16> KnownPoison;
7664 SmallVector<const Instruction *, 8> Worklist;
7665
7666 // We start by assuming \c I, the post-inc add recurrence, is poison. Only
7667 // things that are known to be poison under that assumption go on the
7668 // Worklist.
7669 KnownPoison.insert(Ptr: I);
7670 Worklist.push_back(Elt: I);
7671
7672 while (!Worklist.empty()) {
7673 const Instruction *Poison = Worklist.pop_back_val();
7674
7675 for (const Use &U : Poison->uses()) {
7676 const Instruction *PoisonUser = cast<Instruction>(Val: U.getUser());
7677 if (mustTriggerUB(I: PoisonUser, KnownPoison) &&
7678 DT.dominates(A: PoisonUser->getParent(), B: ExitingBB))
7679 return true;
7680
7681 if (propagatesPoison(PoisonOp: U) && L->contains(Inst: PoisonUser))
7682 if (KnownPoison.insert(Ptr: PoisonUser).second)
7683 Worklist.push_back(Elt: PoisonUser);
7684 }
7685 }
7686
7687 return false;
7688}
7689
7690ScalarEvolution::LoopProperties
7691ScalarEvolution::getLoopProperties(const Loop *L) {
7692 using LoopProperties = ScalarEvolution::LoopProperties;
7693
7694 auto Itr = LoopPropertiesCache.find(Val: L);
7695 if (Itr == LoopPropertiesCache.end()) {
7696 auto HasSideEffects = [](Instruction *I) {
7697 if (auto *SI = dyn_cast<StoreInst>(Val: I))
7698 return !SI->isSimple();
7699
7700 if (I->mayThrow())
7701 return true;
7702
7703 // Non-volatile memset / memcpy do not count as side-effect for forward
7704 // progress.
7705 if (isa<MemIntrinsic>(Val: I) && !I->isVolatile())
7706 return false;
7707
7708 return I->mayWriteToMemory();
7709 };
7710
7711 LoopProperties LP = {/* HasNoAbnormalExits */ true,
7712 /*HasNoSideEffects*/ true};
7713
7714 for (auto *BB : L->getBlocks())
7715 for (auto &I : *BB) {
7716 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7717 LP.HasNoAbnormalExits = false;
7718 if (HasSideEffects(&I))
7719 LP.HasNoSideEffects = false;
7720 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7721 break; // We're already as pessimistic as we can get.
7722 }
7723
7724 auto InsertPair = LoopPropertiesCache.insert(KV: {L, LP});
7725 assert(InsertPair.second && "We just checked!");
7726 Itr = InsertPair.first;
7727 }
7728
7729 return Itr->second;
7730}
7731
7732bool ScalarEvolution::loopIsFiniteByAssumption(const Loop *L) {
7733 // A mustprogress loop without side effects must be finite.
7734 // TODO: The check used here is very conservative. It's only *specific*
7735 // side effects which are well defined in infinite loops.
7736 return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L));
7737}
7738
7739const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
7740 // Worklist item with a Value and a bool indicating whether all operands have
7741 // been visited already.
7742 using PointerTy = PointerIntPair<Value *, 1, bool>;
7743 SmallVector<PointerTy> Stack;
7744
7745 Stack.emplace_back(Args&: V, Args: false);
7746 while (!Stack.empty()) {
7747 auto E = Stack.back();
7748 Value *CurV = E.getPointer();
7749
7750 if (getExistingSCEV(V: CurV)) {
7751 Stack.pop_back();
7752 continue;
7753 }
7754
7755 SmallVector<Value *> Ops;
7756 const SCEV *CreatedSCEV = nullptr;
7757 // If all operands have been visited already, create the SCEV.
7758 if (E.getInt()) {
7759 CreatedSCEV = createSCEV(V: CurV);
7760 } else {
7761 // Otherwise get the operands we need to create SCEV's for before creating
7762 // the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
7763 // just use it.
7764 CreatedSCEV = getOperandsToCreate(V: CurV, Ops);
7765 }
7766
7767 if (CreatedSCEV) {
7768 insertValueToMap(V: CurV, S: CreatedSCEV);
7769 Stack.pop_back();
7770 } else {
7771 Stack.back().setInt(true);
7772 // Queue its operands which need to be constructed.
7773 for (Value *Op : Ops)
7774 Stack.emplace_back(Args&: Op, Args: false);
7775 }
7776 }
7777
7778 return getExistingSCEV(V);
7779}
7780
7781const SCEV *
7782ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
7783 if (!isSCEVable(Ty: V->getType()))
7784 return getUnknown(V);
7785
7786 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7787 // Don't attempt to analyze instructions in blocks that aren't
7788 // reachable. Such instructions don't matter, and they aren't required
7789 // to obey basic rules for definitions dominating uses which this
7790 // analysis depends on.
7791 if (!DT.isReachableFromEntry(A: I->getParent()))
7792 return getUnknown(V: PoisonValue::get(T: V->getType()));
7793 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7794 return getConstant(V: CI);
7795 else if (isa<GlobalAlias>(Val: V))
7796 return getUnknown(V);
7797 else if (!isa<ConstantExpr>(Val: V))
7798 return getUnknown(V);
7799
7800 Operator *U = cast<Operator>(Val: V);
7801 if (auto BO =
7802 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CxtI: dyn_cast<Instruction>(Val: V))) {
7803 bool IsConstArg = isa<ConstantInt>(Val: BO->RHS);
7804 switch (BO->Opcode) {
7805 case Instruction::Add:
7806 case Instruction::Mul: {
7807 // For additions and multiplications, traverse add/mul chains for which we
7808 // can potentially create a single SCEV, to reduce the number of
7809 // get{Add,Mul}Expr calls.
7810 do {
7811 if (BO->Op) {
7812 if (BO->Op != V && getExistingSCEV(V: BO->Op)) {
7813 Ops.push_back(Elt: BO->Op);
7814 break;
7815 }
7816 }
7817 Ops.push_back(Elt: BO->RHS);
7818 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7819 CxtI: dyn_cast<Instruction>(Val: V));
7820 if (!NewBO ||
7821 (BO->Opcode == Instruction::Add &&
7822 (NewBO->Opcode != Instruction::Add &&
7823 NewBO->Opcode != Instruction::Sub)) ||
7824 (BO->Opcode == Instruction::Mul &&
7825 NewBO->Opcode != Instruction::Mul)) {
7826 Ops.push_back(Elt: BO->LHS);
7827 break;
7828 }
7829 // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
7830 // requires a SCEV for the LHS.
7831 if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
7832 auto *I = dyn_cast<Instruction>(Val: BO->Op);
7833 if (I && programUndefinedIfPoison(Inst: I)) {
7834 Ops.push_back(Elt: BO->LHS);
7835 break;
7836 }
7837 }
7838 BO = NewBO;
7839 } while (true);
7840 return nullptr;
7841 }
7842 case Instruction::Sub:
7843 case Instruction::UDiv:
7844 case Instruction::URem:
7845 break;
7846 case Instruction::AShr:
7847 case Instruction::Shl:
7848 case Instruction::Xor:
7849 if (!IsConstArg)
7850 return nullptr;
7851 break;
7852 case Instruction::And:
7853 case Instruction::Or:
7854 if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(BitWidth: 1))
7855 return nullptr;
7856 break;
7857 case Instruction::LShr:
7858 return getUnknown(V);
7859 default:
7860 llvm_unreachable("Unhandled binop");
7861 break;
7862 }
7863
7864 Ops.push_back(Elt: BO->LHS);
7865 Ops.push_back(Elt: BO->RHS);
7866 return nullptr;
7867 }
7868
7869 switch (U->getOpcode()) {
7870 case Instruction::Trunc:
7871 case Instruction::ZExt:
7872 case Instruction::SExt:
7873 case Instruction::PtrToAddr:
7874 case Instruction::PtrToInt:
7875 Ops.push_back(Elt: U->getOperand(i: 0));
7876 return nullptr;
7877
7878 case Instruction::BitCast:
7879 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType())) {
7880 Ops.push_back(Elt: U->getOperand(i: 0));
7881 return nullptr;
7882 }
7883 return getUnknown(V);
7884
7885 case Instruction::SDiv:
7886 case Instruction::SRem:
7887 Ops.push_back(Elt: U->getOperand(i: 0));
7888 Ops.push_back(Elt: U->getOperand(i: 1));
7889 return nullptr;
7890
7891 case Instruction::GetElementPtr:
7892 assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() &&
7893 "GEP source element type must be sized");
7894 llvm::append_range(C&: Ops, R: U->operands());
7895 return nullptr;
7896
7897 case Instruction::IntToPtr:
7898 return getUnknown(V);
7899
7900 case Instruction::PHI:
7901 // getNodeForPHI has four ways to turn a PHI into a SCEV; retrieve the
7902 // relevant nodes for each of them.
7903 //
7904 // The first is just to call simplifyInstruction, and get something back
7905 // that isn't a PHI.
7906 if (Value *V = simplifyInstruction(
7907 I: cast<PHINode>(Val: U),
7908 Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
7909 /*UseInstrInfo=*/true, /*CanUseUndef=*/false})) {
7910 assert(V);
7911 Ops.push_back(Elt: V);
7912 return nullptr;
7913 }
7914 // The second is createNodeForPHIWithIdenticalOperands: this looks for
7915 // operands which all perform the same operation, but haven't been
7916 // CSE'ed for whatever reason.
7917 if (BinaryOperator *BO = getCommonInstForPHI(PN: cast<PHINode>(Val: U))) {
7918 assert(BO);
7919 Ops.push_back(Elt: BO);
7920 return nullptr;
7921 }
7922 // The third is createNodeFromSelectLikePHI; this takes a PHI which
7923 // is equivalent to a select, and analyzes it like a select.
7924 {
7925 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
7926 if (getOperandsForSelectLikePHI(DT, PN: cast<PHINode>(Val: U), Cond, LHS, RHS)) {
7927 assert(Cond);
7928 assert(LHS);
7929 assert(RHS);
7930 if (auto *CondICmp = dyn_cast<ICmpInst>(Val: Cond)) {
7931 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 0));
7932 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 1));
7933 }
7934 Ops.push_back(Elt: Cond);
7935 Ops.push_back(Elt: LHS);
7936 Ops.push_back(Elt: RHS);
7937 return nullptr;
7938 }
7939 }
7940 // The fourth way is createAddRecFromPHI. It's complicated to handle here,
7941 // so just construct it recursively.
7942 //
7943 // In addition to getNodeForPHI, also construct nodes which might be needed
7944 // by getRangeRef.
7945 if (RangeRefPHIAllowedOperands(DT, PHI: cast<PHINode>(Val: U))) {
7946 for (Value *V : cast<PHINode>(Val: U)->operands())
7947 Ops.push_back(Elt: V);
7948 return nullptr;
7949 }
7950 return nullptr;
7951
7952 case Instruction::Select: {
7953 // Check if U is a select that can be simplified to a SCEVUnknown.
7954 auto CanSimplifyToUnknown = [this, U]() {
7955 if (U->getType()->isIntegerTy(BitWidth: 1) || isa<ConstantInt>(Val: U->getOperand(i: 0)))
7956 return false;
7957
7958 auto *ICI = dyn_cast<ICmpInst>(Val: U->getOperand(i: 0));
7959 if (!ICI)
7960 return false;
7961 Value *LHS = ICI->getOperand(i_nocapture: 0);
7962 Value *RHS = ICI->getOperand(i_nocapture: 1);
7963 if (ICI->getPredicate() == CmpInst::ICMP_EQ ||
7964 ICI->getPredicate() == CmpInst::ICMP_NE) {
7965 if (!(isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()))
7966 return true;
7967 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
7968 getTypeSizeInBits(Ty: U->getType()))
7969 return true;
7970 return false;
7971 };
7972 if (CanSimplifyToUnknown())
7973 return getUnknown(V: U);
7974
7975 llvm::append_range(C&: Ops, R: U->operands());
7976 return nullptr;
7977 break;
7978 }
7979 case Instruction::Call:
7980 case Instruction::Invoke:
7981 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand()) {
7982 Ops.push_back(Elt: RV);
7983 return nullptr;
7984 }
7985
7986 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
7987 switch (II->getIntrinsicID()) {
7988 case Intrinsic::abs:
7989 Ops.push_back(Elt: II->getArgOperand(i: 0));
7990 return nullptr;
7991 case Intrinsic::umax:
7992 case Intrinsic::umin:
7993 case Intrinsic::smax:
7994 case Intrinsic::smin:
7995 case Intrinsic::usub_sat:
7996 case Intrinsic::uadd_sat:
7997 Ops.push_back(Elt: II->getArgOperand(i: 0));
7998 Ops.push_back(Elt: II->getArgOperand(i: 1));
7999 return nullptr;
8000 case Intrinsic::start_loop_iterations:
8001 case Intrinsic::annotation:
8002 case Intrinsic::ptr_annotation:
8003 Ops.push_back(Elt: II->getArgOperand(i: 0));
8004 return nullptr;
8005 default:
8006 break;
8007 }
8008 }
8009 break;
8010 }
8011
8012 return nullptr;
8013}
8014
8015const SCEV *ScalarEvolution::createSCEV(Value *V) {
8016 if (!isSCEVable(Ty: V->getType()))
8017 return getUnknown(V);
8018
8019 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
8020 // Don't attempt to analyze instructions in blocks that aren't
8021 // reachable. Such instructions don't matter, and they aren't required
8022 // to obey basic rules for definitions dominating uses which this
8023 // analysis depends on.
8024 if (!DT.isReachableFromEntry(A: I->getParent()))
8025 return getUnknown(V: PoisonValue::get(T: V->getType()));
8026 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
8027 return getConstant(V: CI);
8028 else if (isa<GlobalAlias>(Val: V))
8029 return getUnknown(V);
8030 else if (!isa<ConstantExpr>(Val: V))
8031 return getUnknown(V);
8032
8033 const SCEV *LHS;
8034 const SCEV *RHS;
8035
8036 Operator *U = cast<Operator>(Val: V);
8037 if (auto BO =
8038 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CxtI: dyn_cast<Instruction>(Val: V))) {
8039 switch (BO->Opcode) {
8040 case Instruction::Add: {
8041 // The simple thing to do would be to just call getSCEV on both operands
8042 // and call getAddExpr with the result. However if we're looking at a
8043 // bunch of things all added together, this can be quite inefficient,
8044 // because it leads to N-1 getAddExpr calls for N ultimate operands.
8045 // Instead, gather up all the operands and make a single getAddExpr call.
8046 // LLVM IR canonical form means we need only traverse the left operands.
8047 SmallVector<SCEVUse, 4> AddOps;
8048 do {
8049 if (BO->Op) {
8050 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
8051 AddOps.push_back(Elt: OpSCEV);
8052 break;
8053 }
8054
8055 // If a NUW or NSW flag can be applied to the SCEV for this
8056 // addition, then compute the SCEV for this addition by itself
8057 // with a separate call to getAddExpr. We need to do that
8058 // instead of pushing the operands of the addition onto AddOps,
8059 // since the flags are only known to apply to this particular
8060 // addition - they may not apply to other additions that can be
8061 // formed with operands from AddOps.
8062 const SCEV *RHS = getSCEV(V: BO->RHS);
8063 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
8064 if (Flags != SCEV::FlagAnyWrap) {
8065 const SCEV *LHS = getSCEV(V: BO->LHS);
8066 if (BO->Opcode == Instruction::Sub)
8067 AddOps.push_back(Elt: getMinusSCEV(LHS, RHS, Flags));
8068 else
8069 AddOps.push_back(Elt: getAddExpr(LHS, RHS, Flags));
8070 break;
8071 }
8072 }
8073
8074 if (BO->Opcode == Instruction::Sub)
8075 AddOps.push_back(Elt: getNegativeSCEV(V: getSCEV(V: BO->RHS)));
8076 else
8077 AddOps.push_back(Elt: getSCEV(V: BO->RHS));
8078
8079 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
8080 CxtI: dyn_cast<Instruction>(Val: V));
8081 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
8082 NewBO->Opcode != Instruction::Sub)) {
8083 AddOps.push_back(Elt: getSCEV(V: BO->LHS));
8084 break;
8085 }
8086 BO = NewBO;
8087 } while (true);
8088
8089 return getAddExpr(Ops&: AddOps);
8090 }
8091
8092 case Instruction::Mul: {
8093 SmallVector<SCEVUse, 4> MulOps;
8094 do {
8095 if (BO->Op) {
8096 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
8097 MulOps.push_back(Elt: OpSCEV);
8098 break;
8099 }
8100
8101 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
8102 if (Flags != SCEV::FlagAnyWrap) {
8103 LHS = getSCEV(V: BO->LHS);
8104 RHS = getSCEV(V: BO->RHS);
8105 MulOps.push_back(Elt: getMulExpr(LHS, RHS, Flags));
8106 break;
8107 }
8108 }
8109
8110 MulOps.push_back(Elt: getSCEV(V: BO->RHS));
8111 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
8112 CxtI: dyn_cast<Instruction>(Val: V));
8113 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
8114 MulOps.push_back(Elt: getSCEV(V: BO->LHS));
8115 break;
8116 }
8117 BO = NewBO;
8118 } while (true);
8119
8120 return getMulExpr(Ops&: MulOps);
8121 }
8122 case Instruction::UDiv:
8123 LHS = getSCEV(V: BO->LHS);
8124 RHS = getSCEV(V: BO->RHS);
8125 return getUDivExpr(LHS, RHS);
8126 case Instruction::URem:
8127 LHS = getSCEV(V: BO->LHS);
8128 RHS = getSCEV(V: BO->RHS);
8129 return getURemExpr(LHS, RHS);
8130 case Instruction::Sub: {
8131 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
8132 if (BO->Op)
8133 Flags = getNoWrapFlagsFromUB(V: BO->Op);
8134 LHS = getSCEV(V: BO->LHS);
8135 RHS = getSCEV(V: BO->RHS);
8136 return getMinusSCEV(LHS, RHS, Flags);
8137 }
8138 case Instruction::And:
8139 // For an expression like x&255 that merely masks off the high bits,
8140 // use zext(trunc(x)) as the SCEV expression.
8141 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8142 if (CI->isZero())
8143 return getSCEV(V: BO->RHS);
8144 if (CI->isMinusOne())
8145 return getSCEV(V: BO->LHS);
8146 const APInt &A = CI->getValue();
8147
8148 // Instcombine's ShrinkDemandedConstant may strip bits out of
8149 // constants, obscuring what would otherwise be a low-bits mask.
8150 // Use computeKnownBits to compute what ShrinkDemandedConstant
8151 // knew about to reconstruct a low-bits mask value.
8152 unsigned LZ = A.countl_zero();
8153 unsigned TZ = A.countr_zero();
8154 unsigned BitWidth = A.getBitWidth();
8155 KnownBits Known(BitWidth);
8156 computeKnownBits(V: BO->LHS, Known, DL: getDataLayout(), AC: &AC, CxtI: nullptr, DT: &DT);
8157
8158 APInt EffectiveMask =
8159 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - LZ - TZ).shl(shiftAmt: TZ);
8160 if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
8161 const SCEV *MulCount = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ));
8162 const SCEV *LHS = getSCEV(V: BO->LHS);
8163 const SCEV *ShiftedLHS = nullptr;
8164 if (auto *LHSMul = dyn_cast<SCEVMulExpr>(Val: LHS)) {
8165 if (auto *OpC = dyn_cast<SCEVConstant>(Val: LHSMul->getOperand(i: 0))) {
8166 // For an expression like (x * 8) & 8, simplify the multiply.
8167 unsigned MulZeros = OpC->getAPInt().countr_zero();
8168 unsigned GCD = std::min(a: MulZeros, b: TZ);
8169 APInt DivAmt = APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ - GCD);
8170 SmallVector<SCEVUse, 4> MulOps;
8171 MulOps.push_back(Elt: getConstant(Val: OpC->getAPInt().ashr(ShiftAmt: GCD)));
8172 append_range(C&: MulOps, R: LHSMul->operands().drop_front());
8173 auto *NewMul = getMulExpr(Ops&: MulOps, OrigFlags: LHSMul->getNoWrapFlags());
8174 ShiftedLHS = getUDivExpr(LHS: NewMul, RHS: getConstant(Val: DivAmt));
8175 }
8176 }
8177 if (!ShiftedLHS)
8178 ShiftedLHS = getUDivExpr(LHS, RHS: MulCount);
8179 return getMulExpr(
8180 LHS: getZeroExtendExpr(
8181 Op: getTruncateExpr(Op: ShiftedLHS,
8182 Ty: IntegerType::get(C&: getContext(), NumBits: BitWidth - LZ - TZ)),
8183 Ty: BO->LHS->getType()),
8184 RHS: MulCount);
8185 }
8186 }
8187 // Binary `and` is a bit-wise `umin`.
8188 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
8189 LHS = getSCEV(V: BO->LHS);
8190 RHS = getSCEV(V: BO->RHS);
8191 return getUMinExpr(LHS, RHS);
8192 }
8193 break;
8194
8195 case Instruction::Or:
8196 // Binary `or` is a bit-wise `umax`.
8197 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
8198 LHS = getSCEV(V: BO->LHS);
8199 RHS = getSCEV(V: BO->RHS);
8200 return getUMaxExpr(LHS, RHS);
8201 }
8202 break;
8203
8204 case Instruction::Xor:
8205 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8206 // If the RHS of xor is -1, then this is a not operation.
8207 if (CI->isMinusOne())
8208 return getNotSCEV(V: getSCEV(V: BO->LHS));
8209
8210 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
8211 // This is a variant of the check for xor with -1, and it handles
8212 // the case where instcombine has trimmed non-demanded bits out
8213 // of an xor with -1.
8214 if (auto *LBO = dyn_cast<BinaryOperator>(Val: BO->LHS))
8215 if (ConstantInt *LCI = dyn_cast<ConstantInt>(Val: LBO->getOperand(i_nocapture: 1)))
8216 if (LBO->getOpcode() == Instruction::And &&
8217 LCI->getValue() == CI->getValue())
8218 if (const SCEVZeroExtendExpr *Z =
8219 dyn_cast<SCEVZeroExtendExpr>(Val: getSCEV(V: BO->LHS))) {
8220 Type *UTy = BO->LHS->getType();
8221 const SCEV *Z0 = Z->getOperand();
8222 Type *Z0Ty = Z0->getType();
8223 unsigned Z0TySize = getTypeSizeInBits(Ty: Z0Ty);
8224
8225 // If C is a low-bits mask, the zero extend is serving to
8226 // mask off the high bits. Complement the operand and
8227 // re-apply the zext.
8228 if (CI->getValue().isMask(numBits: Z0TySize))
8229 return getZeroExtendExpr(Op: getNotSCEV(V: Z0), Ty: UTy);
8230
8231 // If C is a single bit, it may be in the sign-bit position
8232 // before the zero-extend. In this case, represent the xor
8233 // using an add, which is equivalent, and re-apply the zext.
8234 APInt Trunc = CI->getValue().trunc(width: Z0TySize);
8235 if (Trunc.zext(width: getTypeSizeInBits(Ty: UTy)) == CI->getValue() &&
8236 Trunc.isSignMask())
8237 return getZeroExtendExpr(Op: getAddExpr(LHS: Z0, RHS: getConstant(Val: Trunc)),
8238 Ty: UTy);
8239 }
8240 }
8241 break;
8242
8243 case Instruction::Shl:
8244 // Turn shift left of a constant amount into a multiply.
8245 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8246 uint32_t BitWidth = cast<IntegerType>(Val: SA->getType())->getBitWidth();
8247
8248 // If the shift count is not less than the bitwidth, the result of
8249 // the shift is undefined. Don't try to analyze it, because the
8250 // resolution chosen here may differ from the resolution chosen in
8251 // other parts of the compiler.
8252 if (SA->getValue().uge(RHS: BitWidth))
8253 break;
8254
8255 // We can safely preserve the nuw flag in all cases. It's also safe to
8256 // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
8257 // requires special handling. It can be preserved as long as we're not
8258 // left shifting by bitwidth - 1.
8259 auto Flags = SCEV::FlagAnyWrap;
8260 if (BO->Op) {
8261 auto MulFlags = getNoWrapFlagsFromUB(V: BO->Op);
8262 if (any(Val: MulFlags & SCEV::FlagNSW) &&
8263 (any(Val: MulFlags & SCEV::FlagNUW) ||
8264 SA->getValue().ult(RHS: BitWidth - 1)))
8265 Flags = Flags | SCEV::FlagNSW;
8266 if (any(Val: MulFlags & SCEV::FlagNUW))
8267 Flags = Flags | SCEV::FlagNUW;
8268 }
8269
8270 ConstantInt *X = ConstantInt::get(
8271 Context&: getContext(), V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
8272 return getMulExpr(LHS: getSCEV(V: BO->LHS), RHS: getConstant(V: X), Flags);
8273 }
8274 break;
8275
8276 case Instruction::AShr:
8277 // AShr X, C, where C is a constant.
8278 ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS);
8279 if (!CI)
8280 break;
8281
8282 Type *OuterTy = BO->LHS->getType();
8283 uint64_t BitWidth = getTypeSizeInBits(Ty: OuterTy);
8284 // If the shift count is not less than the bitwidth, the result of
8285 // the shift is undefined. Don't try to analyze it, because the
8286 // resolution chosen here may differ from the resolution chosen in
8287 // other parts of the compiler.
8288 if (CI->getValue().uge(RHS: BitWidth))
8289 break;
8290
8291 if (CI->isZero())
8292 return getSCEV(V: BO->LHS); // shift by zero --> noop
8293
8294 uint64_t AShrAmt = CI->getZExtValue();
8295 Type *TruncTy = IntegerType::get(C&: getContext(), NumBits: BitWidth - AShrAmt);
8296
8297 Operator *L = dyn_cast<Operator>(Val: BO->LHS);
8298 const SCEV *AddTruncateExpr = nullptr;
8299 ConstantInt *ShlAmtCI = nullptr;
8300 const SCEV *AddConstant = nullptr;
8301
8302 if (L && L->getOpcode() == Instruction::Add) {
8303 // X = Shl A, n
8304 // Y = Add X, c
8305 // Z = AShr Y, m
8306 // n, c and m are constants.
8307
8308 Operator *LShift = dyn_cast<Operator>(Val: L->getOperand(i: 0));
8309 ConstantInt *AddOperandCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8310 if (LShift && LShift->getOpcode() == Instruction::Shl) {
8311 if (AddOperandCI) {
8312 const SCEV *ShlOp0SCEV = getSCEV(V: LShift->getOperand(i: 0));
8313 ShlAmtCI = dyn_cast<ConstantInt>(Val: LShift->getOperand(i: 1));
8314 // since we truncate to TruncTy, the AddConstant should be of the
8315 // same type, so create a new Constant with type same as TruncTy.
8316 // Also, the Add constant should be shifted right by AShr amount.
8317 APInt AddOperand = AddOperandCI->getValue().ashr(ShiftAmt: AShrAmt);
8318 AddConstant = getConstant(Val: AddOperand.trunc(width: BitWidth - AShrAmt));
8319 // we model the expression as sext(add(trunc(A), c << n)), since the
8320 // sext(trunc) part is already handled below, we create a
8321 // AddExpr(TruncExp) which will be used later.
8322 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8323 }
8324 }
8325 } else if (L && L->getOpcode() == Instruction::Shl) {
8326 // X = Shl A, n
8327 // Y = AShr X, m
8328 // Both n and m are constant.
8329
8330 const SCEV *ShlOp0SCEV = getSCEV(V: L->getOperand(i: 0));
8331 ShlAmtCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8332 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8333 }
8334
8335 if (AddTruncateExpr && ShlAmtCI) {
8336 // We can merge the two given cases into a single SCEV statement,
8337 // incase n = m, the mul expression will be 2^0, so it gets resolved to
8338 // a simpler case. The following code handles the two cases:
8339 //
8340 // 1) For a two-shift sext-inreg, i.e. n = m,
8341 // use sext(trunc(x)) as the SCEV expression.
8342 //
8343 // 2) When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
8344 // expression. We already checked that ShlAmt < BitWidth, so
8345 // the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
8346 // ShlAmt - AShrAmt < Amt.
8347 const APInt &ShlAmt = ShlAmtCI->getValue();
8348 if (ShlAmt.ult(RHS: BitWidth) && ShlAmt.uge(RHS: AShrAmt)) {
8349 APInt Mul = APInt::getOneBitSet(numBits: BitWidth - AShrAmt,
8350 BitNo: ShlAmtCI->getZExtValue() - AShrAmt);
8351 const SCEV *CompositeExpr =
8352 getMulExpr(LHS: AddTruncateExpr, RHS: getConstant(Val: Mul));
8353 if (L->getOpcode() != Instruction::Shl)
8354 CompositeExpr = getAddExpr(LHS: CompositeExpr, RHS: AddConstant);
8355
8356 return getSignExtendExpr(Op: CompositeExpr, Ty: OuterTy);
8357 }
8358 }
8359 break;
8360 }
8361 }
8362
8363 switch (U->getOpcode()) {
8364 case Instruction::Trunc:
8365 return getTruncateExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8366
8367 case Instruction::ZExt:
8368 return getZeroExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8369
8370 case Instruction::SExt:
8371 if (auto BO = MatchBinaryOp(V: U->getOperand(i: 0), DL: getDataLayout(), AC, DT,
8372 CxtI: dyn_cast<Instruction>(Val: V))) {
8373 // The NSW flag of a subtract does not always survive the conversion to
8374 // A + (-1)*B. By pushing sign extension onto its operands we are much
8375 // more likely to preserve NSW and allow later AddRec optimisations.
8376 //
8377 // NOTE: This is effectively duplicating this logic from getSignExtend:
8378 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
8379 // but by that point the NSW information has potentially been lost.
8380 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
8381 Type *Ty = U->getType();
8382 auto *V1 = getSignExtendExpr(Op: getSCEV(V: BO->LHS), Ty);
8383 auto *V2 = getSignExtendExpr(Op: getSCEV(V: BO->RHS), Ty);
8384 return getMinusSCEV(LHS: V1, RHS: V2, Flags: SCEV::FlagNSW);
8385 }
8386 }
8387 return getSignExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8388
8389 case Instruction::BitCast:
8390 // BitCasts are no-op casts so we just eliminate the cast.
8391 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType()))
8392 return getSCEV(V: U->getOperand(i: 0));
8393 break;
8394
8395 case Instruction::PtrToAddr: {
8396 const SCEV *IntOp = getPtrToAddrExpr(Op: getSCEV(V: U->getOperand(i: 0)));
8397 if (isa<SCEVCouldNotCompute>(Val: IntOp))
8398 return getUnknown(V);
8399 return IntOp;
8400 }
8401
8402 case Instruction::PtrToInt: {
8403 // Pointer to integer cast is straight-forward, so do model it.
8404 const SCEV *Op = getSCEV(V: U->getOperand(i: 0));
8405 Type *DstIntTy = U->getType();
8406 // But only if effective SCEV (integer) type is wide enough to represent
8407 // all possible pointer values.
8408 const SCEV *IntOp = getPtrToIntExpr(Op, Ty: DstIntTy);
8409 if (isa<SCEVCouldNotCompute>(Val: IntOp))
8410 return getUnknown(V);
8411 return IntOp;
8412 }
8413 case Instruction::IntToPtr:
8414 // Just don't deal with inttoptr casts.
8415 return getUnknown(V);
8416
8417 case Instruction::SDiv:
8418 // If both operands are non-negative, this is just an udiv.
8419 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8420 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8421 return getUDivExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8422 break;
8423
8424 case Instruction::SRem:
8425 // If both operands are non-negative, this is just an urem.
8426 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8427 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8428 return getURemExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8429 break;
8430
8431 case Instruction::GetElementPtr:
8432 return createNodeForGEP(GEP: cast<GEPOperator>(Val: U));
8433
8434 case Instruction::PHI:
8435 return createNodeForPHI(PN: cast<PHINode>(Val: U));
8436
8437 case Instruction::Select:
8438 return createNodeForSelectOrPHI(V: U, Cond: U->getOperand(i: 0), TrueVal: U->getOperand(i: 1),
8439 FalseVal: U->getOperand(i: 2));
8440
8441 case Instruction::Call:
8442 case Instruction::Invoke:
8443 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand())
8444 return getSCEV(V: RV);
8445
8446 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
8447 switch (II->getIntrinsicID()) {
8448 case Intrinsic::abs:
8449 return getAbsExpr(
8450 Op: getSCEV(V: II->getArgOperand(i: 0)),
8451 /*IsNSW=*/cast<ConstantInt>(Val: II->getArgOperand(i: 1))->isOne());
8452 case Intrinsic::umax:
8453 LHS = getSCEV(V: II->getArgOperand(i: 0));
8454 RHS = getSCEV(V: II->getArgOperand(i: 1));
8455 return getUMaxExpr(LHS, RHS);
8456 case Intrinsic::umin:
8457 LHS = getSCEV(V: II->getArgOperand(i: 0));
8458 RHS = getSCEV(V: II->getArgOperand(i: 1));
8459 return getUMinExpr(LHS, RHS);
8460 case Intrinsic::smax:
8461 LHS = getSCEV(V: II->getArgOperand(i: 0));
8462 RHS = getSCEV(V: II->getArgOperand(i: 1));
8463 return getSMaxExpr(LHS, RHS);
8464 case Intrinsic::smin:
8465 LHS = getSCEV(V: II->getArgOperand(i: 0));
8466 RHS = getSCEV(V: II->getArgOperand(i: 1));
8467 return getSMinExpr(LHS, RHS);
8468 case Intrinsic::usub_sat: {
8469 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8470 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8471 const SCEV *ClampedY = getUMinExpr(LHS: X, RHS: Y);
8472 return getMinusSCEV(LHS: X, RHS: ClampedY, Flags: SCEV::FlagNUW);
8473 }
8474 case Intrinsic::uadd_sat: {
8475 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8476 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8477 const SCEV *ClampedX = getUMinExpr(LHS: X, RHS: getNotSCEV(V: Y));
8478 return getAddExpr(LHS: ClampedX, RHS: Y, Flags: SCEV::FlagNUW);
8479 }
8480 case Intrinsic::start_loop_iterations:
8481 case Intrinsic::annotation:
8482 case Intrinsic::ptr_annotation:
8483 // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is
8484 // just eqivalent to the first operand for SCEV purposes.
8485 return getSCEV(V: II->getArgOperand(i: 0));
8486 case Intrinsic::vscale:
8487 return getVScale(Ty: II->getType());
8488 default:
8489 break;
8490 }
8491 }
8492 break;
8493 }
8494
8495 return getUnknown(V);
8496}
8497
8498//===----------------------------------------------------------------------===//
8499// Iteration Count Computation Code
8500//
8501
8502const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount) {
8503 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8504 return getCouldNotCompute();
8505
8506 auto *ExitCountType = ExitCount->getType();
8507 assert(ExitCountType->isIntegerTy());
8508 auto *EvalTy = Type::getIntNTy(C&: ExitCountType->getContext(),
8509 N: 1 + ExitCountType->getScalarSizeInBits());
8510 return getTripCountFromExitCount(ExitCount, EvalTy, L: nullptr);
8511}
8512
8513const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
8514 Type *EvalTy,
8515 const Loop *L) {
8516 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8517 return getCouldNotCompute();
8518
8519 unsigned ExitCountSize = getTypeSizeInBits(Ty: ExitCount->getType());
8520 unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
8521
8522 auto CanAddOneWithoutOverflow = [&]() {
8523 ConstantRange ExitCountRange =
8524 getRangeRef(S: ExitCount, SignHint: RangeSignHint::HINT_RANGE_UNSIGNED);
8525 if (!ExitCountRange.contains(Val: APInt::getMaxValue(numBits: ExitCountSize)))
8526 return true;
8527
8528 return L && isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: ExitCount,
8529 RHS: getMinusOne(Ty: ExitCount->getType()));
8530 };
8531
8532 // If we need to zero extend the backedge count, check if we can add one to
8533 // it prior to zero extending without overflow. Provided this is safe, it
8534 // allows better simplification of the +1.
8535 if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
8536 return getZeroExtendExpr(
8537 Op: getAddExpr(LHS: ExitCount, RHS: getOne(Ty: ExitCount->getType())), Ty: EvalTy);
8538
8539 // Get the total trip count from the count by adding 1. This may wrap.
8540 return getAddExpr(LHS: getTruncateOrZeroExtend(V: ExitCount, Ty: EvalTy), RHS: getOne(Ty: EvalTy));
8541}
8542
8543static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
8544 if (!ExitCount)
8545 return 0;
8546
8547 ConstantInt *ExitConst = ExitCount->getValue();
8548
8549 // Guard against huge trip counts.
8550 if (ExitConst->getValue().getActiveBits() > 32)
8551 return 0;
8552
8553 // In case of integer overflow, this returns 0, which is correct.
8554 return ((unsigned)ExitConst->getZExtValue()) + 1;
8555}
8556
8557unsigned ScalarEvolution::getSmallConstantTripCount(const Loop *L) {
8558 auto *ExitCount = dyn_cast<SCEVConstant>(Val: getBackedgeTakenCount(L, Kind: Exact));
8559 return getConstantTripCount(ExitCount);
8560}
8561
8562unsigned
8563ScalarEvolution::getSmallConstantTripCount(const Loop *L,
8564 const BasicBlock *ExitingBlock) {
8565 assert(ExitingBlock && "Must pass a non-null exiting block!");
8566 assert(L->isLoopExiting(ExitingBlock) &&
8567 "Exiting block must actually branch out of the loop!");
8568 const SCEVConstant *ExitCount =
8569 dyn_cast<SCEVConstant>(Val: getExitCount(L, ExitingBlock));
8570 return getConstantTripCount(ExitCount);
8571}
8572
8573unsigned ScalarEvolution::getSmallConstantMaxTripCount(
8574 const Loop *L, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8575
8576 const auto *MaxExitCount =
8577 Predicates ? getPredicatedConstantMaxBackedgeTakenCount(L, Predicates&: *Predicates)
8578 : getConstantMaxBackedgeTakenCount(L);
8579 return getConstantTripCount(ExitCount: dyn_cast<SCEVConstant>(Val: MaxExitCount));
8580}
8581
8582unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L) {
8583 SmallVector<BasicBlock *, 8> ExitingBlocks;
8584 L->getExitingBlocks(ExitingBlocks);
8585
8586 std::optional<unsigned> Res;
8587 for (auto *ExitingBB : ExitingBlocks) {
8588 unsigned Multiple = getSmallConstantTripMultiple(L, ExitingBlock: ExitingBB);
8589 if (!Res)
8590 Res = Multiple;
8591 Res = std::gcd(m: *Res, n: Multiple);
8592 }
8593 return Res.value_or(u: 1);
8594}
8595
8596unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8597 const SCEV *ExitCount) {
8598 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8599 return 1;
8600
8601 // Get the trip count
8602 const SCEV *TCExpr = getTripCountFromExitCount(ExitCount: applyLoopGuards(Expr: ExitCount, L));
8603
8604 APInt Multiple = getNonZeroConstantMultiple(S: TCExpr);
8605 // If a trip multiple is huge (>=2^32), the trip count is still divisible by
8606 // the greatest power of 2 divisor less than 2^32.
8607 return Multiple.getActiveBits() > 32
8608 ? 1U << std::min(a: 31U, b: Multiple.countTrailingZeros())
8609 : (unsigned)Multiple.getZExtValue();
8610}
8611
8612/// Returns the largest constant divisor of the trip count of this loop as a
8613/// normal unsigned value, if possible. This means that the actual trip count is
8614/// always a multiple of the returned value (don't forget the trip count could
8615/// very well be zero as well!).
8616///
8617/// Returns 1 if the trip count is unknown or not guaranteed to be the
8618/// multiple of a constant (which is also the case if the trip count is simply
8619/// constant, use getSmallConstantTripCount for that case), Will also return 1
8620/// if the trip count is very large (>= 2^32).
8621///
8622/// As explained in the comments for getSmallConstantTripCount, this assumes
8623/// that control exits the loop via ExitingBlock.
8624unsigned
8625ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8626 const BasicBlock *ExitingBlock) {
8627 assert(ExitingBlock && "Must pass a non-null exiting block!");
8628 assert(L->isLoopExiting(ExitingBlock) &&
8629 "Exiting block must actually branch out of the loop!");
8630 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
8631 return getSmallConstantTripMultiple(L, ExitCount);
8632}
8633
8634const SCEV *ScalarEvolution::getExitCount(const Loop *L,
8635 const BasicBlock *ExitingBlock,
8636 ExitCountKind Kind) {
8637 switch (Kind) {
8638 case Exact:
8639 return getBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this);
8640 case SymbolicMaximum:
8641 return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this);
8642 case ConstantMaximum:
8643 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this);
8644 };
8645 llvm_unreachable("Invalid ExitCountKind!");
8646}
8647
8648const SCEV *ScalarEvolution::getPredicatedExitCount(
8649 const Loop *L, const BasicBlock *ExitingBlock,
8650 SmallVectorImpl<const SCEVPredicate *> *Predicates, ExitCountKind Kind) {
8651 switch (Kind) {
8652 case Exact:
8653 return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this,
8654 Predicates);
8655 case SymbolicMaximum:
8656 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this,
8657 Predicates);
8658 case ConstantMaximum:
8659 return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this,
8660 Predicates);
8661 };
8662 llvm_unreachable("Invalid ExitCountKind!");
8663}
8664
8665const SCEV *ScalarEvolution::getPredicatedBackedgeTakenCount(
8666 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8667 return getPredicatedBackedgeTakenInfo(L).getExact(L, SE: this, Predicates: &Preds);
8668}
8669
8670const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
8671 ExitCountKind Kind) {
8672 switch (Kind) {
8673 case Exact:
8674 return getBackedgeTakenInfo(L).getExact(L, SE: this);
8675 case ConstantMaximum:
8676 return getBackedgeTakenInfo(L).getConstantMax(SE: this);
8677 case SymbolicMaximum:
8678 return getBackedgeTakenInfo(L).getSymbolicMax(L, SE: this);
8679 };
8680 llvm_unreachable("Invalid ExitCountKind!");
8681}
8682
8683const SCEV *ScalarEvolution::getPredicatedSymbolicMaxBackedgeTakenCount(
8684 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8685 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, SE: this, Predicates: &Preds);
8686}
8687
8688const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
8689 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8690 return getPredicatedBackedgeTakenInfo(L).getConstantMax(SE: this, Predicates: &Preds);
8691}
8692
8693bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
8694 return getBackedgeTakenInfo(L).isConstantMaxOrZero(SE: this);
8695}
8696
8697ScalarEvolution::BackedgeTakenInfo &
8698ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
8699 auto &BTI = getBackedgeTakenInfo(L);
8700 if (BTI.hasFullInfo())
8701 return BTI;
8702
8703 auto Pair = PredicatedBackedgeTakenCounts.try_emplace(Key: L);
8704
8705 if (!Pair.second)
8706 return Pair.first->second;
8707
8708 BackedgeTakenInfo Result =
8709 computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
8710
8711 return PredicatedBackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8712}
8713
8714ScalarEvolution::BackedgeTakenInfo &
8715ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
8716 // Initially insert an invalid entry for this loop. If the insertion
8717 // succeeds, proceed to actually compute a backedge-taken count and
8718 // update the value. The temporary CouldNotCompute value tells SCEV
8719 // code elsewhere that it shouldn't attempt to request a new
8720 // backedge-taken count, which could result in infinite recursion.
8721 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
8722 BackedgeTakenCounts.try_emplace(Key: L);
8723 if (!Pair.second)
8724 return Pair.first->second;
8725
8726 // computeBackedgeTakenCount may allocate memory for its result. Inserting it
8727 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
8728 // must be cleared in this scope.
8729 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
8730
8731 // Now that we know more about the trip count for this loop, forget any
8732 // existing SCEV values for PHI nodes in this loop since they are only
8733 // conservative estimates made without the benefit of trip count
8734 // information. This invalidation is not necessary for correctness, and is
8735 // only done to produce more precise results.
8736 if (Result.hasAnyInfo()) {
8737 // Invalidate any expression using an addrec in this loop.
8738 SmallVector<SCEVUse, 8> ToForget;
8739 auto LoopUsersIt = LoopUsers.find(Val: L);
8740 if (LoopUsersIt != LoopUsers.end())
8741 append_range(C&: ToForget, R&: LoopUsersIt->second);
8742 forgetMemoizedResults(SCEVs: ToForget);
8743
8744 // Invalidate constant-evolved loop header phis.
8745 for (PHINode &PN : L->getHeader()->phis())
8746 ConstantEvolutionLoopExitValue.erase(Val: &PN);
8747 }
8748
8749 // Re-lookup the insert position, since the call to
8750 // computeBackedgeTakenCount above could result in a
8751 // recusive call to getBackedgeTakenInfo (on a different
8752 // loop), which would invalidate the iterator computed
8753 // earlier.
8754 return BackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8755}
8756
8757void ScalarEvolution::forgetAllLoops() {
8758 // This method is intended to forget all info about loops. It should
8759 // invalidate caches as if the following happened:
8760 // - The trip counts of all loops have changed arbitrarily
8761 // - Every llvm::Value has been updated in place to produce a different
8762 // result.
8763 BackedgeTakenCounts.clear();
8764 PredicatedBackedgeTakenCounts.clear();
8765 BECountUsers.clear();
8766 LoopPropertiesCache.clear();
8767 ConstantEvolutionLoopExitValue.clear();
8768 ValueExprMap.clear();
8769 ValuesAtScopes.clear();
8770 ValuesAtScopesUsers.clear();
8771 LoopDispositions.clear();
8772 BlockDispositions.clear();
8773 UnsignedRanges.clear();
8774 SignedRanges.clear();
8775 ExprValueMap.clear();
8776 HasRecMap.clear();
8777 ConstantMultipleCache.clear();
8778 PredicatedSCEVRewrites.clear();
8779 FoldCache.clear();
8780 FoldCacheUser.clear();
8781}
8782void ScalarEvolution::visitAndClearUsers(
8783 SmallVectorImpl<Instruction *> &Worklist,
8784 SmallPtrSetImpl<Instruction *> &Visited,
8785 SmallVectorImpl<SCEVUse> &ToForget) {
8786 while (!Worklist.empty()) {
8787 Instruction *I = Worklist.pop_back_val();
8788 if (!isSCEVable(Ty: I->getType()) && !isa<WithOverflowInst>(Val: I))
8789 continue;
8790
8791 ValueExprMapType::iterator It =
8792 ValueExprMap.find_as(Val: static_cast<Value *>(I));
8793 if (It != ValueExprMap.end()) {
8794 ToForget.push_back(Elt: It->second);
8795 eraseValueFromMap(V: It->first);
8796 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
8797 ConstantEvolutionLoopExitValue.erase(Val: PN);
8798 }
8799
8800 PushDefUseChildren(I, Worklist, Visited);
8801 }
8802}
8803
8804void ScalarEvolution::forgetLoop(const Loop *L) {
8805 SmallVector<const Loop *, 16> LoopWorklist(1, L);
8806 SmallVector<SCEVUse, 16> ToForget;
8807
8808 // Iterate over all the loops and sub-loops to drop SCEV information.
8809 while (!LoopWorklist.empty()) {
8810 auto *CurrL = LoopWorklist.pop_back_val();
8811
8812 // Drop any stored trip count value.
8813 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ false);
8814 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ true);
8815
8816 // Drop information about predicated SCEV rewrites for this loop.
8817 PredicatedSCEVRewrites.remove_if(
8818 Pred: [&](const auto &Entry) { return Entry.first.second == CurrL; });
8819
8820 auto LoopUsersItr = LoopUsers.find(Val: CurrL);
8821 if (LoopUsersItr != LoopUsers.end())
8822 llvm::append_range(C&: ToForget, R&: LoopUsersItr->second);
8823
8824 // Drop information about expressions based on loop-header PHIs.
8825 for (PHINode &PN : CurrL->getHeader()->phis()) {
8826 ConstantEvolutionLoopExitValue.erase(Val: &PN);
8827 auto VIt = ValueExprMap.find_as(Val: static_cast<Value *>(&PN));
8828 if (VIt != ValueExprMap.end())
8829 ToForget.push_back(Elt: VIt->second);
8830 }
8831
8832 LoopPropertiesCache.erase(Val: CurrL);
8833 // Forget all contained loops too, to avoid dangling entries in the
8834 // ValuesAtScopes map.
8835 LoopWorklist.append(in_start: CurrL->begin(), in_end: CurrL->end());
8836 }
8837 forgetMemoizedResults(SCEVs: ToForget);
8838}
8839
8840void ScalarEvolution::forgetTopmostLoop(const Loop *L) {
8841 forgetLoop(L: L->getOutermostLoop());
8842}
8843
8844void ScalarEvolution::forgetValue(Value *V) {
8845 Instruction *I = dyn_cast<Instruction>(Val: V);
8846 if (!I) return;
8847
8848 // Drop information about expressions based on loop-header PHIs.
8849 SmallVector<Instruction *, 16> Worklist;
8850 SmallPtrSet<Instruction *, 8> Visited;
8851 SmallVector<SCEVUse, 8> ToForget;
8852 Worklist.push_back(Elt: I);
8853 Visited.insert(Ptr: I);
8854 visitAndClearUsers(Worklist, Visited, ToForget);
8855
8856 forgetMemoizedResults(SCEVs: ToForget);
8857}
8858
8859void ScalarEvolution::forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V) {
8860 if (!isSCEVable(Ty: V->getType()))
8861 return;
8862
8863 // If SCEV looked through a trivial LCSSA phi node, we might have SCEV's
8864 // directly using a SCEVUnknown/SCEVAddRec defined in the loop. After an
8865 // extra predecessor is added, this is no longer valid. Find all Unknowns and
8866 // AddRecs defined in the loop and invalidate any SCEV's making use of them.
8867 if (const SCEV *S = getExistingSCEV(V)) {
8868 struct InvalidationRootCollector {
8869 Loop *L;
8870 SmallVector<SCEVUse, 8> Roots;
8871
8872 InvalidationRootCollector(Loop *L) : L(L) {}
8873
8874 bool follow(const SCEV *S) {
8875 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
8876 if (auto *I = dyn_cast<Instruction>(Val: SU->getValue()))
8877 if (L->contains(Inst: I))
8878 Roots.push_back(Elt: S);
8879 } else if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S)) {
8880 if (L->contains(L: AddRec->getLoop()))
8881 Roots.push_back(Elt: S);
8882 }
8883 return true;
8884 }
8885 bool isDone() const { return false; }
8886 };
8887
8888 InvalidationRootCollector C(L);
8889 visitAll(Root: S, Visitor&: C);
8890 forgetMemoizedResults(SCEVs: C.Roots);
8891 }
8892
8893 // Also perform the normal invalidation.
8894 forgetValue(V);
8895}
8896
8897void ScalarEvolution::forgetLoopDispositions() { LoopDispositions.clear(); }
8898
8899void ScalarEvolution::forgetBlockAndLoopDispositions(Value *V) {
8900 // Unless a specific value is passed to invalidation, completely clear both
8901 // caches.
8902 if (!V) {
8903 BlockDispositions.clear();
8904 LoopDispositions.clear();
8905 return;
8906 }
8907
8908 if (!isSCEVable(Ty: V->getType()))
8909 return;
8910
8911 const SCEV *S = getExistingSCEV(V);
8912 if (!S)
8913 return;
8914
8915 // Invalidate the block and loop dispositions cached for S. Dispositions of
8916 // S's users may change if S's disposition changes (i.e. a user may change to
8917 // loop-invariant, if S changes to loop invariant), so also invalidate
8918 // dispositions of S's users recursively.
8919 SmallVector<SCEVUse, 8> Worklist = {S};
8920 SmallPtrSet<const SCEV *, 8> Seen = {S};
8921 while (!Worklist.empty()) {
8922 const SCEV *Curr = Worklist.pop_back_val();
8923 bool LoopDispoRemoved = LoopDispositions.erase(Val: Curr);
8924 bool BlockDispoRemoved = BlockDispositions.erase(Val: Curr);
8925 if (!LoopDispoRemoved && !BlockDispoRemoved)
8926 continue;
8927 auto Users = SCEVUsers.find(Val: Curr);
8928 if (Users != SCEVUsers.end())
8929 for (const auto *User : Users->second)
8930 if (Seen.insert(Ptr: User).second)
8931 Worklist.push_back(Elt: User);
8932 }
8933}
8934
8935/// Get the exact loop backedge taken count considering all loop exits. A
8936/// computable result can only be returned for loops with all exiting blocks
8937/// dominating the latch. howFarToZero assumes that the limit of each loop test
8938/// is never skipped. This is a valid assumption as long as the loop exits via
8939/// that test. For precise results, it is the caller's responsibility to specify
8940/// the relevant loop exiting block using getExact(ExitingBlock, SE).
8941const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
8942 const Loop *L, ScalarEvolution *SE,
8943 SmallVectorImpl<const SCEVPredicate *> *Preds) const {
8944 // If any exits were not computable, the loop is not computable.
8945 if (!isComplete() || ExitNotTaken.empty())
8946 return SE->getCouldNotCompute();
8947
8948 const BasicBlock *Latch = L->getLoopLatch();
8949 // All exiting blocks we have collected must dominate the only backedge.
8950 if (!Latch)
8951 return SE->getCouldNotCompute();
8952
8953 // All exiting blocks we have gathered dominate loop's latch, so exact trip
8954 // count is simply a minimum out of all these calculated exit counts.
8955 SmallVector<SCEVUse, 2> Ops;
8956 for (const auto &ENT : ExitNotTaken) {
8957 const SCEV *BECount = ENT.ExactNotTaken;
8958 assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!");
8959 assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&
8960 "We should only have known counts for exiting blocks that dominate "
8961 "latch!");
8962
8963 Ops.push_back(Elt: BECount);
8964
8965 if (Preds)
8966 append_range(C&: *Preds, R: ENT.Predicates);
8967
8968 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
8969 "Predicate should be always true!");
8970 }
8971
8972 // If an earlier exit exits on the first iteration (exit count zero), then
8973 // a later poison exit count should not propagate into the result. This are
8974 // exactly the semantics provided by umin_seq.
8975 return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true);
8976}
8977
8978const ScalarEvolution::ExitNotTakenInfo *
8979ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
8980 const BasicBlock *ExitingBlock,
8981 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8982 for (const auto &ENT : ExitNotTaken)
8983 if (ENT.ExitingBlock == ExitingBlock) {
8984 if (ENT.hasAlwaysTruePredicate())
8985 return &ENT;
8986 else if (Predicates) {
8987 append_range(C&: *Predicates, R: ENT.Predicates);
8988 return &ENT;
8989 }
8990 }
8991
8992 return nullptr;
8993}
8994
8995/// getConstantMax - Get the constant max backedge taken count for the loop.
8996const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
8997 ScalarEvolution *SE,
8998 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8999 if (!getConstantMax())
9000 return SE->getCouldNotCompute();
9001
9002 for (const auto &ENT : ExitNotTaken)
9003 if (!ENT.hasAlwaysTruePredicate()) {
9004 if (!Predicates)
9005 return SE->getCouldNotCompute();
9006 append_range(C&: *Predicates, R: ENT.Predicates);
9007 }
9008
9009 assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||
9010 isa<SCEVConstant>(getConstantMax())) &&
9011 "No point in having a non-constant max backedge taken count!");
9012 return getConstantMax();
9013}
9014
9015const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
9016 const Loop *L, ScalarEvolution *SE,
9017 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
9018 if (!SymbolicMax) {
9019 // Form an expression for the maximum exit count possible for this loop. We
9020 // merge the max and exact information to approximate a version of
9021 // getConstantMaxBackedgeTakenCount which isn't restricted to just
9022 // constants.
9023 SmallVector<SCEVUse, 4> ExitCounts;
9024
9025 for (const auto &ENT : ExitNotTaken) {
9026 const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
9027 if (!isa<SCEVCouldNotCompute>(Val: ExitCount)) {
9028 assert(SE->DT.dominates(ENT.ExitingBlock, L->getLoopLatch()) &&
9029 "We should only have known counts for exiting blocks that "
9030 "dominate latch!");
9031 ExitCounts.push_back(Elt: ExitCount);
9032 if (Predicates)
9033 append_range(C&: *Predicates, R: ENT.Predicates);
9034
9035 assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
9036 "Predicate should be always true!");
9037 }
9038 }
9039 if (ExitCounts.empty())
9040 SymbolicMax = SE->getCouldNotCompute();
9041 else
9042 SymbolicMax =
9043 SE->getUMinFromMismatchedTypes(Ops&: ExitCounts, /*Sequential*/ true);
9044 }
9045 return SymbolicMax;
9046}
9047
9048bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
9049 ScalarEvolution *SE) const {
9050 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
9051 return !ENT.hasAlwaysTruePredicate();
9052 };
9053 return MaxOrZero && !any_of(Range: ExitNotTaken, P: PredicateNotAlwaysTrue);
9054}
9055
9056ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E)
9057 : ExitLimit(E, E, E, false) {}
9058
9059ScalarEvolution::ExitLimit::ExitLimit(
9060 const SCEV *E, const SCEV *ConstantMaxNotTaken,
9061 const SCEV *SymbolicMaxNotTaken, bool MaxOrZero,
9062 ArrayRef<ArrayRef<const SCEVPredicate *>> PredLists)
9063 : ExactNotTaken(E), ConstantMaxNotTaken(ConstantMaxNotTaken),
9064 SymbolicMaxNotTaken(SymbolicMaxNotTaken), MaxOrZero(MaxOrZero) {
9065 // If we prove the max count is zero, so is the symbolic bound. This happens
9066 // in practice due to differences in a) how context sensitive we've chosen
9067 // to be and b) how we reason about bounds implied by UB.
9068 if (ConstantMaxNotTaken->isZero()) {
9069 this->ExactNotTaken = E = ConstantMaxNotTaken;
9070 this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
9071 }
9072
9073 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
9074 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
9075 "Exact is not allowed to be less precise than Constant Max");
9076 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
9077 !isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken)) &&
9078 "Exact is not allowed to be less precise than Symbolic Max");
9079 assert((isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken) ||
9080 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
9081 "Symbolic Max is not allowed to be less precise than Constant Max");
9082 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
9083 isa<SCEVConstant>(ConstantMaxNotTaken)) &&
9084 "No point in having a non-constant max backedge taken count!");
9085 SmallPtrSet<const SCEVPredicate *, 4> SeenPreds;
9086 for (const auto PredList : PredLists)
9087 for (const auto *P : PredList) {
9088 if (SeenPreds.contains(Ptr: P))
9089 continue;
9090 assert(!isa<SCEVUnionPredicate>(P) && "Only add leaf predicates here!");
9091 SeenPreds.insert(Ptr: P);
9092 Predicates.push_back(Elt: P);
9093 }
9094 assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&
9095 "Backedge count should be int");
9096 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
9097 !ConstantMaxNotTaken->getType()->isPointerTy()) &&
9098 "Max backedge count should be int");
9099}
9100
9101ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E,
9102 const SCEV *ConstantMaxNotTaken,
9103 const SCEV *SymbolicMaxNotTaken,
9104 bool MaxOrZero,
9105 ArrayRef<const SCEVPredicate *> PredList)
9106 : ExitLimit(E, ConstantMaxNotTaken, SymbolicMaxNotTaken, MaxOrZero,
9107 ArrayRef({PredList})) {}
9108
9109/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
9110/// computable exit into a persistent ExitNotTakenInfo array.
9111ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
9112 ArrayRef<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo> ExitCounts,
9113 bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
9114 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
9115 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
9116
9117 ExitNotTaken.reserve(N: ExitCounts.size());
9118 std::transform(first: ExitCounts.begin(), last: ExitCounts.end(),
9119 result: std::back_inserter(x&: ExitNotTaken),
9120 unary_op: [&](const EdgeExitInfo &EEI) {
9121 BasicBlock *ExitBB = EEI.first;
9122 const ExitLimit &EL = EEI.second;
9123 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
9124 EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
9125 EL.Predicates);
9126 });
9127 assert((isa<SCEVCouldNotCompute>(ConstantMax) ||
9128 isa<SCEVConstant>(ConstantMax)) &&
9129 "No point in having a non-constant max backedge taken count!");
9130}
9131
9132/// Compute the number of times the backedge of the specified loop will execute.
9133ScalarEvolution::BackedgeTakenInfo
9134ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
9135 bool AllowPredicates) {
9136 SmallVector<BasicBlock *, 8> ExitingBlocks;
9137 L->getExitingBlocks(ExitingBlocks);
9138
9139 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
9140
9141 SmallVector<EdgeExitInfo, 4> ExitCounts;
9142 bool CouldComputeBECount = true;
9143 BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
9144 const SCEV *MustExitMaxBECount = nullptr;
9145 const SCEV *MayExitMaxBECount = nullptr;
9146 bool MustExitMaxOrZero = false;
9147 bool IsOnlyExit = ExitingBlocks.size() == 1;
9148
9149 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
9150 // and compute maxBECount.
9151 // Do a union of all the predicates here.
9152 for (BasicBlock *ExitBB : ExitingBlocks) {
9153 // We canonicalize untaken exits to br (constant), ignore them so that
9154 // proving an exit untaken doesn't negatively impact our ability to reason
9155 // about the loop as whole.
9156 if (auto *BI = dyn_cast<CondBrInst>(Val: ExitBB->getTerminator()))
9157 if (auto *CI = dyn_cast<ConstantInt>(Val: BI->getCondition())) {
9158 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9159 if (ExitIfTrue == CI->isZero())
9160 continue;
9161 }
9162
9163 ExitLimit EL = computeExitLimit(L, ExitingBlock: ExitBB, IsOnlyExit, AllowPredicates);
9164
9165 assert((AllowPredicates || EL.Predicates.empty()) &&
9166 "Predicated exit limit when predicates are not allowed!");
9167
9168 // 1. For each exit that can be computed, add an entry to ExitCounts.
9169 // CouldComputeBECount is true only if all exits can be computed.
9170 if (EL.ExactNotTaken != getCouldNotCompute())
9171 ++NumExitCountsComputed;
9172 else
9173 // We couldn't compute an exact value for this exit, so
9174 // we won't be able to compute an exact value for the loop.
9175 CouldComputeBECount = false;
9176 // Remember exit count if either exact or symbolic is known. Because
9177 // Exact always implies symbolic, only check symbolic.
9178 if (EL.SymbolicMaxNotTaken != getCouldNotCompute())
9179 ExitCounts.emplace_back(Args&: ExitBB, Args&: EL);
9180 else {
9181 assert(EL.ExactNotTaken == getCouldNotCompute() &&
9182 "Exact is known but symbolic isn't?");
9183 ++NumExitCountsNotComputed;
9184 }
9185
9186 // 2. Derive the loop's MaxBECount from each exit's max number of
9187 // non-exiting iterations. Partition the loop exits into two kinds:
9188 // LoopMustExits and LoopMayExits.
9189 //
9190 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
9191 // is a LoopMayExit. If any computable LoopMustExit is found, then
9192 // MaxBECount is the minimum EL.ConstantMaxNotTaken of computable
9193 // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
9194 // EL.ConstantMaxNotTaken, where CouldNotCompute is considered greater than
9195 // any
9196 // computable EL.ConstantMaxNotTaken.
9197 if (EL.ConstantMaxNotTaken != getCouldNotCompute() && Latch &&
9198 DT.dominates(A: ExitBB, B: Latch)) {
9199 if (!MustExitMaxBECount) {
9200 MustExitMaxBECount = EL.ConstantMaxNotTaken;
9201 MustExitMaxOrZero = EL.MaxOrZero;
9202 } else {
9203 MustExitMaxBECount = getUMinFromMismatchedTypes(LHS: MustExitMaxBECount,
9204 RHS: EL.ConstantMaxNotTaken);
9205 }
9206 } else if (MayExitMaxBECount != getCouldNotCompute()) {
9207 if (!MayExitMaxBECount || EL.ConstantMaxNotTaken == getCouldNotCompute())
9208 MayExitMaxBECount = EL.ConstantMaxNotTaken;
9209 else {
9210 MayExitMaxBECount = getUMaxFromMismatchedTypes(LHS: MayExitMaxBECount,
9211 RHS: EL.ConstantMaxNotTaken);
9212 }
9213 }
9214 }
9215 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
9216 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
9217 // The loop backedge will be taken the maximum or zero times if there's
9218 // a single exit that must be taken the maximum or zero times.
9219 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
9220
9221 // Remember which SCEVs are used in exit limits for invalidation purposes.
9222 // We only care about non-constant SCEVs here, so we can ignore
9223 // EL.ConstantMaxNotTaken
9224 // and MaxBECount, which must be SCEVConstant.
9225 for (const auto &Pair : ExitCounts) {
9226 if (!isa<SCEVConstant>(Val: Pair.second.ExactNotTaken))
9227 BECountUsers[Pair.second.ExactNotTaken].insert(Ptr: {L, AllowPredicates});
9228 if (!isa<SCEVConstant>(Val: Pair.second.SymbolicMaxNotTaken))
9229 BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
9230 Ptr: {L, AllowPredicates});
9231 }
9232 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
9233 MaxBECount, MaxOrZero);
9234}
9235
9236ScalarEvolution::ExitLimit
9237ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
9238 bool IsOnlyExit, bool AllowPredicates) {
9239 assert(L->contains(ExitingBlock) && "Exit count for non-loop block?");
9240 // If our exiting block does not dominate the latch, then its connection with
9241 // loop's exit limit may be far from trivial.
9242 const BasicBlock *Latch = L->getLoopLatch();
9243 if (!Latch || !DT.dominates(A: ExitingBlock, B: Latch))
9244 return getCouldNotCompute();
9245
9246 Instruction *Term = ExitingBlock->getTerminator();
9247 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Term)) {
9248 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9249 assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&
9250 "It should have one successor in loop and one exit block!");
9251 // Proceed to the next level to examine the exit condition expression.
9252 return computeExitLimitFromCond(L, ExitCond: BI->getCondition(), ExitIfTrue,
9253 /*ControlsOnlyExit=*/IsOnlyExit,
9254 AllowPredicates);
9255 }
9256
9257 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: Term)) {
9258 // For switch, make sure that there is a single exit from the loop.
9259 BasicBlock *Exit = nullptr;
9260 for (auto *SBB : successors(BB: ExitingBlock))
9261 if (!L->contains(BB: SBB)) {
9262 if (Exit) // Multiple exit successors.
9263 return getCouldNotCompute();
9264 Exit = SBB;
9265 }
9266 assert(Exit && "Exiting block must have at least one exit");
9267 return computeExitLimitFromSingleExitSwitch(
9268 L, Switch: SI, ExitingBB: Exit, /*ControlsOnlyExit=*/IsSubExpr: IsOnlyExit);
9269 }
9270
9271 return getCouldNotCompute();
9272}
9273
9274ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCond(
9275 const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9276 bool AllowPredicates) {
9277 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
9278 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
9279 ControlsOnlyExit, AllowPredicates);
9280}
9281
9282std::optional<ScalarEvolution::ExitLimit>
9283ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
9284 bool ExitIfTrue, bool ControlsOnlyExit,
9285 bool AllowPredicates) {
9286 (void)this->L;
9287 (void)this->ExitIfTrue;
9288 (void)this->AllowPredicates;
9289
9290 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9291 this->AllowPredicates == AllowPredicates &&
9292 "Variance in assumed invariant key components!");
9293 auto Itr = TripCountMap.find(Val: {ExitCond, ControlsOnlyExit});
9294 if (Itr == TripCountMap.end())
9295 return std::nullopt;
9296 return Itr->second;
9297}
9298
9299void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond,
9300 bool ExitIfTrue,
9301 bool ControlsOnlyExit,
9302 bool AllowPredicates,
9303 const ExitLimit &EL) {
9304 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9305 this->AllowPredicates == AllowPredicates &&
9306 "Variance in assumed invariant key components!");
9307
9308 auto InsertResult = TripCountMap.insert(KV: {{ExitCond, ControlsOnlyExit}, EL});
9309 assert(InsertResult.second && "Expected successful insertion!");
9310 (void)InsertResult;
9311 (void)ExitIfTrue;
9312}
9313
9314ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
9315 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9316 bool ControlsOnlyExit, bool AllowPredicates) {
9317
9318 if (auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit,
9319 AllowPredicates))
9320 return *MaybeEL;
9321
9322 ExitLimit EL = computeExitLimitFromCondImpl(
9323 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates);
9324 Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL);
9325 return EL;
9326}
9327
9328ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
9329 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9330 bool ControlsOnlyExit, bool AllowPredicates) {
9331 // Handle BinOp conditions (And, Or).
9332 if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
9333 Cache, L, ExitCond, ExitIfTrue, AllowPredicates))
9334 return *LimitFromBinOp;
9335
9336 // With an icmp, it may be feasible to compute an exact backedge-taken count.
9337 // Proceed to the next level to examine the icmp.
9338 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(Val: ExitCond)) {
9339 ExitLimit EL =
9340 computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue, IsSubExpr: ControlsOnlyExit);
9341 if (EL.hasFullInfo() || !AllowPredicates)
9342 return EL;
9343
9344 // Try again, but use SCEV predicates this time.
9345 return computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue,
9346 IsSubExpr: ControlsOnlyExit,
9347 /*AllowPredicates=*/true);
9348 }
9349
9350 // Check for a constant condition. These are normally stripped out by
9351 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
9352 // preserve the CFG and is temporarily leaving constant conditions
9353 // in place.
9354 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: ExitCond)) {
9355 if (ExitIfTrue == !CI->getZExtValue())
9356 // The backedge is always taken.
9357 return getCouldNotCompute();
9358 // The backedge is never taken.
9359 return getZero(Ty: CI->getType());
9360 }
9361
9362 // If we're exiting based on the overflow flag of an x.with.overflow intrinsic
9363 // with a constant step, we can form an equivalent icmp predicate and figure
9364 // out how many iterations will be taken before we exit.
9365 const WithOverflowInst *WO;
9366 const APInt *C;
9367 if (match(V: ExitCond, P: m_ExtractValue<1>(V: m_WithOverflowInst(I&: WO))) &&
9368 match(V: WO->getRHS(), P: m_APInt(Res&: C))) {
9369 ConstantRange NWR =
9370 ConstantRange::makeExactNoWrapRegion(BinOp: WO->getBinaryOp(), Other: *C,
9371 NoWrapKind: WO->getNoWrapKind());
9372 CmpInst::Predicate Pred;
9373 APInt NewRHSC, Offset;
9374 NWR.getEquivalentICmp(Pred, RHS&: NewRHSC, Offset);
9375 if (!ExitIfTrue)
9376 Pred = ICmpInst::getInversePredicate(pred: Pred);
9377 auto *LHS = getSCEV(V: WO->getLHS());
9378 if (Offset != 0)
9379 LHS = getAddExpr(LHS, RHS: getConstant(Val: Offset));
9380 auto EL = computeExitLimitFromICmp(L, Pred, LHS, RHS: getConstant(Val: NewRHSC),
9381 IsSubExpr: ControlsOnlyExit, AllowPredicates);
9382 if (EL.hasAnyInfo())
9383 return EL;
9384 }
9385
9386 // If it's not an integer or pointer comparison then compute it the hard way.
9387 return computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9388}
9389
9390std::optional<ScalarEvolution::ExitLimit>
9391ScalarEvolution::computeExitLimitFromCondFromBinOp(ExitLimitCacheTy &Cache,
9392 const Loop *L,
9393 Value *ExitCond,
9394 bool ExitIfTrue,
9395 bool AllowPredicates) {
9396 // Check if the controlling expression for this loop is an And or Or.
9397 Value *Op0, *Op1;
9398 bool IsAnd;
9399 if (match(V: ExitCond, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9400 IsAnd = true;
9401 else if (match(V: ExitCond, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9402 IsAnd = false;
9403 else
9404 return std::nullopt;
9405
9406 // A sub-condition of a non-trivial binop never solely controls the exit,
9407 // whether we exit always depends on both conditions.
9408 ExitLimit EL0 = computeExitLimitFromCondCached(
9409 Cache, L, ExitCond: Op0, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9410 ExitLimit EL1 = computeExitLimitFromCondCached(
9411 Cache, L, ExitCond: Op1, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9412
9413 // EitherMayExit is true in these two cases:
9414 // br (and Op0 Op1), loop, exit
9415 // br (or Op0 Op1), exit, loop
9416 bool EitherMayExit = IsAnd ^ ExitIfTrue;
9417
9418 const SCEV *BECount = getCouldNotCompute();
9419 const SCEV *ConstantMaxBECount = getCouldNotCompute();
9420 const SCEV *SymbolicMaxBECount = getCouldNotCompute();
9421 if (EitherMayExit) {
9422 bool UseSequentialUMin = !isa<BinaryOperator>(Val: ExitCond);
9423 // Both conditions must be same for the loop to continue executing.
9424 // Choose the less conservative count.
9425 if (EL0.ExactNotTaken != getCouldNotCompute() &&
9426 EL1.ExactNotTaken != getCouldNotCompute()) {
9427 BECount = getUMinFromMismatchedTypes(LHS: EL0.ExactNotTaken, RHS: EL1.ExactNotTaken,
9428 Sequential: UseSequentialUMin);
9429 }
9430 if (EL0.ConstantMaxNotTaken == getCouldNotCompute())
9431 ConstantMaxBECount = EL1.ConstantMaxNotTaken;
9432 else if (EL1.ConstantMaxNotTaken == getCouldNotCompute())
9433 ConstantMaxBECount = EL0.ConstantMaxNotTaken;
9434 else
9435 ConstantMaxBECount = getUMinFromMismatchedTypes(LHS: EL0.ConstantMaxNotTaken,
9436 RHS: EL1.ConstantMaxNotTaken);
9437 if (EL0.SymbolicMaxNotTaken == getCouldNotCompute())
9438 SymbolicMaxBECount = EL1.SymbolicMaxNotTaken;
9439 else if (EL1.SymbolicMaxNotTaken == getCouldNotCompute())
9440 SymbolicMaxBECount = EL0.SymbolicMaxNotTaken;
9441 else
9442 SymbolicMaxBECount = getUMinFromMismatchedTypes(
9443 LHS: EL0.SymbolicMaxNotTaken, RHS: EL1.SymbolicMaxNotTaken, Sequential: UseSequentialUMin);
9444 } else {
9445 // Both conditions must be same at the same time for the loop to exit.
9446 // For now, be conservative.
9447 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
9448 BECount = EL0.ExactNotTaken;
9449 }
9450
9451 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able
9452 // to be more aggressive when computing BECount than when computing
9453 // ConstantMaxBECount. In these cases it is possible for EL0.ExactNotTaken
9454 // and
9455 // EL1.ExactNotTaken to match, but for EL0.ConstantMaxNotTaken and
9456 // EL1.ConstantMaxNotTaken to not.
9457 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
9458 !isa<SCEVCouldNotCompute>(Val: BECount))
9459 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
9460 if (isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount))
9461 SymbolicMaxBECount =
9462 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
9463 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
9464 {ArrayRef(EL0.Predicates), ArrayRef(EL1.Predicates)});
9465}
9466
9467ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9468 const Loop *L, ICmpInst *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9469 bool AllowPredicates) {
9470 // If the condition was exit on true, convert the condition to exit on false
9471 CmpPredicate Pred;
9472 if (!ExitIfTrue)
9473 Pred = ExitCond->getCmpPredicate();
9474 else
9475 Pred = ExitCond->getInverseCmpPredicate();
9476 const ICmpInst::Predicate OriginalPred = Pred;
9477
9478 const SCEV *LHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 0));
9479 const SCEV *RHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 1));
9480
9481 ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, IsSubExpr: ControlsOnlyExit,
9482 AllowPredicates);
9483 if (EL.hasAnyInfo())
9484 return EL;
9485
9486 auto *ExhaustiveCount =
9487 computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9488
9489 if (!isa<SCEVCouldNotCompute>(Val: ExhaustiveCount))
9490 return ExhaustiveCount;
9491
9492 return computeShiftCompareExitLimit(LHS: ExitCond->getOperand(i_nocapture: 0),
9493 RHS: ExitCond->getOperand(i_nocapture: 1), L, Pred: OriginalPred);
9494}
9495ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9496 const Loop *L, CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS,
9497 bool ControlsOnlyExit, bool AllowPredicates) {
9498
9499 // Try to evaluate any dependencies out of the loop.
9500 LHS = getSCEVAtScope(S: LHS, L);
9501 RHS = getSCEVAtScope(S: RHS, L);
9502
9503 // At this point, we would like to compute how many iterations of the
9504 // loop the predicate will return true for these inputs.
9505 if (isLoopInvariant(S: LHS, L) && !isLoopInvariant(S: RHS, L)) {
9506 // If there is a loop-invariant, force it into the RHS.
9507 std::swap(a&: LHS, b&: RHS);
9508 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
9509 }
9510
9511 bool ControllingFiniteLoop = ControlsOnlyExit && loopHasNoAbnormalExits(L) &&
9512 loopIsFiniteByAssumption(L);
9513 // Simplify the operands before analyzing them.
9514 (void)SimplifyICmpOperands(Pred, LHS, RHS, /*Depth=*/0);
9515
9516 // If we have a comparison of a chrec against a constant, try to use value
9517 // ranges to answer this query.
9518 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS))
9519 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
9520 if (AddRec->getLoop() == L) {
9521 // Form the constant range.
9522 ConstantRange CompRange =
9523 ConstantRange::makeExactICmpRegion(Pred, Other: RHSC->getAPInt());
9524
9525 const SCEV *Ret = AddRec->getNumIterationsInRange(Range: CompRange, SE&: *this);
9526 if (!isa<SCEVCouldNotCompute>(Val: Ret)) return Ret;
9527 }
9528
9529 // If this loop must exit based on this condition (or execute undefined
9530 // behaviour), see if we can improve wrap flags. This is essentially
9531 // a must execute style proof.
9532 if (ControllingFiniteLoop && isLoopInvariant(S: RHS, L)) {
9533 // If we can prove the test sequence produced must repeat the same values
9534 // on self-wrap of the IV, then we can infer that IV doesn't self wrap
9535 // because if it did, we'd have an infinite (undefined) loop.
9536 // TODO: We can peel off any functions which are invertible *in L*. Loop
9537 // invariant terms are effectively constants for our purposes here.
9538 SCEVUse InnerLHS = LHS;
9539 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val&: LHS))
9540 InnerLHS = ZExt->getOperand();
9541 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: InnerLHS);
9542 AR && !AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() &&
9543 isKnownToBeAPowerOfTwo(S: AR->getStepRecurrence(SE&: *this), /*OrZero=*/true,
9544 /*OrNegative=*/true)) {
9545 auto Flags = AR->getNoWrapFlags();
9546 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
9547 SmallVector<SCEVUse> Operands{AR->operands()};
9548 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9549 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9550 }
9551
9552 // For a slt/ult condition with a positive step, can we prove nsw/nuw?
9553 // From no-self-wrap, this follows trivially from the fact that every
9554 // (un)signed-wrapped, but not self-wrapped value must be LT than the
9555 // last value before (un)signed wrap. Since we know that last value
9556 // didn't exit, nor will any smaller one.
9557 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT) {
9558 auto WrapType = Pred == ICmpInst::ICMP_SLT ? SCEV::FlagNSW : SCEV::FlagNUW;
9559 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS);
9560 AR && AR->getLoop() == L && AR->isAffine() &&
9561 !AR->getNoWrapFlags(Mask: WrapType) && AR->hasNoSelfWrap() &&
9562 isKnownPositive(S: AR->getStepRecurrence(SE&: *this))) {
9563 auto Flags = AR->getNoWrapFlags();
9564 Flags = setFlags(Flags, OnFlags: WrapType);
9565 SmallVector<SCEVUse> Operands{AR->operands()};
9566 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9567 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9568 }
9569 }
9570 }
9571
9572 switch (Pred) {
9573 case ICmpInst::ICMP_NE: { // while (X != Y)
9574 // Convert to: while (X-Y != 0)
9575 if (LHS->getType()->isPointerTy()) {
9576 LHS = getLosslessPtrToIntExpr(Op: LHS);
9577 if (isa<SCEVCouldNotCompute>(Val: LHS))
9578 return LHS;
9579 }
9580 if (RHS->getType()->isPointerTy()) {
9581 RHS = getLosslessPtrToIntExpr(Op: RHS);
9582 if (isa<SCEVCouldNotCompute>(Val: RHS))
9583 return RHS;
9584 }
9585 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit,
9586 AllowPredicates);
9587 if (EL.hasAnyInfo())
9588 return EL;
9589 break;
9590 }
9591 case ICmpInst::ICMP_EQ: { // while (X == Y)
9592 // Convert to: while (X-Y == 0)
9593 if (LHS->getType()->isPointerTy()) {
9594 LHS = getLosslessPtrToIntExpr(Op: LHS);
9595 if (isa<SCEVCouldNotCompute>(Val: LHS))
9596 return LHS;
9597 }
9598 if (RHS->getType()->isPointerTy()) {
9599 RHS = getLosslessPtrToIntExpr(Op: RHS);
9600 if (isa<SCEVCouldNotCompute>(Val: RHS))
9601 return RHS;
9602 }
9603 ExitLimit EL = howFarToNonZero(V: getMinusSCEV(LHS, RHS), L);
9604 if (EL.hasAnyInfo()) return EL;
9605 break;
9606 }
9607 case ICmpInst::ICMP_SLE:
9608 case ICmpInst::ICMP_ULE:
9609 // Since the loop is finite, an invariant RHS cannot include the boundary
9610 // value, otherwise it would loop forever.
9611 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9612 !isLoopInvariant(S: RHS, L)) {
9613 // Otherwise, perform the addition in a wider type, to avoid overflow.
9614 // If the LHS is an addrec with the appropriate nowrap flag, the
9615 // extension will be sunk into it and the exit count can be analyzed.
9616 auto *OldType = dyn_cast<IntegerType>(Val: LHS->getType());
9617 if (!OldType)
9618 break;
9619 // Prefer doubling the bitwidth over adding a single bit to make it more
9620 // likely that we use a legal type.
9621 auto *NewType =
9622 Type::getIntNTy(C&: OldType->getContext(), N: OldType->getBitWidth() * 2);
9623 if (ICmpInst::isSigned(Pred)) {
9624 LHS = getSignExtendExpr(Op: LHS, Ty: NewType);
9625 RHS = getSignExtendExpr(Op: RHS, Ty: NewType);
9626 } else {
9627 LHS = getZeroExtendExpr(Op: LHS, Ty: NewType);
9628 RHS = getZeroExtendExpr(Op: RHS, Ty: NewType);
9629 }
9630 }
9631 RHS = getAddExpr(LHS: getOne(Ty: RHS->getType()), RHS);
9632 [[fallthrough]];
9633 case ICmpInst::ICMP_SLT:
9634 case ICmpInst::ICMP_ULT: { // while (X < Y)
9635 bool IsSigned = ICmpInst::isSigned(Pred);
9636 ExitLimit EL = howManyLessThans(LHS, RHS, L, isSigned: IsSigned, ControlsOnlyExit,
9637 AllowPredicates);
9638 if (EL.hasAnyInfo())
9639 return EL;
9640 break;
9641 }
9642 case ICmpInst::ICMP_SGE:
9643 case ICmpInst::ICMP_UGE:
9644 // Since the loop is finite, an invariant RHS cannot include the boundary
9645 // value, otherwise it would loop forever.
9646 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9647 !isLoopInvariant(S: RHS, L))
9648 break;
9649 RHS = getAddExpr(LHS: getMinusOne(Ty: RHS->getType()), RHS);
9650 [[fallthrough]];
9651 case ICmpInst::ICMP_SGT:
9652 case ICmpInst::ICMP_UGT: { // while (X > Y)
9653 bool IsSigned = ICmpInst::isSigned(Pred);
9654 ExitLimit EL = howManyGreaterThans(LHS, RHS, L, isSigned: IsSigned, IsSubExpr: ControlsOnlyExit,
9655 AllowPredicates);
9656 if (EL.hasAnyInfo())
9657 return EL;
9658 break;
9659 }
9660 default:
9661 break;
9662 }
9663
9664 return getCouldNotCompute();
9665}
9666
9667ScalarEvolution::ExitLimit
9668ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L,
9669 SwitchInst *Switch,
9670 BasicBlock *ExitingBlock,
9671 bool ControlsOnlyExit) {
9672 assert(!L->contains(ExitingBlock) && "Not an exiting block!");
9673
9674 // Give up if the exit is the default dest of a switch.
9675 if (Switch->getDefaultDest() == ExitingBlock)
9676 return getCouldNotCompute();
9677
9678 assert(L->contains(Switch->getDefaultDest()) &&
9679 "Default case must not exit the loop!");
9680 const SCEV *LHS = getSCEVAtScope(V: Switch->getCondition(), L);
9681 const SCEV *RHS = getConstant(V: Switch->findCaseDest(BB: ExitingBlock));
9682
9683 // while (X != Y) --> while (X-Y != 0)
9684 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit);
9685 if (EL.hasAnyInfo())
9686 return EL;
9687
9688 return getCouldNotCompute();
9689}
9690
9691static ConstantInt *
9692EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
9693 ScalarEvolution &SE) {
9694 const SCEV *InVal = SE.getConstant(V: C);
9695 const SCEV *Val = AddRec->evaluateAtIteration(It: InVal, SE);
9696 assert(isa<SCEVConstant>(Val) &&
9697 "Evaluation of SCEV at constant didn't fold correctly?");
9698 return cast<SCEVConstant>(Val)->getValue();
9699}
9700
9701ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
9702 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) {
9703 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: RHSV);
9704 if (!RHS)
9705 return getCouldNotCompute();
9706
9707 const BasicBlock *Latch = L->getLoopLatch();
9708 if (!Latch)
9709 return getCouldNotCompute();
9710
9711 const BasicBlock *Predecessor = L->getLoopPredecessor();
9712 if (!Predecessor)
9713 return getCouldNotCompute();
9714
9715 // Return true if V is of the form "LHS `shift_op` <positive constant>".
9716 // Return LHS in OutLHS, shift_op in OutOpCode, and the shift amount in
9717 // OutShiftAmt.
9718 auto MatchPositiveShift = [](Value *V, Value *&OutLHS,
9719 Instruction::BinaryOps &OutOpCode,
9720 unsigned &OutShiftAmt) {
9721 using namespace PatternMatch;
9722
9723 ConstantInt *ShiftAmt;
9724 if (match(V, P: m_LShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9725 OutOpCode = Instruction::LShr;
9726 else if (match(V, P: m_AShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9727 OutOpCode = Instruction::AShr;
9728 else if (match(V, P: m_Shl(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9729 OutOpCode = Instruction::Shl;
9730 else
9731 return false;
9732
9733 uint64_t Amt = ShiftAmt->getValue().getLimitedValue();
9734 if (Amt == 0 || Amt >= OutLHS->getType()->getScalarSizeInBits())
9735 return false;
9736 OutShiftAmt = Amt;
9737 return true;
9738 };
9739
9740 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in
9741 //
9742 // loop:
9743 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ]
9744 // %iv.shifted = lshr i32 %iv, <positive constant>
9745 //
9746 // Return true on a successful match. Return the corresponding PHI node (%iv
9747 // above) in PNOut, the opcode of the shift operation in OpCodeOut, and the
9748 // shift amount in ShiftAmtOut.
9749 auto MatchShiftRecurrence = [&](Value *V, PHINode *&PNOut,
9750 Instruction::BinaryOps &OpCodeOut,
9751 unsigned &ShiftAmtOut) {
9752 std::optional<Instruction::BinaryOps> PostShiftOpCode;
9753
9754 {
9755 Instruction::BinaryOps OpC;
9756 Value *V;
9757 unsigned Amt;
9758
9759 // If we encounter a shift instruction, "peel off" the shift operation,
9760 // and remember that we did so. Later when we inspect %iv's backedge
9761 // value, we will make sure that the backedge value uses the same
9762 // operation.
9763 //
9764 // Note: the peeled shift operation does not have to be the same
9765 // instruction as the one feeding into the PHI's backedge value. We only
9766 // really care about it being the same *kind* of shift instruction --
9767 // that's all that is required for our later inferences to hold.
9768 if (MatchPositiveShift(LHS, V, OpC, Amt)) {
9769 PostShiftOpCode = OpC;
9770 LHS = V;
9771 }
9772 }
9773
9774 PNOut = dyn_cast<PHINode>(Val: LHS);
9775 if (!PNOut || PNOut->getParent() != L->getHeader())
9776 return false;
9777
9778 Value *BEValue = PNOut->getIncomingValueForBlock(BB: Latch);
9779 Value *OpLHS;
9780
9781 return
9782 // The backedge value for the PHI node must be a shift by a positive
9783 // amount
9784 MatchPositiveShift(BEValue, OpLHS, OpCodeOut, ShiftAmtOut) &&
9785
9786 // of the PHI node itself
9787 OpLHS == PNOut &&
9788
9789 // and the kind of shift should be match the kind of shift we peeled
9790 // off, if any.
9791 (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut);
9792 };
9793
9794 PHINode *PN;
9795 Instruction::BinaryOps OpCode;
9796 unsigned ShiftAmt;
9797 if (!MatchShiftRecurrence(LHS, PN, OpCode, ShiftAmt))
9798 return getCouldNotCompute();
9799
9800 const DataLayout &DL = getDataLayout();
9801
9802 // The key rationale for this optimization is that for some kinds of shift
9803 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1
9804 // within a finite number of iterations. If the condition guarding the
9805 // backedge (in the sense that the backedge is taken if the condition is true)
9806 // is false for the value the shift recurrence stabilizes to, then we know
9807 // that the backedge is taken only a finite number of times.
9808
9809 ConstantInt *StableValue = nullptr;
9810 switch (OpCode) {
9811 default:
9812 llvm_unreachable("Impossible case!");
9813
9814 case Instruction::AShr: {
9815 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
9816 // bitwidth(K) iterations.
9817 Value *FirstValue = PN->getIncomingValueForBlock(BB: Predecessor);
9818 KnownBits Known = computeKnownBits(V: FirstValue, DL, AC: &AC,
9819 CxtI: Predecessor->getTerminator(), DT: &DT);
9820 auto *Ty = cast<IntegerType>(Val: RHS->getType());
9821 if (Known.isNonNegative())
9822 StableValue = ConstantInt::get(Ty, V: 0);
9823 else if (Known.isNegative())
9824 StableValue = ConstantInt::get(Ty, V: -1, IsSigned: true);
9825 else
9826 return getCouldNotCompute();
9827
9828 break;
9829 }
9830 case Instruction::LShr:
9831 case Instruction::Shl:
9832 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>}
9833 // stabilize to 0 in at most bitwidth(K) iterations.
9834 StableValue = ConstantInt::get(Ty: cast<IntegerType>(Val: RHS->getType()), V: 0);
9835 break;
9836 }
9837
9838 auto *Result =
9839 ConstantFoldCompareInstOperands(Predicate: Pred, LHS: StableValue, RHS, DL, TLI: &TLI);
9840 assert(Result->getType()->isIntegerTy(1) &&
9841 "Otherwise cannot be an operand to a branch instruction");
9842
9843 if (Result->isNullValue()) {
9844 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
9845 unsigned MaxBTC = BitWidth;
9846
9847 // For right-shift recurrences (lshr/ashr with non-negative start), we can
9848 // compute a tighter max backedge-taken count from the range of the start
9849 // value. After k shifts of ShiftAmt, value = start >> (k * ShiftAmt).
9850 // The value reaches 0 (the stable value) when k * ShiftAmt >=
9851 // activeBits(start), so max BTC = ceil(activeBits(maxStart) / ShiftAmt).
9852 if (OpCode == Instruction::LShr || OpCode == Instruction::AShr) {
9853 Value *StartValue = PN->getIncomingValueForBlock(BB: Predecessor);
9854 const SCEV *StartSCEV = getSCEV(V: StartValue);
9855 APInt MaxStart = getUnsignedRangeMax(S: StartSCEV);
9856 if (MaxStart.isStrictlyPositive()) {
9857 unsigned ActiveBits = MaxStart.getActiveBits();
9858 unsigned RangeBTC = divideCeil(Numerator: ActiveBits, Denominator: ShiftAmt);
9859 MaxBTC = std::min(a: MaxBTC, b: RangeBTC);
9860 }
9861 }
9862
9863 const SCEV *UpperBound =
9864 getConstant(Ty: getEffectiveSCEVType(Ty: RHS->getType()), V: MaxBTC);
9865 return ExitLimit(getCouldNotCompute(), UpperBound, UpperBound, false);
9866 }
9867
9868 return getCouldNotCompute();
9869}
9870
9871/// Return true if we can constant fold an instruction of the specified type,
9872/// assuming that all operands were constants.
9873static bool CanConstantFold(const Instruction *I) {
9874 if (isa<BinaryOperator>(Val: I) || isa<CmpInst>(Val: I) ||
9875 isa<SelectInst>(Val: I) || isa<CastInst>(Val: I) || isa<GetElementPtrInst>(Val: I) ||
9876 isa<LoadInst>(Val: I) || isa<ExtractValueInst>(Val: I))
9877 return true;
9878
9879 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
9880 if (const Function *F = CI->getCalledFunction())
9881 return canConstantFoldCallTo(Call: CI, F);
9882 return false;
9883}
9884
9885/// Determine whether this instruction can constant evolve within this loop
9886/// assuming its operands can all constant evolve.
9887static bool canConstantEvolve(Instruction *I, const Loop *L) {
9888 // An instruction outside of the loop can't be derived from a loop PHI.
9889 if (!L->contains(Inst: I)) return false;
9890
9891 if (isa<PHINode>(Val: I)) {
9892 // We don't currently keep track of the control flow needed to evaluate
9893 // PHIs, so we cannot handle PHIs inside of loops.
9894 return L->getHeader() == I->getParent();
9895 }
9896
9897 // If we won't be able to constant fold this expression even if the operands
9898 // are constants, bail early.
9899 return CanConstantFold(I);
9900}
9901
9902/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
9903/// recursing through each instruction operand until reaching a loop header phi.
9904static PHINode *
9905getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
9906 DenseMap<Instruction *, PHINode *> &PHIMap,
9907 unsigned Depth) {
9908 if (Depth > MaxConstantEvolvingDepth)
9909 return nullptr;
9910
9911 // Otherwise, we can evaluate this instruction if all of its operands are
9912 // constant or derived from a PHI node themselves.
9913 PHINode *PHI = nullptr;
9914 for (Value *Op : UseInst->operands()) {
9915 if (isa<Constant>(Val: Op)) continue;
9916
9917 Instruction *OpInst = dyn_cast<Instruction>(Val: Op);
9918 if (!OpInst || !canConstantEvolve(I: OpInst, L)) return nullptr;
9919
9920 PHINode *P = dyn_cast<PHINode>(Val: OpInst);
9921 if (!P)
9922 // If this operand is already visited, reuse the prior result.
9923 // We may have P != PHI if this is the deepest point at which the
9924 // inconsistent paths meet.
9925 P = PHIMap.lookup(Val: OpInst);
9926 if (!P) {
9927 // Recurse and memoize the results, whether a phi is found or not.
9928 // This recursive call invalidates pointers into PHIMap.
9929 P = getConstantEvolvingPHIOperands(UseInst: OpInst, L, PHIMap, Depth: Depth + 1);
9930 PHIMap[OpInst] = P;
9931 }
9932 if (!P)
9933 return nullptr; // Not evolving from PHI
9934 if (PHI && PHI != P)
9935 return nullptr; // Evolving from multiple different PHIs.
9936 PHI = P;
9937 }
9938 // This is a expression evolving from a constant PHI!
9939 return PHI;
9940}
9941
9942/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
9943/// in the loop that V is derived from. We allow arbitrary operations along the
9944/// way, but the operands of an operation must either be constants or a value
9945/// derived from a constant PHI. If this expression does not fit with these
9946/// constraints, return null.
9947static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
9948 Instruction *I = dyn_cast<Instruction>(Val: V);
9949 if (!I || !canConstantEvolve(I, L)) return nullptr;
9950
9951 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
9952 return PN;
9953
9954 // Record non-constant instructions contained by the loop.
9955 DenseMap<Instruction *, PHINode *> PHIMap;
9956 return getConstantEvolvingPHIOperands(UseInst: I, L, PHIMap, Depth: 0);
9957}
9958
9959/// EvaluateExpression - Given an expression that passes the
9960/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
9961/// in the loop has the value PHIVal. If we can't fold this expression for some
9962/// reason, return null.
9963static Constant *EvaluateExpression(Value *V, const Loop *L,
9964 DenseMap<Instruction *, Constant *> &Vals,
9965 const DataLayout &DL,
9966 const TargetLibraryInfo *TLI) {
9967 // Convenient constant check, but redundant for recursive calls.
9968 if (Constant *C = dyn_cast<Constant>(Val: V)) return C;
9969 Instruction *I = dyn_cast<Instruction>(Val: V);
9970 if (!I) return nullptr;
9971
9972 if (Constant *C = Vals.lookup(Val: I)) return C;
9973
9974 // An instruction inside the loop depends on a value outside the loop that we
9975 // weren't given a mapping for, or a value such as a call inside the loop.
9976 if (!canConstantEvolve(I, L)) return nullptr;
9977
9978 // An unmapped PHI can be due to a branch or another loop inside this loop,
9979 // or due to this not being the initial iteration through a loop where we
9980 // couldn't compute the evolution of this particular PHI last time.
9981 if (isa<PHINode>(Val: I)) return nullptr;
9982
9983 std::vector<Constant*> Operands(I->getNumOperands());
9984
9985 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
9986 Instruction *Operand = dyn_cast<Instruction>(Val: I->getOperand(i));
9987 if (!Operand) {
9988 Operands[i] = dyn_cast<Constant>(Val: I->getOperand(i));
9989 if (!Operands[i]) return nullptr;
9990 continue;
9991 }
9992 Constant *C = EvaluateExpression(V: Operand, L, Vals, DL, TLI);
9993 Vals[Operand] = C;
9994 if (!C) return nullptr;
9995 Operands[i] = C;
9996 }
9997
9998 return ConstantFoldInstOperands(I, Ops: Operands, DL, TLI,
9999 /*AllowNonDeterministic=*/false);
10000}
10001
10002
10003// If every incoming value to PN except the one for BB is a specific Constant,
10004// return that, else return nullptr.
10005static Constant *getOtherIncomingValue(PHINode *PN, BasicBlock *BB) {
10006 Constant *IncomingVal = nullptr;
10007
10008 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
10009 if (PN->getIncomingBlock(i) == BB)
10010 continue;
10011
10012 auto *CurrentVal = dyn_cast<Constant>(Val: PN->getIncomingValue(i));
10013 if (!CurrentVal)
10014 return nullptr;
10015
10016 if (IncomingVal != CurrentVal) {
10017 if (IncomingVal)
10018 return nullptr;
10019 IncomingVal = CurrentVal;
10020 }
10021 }
10022
10023 return IncomingVal;
10024}
10025
10026/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
10027/// in the header of its containing loop, we know the loop executes a
10028/// constant number of times, and the PHI node is just a recurrence
10029/// involving constants, fold it.
10030Constant *
10031ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
10032 const APInt &BEs,
10033 const Loop *L) {
10034 auto [I, Inserted] = ConstantEvolutionLoopExitValue.try_emplace(Key: PN);
10035 if (!Inserted)
10036 return I->second;
10037
10038 if (BEs.ugt(RHS: MaxBruteForceIterations))
10039 return nullptr; // Not going to evaluate it.
10040
10041 Constant *&RetVal = I->second;
10042
10043 DenseMap<Instruction *, Constant *> CurrentIterVals;
10044 BasicBlock *Header = L->getHeader();
10045 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
10046
10047 BasicBlock *Latch = L->getLoopLatch();
10048 if (!Latch)
10049 return nullptr;
10050
10051 for (PHINode &PHI : Header->phis()) {
10052 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
10053 CurrentIterVals[&PHI] = StartCST;
10054 }
10055 if (!CurrentIterVals.count(Val: PN))
10056 return RetVal = nullptr;
10057
10058 Value *BEValue = PN->getIncomingValueForBlock(BB: Latch);
10059
10060 // Execute the loop symbolically to determine the exit value.
10061 assert(BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) &&
10062 "BEs is <= MaxBruteForceIterations which is an 'unsigned'!");
10063
10064 unsigned NumIterations = BEs.getZExtValue(); // must be in range
10065 unsigned IterationNum = 0;
10066 const DataLayout &DL = getDataLayout();
10067 for (; ; ++IterationNum) {
10068 if (IterationNum == NumIterations)
10069 return RetVal = CurrentIterVals[PN]; // Got exit value!
10070
10071 // Compute the value of the PHIs for the next iteration.
10072 // EvaluateExpression adds non-phi values to the CurrentIterVals map.
10073 DenseMap<Instruction *, Constant *> NextIterVals;
10074 Constant *NextPHI =
10075 EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
10076 if (!NextPHI)
10077 return nullptr; // Couldn't evaluate!
10078 NextIterVals[PN] = NextPHI;
10079
10080 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
10081
10082 // Also evaluate the other PHI nodes. However, we don't get to stop if we
10083 // cease to be able to evaluate one of them or if they stop evolving,
10084 // because that doesn't necessarily prevent us from computing PN.
10085 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
10086 for (const auto &I : CurrentIterVals) {
10087 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
10088 if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
10089 PHIsToCompute.emplace_back(Args&: PHI, Args: I.second);
10090 }
10091 // We use two distinct loops because EvaluateExpression may invalidate any
10092 // iterators into CurrentIterVals.
10093 for (const auto &I : PHIsToCompute) {
10094 PHINode *PHI = I.first;
10095 Constant *&NextPHI = NextIterVals[PHI];
10096 if (!NextPHI) { // Not already computed.
10097 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
10098 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
10099 }
10100 if (NextPHI != I.second)
10101 StoppedEvolving = false;
10102 }
10103
10104 // If all entries in CurrentIterVals == NextIterVals then we can stop
10105 // iterating, the loop can't continue to change.
10106 if (StoppedEvolving)
10107 return RetVal = CurrentIterVals[PN];
10108
10109 CurrentIterVals.swap(RHS&: NextIterVals);
10110 }
10111}
10112
10113const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L,
10114 Value *Cond,
10115 bool ExitWhen) {
10116 PHINode *PN = getConstantEvolvingPHI(V: Cond, L);
10117 if (!PN) return getCouldNotCompute();
10118
10119 // If the loop is canonicalized, the PHI will have exactly two entries.
10120 // That's the only form we support here.
10121 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
10122
10123 DenseMap<Instruction *, Constant *> CurrentIterVals;
10124 BasicBlock *Header = L->getHeader();
10125 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
10126
10127 BasicBlock *Latch = L->getLoopLatch();
10128 assert(Latch && "Should follow from NumIncomingValues == 2!");
10129
10130 for (PHINode &PHI : Header->phis()) {
10131 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
10132 CurrentIterVals[&PHI] = StartCST;
10133 }
10134 if (!CurrentIterVals.count(Val: PN))
10135 return getCouldNotCompute();
10136
10137 // Okay, we find a PHI node that defines the trip count of this loop. Execute
10138 // the loop symbolically to determine when the condition gets a value of
10139 // "ExitWhen".
10140 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis.
10141 const DataLayout &DL = getDataLayout();
10142 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
10143 auto *CondVal = dyn_cast_or_null<ConstantInt>(
10144 Val: EvaluateExpression(V: Cond, L, Vals&: CurrentIterVals, DL, TLI: &TLI));
10145
10146 // Couldn't symbolically evaluate.
10147 if (!CondVal) return getCouldNotCompute();
10148
10149 if (CondVal->getValue() == uint64_t(ExitWhen)) {
10150 ++NumBruteForceTripCountsComputed;
10151 return getConstant(Ty: Type::getInt32Ty(C&: getContext()), V: IterationNum);
10152 }
10153
10154 // Update all the PHI nodes for the next iteration.
10155 DenseMap<Instruction *, Constant *> NextIterVals;
10156
10157 // Create a list of which PHIs we need to compute. We want to do this before
10158 // calling EvaluateExpression on them because that may invalidate iterators
10159 // into CurrentIterVals.
10160 SmallVector<PHINode *, 8> PHIsToCompute;
10161 for (const auto &I : CurrentIterVals) {
10162 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
10163 if (!PHI || PHI->getParent() != Header) continue;
10164 PHIsToCompute.push_back(Elt: PHI);
10165 }
10166 for (PHINode *PHI : PHIsToCompute) {
10167 Constant *&NextPHI = NextIterVals[PHI];
10168 if (NextPHI) continue; // Already computed!
10169
10170 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
10171 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
10172 }
10173 CurrentIterVals.swap(RHS&: NextIterVals);
10174 }
10175
10176 // Too many iterations were needed to evaluate.
10177 return getCouldNotCompute();
10178}
10179
10180const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
10181 SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values =
10182 ValuesAtScopes[V];
10183 // Check to see if we've folded this expression at this loop before.
10184 for (auto &LS : Values)
10185 if (LS.first == L)
10186 return LS.second ? LS.second : V;
10187
10188 Values.emplace_back(Args&: L, Args: nullptr);
10189
10190 // Otherwise compute it.
10191 const SCEV *C = computeSCEVAtScope(S: V, L);
10192 for (auto &LS : reverse(C&: ValuesAtScopes[V]))
10193 if (LS.first == L) {
10194 LS.second = C;
10195 if (!isa<SCEVConstant>(Val: C))
10196 ValuesAtScopesUsers[C].push_back(Elt: {L, V});
10197 break;
10198 }
10199 return C;
10200}
10201
10202/// This builds up a Constant using the ConstantExpr interface. That way, we
10203/// will return Constants for objects which aren't represented by a
10204/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
10205/// Returns NULL if the SCEV isn't representable as a Constant.
10206static Constant *BuildConstantFromSCEV(const SCEV *V) {
10207 switch (V->getSCEVType()) {
10208 case scCouldNotCompute:
10209 case scAddRecExpr:
10210 case scVScale:
10211 return nullptr;
10212 case scConstant:
10213 return cast<SCEVConstant>(Val: V)->getValue();
10214 case scUnknown:
10215 return dyn_cast<Constant>(Val: cast<SCEVUnknown>(Val: V)->getValue());
10216 case scPtrToAddr: {
10217 const SCEVPtrToAddrExpr *P2I = cast<SCEVPtrToAddrExpr>(Val: V);
10218 if (Constant *CastOp = BuildConstantFromSCEV(V: P2I->getOperand()))
10219 return ConstantExpr::getPtrToAddr(C: CastOp, Ty: P2I->getType());
10220
10221 return nullptr;
10222 }
10223 case scPtrToInt: {
10224 const SCEVPtrToIntExpr *P2I = cast<SCEVPtrToIntExpr>(Val: V);
10225 if (Constant *CastOp = BuildConstantFromSCEV(V: P2I->getOperand()))
10226 return ConstantExpr::getPtrToInt(C: CastOp, Ty: P2I->getType());
10227
10228 return nullptr;
10229 }
10230 case scTruncate: {
10231 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(Val: V);
10232 if (Constant *CastOp = BuildConstantFromSCEV(V: ST->getOperand()))
10233 return ConstantExpr::getTrunc(C: CastOp, Ty: ST->getType());
10234 return nullptr;
10235 }
10236 case scAddExpr: {
10237 const SCEVAddExpr *SA = cast<SCEVAddExpr>(Val: V);
10238 Constant *C = nullptr;
10239 for (const SCEV *Op : SA->operands()) {
10240 Constant *OpC = BuildConstantFromSCEV(V: Op);
10241 if (!OpC)
10242 return nullptr;
10243 if (!C) {
10244 C = OpC;
10245 continue;
10246 }
10247 assert(!C->getType()->isPointerTy() &&
10248 "Can only have one pointer, and it must be last");
10249 if (OpC->getType()->isPointerTy()) {
10250 // The offsets have been converted to bytes. We can add bytes using
10251 // an i8 GEP.
10252 C = ConstantExpr::getPtrAdd(Ptr: OpC, Offset: C);
10253 } else {
10254 C = ConstantExpr::getAdd(C1: C, C2: OpC);
10255 }
10256 }
10257 return C;
10258 }
10259 case scMulExpr:
10260 case scSignExtend:
10261 case scZeroExtend:
10262 case scUDivExpr:
10263 case scSMaxExpr:
10264 case scUMaxExpr:
10265 case scSMinExpr:
10266 case scUMinExpr:
10267 case scSequentialUMinExpr:
10268 return nullptr;
10269 }
10270 llvm_unreachable("Unknown SCEV kind!");
10271}
10272
10273const SCEV *ScalarEvolution::getWithOperands(const SCEV *S,
10274 SmallVectorImpl<SCEVUse> &NewOps) {
10275 switch (S->getSCEVType()) {
10276 case scTruncate:
10277 case scZeroExtend:
10278 case scSignExtend:
10279 case scPtrToAddr:
10280 case scPtrToInt:
10281 return getCastExpr(Kind: S->getSCEVType(), Op: NewOps[0], Ty: S->getType());
10282 case scAddRecExpr: {
10283 auto *AddRec = cast<SCEVAddRecExpr>(Val: S);
10284 return getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags: AddRec->getNoWrapFlags());
10285 }
10286 case scAddExpr:
10287 return getAddExpr(Ops&: NewOps, OrigFlags: cast<SCEVAddExpr>(Val: S)->getNoWrapFlags());
10288 case scMulExpr:
10289 return getMulExpr(Ops&: NewOps, OrigFlags: cast<SCEVMulExpr>(Val: S)->getNoWrapFlags());
10290 case scUDivExpr:
10291 return getUDivExpr(LHS: NewOps[0], RHS: NewOps[1]);
10292 case scUMaxExpr:
10293 case scSMaxExpr:
10294 case scUMinExpr:
10295 case scSMinExpr:
10296 return getMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10297 case scSequentialUMinExpr:
10298 return getSequentialMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10299 case scConstant:
10300 case scVScale:
10301 case scUnknown:
10302 return S;
10303 case scCouldNotCompute:
10304 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10305 }
10306 llvm_unreachable("Unknown SCEV kind!");
10307}
10308
10309const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
10310 switch (V->getSCEVType()) {
10311 case scConstant:
10312 case scVScale:
10313 return V;
10314 case scAddRecExpr: {
10315 // If this is a loop recurrence for a loop that does not contain L, then we
10316 // are dealing with the final value computed by the loop.
10317 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: V);
10318 // First, attempt to evaluate each operand.
10319 // Avoid performing the look-up in the common case where the specified
10320 // expression has no loop-variant portions.
10321 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
10322 const SCEV *OpAtScope = getSCEVAtScope(V: AddRec->getOperand(i), L);
10323 if (OpAtScope == AddRec->getOperand(i))
10324 continue;
10325
10326 // Okay, at least one of these operands is loop variant but might be
10327 // foldable. Build a new instance of the folded commutative expression.
10328 SmallVector<SCEVUse, 8> NewOps;
10329 NewOps.reserve(N: AddRec->getNumOperands());
10330 append_range(C&: NewOps, R: AddRec->operands().take_front(N: i));
10331 NewOps.push_back(Elt: OpAtScope);
10332 for (++i; i != e; ++i)
10333 NewOps.push_back(Elt: getSCEVAtScope(V: AddRec->getOperand(i), L));
10334
10335 const SCEV *FoldedRec = getAddRecExpr(
10336 Operands&: NewOps, L: AddRec->getLoop(), Flags: AddRec->getNoWrapFlags(Mask: SCEV::FlagNW));
10337 AddRec = dyn_cast<SCEVAddRecExpr>(Val: FoldedRec);
10338 // The addrec may be folded to a nonrecurrence, for example, if the
10339 // induction variable is multiplied by zero after constant folding. Go
10340 // ahead and return the folded value.
10341 if (!AddRec)
10342 return FoldedRec;
10343 break;
10344 }
10345
10346 // If the scope is outside the addrec's loop, evaluate it by using the
10347 // loop exit value of the addrec.
10348 if (!AddRec->getLoop()->contains(L)) {
10349 // To evaluate this recurrence, we need to know how many times the AddRec
10350 // loop iterates. Compute this now.
10351 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: AddRec->getLoop());
10352 if (BackedgeTakenCount == getCouldNotCompute())
10353 return AddRec;
10354
10355 // Then, evaluate the AddRec.
10356 return AddRec->evaluateAtIteration(It: BackedgeTakenCount, SE&: *this);
10357 }
10358
10359 return AddRec;
10360 }
10361 case scTruncate:
10362 case scZeroExtend:
10363 case scSignExtend:
10364 case scPtrToAddr:
10365 case scPtrToInt:
10366 case scAddExpr:
10367 case scMulExpr:
10368 case scUDivExpr:
10369 case scUMaxExpr:
10370 case scSMaxExpr:
10371 case scUMinExpr:
10372 case scSMinExpr:
10373 case scSequentialUMinExpr: {
10374 ArrayRef<SCEVUse> Ops = V->operands();
10375 // Avoid performing the look-up in the common case where the specified
10376 // expression has no loop-variant portions.
10377 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
10378 const SCEV *OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10379 if (OpAtScope != Ops[i].getPointer()) {
10380 // Okay, at least one of these operands is loop variant but might be
10381 // foldable. Build a new instance of the folded commutative expression.
10382 SmallVector<SCEVUse, 8> NewOps;
10383 NewOps.reserve(N: Ops.size());
10384 append_range(C&: NewOps, R: Ops.take_front(N: i));
10385 NewOps.push_back(Elt: OpAtScope);
10386
10387 for (++i; i != e; ++i) {
10388 OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10389 NewOps.push_back(Elt: OpAtScope);
10390 }
10391
10392 return getWithOperands(S: V, NewOps);
10393 }
10394 }
10395 // If we got here, all operands are loop invariant.
10396 return V;
10397 }
10398 case scUnknown: {
10399 // If this instruction is evolved from a constant-evolving PHI, compute the
10400 // exit value from the loop without using SCEVs.
10401 const SCEVUnknown *SU = cast<SCEVUnknown>(Val: V);
10402 Instruction *I = dyn_cast<Instruction>(Val: SU->getValue());
10403 if (!I)
10404 return V; // This is some other type of SCEVUnknown, just return it.
10405
10406 if (PHINode *PN = dyn_cast<PHINode>(Val: I)) {
10407 const Loop *CurrLoop = this->LI[I->getParent()];
10408 // Looking for loop exit value.
10409 if (CurrLoop && CurrLoop->getParentLoop() == L &&
10410 PN->getParent() == CurrLoop->getHeader()) {
10411 // Okay, there is no closed form solution for the PHI node. Check
10412 // to see if the loop that contains it has a known backedge-taken
10413 // count. If so, we may be able to force computation of the exit
10414 // value.
10415 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: CurrLoop);
10416 // This trivial case can show up in some degenerate cases where
10417 // the incoming IR has not yet been fully simplified.
10418 if (BackedgeTakenCount->isZero()) {
10419 Value *InitValue = nullptr;
10420 bool MultipleInitValues = false;
10421 for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
10422 if (!CurrLoop->contains(BB: PN->getIncomingBlock(i))) {
10423 if (!InitValue)
10424 InitValue = PN->getIncomingValue(i);
10425 else if (InitValue != PN->getIncomingValue(i)) {
10426 MultipleInitValues = true;
10427 break;
10428 }
10429 }
10430 }
10431 if (!MultipleInitValues && InitValue)
10432 return getSCEV(V: InitValue);
10433 }
10434 // Do we have a loop invariant value flowing around the backedge
10435 // for a loop which must execute the backedge?
10436 if (!isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount) &&
10437 isKnownNonZero(S: BackedgeTakenCount) &&
10438 PN->getNumIncomingValues() == 2) {
10439
10440 unsigned InLoopPred =
10441 CurrLoop->contains(BB: PN->getIncomingBlock(i: 0)) ? 0 : 1;
10442 Value *BackedgeVal = PN->getIncomingValue(i: InLoopPred);
10443 if (CurrLoop->isLoopInvariant(V: BackedgeVal))
10444 return getSCEV(V: BackedgeVal);
10445 }
10446 if (auto *BTCC = dyn_cast<SCEVConstant>(Val: BackedgeTakenCount)) {
10447 // Okay, we know how many times the containing loop executes. If
10448 // this is a constant evolving PHI node, get the final value at
10449 // the specified iteration number.
10450 Constant *RV =
10451 getConstantEvolutionLoopExitValue(PN, BEs: BTCC->getAPInt(), L: CurrLoop);
10452 if (RV)
10453 return getSCEV(V: RV);
10454 }
10455 }
10456 }
10457
10458 // Okay, this is an expression that we cannot symbolically evaluate
10459 // into a SCEV. Check to see if it's possible to symbolically evaluate
10460 // the arguments into constants, and if so, try to constant propagate the
10461 // result. This is particularly useful for computing loop exit values.
10462 if (!CanConstantFold(I))
10463 return V; // This is some other type of SCEVUnknown, just return it.
10464
10465 SmallVector<Constant *, 4> Operands;
10466 Operands.reserve(N: I->getNumOperands());
10467 bool MadeImprovement = false;
10468 for (Value *Op : I->operands()) {
10469 if (Constant *C = dyn_cast<Constant>(Val: Op)) {
10470 Operands.push_back(Elt: C);
10471 continue;
10472 }
10473
10474 // If any of the operands is non-constant and if they are
10475 // non-integer and non-pointer, don't even try to analyze them
10476 // with scev techniques.
10477 if (!isSCEVable(Ty: Op->getType()))
10478 return V;
10479
10480 const SCEV *OrigV = getSCEV(V: Op);
10481 const SCEV *OpV = getSCEVAtScope(V: OrigV, L);
10482 MadeImprovement |= OrigV != OpV;
10483
10484 Constant *C = BuildConstantFromSCEV(V: OpV);
10485 if (!C)
10486 return V;
10487 assert(C->getType() == Op->getType() && "Type mismatch");
10488 Operands.push_back(Elt: C);
10489 }
10490
10491 // Check to see if getSCEVAtScope actually made an improvement.
10492 if (!MadeImprovement)
10493 return V; // This is some other type of SCEVUnknown, just return it.
10494
10495 Constant *C = nullptr;
10496 const DataLayout &DL = getDataLayout();
10497 C = ConstantFoldInstOperands(I, Ops: Operands, DL, TLI: &TLI,
10498 /*AllowNonDeterministic=*/false);
10499 if (!C)
10500 return V;
10501 return getSCEV(V: C);
10502 }
10503 case scCouldNotCompute:
10504 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10505 }
10506 llvm_unreachable("Unknown SCEV type!");
10507}
10508
10509const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
10510 return getSCEVAtScope(V: getSCEV(V), L);
10511}
10512
10513const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const {
10514 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: S))
10515 return stripInjectiveFunctions(S: ZExt->getOperand());
10516 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
10517 return stripInjectiveFunctions(S: SExt->getOperand());
10518 return S;
10519}
10520
10521/// Finds the minimum unsigned root of the following equation:
10522///
10523/// A * X = B (mod N)
10524///
10525/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
10526/// A and B isn't important.
10527///
10528/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
10529static const SCEV *
10530SolveLinEquationWithOverflow(const APInt &A, const SCEV *B,
10531 SmallVectorImpl<const SCEVPredicate *> *Predicates,
10532 ScalarEvolution &SE, const Loop *L) {
10533 uint32_t BW = A.getBitWidth();
10534 assert(BW == SE.getTypeSizeInBits(B->getType()));
10535 assert(A != 0 && "A must be non-zero.");
10536
10537 // 1. D = gcd(A, N)
10538 //
10539 // The gcd of A and N may have only one prime factor: 2. The number of
10540 // trailing zeros in A is its multiplicity
10541 uint32_t Mult2 = A.countr_zero();
10542 // D = 2^Mult2
10543
10544 // 2. Check if B is divisible by D.
10545 //
10546 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
10547 // is not less than multiplicity of this prime factor for D.
10548 unsigned MinTZ = SE.getMinTrailingZeros(S: B);
10549 // Try again with the terminator of the loop predecessor for context-specific
10550 // result, if MinTZ s too small.
10551 if (MinTZ < Mult2 && L->getLoopPredecessor())
10552 MinTZ = SE.getMinTrailingZeros(S: B, CtxI: L->getLoopPredecessor()->getTerminator());
10553 if (MinTZ < Mult2) {
10554 // Check if we can prove there's no remainder using URem.
10555 const SCEV *URem =
10556 SE.getURemExpr(LHS: B, RHS: SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2)));
10557 const SCEV *Zero = SE.getZero(Ty: B->getType());
10558 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: URem, RHS: Zero)) {
10559 // Try to add a predicate ensuring B is a multiple of 1 << Mult2.
10560 if (!Predicates)
10561 return SE.getCouldNotCompute();
10562
10563 // Avoid adding a predicate that is known to be false.
10564 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_NE, LHS: URem, RHS: Zero))
10565 return SE.getCouldNotCompute();
10566 Predicates->push_back(Elt: SE.getEqualPredicate(LHS: URem, RHS: Zero));
10567 }
10568 }
10569
10570 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
10571 // modulo (N / D).
10572 //
10573 // If D == 1, (N / D) == N == 2^BW, so we need one extra bit to represent
10574 // (N / D) in general. The inverse itself always fits into BW bits, though,
10575 // so we immediately truncate it.
10576 APInt AD = A.lshr(shiftAmt: Mult2).trunc(width: BW - Mult2); // AD = A / D
10577 APInt I = AD.multiplicativeInverse().zext(width: BW);
10578
10579 // 4. Compute the minimum unsigned root of the equation:
10580 // I * (B / D) mod (N / D)
10581 // To simplify the computation, we factor out the divide by D:
10582 // (I * B mod N) / D
10583 const SCEV *D = SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2));
10584 return SE.getUDivExactExpr(LHS: SE.getMulExpr(LHS: B, RHS: SE.getConstant(Val: I)), RHS: D);
10585}
10586
10587/// For a given quadratic addrec, generate coefficients of the corresponding
10588/// quadratic equation, multiplied by a common value to ensure that they are
10589/// integers.
10590/// The returned value is a tuple { A, B, C, M, BitWidth }, where
10591/// Ax^2 + Bx + C is the quadratic function, M is the value that A, B and C
10592/// were multiplied by, and BitWidth is the bit width of the original addrec
10593/// coefficients.
10594/// This function returns std::nullopt if the addrec coefficients are not
10595/// compile- time constants.
10596static std::optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
10597GetQuadraticEquation(const SCEVAddRecExpr *AddRec) {
10598 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
10599 const SCEVConstant *LC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 0));
10600 const SCEVConstant *MC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 1));
10601 const SCEVConstant *NC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 2));
10602 LLVM_DEBUG(dbgs() << __func__ << ": analyzing quadratic addrec: "
10603 << *AddRec << '\n');
10604
10605 // We currently can only solve this if the coefficients are constants.
10606 if (!LC || !MC || !NC) {
10607 LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n");
10608 return std::nullopt;
10609 }
10610
10611 APInt L = LC->getAPInt();
10612 APInt M = MC->getAPInt();
10613 APInt N = NC->getAPInt();
10614 assert(!N.isZero() && "This is not a quadratic addrec");
10615
10616 unsigned BitWidth = LC->getAPInt().getBitWidth();
10617 unsigned NewWidth = BitWidth + 1;
10618 LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "
10619 << BitWidth << '\n');
10620 // The sign-extension (as opposed to a zero-extension) here matches the
10621 // extension used in SolveQuadraticEquationWrap (with the same motivation).
10622 N = N.sext(width: NewWidth);
10623 M = M.sext(width: NewWidth);
10624 L = L.sext(width: NewWidth);
10625
10626 // The increments are M, M+N, M+2N, ..., so the accumulated values are
10627 // L+M, (L+M)+(M+N), (L+M)+(M+N)+(M+2N), ..., that is,
10628 // L+M, L+2M+N, L+3M+3N, ...
10629 // After n iterations the accumulated value Acc is L + nM + n(n-1)/2 N.
10630 //
10631 // The equation Acc = 0 is then
10632 // L + nM + n(n-1)/2 N = 0, or 2L + 2M n + n(n-1) N = 0.
10633 // In a quadratic form it becomes:
10634 // N n^2 + (2M-N) n + 2L = 0.
10635
10636 APInt A = N;
10637 APInt B = 2 * M - A;
10638 APInt C = 2 * L;
10639 APInt T = APInt(NewWidth, 2);
10640 LLVM_DEBUG(dbgs() << __func__ << ": equation " << A << "x^2 + " << B
10641 << "x + " << C << ", coeff bw: " << NewWidth
10642 << ", multiplied by " << T << '\n');
10643 return std::make_tuple(args&: A, args&: B, args&: C, args&: T, args&: BitWidth);
10644}
10645
10646/// Helper function to compare optional APInts:
10647/// (a) if X and Y both exist, return min(X, Y),
10648/// (b) if neither X nor Y exist, return std::nullopt,
10649/// (c) if exactly one of X and Y exists, return that value.
10650static std::optional<APInt> MinOptional(std::optional<APInt> X,
10651 std::optional<APInt> Y) {
10652 if (X && Y) {
10653 unsigned W = std::max(a: X->getBitWidth(), b: Y->getBitWidth());
10654 APInt XW = X->sext(width: W);
10655 APInt YW = Y->sext(width: W);
10656 return XW.slt(RHS: YW) ? *X : *Y;
10657 }
10658 if (!X && !Y)
10659 return std::nullopt;
10660 return X ? *X : *Y;
10661}
10662
10663/// Helper function to truncate an optional APInt to a given BitWidth.
10664/// When solving addrec-related equations, it is preferable to return a value
10665/// that has the same bit width as the original addrec's coefficients. If the
10666/// solution fits in the original bit width, truncate it (except for i1).
10667/// Returning a value of a different bit width may inhibit some optimizations.
10668///
10669/// In general, a solution to a quadratic equation generated from an addrec
10670/// may require BW+1 bits, where BW is the bit width of the addrec's
10671/// coefficients. The reason is that the coefficients of the quadratic
10672/// equation are BW+1 bits wide (to avoid truncation when converting from
10673/// the addrec to the equation).
10674static std::optional<APInt> TruncIfPossible(std::optional<APInt> X,
10675 unsigned BitWidth) {
10676 if (!X)
10677 return std::nullopt;
10678 unsigned W = X->getBitWidth();
10679 if (BitWidth > 1 && BitWidth < W && X->isIntN(N: BitWidth))
10680 return X->trunc(width: BitWidth);
10681 return X;
10682}
10683
10684/// Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n
10685/// iterations. The values L, M, N are assumed to be signed, and they
10686/// should all have the same bit widths.
10687/// Find the least n >= 0 such that c(n) = 0 in the arithmetic modulo 2^BW,
10688/// where BW is the bit width of the addrec's coefficients.
10689/// If the calculated value is a BW-bit integer (for BW > 1), it will be
10690/// returned as such, otherwise the bit width of the returned value may
10691/// be greater than BW.
10692///
10693/// This function returns std::nullopt if
10694/// (a) the addrec coefficients are not constant, or
10695/// (b) SolveQuadraticEquationWrap was unable to find a solution. For cases
10696/// like x^2 = 5, no integer solutions exist, in other cases an integer
10697/// solution may exist, but SolveQuadraticEquationWrap may fail to find it.
10698static std::optional<APInt>
10699SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
10700 APInt A, B, C, M;
10701 unsigned BitWidth;
10702 auto T = GetQuadraticEquation(AddRec);
10703 if (!T)
10704 return std::nullopt;
10705
10706 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10707 LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n");
10708 std::optional<APInt> X =
10709 APIntOps::SolveQuadraticEquationWrap(A, B, C, RangeWidth: BitWidth + 1);
10710 if (!X)
10711 return std::nullopt;
10712
10713 ConstantInt *CX = ConstantInt::get(Context&: SE.getContext(), V: *X);
10714 ConstantInt *V = EvaluateConstantChrecAtConstant(AddRec, C: CX, SE);
10715 if (!V->isZero())
10716 return std::nullopt;
10717
10718 return TruncIfPossible(X, BitWidth);
10719}
10720
10721/// Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n
10722/// iterations. The values M, N are assumed to be signed, and they
10723/// should all have the same bit widths.
10724/// Find the least n such that c(n) does not belong to the given range,
10725/// while c(n-1) does.
10726///
10727/// This function returns std::nullopt if
10728/// (a) the addrec coefficients are not constant, or
10729/// (b) SolveQuadraticEquationWrap was unable to find a solution for the
10730/// bounds of the range.
10731static std::optional<APInt>
10732SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec,
10733 const ConstantRange &Range, ScalarEvolution &SE) {
10734 assert(AddRec->getOperand(0)->isZero() &&
10735 "Starting value of addrec should be 0");
10736 LLVM_DEBUG(dbgs() << __func__ << ": solving boundary crossing for range "
10737 << Range << ", addrec " << *AddRec << '\n');
10738 // This case is handled in getNumIterationsInRange. Here we can assume that
10739 // we start in the range.
10740 assert(Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) &&
10741 "Addrec's initial value should be in range");
10742
10743 APInt A, B, C, M;
10744 unsigned BitWidth;
10745 auto T = GetQuadraticEquation(AddRec);
10746 if (!T)
10747 return std::nullopt;
10748
10749 // Be careful about the return value: there can be two reasons for not
10750 // returning an actual number. First, if no solutions to the equations
10751 // were found, and second, if the solutions don't leave the given range.
10752 // The first case means that the actual solution is "unknown", the second
10753 // means that it's known, but not valid. If the solution is unknown, we
10754 // cannot make any conclusions.
10755 // Return a pair: the optional solution and a flag indicating if the
10756 // solution was found.
10757 auto SolveForBoundary =
10758 [&](APInt Bound) -> std::pair<std::optional<APInt>, bool> {
10759 // Solve for signed overflow and unsigned overflow, pick the lower
10760 // solution.
10761 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: checking boundary "
10762 << Bound << " (before multiplying by " << M << ")\n");
10763 Bound *= M; // The quadratic equation multiplier.
10764
10765 std::optional<APInt> SO;
10766 if (BitWidth > 1) {
10767 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10768 "signed overflow\n");
10769 SO = APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth);
10770 }
10771 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10772 "unsigned overflow\n");
10773 std::optional<APInt> UO =
10774 APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth + 1);
10775
10776 auto LeavesRange = [&] (const APInt &X) {
10777 ConstantInt *C0 = ConstantInt::get(Context&: SE.getContext(), V: X);
10778 ConstantInt *V0 = EvaluateConstantChrecAtConstant(AddRec, C: C0, SE);
10779 if (Range.contains(Val: V0->getValue()))
10780 return false;
10781 // X should be at least 1, so X-1 is non-negative.
10782 ConstantInt *C1 = ConstantInt::get(Context&: SE.getContext(), V: X-1);
10783 ConstantInt *V1 = EvaluateConstantChrecAtConstant(AddRec, C: C1, SE);
10784 if (Range.contains(Val: V1->getValue()))
10785 return true;
10786 return false;
10787 };
10788
10789 // If SolveQuadraticEquationWrap returns std::nullopt, it means that there
10790 // can be a solution, but the function failed to find it. We cannot treat it
10791 // as "no solution".
10792 if (!SO || !UO)
10793 return {std::nullopt, false};
10794
10795 // Check the smaller value first to see if it leaves the range.
10796 // At this point, both SO and UO must have values.
10797 std::optional<APInt> Min = MinOptional(X: SO, Y: UO);
10798 if (LeavesRange(*Min))
10799 return { Min, true };
10800 std::optional<APInt> Max = Min == SO ? UO : SO;
10801 if (LeavesRange(*Max))
10802 return { Max, true };
10803
10804 // Solutions were found, but were eliminated, hence the "true".
10805 return {std::nullopt, true};
10806 };
10807
10808 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10809 // Lower bound is inclusive, subtract 1 to represent the exiting value.
10810 APInt Lower = Range.getLower().sext(width: A.getBitWidth()) - 1;
10811 APInt Upper = Range.getUpper().sext(width: A.getBitWidth());
10812 auto SL = SolveForBoundary(Lower);
10813 auto SU = SolveForBoundary(Upper);
10814 // If any of the solutions was unknown, no meaninigful conclusions can
10815 // be made.
10816 if (!SL.second || !SU.second)
10817 return std::nullopt;
10818
10819 // Claim: The correct solution is not some value between Min and Max.
10820 //
10821 // Justification: Assuming that Min and Max are different values, one of
10822 // them is when the first signed overflow happens, the other is when the
10823 // first unsigned overflow happens. Crossing the range boundary is only
10824 // possible via an overflow (treating 0 as a special case of it, modeling
10825 // an overflow as crossing k*2^W for some k).
10826 //
10827 // The interesting case here is when Min was eliminated as an invalid
10828 // solution, but Max was not. The argument is that if there was another
10829 // overflow between Min and Max, it would also have been eliminated if
10830 // it was considered.
10831 //
10832 // For a given boundary, it is possible to have two overflows of the same
10833 // type (signed/unsigned) without having the other type in between: this
10834 // can happen when the vertex of the parabola is between the iterations
10835 // corresponding to the overflows. This is only possible when the two
10836 // overflows cross k*2^W for the same k. In such case, if the second one
10837 // left the range (and was the first one to do so), the first overflow
10838 // would have to enter the range, which would mean that either we had left
10839 // the range before or that we started outside of it. Both of these cases
10840 // are contradictions.
10841 //
10842 // Claim: In the case where SolveForBoundary returns std::nullopt, the correct
10843 // solution is not some value between the Max for this boundary and the
10844 // Min of the other boundary.
10845 //
10846 // Justification: Assume that we had such Max_A and Min_B corresponding
10847 // to range boundaries A and B and such that Max_A < Min_B. If there was
10848 // a solution between Max_A and Min_B, it would have to be caused by an
10849 // overflow corresponding to either A or B. It cannot correspond to B,
10850 // since Min_B is the first occurrence of such an overflow. If it
10851 // corresponded to A, it would have to be either a signed or an unsigned
10852 // overflow that is larger than both eliminated overflows for A. But
10853 // between the eliminated overflows and this overflow, the values would
10854 // cover the entire value space, thus crossing the other boundary, which
10855 // is a contradiction.
10856
10857 return TruncIfPossible(X: MinOptional(X: SL.first, Y: SU.first), BitWidth);
10858}
10859
10860ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V,
10861 const Loop *L,
10862 bool ControlsOnlyExit,
10863 bool AllowPredicates) {
10864
10865 // This is only used for loops with a "x != y" exit test. The exit condition
10866 // is now expressed as a single expression, V = x-y. So the exit test is
10867 // effectively V != 0. We know and take advantage of the fact that this
10868 // expression only being used in a comparison by zero context.
10869
10870 SmallVector<const SCEVPredicate *> Predicates;
10871 // If the value is a constant
10872 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10873 // If the value is already zero, the branch will execute zero times.
10874 if (C->getValue()->isZero()) return C;
10875 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10876 }
10877
10878 const SCEVAddRecExpr *AddRec =
10879 dyn_cast<SCEVAddRecExpr>(Val: stripInjectiveFunctions(S: V));
10880
10881 if (!AddRec && AllowPredicates)
10882 // Try to make this an AddRec using runtime tests, in the first X
10883 // iterations of this loop, where X is the SCEV expression found by the
10884 // algorithm below.
10885 AddRec = convertSCEVToAddRecWithPredicates(S: V, L, Preds&: Predicates);
10886
10887 if (!AddRec || AddRec->getLoop() != L)
10888 return getCouldNotCompute();
10889
10890 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
10891 // the quadratic equation to solve it.
10892 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
10893 // We can only use this value if the chrec ends up with an exact zero
10894 // value at this index. When solving for "X*X != 5", for example, we
10895 // should not accept a root of 2.
10896 if (auto S = SolveQuadraticAddRecExact(AddRec, SE&: *this)) {
10897 const auto *R = cast<SCEVConstant>(Val: getConstant(Val: *S));
10898 return ExitLimit(R, R, R, false, Predicates);
10899 }
10900 return getCouldNotCompute();
10901 }
10902
10903 // Otherwise we can only handle this if it is affine.
10904 if (!AddRec->isAffine())
10905 return getCouldNotCompute();
10906
10907 // If this is an affine expression, the execution count of this branch is
10908 // the minimum unsigned root of the following equation:
10909 //
10910 // Start + Step*N = 0 (mod 2^BW)
10911 //
10912 // equivalent to:
10913 //
10914 // Step*N = -Start (mod 2^BW)
10915 //
10916 // where BW is the common bit width of Start and Step.
10917
10918 // Get the initial value for the loop.
10919 const SCEV *Start = getSCEVAtScope(V: AddRec->getStart(), L: L->getParentLoop());
10920 const SCEV *Step = getSCEVAtScope(V: AddRec->getOperand(i: 1), L: L->getParentLoop());
10921
10922 if (!isLoopInvariant(S: Step, L))
10923 return getCouldNotCompute();
10924
10925 LoopGuards Guards = LoopGuards::collect(L, SE&: *this);
10926 // Specialize step for this loop so we get context sensitive facts below.
10927 const SCEV *StepWLG = applyLoopGuards(Expr: Step, Guards);
10928
10929 // For positive steps (counting up until unsigned overflow):
10930 // N = -Start/Step (as unsigned)
10931 // For negative steps (counting down to zero):
10932 // N = Start/-Step
10933 // First compute the unsigned distance from zero in the direction of Step.
10934 bool CountDown = isKnownNegative(S: StepWLG);
10935 if (!CountDown && !isKnownNonNegative(S: StepWLG))
10936 return getCouldNotCompute();
10937
10938 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(V: Start);
10939 // Handle unitary steps, which cannot wraparound.
10940 // 1*N = -Start; -1*N = Start (mod 2^BW), so:
10941 // N = Distance (as unsigned)
10942
10943 if (match(S: Step, P: m_CombineOr(Ps: m_scev_One(), Ps: m_scev_AllOnes()))) {
10944 APInt MaxBECount = getUnsignedRangeMax(S: applyLoopGuards(Expr: Distance, Guards));
10945 MaxBECount = APIntOps::umin(A: MaxBECount, B: getUnsignedRangeMax(S: Distance));
10946
10947 // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated,
10948 // we end up with a loop whose backedge-taken count is n - 1. Detect this
10949 // case, and see if we can improve the bound.
10950 //
10951 // Explicitly handling this here is necessary because getUnsignedRange
10952 // isn't context-sensitive; it doesn't know that we only care about the
10953 // range inside the loop.
10954 const SCEV *Zero = getZero(Ty: Distance->getType());
10955 const SCEV *One = getOne(Ty: Distance->getType());
10956 const SCEV *DistancePlusOne = getAddExpr(LHS: Distance, RHS: One);
10957 if (isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: DistancePlusOne, RHS: Zero)) {
10958 // If Distance + 1 doesn't overflow, we can compute the maximum distance
10959 // as "unsigned_max(Distance + 1) - 1".
10960 ConstantRange CR = getUnsignedRange(S: DistancePlusOne);
10961 MaxBECount = APIntOps::umin(A: MaxBECount, B: CR.getUnsignedMax() - 1);
10962 }
10963 return ExitLimit(Distance, getConstant(Val: MaxBECount), Distance, false,
10964 Predicates);
10965 }
10966
10967 // If the condition controls loop exit (the loop exits only if the expression
10968 // is true) and the addition is no-wrap we can use unsigned divide to
10969 // compute the backedge count. In this case, the step may not divide the
10970 // distance, but we don't care because if the condition is "missed" the loop
10971 // will have undefined behavior due to wrapping.
10972 if (ControlsOnlyExit && AddRec->hasNoSelfWrap() &&
10973 loopHasNoAbnormalExits(L: AddRec->getLoop())) {
10974
10975 // If the stride is zero and the start is non-zero, the loop must be
10976 // infinite. In C++, most loops are finite by assumption, in which case the
10977 // step being zero implies UB must execute if the loop is entered.
10978 if (!(loopIsFiniteByAssumption(L) && isKnownNonZero(S: Start)) &&
10979 !isKnownNonZero(S: StepWLG))
10980 return getCouldNotCompute();
10981
10982 const SCEV *Exact =
10983 getUDivExpr(LHS: Distance, RHS: CountDown ? getNegativeSCEV(V: Step) : Step);
10984 const SCEV *ConstantMax = getCouldNotCompute();
10985 if (Exact != getCouldNotCompute()) {
10986 APInt MaxInt = getUnsignedRangeMax(S: applyLoopGuards(Expr: Exact, Guards));
10987 ConstantMax =
10988 getConstant(Val: APIntOps::umin(A: MaxInt, B: getUnsignedRangeMax(S: Exact)));
10989 }
10990 const SCEV *SymbolicMax =
10991 isa<SCEVCouldNotCompute>(Val: Exact) ? ConstantMax : Exact;
10992 return ExitLimit(Exact, ConstantMax, SymbolicMax, false, Predicates);
10993 }
10994
10995 // Solve the general equation.
10996 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Val: Step);
10997 if (!StepC || StepC->getValue()->isZero())
10998 return getCouldNotCompute();
10999 const SCEV *E = SolveLinEquationWithOverflow(
11000 A: StepC->getAPInt(), B: getNegativeSCEV(V: Start),
11001 Predicates: AllowPredicates ? &Predicates : nullptr, SE&: *this, L);
11002
11003 const SCEV *M = E;
11004 if (E != getCouldNotCompute()) {
11005 APInt MaxWithGuards = getUnsignedRangeMax(S: applyLoopGuards(Expr: E, Guards));
11006 M = getConstant(Val: APIntOps::umin(A: MaxWithGuards, B: getUnsignedRangeMax(S: E)));
11007 }
11008 auto *S = isa<SCEVCouldNotCompute>(Val: E) ? M : E;
11009 return ExitLimit(E, M, S, false, Predicates);
11010}
11011
11012ScalarEvolution::ExitLimit
11013ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) {
11014 // Loops that look like: while (X == 0) are very strange indeed. We don't
11015 // handle them yet except for the trivial case. This could be expanded in the
11016 // future as needed.
11017
11018 // If the value is a constant, check to see if it is known to be non-zero
11019 // already. If so, the backedge will execute zero times.
11020 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
11021 if (!C->getValue()->isZero())
11022 return getZero(Ty: C->getType());
11023 return getCouldNotCompute(); // Otherwise it will loop infinitely.
11024 }
11025
11026 // We could implement others, but I really doubt anyone writes loops like
11027 // this, and if they did, they would already be constant folded.
11028 return getCouldNotCompute();
11029}
11030
11031std::pair<const BasicBlock *, const BasicBlock *>
11032ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(const BasicBlock *BB)
11033 const {
11034 // If the block has a unique predecessor, then there is no path from the
11035 // predecessor to the block that does not go through the direct edge
11036 // from the predecessor to the block.
11037 if (const BasicBlock *Pred = BB->getSinglePredecessor())
11038 return {Pred, BB};
11039
11040 // A loop's header is defined to be a block that dominates the loop.
11041 // If the header has a unique predecessor outside the loop, it must be
11042 // a block that has exactly one successor that can reach the loop.
11043 if (const Loop *L = LI.getLoopFor(BB))
11044 return {L->getLoopPredecessor(), L->getHeader()};
11045
11046 return {nullptr, BB};
11047}
11048
11049/// SCEV structural equivalence is usually sufficient for testing whether two
11050/// expressions are equal, however for the purposes of looking for a condition
11051/// guarding a loop, it can be useful to be a little more general, since a
11052/// front-end may have replicated the controlling expression.
11053static bool HasSameValue(const SCEV *A, const SCEV *B) {
11054 // Quick check to see if they are the same SCEV.
11055 if (A == B) return true;
11056
11057 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) {
11058 // Not all instructions that are "identical" compute the same value. For
11059 // instance, two distinct alloca instructions allocating the same type are
11060 // identical and do not read memory; but compute distinct values.
11061 return A->isIdenticalTo(I: B) && (isa<BinaryOperator>(Val: A) || isa<GetElementPtrInst>(Val: A));
11062 };
11063
11064 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
11065 // two different instructions with the same value. Check for this case.
11066 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(Val: A))
11067 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(Val: B))
11068 if (const Instruction *AI = dyn_cast<Instruction>(Val: AU->getValue()))
11069 if (const Instruction *BI = dyn_cast<Instruction>(Val: BU->getValue()))
11070 if (ComputesEqualValues(AI, BI))
11071 return true;
11072
11073 // Otherwise assume they may have a different value.
11074 return false;
11075}
11076
11077static bool MatchBinarySub(const SCEV *S, SCEVUse &LHS, SCEVUse &RHS) {
11078 const SCEV *Op0, *Op1;
11079 if (!match(S, P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))))
11080 return false;
11081 if (match(S: Op0, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
11082 LHS = Op1;
11083 return true;
11084 }
11085 if (match(S: Op1, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
11086 LHS = Op0;
11087 return true;
11088 }
11089 return false;
11090}
11091
11092bool ScalarEvolution::SimplifyICmpOperands(CmpPredicate &Pred, SCEVUse &LHS,
11093 SCEVUse &RHS, unsigned Depth) {
11094 bool Changed = false;
11095 // Simplifies ICMP to trivial true or false by turning it into '0 == 0' or
11096 // '0 != 0'.
11097 auto TrivialCase = [&](bool TriviallyTrue) {
11098 LHS = RHS = getConstant(V: ConstantInt::getFalse(Context&: getContext()));
11099 Pred = TriviallyTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
11100 return true;
11101 };
11102 // If we hit the max recursion limit bail out.
11103 if (Depth >= 3)
11104 return false;
11105
11106 const SCEV *NewLHS, *NewRHS;
11107 if (match(U: LHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
11108 match(U: RHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale()))) {
11109 const SCEVMulExpr *LMul = cast<SCEVMulExpr>(Val&: LHS);
11110 const SCEVMulExpr *RMul = cast<SCEVMulExpr>(Val&: RHS);
11111
11112 // (X * vscale) pred (Y * vscale) ==> X pred Y
11113 // when both multiples are NSW.
11114 // (X * vscale) uicmp/eq/ne (Y * vscale) ==> X uicmp/eq/ne Y
11115 // when both multiples are NUW.
11116 if ((LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap()) ||
11117 (LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap() &&
11118 !ICmpInst::isSigned(Pred))) {
11119 LHS = NewLHS;
11120 RHS = NewRHS;
11121 Changed = true;
11122 }
11123 }
11124
11125 // Canonicalize a constant to the right side.
11126 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS)) {
11127 // Check for both operands constant.
11128 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
11129 if (!ICmpInst::compare(LHS: LHSC->getAPInt(), RHS: RHSC->getAPInt(), Pred))
11130 return TrivialCase(false);
11131 return TrivialCase(true);
11132 }
11133 // Otherwise swap the operands to put the constant on the right.
11134 std::swap(a&: LHS, b&: RHS);
11135 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11136 Changed = true;
11137 }
11138
11139 // (K + A) pred (K + B) --> A pred B
11140 // For equality, no flags are needed.
11141 // For signed, both adds must be NSW. For unsigned, both must be NUW.
11142 {
11143 const SCEVConstant *C = nullptr;
11144 if (match(U: LHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: NewLHS))) &&
11145 match(U: RHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: NewRHS)))) {
11146 const auto *LAdd = cast<SCEVAddExpr>(Val&: LHS);
11147 const auto *RAdd = cast<SCEVAddExpr>(Val&: RHS);
11148 if (ICmpInst::isEquality(P: Pred) ||
11149 (ICmpInst::isSigned(Pred) && LAdd->hasNoSignedWrap() &&
11150 RAdd->hasNoSignedWrap()) ||
11151 (ICmpInst::isUnsigned(Pred) && LAdd->hasNoUnsignedWrap() &&
11152 RAdd->hasNoUnsignedWrap())) {
11153 LHS = NewLHS;
11154 RHS = NewRHS;
11155 Changed = true;
11156 }
11157 }
11158 }
11159
11160 // (C * A) pred (C * B) --> A pred B
11161 // For equality predicates, both muls must be NUW or both must be NSW
11162 // (either suffices to make multiplication by C injective; C == 0 is
11163 // impossible because SCEV folds 0 * X to 0).
11164 // For signed ordering, C must be positive and both muls must be NSW.
11165 // For unsigned ordering, both muls must be NUW.
11166 {
11167 const SCEVConstant *C = nullptr;
11168 if (match(U: LHS, P: m_scev_Mul(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: NewLHS))) &&
11169 match(U: RHS, P: m_scev_Mul(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: NewRHS)))) {
11170 const auto *LMul = cast<SCEVMulExpr>(Val&: LHS);
11171 const auto *RMul = cast<SCEVMulExpr>(Val&: RHS);
11172 bool BothNUW = LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap();
11173 bool BothNSW = LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap();
11174 if ((ICmpInst::isEquality(P: Pred) && (BothNUW || BothNSW)) ||
11175 (ICmpInst::isSigned(Pred) && BothNSW &&
11176 C->getAPInt().isStrictlyPositive()) ||
11177 (ICmpInst::isUnsigned(Pred) && BothNUW)) {
11178 LHS = NewLHS;
11179 RHS = NewRHS;
11180 Changed = true;
11181 }
11182 }
11183 }
11184
11185 // If we're comparing an addrec with a value which is loop-invariant in the
11186 // addrec's loop, put the addrec on the left. Also make a dominance check,
11187 // as both operands could be addrecs loop-invariant in each other's loop.
11188 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: RHS)) {
11189 const Loop *L = AR->getLoop();
11190 if (isLoopInvariant(S: LHS, L) && properlyDominates(S: LHS, BB: L->getHeader())) {
11191 std::swap(a&: LHS, b&: RHS);
11192 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11193 Changed = true;
11194 }
11195 }
11196
11197 // If there's a constant operand, canonicalize comparisons with boundary
11198 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
11199 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(Val&: RHS)) {
11200 const APInt &RA = RC->getAPInt();
11201
11202 bool SimplifiedByConstantRange = false;
11203
11204 if (!ICmpInst::isEquality(P: Pred)) {
11205 ConstantRange ExactCR = ConstantRange::makeExactICmpRegion(Pred, Other: RA);
11206 if (ExactCR.isFullSet())
11207 return TrivialCase(true);
11208 if (ExactCR.isEmptySet())
11209 return TrivialCase(false);
11210
11211 APInt NewRHS;
11212 CmpInst::Predicate NewPred;
11213 if (ExactCR.getEquivalentICmp(Pred&: NewPred, RHS&: NewRHS) &&
11214 ICmpInst::isEquality(P: NewPred)) {
11215 // We were able to convert an inequality to an equality.
11216 Pred = NewPred;
11217 RHS = getConstant(Val: NewRHS);
11218 Changed = SimplifiedByConstantRange = true;
11219 }
11220 }
11221
11222 if (!SimplifiedByConstantRange) {
11223 switch (Pred) {
11224 default:
11225 break;
11226 case ICmpInst::ICMP_EQ:
11227 case ICmpInst::ICMP_NE:
11228 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
11229 if (RA.isZero() && MatchBinarySub(S: LHS, LHS, RHS))
11230 Changed = true;
11231 break;
11232
11233 // The "Should have been caught earlier!" messages refer to the fact
11234 // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above
11235 // should have fired on the corresponding cases, and canonicalized the
11236 // check to trivial case.
11237
11238 case ICmpInst::ICMP_UGE:
11239 assert(!RA.isMinValue() && "Should have been caught earlier!");
11240 Pred = ICmpInst::ICMP_UGT;
11241 RHS = getConstant(Val: RA - 1);
11242 Changed = true;
11243 break;
11244 case ICmpInst::ICMP_ULE:
11245 assert(!RA.isMaxValue() && "Should have been caught earlier!");
11246 Pred = ICmpInst::ICMP_ULT;
11247 RHS = getConstant(Val: RA + 1);
11248 Changed = true;
11249 break;
11250 case ICmpInst::ICMP_SGE:
11251 assert(!RA.isMinSignedValue() && "Should have been caught earlier!");
11252 Pred = ICmpInst::ICMP_SGT;
11253 RHS = getConstant(Val: RA - 1);
11254 Changed = true;
11255 break;
11256 case ICmpInst::ICMP_SLE:
11257 assert(!RA.isMaxSignedValue() && "Should have been caught earlier!");
11258 Pred = ICmpInst::ICMP_SLT;
11259 RHS = getConstant(Val: RA + 1);
11260 Changed = true;
11261 break;
11262 }
11263 }
11264 }
11265
11266 // Check for obvious equality.
11267 if (HasSameValue(A: LHS, B: RHS)) {
11268 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
11269 return TrivialCase(true);
11270 if (ICmpInst::isFalseWhenEqual(predicate: Pred))
11271 return TrivialCase(false);
11272 }
11273
11274 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
11275 // adding or subtracting 1 from one of the operands.
11276 switch (Pred) {
11277 case ICmpInst::ICMP_SLE:
11278 if (!getSignedRangeMax(S: RHS).isMaxSignedValue()) {
11279 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11280 Flags: SCEV::FlagNSW);
11281 Pred = ICmpInst::ICMP_SLT;
11282 Changed = true;
11283 } else if (!getSignedRangeMin(S: LHS).isMinSignedValue()) {
11284 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS,
11285 Flags: SCEV::FlagNSW);
11286 Pred = ICmpInst::ICMP_SLT;
11287 Changed = true;
11288 }
11289 break;
11290 case ICmpInst::ICMP_SGE:
11291 if (!getSignedRangeMin(S: RHS).isMinSignedValue()) {
11292 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS,
11293 Flags: SCEV::FlagNSW);
11294 Pred = ICmpInst::ICMP_SGT;
11295 Changed = true;
11296 } else if (!getSignedRangeMax(S: LHS).isMaxSignedValue()) {
11297 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11298 Flags: SCEV::FlagNSW);
11299 Pred = ICmpInst::ICMP_SGT;
11300 Changed = true;
11301 }
11302 break;
11303 case ICmpInst::ICMP_ULE:
11304 if (!getUnsignedRangeMax(S: RHS).isMaxValue()) {
11305 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11306 Flags: SCEV::FlagNUW);
11307 Pred = ICmpInst::ICMP_ULT;
11308 Changed = true;
11309 } else if (!getUnsignedRangeMin(S: LHS).isMinValue()) {
11310 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS);
11311 Pred = ICmpInst::ICMP_ULT;
11312 Changed = true;
11313 }
11314 break;
11315 case ICmpInst::ICMP_UGE:
11316 // If RHS is an op we can fold the -1, try that first.
11317 // Otherwise prefer LHS to preserve the nuw flag.
11318 if ((isa<SCEVConstant>(Val: RHS) ||
11319 (isa<SCEVAddExpr, SCEVAddRecExpr>(Val: RHS) &&
11320 isa<SCEVConstant>(Val: cast<SCEVNAryExpr>(Val&: RHS)->getOperand(i: 0)))) &&
11321 !getUnsignedRangeMin(S: RHS).isMinValue()) {
11322 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11323 Pred = ICmpInst::ICMP_UGT;
11324 Changed = true;
11325 } else if (!getUnsignedRangeMax(S: LHS).isMaxValue()) {
11326 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11327 Flags: SCEV::FlagNUW);
11328 Pred = ICmpInst::ICMP_UGT;
11329 Changed = true;
11330 } else if (!getUnsignedRangeMin(S: RHS).isMinValue()) {
11331 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11332 Pred = ICmpInst::ICMP_UGT;
11333 Changed = true;
11334 }
11335 break;
11336 default:
11337 break;
11338 }
11339
11340 // TODO: More simplifications are possible here.
11341
11342 // Recursively simplify until we either hit a recursion limit or nothing
11343 // changes.
11344 if (Changed)
11345 (void)SimplifyICmpOperands(Pred, LHS, RHS, Depth: Depth + 1);
11346
11347 return Changed;
11348}
11349
11350bool ScalarEvolution::isKnownNegative(const SCEV *S) {
11351 return getSignedRangeMax(S).isNegative();
11352}
11353
11354bool ScalarEvolution::isKnownPositive(const SCEV *S) {
11355 return getSignedRangeMin(S).isStrictlyPositive();
11356}
11357
11358bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
11359 return !getSignedRangeMin(S).isNegative();
11360}
11361
11362bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
11363 return !getSignedRangeMax(S).isStrictlyPositive();
11364}
11365
11366bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
11367 // Query push down for cases where the unsigned range is
11368 // less than sufficient.
11369 if (const auto *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
11370 return isKnownNonZero(S: SExt->getOperand(i: 0));
11371 return getUnsignedRangeMin(S) != 0;
11372}
11373
11374bool ScalarEvolution::isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero,
11375 bool OrNegative) {
11376 auto NonRecursive = [OrNegative](const SCEV *S) {
11377 if (auto *C = dyn_cast<SCEVConstant>(Val: S))
11378 return C->getAPInt().isPowerOf2() ||
11379 (OrNegative && C->getAPInt().isNegatedPowerOf2());
11380
11381 // vscale is a power-of-two.
11382 return isa<SCEVVScale>(Val: S);
11383 };
11384
11385 if (NonRecursive(S))
11386 return true;
11387
11388 auto *Mul = dyn_cast<SCEVMulExpr>(Val: S);
11389 if (!Mul)
11390 return false;
11391 return all_of(Range: Mul->operands(), P: NonRecursive) && (OrZero || isKnownNonZero(S));
11392}
11393
11394bool ScalarEvolution::isKnownMultipleOf(
11395 const SCEV *S, uint64_t M,
11396 SmallVectorImpl<const SCEVPredicate *> &Assumptions) {
11397 if (M == 0)
11398 return false;
11399 if (M == 1)
11400 return true;
11401
11402 // Recursively check AddRec operands. An AddRecExpr S is a multiple of M if S
11403 // starts with a multiple of M and at every iteration step S only adds
11404 // multiples of M.
11405 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
11406 return isKnownMultipleOf(S: AddRec->getStart(), M, Assumptions) &&
11407 isKnownMultipleOf(S: AddRec->getStepRecurrence(SE&: *this), M, Assumptions);
11408
11409 // For a constant, check that "S % M == 0".
11410 if (auto *Cst = dyn_cast<SCEVConstant>(Val: S)) {
11411 APInt C = Cst->getAPInt();
11412 return C.urem(RHS: M) == 0;
11413 }
11414
11415 // TODO: Also check other SCEV expressions, i.e., SCEVAddRecExpr, etc.
11416
11417 // Basic tests have failed.
11418 // Check "S % M == 0" at compile time and record runtime Assumptions.
11419 auto *STy = dyn_cast<IntegerType>(Val: S->getType());
11420 const SCEV *SmodM =
11421 getURemExpr(LHS: S, RHS: getConstant(V: ConstantInt::get(Ty: STy, V: M, IsSigned: false)));
11422 const SCEV *Zero = getZero(Ty: STy);
11423
11424 // Check whether "S % M == 0" is known at compile time.
11425 if (isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero))
11426 return true;
11427
11428 // Check whether "S % M != 0" is known at compile time.
11429 if (isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: SmodM, RHS: Zero))
11430 return false;
11431
11432 const SCEVPredicate *P = getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero);
11433
11434 // Detect redundant predicates.
11435 for (auto *A : Assumptions)
11436 if (A->implies(N: P, SE&: *this))
11437 return true;
11438
11439 // Only record non-redundant predicates.
11440 Assumptions.push_back(Elt: P);
11441 return true;
11442}
11443
11444bool ScalarEvolution::haveSameSign(const SCEV *S1, const SCEV *S2) {
11445 return ((isKnownNonNegative(S: S1) && isKnownNonNegative(S: S2)) ||
11446 (isKnownNegative(S: S1) && isKnownNegative(S: S2)));
11447}
11448
11449std::pair<const SCEV *, const SCEV *>
11450ScalarEvolution::SplitIntoInitAndPostInc(const Loop *L, const SCEV *S) {
11451 // Compute SCEV on entry of loop L.
11452 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, SE&: *this);
11453 if (Start == getCouldNotCompute())
11454 return { Start, Start };
11455 // Compute post increment SCEV for loop L.
11456 const SCEV *PostInc = SCEVPostIncRewriter::rewrite(S, L, SE&: *this);
11457 assert(PostInc != getCouldNotCompute() && "Unexpected could not compute");
11458 return { Start, PostInc };
11459}
11460
11461bool ScalarEvolution::isKnownViaInduction(CmpPredicate Pred, SCEVUse LHS,
11462 SCEVUse RHS) {
11463 // First collect all loops.
11464 SmallPtrSet<const Loop *, 8> LoopsUsed;
11465 getUsedLoops(S: LHS, LoopsUsed);
11466 getUsedLoops(S: RHS, LoopsUsed);
11467
11468 if (LoopsUsed.empty())
11469 return false;
11470
11471 // Domination relationship must be a linear order on collected loops.
11472#ifndef NDEBUG
11473 for (const auto *L1 : LoopsUsed)
11474 for (const auto *L2 : LoopsUsed)
11475 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||
11476 DT.dominates(L2->getHeader(), L1->getHeader())) &&
11477 "Domination relationship is not a linear order");
11478#endif
11479
11480 const Loop *MDL =
11481 *llvm::max_element(Range&: LoopsUsed, C: [&](const Loop *L1, const Loop *L2) {
11482 return DT.properlyDominates(A: L1->getHeader(), B: L2->getHeader());
11483 });
11484
11485 // Get init and post increment value for LHS.
11486 auto SplitLHS = SplitIntoInitAndPostInc(L: MDL, S: LHS);
11487 // if LHS contains unknown non-invariant SCEV then bail out.
11488 if (SplitLHS.first == getCouldNotCompute())
11489 return false;
11490 assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
11491 // Get init and post increment value for RHS.
11492 auto SplitRHS = SplitIntoInitAndPostInc(L: MDL, S: RHS);
11493 // if RHS contains unknown non-invariant SCEV then bail out.
11494 if (SplitRHS.first == getCouldNotCompute())
11495 return false;
11496 assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
11497 // It is possible that init SCEV contains an invariant load but it does
11498 // not dominate MDL and is not available at MDL loop entry, so we should
11499 // check it here.
11500 if (!isAvailableAtLoopEntry(S: SplitLHS.first, L: MDL) ||
11501 !isAvailableAtLoopEntry(S: SplitRHS.first, L: MDL))
11502 return false;
11503
11504 // It seems backedge guard check is faster than entry one so in some cases
11505 // it can speed up whole estimation by short circuit
11506 return isLoopBackedgeGuardedByCond(L: MDL, Pred, LHS: SplitLHS.second,
11507 RHS: SplitRHS.second) &&
11508 isLoopEntryGuardedByCond(L: MDL, Pred, LHS: SplitLHS.first, RHS: SplitRHS.first);
11509}
11510
11511bool ScalarEvolution::isKnownPredicate(CmpPredicate Pred, SCEVUse LHS,
11512 SCEVUse RHS) {
11513 // Canonicalize the inputs first.
11514 (void)SimplifyICmpOperands(Pred, LHS, RHS);
11515
11516 if (isKnownViaInduction(Pred, LHS, RHS))
11517 return true;
11518
11519 if (isKnownPredicateViaSplitting(Pred, LHS, RHS))
11520 return true;
11521
11522 // Otherwise see what can be done with some simple reasoning.
11523 return isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
11524}
11525
11526std::optional<bool> ScalarEvolution::evaluatePredicate(CmpPredicate Pred,
11527 const SCEV *LHS,
11528 const SCEV *RHS) {
11529 if (isKnownPredicate(Pred, LHS, RHS))
11530 return true;
11531 if (isKnownPredicate(Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11532 return false;
11533 return std::nullopt;
11534}
11535
11536bool ScalarEvolution::isKnownPredicateAt(CmpPredicate Pred, const SCEV *LHS,
11537 const SCEV *RHS,
11538 const Instruction *CtxI) {
11539 // TODO: Analyze guards and assumes from Context's block.
11540 return isKnownPredicate(Pred, LHS, RHS) ||
11541 isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS);
11542}
11543
11544std::optional<bool>
11545ScalarEvolution::evaluatePredicateAt(CmpPredicate Pred, const SCEV *LHS,
11546 const SCEV *RHS, const Instruction *CtxI) {
11547 std::optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
11548 if (KnownWithoutContext)
11549 return KnownWithoutContext;
11550
11551 if (isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS))
11552 return true;
11553 if (isBasicBlockEntryGuardedByCond(
11554 BB: CtxI->getParent(), Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11555 return false;
11556 return std::nullopt;
11557}
11558
11559bool ScalarEvolution::isKnownOnEveryIteration(CmpPredicate Pred,
11560 const SCEVAddRecExpr *LHS,
11561 const SCEV *RHS) {
11562 const Loop *L = LHS->getLoop();
11563 return isLoopEntryGuardedByCond(L, Pred, LHS: LHS->getStart(), RHS) &&
11564 isLoopBackedgeGuardedByCond(L, Pred, LHS: LHS->getPostIncExpr(SE&: *this), RHS);
11565}
11566
11567std::optional<ScalarEvolution::MonotonicPredicateType>
11568ScalarEvolution::getMonotonicPredicateType(const SCEVAddRecExpr *LHS,
11569 ICmpInst::Predicate Pred) {
11570 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
11571
11572#ifndef NDEBUG
11573 // Verify an invariant: inverting the predicate should turn a monotonically
11574 // increasing change to a monotonically decreasing one, and vice versa.
11575 if (Result) {
11576 auto ResultSwapped =
11577 getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred));
11578
11579 assert(*ResultSwapped != *Result &&
11580 "monotonicity should flip as we flip the predicate");
11581 }
11582#endif
11583
11584 return Result;
11585}
11586
11587std::optional<ScalarEvolution::MonotonicPredicateType>
11588ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
11589 ICmpInst::Predicate Pred) {
11590 // A zero step value for LHS means the induction variable is essentially a
11591 // loop invariant value. We don't really depend on the predicate actually
11592 // flipping from false to true (for increasing predicates, and the other way
11593 // around for decreasing predicates), all we care about is that *if* the
11594 // predicate changes then it only changes from false to true.
11595 //
11596 // A zero step value in itself is not very useful, but there may be places
11597 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
11598 // as general as possible.
11599
11600 // Only handle LE/LT/GE/GT predicates.
11601 if (!ICmpInst::isRelational(P: Pred))
11602 return std::nullopt;
11603
11604 bool IsGreater = ICmpInst::isGE(P: Pred) || ICmpInst::isGT(P: Pred);
11605 assert((IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) &&
11606 "Should be greater or less!");
11607
11608 // Check that AR does not wrap.
11609 if (ICmpInst::isUnsigned(Pred)) {
11610 if (!LHS->hasNoUnsignedWrap())
11611 return std::nullopt;
11612 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11613 }
11614 assert(ICmpInst::isSigned(Pred) &&
11615 "Relational predicate is either signed or unsigned!");
11616 if (!LHS->hasNoSignedWrap())
11617 return std::nullopt;
11618
11619 const SCEV *Step = LHS->getStepRecurrence(SE&: *this);
11620
11621 if (isKnownNonNegative(S: Step))
11622 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11623
11624 if (isKnownNonPositive(S: Step))
11625 return !IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11626
11627 return std::nullopt;
11628}
11629
11630std::optional<ScalarEvolution::LoopInvariantPredicate>
11631ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
11632 const SCEV *RHS, const Loop *L,
11633 const Instruction *CtxI) {
11634 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11635 if (!isLoopInvariant(S: RHS, L)) {
11636 if (!isLoopInvariant(S: LHS, L))
11637 return std::nullopt;
11638
11639 std::swap(a&: LHS, b&: RHS);
11640 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11641 }
11642
11643 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11644 if (!ArLHS || ArLHS->getLoop() != L)
11645 return std::nullopt;
11646
11647 auto MonotonicType = getMonotonicPredicateType(LHS: ArLHS, Pred);
11648 if (!MonotonicType)
11649 return std::nullopt;
11650 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
11651 // true as the loop iterates, and the backedge is control dependent on
11652 // "ArLHS `Pred` RHS" == true then we can reason as follows:
11653 //
11654 // * if the predicate was false in the first iteration then the predicate
11655 // is never evaluated again, since the loop exits without taking the
11656 // backedge.
11657 // * if the predicate was true in the first iteration then it will
11658 // continue to be true for all future iterations since it is
11659 // monotonically increasing.
11660 //
11661 // For both the above possibilities, we can replace the loop varying
11662 // predicate with its value on the first iteration of the loop (which is
11663 // loop invariant).
11664 //
11665 // A similar reasoning applies for a monotonically decreasing predicate, by
11666 // replacing true with false and false with true in the above two bullets.
11667 bool Increasing = *MonotonicType == ScalarEvolution::MonotonicallyIncreasing;
11668 auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
11669
11670 if (isLoopBackedgeGuardedByCond(L, Pred: P, LHS, RHS))
11671 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11672 RHS);
11673
11674 if (!CtxI)
11675 return std::nullopt;
11676 // Try to prove via context.
11677 // TODO: Support other cases.
11678 switch (Pred) {
11679 default:
11680 break;
11681 case ICmpInst::ICMP_ULE:
11682 case ICmpInst::ICMP_ULT: {
11683 assert(ArLHS->hasNoUnsignedWrap() && "Is a requirement of monotonicity!");
11684 // Given preconditions
11685 // (1) ArLHS does not cross the border of positive and negative parts of
11686 // range because of:
11687 // - Positive step; (TODO: lift this limitation)
11688 // - nuw - does not cross zero boundary;
11689 // - nsw - does not cross SINT_MAX boundary;
11690 // (2) ArLHS <s RHS
11691 // (3) RHS >=s 0
11692 // we can replace the loop variant ArLHS <u RHS condition with loop
11693 // invariant Start(ArLHS) <u RHS.
11694 //
11695 // Because of (1) there are two options:
11696 // - ArLHS is always negative. It means that ArLHS <u RHS is always false;
11697 // - ArLHS is always non-negative. Because of (3) RHS is also non-negative.
11698 // It means that ArLHS <s RHS <=> ArLHS <u RHS.
11699 // Because of (2) ArLHS <u RHS is trivially true.
11700 // All together it means that ArLHS <u RHS <=> Start(ArLHS) >=s 0.
11701 // We can strengthen this to Start(ArLHS) <u RHS.
11702 auto SignFlippedPred = ICmpInst::getFlippedSignednessPredicate(Pred);
11703 if (ArLHS->hasNoSignedWrap() && ArLHS->isAffine() &&
11704 isKnownPositive(S: ArLHS->getStepRecurrence(SE&: *this)) &&
11705 isKnownNonNegative(S: RHS) &&
11706 isKnownPredicateAt(Pred: SignFlippedPred, LHS: ArLHS, RHS, CtxI))
11707 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11708 RHS);
11709 }
11710 }
11711
11712 return std::nullopt;
11713}
11714
11715std::optional<ScalarEvolution::LoopInvariantPredicate>
11716ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations(
11717 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11718 const Instruction *CtxI, const SCEV *MaxIter) {
11719 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11720 Pred, LHS, RHS, L, CtxI, MaxIter))
11721 return LIP;
11722 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: MaxIter))
11723 // Number of iterations expressed as UMIN isn't always great for expressing
11724 // the value on the last iteration. If the straightforward approach didn't
11725 // work, try the following trick: if the a predicate is invariant for X, it
11726 // is also invariant for umin(X, ...). So try to find something that works
11727 // among subexpressions of MaxIter expressed as umin.
11728 for (SCEVUse Op : UMin->operands())
11729 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11730 Pred, LHS, RHS, L, CtxI, MaxIter: Op))
11731 return LIP;
11732 return std::nullopt;
11733}
11734
11735std::optional<ScalarEvolution::LoopInvariantPredicate>
11736ScalarEvolution::getLoopInvariantExitCondDuringFirstIterationsImpl(
11737 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11738 const Instruction *CtxI, const SCEV *MaxIter) {
11739 // Try to prove the following set of facts:
11740 // - The predicate is monotonic in the iteration space.
11741 // - If the check does not fail on the 1st iteration:
11742 // - No overflow will happen during first MaxIter iterations;
11743 // - It will not fail on the MaxIter'th iteration.
11744 // If the check does fail on the 1st iteration, we leave the loop and no
11745 // other checks matter.
11746
11747 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11748 if (!isLoopInvariant(S: RHS, L)) {
11749 if (!isLoopInvariant(S: LHS, L))
11750 return std::nullopt;
11751
11752 std::swap(a&: LHS, b&: RHS);
11753 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11754 }
11755
11756 auto *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11757 if (!AR || AR->getLoop() != L)
11758 return std::nullopt;
11759
11760 // Even if both are valid, we need to consistently chose the unsigned or the
11761 // signed predicate below, not mixtures of both. For now, prefer the unsigned
11762 // predicate.
11763 Pred = Pred.dropSameSign();
11764
11765 // The predicate must be relational (i.e. <, <=, >=, >).
11766 if (!ICmpInst::isRelational(P: Pred))
11767 return std::nullopt;
11768
11769 // TODO: Support steps other than +/- 1.
11770 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
11771 auto *One = getOne(Ty: Step->getType());
11772 auto *MinusOne = getNegativeSCEV(V: One);
11773 if (Step != One && Step != MinusOne)
11774 return std::nullopt;
11775
11776 // Type mismatch here means that MaxIter is potentially larger than max
11777 // unsigned value in start type, which mean we cannot prove no wrap for the
11778 // indvar.
11779 if (AR->getType() != MaxIter->getType())
11780 return std::nullopt;
11781
11782 // Value of IV on suggested last iteration.
11783 const SCEV *Last = AR->evaluateAtIteration(It: MaxIter, SE&: *this);
11784 // Does it still meet the requirement?
11785 if (!isLoopBackedgeGuardedByCond(L, Pred, LHS: Last, RHS))
11786 return std::nullopt;
11787 // Because step is +/- 1 and MaxIter has same type as Start (i.e. it does
11788 // not exceed max unsigned value of this type), this effectively proves
11789 // that there is no wrap during the iteration. To prove that there is no
11790 // signed/unsigned wrap, we need to check that
11791 // Start <= Last for step = 1 or Start >= Last for step = -1.
11792 ICmpInst::Predicate NoOverflowPred =
11793 CmpInst::isSigned(Pred) ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
11794 if (Step == MinusOne)
11795 NoOverflowPred = ICmpInst::getSwappedPredicate(pred: NoOverflowPred);
11796 const SCEV *Start = AR->getStart();
11797 if (!isKnownPredicateAt(Pred: NoOverflowPred, LHS: Start, RHS: Last, CtxI))
11798 return std::nullopt;
11799
11800 // Everything is fine.
11801 return ScalarEvolution::LoopInvariantPredicate(Pred, Start, RHS);
11802}
11803
11804bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
11805 SCEVUse LHS,
11806 SCEVUse RHS) {
11807 if (HasSameValue(A: LHS, B: RHS))
11808 return ICmpInst::isTrueWhenEqual(predicate: Pred);
11809
11810 auto CheckRange = [&](bool IsSigned) {
11811 auto RangeLHS = IsSigned ? getSignedRange(S: LHS) : getUnsignedRange(S: LHS);
11812 auto RangeRHS = IsSigned ? getSignedRange(S: RHS) : getUnsignedRange(S: RHS);
11813 return RangeLHS.icmp(Pred, Other: RangeRHS);
11814 };
11815
11816 // The check at the top of the function catches the case where the values are
11817 // known to be equal.
11818 if (Pred == CmpInst::ICMP_EQ)
11819 return false;
11820
11821 if (Pred == CmpInst::ICMP_NE) {
11822 if (CheckRange(true) || CheckRange(false))
11823 return true;
11824 auto *Diff = getMinusSCEV(LHS, RHS);
11825 return !isa<SCEVCouldNotCompute>(Val: Diff) && isKnownNonZero(S: Diff);
11826 }
11827
11828 return CheckRange(CmpInst::isSigned(Pred));
11829}
11830
11831bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
11832 SCEVUse LHS, SCEVUse RHS) {
11833 // Match X to (A + C1)<ExpectedFlags> and Y to (A + C2)<ExpectedFlags>, where
11834 // C1 and C2 are constant integers. If either X or Y are not add expressions,
11835 // consider them as X + 0 and Y + 0 respectively. C1 and C2 are returned via
11836 // OutC1 and OutC2.
11837 auto MatchBinaryAddToConst = [this](SCEVUse X, SCEVUse Y, APInt &OutC1,
11838 APInt &OutC2,
11839 SCEV::NoWrapFlags ExpectedFlags) {
11840 SCEVUse XNonConstOp, XConstOp;
11841 SCEVUse YNonConstOp, YConstOp;
11842 SCEV::NoWrapFlags XFlagsPresent;
11843 SCEV::NoWrapFlags YFlagsPresent;
11844
11845 if (!splitBinaryAdd(Expr: X, L&: XConstOp, R&: XNonConstOp, Flags&: XFlagsPresent)) {
11846 XConstOp = getZero(Ty: X->getType());
11847 XNonConstOp = X;
11848 XFlagsPresent = ExpectedFlags;
11849 }
11850 if (!isa<SCEVConstant>(Val: XConstOp))
11851 return false;
11852
11853 if (!splitBinaryAdd(Expr: Y, L&: YConstOp, R&: YNonConstOp, Flags&: YFlagsPresent)) {
11854 YConstOp = getZero(Ty: Y->getType());
11855 YNonConstOp = Y;
11856 YFlagsPresent = ExpectedFlags;
11857 }
11858
11859 if (YNonConstOp != XNonConstOp)
11860 return false;
11861
11862 if (!isa<SCEVConstant>(Val: YConstOp))
11863 return false;
11864
11865 // When matching ADDs with NUW flags (and unsigned predicates), only the
11866 // second ADD (with the larger constant) requires NUW.
11867 if ((YFlagsPresent & ExpectedFlags) != ExpectedFlags)
11868 return false;
11869 if (ExpectedFlags != SCEV::FlagNUW &&
11870 (XFlagsPresent & ExpectedFlags) != ExpectedFlags) {
11871 return false;
11872 }
11873
11874 OutC1 = cast<SCEVConstant>(Val&: XConstOp)->getAPInt();
11875 OutC2 = cast<SCEVConstant>(Val&: YConstOp)->getAPInt();
11876
11877 return true;
11878 };
11879
11880 APInt C1;
11881 APInt C2;
11882
11883 switch (Pred) {
11884 default:
11885 break;
11886
11887 case ICmpInst::ICMP_SGE:
11888 std::swap(a&: LHS, b&: RHS);
11889 [[fallthrough]];
11890 case ICmpInst::ICMP_SLE:
11891 // (X + C1)<nsw> s<= (X + C2)<nsw> if C1 s<= C2.
11892 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.sle(RHS: C2))
11893 return true;
11894
11895 break;
11896
11897 case ICmpInst::ICMP_SGT:
11898 std::swap(a&: LHS, b&: RHS);
11899 [[fallthrough]];
11900 case ICmpInst::ICMP_SLT:
11901 // (X + C1)<nsw> s< (X + C2)<nsw> if C1 s< C2.
11902 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.slt(RHS: C2))
11903 return true;
11904
11905 break;
11906
11907 case ICmpInst::ICMP_UGE:
11908 std::swap(a&: LHS, b&: RHS);
11909 [[fallthrough]];
11910 case ICmpInst::ICMP_ULE:
11911 // (X + C1) u<= (X + C2)<nuw> for C1 u<= C2.
11912 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(RHS: C2))
11913 return true;
11914
11915 break;
11916
11917 case ICmpInst::ICMP_UGT:
11918 std::swap(a&: LHS, b&: RHS);
11919 [[fallthrough]];
11920 case ICmpInst::ICMP_ULT:
11921 // (X + C1) u< (X + C2)<nuw> if C1 u< C2.
11922 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(RHS: C2))
11923 return true;
11924 break;
11925 }
11926
11927 return false;
11928}
11929
11930bool ScalarEvolution::isKnownPredicateViaSplitting(CmpPredicate Pred,
11931 SCEVUse LHS, SCEVUse RHS) {
11932 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate)
11933 return false;
11934
11935 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on
11936 // the stack can result in exponential time complexity.
11937 SaveAndRestore Restore(ProvingSplitPredicate, true);
11938
11939 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L
11940 //
11941 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use
11942 // isKnownPredicate. isKnownPredicate is more powerful, but also more
11943 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the
11944 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to
11945 // use isKnownPredicate later if needed.
11946 return isKnownNonNegative(S: RHS) &&
11947 isKnownPredicate(Pred: CmpInst::ICMP_SGE, LHS, RHS: getZero(Ty: LHS->getType())) &&
11948 isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS, RHS);
11949}
11950
11951bool ScalarEvolution::isImpliedViaGuard(const BasicBlock *BB, CmpPredicate Pred,
11952 const SCEV *LHS, const SCEV *RHS) {
11953 // No need to even try if we know the module has no guards.
11954 if (!HasGuards)
11955 return false;
11956
11957 return any_of(Range: *BB, P: [&](const Instruction &I) {
11958 using namespace llvm::PatternMatch;
11959
11960 Value *Condition;
11961 return match(V: &I, P: m_Intrinsic<Intrinsic::experimental_guard>(
11962 Ops: m_Value(V&: Condition))) &&
11963 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse: false);
11964 });
11965}
11966
11967/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
11968/// protected by a conditional between LHS and RHS. This is used to
11969/// to eliminate casts.
11970bool ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
11971 CmpPredicate Pred,
11972 const SCEV *LHS,
11973 const SCEV *RHS) {
11974 // Interpret a null as meaning no loop, where there is obviously no guard
11975 // (interprocedural conditions notwithstanding). Do not bother about
11976 // unreachable loops.
11977 if (!L || !DT.isReachableFromEntry(A: L->getHeader()))
11978 return true;
11979
11980 if (VerifyIR)
11981 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) &&
11982 "This cannot be done on broken IR!");
11983
11984
11985 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11986 return true;
11987
11988 BasicBlock *Latch = L->getLoopLatch();
11989 if (!Latch)
11990 return false;
11991
11992 CondBrInst *LoopContinuePredicate =
11993 dyn_cast<CondBrInst>(Val: Latch->getTerminator());
11994 if (LoopContinuePredicate &&
11995 isImpliedCond(Pred, LHS, RHS, FoundCondValue: LoopContinuePredicate->getCondition(),
11996 Inverse: LoopContinuePredicate->getSuccessor(i: 0) != L->getHeader()))
11997 return true;
11998
11999 // We don't want more than one activation of the following loops on the stack
12000 // -- that can lead to O(n!) time complexity.
12001 if (WalkingBEDominatingConds)
12002 return false;
12003
12004 SaveAndRestore ClearOnExit(WalkingBEDominatingConds, true);
12005
12006 // See if we can exploit a trip count to prove the predicate.
12007 const auto &BETakenInfo = getBackedgeTakenInfo(L);
12008 const SCEV *LatchBECount = BETakenInfo.getExact(ExitingBlock: Latch, SE: this);
12009 if (LatchBECount != getCouldNotCompute()) {
12010 // We know that Latch branches back to the loop header exactly
12011 // LatchBECount times. This means the backdege condition at Latch is
12012 // equivalent to "{0,+,1} u< LatchBECount".
12013 Type *Ty = LatchBECount->getType();
12014 auto NoWrapFlags = SCEV::NoWrapFlags(SCEV::FlagNUW | SCEV::FlagNW);
12015 const SCEV *LoopCounter =
12016 getAddRecExpr(Start: getZero(Ty), Step: getOne(Ty), L, Flags: NoWrapFlags);
12017 if (isImpliedCond(Pred, LHS, RHS, FoundPred: ICmpInst::ICMP_ULT, FoundLHS: LoopCounter,
12018 FoundRHS: LatchBECount))
12019 return true;
12020 }
12021
12022 // Check conditions due to any @llvm.assume intrinsics.
12023 for (auto &AssumeVH : AC.assumptions()) {
12024 if (!AssumeVH)
12025 continue;
12026 auto *CI = cast<CallInst>(Val&: AssumeVH);
12027 if (!DT.dominates(Def: CI, User: Latch->getTerminator()))
12028 continue;
12029
12030 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: CI->getArgOperand(i: 0), Inverse: false))
12031 return true;
12032 }
12033
12034 if (isImpliedViaGuard(BB: Latch, Pred, LHS, RHS))
12035 return true;
12036
12037 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
12038 DTN != HeaderDTN; DTN = DTN->getIDom()) {
12039 assert(DTN && "should reach the loop header before reaching the root!");
12040
12041 BasicBlock *BB = DTN->getBlock();
12042 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
12043 return true;
12044
12045 BasicBlock *PBB = BB->getSinglePredecessor();
12046 if (!PBB)
12047 continue;
12048
12049 CondBrInst *ContBr = dyn_cast<CondBrInst>(Val: PBB->getTerminator());
12050 if (!ContBr || ContBr->getSuccessor(i: 0) == ContBr->getSuccessor(i: 1))
12051 continue;
12052
12053 // If we have an edge `E` within the loop body that dominates the only
12054 // latch, the condition guarding `E` also guards the backedge. This
12055 // reasoning works only for loops with a single latch.
12056 // We're constructively (and conservatively) enumerating edges within the
12057 // loop body that dominate the latch. The dominator tree better agree
12058 // with us on this:
12059 assert(DT.dominates(BasicBlockEdge(PBB, BB), Latch) && "should be!");
12060 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: ContBr->getCondition(),
12061 Inverse: BB != ContBr->getSuccessor(i: 0)))
12062 return true;
12063 }
12064
12065 return false;
12066}
12067
12068bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
12069 CmpPredicate Pred,
12070 const SCEV *LHS,
12071 const SCEV *RHS) {
12072 // Do not bother proving facts for unreachable code.
12073 if (!DT.isReachableFromEntry(A: BB))
12074 return true;
12075 if (VerifyIR)
12076 assert(!verifyFunction(*BB->getParent(), &dbgs()) &&
12077 "This cannot be done on broken IR!");
12078
12079 // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
12080 // the facts (a >= b && a != b) separately. A typical situation is when the
12081 // non-strict comparison is known from ranges and non-equality is known from
12082 // dominating predicates. If we are proving strict comparison, we always try
12083 // to prove non-equality and non-strict comparison separately.
12084 CmpPredicate NonStrictPredicate = ICmpInst::getNonStrictCmpPredicate(Pred);
12085 const bool ProvingStrictComparison =
12086 Pred != NonStrictPredicate.dropSameSign();
12087 bool ProvedNonStrictComparison = false;
12088 bool ProvedNonEquality = false;
12089
12090 auto SplitAndProve = [&](std::function<bool(CmpPredicate)> Fn) -> bool {
12091 if (!ProvedNonStrictComparison)
12092 ProvedNonStrictComparison = Fn(NonStrictPredicate);
12093 if (!ProvedNonEquality)
12094 ProvedNonEquality = Fn(ICmpInst::ICMP_NE);
12095 if (ProvedNonStrictComparison && ProvedNonEquality)
12096 return true;
12097 return false;
12098 };
12099
12100 if (ProvingStrictComparison) {
12101 auto ProofFn = [&](CmpPredicate P) {
12102 return isKnownViaNonRecursiveReasoning(Pred: P, LHS, RHS);
12103 };
12104 if (SplitAndProve(ProofFn))
12105 return true;
12106 }
12107
12108 // Try to prove (Pred, LHS, RHS) using isImpliedCond.
12109 auto ProveViaCond = [&](const Value *Condition, bool Inverse) {
12110 const Instruction *CtxI = &BB->front();
12111 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI))
12112 return true;
12113 if (ProvingStrictComparison) {
12114 auto ProofFn = [&](CmpPredicate P) {
12115 return isImpliedCond(Pred: P, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI);
12116 };
12117 if (SplitAndProve(ProofFn))
12118 return true;
12119 }
12120 return false;
12121 };
12122
12123 // Starting at the block's predecessor, climb up the predecessor chain, as long
12124 // as there are predecessors that can be found that have unique successors
12125 // leading to the original block.
12126 const Loop *ContainingLoop = LI.getLoopFor(BB);
12127 const BasicBlock *PredBB;
12128 if (ContainingLoop && ContainingLoop->getHeader() == BB)
12129 PredBB = ContainingLoop->getLoopPredecessor();
12130 else
12131 PredBB = BB->getSinglePredecessor();
12132 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
12133 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
12134 const CondBrInst *BlockEntryPredicate =
12135 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
12136 if (!BlockEntryPredicate)
12137 continue;
12138
12139 if (ProveViaCond(BlockEntryPredicate->getCondition(),
12140 BlockEntryPredicate->getSuccessor(i: 0) != Pair.second))
12141 return true;
12142 }
12143
12144 // Check conditions due to any @llvm.assume intrinsics.
12145 for (auto &AssumeVH : AC.assumptions()) {
12146 if (!AssumeVH)
12147 continue;
12148 auto *CI = cast<CallInst>(Val&: AssumeVH);
12149 if (!DT.dominates(Def: CI, BB))
12150 continue;
12151
12152 if (ProveViaCond(CI->getArgOperand(i: 0), false))
12153 return true;
12154 }
12155
12156 // Check conditions due to any @llvm.experimental.guard intrinsics.
12157 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
12158 M: F.getParent(), id: Intrinsic::experimental_guard);
12159 if (GuardDecl)
12160 for (const auto *GU : GuardDecl->users())
12161 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
12162 if (Guard->getFunction() == BB->getParent() && DT.dominates(Def: Guard, BB))
12163 if (ProveViaCond(Guard->getArgOperand(i: 0), false))
12164 return true;
12165 return false;
12166}
12167
12168bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred,
12169 const SCEV *LHS,
12170 const SCEV *RHS) {
12171 // Interpret a null as meaning no loop, where there is obviously no guard
12172 // (interprocedural conditions notwithstanding).
12173 if (!L)
12174 return false;
12175
12176 // Both LHS and RHS must be available at loop entry.
12177 assert(isAvailableAtLoopEntry(LHS, L) &&
12178 "LHS is not available at Loop Entry");
12179 assert(isAvailableAtLoopEntry(RHS, L) &&
12180 "RHS is not available at Loop Entry");
12181
12182 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
12183 return true;
12184
12185 return isBasicBlockEntryGuardedByCond(BB: L->getHeader(), Pred, LHS, RHS);
12186}
12187
12188bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12189 const SCEV *RHS,
12190 const Value *FoundCondValue, bool Inverse,
12191 const Instruction *CtxI) {
12192 // False conditions implies anything. Do not bother analyzing it further.
12193 if (FoundCondValue ==
12194 ConstantInt::getBool(Context&: FoundCondValue->getContext(), V: Inverse))
12195 return true;
12196
12197 if (!PendingLoopPredicates.insert(Ptr: FoundCondValue).second)
12198 return false;
12199
12200 llvm::scope_exit ClearOnExit(
12201 [&]() { PendingLoopPredicates.erase(Ptr: FoundCondValue); });
12202
12203 // Recursively handle And and Or conditions.
12204 const Value *Op0, *Op1;
12205 if (match(V: FoundCondValue, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12206 if (!Inverse)
12207 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12208 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12209 } else if (match(V: FoundCondValue, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12210 if (Inverse)
12211 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12212 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12213 }
12214
12215 const ICmpInst *ICI = dyn_cast<ICmpInst>(Val: FoundCondValue);
12216 if (!ICI) return false;
12217
12218 // Now that we found a conditional branch that dominates the loop or controls
12219 // the loop latch. Check to see if it is the comparison we are looking for.
12220 CmpPredicate FoundPred;
12221 if (Inverse)
12222 FoundPred = ICI->getInverseCmpPredicate();
12223 else
12224 FoundPred = ICI->getCmpPredicate();
12225
12226 const SCEV *FoundLHS = getSCEV(V: ICI->getOperand(i_nocapture: 0));
12227 const SCEV *FoundRHS = getSCEV(V: ICI->getOperand(i_nocapture: 1));
12228
12229 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, Context: CtxI);
12230}
12231
12232bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12233 const SCEV *RHS, CmpPredicate FoundPred,
12234 const SCEV *FoundLHS, const SCEV *FoundRHS,
12235 const Instruction *CtxI) {
12236 // Balance the types.
12237 if (getTypeSizeInBits(Ty: LHS->getType()) <
12238 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12239 // For unsigned and equality predicates, try to prove that both found
12240 // operands fit into narrow unsigned range. If so, try to prove facts in
12241 // narrow types.
12242 if (!CmpInst::isSigned(Pred: FoundPred) && !FoundLHS->getType()->isPointerTy() &&
12243 !FoundRHS->getType()->isPointerTy()) {
12244 auto *NarrowType = LHS->getType();
12245 auto *WideType = FoundLHS->getType();
12246 auto BitWidth = getTypeSizeInBits(Ty: NarrowType);
12247 const SCEV *MaxValue = getZeroExtendExpr(
12248 Op: getConstant(Val: APInt::getMaxValue(numBits: BitWidth)), Ty: WideType);
12249 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundLHS,
12250 RHS: MaxValue) &&
12251 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundRHS,
12252 RHS: MaxValue)) {
12253 const SCEV *TruncFoundLHS = getTruncateExpr(Op: FoundLHS, Ty: NarrowType);
12254 const SCEV *TruncFoundRHS = getTruncateExpr(Op: FoundRHS, Ty: NarrowType);
12255 // We cannot preserve samesign after truncation.
12256 if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred: FoundPred.dropSameSign(),
12257 FoundLHS: TruncFoundLHS, FoundRHS: TruncFoundRHS, CtxI))
12258 return true;
12259 }
12260 }
12261
12262 if (LHS->getType()->isPointerTy() || RHS->getType()->isPointerTy())
12263 return false;
12264 if (CmpInst::isSigned(Pred)) {
12265 LHS = getSignExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12266 RHS = getSignExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12267 } else {
12268 LHS = getZeroExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12269 RHS = getZeroExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12270 }
12271 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
12272 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12273 if (FoundLHS->getType()->isPointerTy() || FoundRHS->getType()->isPointerTy())
12274 return false;
12275 if (CmpInst::isSigned(Pred: FoundPred)) {
12276 FoundLHS = getSignExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12277 FoundRHS = getSignExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12278 } else {
12279 FoundLHS = getZeroExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12280 FoundRHS = getZeroExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12281 }
12282 }
12283 return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS,
12284 FoundRHS, CtxI);
12285}
12286
12287bool ScalarEvolution::isImpliedCondBalancedTypes(
12288 CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS, CmpPredicate FoundPred,
12289 SCEVUse FoundLHS, SCEVUse FoundRHS, const Instruction *CtxI) {
12290 assert(getTypeSizeInBits(LHS->getType()) ==
12291 getTypeSizeInBits(FoundLHS->getType()) &&
12292 "Types should be balanced!");
12293 // Canonicalize the query to match the way instcombine will have
12294 // canonicalized the comparison.
12295 if (SimplifyICmpOperands(Pred, LHS, RHS))
12296 if (LHS == RHS)
12297 return CmpInst::isTrueWhenEqual(predicate: Pred);
12298 if (SimplifyICmpOperands(Pred&: FoundPred, LHS&: FoundLHS, RHS&: FoundRHS))
12299 if (FoundLHS == FoundRHS)
12300 return CmpInst::isFalseWhenEqual(predicate: FoundPred);
12301
12302 // Check to see if we can make the LHS or RHS match.
12303 if (LHS == FoundRHS || RHS == FoundLHS) {
12304 if (isa<SCEVConstant>(Val: RHS)) {
12305 std::swap(a&: FoundLHS, b&: FoundRHS);
12306 FoundPred = ICmpInst::getSwappedCmpPredicate(Pred: FoundPred);
12307 } else {
12308 std::swap(a&: LHS, b&: RHS);
12309 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12310 }
12311 }
12312
12313 // Check whether the found predicate is the same as the desired predicate.
12314 if (auto P = CmpPredicate::getMatching(A: FoundPred, B: Pred))
12315 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12316
12317 // Check whether swapping the found predicate makes it the same as the
12318 // desired predicate.
12319 if (auto P = CmpPredicate::getMatching(
12320 A: ICmpInst::getSwappedCmpPredicate(Pred: FoundPred), B: Pred)) {
12321 // We can write the implication
12322 // 0. LHS Pred RHS <- FoundLHS SwapPred FoundRHS
12323 // using one of the following ways:
12324 // 1. LHS Pred RHS <- FoundRHS Pred FoundLHS
12325 // 2. RHS SwapPred LHS <- FoundLHS SwapPred FoundRHS
12326 // 3. LHS Pred RHS <- ~FoundLHS Pred ~FoundRHS
12327 // 4. ~LHS SwapPred ~RHS <- FoundLHS SwapPred FoundRHS
12328 // Forms 1. and 2. require swapping the operands of one condition. Don't
12329 // do this if it would break canonical constant/addrec ordering.
12330 if (!isa<SCEVConstant>(Val: RHS) && !isa<SCEVAddRecExpr>(Val: LHS))
12331 return isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P), LHS: RHS,
12332 RHS: LHS, FoundLHS, FoundRHS, Context: CtxI);
12333 if (!isa<SCEVConstant>(Val: FoundRHS) && !isa<SCEVAddRecExpr>(Val: FoundLHS))
12334 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: FoundRHS, FoundRHS: FoundLHS, Context: CtxI);
12335
12336 // There's no clear preference between forms 3. and 4., try both. Avoid
12337 // forming getNotSCEV of pointer values as the resulting subtract is
12338 // not legal.
12339 if (!LHS->getType()->isPointerTy() && !RHS->getType()->isPointerTy() &&
12340 isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P),
12341 LHS: getNotSCEV(V: LHS), RHS: getNotSCEV(V: RHS), FoundLHS,
12342 FoundRHS, Context: CtxI))
12343 return true;
12344
12345 if (!FoundLHS->getType()->isPointerTy() &&
12346 !FoundRHS->getType()->isPointerTy() &&
12347 isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: getNotSCEV(V: FoundLHS),
12348 FoundRHS: getNotSCEV(V: FoundRHS), Context: CtxI))
12349 return true;
12350
12351 return false;
12352 }
12353
12354 auto IsSignFlippedPredicate = [](CmpInst::Predicate P1,
12355 CmpInst::Predicate P2) {
12356 assert(P1 != P2 && "Handled earlier!");
12357 return CmpInst::isRelational(P: P2) &&
12358 P1 == ICmpInst::getFlippedSignednessPredicate(Pred: P2);
12359 };
12360 if (IsSignFlippedPredicate(Pred, FoundPred)) {
12361 // Unsigned comparison is the same as signed comparison when both the
12362 // operands are non-negative or negative.
12363 if (haveSameSign(S1: FoundLHS, S2: FoundRHS))
12364 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12365 // Create local copies that we can freely swap and canonicalize our
12366 // conditions to "le/lt".
12367 CmpPredicate CanonicalPred = Pred, CanonicalFoundPred = FoundPred;
12368 const SCEV *CanonicalLHS = LHS, *CanonicalRHS = RHS,
12369 *CanonicalFoundLHS = FoundLHS, *CanonicalFoundRHS = FoundRHS;
12370 if (ICmpInst::isGT(P: CanonicalPred) || ICmpInst::isGE(P: CanonicalPred)) {
12371 CanonicalPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalPred);
12372 CanonicalFoundPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalFoundPred);
12373 std::swap(a&: CanonicalLHS, b&: CanonicalRHS);
12374 std::swap(a&: CanonicalFoundLHS, b&: CanonicalFoundRHS);
12375 }
12376 assert((ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) &&
12377 "Must be!");
12378 assert((ICmpInst::isLT(CanonicalFoundPred) ||
12379 ICmpInst::isLE(CanonicalFoundPred)) &&
12380 "Must be!");
12381 if (ICmpInst::isSigned(Pred: CanonicalPred) && isKnownNonNegative(S: CanonicalRHS))
12382 // Use implication:
12383 // x <u y && y >=s 0 --> x <s y.
12384 // If we can prove the left part, the right part is also proven.
12385 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
12386 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
12387 FoundRHS: CanonicalFoundRHS);
12388 if (ICmpInst::isUnsigned(Pred: CanonicalPred) && isKnownNegative(S: CanonicalRHS))
12389 // Use implication:
12390 // x <s y && y <s 0 --> x <u y.
12391 // If we can prove the left part, the right part is also proven.
12392 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
12393 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
12394 FoundRHS: CanonicalFoundRHS);
12395 }
12396
12397 // Check if we can make progress by sharpening ranges.
12398 if (FoundPred == ICmpInst::ICMP_NE &&
12399 (isa<SCEVConstant>(Val: FoundLHS) || isa<SCEVConstant>(Val: FoundRHS))) {
12400
12401 const SCEVConstant *C = nullptr;
12402 const SCEV *V = nullptr;
12403
12404 if (isa<SCEVConstant>(Val: FoundLHS)) {
12405 C = cast<SCEVConstant>(Val&: FoundLHS);
12406 V = FoundRHS;
12407 } else {
12408 C = cast<SCEVConstant>(Val&: FoundRHS);
12409 V = FoundLHS;
12410 }
12411
12412 // The guarding predicate tells us that C != V. If the known range
12413 // of V is [C, t), we can sharpen the range to [C + 1, t). The
12414 // range we consider has to correspond to same signedness as the
12415 // predicate we're interested in folding.
12416
12417 APInt Min = ICmpInst::isSigned(Pred) ?
12418 getSignedRangeMin(S: V) : getUnsignedRangeMin(S: V);
12419
12420 if (Min == C->getAPInt()) {
12421 // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
12422 // This is true even if (Min + 1) wraps around -- in case of
12423 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
12424
12425 APInt SharperMin = Min + 1;
12426
12427 switch (Pred) {
12428 case ICmpInst::ICMP_SGE:
12429 case ICmpInst::ICMP_UGE:
12430 // We know V `Pred` SharperMin. If this implies LHS `Pred`
12431 // RHS, we're done.
12432 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin),
12433 Context: CtxI))
12434 return true;
12435 [[fallthrough]];
12436
12437 case ICmpInst::ICMP_SGT:
12438 case ICmpInst::ICMP_UGT:
12439 // We know from the range information that (V `Pred` Min ||
12440 // V == Min). We know from the guarding condition that !(V
12441 // == Min). This gives us
12442 //
12443 // V `Pred` Min || V == Min && !(V == Min)
12444 // => V `Pred` Min
12445 //
12446 // If V `Pred` Min implies LHS `Pred` RHS, we're done.
12447
12448 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12449 return true;
12450 break;
12451
12452 // `LHS < RHS` and `LHS <= RHS` are handled in the same way as `RHS > LHS` and `RHS >= LHS` respectively.
12453 case ICmpInst::ICMP_SLE:
12454 case ICmpInst::ICMP_ULE:
12455 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12456 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin), Context: CtxI))
12457 return true;
12458 [[fallthrough]];
12459
12460 case ICmpInst::ICMP_SLT:
12461 case ICmpInst::ICMP_ULT:
12462 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12463 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12464 return true;
12465 break;
12466
12467 default:
12468 // No change
12469 break;
12470 }
12471 }
12472 }
12473
12474 // Check whether the actual condition is beyond sufficient.
12475 if (FoundPred == ICmpInst::ICMP_EQ)
12476 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
12477 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12478 return true;
12479 if (Pred == ICmpInst::ICMP_NE)
12480 if (!ICmpInst::isTrueWhenEqual(predicate: FoundPred))
12481 if (isImpliedCondOperands(Pred: FoundPred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12482 return true;
12483
12484 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS))
12485 return true;
12486
12487 // Otherwise assume the worst.
12488 return false;
12489}
12490
12491bool ScalarEvolution::splitBinaryAdd(SCEVUse Expr, SCEVUse &L, SCEVUse &R,
12492 SCEV::NoWrapFlags &Flags) {
12493 if (!match(U: Expr, P: m_scev_Add(Op0: m_SCEV(V&: L), Op1: m_SCEV(V&: R))))
12494 return false;
12495
12496 Flags = cast<SCEVAddExpr>(Val&: Expr)->getNoWrapFlags();
12497 return true;
12498}
12499
12500std::optional<APInt>
12501ScalarEvolution::computeConstantDifference(const SCEV *More, const SCEV *Less) {
12502 // We avoid subtracting expressions here because this function is usually
12503 // fairly deep in the call stack (i.e. is called many times).
12504
12505 unsigned BW = getTypeSizeInBits(Ty: More->getType());
12506 APInt Diff(BW, 0);
12507 APInt DiffMul(BW, 1);
12508 // Try various simplifications to reduce the difference to a constant. Limit
12509 // the number of allowed simplifications to keep compile-time low.
12510 for (unsigned I = 0; I < 8; ++I) {
12511 if (More == Less)
12512 return Diff;
12513
12514 // Reduce addrecs with identical steps to their start value.
12515 if (isa<SCEVAddRecExpr>(Val: Less) && isa<SCEVAddRecExpr>(Val: More)) {
12516 const auto *LAR = cast<SCEVAddRecExpr>(Val: Less);
12517 const auto *MAR = cast<SCEVAddRecExpr>(Val: More);
12518
12519 if (LAR->getLoop() != MAR->getLoop())
12520 return std::nullopt;
12521
12522 // We look at affine expressions only; not for correctness but to keep
12523 // getStepRecurrence cheap.
12524 if (!LAR->isAffine() || !MAR->isAffine())
12525 return std::nullopt;
12526
12527 if (LAR->getStepRecurrence(SE&: *this) != MAR->getStepRecurrence(SE&: *this))
12528 return std::nullopt;
12529
12530 Less = LAR->getStart();
12531 More = MAR->getStart();
12532 continue;
12533 }
12534
12535 // Try to match a common constant multiply.
12536 auto MatchConstMul =
12537 [](const SCEV *S) -> std::optional<std::pair<const SCEV *, APInt>> {
12538 const APInt *C;
12539 const SCEV *Op;
12540 if (match(S, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEV(V&: Op))))
12541 return {{Op, *C}};
12542 return std::nullopt;
12543 };
12544 if (auto MatchedMore = MatchConstMul(More)) {
12545 if (auto MatchedLess = MatchConstMul(Less)) {
12546 if (MatchedMore->second == MatchedLess->second) {
12547 More = MatchedMore->first;
12548 Less = MatchedLess->first;
12549 DiffMul *= MatchedMore->second;
12550 continue;
12551 }
12552 }
12553 }
12554
12555 // Try to cancel out common factors in two add expressions.
12556 SmallDenseMap<const SCEV *, int, 8> Multiplicity;
12557 auto Add = [&](const SCEV *S, int Mul) {
12558 if (auto *C = dyn_cast<SCEVConstant>(Val: S)) {
12559 if (Mul == 1) {
12560 Diff += C->getAPInt() * DiffMul;
12561 } else {
12562 assert(Mul == -1);
12563 Diff -= C->getAPInt() * DiffMul;
12564 }
12565 } else
12566 Multiplicity[S] += Mul;
12567 };
12568 auto Decompose = [&](const SCEV *S, int Mul) {
12569 if (isa<SCEVAddExpr>(Val: S)) {
12570 for (const SCEV *Op : S->operands())
12571 Add(Op, Mul);
12572 } else
12573 Add(S, Mul);
12574 };
12575 Decompose(More, 1);
12576 Decompose(Less, -1);
12577
12578 // Check whether all the non-constants cancel out, or reduce to new
12579 // More/Less values.
12580 const SCEV *NewMore = nullptr, *NewLess = nullptr;
12581 for (const auto &[S, Mul] : Multiplicity) {
12582 if (Mul == 0)
12583 continue;
12584 if (Mul == 1) {
12585 if (NewMore)
12586 return std::nullopt;
12587 NewMore = S;
12588 } else if (Mul == -1) {
12589 if (NewLess)
12590 return std::nullopt;
12591 NewLess = S;
12592 } else
12593 return std::nullopt;
12594 }
12595
12596 // Values stayed the same, no point in trying further.
12597 if (NewMore == More || NewLess == Less)
12598 return std::nullopt;
12599
12600 More = NewMore;
12601 Less = NewLess;
12602
12603 // Reduced to constant.
12604 if (!More && !Less)
12605 return Diff;
12606
12607 // Left with variable on only one side, bail out.
12608 if (!More || !Less)
12609 return std::nullopt;
12610 }
12611
12612 // Did not reduce to constant.
12613 return std::nullopt;
12614}
12615
12616bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
12617 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12618 const SCEV *FoundRHS, const Instruction *CtxI) {
12619 // Try to recognize the following pattern:
12620 //
12621 // FoundRHS = ...
12622 // ...
12623 // loop:
12624 // FoundLHS = {Start,+,W}
12625 // context_bb: // Basic block from the same loop
12626 // known(Pred, FoundLHS, FoundRHS)
12627 //
12628 // If some predicate is known in the context of a loop, it is also known on
12629 // each iteration of this loop, including the first iteration. Therefore, in
12630 // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to
12631 // prove the original pred using this fact.
12632 if (!CtxI)
12633 return false;
12634 const BasicBlock *ContextBB = CtxI->getParent();
12635 // Make sure AR varies in the context block.
12636 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS)) {
12637 const Loop *L = AR->getLoop();
12638 const auto *Latch = L->getLoopLatch();
12639 // Make sure that context belongs to the loop and executes on 1st iteration
12640 // (if it ever executes at all).
12641 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12642 return false;
12643 if (!isAvailableAtLoopEntry(S: FoundRHS, L: AR->getLoop()))
12644 return false;
12645 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: AR->getStart(), FoundRHS);
12646 }
12647
12648 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundRHS)) {
12649 const Loop *L = AR->getLoop();
12650 const auto *Latch = L->getLoopLatch();
12651 // Make sure that context belongs to the loop and executes on 1st iteration
12652 // (if it ever executes at all).
12653 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12654 return false;
12655 if (!isAvailableAtLoopEntry(S: FoundLHS, L: AR->getLoop()))
12656 return false;
12657 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS: AR->getStart());
12658 }
12659
12660 return false;
12661}
12662
12663bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(CmpPredicate Pred,
12664 const SCEV *LHS,
12665 const SCEV *RHS,
12666 const SCEV *FoundLHS,
12667 const SCEV *FoundRHS) {
12668 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT)
12669 return false;
12670
12671 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12672 if (!AddRecLHS)
12673 return false;
12674
12675 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12676 if (!AddRecFoundLHS)
12677 return false;
12678
12679 // We'd like to let SCEV reason about control dependencies, so we constrain
12680 // both the inequalities to be about add recurrences on the same loop. This
12681 // way we can use isLoopEntryGuardedByCond later.
12682
12683 const Loop *L = AddRecFoundLHS->getLoop();
12684 if (L != AddRecLHS->getLoop())
12685 return false;
12686
12687 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1)
12688 //
12689 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C)
12690 // ... (2)
12691 //
12692 // Informal proof for (2), assuming (1) [*]:
12693 //
12694 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**]
12695 //
12696 // Then
12697 //
12698 // FoundLHS s< FoundRHS s< INT_MIN - C
12699 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ]
12700 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ]
12701 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s<
12702 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ]
12703 // <=> FoundLHS + C s< FoundRHS + C
12704 //
12705 // [*]: (1) can be proved by ruling out overflow.
12706 //
12707 // [**]: This can be proved by analyzing all the four possibilities:
12708 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and
12709 // (A s>= 0, B s>= 0).
12710 //
12711 // Note:
12712 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C"
12713 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS
12714 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS
12715 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is
12716 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS +
12717 // C)".
12718
12719 std::optional<APInt> LDiff = computeConstantDifference(More: LHS, Less: FoundLHS);
12720 if (!LDiff)
12721 return false;
12722 std::optional<APInt> RDiff = computeConstantDifference(More: RHS, Less: FoundRHS);
12723 if (!RDiff || *LDiff != *RDiff)
12724 return false;
12725
12726 if (LDiff->isMinValue())
12727 return true;
12728
12729 APInt FoundRHSLimit;
12730
12731 if (Pred == CmpInst::ICMP_ULT) {
12732 FoundRHSLimit = -(*RDiff);
12733 } else {
12734 assert(Pred == CmpInst::ICMP_SLT && "Checked above!");
12735 FoundRHSLimit = APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: RHS->getType())) - *RDiff;
12736 }
12737
12738 // Try to prove (1) or (2), as needed.
12739 return isAvailableAtLoopEntry(S: FoundRHS, L) &&
12740 isLoopEntryGuardedByCond(L, Pred, LHS: FoundRHS,
12741 RHS: getConstant(Val: FoundRHSLimit));
12742}
12743
12744bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred, const SCEV *LHS,
12745 const SCEV *RHS, const SCEV *FoundLHS,
12746 const SCEV *FoundRHS, unsigned Depth) {
12747 const PHINode *LPhi = nullptr, *RPhi = nullptr;
12748
12749 llvm::scope_exit ClearOnExit([&]() {
12750 if (LPhi) {
12751 bool Erased = PendingMerges.erase(Ptr: LPhi);
12752 assert(Erased && "Failed to erase LPhi!");
12753 (void)Erased;
12754 }
12755 if (RPhi) {
12756 bool Erased = PendingMerges.erase(Ptr: RPhi);
12757 assert(Erased && "Failed to erase RPhi!");
12758 (void)Erased;
12759 }
12760 });
12761
12762 // Find respective Phis and check that they are not being pending.
12763 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(Val: LHS))
12764 if (auto *Phi = dyn_cast<PHINode>(Val: LU->getValue())) {
12765 if (!PendingMerges.insert(Ptr: Phi).second)
12766 return false;
12767 LPhi = Phi;
12768 }
12769 if (const SCEVUnknown *RU = dyn_cast<SCEVUnknown>(Val: RHS))
12770 if (auto *Phi = dyn_cast<PHINode>(Val: RU->getValue())) {
12771 // If we detect a loop of Phi nodes being processed by this method, for
12772 // example:
12773 //
12774 // %a = phi i32 [ %some1, %preheader ], [ %b, %latch ]
12775 // %b = phi i32 [ %some2, %preheader ], [ %a, %latch ]
12776 //
12777 // we don't want to deal with a case that complex, so return conservative
12778 // answer false.
12779 if (!PendingMerges.insert(Ptr: Phi).second)
12780 return false;
12781 RPhi = Phi;
12782 }
12783
12784 // If none of LHS, RHS is a Phi, nothing to do here.
12785 if (!LPhi && !RPhi)
12786 return false;
12787
12788 // If there is a SCEVUnknown Phi we are interested in, make it left.
12789 if (!LPhi) {
12790 std::swap(a&: LHS, b&: RHS);
12791 std::swap(a&: FoundLHS, b&: FoundRHS);
12792 std::swap(a&: LPhi, b&: RPhi);
12793 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12794 }
12795
12796 assert(LPhi && "LPhi should definitely be a SCEVUnknown Phi!");
12797 const BasicBlock *LBB = LPhi->getParent();
12798 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(Val: RHS);
12799
12800 auto ProvedEasily = [&](const SCEV *S1, const SCEV *S2) {
12801 return isKnownViaNonRecursiveReasoning(Pred, LHS: S1, RHS: S2) ||
12802 isImpliedCondOperandsViaRanges(Pred, LHS: S1, RHS: S2, FoundPred: Pred, FoundLHS, FoundRHS) ||
12803 isImpliedViaOperations(Pred, LHS: S1, RHS: S2, FoundLHS, FoundRHS, Depth);
12804 };
12805
12806 if (RPhi && RPhi->getParent() == LBB) {
12807 // Case one: RHS is also a SCEVUnknown Phi from the same basic block.
12808 // If we compare two Phis from the same block, and for each entry block
12809 // the predicate is true for incoming values from this block, then the
12810 // predicate is also true for the Phis.
12811 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12812 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12813 const SCEV *R = getSCEV(V: RPhi->getIncomingValueForBlock(BB: IncBB));
12814 if (!ProvedEasily(L, R))
12815 return false;
12816 }
12817 } else if (RAR && RAR->getLoop()->getHeader() == LBB) {
12818 // Case two: RHS is also a Phi from the same basic block, and it is an
12819 // AddRec. It means that there is a loop which has both AddRec and Unknown
12820 // PHIs, for it we can compare incoming values of AddRec from above the loop
12821 // and latch with their respective incoming values of LPhi.
12822 // TODO: Generalize to handle loops with many inputs in a header.
12823 if (LPhi->getNumIncomingValues() != 2) return false;
12824
12825 auto *RLoop = RAR->getLoop();
12826 auto *Predecessor = RLoop->getLoopPredecessor();
12827 assert(Predecessor && "Loop with AddRec with no predecessor?");
12828 const SCEV *L1 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Predecessor));
12829 if (!ProvedEasily(L1, RAR->getStart()))
12830 return false;
12831 auto *Latch = RLoop->getLoopLatch();
12832 assert(Latch && "Loop with AddRec with no latch?");
12833 const SCEV *L2 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Latch));
12834 if (!ProvedEasily(L2, RAR->getPostIncExpr(SE&: *this)))
12835 return false;
12836 } else {
12837 // In all other cases go over inputs of LHS and compare each of them to RHS,
12838 // the predicate is true for (LHS, RHS) if it is true for all such pairs.
12839 // At this point RHS is either a non-Phi, or it is a Phi from some block
12840 // different from LBB.
12841 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12842 // Check that RHS is available in this block.
12843 if (!dominates(S: RHS, BB: IncBB))
12844 return false;
12845 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12846 // Make sure L does not refer to a value from a potentially previous
12847 // iteration of a loop.
12848 if (!properlyDominates(S: L, BB: LBB))
12849 return false;
12850 // Addrecs are considered to properly dominate their loop, so are missed
12851 // by the previous check. Discard any values that have computable
12852 // evolution in this loop.
12853 if (auto *Loop = LI.getLoopFor(BB: LBB))
12854 if (hasComputableLoopEvolution(S: L, L: Loop))
12855 return false;
12856 if (!ProvedEasily(L, RHS))
12857 return false;
12858 }
12859 }
12860 return true;
12861}
12862
12863bool ScalarEvolution::isImpliedCondOperandsViaShift(CmpPredicate Pred,
12864 const SCEV *LHS,
12865 const SCEV *RHS,
12866 const SCEV *FoundLHS,
12867 const SCEV *FoundRHS) {
12868 // We want to imply LHS < RHS from LHS < (RHS >> shiftvalue). First, make
12869 // sure that we are dealing with same LHS.
12870 if (RHS == FoundRHS) {
12871 std::swap(a&: LHS, b&: RHS);
12872 std::swap(a&: FoundLHS, b&: FoundRHS);
12873 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12874 }
12875 if (LHS != FoundLHS)
12876 return false;
12877
12878 auto *SUFoundRHS = dyn_cast<SCEVUnknown>(Val: FoundRHS);
12879 if (!SUFoundRHS)
12880 return false;
12881
12882 Value *Shiftee, *ShiftValue;
12883
12884 using namespace PatternMatch;
12885 if (match(V: SUFoundRHS->getValue(),
12886 P: m_LShr(L: m_Value(V&: Shiftee), R: m_Value(V&: ShiftValue)))) {
12887 auto *ShifteeS = getSCEV(V: Shiftee);
12888 // Prove one of the following:
12889 // LHS <u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <u RHS
12890 // LHS <=u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <=u RHS
12891 // LHS <s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12892 // ---> LHS <s RHS
12893 // LHS <=s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12894 // ---> LHS <=s RHS
12895 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
12896 return isKnownPredicate(Pred: ICmpInst::ICMP_ULE, LHS: ShifteeS, RHS);
12897 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
12898 if (isKnownNonNegative(S: ShifteeS))
12899 return isKnownPredicate(Pred: ICmpInst::ICMP_SLE, LHS: ShifteeS, RHS);
12900 }
12901
12902 return false;
12903}
12904
12905bool ScalarEvolution::isImpliedCondOperandsViaMatchingDiff(
12906 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12907 const SCEV *FoundRHS) {
12908 // Only valid for equality predicates: (A == B) implies (C == D) when
12909 // the SCEV difference A - B equals C - D (they check the same
12910 // underlying relationship at every iteration).
12911 if (!ICmpInst::isEquality(P: Pred))
12912 return false;
12913
12914 // Restrict to cases involving loop recurrences - that's where this
12915 // pattern arises (correlated IV comparisons). This avoids calling
12916 // getMinusSCEV on arbitrary non-loop expressions.
12917 if ((!isa<SCEVAddRecExpr>(Val: LHS) && !isa<SCEVAddRecExpr>(Val: RHS)) ||
12918 (!isa<SCEVAddRecExpr>(Val: FoundLHS) && !isa<SCEVAddRecExpr>(Val: FoundRHS)))
12919 return false;
12920
12921 // AddRecs from different loops can never produce matching differences.
12922 const SCEVAddRecExpr *QueryAddRec = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12923 if (!QueryAddRec)
12924 QueryAddRec = cast<SCEVAddRecExpr>(Val: RHS);
12925 const SCEVAddRecExpr *FoundAddRec = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12926 if (!FoundAddRec)
12927 FoundAddRec = cast<SCEVAddRecExpr>(Val: FoundRHS);
12928 if (QueryAddRec->getLoop() != FoundAddRec->getLoop())
12929 return false;
12930
12931 // If the strides differ, the differences can never match.
12932 if (QueryAddRec->getStepRecurrence(SE&: *this) !=
12933 FoundAddRec->getStepRecurrence(SE&: *this))
12934 return false;
12935
12936 // Compute differences. For pointer-typed operands sharing the same base,
12937 // getMinusSCEV strips the common base and returns an integer SCEV.
12938 // For example, {base,+,8} - (base+8*n) = {-8n,+,8}
12939 const SCEV *FoundDiff = getMinusSCEV(LHS: FoundLHS, RHS: FoundRHS);
12940 if (isa<SCEVCouldNotCompute>(Val: FoundDiff))
12941 return false;
12942
12943 const SCEV *Diff = getMinusSCEV(LHS, RHS);
12944 if (isa<SCEVCouldNotCompute>(Val: Diff))
12945 return false;
12946
12947 return Diff == FoundDiff;
12948}
12949
12950bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred, const SCEV *LHS,
12951 const SCEV *RHS,
12952 const SCEV *FoundLHS,
12953 const SCEV *FoundRHS,
12954 const Instruction *CtxI) {
12955 return isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred: Pred, FoundLHS,
12956 FoundRHS) ||
12957 isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS,
12958 FoundRHS) ||
12959 isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS) ||
12960 isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
12961 CtxI) ||
12962 isImpliedCondOperandsViaMatchingDiff(Pred, LHS, RHS, FoundLHS,
12963 FoundRHS) ||
12964 isImpliedCondOperandsHelper(Pred, LHS, RHS, FoundLHS, FoundRHS);
12965}
12966
12967/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
12968template <typename MinMaxExprType>
12969static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
12970 const SCEV *Candidate) {
12971 const MinMaxExprType *MinMaxExpr = dyn_cast<MinMaxExprType>(MaybeMinMaxExpr);
12972 if (!MinMaxExpr)
12973 return false;
12974
12975 return is_contained(MinMaxExpr->operands(), Candidate);
12976}
12977
12978static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE,
12979 CmpPredicate Pred, const SCEV *LHS,
12980 const SCEV *RHS) {
12981 // If both sides are affine addrecs for the same loop, with equal
12982 // steps, and we know the recurrences don't wrap, then we only
12983 // need to check the predicate on the starting values.
12984
12985 if (!ICmpInst::isRelational(P: Pred))
12986 return false;
12987
12988 const SCEV *LStart, *RStart, *Step;
12989 const Loop *L;
12990 if (!match(S: LHS,
12991 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: LStart), Op1: m_SCEV(V&: Step), L: m_Loop(L))) ||
12992 !match(S: RHS, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: RStart), Op1: m_scev_Specific(S: Step),
12993 L: m_SpecificLoop(L))))
12994 return false;
12995 const SCEVAddRecExpr *LAR = cast<SCEVAddRecExpr>(Val: LHS);
12996 const SCEVAddRecExpr *RAR = cast<SCEVAddRecExpr>(Val: RHS);
12997 SCEV::NoWrapFlags NW = ICmpInst::isSigned(Pred) ?
12998 SCEV::FlagNSW : SCEV::FlagNUW;
12999 if (!LAR->getNoWrapFlags(Mask: NW) || !RAR->getNoWrapFlags(Mask: NW))
13000 return false;
13001
13002 return SE.isKnownPredicate(Pred, LHS: LStart, RHS: RStart);
13003}
13004
13005/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
13006/// expression?
13007static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, CmpPredicate Pred,
13008 const SCEV *LHS, const SCEV *RHS) {
13009 switch (Pred) {
13010 default:
13011 return false;
13012
13013 case ICmpInst::ICMP_SGE:
13014 std::swap(a&: LHS, b&: RHS);
13015 [[fallthrough]];
13016 case ICmpInst::ICMP_SLE:
13017 return
13018 // min(A, ...) <= A
13019 IsMinMaxConsistingOf<SCEVSMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
13020 // A <= max(A, ...)
13021 IsMinMaxConsistingOf<SCEVSMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
13022
13023 case ICmpInst::ICMP_UGE:
13024 std::swap(a&: LHS, b&: RHS);
13025 [[fallthrough]];
13026 case ICmpInst::ICMP_ULE:
13027 return
13028 // min(A, ...) <= A
13029 // FIXME: what about umin_seq?
13030 IsMinMaxConsistingOf<SCEVUMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
13031 // A <= max(A, ...)
13032 IsMinMaxConsistingOf<SCEVUMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
13033
13034 case ICmpInst::ICMP_UGT:
13035 std::swap(a&: LHS, b&: RHS);
13036 [[fallthrough]];
13037 case ICmpInst::ICMP_ULT:
13038 // umin(Ops) u<= each Op, so proving Op u< RHS for any Op proves
13039 // umin(Ops) u< RHS.
13040 //
13041 // Use computeConstantDifference instead of the more powerful
13042 // isKnownPredicate to keep this check cheap: isKnownPredicateViaMinOrMax
13043 // is called from isKnownViaNonRecursiveReasoning, so recursing into
13044 // the full predicate prover would be expensive.
13045 if (const auto *Min = dyn_cast<SCEVUMinExpr>(Val: LHS)) {
13046 for (SCEVUse Op : Min->operands()) {
13047 std::optional<APInt> Diff = SE.computeConstantDifference(More: RHS, Less: Op);
13048 // When Op and RHS share a common base differing by a
13049 // constant offset D (RHS - Op = D), Op u< RHS holds iff D != 0 and
13050 // RHS >= D (unsigned), i.e. the subtraction doesn't underflow.
13051 if (Diff && !Diff->isZero() && SE.getUnsignedRangeMin(S: RHS).uge(RHS: *Diff))
13052 return true;
13053 }
13054 }
13055 return false;
13056 }
13057
13058 llvm_unreachable("covered switch fell through?!");
13059}
13060
13061bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
13062 const SCEV *RHS,
13063 const SCEV *FoundLHS,
13064 const SCEV *FoundRHS,
13065 unsigned Depth) {
13066 assert(getTypeSizeInBits(LHS->getType()) ==
13067 getTypeSizeInBits(RHS->getType()) &&
13068 "LHS and RHS have different sizes?");
13069 assert(getTypeSizeInBits(FoundLHS->getType()) ==
13070 getTypeSizeInBits(FoundRHS->getType()) &&
13071 "FoundLHS and FoundRHS have different sizes?");
13072 // We want to avoid hurting the compile time with analysis of too big trees.
13073 if (Depth > MaxSCEVOperationsImplicationDepth)
13074 return false;
13075
13076 // We only want to work with GT comparison so far.
13077 if (ICmpInst::isLT(P: Pred)) {
13078 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
13079 std::swap(a&: LHS, b&: RHS);
13080 std::swap(a&: FoundLHS, b&: FoundRHS);
13081 }
13082
13083 CmpInst::Predicate P = Pred.getPreferredSignedPredicate();
13084
13085 // For unsigned, try to reduce it to corresponding signed comparison.
13086 if (P == ICmpInst::ICMP_UGT)
13087 // We can replace unsigned predicate with its signed counterpart if all
13088 // involved values are non-negative.
13089 // TODO: We could have better support for unsigned.
13090 if (isKnownNonNegative(S: FoundLHS) && isKnownNonNegative(S: FoundRHS)) {
13091 // Knowing that both FoundLHS and FoundRHS are non-negative, and knowing
13092 // FoundLHS >u FoundRHS, we also know that FoundLHS >s FoundRHS. Let us
13093 // use this fact to prove that LHS and RHS are non-negative.
13094 const SCEV *MinusOne = getMinusOne(Ty: LHS->getType());
13095 if (isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS, RHS: MinusOne, FoundLHS,
13096 FoundRHS) &&
13097 isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS: RHS, RHS: MinusOne, FoundLHS,
13098 FoundRHS))
13099 P = ICmpInst::ICMP_SGT;
13100 }
13101
13102 if (P != ICmpInst::ICMP_SGT)
13103 return false;
13104
13105 auto GetOpFromSExt = [&](const SCEV *S) -> const SCEV * {
13106 if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(Val: S))
13107 return Ext->getOperand();
13108 // TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
13109 // the constant in some cases.
13110 return S;
13111 };
13112
13113 // Acquire values from extensions.
13114 auto *OrigLHS = LHS;
13115 auto *OrigFoundLHS = FoundLHS;
13116 LHS = GetOpFromSExt(LHS);
13117 FoundLHS = GetOpFromSExt(FoundLHS);
13118
13119 // Is the SGT predicate can be proved trivially or using the found context.
13120 auto IsSGTViaContext = [&](const SCEV *S1, const SCEV *S2) {
13121 return isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2) ||
13122 isImpliedViaOperations(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2, FoundLHS: OrigFoundLHS,
13123 FoundRHS, Depth: Depth + 1);
13124 };
13125
13126 if (auto *LHSAddExpr = dyn_cast<SCEVAddExpr>(Val: LHS)) {
13127 // We want to avoid creation of any new non-constant SCEV. Since we are
13128 // going to compare the operands to RHS, we should be certain that we don't
13129 // need any size extensions for this. So let's decline all cases when the
13130 // sizes of types of LHS and RHS do not match.
13131 // TODO: Maybe try to get RHS from sext to catch more cases?
13132 if (getTypeSizeInBits(Ty: LHS->getType()) != getTypeSizeInBits(Ty: RHS->getType()))
13133 return false;
13134
13135 // Should not overflow.
13136 if (!LHSAddExpr->hasNoSignedWrap())
13137 return false;
13138
13139 SCEVUse LL = LHSAddExpr->getOperand(i: 0);
13140 SCEVUse LR = LHSAddExpr->getOperand(i: 1);
13141 auto *MinusOne = getMinusOne(Ty: RHS->getType());
13142
13143 // Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
13144 auto IsSumGreaterThanRHS = [&](const SCEV *S1, const SCEV *S2) {
13145 return IsSGTViaContext(S1, MinusOne) && IsSGTViaContext(S2, RHS);
13146 };
13147 // Try to prove the following rule:
13148 // (LHS = LL + LR) && (LL >= 0) && (LR > RHS) => (LHS > RHS).
13149 // (LHS = LL + LR) && (LR >= 0) && (LL > RHS) => (LHS > RHS).
13150 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
13151 return true;
13152 } else if (auto *LHSUnknownExpr = dyn_cast<SCEVUnknown>(Val: LHS)) {
13153 Value *LL, *LR;
13154 // FIXME: Once we have SDiv implemented, we can get rid of this matching.
13155
13156 using namespace llvm::PatternMatch;
13157
13158 if (match(V: LHSUnknownExpr->getValue(), P: m_SDiv(L: m_Value(V&: LL), R: m_Value(V&: LR)))) {
13159 // Rules for division.
13160 // We are going to perform some comparisons with Denominator and its
13161 // derivative expressions. In general case, creating a SCEV for it may
13162 // lead to a complex analysis of the entire graph, and in particular it
13163 // can request trip count recalculation for the same loop. This would
13164 // cache as SCEVCouldNotCompute to avoid the infinite recursion. To avoid
13165 // this, we only want to create SCEVs that are constants in this section.
13166 // So we bail if Denominator is not a constant.
13167 if (!isa<ConstantInt>(Val: LR))
13168 return false;
13169
13170 auto *Denominator = cast<SCEVConstant>(Val: getSCEV(V: LR));
13171
13172 // We want to make sure that LHS = FoundLHS / Denominator. If it is so,
13173 // then a SCEV for the numerator already exists and matches with FoundLHS.
13174 auto *Numerator = getExistingSCEV(V: LL);
13175 if (!Numerator || Numerator->getType() != FoundLHS->getType())
13176 return false;
13177
13178 // Make sure that the numerator matches with FoundLHS and the denominator
13179 // is positive.
13180 if (!HasSameValue(A: Numerator, B: FoundLHS) || !isKnownPositive(S: Denominator))
13181 return false;
13182
13183 auto *DTy = Denominator->getType();
13184 auto *FRHSTy = FoundRHS->getType();
13185 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
13186 // One of types is a pointer and another one is not. We cannot extend
13187 // them properly to a wider type, so let us just reject this case.
13188 // TODO: Usage of getEffectiveSCEVType for DTy, FRHSTy etc should help
13189 // to avoid this check.
13190 return false;
13191
13192 // Given that:
13193 // FoundLHS > FoundRHS, LHS = FoundLHS / Denominator, Denominator > 0.
13194 auto *WTy = getWiderType(T1: DTy, T2: FRHSTy);
13195 auto *DenominatorExt = getNoopOrSignExtend(V: Denominator, Ty: WTy);
13196 auto *FoundRHSExt = getNoopOrSignExtend(V: FoundRHS, Ty: WTy);
13197
13198 // Try to prove the following rule:
13199 // (FoundRHS > Denominator - 2) && (RHS <= 0) => (LHS > RHS).
13200 // For example, given that FoundLHS > 2. It means that FoundLHS is at
13201 // least 3. If we divide it by Denominator < 4, we will have at least 1.
13202 auto *DenomMinusTwo = getMinusSCEV(LHS: DenominatorExt, RHS: getConstant(Ty: WTy, V: 2));
13203 if (isKnownNonPositive(S: RHS) &&
13204 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
13205 return true;
13206
13207 // Try to prove the following rule:
13208 // (FoundRHS > -1 - Denominator) && (RHS < 0) => (LHS > RHS).
13209 // For example, given that FoundLHS > -3. Then FoundLHS is at least -2.
13210 // If we divide it by Denominator > 2, then:
13211 // 1. If FoundLHS is negative, then the result is 0.
13212 // 2. If FoundLHS is non-negative, then the result is non-negative.
13213 // Anyways, the result is non-negative.
13214 auto *MinusOne = getMinusOne(Ty: WTy);
13215 auto *NegDenomMinusOne = getMinusSCEV(LHS: MinusOne, RHS: DenominatorExt);
13216 if (isKnownNegative(S: RHS) &&
13217 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
13218 return true;
13219 }
13220 }
13221
13222 // If our expression contained SCEVUnknown Phis, and we split it down and now
13223 // need to prove something for them, try to prove the predicate for every
13224 // possible incoming values of those Phis.
13225 if (isImpliedViaMerge(Pred, LHS: OrigLHS, RHS, FoundLHS: OrigFoundLHS, FoundRHS, Depth: Depth + 1))
13226 return true;
13227
13228 return false;
13229}
13230
13231static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS,
13232 const SCEV *RHS) {
13233 // zext x u<= sext x, sext x s<= zext x
13234 const SCEV *Op;
13235 switch (Pred) {
13236 case ICmpInst::ICMP_SGE:
13237 std::swap(a&: LHS, b&: RHS);
13238 [[fallthrough]];
13239 case ICmpInst::ICMP_SLE: {
13240 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
13241 return match(S: LHS, P: m_scev_SExt(Op0: m_SCEV(V&: Op))) &&
13242 match(S: RHS, P: m_scev_ZExt(Op0: m_scev_Specific(S: Op)));
13243 }
13244 case ICmpInst::ICMP_UGE:
13245 std::swap(a&: LHS, b&: RHS);
13246 [[fallthrough]];
13247 case ICmpInst::ICMP_ULE: {
13248 // If operand >=u 0 then ZExt == SExt. If operand <u 0 then ZExt <u SExt.
13249 return match(S: LHS, P: m_scev_ZExt(Op0: m_SCEV(V&: Op))) &&
13250 match(S: RHS, P: m_scev_SExt(Op0: m_scev_Specific(S: Op)));
13251 }
13252 default:
13253 return false;
13254 };
13255 llvm_unreachable("unhandled case");
13256}
13257
13258bool ScalarEvolution::isKnownViaNonRecursiveReasoning(CmpPredicate Pred,
13259 SCEVUse LHS,
13260 SCEVUse RHS) {
13261 return isKnownPredicateExtendIdiom(Pred, LHS, RHS) ||
13262 isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
13263 IsKnownPredicateViaMinOrMax(SE&: *this, Pred, LHS, RHS) ||
13264 IsKnownPredicateViaAddRecStart(SE&: *this, Pred, LHS, RHS) ||
13265 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
13266}
13267
13268bool ScalarEvolution::isImpliedCondOperandsHelper(CmpPredicate Pred,
13269 const SCEV *LHS,
13270 const SCEV *RHS,
13271 const SCEV *FoundLHS,
13272 const SCEV *FoundRHS) {
13273 switch (Pred) {
13274 default:
13275 llvm_unreachable("Unexpected CmpPredicate value!");
13276 case ICmpInst::ICMP_EQ:
13277 case ICmpInst::ICMP_NE:
13278 if (HasSameValue(A: LHS, B: FoundLHS) && HasSameValue(A: RHS, B: FoundRHS))
13279 return true;
13280 break;
13281 case ICmpInst::ICMP_SLT:
13282 case ICmpInst::ICMP_SLE:
13283 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS, RHS: FoundLHS) &&
13284 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS: RHS, RHS: FoundRHS))
13285 return true;
13286 break;
13287 case ICmpInst::ICMP_SGT:
13288 case ICmpInst::ICMP_SGE:
13289 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS, RHS: FoundLHS) &&
13290 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS: RHS, RHS: FoundRHS))
13291 return true;
13292 break;
13293 case ICmpInst::ICMP_ULT:
13294 case ICmpInst::ICMP_ULE:
13295 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS, RHS: FoundLHS) &&
13296 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS: RHS, RHS: FoundRHS))
13297 return true;
13298 break;
13299 case ICmpInst::ICMP_UGT:
13300 case ICmpInst::ICMP_UGE:
13301 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS, RHS: FoundLHS) &&
13302 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: RHS, RHS: FoundRHS))
13303 return true;
13304 break;
13305 }
13306
13307 // Maybe it can be proved via operations?
13308 if (isImpliedViaOperations(Pred, LHS, RHS, FoundLHS, FoundRHS))
13309 return true;
13310
13311 return false;
13312}
13313
13314bool ScalarEvolution::isImpliedCondOperandsViaRanges(
13315 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
13316 const SCEV *FoundLHS, const SCEV *FoundRHS) {
13317 if (!isa<SCEVConstant>(Val: RHS) || !isa<SCEVConstant>(Val: FoundRHS))
13318 // The restriction on `FoundRHS` be lifted easily -- it exists only to
13319 // reduce the compile time impact of this optimization.
13320 return false;
13321
13322 std::optional<APInt> Addend = computeConstantDifference(More: LHS, Less: FoundLHS);
13323 if (!Addend)
13324 return false;
13325
13326 const APInt &ConstFoundRHS = cast<SCEVConstant>(Val: FoundRHS)->getAPInt();
13327
13328 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the
13329 // antecedent "`FoundLHS` `FoundPred` `FoundRHS`".
13330 ConstantRange FoundLHSRange =
13331 ConstantRange::makeExactICmpRegion(Pred: FoundPred, Other: ConstFoundRHS);
13332
13333 // Since `LHS` is `FoundLHS` + `Addend`, we can compute a range for `LHS`:
13334 ConstantRange LHSRange = FoundLHSRange.add(Other: ConstantRange(*Addend));
13335
13336 // We can also compute the range of values for `LHS` that satisfy the
13337 // consequent, "`LHS` `Pred` `RHS`":
13338 const APInt &ConstRHS = cast<SCEVConstant>(Val: RHS)->getAPInt();
13339 // The antecedent implies the consequent if every value of `LHS` that
13340 // satisfies the antecedent also satisfies the consequent.
13341 return LHSRange.icmp(Pred, Other: ConstRHS);
13342}
13343
13344bool ScalarEvolution::canIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
13345 bool IsSigned) {
13346 assert(isKnownPositive(Stride) && "Positive stride expected!");
13347
13348 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
13349 const SCEV *One = getOne(Ty: Stride->getType());
13350
13351 if (IsSigned) {
13352 APInt MaxRHS = getSignedRangeMax(S: RHS);
13353 APInt MaxValue = APInt::getSignedMaxValue(numBits: BitWidth);
13354 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13355
13356 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
13357 return (std::move(MaxValue) - MaxStrideMinusOne).slt(RHS: MaxRHS);
13358 }
13359
13360 APInt MaxRHS = getUnsignedRangeMax(S: RHS);
13361 APInt MaxValue = APInt::getMaxValue(numBits: BitWidth);
13362 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13363
13364 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
13365 return (std::move(MaxValue) - MaxStrideMinusOne).ult(RHS: MaxRHS);
13366}
13367
13368bool ScalarEvolution::canIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride,
13369 bool IsSigned) {
13370
13371 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
13372 const SCEV *One = getOne(Ty: Stride->getType());
13373
13374 if (IsSigned) {
13375 APInt MinRHS = getSignedRangeMin(S: RHS);
13376 APInt MinValue = APInt::getSignedMinValue(numBits: BitWidth);
13377 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13378
13379 // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow!
13380 return (std::move(MinValue) + MaxStrideMinusOne).sgt(RHS: MinRHS);
13381 }
13382
13383 APInt MinRHS = getUnsignedRangeMin(S: RHS);
13384 APInt MinValue = APInt::getMinValue(numBits: BitWidth);
13385 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13386
13387 // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow!
13388 return (std::move(MinValue) + MaxStrideMinusOne).ugt(RHS: MinRHS);
13389}
13390
13391const SCEV *ScalarEvolution::getUDivCeilSCEV(const SCEV *N, const SCEV *D) {
13392 // umin(N, 1) + floor((N - umin(N, 1)) / D)
13393 // This is equivalent to "1 + floor((N - 1) / D)" for N != 0. The umin
13394 // expression fixes the case of N=0.
13395 const SCEV *MinNOne = getUMinExpr(LHS: N, RHS: getOne(Ty: N->getType()));
13396 const SCEV *NMinusOne = getMinusSCEV(LHS: N, RHS: MinNOne);
13397 return getAddExpr(LHS: MinNOne, RHS: getUDivExpr(LHS: NMinusOne, RHS: D));
13398}
13399
13400const SCEV *ScalarEvolution::computeMaxBECountForLT(const SCEV *Start,
13401 const SCEV *Stride,
13402 const SCEV *End,
13403 unsigned BitWidth,
13404 bool IsSigned) {
13405 // The logic in this function assumes we can represent a positive stride.
13406 // If we can't, the backedge-taken count must be zero.
13407 if (IsSigned && BitWidth == 1)
13408 return getZero(Ty: Stride->getType());
13409
13410 // This code below only been closely audited for negative strides in the
13411 // unsigned comparison case, it may be correct for signed comparison, but
13412 // that needs to be established.
13413 if (IsSigned && isKnownNegative(S: Stride))
13414 return getCouldNotCompute();
13415
13416 // Calculate the maximum backedge count based on the range of values
13417 // permitted by Start, End, and Stride.
13418 APInt MinStart =
13419 IsSigned ? getSignedRangeMin(S: Start) : getUnsignedRangeMin(S: Start);
13420
13421 APInt MinStride =
13422 IsSigned ? getSignedRangeMin(S: Stride) : getUnsignedRangeMin(S: Stride);
13423
13424 // We assume either the stride is positive, or the backedge-taken count
13425 // is zero. So force StrideForMaxBECount to be at least one.
13426 APInt One(BitWidth, 1);
13427 APInt StrideForMaxBECount = IsSigned ? APIntOps::smax(A: One, B: MinStride)
13428 : APIntOps::umax(A: One, B: MinStride);
13429
13430 APInt MaxValue = IsSigned ? APInt::getSignedMaxValue(numBits: BitWidth)
13431 : APInt::getMaxValue(numBits: BitWidth);
13432 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
13433
13434 // Although End can be a MAX expression we estimate MaxEnd considering only
13435 // the case End = RHS of the loop termination condition. This is safe because
13436 // in the other case (End - Start) is zero, leading to a zero maximum backedge
13437 // taken count.
13438 APInt MaxEnd = IsSigned ? APIntOps::smin(A: getSignedRangeMax(S: End), B: Limit)
13439 : APIntOps::umin(A: getUnsignedRangeMax(S: End), B: Limit);
13440
13441 // MaxBECount = ceil((max(MaxEnd, MinStart) - MinStart) / Stride)
13442 MaxEnd = IsSigned ? APIntOps::smax(A: MaxEnd, B: MinStart)
13443 : APIntOps::umax(A: MaxEnd, B: MinStart);
13444
13445 return getUDivCeilSCEV(N: getConstant(Val: MaxEnd - MinStart) /* Delta */,
13446 D: getConstant(Val: StrideForMaxBECount) /* Step */);
13447}
13448
13449ScalarEvolution::ExitLimit
13450ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
13451 const Loop *L, bool IsSigned,
13452 bool ControlsOnlyExit, bool AllowPredicates) {
13453 SmallVector<const SCEVPredicate *> Predicates;
13454
13455 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13456 bool PredicatedIV = false;
13457 if (!IV) {
13458 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: LHS)) {
13459 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: ZExt->getOperand());
13460 if (AR && AR->getLoop() == L && AR->isAffine()) {
13461 auto canProveNUW = [&]() {
13462 // We can use the comparison to infer no-wrap flags only if it fully
13463 // controls the loop exit.
13464 if (!ControlsOnlyExit)
13465 return false;
13466
13467 if (!isLoopInvariant(S: RHS, L))
13468 return false;
13469
13470 if (!isKnownNonZero(S: AR->getStepRecurrence(SE&: *this)))
13471 // We need the sequence defined by AR to strictly increase in the
13472 // unsigned integer domain for the logic below to hold.
13473 return false;
13474
13475 const unsigned InnerBitWidth = getTypeSizeInBits(Ty: AR->getType());
13476 const unsigned OuterBitWidth = getTypeSizeInBits(Ty: RHS->getType());
13477 // If RHS <=u Limit, then there must exist a value V in the sequence
13478 // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and
13479 // V <=u UINT_MAX. Thus, we must exit the loop before unsigned
13480 // overflow occurs. This limit also implies that a signed comparison
13481 // (in the wide bitwidth) is equivalent to an unsigned comparison as
13482 // the high bits on both sides must be zero.
13483 APInt StrideMax = getUnsignedRangeMax(S: AR->getStepRecurrence(SE&: *this));
13484 APInt Limit = APInt::getMaxValue(numBits: InnerBitWidth) - (StrideMax - 1);
13485 Limit = Limit.zext(width: OuterBitWidth);
13486 return getUnsignedRangeMax(S: applyLoopGuards(Expr: RHS, L)).ule(RHS: Limit);
13487 };
13488 auto Flags = AR->getNoWrapFlags();
13489 if (!hasFlags(Flags, TestFlags: SCEV::FlagNUW) && canProveNUW())
13490 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
13491
13492 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
13493 if (AR->hasNoUnsignedWrap()) {
13494 // Emulate what getZeroExtendExpr would have done during construction
13495 // if we'd been able to infer the fact just above at that time.
13496 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
13497 Type *Ty = ZExt->getType();
13498 auto *S = getAddRecExpr(
13499 Start: getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: 0),
13500 Step: getZeroExtendExpr(Op: Step, Ty, Depth: 0), L, Flags: AR->getNoWrapFlags());
13501 IV = dyn_cast<SCEVAddRecExpr>(Val: S);
13502 }
13503 }
13504 }
13505 }
13506
13507
13508 if (!IV && AllowPredicates) {
13509 // Try to make this an AddRec using runtime tests, in the first X
13510 // iterations of this loop, where X is the SCEV expression found by the
13511 // algorithm below.
13512 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13513 PredicatedIV = true;
13514 }
13515
13516 // Avoid weird loops
13517 if (!IV || IV->getLoop() != L || !IV->isAffine())
13518 return getCouldNotCompute();
13519
13520 // A precondition of this method is that the condition being analyzed
13521 // reaches an exiting branch which dominates the latch. Given that, we can
13522 // assume that an increment which violates the nowrap specification and
13523 // produces poison must cause undefined behavior when the resulting poison
13524 // value is branched upon and thus we can conclude that the backedge is
13525 // taken no more often than would be required to produce that poison value.
13526 // Note that a well defined loop can exit on the iteration which violates
13527 // the nowrap specification if there is another exit (either explicit or
13528 // implicit/exceptional) which causes the loop to execute before the
13529 // exiting instruction we're analyzing would trigger UB.
13530 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13531 bool NoWrap = ControlsOnlyExit && any(Val: IV->getNoWrapFlags(Mask: WrapType));
13532 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13533
13534 const SCEV *Stride = IV->getStepRecurrence(SE&: *this);
13535
13536 bool PositiveStride = isKnownPositive(S: Stride);
13537
13538 // Avoid negative or zero stride values.
13539 if (!PositiveStride) {
13540 // We can compute the correct backedge taken count for loops with unknown
13541 // strides if we can prove that the loop is not an infinite loop with side
13542 // effects. Here's the loop structure we are trying to handle -
13543 //
13544 // i = start
13545 // do {
13546 // A[i] = i;
13547 // i += s;
13548 // } while (i < end);
13549 //
13550 // The backedge taken count for such loops is evaluated as -
13551 // (max(end, start + stride) - start - 1) /u stride
13552 //
13553 // The additional preconditions that we need to check to prove correctness
13554 // of the above formula is as follows -
13555 //
13556 // a) IV is either nuw or nsw depending upon signedness (indicated by the
13557 // NoWrap flag).
13558 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13559 // no side effects within the loop)
13560 // c) loop has a single static exit (with no abnormal exits)
13561 //
13562 // Precondition a) implies that if the stride is negative, this is a single
13563 // trip loop. The backedge taken count formula reduces to zero in this case.
13564 //
13565 // Precondition b) and c) combine to imply that if rhs is invariant in L,
13566 // then a zero stride means the backedge can't be taken without executing
13567 // undefined behavior.
13568 //
13569 // The positive stride case is the same as isKnownPositive(Stride) returning
13570 // true (original behavior of the function).
13571 //
13572 if (PredicatedIV || !NoWrap || !loopIsFiniteByAssumption(L) ||
13573 !loopHasNoAbnormalExits(L))
13574 return getCouldNotCompute();
13575
13576 if (!isKnownNonZero(S: Stride)) {
13577 // If we have a step of zero, and RHS isn't invariant in L, we don't know
13578 // if it might eventually be greater than start and if so, on which
13579 // iteration. We can't even produce a useful upper bound.
13580 if (!isLoopInvariant(S: RHS, L))
13581 return getCouldNotCompute();
13582
13583 // We allow a potentially zero stride, but we need to divide by stride
13584 // below. Since the loop can't be infinite and this check must control
13585 // the sole exit, we can infer the exit must be taken on the first
13586 // iteration (e.g. backedge count = 0) if the stride is zero. Given that,
13587 // we know the numerator in the divides below must be zero, so we can
13588 // pick an arbitrary non-zero value for the denominator (e.g. stride)
13589 // and produce the right result.
13590 // FIXME: Handle the case where Stride is poison?
13591 auto wouldZeroStrideBeUB = [&]() {
13592 // Proof by contradiction. Suppose the stride were zero. If we can
13593 // prove that the backedge *is* taken on the first iteration, then since
13594 // we know this condition controls the sole exit, we must have an
13595 // infinite loop. We can't have a (well defined) infinite loop per
13596 // check just above.
13597 // Note: The (Start - Stride) term is used to get the start' term from
13598 // (start' + stride,+,stride). Remember that we only care about the
13599 // result of this expression when stride == 0 at runtime.
13600 auto *StartIfZero = getMinusSCEV(LHS: IV->getStart(), RHS: Stride);
13601 return isLoopEntryGuardedByCond(L, Pred: Cond, LHS: StartIfZero, RHS);
13602 };
13603 if (!wouldZeroStrideBeUB()) {
13604 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13605 }
13606 }
13607 } else if (!NoWrap) {
13608 // Avoid proven overflow cases: this will ensure that the backedge taken
13609 // count will not generate any unsigned overflow.
13610 if (canIVOverflowOnLT(RHS, Stride, IsSigned))
13611 return getCouldNotCompute();
13612 }
13613
13614 // On all paths just preceeding, we established the following invariant:
13615 // IV can be assumed not to overflow up to and including the exiting
13616 // iteration. We proved this in one of two ways:
13617 // 1) We can show overflow doesn't occur before the exiting iteration
13618 // 1a) canIVOverflowOnLT, and b) step of one
13619 // 2) We can show that if overflow occurs, the loop must execute UB
13620 // before any possible exit.
13621 // Note that we have not yet proved RHS invariant (in general).
13622
13623 const SCEV *Start = IV->getStart();
13624
13625 // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond.
13626 // If we convert to integers, isLoopEntryGuardedByCond will miss some cases.
13627 // Use integer-typed versions for actual computation; we can't subtract
13628 // pointers in general.
13629 const SCEV *OrigStart = Start;
13630 const SCEV *OrigRHS = RHS;
13631 if (Start->getType()->isPointerTy()) {
13632 Start = getLosslessPtrToIntExpr(Op: Start);
13633 if (isa<SCEVCouldNotCompute>(Val: Start))
13634 return Start;
13635 }
13636 if (RHS->getType()->isPointerTy()) {
13637 RHS = getLosslessPtrToIntExpr(Op: RHS);
13638 if (isa<SCEVCouldNotCompute>(Val: RHS))
13639 return RHS;
13640 }
13641
13642 const SCEV *End = nullptr, *BECount = nullptr,
13643 *BECountIfBackedgeTaken = nullptr;
13644 if (!isLoopInvariant(S: RHS, L)) {
13645 const auto *RHSAddRec = dyn_cast<SCEVAddRecExpr>(Val: RHS);
13646 if (PositiveStride && RHSAddRec != nullptr && RHSAddRec->getLoop() == L &&
13647 any(Val: RHSAddRec->getNoWrapFlags())) {
13648 // The structure of loop we are trying to calculate backedge count of:
13649 //
13650 // left = left_start
13651 // right = right_start
13652 //
13653 // while(left < right){
13654 // ... do something here ...
13655 // left += s1; // stride of left is s1 (s1 > 0)
13656 // right += s2; // stride of right is s2 (s2 < 0)
13657 // }
13658 //
13659
13660 const SCEV *RHSStart = RHSAddRec->getStart();
13661 const SCEV *RHSStride = RHSAddRec->getStepRecurrence(SE&: *this);
13662
13663 // If Stride - RHSStride is positive and does not overflow, we can write
13664 // backedge count as ->
13665 // ceil((End - Start) /u (Stride - RHSStride))
13666 // Where, End = max(RHSStart, Start)
13667
13668 // Check if RHSStride < 0 and Stride - RHSStride will not overflow.
13669 if (isKnownNegative(S: RHSStride) &&
13670 willNotOverflow(BinOp: Instruction::Sub, /*Signed=*/true, LHS: Stride,
13671 RHS: RHSStride)) {
13672
13673 const SCEV *Denominator = getMinusSCEV(LHS: Stride, RHS: RHSStride);
13674 if (isKnownPositive(S: Denominator)) {
13675 End = IsSigned ? getSMaxExpr(LHS: RHSStart, RHS: Start)
13676 : getUMaxExpr(LHS: RHSStart, RHS: Start);
13677
13678 // We can do this because End >= Start, as End = max(RHSStart, Start)
13679 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13680
13681 BECount = getUDivCeilSCEV(N: Delta, D: Denominator);
13682 BECountIfBackedgeTaken =
13683 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHSStart, RHS: Start), D: Denominator);
13684 }
13685 }
13686 }
13687 if (BECount == nullptr) {
13688 // If we cannot calculate ExactBECount, we can calculate the MaxBECount,
13689 // given the start, stride and max value for the end bound of the
13690 // loop (RHS), and the fact that IV does not overflow (which is
13691 // checked above).
13692 const SCEV *MaxBECount = computeMaxBECountForLT(
13693 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned);
13694 return ExitLimit(getCouldNotCompute() /* ExactNotTaken */, MaxBECount,
13695 MaxBECount, false /*MaxOrZero*/, Predicates);
13696 }
13697 } else {
13698 // We use the expression (max(End,Start)-Start)/Stride to describe the
13699 // backedge count, as if the backedge is taken at least once
13700 // max(End,Start) is End and so the result is as above, and if not
13701 // max(End,Start) is Start so we get a backedge count of zero.
13702 auto *OrigStartMinusStride = getMinusSCEV(LHS: OrigStart, RHS: Stride);
13703 assert(isAvailableAtLoopEntry(OrigStartMinusStride, L) && "Must be!");
13704 assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!");
13705 assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!");
13706 // Can we prove (max(RHS,Start) > Start - Stride?
13707 if (isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigStartMinusStride, RHS: OrigStart) &&
13708 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigStartMinusStride, RHS: OrigRHS)) {
13709 // In this case, we can use a refined formula for computing backedge
13710 // taken count. The general formula remains:
13711 // "End-Start /uceiling Stride" where "End = max(RHS,Start)"
13712 // We want to use the alternate formula:
13713 // "((End - 1) - (Start - Stride)) /u Stride"
13714 // Let's do a quick case analysis to show these are equivalent under
13715 // our precondition that max(RHS,Start) > Start - Stride.
13716 // * For RHS <= Start, the backedge-taken count must be zero.
13717 // "((End - 1) - (Start - Stride)) /u Stride" reduces to
13718 // "((Start - 1) - (Start - Stride)) /u Stride" which simplies to
13719 // "Stride - 1 /u Stride" which is indeed zero for all non-zero values
13720 // of Stride. For 0 stride, we've use umin(1,Stride) above,
13721 // reducing this to the stride of 1 case.
13722 // * For RHS >= Start, the backedge count must be "RHS-Start /uceil
13723 // Stride".
13724 // "((End - 1) - (Start - Stride)) /u Stride" reduces to
13725 // "((RHS - 1) - (Start - Stride)) /u Stride" reassociates to
13726 // "((RHS - (Start - Stride) - 1) /u Stride".
13727 // Our preconditions trivially imply no overflow in that form.
13728 const SCEV *MinusOne = getMinusOne(Ty: Stride->getType());
13729 const SCEV *Numerator =
13730 getMinusSCEV(LHS: getAddExpr(LHS: RHS, RHS: MinusOne), RHS: getMinusSCEV(LHS: Start, RHS: Stride));
13731 BECount = getUDivExpr(LHS: Numerator, RHS: Stride);
13732 }
13733
13734 if (!BECount) {
13735 auto canProveRHSGreaterThanEqualStart = [&]() {
13736 auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13737 const SCEV *GuardedRHS = applyLoopGuards(Expr: OrigRHS, L);
13738 const SCEV *GuardedStart = applyLoopGuards(Expr: OrigStart, L);
13739
13740 if (isLoopEntryGuardedByCond(L, Pred: CondGE, LHS: OrigRHS, RHS: OrigStart) ||
13741 isKnownPredicate(Pred: CondGE, LHS: GuardedRHS, RHS: GuardedStart))
13742 return true;
13743
13744 // (RHS > Start - 1) implies RHS >= Start.
13745 // * "RHS >= Start" is trivially equivalent to "RHS > Start - 1" if
13746 // "Start - 1" doesn't overflow.
13747 // * For signed comparison, if Start - 1 does overflow, it's equal
13748 // to INT_MAX, and "RHS >s INT_MAX" is trivially false.
13749 // * For unsigned comparison, if Start - 1 does overflow, it's equal
13750 // to UINT_MAX, and "RHS >u UINT_MAX" is trivially false.
13751 //
13752 // FIXME: Should isLoopEntryGuardedByCond do this for us?
13753 auto CondGT = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13754 auto *StartMinusOne =
13755 getAddExpr(LHS: OrigStart, RHS: getMinusOne(Ty: OrigStart->getType()));
13756 return isLoopEntryGuardedByCond(L, Pred: CondGT, LHS: OrigRHS, RHS: StartMinusOne);
13757 };
13758
13759 // If we know that RHS >= Start in the context of loop, then we know
13760 // that max(RHS, Start) = RHS at this point.
13761 if (canProveRHSGreaterThanEqualStart()) {
13762 End = RHS;
13763 } else {
13764 // If RHS < Start, the backedge will be taken zero times. So in
13765 // general, we can write the backedge-taken count as:
13766 //
13767 // RHS >= Start ? ceil(RHS - Start) / Stride : 0
13768 //
13769 // We convert it to the following to make it more convenient for SCEV:
13770 //
13771 // ceil(max(RHS, Start) - Start) / Stride
13772 End = IsSigned ? getSMaxExpr(LHS: RHS, RHS: Start) : getUMaxExpr(LHS: RHS, RHS: Start);
13773
13774 // See what would happen if we assume the backedge is taken. This is
13775 // used to compute MaxBECount.
13776 BECountIfBackedgeTaken =
13777 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHS, RHS: Start), D: Stride);
13778 }
13779
13780 // At this point, we know:
13781 //
13782 // 1. If IsSigned, Start <=s End; otherwise, Start <=u End
13783 // 2. The index variable doesn't overflow.
13784 //
13785 // Therefore, we know N exists such that
13786 // (Start + Stride * N) >= End, and computing "(Start + Stride * N)"
13787 // doesn't overflow.
13788 //
13789 // Using this information, try to prove whether the addition in
13790 // "(Start - End) + (Stride - 1)" has unsigned overflow.
13791 const SCEV *One = getOne(Ty: Stride->getType());
13792 bool MayAddOverflow = [&] {
13793 if (isKnownToBeAPowerOfTwo(S: Stride)) {
13794 // Suppose Stride is a power of two, and Start/End are unsigned
13795 // integers. Let UMAX be the largest representable unsigned
13796 // integer.
13797 //
13798 // By the preconditions of this function, we know
13799 // "(Start + Stride * N) >= End", and this doesn't overflow.
13800 // As a formula:
13801 //
13802 // End <= (Start + Stride * N) <= UMAX
13803 //
13804 // Subtracting Start from all the terms:
13805 //
13806 // End - Start <= Stride * N <= UMAX - Start
13807 //
13808 // Since Start is unsigned, UMAX - Start <= UMAX. Therefore:
13809 //
13810 // End - Start <= Stride * N <= UMAX
13811 //
13812 // Stride * N is a multiple of Stride. Therefore,
13813 //
13814 // End - Start <= Stride * N <= UMAX - (UMAX mod Stride)
13815 //
13816 // Since Stride is a power of two, UMAX + 1 is divisible by
13817 // Stride. Therefore, UMAX mod Stride == Stride - 1. So we can
13818 // write:
13819 //
13820 // End - Start <= Stride * N <= UMAX - Stride - 1
13821 //
13822 // Dropping the middle term:
13823 //
13824 // End - Start <= UMAX - Stride - 1
13825 //
13826 // Adding Stride - 1 to both sides:
13827 //
13828 // (End - Start) + (Stride - 1) <= UMAX
13829 //
13830 // In other words, the addition doesn't have unsigned overflow.
13831 //
13832 // A similar proof works if we treat Start/End as signed values.
13833 // Just rewrite steps before "End - Start <= Stride * N <= UMAX"
13834 // to use signed max instead of unsigned max. Note that we're
13835 // trying to prove a lack of unsigned overflow in either case.
13836 return false;
13837 }
13838 if (Start == Stride || Start == getMinusSCEV(LHS: Stride, RHS: One)) {
13839 // If Start is equal to Stride, (End - Start) + (Stride - 1) == End
13840 // - 1. If !IsSigned, 0 <u Stride == Start <=u End; so 0 <u End - 1
13841 // <u End. If IsSigned, 0 <s Stride == Start <=s End; so 0 <s End -
13842 // 1 <s End.
13843 //
13844 // If Start is equal to Stride - 1, (End - Start) + Stride - 1 ==
13845 // End.
13846 return false;
13847 }
13848 return true;
13849 }();
13850
13851 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13852 if (!MayAddOverflow) {
13853 // floor((D + (S - 1)) / S)
13854 // We prefer this formulation if it's legal because it's fewer
13855 // operations.
13856 BECount =
13857 getUDivExpr(LHS: getAddExpr(LHS: Delta, RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13858 } else {
13859 BECount = getUDivCeilSCEV(N: Delta, D: Stride);
13860 }
13861 }
13862 }
13863
13864 const SCEV *ConstantMaxBECount;
13865 bool MaxOrZero = false;
13866 if (isa<SCEVConstant>(Val: BECount)) {
13867 ConstantMaxBECount = BECount;
13868 } else if (BECountIfBackedgeTaken &&
13869 isa<SCEVConstant>(Val: BECountIfBackedgeTaken)) {
13870 // If we know exactly how many times the backedge will be taken if it's
13871 // taken at least once, then the backedge count will either be that or
13872 // zero.
13873 ConstantMaxBECount = BECountIfBackedgeTaken;
13874 MaxOrZero = true;
13875 } else {
13876 ConstantMaxBECount = computeMaxBECountForLT(
13877 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned);
13878 }
13879
13880 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
13881 !isa<SCEVCouldNotCompute>(Val: BECount))
13882 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
13883
13884 const SCEV *SymbolicMaxBECount =
13885 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13886 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, MaxOrZero,
13887 Predicates);
13888}
13889
13890ScalarEvolution::ExitLimit ScalarEvolution::howManyGreaterThans(
13891 const SCEV *LHS, const SCEV *RHS, const Loop *L, bool IsSigned,
13892 bool ControlsOnlyExit, bool AllowPredicates) {
13893 SmallVector<const SCEVPredicate *> Predicates;
13894 // We handle only IV > Invariant
13895 if (!isLoopInvariant(S: RHS, L))
13896 return getCouldNotCompute();
13897
13898 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13899 if (!IV && AllowPredicates)
13900 // Try to make this an AddRec using runtime tests, in the first X
13901 // iterations of this loop, where X is the SCEV expression found by the
13902 // algorithm below.
13903 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13904
13905 // Avoid weird loops
13906 if (!IV || IV->getLoop() != L || !IV->isAffine())
13907 return getCouldNotCompute();
13908
13909 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13910 bool NoWrap = ControlsOnlyExit && any(Val: IV->getNoWrapFlags(Mask: WrapType));
13911 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13912
13913 const SCEV *Stride = getNegativeSCEV(V: IV->getStepRecurrence(SE&: *this));
13914
13915 // Avoid negative or zero stride values
13916 if (!isKnownPositive(S: Stride))
13917 return getCouldNotCompute();
13918
13919 // Avoid proven overflow cases: this will ensure that the backedge taken count
13920 // will not generate any unsigned overflow. Relaxed no-overflow conditions
13921 // exploit NoWrapFlags, allowing to optimize in presence of undefined
13922 // behaviors like the case of C language.
13923 if (!Stride->isOne() && !NoWrap)
13924 if (canIVOverflowOnGT(RHS, Stride, IsSigned))
13925 return getCouldNotCompute();
13926
13927 const SCEV *Start = IV->getStart();
13928 const SCEV *End = RHS;
13929 if (!isLoopEntryGuardedByCond(L, Pred: Cond, LHS: getAddExpr(LHS: Start, RHS: Stride), RHS)) {
13930 // If we know that Start >= RHS in the context of loop, then we know that
13931 // min(RHS, Start) = RHS at this point.
13932 if (isLoopEntryGuardedByCond(
13933 L, Pred: IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, LHS: Start, RHS))
13934 End = RHS;
13935 else
13936 End = IsSigned ? getSMinExpr(LHS: RHS, RHS: Start) : getUMinExpr(LHS: RHS, RHS: Start);
13937 }
13938
13939 if (Start->getType()->isPointerTy()) {
13940 Start = getLosslessPtrToIntExpr(Op: Start);
13941 if (isa<SCEVCouldNotCompute>(Val: Start))
13942 return Start;
13943 }
13944 if (End->getType()->isPointerTy()) {
13945 End = getLosslessPtrToIntExpr(Op: End);
13946 if (isa<SCEVCouldNotCompute>(Val: End))
13947 return End;
13948 }
13949
13950 // Compute ((Start - End) + (Stride - 1)) / Stride.
13951 // FIXME: This can overflow. Holding off on fixing this for now;
13952 // howManyGreaterThans will hopefully be gone soon.
13953 const SCEV *One = getOne(Ty: Stride->getType());
13954 const SCEV *BECount = getUDivExpr(
13955 LHS: getAddExpr(LHS: getMinusSCEV(LHS: Start, RHS: End), RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13956
13957 APInt MaxStart = IsSigned ? getSignedRangeMax(S: Start)
13958 : getUnsignedRangeMax(S: Start);
13959
13960 APInt MinStride = IsSigned ? getSignedRangeMin(S: Stride)
13961 : getUnsignedRangeMin(S: Stride);
13962
13963 unsigned BitWidth = getTypeSizeInBits(Ty: LHS->getType());
13964 APInt Limit = IsSigned ? APInt::getSignedMinValue(numBits: BitWidth) + (MinStride - 1)
13965 : APInt::getMinValue(numBits: BitWidth) + (MinStride - 1);
13966
13967 // Although End can be a MIN expression we estimate MinEnd considering only
13968 // the case End = RHS. This is safe because in the other case (Start - End)
13969 // is zero, leading to a zero maximum backedge taken count.
13970 APInt MinEnd =
13971 IsSigned ? APIntOps::smax(A: getSignedRangeMin(S: RHS), B: Limit)
13972 : APIntOps::umax(A: getUnsignedRangeMin(S: RHS), B: Limit);
13973
13974 const SCEV *ConstantMaxBECount =
13975 isa<SCEVConstant>(Val: BECount)
13976 ? BECount
13977 : getUDivCeilSCEV(N: getConstant(Val: MaxStart - MinEnd),
13978 D: getConstant(Val: MinStride));
13979
13980 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount))
13981 ConstantMaxBECount = BECount;
13982 const SCEV *SymbolicMaxBECount =
13983 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13984
13985 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
13986 Predicates);
13987}
13988
13989const SCEV *SCEVAddRecExpr::getNumIterationsInRange(const ConstantRange &Range,
13990 ScalarEvolution &SE) const {
13991 if (Range.isFullSet()) // Infinite loop.
13992 return SE.getCouldNotCompute();
13993
13994 // If the start is a non-zero constant, shift the range to simplify things.
13995 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: getStart()))
13996 if (!SC->getValue()->isZero()) {
13997 SmallVector<SCEVUse, 4> Operands(operands());
13998 Operands[0] = SE.getZero(Ty: SC->getType());
13999 const SCEV *Shifted = SE.getAddRecExpr(Operands, L: getLoop(),
14000 Flags: getNoWrapFlags(Mask: FlagNW));
14001 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Val: Shifted))
14002 return ShiftedAddRec->getNumIterationsInRange(
14003 Range: Range.subtract(CI: SC->getAPInt()), SE);
14004 // This is strange and shouldn't happen.
14005 return SE.getCouldNotCompute();
14006 }
14007
14008 // The only time we can solve this is when we have all constant indices.
14009 // Otherwise, we cannot determine the overflow conditions.
14010 if (any_of(Range: operands(), P: [](const SCEV *Op) { return !isa<SCEVConstant>(Val: Op); }))
14011 return SE.getCouldNotCompute();
14012
14013 // Okay at this point we know that all elements of the chrec are constants and
14014 // that the start element is zero.
14015
14016 // First check to see if the range contains zero. If not, the first
14017 // iteration exits.
14018 unsigned BitWidth = SE.getTypeSizeInBits(Ty: getType());
14019 if (!Range.contains(Val: APInt(BitWidth, 0)))
14020 return SE.getZero(Ty: getType());
14021
14022 if (isAffine()) {
14023 // If this is an affine expression then we have this situation:
14024 // Solve {0,+,A} in Range === Ax in Range
14025
14026 // We know that zero is in the range. If A is positive then we know that
14027 // the upper value of the range must be the first possible exit value.
14028 // If A is negative then the lower of the range is the last possible loop
14029 // value. Also note that we already checked for a full range.
14030 APInt A = cast<SCEVConstant>(Val: getOperand(i: 1))->getAPInt();
14031 APInt End = A.sge(RHS: 1) ? (Range.getUpper() - 1) : Range.getLower();
14032
14033 // The exit value should be (End+A)/A.
14034 APInt ExitVal = (End + A).udiv(RHS: A);
14035 ConstantInt *ExitValue = ConstantInt::get(Context&: SE.getContext(), V: ExitVal);
14036
14037 // Evaluate at the exit value. If we really did fall out of the valid
14038 // range, then we computed our trip count, otherwise wrap around or other
14039 // things must have happened.
14040 ConstantInt *Val = EvaluateConstantChrecAtConstant(AddRec: this, C: ExitValue, SE);
14041 if (Range.contains(Val: Val->getValue()))
14042 return SE.getCouldNotCompute(); // Something strange happened
14043
14044 // Ensure that the previous value is in the range.
14045 assert(Range.contains(
14046 EvaluateConstantChrecAtConstant(this,
14047 ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) &&
14048 "Linear scev computation is off in a bad way!");
14049 return SE.getConstant(V: ExitValue);
14050 }
14051
14052 if (isQuadratic()) {
14053 if (auto S = SolveQuadraticAddRecRange(AddRec: this, Range, SE))
14054 return SE.getConstant(Val: *S);
14055 }
14056
14057 return SE.getCouldNotCompute();
14058}
14059
14060const SCEVAddRecExpr *
14061SCEVAddRecExpr::getPostIncExpr(ScalarEvolution &SE) const {
14062 assert(getNumOperands() > 1 && "AddRec with zero step?");
14063 // There is a temptation to just call getAddExpr(this, getStepRecurrence(SE)),
14064 // but in this case we cannot guarantee that the value returned will be an
14065 // AddRec because SCEV does not have a fixed point where it stops
14066 // simplification: it is legal to return ({rec1} + {rec2}). For example, it
14067 // may happen if we reach arithmetic depth limit while simplifying. So we
14068 // construct the returned value explicitly.
14069 SmallVector<SCEVUse, 3> Ops;
14070 // If this is {A,+,B,+,C,...,+,N}, then its step is {B,+,C,+,...,+,N}, and
14071 // (this + Step) is {A+B,+,B+C,+...,+,N}.
14072 for (unsigned i = 0, e = getNumOperands() - 1; i < e; ++i)
14073 Ops.push_back(Elt: SE.getAddExpr(LHS: getOperand(i), RHS: getOperand(i: i + 1)));
14074 // We know that the last operand is not a constant zero (otherwise it would
14075 // have been popped out earlier). This guarantees us that if the result has
14076 // the same last operand, then it will also not be popped out, meaning that
14077 // the returned value will be an AddRec.
14078 const SCEV *Last = getOperand(i: getNumOperands() - 1);
14079 assert(!Last->isZero() && "Recurrency with zero step?");
14080 Ops.push_back(Elt: Last);
14081 return cast<SCEVAddRecExpr>(Val: SE.getAddRecExpr(Operands&: Ops, L: getLoop(),
14082 Flags: SCEV::FlagAnyWrap));
14083}
14084
14085// Return true when S contains at least an undef value.
14086bool ScalarEvolution::containsUndefs(const SCEV *S) const {
14087 return SCEVExprContains(
14088 Root: S, Pred: [](const SCEV *S) { return match(S, P: m_scev_UndefOrPoison()); });
14089}
14090
14091// Return true when S contains a value that is a nullptr.
14092bool ScalarEvolution::containsErasedValue(const SCEV *S) const {
14093 return SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
14094 if (const auto *SU = dyn_cast<SCEVUnknown>(Val: S))
14095 return SU->getValue() == nullptr;
14096 return false;
14097 });
14098}
14099
14100/// Return the size of an element read or written by Inst.
14101const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) {
14102 Type *Ty;
14103 Type *PtrTy;
14104 if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst)) {
14105 Ty = Store->getValueOperand()->getType();
14106 PtrTy = Store->getPointerOperandType();
14107 } else if (LoadInst *Load = dyn_cast<LoadInst>(Val: Inst)) {
14108 Ty = Load->getType();
14109 PtrTy = Load->getPointerOperandType();
14110 } else {
14111 return nullptr;
14112 }
14113
14114 Type *ETy = getEffectiveSCEVType(Ty: PtrTy);
14115 return getSizeOfExpr(IntTy: ETy, AllocTy: Ty);
14116}
14117
14118//===----------------------------------------------------------------------===//
14119// SCEVCallbackVH Class Implementation
14120//===----------------------------------------------------------------------===//
14121
14122void ScalarEvolution::SCEVCallbackVH::deleted() {
14123 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
14124 if (PHINode *PN = dyn_cast<PHINode>(Val: getValPtr()))
14125 SE->ConstantEvolutionLoopExitValue.erase(Val: PN);
14126 SE->eraseValueFromMap(V: getValPtr());
14127 // this now dangles!
14128}
14129
14130void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
14131 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
14132
14133 // Forget all the expressions associated with users of the old value,
14134 // so that future queries will recompute the expressions using the new
14135 // value.
14136 SE->forgetValue(V: getValPtr());
14137 // this now dangles!
14138}
14139
14140ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
14141 : CallbackVH(V), SE(se) {}
14142
14143//===----------------------------------------------------------------------===//
14144// ScalarEvolution Class Implementation
14145//===----------------------------------------------------------------------===//
14146
14147ScalarEvolution::ScalarEvolution(Function &F, TargetLibraryInfo &TLI,
14148 AssumptionCache &AC, DominatorTree &DT,
14149 LoopInfo &LI)
14150 : F(F), DL(F.getDataLayout()), TLI(TLI), AC(AC), DT(DT), LI(LI),
14151 CouldNotCompute(new SCEVCouldNotCompute()), ValuesAtScopes(64),
14152 LoopDispositions(64), BlockDispositions(64) {
14153 // To use guards for proving predicates, we need to scan every instruction in
14154 // relevant basic blocks, and not just terminators. Doing this is a waste of
14155 // time if the IR does not actually contain any calls to
14156 // @llvm.experimental.guard, so do a quick check and remember this beforehand.
14157 //
14158 // This pessimizes the case where a pass that preserves ScalarEvolution wants
14159 // to _add_ guards to the module when there weren't any before, and wants
14160 // ScalarEvolution to optimize based on those guards. For now we prefer to be
14161 // efficient in lieu of being smart in that rather obscure case.
14162
14163 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
14164 M: F.getParent(), id: Intrinsic::experimental_guard);
14165 HasGuards = GuardDecl && !GuardDecl->use_empty();
14166}
14167
14168ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg)
14169 : F(Arg.F), DL(Arg.DL), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC),
14170 DT(Arg.DT), LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)),
14171 ValueExprMap(std::move(Arg.ValueExprMap)),
14172 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
14173 PendingMerges(std::move(Arg.PendingMerges)),
14174 ConstantMultipleCache(std::move(Arg.ConstantMultipleCache)),
14175 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
14176 PredicatedBackedgeTakenCounts(
14177 std::move(Arg.PredicatedBackedgeTakenCounts)),
14178 BECountUsers(std::move(Arg.BECountUsers)),
14179 ConstantEvolutionLoopExitValue(
14180 std::move(Arg.ConstantEvolutionLoopExitValue)),
14181 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
14182 ValuesAtScopesUsers(std::move(Arg.ValuesAtScopesUsers)),
14183 LoopDispositions(std::move(Arg.LoopDispositions)),
14184 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
14185 BlockDispositions(std::move(Arg.BlockDispositions)),
14186 SCEVUsers(std::move(Arg.SCEVUsers)),
14187 UnsignedRanges(std::move(Arg.UnsignedRanges)),
14188 SignedRanges(std::move(Arg.SignedRanges)),
14189 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
14190 UniquePreds(std::move(Arg.UniquePreds)),
14191 SCEVAllocator(std::move(Arg.SCEVAllocator)),
14192 LoopUsers(std::move(Arg.LoopUsers)),
14193 PredicatedSCEVRewrites(std::move(Arg.PredicatedSCEVRewrites)),
14194 FirstUnknown(Arg.FirstUnknown) {
14195 Arg.FirstUnknown = nullptr;
14196}
14197
14198ScalarEvolution::~ScalarEvolution() {
14199 // Iterate through all the SCEVUnknown instances and call their
14200 // destructors, so that they release their references to their values.
14201 for (SCEVUnknown *U = FirstUnknown; U;) {
14202 SCEVUnknown *Tmp = U;
14203 U = U->Next;
14204 Tmp->~SCEVUnknown();
14205 }
14206 FirstUnknown = nullptr;
14207
14208 ExprValueMap.clear();
14209 ValueExprMap.clear();
14210 HasRecMap.clear();
14211 BackedgeTakenCounts.clear();
14212 PredicatedBackedgeTakenCounts.clear();
14213
14214 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage");
14215 assert(PendingMerges.empty() && "isImpliedViaMerge garbage");
14216 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!");
14217 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!");
14218}
14219
14220bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) {
14221 return !isa<SCEVCouldNotCompute>(Val: getBackedgeTakenCount(L));
14222}
14223
14224/// When printing a top-level SCEV for trip counts, it's helpful to include
14225/// a type for constants which are otherwise hard to disambiguate.
14226static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV* S) {
14227 if (isa<SCEVConstant>(Val: S))
14228 OS << *S->getType() << " ";
14229 OS << *S;
14230}
14231
14232static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE,
14233 const Loop *L) {
14234 // Print all inner loops first
14235 for (Loop *I : *L)
14236 PrintLoopInfo(OS, SE, L: I);
14237
14238 OS << "Loop ";
14239 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14240 OS << ": ";
14241
14242 SmallVector<BasicBlock *, 8> ExitingBlocks;
14243 L->getExitingBlocks(ExitingBlocks);
14244 if (ExitingBlocks.size() != 1)
14245 OS << "<multiple exits> ";
14246
14247 auto *BTC = SE->getBackedgeTakenCount(L);
14248 if (!isa<SCEVCouldNotCompute>(Val: BTC)) {
14249 OS << "backedge-taken count is ";
14250 PrintSCEVWithTypeHint(OS, S: BTC);
14251 } else
14252 OS << "Unpredictable backedge-taken count.";
14253 OS << "\n";
14254
14255 if (ExitingBlocks.size() > 1)
14256 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14257 OS << " exit count for " << ExitingBlock->getName() << ": ";
14258 const SCEV *EC = SE->getExitCount(L, ExitingBlock);
14259 PrintSCEVWithTypeHint(OS, S: EC);
14260 if (isa<SCEVCouldNotCompute>(Val: EC)) {
14261 // Retry with predicates.
14262 SmallVector<const SCEVPredicate *> Predicates;
14263 EC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates);
14264 if (!isa<SCEVCouldNotCompute>(Val: EC)) {
14265 OS << "\n predicated exit count for " << ExitingBlock->getName()
14266 << ": ";
14267 PrintSCEVWithTypeHint(OS, S: EC);
14268 OS << "\n Predicates:\n";
14269 for (const auto *P : Predicates)
14270 P->print(OS, Depth: 4);
14271 }
14272 }
14273 OS << "\n";
14274 }
14275
14276 OS << "Loop ";
14277 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14278 OS << ": ";
14279
14280 auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L);
14281 if (!isa<SCEVCouldNotCompute>(Val: ConstantBTC)) {
14282 OS << "constant max backedge-taken count is ";
14283 PrintSCEVWithTypeHint(OS, S: ConstantBTC);
14284 if (SE->isBackedgeTakenCountMaxOrZero(L))
14285 OS << ", actual taken count either this or zero.";
14286 } else {
14287 OS << "Unpredictable constant max backedge-taken count. ";
14288 }
14289
14290 OS << "\n"
14291 "Loop ";
14292 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14293 OS << ": ";
14294
14295 auto *SymbolicBTC = SE->getSymbolicMaxBackedgeTakenCount(L);
14296 if (!isa<SCEVCouldNotCompute>(Val: SymbolicBTC)) {
14297 OS << "symbolic max backedge-taken count is ";
14298 PrintSCEVWithTypeHint(OS, S: SymbolicBTC);
14299 if (SE->isBackedgeTakenCountMaxOrZero(L))
14300 OS << ", actual taken count either this or zero.";
14301 } else {
14302 OS << "Unpredictable symbolic max backedge-taken count. ";
14303 }
14304 OS << "\n";
14305
14306 if (ExitingBlocks.size() > 1)
14307 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14308 OS << " symbolic max exit count for " << ExitingBlock->getName() << ": ";
14309 auto *ExitBTC = SE->getExitCount(L, ExitingBlock,
14310 Kind: ScalarEvolution::SymbolicMaximum);
14311 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14312 if (isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14313 // Retry with predicates.
14314 SmallVector<const SCEVPredicate *> Predicates;
14315 ExitBTC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates,
14316 Kind: ScalarEvolution::SymbolicMaximum);
14317 if (!isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14318 OS << "\n predicated symbolic max exit count for "
14319 << ExitingBlock->getName() << ": ";
14320 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14321 OS << "\n Predicates:\n";
14322 for (const auto *P : Predicates)
14323 P->print(OS, Depth: 4);
14324 }
14325 }
14326 OS << "\n";
14327 }
14328
14329 SmallVector<const SCEVPredicate *, 4> Preds;
14330 auto *PBT = SE->getPredicatedBackedgeTakenCount(L, Preds);
14331 if (PBT != BTC) {
14332 OS << "Loop ";
14333 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14334 OS << ": ";
14335 if (!isa<SCEVCouldNotCompute>(Val: PBT)) {
14336 OS << "Predicated backedge-taken count is ";
14337 PrintSCEVWithTypeHint(OS, S: PBT);
14338 } else
14339 OS << "Unpredictable predicated backedge-taken count.";
14340 OS << "\n";
14341 OS << " Predicates:\n";
14342 for (const auto *P : Preds)
14343 P->print(OS, Depth: 4);
14344 }
14345 Preds.clear();
14346
14347 auto *PredConstantMax =
14348 SE->getPredicatedConstantMaxBackedgeTakenCount(L, Preds);
14349 if (PredConstantMax != ConstantBTC) {
14350 OS << "Loop ";
14351 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14352 OS << ": ";
14353 if (!isa<SCEVCouldNotCompute>(Val: PredConstantMax)) {
14354 OS << "Predicated constant max backedge-taken count is ";
14355 PrintSCEVWithTypeHint(OS, S: PredConstantMax);
14356 } else
14357 OS << "Unpredictable predicated constant max backedge-taken count.";
14358 OS << "\n";
14359 OS << " Predicates:\n";
14360 for (const auto *P : Preds)
14361 P->print(OS, Depth: 4);
14362 }
14363 Preds.clear();
14364
14365 auto *PredSymbolicMax =
14366 SE->getPredicatedSymbolicMaxBackedgeTakenCount(L, Preds);
14367 if (SymbolicBTC != PredSymbolicMax) {
14368 OS << "Loop ";
14369 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14370 OS << ": ";
14371 if (!isa<SCEVCouldNotCompute>(Val: PredSymbolicMax)) {
14372 OS << "Predicated symbolic max backedge-taken count is ";
14373 PrintSCEVWithTypeHint(OS, S: PredSymbolicMax);
14374 } else
14375 OS << "Unpredictable predicated symbolic max backedge-taken count.";
14376 OS << "\n";
14377 OS << " Predicates:\n";
14378 for (const auto *P : Preds)
14379 P->print(OS, Depth: 4);
14380 }
14381
14382 if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
14383 OS << "Loop ";
14384 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14385 OS << ": ";
14386 OS << "Trip multiple is " << SE->getSmallConstantTripMultiple(L) << "\n";
14387 }
14388}
14389
14390namespace llvm {
14391// Note: these overloaded operators need to be in the llvm namespace for them
14392// to be resolved correctly. If we put them outside the llvm namespace, the
14393//
14394// OS << ": " << SE.getLoopDisposition(SV, InnerL);
14395//
14396// code below "breaks" and start printing raw enum values as opposed to the
14397// string values.
14398static raw_ostream &operator<<(raw_ostream &OS,
14399 ScalarEvolution::LoopDisposition LD) {
14400 switch (LD) {
14401 case ScalarEvolution::LoopVariant:
14402 OS << "Variant";
14403 break;
14404 case ScalarEvolution::LoopInvariant:
14405 OS << "Invariant";
14406 break;
14407 case ScalarEvolution::LoopUniform:
14408 OS << "Uniform";
14409 break;
14410 case ScalarEvolution::LoopComputable:
14411 OS << "Computable";
14412 break;
14413 }
14414 return OS;
14415}
14416
14417static raw_ostream &operator<<(raw_ostream &OS,
14418 llvm::ScalarEvolution::BlockDisposition BD) {
14419 switch (BD) {
14420 case ScalarEvolution::DoesNotDominateBlock:
14421 OS << "DoesNotDominate";
14422 break;
14423 case ScalarEvolution::DominatesBlock:
14424 OS << "Dominates";
14425 break;
14426 case ScalarEvolution::ProperlyDominatesBlock:
14427 OS << "ProperlyDominates";
14428 break;
14429 }
14430 return OS;
14431}
14432} // namespace llvm
14433
14434void ScalarEvolution::print(raw_ostream &OS) const {
14435 // ScalarEvolution's implementation of the print method is to print
14436 // out SCEV values of all instructions that are interesting. Doing
14437 // this potentially causes it to create new SCEV objects though,
14438 // which technically conflicts with the const qualifier. This isn't
14439 // observable from outside the class though, so casting away the
14440 // const isn't dangerous.
14441 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14442
14443 if (ClassifyExpressions) {
14444 OS << "Classifying expressions for: ";
14445 F.printAsOperand(O&: OS, /*PrintType=*/false);
14446 OS << "\n";
14447 for (Instruction &I : instructions(F))
14448 if (isSCEVable(Ty: I.getType()) && !isa<CmpInst>(Val: I)) {
14449 OS << I << '\n';
14450 OS << " --> ";
14451 const SCEV *SV = SE.getSCEV(V: &I);
14452 SV->print(OS);
14453 if (!isa<SCEVCouldNotCompute>(Val: SV)) {
14454 OS << " U: ";
14455 SE.getUnsignedRange(S: SV).print(OS);
14456 OS << " S: ";
14457 SE.getSignedRange(S: SV).print(OS);
14458 }
14459
14460 const Loop *L = LI.getLoopFor(BB: I.getParent());
14461
14462 const SCEV *AtUse = SE.getSCEVAtScope(V: SV, L);
14463 if (AtUse != SV) {
14464 OS << " --> ";
14465 AtUse->print(OS);
14466 if (!isa<SCEVCouldNotCompute>(Val: AtUse)) {
14467 OS << " U: ";
14468 SE.getUnsignedRange(S: AtUse).print(OS);
14469 OS << " S: ";
14470 SE.getSignedRange(S: AtUse).print(OS);
14471 }
14472 }
14473
14474 if (L) {
14475 OS << "\t\t" "Exits: ";
14476 const SCEV *ExitValue = SE.getSCEVAtScope(V: SV, L: L->getParentLoop());
14477 if (!SE.isLoopInvariant(S: ExitValue, L)) {
14478 OS << "<<Unknown>>";
14479 } else {
14480 OS << *ExitValue;
14481 }
14482
14483 ListSeparator LS(", ", "\t\tLoopDispositions: { ");
14484 for (const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
14485 OS << LS;
14486 Iter->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14487 OS << ": " << SE.getLoopDisposition(S: SV, L: Iter);
14488 }
14489
14490 for (const auto *InnerL : depth_first(G: L)) {
14491 if (InnerL == L)
14492 continue;
14493 OS << LS;
14494 InnerL->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14495 OS << ": " << SE.getLoopDisposition(S: SV, L: InnerL);
14496 }
14497
14498 OS << " }";
14499 }
14500
14501 OS << "\n";
14502 }
14503 }
14504
14505 OS << "Determining loop execution counts for: ";
14506 F.printAsOperand(O&: OS, /*PrintType=*/false);
14507 OS << "\n";
14508 for (Loop *I : LI)
14509 PrintLoopInfo(OS, SE: &SE, L: I);
14510}
14511
14512ScalarEvolution::LoopDisposition
14513ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) {
14514 auto &Values = LoopDispositions[S];
14515 for (auto &V : Values) {
14516 if (V.getPointer() == L)
14517 return V.getInt();
14518 }
14519 Values.emplace_back(Args&: L, Args: LoopVariant);
14520 LoopDisposition D = computeLoopDisposition(S, L);
14521 auto &Values2 = LoopDispositions[S];
14522 for (auto &V : llvm::reverse(C&: Values2)) {
14523 if (V.getPointer() == L) {
14524 V.setInt(D);
14525 break;
14526 }
14527 }
14528 return D;
14529}
14530
14531ScalarEvolution::LoopDisposition
14532ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
14533 switch (S->getSCEVType()) {
14534 case scConstant:
14535 case scVScale:
14536 return LoopInvariant;
14537 case scAddRecExpr: {
14538 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14539
14540 // If L is the addrec's loop, it's computable.
14541 if (AR->getLoop() == L)
14542 return LoopComputable;
14543
14544 // Add recurrences are never invariant in the function-body (null loop).
14545 if (!L)
14546 return LoopVariant;
14547
14548 // Everything that is not defined at loop entry is variant.
14549 if (DT.dominates(A: L->getHeader(), B: AR->getLoop()->getHeader())) {
14550 if (L->contains(L: AR->getLoop()) &&
14551 llvm::all_of(Range: AR->operands(),
14552 P: [&](const SCEV *Op) { return isLoopUniform(S: Op, L); }))
14553 return LoopUniform;
14554
14555 return LoopVariant;
14556 }
14557 assert(!L->contains(AR->getLoop()) && "Containing loop's header does not"
14558 " dominate the contained loop's header?");
14559
14560 // This recurrence is invariant w.r.t. L if AR's loop contains L.
14561 if (AR->getLoop()->contains(L))
14562 return LoopInvariant;
14563
14564 // This recurrence is variant w.r.t. L if any of its operands
14565 // are variant.
14566 for (SCEVUse Op : AR->operands())
14567 if (!isLoopInvariant(S: Op, L))
14568 return LoopVariant;
14569
14570 // Otherwise it's loop-invariant.
14571 return LoopInvariant;
14572 }
14573 case scTruncate:
14574 case scZeroExtend:
14575 case scSignExtend:
14576 case scPtrToAddr:
14577 case scPtrToInt:
14578 case scAddExpr:
14579 case scMulExpr:
14580 case scUDivExpr:
14581 case scUMaxExpr:
14582 case scSMaxExpr:
14583 case scUMinExpr:
14584 case scSMinExpr:
14585 case scSequentialUMinExpr: {
14586 bool HasVarying = false;
14587 bool HasUniform = false;
14588 for (SCEVUse Op : S->operands()) {
14589 LoopDisposition D = getLoopDisposition(S: Op, L);
14590 if (D == LoopVariant)
14591 return LoopVariant;
14592 if (D == LoopComputable)
14593 HasVarying = true;
14594 if (D == LoopUniform)
14595 HasUniform = true;
14596 }
14597 return HasVarying ? (HasUniform ? LoopVariant : LoopComputable)
14598 : (HasUniform ? LoopUniform : LoopInvariant);
14599 }
14600 case scUnknown:
14601 // All non-instruction values are loop invariant. All instructions are loop
14602 // invariant if they are not contained in the specified loop.
14603 // Instructions are never considered invariant in the function body
14604 // (null loop) because they are defined within the "loop".
14605 if (auto *I = dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue()))
14606 return (L && !L->contains(Inst: I)) ? LoopInvariant : LoopVariant;
14607 return LoopInvariant;
14608 case scCouldNotCompute:
14609 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14610 }
14611 llvm_unreachable("Unknown SCEV kind!");
14612}
14613
14614bool ScalarEvolution::isLoopUniform(const SCEV *S, const Loop *L) {
14615 LoopDisposition D = getLoopDisposition(S, L);
14616 return D == LoopUniform || D == LoopInvariant;
14617}
14618
14619bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) {
14620 return getLoopDisposition(S, L) == LoopInvariant;
14621}
14622
14623bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) {
14624 return getLoopDisposition(S, L) == LoopComputable;
14625}
14626
14627ScalarEvolution::BlockDisposition
14628ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14629 auto &Values = BlockDispositions[S];
14630 for (auto &V : Values) {
14631 if (V.getPointer() == BB)
14632 return V.getInt();
14633 }
14634 Values.emplace_back(Args&: BB, Args: DoesNotDominateBlock);
14635 BlockDisposition D = computeBlockDisposition(S, BB);
14636 auto &Values2 = BlockDispositions[S];
14637 for (auto &V : llvm::reverse(C&: Values2)) {
14638 if (V.getPointer() == BB) {
14639 V.setInt(D);
14640 break;
14641 }
14642 }
14643 return D;
14644}
14645
14646ScalarEvolution::BlockDisposition
14647ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14648 switch (S->getSCEVType()) {
14649 case scConstant:
14650 case scVScale:
14651 return ProperlyDominatesBlock;
14652 case scAddRecExpr: {
14653 // This uses a "dominates" query instead of "properly dominates" query
14654 // to test for proper dominance too, because the instruction which
14655 // produces the addrec's value is a PHI, and a PHI effectively properly
14656 // dominates its entire containing block.
14657 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14658 if (!DT.dominates(A: AR->getLoop()->getHeader(), B: BB))
14659 return DoesNotDominateBlock;
14660
14661 // Fall through into SCEVNAryExpr handling.
14662 [[fallthrough]];
14663 }
14664 case scTruncate:
14665 case scZeroExtend:
14666 case scSignExtend:
14667 case scPtrToAddr:
14668 case scPtrToInt:
14669 case scAddExpr:
14670 case scMulExpr:
14671 case scUDivExpr:
14672 case scUMaxExpr:
14673 case scSMaxExpr:
14674 case scUMinExpr:
14675 case scSMinExpr:
14676 case scSequentialUMinExpr: {
14677 bool Proper = true;
14678 for (const SCEV *NAryOp : S->operands()) {
14679 BlockDisposition D = getBlockDisposition(S: NAryOp, BB);
14680 if (D == DoesNotDominateBlock)
14681 return DoesNotDominateBlock;
14682 if (D == DominatesBlock)
14683 Proper = false;
14684 }
14685 return Proper ? ProperlyDominatesBlock : DominatesBlock;
14686 }
14687 case scUnknown:
14688 if (Instruction *I =
14689 dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue())) {
14690 if (I->getParent() == BB)
14691 return DominatesBlock;
14692 if (DT.properlyDominates(A: I->getParent(), B: BB))
14693 return ProperlyDominatesBlock;
14694 return DoesNotDominateBlock;
14695 }
14696 return ProperlyDominatesBlock;
14697 case scCouldNotCompute:
14698 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14699 }
14700 llvm_unreachable("Unknown SCEV kind!");
14701}
14702
14703bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
14704 return getBlockDisposition(S, BB) >= DominatesBlock;
14705}
14706
14707bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) {
14708 return getBlockDisposition(S, BB) == ProperlyDominatesBlock;
14709}
14710
14711bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const {
14712 return SCEVExprContains(Root: S, Pred: [&](const SCEV *Expr) { return Expr == Op; });
14713}
14714
14715void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
14716 bool Predicated) {
14717 auto &BECounts =
14718 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14719 auto It = BECounts.find(Val: L);
14720 if (It != BECounts.end()) {
14721 for (const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
14722 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14723 if (!isa<SCEVConstant>(Val: S)) {
14724 auto UserIt = BECountUsers.find(Val: S);
14725 assert(UserIt != BECountUsers.end());
14726 UserIt->second.erase(Ptr: {L, Predicated});
14727 }
14728 }
14729 }
14730 BECounts.erase(I: It);
14731 }
14732}
14733
14734void ScalarEvolution::forgetMemoizedResults(ArrayRef<SCEVUse> SCEVs) {
14735 SmallPtrSet<const SCEV *, 8> ToForget(llvm::from_range, SCEVs);
14736 SmallVector<SCEVUse, 8> Worklist(ToForget.begin(), ToForget.end());
14737
14738 while (!Worklist.empty()) {
14739 const SCEV *Curr = Worklist.pop_back_val();
14740 auto Users = SCEVUsers.find(Val: Curr);
14741 if (Users != SCEVUsers.end())
14742 for (const auto *User : Users->second)
14743 if (ToForget.insert(Ptr: User).second)
14744 Worklist.push_back(Elt: User);
14745 }
14746
14747 for (const auto *S : ToForget)
14748 forgetMemoizedResultsImpl(S);
14749
14750 PredicatedSCEVRewrites.remove_if(
14751 Pred: [&](const auto &Entry) { return ToForget.count(Ptr: Entry.first.first); });
14752}
14753
14754void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) {
14755 LoopDispositions.erase(Val: S);
14756 BlockDispositions.erase(Val: S);
14757 UnsignedRanges.erase(Val: S);
14758 SignedRanges.erase(Val: S);
14759 HasRecMap.erase(Val: S);
14760 ConstantMultipleCache.erase(Val: S);
14761
14762 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
14763 UnsignedWrapViaInductionTried.erase(Ptr: AR);
14764 SignedWrapViaInductionTried.erase(Ptr: AR);
14765 }
14766
14767 auto ExprIt = ExprValueMap.find(Val: S);
14768 if (ExprIt != ExprValueMap.end()) {
14769 for (Value *V : ExprIt->second) {
14770 auto ValueIt = ValueExprMap.find_as(Val: V);
14771 if (ValueIt != ValueExprMap.end())
14772 ValueExprMap.erase(I: ValueIt);
14773 }
14774 ExprValueMap.erase(I: ExprIt);
14775 }
14776
14777 auto ScopeIt = ValuesAtScopes.find(Val: S);
14778 if (ScopeIt != ValuesAtScopes.end()) {
14779 for (const auto &Pair : ScopeIt->second)
14780 if (!isa_and_nonnull<SCEVConstant>(Val: Pair.second))
14781 llvm::erase(C&: ValuesAtScopesUsers[Pair.second],
14782 V: std::make_pair(x: Pair.first, y&: S));
14783 ValuesAtScopes.erase(I: ScopeIt);
14784 }
14785
14786 auto ScopeUserIt = ValuesAtScopesUsers.find(Val: S);
14787 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
14788 for (const auto &Pair : ScopeUserIt->second)
14789 llvm::erase(C&: ValuesAtScopes[Pair.second], V: std::make_pair(x: Pair.first, y&: S));
14790 ValuesAtScopesUsers.erase(I: ScopeUserIt);
14791 }
14792
14793 auto BEUsersIt = BECountUsers.find(Val: S);
14794 if (BEUsersIt != BECountUsers.end()) {
14795 // Work on a copy, as forgetBackedgeTakenCounts() will modify the original.
14796 auto Copy = BEUsersIt->second;
14797 for (const auto &Pair : Copy)
14798 forgetBackedgeTakenCounts(L: Pair.getPointer(), Predicated: Pair.getInt());
14799 BECountUsers.erase(I: BEUsersIt);
14800 }
14801
14802 auto FoldUser = FoldCacheUser.find(Val: S);
14803 if (FoldUser != FoldCacheUser.end())
14804 for (auto &KV : FoldUser->second)
14805 FoldCache.erase(Val: KV);
14806 FoldCacheUser.erase(Val: S);
14807}
14808
14809void
14810ScalarEvolution::getUsedLoops(const SCEV *S,
14811 SmallPtrSetImpl<const Loop *> &LoopsUsed) {
14812 struct FindUsedLoops {
14813 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
14814 : LoopsUsed(LoopsUsed) {}
14815 SmallPtrSetImpl<const Loop *> &LoopsUsed;
14816 bool follow(const SCEV *S) {
14817 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S))
14818 LoopsUsed.insert(Ptr: AR->getLoop());
14819 return true;
14820 }
14821
14822 bool isDone() const { return false; }
14823 };
14824
14825 FindUsedLoops F(LoopsUsed);
14826 SCEVTraversal<FindUsedLoops>(F).visitAll(Root: S);
14827}
14828
14829void ScalarEvolution::getReachableBlocks(
14830 SmallPtrSetImpl<BasicBlock *> &Reachable, Function &F) {
14831 SmallVector<BasicBlock *> Worklist;
14832 Worklist.push_back(Elt: &F.getEntryBlock());
14833 while (!Worklist.empty()) {
14834 BasicBlock *BB = Worklist.pop_back_val();
14835 if (!Reachable.insert(Ptr: BB).second)
14836 continue;
14837
14838 Value *Cond;
14839 BasicBlock *TrueBB, *FalseBB;
14840 if (match(V: BB->getTerminator(), P: m_Br(C: m_Value(V&: Cond), T: m_BasicBlock(V&: TrueBB),
14841 F: m_BasicBlock(V&: FalseBB)))) {
14842 if (auto *C = dyn_cast<ConstantInt>(Val: Cond)) {
14843 Worklist.push_back(Elt: C->isOne() ? TrueBB : FalseBB);
14844 continue;
14845 }
14846
14847 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
14848 const SCEV *L = getSCEV(V: Cmp->getOperand(i_nocapture: 0));
14849 const SCEV *R = getSCEV(V: Cmp->getOperand(i_nocapture: 1));
14850 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getCmpPredicate(), LHS: L, RHS: R)) {
14851 Worklist.push_back(Elt: TrueBB);
14852 continue;
14853 }
14854 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getInverseCmpPredicate(), LHS: L,
14855 RHS: R)) {
14856 Worklist.push_back(Elt: FalseBB);
14857 continue;
14858 }
14859 }
14860 }
14861
14862 append_range(C&: Worklist, R: successors(BB));
14863 }
14864}
14865
14866void ScalarEvolution::verify() const {
14867 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14868 ScalarEvolution SE2(F, TLI, AC, DT, LI);
14869
14870 SmallVector<Loop *, 8> LoopStack(LI.begin(), LI.end());
14871
14872 // Map's SCEV expressions from one ScalarEvolution "universe" to another.
14873 struct SCEVMapper : public SCEVRewriteVisitor<SCEVMapper> {
14874 SCEVMapper(ScalarEvolution &SE) : SCEVRewriteVisitor<SCEVMapper>(SE) {}
14875
14876 const SCEV *visitConstant(const SCEVConstant *Constant) {
14877 return SE.getConstant(Val: Constant->getAPInt());
14878 }
14879
14880 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14881 return SE.getUnknown(V: Expr->getValue());
14882 }
14883
14884 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
14885 return SE.getCouldNotCompute();
14886 }
14887 };
14888
14889 SCEVMapper SCM(SE2);
14890 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
14891 SE2.getReachableBlocks(Reachable&: ReachableBlocks, F);
14892
14893 auto GetDelta = [&](const SCEV *Old, const SCEV *New) -> const SCEV * {
14894 if (containsUndefs(S: Old) || containsUndefs(S: New)) {
14895 // SCEV treats "undef" as an unknown but consistent value (i.e. it does
14896 // not propagate undef aggressively). This means we can (and do) fail
14897 // verification in cases where a transform makes a value go from "undef"
14898 // to "undef+1" (say). The transform is fine, since in both cases the
14899 // result is "undef", but SCEV thinks the value increased by 1.
14900 return nullptr;
14901 }
14902
14903 // Unless VerifySCEVStrict is set, we only compare constant deltas.
14904 const SCEV *Delta = SE2.getMinusSCEV(LHS: Old, RHS: New);
14905 if (!VerifySCEVStrict && !isa<SCEVConstant>(Val: Delta))
14906 return nullptr;
14907
14908 return Delta;
14909 };
14910
14911 while (!LoopStack.empty()) {
14912 auto *L = LoopStack.pop_back_val();
14913 llvm::append_range(C&: LoopStack, R&: *L);
14914
14915 // Only verify BECounts in reachable loops. For an unreachable loop,
14916 // any BECount is legal.
14917 if (!ReachableBlocks.contains(Ptr: L->getHeader()))
14918 continue;
14919
14920 // Only verify cached BECounts. Computing new BECounts may change the
14921 // results of subsequent SCEV uses.
14922 auto It = BackedgeTakenCounts.find(Val: L);
14923 if (It == BackedgeTakenCounts.end())
14924 continue;
14925
14926 auto *CurBECount =
14927 SCM.visit(S: It->second.getExact(L, SE: const_cast<ScalarEvolution *>(this)));
14928 auto *NewBECount = SE2.getBackedgeTakenCount(L);
14929
14930 if (CurBECount == SE2.getCouldNotCompute() ||
14931 NewBECount == SE2.getCouldNotCompute()) {
14932 // NB! This situation is legal, but is very suspicious -- whatever pass
14933 // change the loop to make a trip count go from could not compute to
14934 // computable or vice-versa *should have* invalidated SCEV. However, we
14935 // choose not to assert here (for now) since we don't want false
14936 // positives.
14937 continue;
14938 }
14939
14940 if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) >
14941 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14942 NewBECount = SE2.getZeroExtendExpr(Op: NewBECount, Ty: CurBECount->getType());
14943 else if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) <
14944 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14945 CurBECount = SE2.getZeroExtendExpr(Op: CurBECount, Ty: NewBECount->getType());
14946
14947 const SCEV *Delta = GetDelta(CurBECount, NewBECount);
14948 if (Delta && !Delta->isZero()) {
14949 dbgs() << "Trip Count for " << *L << " Changed!\n";
14950 dbgs() << "Old: " << *CurBECount << "\n";
14951 dbgs() << "New: " << *NewBECount << "\n";
14952 dbgs() << "Delta: " << *Delta << "\n";
14953 std::abort();
14954 }
14955 }
14956
14957 // Collect all valid loops currently in LoopInfo.
14958 SmallPtrSet<Loop *, 32> ValidLoops;
14959 SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end());
14960 while (!Worklist.empty()) {
14961 Loop *L = Worklist.pop_back_val();
14962 if (ValidLoops.insert(Ptr: L).second)
14963 Worklist.append(in_start: L->begin(), in_end: L->end());
14964 }
14965 for (const auto &KV : ValueExprMap) {
14966#ifndef NDEBUG
14967 // Check for SCEV expressions referencing invalid/deleted loops.
14968 if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
14969 assert(ValidLoops.contains(AR->getLoop()) &&
14970 "AddRec references invalid loop");
14971 }
14972#endif
14973
14974 // Check that the value is also part of the reverse map.
14975 auto It = ExprValueMap.find(Val: KV.second);
14976 if (It == ExprValueMap.end() || !It->second.contains(key: KV.first)) {
14977 dbgs() << "Value " << *KV.first
14978 << " is in ValueExprMap but not in ExprValueMap\n";
14979 std::abort();
14980 }
14981
14982 if (auto *I = dyn_cast<Instruction>(Val: &*KV.first)) {
14983 if (!ReachableBlocks.contains(Ptr: I->getParent()))
14984 continue;
14985 const SCEV *OldSCEV = SCM.visit(S: KV.second);
14986 const SCEV *NewSCEV = SE2.getSCEV(V: I);
14987 const SCEV *Delta = GetDelta(OldSCEV, NewSCEV);
14988 if (Delta && !Delta->isZero()) {
14989 dbgs() << "SCEV for value " << *I << " changed!\n"
14990 << "Old: " << *OldSCEV << "\n"
14991 << "New: " << *NewSCEV << "\n"
14992 << "Delta: " << *Delta << "\n";
14993 std::abort();
14994 }
14995 }
14996 }
14997
14998 for (const auto &KV : ExprValueMap) {
14999 for (Value *V : KV.second) {
15000 const SCEV *S = ValueExprMap.lookup(Val: V);
15001 if (!S) {
15002 dbgs() << "Value " << *V
15003 << " is in ExprValueMap but not in ValueExprMap\n";
15004 std::abort();
15005 }
15006 if (S != KV.first) {
15007 dbgs() << "Value " << *V << " mapped to " << *S << " rather than "
15008 << *KV.first << "\n";
15009 std::abort();
15010 }
15011 }
15012 }
15013
15014 // Verify integrity of SCEV users.
15015 for (const auto &S : UniqueSCEVs) {
15016 for (SCEVUse Op : S.operands()) {
15017 // We do not store dependencies of constants.
15018 if (isa<SCEVConstant>(Val: Op))
15019 continue;
15020 auto It = SCEVUsers.find(Val: Op);
15021 if (It != SCEVUsers.end() && It->second.count(Ptr: &S))
15022 continue;
15023 dbgs() << "Use of operand " << *Op << " by user " << S
15024 << " is not being tracked!\n";
15025 std::abort();
15026 }
15027 }
15028
15029 // Verify integrity of ValuesAtScopes users.
15030 for (const auto &ValueAndVec : ValuesAtScopes) {
15031 const SCEV *Value = ValueAndVec.first;
15032 for (const auto &LoopAndValueAtScope : ValueAndVec.second) {
15033 const Loop *L = LoopAndValueAtScope.first;
15034 const SCEV *ValueAtScope = LoopAndValueAtScope.second;
15035 if (!isa<SCEVConstant>(Val: ValueAtScope)) {
15036 auto It = ValuesAtScopesUsers.find(Val: ValueAtScope);
15037 if (It != ValuesAtScopesUsers.end() &&
15038 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: Value)))
15039 continue;
15040 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
15041 << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
15042 std::abort();
15043 }
15044 }
15045 }
15046
15047 for (const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
15048 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
15049 for (const auto &LoopAndValue : ValueAtScopeAndVec.second) {
15050 const Loop *L = LoopAndValue.first;
15051 const SCEV *Value = LoopAndValue.second;
15052 assert(!isa<SCEVConstant>(Value));
15053 auto It = ValuesAtScopes.find(Val: Value);
15054 if (It != ValuesAtScopes.end() &&
15055 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: ValueAtScope)))
15056 continue;
15057 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
15058 << *ValueAtScope << " missing in ValuesAtScopes\n";
15059 std::abort();
15060 }
15061 }
15062
15063 // Verify integrity of BECountUsers.
15064 auto VerifyBECountUsers = [&](bool Predicated) {
15065 auto &BECounts =
15066 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
15067 for (const auto &LoopAndBEInfo : BECounts) {
15068 for (const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
15069 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
15070 if (!isa<SCEVConstant>(Val: S)) {
15071 auto UserIt = BECountUsers.find(Val: S);
15072 if (UserIt != BECountUsers.end() &&
15073 UserIt->second.contains(Ptr: { LoopAndBEInfo.first, Predicated }))
15074 continue;
15075 dbgs() << "Value " << *S << " for loop " << *LoopAndBEInfo.first
15076 << " missing from BECountUsers\n";
15077 std::abort();
15078 }
15079 }
15080 }
15081 }
15082 };
15083 VerifyBECountUsers(/* Predicated */ false);
15084 VerifyBECountUsers(/* Predicated */ true);
15085
15086 // Verify intergity of loop disposition cache.
15087 for (auto &[S, Values] : LoopDispositions) {
15088 for (auto [Loop, CachedDisposition] : Values) {
15089 const auto RecomputedDisposition = SE2.getLoopDisposition(S, L: Loop);
15090 if (CachedDisposition != RecomputedDisposition) {
15091 dbgs() << "Cached disposition of " << *S << " for loop " << *Loop
15092 << " is incorrect: cached " << CachedDisposition << ", actual "
15093 << RecomputedDisposition << "\n";
15094 std::abort();
15095 }
15096 }
15097 }
15098
15099 // Verify integrity of the block disposition cache.
15100 for (auto &[S, Values] : BlockDispositions) {
15101 for (auto [BB, CachedDisposition] : Values) {
15102 const auto RecomputedDisposition = SE2.getBlockDisposition(S, BB);
15103 if (CachedDisposition != RecomputedDisposition) {
15104 dbgs() << "Cached disposition of " << *S << " for block %"
15105 << BB->getName() << " is incorrect: cached " << CachedDisposition
15106 << ", actual " << RecomputedDisposition << "\n";
15107 std::abort();
15108 }
15109 }
15110 }
15111
15112 // Verify FoldCache/FoldCacheUser caches.
15113 for (auto [FoldID, Expr] : FoldCache) {
15114 auto I = FoldCacheUser.find(Val: Expr);
15115 if (I == FoldCacheUser.end()) {
15116 dbgs() << "Missing entry in FoldCacheUser for cached expression " << *Expr
15117 << "!\n";
15118 std::abort();
15119 }
15120 if (!is_contained(Range: I->second, Element: FoldID)) {
15121 dbgs() << "Missing FoldID in cached users of " << *Expr << "!\n";
15122 std::abort();
15123 }
15124 }
15125 for (auto [Expr, IDs] : FoldCacheUser) {
15126 for (auto &FoldID : IDs) {
15127 const SCEV *S = FoldCache.lookup(Val: FoldID);
15128 if (!S) {
15129 dbgs() << "Missing entry in FoldCache for expression " << *Expr
15130 << "!\n";
15131 std::abort();
15132 }
15133 if (S != Expr) {
15134 dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: " << *S
15135 << " != " << *Expr << "!\n";
15136 std::abort();
15137 }
15138 }
15139 }
15140
15141 // Verify that ConstantMultipleCache computations are correct. We check that
15142 // cached multiples and recomputed multiples are multiples of each other to
15143 // verify correctness. It is possible that a recomputed multiple is different
15144 // from the cached multiple due to strengthened no wrap flags or changes in
15145 // KnownBits computations.
15146 for (auto [S, Multiple] : ConstantMultipleCache) {
15147 APInt RecomputedMultiple = SE2.getConstantMultiple(S);
15148 if ((Multiple != 0 && RecomputedMultiple != 0 &&
15149 Multiple.urem(RHS: RecomputedMultiple) != 0 &&
15150 RecomputedMultiple.urem(RHS: Multiple) != 0)) {
15151 dbgs() << "Incorrect cached computation in ConstantMultipleCache for "
15152 << *S << " : Computed " << RecomputedMultiple
15153 << " but cache contains " << Multiple << "!\n";
15154 std::abort();
15155 }
15156 }
15157}
15158
15159bool ScalarEvolution::invalidate(
15160 Function &F, const PreservedAnalyses &PA,
15161 FunctionAnalysisManager::Invalidator &Inv) {
15162 // Invalidate the ScalarEvolution object whenever it isn't preserved or one
15163 // of its dependencies is invalidated.
15164 auto PAC = PA.getChecker<ScalarEvolutionAnalysis>();
15165 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) ||
15166 Inv.invalidate<AssumptionAnalysis>(IR&: F, PA) ||
15167 Inv.invalidate<DominatorTreeAnalysis>(IR&: F, PA) ||
15168 Inv.invalidate<LoopAnalysis>(IR&: F, PA);
15169}
15170
15171AnalysisKey ScalarEvolutionAnalysis::Key;
15172
15173ScalarEvolution ScalarEvolutionAnalysis::run(Function &F,
15174 FunctionAnalysisManager &AM) {
15175 auto &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
15176 auto &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
15177 auto &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
15178 auto &LI = AM.getResult<LoopAnalysis>(IR&: F);
15179 return ScalarEvolution(F, TLI, AC, DT, LI);
15180}
15181
15182PreservedAnalyses
15183ScalarEvolutionVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
15184 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).verify();
15185 return PreservedAnalyses::all();
15186}
15187
15188PreservedAnalyses
15189ScalarEvolutionPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
15190 // For compatibility with opt's -analyze feature under legacy pass manager
15191 // which was not ported to NPM. This keeps tests using
15192 // update_analyze_test_checks.py working.
15193 OS << "Printing analysis 'Scalar Evolution Analysis' for function '"
15194 << F.getName() << "':\n";
15195 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).print(OS);
15196 return PreservedAnalyses::all();
15197}
15198
15199INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass, "scalar-evolution",
15200 "Scalar Evolution Analysis", false, true)
15201INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
15202INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
15203INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
15204INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
15205INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution",
15206 "Scalar Evolution Analysis", false, true)
15207
15208char ScalarEvolutionWrapperPass::ID = 0;
15209
15210ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) {}
15211
15212bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) {
15213 SE.reset(p: new ScalarEvolution(
15214 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F),
15215 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
15216 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
15217 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
15218 return false;
15219}
15220
15221void ScalarEvolutionWrapperPass::releaseMemory() { SE.reset(); }
15222
15223void ScalarEvolutionWrapperPass::print(raw_ostream &OS, const Module *) const {
15224 SE->print(OS);
15225}
15226
15227void ScalarEvolutionWrapperPass::verifyAnalysis() const {
15228 if (!VerifySCEV)
15229 return;
15230
15231 SE->verify();
15232}
15233
15234void ScalarEvolutionWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
15235 AU.setPreservesAll();
15236 AU.addRequiredTransitive<AssumptionCacheTracker>();
15237 AU.addRequiredTransitive<LoopInfoWrapperPass>();
15238 AU.addRequiredTransitive<DominatorTreeWrapperPass>();
15239 AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>();
15240}
15241
15242const SCEVPredicate *ScalarEvolution::getEqualPredicate(const SCEV *LHS,
15243 const SCEV *RHS) {
15244 return getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS, RHS);
15245}
15246
15247const SCEVPredicate *
15248ScalarEvolution::getComparePredicate(const ICmpInst::Predicate Pred,
15249 const SCEV *LHS, const SCEV *RHS) {
15250 FoldingSetNodeID ID;
15251 assert(LHS->getType() == RHS->getType() &&
15252 "Type mismatch between LHS and RHS");
15253 // Unique this node based on the arguments
15254 ID.AddInteger(I: SCEVPredicate::P_Compare);
15255 ID.AddInteger(I: Pred);
15256 ID.AddPointer(Ptr: LHS);
15257 ID.AddPointer(Ptr: RHS);
15258 void *IP = nullptr;
15259 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, InsertPos&: IP))
15260 return S;
15261 SCEVComparePredicate *Eq = new (SCEVAllocator)
15262 SCEVComparePredicate(ID.Intern(Allocator&: SCEVAllocator), Pred, LHS, RHS);
15263 UniquePreds.InsertNode(N: Eq, InsertPos: IP);
15264 return Eq;
15265}
15266
15267const SCEVPredicate *ScalarEvolution::getWrapPredicate(
15268 const SCEVAddRecExpr *AR,
15269 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15270 FoldingSetNodeID ID;
15271 // Unique this node based on the arguments
15272 ID.AddInteger(I: SCEVPredicate::P_Wrap);
15273 ID.AddPointer(Ptr: AR);
15274 ID.AddInteger(I: AddedFlags);
15275 void *IP = nullptr;
15276 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, InsertPos&: IP))
15277 return S;
15278 auto *OF = new (SCEVAllocator)
15279 SCEVWrapPredicate(ID.Intern(Allocator&: SCEVAllocator), AR, AddedFlags);
15280 UniquePreds.InsertNode(N: OF, InsertPos: IP);
15281 return OF;
15282}
15283
15284namespace {
15285
15286class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
15287public:
15288
15289 /// Rewrites \p S in the context of a loop L and the SCEV predication
15290 /// infrastructure.
15291 ///
15292 /// If \p Pred is non-null, the SCEV expression is rewritten to respect the
15293 /// equivalences present in \p Pred.
15294 ///
15295 /// If \p NewPreds is non-null, rewrite is free to add further predicates to
15296 /// \p NewPreds such that the result will be an AddRecExpr.
15297 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
15298 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15299 const SCEVPredicate *Pred) {
15300 SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred);
15301 return Rewriter.visit(S);
15302 }
15303
15304 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
15305 if (Pred) {
15306 if (auto *U = dyn_cast<SCEVUnionPredicate>(Val: Pred)) {
15307 for (const auto *Pred : U->getPredicates())
15308 if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred))
15309 if (IPred->getLHS() == Expr &&
15310 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15311 return IPred->getRHS();
15312 } else if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred)) {
15313 if (IPred->getLHS() == Expr &&
15314 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15315 return IPred->getRHS();
15316 }
15317 }
15318 return convertToAddRecWithPreds(Expr);
15319 }
15320
15321 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
15322 const SCEV *Operand = visit(S: Expr->getOperand());
15323 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15324 if (AR && AR->getLoop() == L && AR->isAffine()) {
15325 // This couldn't be folded because the operand didn't have the nuw
15326 // flag. Add the nusw flag as an assumption that we could make.
15327 const SCEV *Step = AR->getStepRecurrence(SE);
15328 Type *Ty = Expr->getType();
15329 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNUSW))
15330 return SE.getAddRecExpr(Start: SE.getZeroExtendExpr(Op: AR->getStart(), Ty),
15331 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15332 Flags: AR->getNoWrapFlags());
15333 }
15334 return SE.getZeroExtendExpr(Op: Operand, Ty: Expr->getType());
15335 }
15336
15337 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
15338 const SCEV *Operand = visit(S: Expr->getOperand());
15339 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15340 if (AR && AR->getLoop() == L && AR->isAffine()) {
15341 // This couldn't be folded because the operand didn't have the nsw
15342 // flag. Add the nssw flag as an assumption that we could make.
15343 const SCEV *Step = AR->getStepRecurrence(SE);
15344 Type *Ty = Expr->getType();
15345 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNSSW))
15346 return SE.getAddRecExpr(Start: SE.getSignExtendExpr(Op: AR->getStart(), Ty),
15347 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15348 Flags: AR->getNoWrapFlags());
15349 }
15350 return SE.getSignExtendExpr(Op: Operand, Ty: Expr->getType());
15351 }
15352
15353private:
15354 explicit SCEVPredicateRewriter(
15355 const Loop *L, ScalarEvolution &SE,
15356 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15357 const SCEVPredicate *Pred)
15358 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {}
15359
15360 bool addOverflowAssumption(const SCEVPredicate *P) {
15361 if (!NewPreds) {
15362 // Check if we've already made this assumption.
15363 return Pred && Pred->implies(N: P, SE);
15364 }
15365 NewPreds->push_back(Elt: P);
15366 return true;
15367 }
15368
15369 bool addOverflowAssumption(const SCEVAddRecExpr *AR,
15370 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15371 auto *A = SE.getWrapPredicate(AR, AddedFlags);
15372 return addOverflowAssumption(P: A);
15373 }
15374
15375 // If \p Expr represents a PHINode, we try to see if it can be represented
15376 // as an AddRec, possibly under a predicate (PHISCEVPred). If it is possible
15377 // to add this predicate as a runtime overflow check, we return the AddRec.
15378 // If \p Expr does not meet these conditions (is not a PHI node, or we
15379 // couldn't create an AddRec for it, or couldn't add the predicate), we just
15380 // return \p Expr.
15381 const SCEV *convertToAddRecWithPreds(const SCEVUnknown *Expr) {
15382 if (!isa<PHINode>(Val: Expr->getValue()))
15383 return Expr;
15384 std::optional<
15385 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
15386 PredicatedRewrite = SE.createAddRecFromPHIWithCasts(SymbolicPHI: Expr);
15387 if (!PredicatedRewrite)
15388 return Expr;
15389 for (const auto *P : PredicatedRewrite->second){
15390 // Wrap predicates from outer loops are not supported.
15391 if (auto *WP = dyn_cast<const SCEVWrapPredicate>(Val: P)) {
15392 if (L != WP->getExpr()->getLoop())
15393 return Expr;
15394 }
15395 if (!addOverflowAssumption(P))
15396 return Expr;
15397 }
15398 return PredicatedRewrite->first;
15399 }
15400
15401 SmallVectorImpl<const SCEVPredicate *> *NewPreds;
15402 const SCEVPredicate *Pred;
15403 const Loop *L;
15404};
15405
15406} // end anonymous namespace
15407
15408const SCEV *
15409ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L,
15410 const SCEVPredicate &Preds) {
15411 return SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: nullptr, Pred: &Preds);
15412}
15413
15414const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
15415 const SCEV *S, const Loop *L,
15416 SmallVectorImpl<const SCEVPredicate *> &Preds) {
15417 SmallVector<const SCEVPredicate *> TransformPreds;
15418 S = SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: &TransformPreds, Pred: nullptr);
15419 auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S);
15420
15421 if (!AddRec)
15422 return nullptr;
15423
15424 // Check if any of the transformed predicates is known to be false. In that
15425 // case, it doesn't make sense to convert to a predicated AddRec, as the
15426 // versioned loop will never execute.
15427 for (const SCEVPredicate *Pred : TransformPreds) {
15428 auto *WrapPred = dyn_cast<SCEVWrapPredicate>(Val: Pred);
15429 if (!WrapPred || WrapPred->getFlags() != SCEVWrapPredicate::IncrementNSSW)
15430 continue;
15431
15432 const SCEVAddRecExpr *AddRecToCheck = WrapPred->getExpr();
15433 const SCEV *ExitCount = getBackedgeTakenCount(L: AddRecToCheck->getLoop());
15434 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
15435 continue;
15436
15437 const SCEV *Step = AddRecToCheck->getStepRecurrence(SE&: *this);
15438 if (!Step->isOne())
15439 continue;
15440
15441 ExitCount = getTruncateOrSignExtend(V: ExitCount, Ty: Step->getType());
15442 const SCEV *Add = getAddExpr(LHS: AddRecToCheck->getStart(), RHS: ExitCount);
15443 if (isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS: Add, RHS: AddRecToCheck->getStart()))
15444 return nullptr;
15445 }
15446
15447 // Since the transformation was successful, we can now transfer the SCEV
15448 // predicates.
15449 Preds.append(in_start: TransformPreds.begin(), in_end: TransformPreds.end());
15450
15451 return AddRec;
15452}
15453
15454/// SCEV predicates
15455SCEVPredicate::SCEVPredicate(const FoldingSetNodeIDRef ID,
15456 SCEVPredicateKind Kind)
15457 : FastID(ID), Kind(Kind) {}
15458
15459SCEVComparePredicate::SCEVComparePredicate(const FoldingSetNodeIDRef ID,
15460 const ICmpInst::Predicate Pred,
15461 const SCEV *LHS, const SCEV *RHS)
15462 : SCEVPredicate(ID, P_Compare), Pred(Pred), LHS(LHS), RHS(RHS) {
15463 assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match");
15464 assert(LHS != RHS && "LHS and RHS are the same SCEV");
15465}
15466
15467bool SCEVComparePredicate::implies(const SCEVPredicate *N,
15468 ScalarEvolution &SE) const {
15469 const auto *Op = dyn_cast<SCEVComparePredicate>(Val: N);
15470
15471 if (!Op)
15472 return false;
15473
15474 if (Pred != ICmpInst::ICMP_EQ)
15475 return false;
15476
15477 return Op->LHS == LHS && Op->RHS == RHS;
15478}
15479
15480bool SCEVComparePredicate::isAlwaysTrue() const { return false; }
15481
15482void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
15483 if (Pred == ICmpInst::ICMP_EQ)
15484 OS.indent(NumSpaces: Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n";
15485 else
15486 OS.indent(NumSpaces: Depth) << "Compare predicate: " << *LHS << " " << Pred << ") "
15487 << *RHS << "\n";
15488
15489}
15490
15491SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
15492 const SCEVAddRecExpr *AR,
15493 IncrementWrapFlags Flags)
15494 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {}
15495
15496const SCEVAddRecExpr *SCEVWrapPredicate::getExpr() const { return AR; }
15497
15498bool SCEVWrapPredicate::implies(const SCEVPredicate *N,
15499 ScalarEvolution &SE) const {
15500 const auto *Op = dyn_cast<SCEVWrapPredicate>(Val: N);
15501 if (!Op || setFlags(Flags, OnFlags: Op->Flags) != Flags)
15502 return false;
15503
15504 if (Op->AR == AR)
15505 return true;
15506
15507 if (Flags != SCEVWrapPredicate::IncrementNSSW &&
15508 Flags != SCEVWrapPredicate::IncrementNUSW)
15509 return false;
15510
15511 const SCEV *Start = AR->getStart();
15512 const SCEV *OpStart = Op->AR->getStart();
15513 if (Start->getType()->isPointerTy() != OpStart->getType()->isPointerTy())
15514 return false;
15515
15516 // Reject pointers to different address spaces.
15517 if (Start->getType()->isPointerTy() && Start->getType() != OpStart->getType())
15518 return false;
15519
15520 // NUSW/NSSW on a wider-type AddRec does not imply the same on a
15521 // narrower-type AddRec.
15522 if (SE.getTypeSizeInBits(Ty: AR->getType()) >
15523 SE.getTypeSizeInBits(Ty: Op->AR->getType()))
15524 return false;
15525
15526 const SCEV *Step = AR->getStepRecurrence(SE);
15527 const SCEV *OpStep = Op->AR->getStepRecurrence(SE);
15528 if (!SE.isKnownPositive(S: Step) || !SE.isKnownPositive(S: OpStep))
15529 return false;
15530
15531 // If both steps are positive, this implies N, if N's start and step are
15532 // ULE/SLE (for NSUW/NSSW) than this'.
15533 Type *WiderTy = SE.getWiderType(T1: Step->getType(), T2: OpStep->getType());
15534 Step = SE.getNoopOrZeroExtend(V: Step, Ty: WiderTy);
15535 OpStep = SE.getNoopOrZeroExtend(V: OpStep, Ty: WiderTy);
15536
15537 bool IsNUW = Flags == SCEVWrapPredicate::IncrementNUSW;
15538 OpStart = IsNUW ? SE.getNoopOrZeroExtend(V: OpStart, Ty: WiderTy)
15539 : SE.getNoopOrSignExtend(V: OpStart, Ty: WiderTy);
15540 Start = IsNUW ? SE.getNoopOrZeroExtend(V: Start, Ty: WiderTy)
15541 : SE.getNoopOrSignExtend(V: Start, Ty: WiderTy);
15542 CmpInst::Predicate Pred = IsNUW ? CmpInst::ICMP_ULE : CmpInst::ICMP_SLE;
15543 return SE.isKnownPredicate(Pred, LHS: OpStep, RHS: Step) &&
15544 SE.isKnownPredicate(Pred, LHS: OpStart, RHS: Start);
15545}
15546
15547bool SCEVWrapPredicate::isAlwaysTrue() const {
15548 SCEV::NoWrapFlags ScevFlags = AR->getNoWrapFlags();
15549 IncrementWrapFlags IFlags = Flags;
15550
15551 if (ScalarEvolution::setFlags(Flags: ScevFlags, OnFlags: SCEV::FlagNSW) == ScevFlags)
15552 IFlags = clearFlags(Flags: IFlags, OffFlags: IncrementNSSW);
15553
15554 return IFlags == IncrementAnyWrap;
15555}
15556
15557void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const {
15558 OS.indent(NumSpaces: Depth) << *getExpr() << " Added Flags: ";
15559 if (SCEVWrapPredicate::IncrementNUSW & getFlags())
15560 OS << "<nusw>";
15561 if (SCEVWrapPredicate::IncrementNSSW & getFlags())
15562 OS << "<nssw>";
15563 OS << "\n";
15564}
15565
15566SCEVWrapPredicate::IncrementWrapFlags
15567SCEVWrapPredicate::getImpliedFlags(const SCEVAddRecExpr *AR,
15568 ScalarEvolution &SE) {
15569 IncrementWrapFlags ImpliedFlags = IncrementAnyWrap;
15570 SCEV::NoWrapFlags StaticFlags = AR->getNoWrapFlags();
15571
15572 // We can safely transfer the NSW flag as NSSW.
15573 if (ScalarEvolution::setFlags(Flags: StaticFlags, OnFlags: SCEV::FlagNSW) == StaticFlags)
15574 ImpliedFlags = IncrementNSSW;
15575
15576 if (ScalarEvolution::setFlags(Flags: StaticFlags, OnFlags: SCEV::FlagNUW) == StaticFlags) {
15577 // If the increment is positive, the SCEV NUW flag will also imply the
15578 // WrapPredicate NUSW flag.
15579 if (const auto *Step = dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE)))
15580 if (Step->getValue()->getValue().isNonNegative())
15581 ImpliedFlags = setFlags(Flags: ImpliedFlags, OnFlags: IncrementNUSW);
15582 }
15583
15584 return ImpliedFlags;
15585}
15586
15587/// Union predicates don't get cached so create a dummy set ID for it.
15588SCEVUnionPredicate::SCEVUnionPredicate(ArrayRef<const SCEVPredicate *> Preds,
15589 ScalarEvolution &SE)
15590 : SCEVPredicate(FoldingSetNodeIDRef(nullptr, 0), P_Union) {
15591 for (const auto *P : Preds)
15592 add(N: P, SE);
15593}
15594
15595bool SCEVUnionPredicate::isAlwaysTrue() const {
15596 return all_of(Range: Preds,
15597 P: [](const SCEVPredicate *I) { return I->isAlwaysTrue(); });
15598}
15599
15600bool SCEVUnionPredicate::implies(const SCEVPredicate *N,
15601 ScalarEvolution &SE) const {
15602 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N))
15603 return all_of(Range: Set->Preds, P: [this, &SE](const SCEVPredicate *I) {
15604 return this->implies(N: I, SE);
15605 });
15606
15607 if (any_of(Range: Preds,
15608 P: [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); }))
15609 return true;
15610
15611 // A wrap predicate may be implied by a wrap predicate in Preds after applying
15612 // equal predicates.
15613 const auto *NWrap = dyn_cast<SCEVWrapPredicate>(Val: N);
15614 if (!NWrap)
15615 return false;
15616 const Loop *L = NWrap->getExpr()->getLoop();
15617 return any_of(Range: Preds, P: [&](const SCEVPredicate *I) {
15618 const auto *IWrap = dyn_cast<SCEVWrapPredicate>(Val: I);
15619 if (!IWrap)
15620 return false;
15621 const auto *RewrittenAR = dyn_cast<SCEVAddRecExpr>(
15622 Val: SE.rewriteUsingPredicate(S: IWrap->getExpr(), L, Preds: *this));
15623 return RewrittenAR &&
15624 SE.getWrapPredicate(AR: RewrittenAR, AddedFlags: IWrap->getFlags())->implies(N, SE);
15625 });
15626}
15627
15628void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const {
15629 for (const auto *Pred : Preds)
15630 Pred->print(OS, Depth);
15631}
15632
15633void SCEVUnionPredicate::add(const SCEVPredicate *N, ScalarEvolution &SE) {
15634 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N)) {
15635 for (const auto *Pred : Set->Preds)
15636 add(N: Pred, SE);
15637 return;
15638 }
15639
15640 // Implication checks are quadratic in the number of predicates. Stop doing
15641 // them if there are many predicates, as they should be too expensive to use
15642 // anyway at that point.
15643 bool CheckImplies = Preds.size() < 16;
15644
15645 // Only add predicate if it is not already implied by this union predicate.
15646 if (CheckImplies && implies(N, SE))
15647 return;
15648
15649 // Build a new vector containing the current predicates, except the ones that
15650 // are implied by the new predicate N.
15651 SmallVector<const SCEVPredicate *> PrunedPreds;
15652 for (auto *P : Preds) {
15653 if (CheckImplies && N->implies(N: P, SE))
15654 continue;
15655 PrunedPreds.push_back(Elt: P);
15656 }
15657 Preds = std::move(PrunedPreds);
15658 Preds.push_back(Elt: N);
15659}
15660
15661PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE,
15662 Loop &L)
15663 : SE(SE), L(L) {
15664 SmallVector<const SCEVPredicate*, 4> Empty;
15665 Preds = std::make_unique<SCEVUnionPredicate>(args&: Empty, args&: SE);
15666}
15667
15668void ScalarEvolution::registerUser(const SCEV *User,
15669 ArrayRef<const SCEV *> Ops) {
15670 for (const auto *Op : Ops)
15671 // We do not expect that forgetting cached data for SCEVConstants will ever
15672 // open any prospects for sharpening or introduce any correctness issues,
15673 // so we don't bother storing their dependencies.
15674 if (!isa<SCEVConstant>(Val: Op))
15675 SCEVUsers[Op].insert(Ptr: User);
15676}
15677
15678void ScalarEvolution::registerUser(const SCEV *User, ArrayRef<SCEVUse> Ops) {
15679 for (const SCEV *Op : Ops)
15680 // We do not expect that forgetting cached data for SCEVConstants will ever
15681 // open any prospects for sharpening or introduce any correctness issues,
15682 // so we don't bother storing their dependencies.
15683 if (!isa<SCEVConstant>(Val: Op))
15684 SCEVUsers[Op].insert(Ptr: User);
15685}
15686
15687const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) {
15688 const SCEV *Expr = SE.getSCEV(V);
15689 return getPredicatedSCEV(Expr);
15690}
15691
15692const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
15693 RewriteEntry &Entry = RewriteMap[Expr];
15694
15695 // If we already have an entry and the version matches, return it.
15696 if (Entry.second && Generation == Entry.first)
15697 return Entry.second;
15698
15699 // We found an entry but it's stale. Rewrite the stale entry
15700 // according to the current predicate.
15701 if (Entry.second)
15702 Expr = Entry.second;
15703
15704 const SCEV *NewSCEV = SE.rewriteUsingPredicate(S: Expr, L: &L, Preds: *Preds);
15705 Entry = {Generation, NewSCEV};
15706
15707 return NewSCEV;
15708}
15709
15710const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
15711 if (!BackedgeCount) {
15712 SmallVector<const SCEVPredicate *, 4> Preds;
15713 BackedgeCount = SE.getPredicatedBackedgeTakenCount(L: &L, Preds);
15714 for (const auto *P : Preds)
15715 addPredicate(Pred: *P);
15716 }
15717 return BackedgeCount;
15718}
15719
15720const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
15721 if (!SymbolicMaxBackedgeCount) {
15722 SmallVector<const SCEVPredicate *, 4> Preds;
15723 SymbolicMaxBackedgeCount =
15724 SE.getPredicatedSymbolicMaxBackedgeTakenCount(L: &L, Preds);
15725 for (const auto *P : Preds)
15726 addPredicate(Pred: *P);
15727 }
15728 return SymbolicMaxBackedgeCount;
15729}
15730
15731unsigned PredicatedScalarEvolution::getSmallConstantMaxTripCount() {
15732 if (!SmallConstantMaxTripCount) {
15733 SmallVector<const SCEVPredicate *, 4> Preds;
15734 SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(L: &L, Predicates: &Preds);
15735 for (const auto *P : Preds)
15736 addPredicate(Pred: *P);
15737 }
15738 return *SmallConstantMaxTripCount;
15739}
15740
15741void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
15742 if (Preds->implies(N: &Pred, SE))
15743 return;
15744
15745 SmallVector<const SCEVPredicate *, 4> NewPreds(Preds->getPredicates());
15746 NewPreds.push_back(Elt: &Pred);
15747 Preds = std::make_unique<SCEVUnionPredicate>(args&: NewPreds, args&: SE);
15748 updateGeneration();
15749}
15750
15751void PredicatedScalarEvolution::addPredicates(
15752 ArrayRef<const SCEVPredicate *> Preds) {
15753 for (const SCEVPredicate *P : Preds)
15754 addPredicate(Pred: *P);
15755}
15756
15757const SCEVPredicate &PredicatedScalarEvolution::getPredicate() const {
15758 return *Preds;
15759}
15760
15761void PredicatedScalarEvolution::updateGeneration() {
15762 // If the generation number wrapped recompute everything.
15763 if (++Generation == 0) {
15764 for (auto &II : RewriteMap) {
15765 const SCEV *Rewritten = II.second.second;
15766 II.second = {Generation, SE.rewriteUsingPredicate(S: Rewritten, L: &L, Preds: *Preds)};
15767 }
15768 }
15769}
15770
15771bool PredicatedScalarEvolution::hasNoOverflow(
15772 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) {
15773 const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: getSCEV(V));
15774 if (!AR)
15775 return false;
15776
15777 Flags = SCEVWrapPredicate::clearFlags(
15778 Flags, OffFlags: SCEVWrapPredicate::getImpliedFlags(AR, SE));
15779
15780 return Flags == SCEVWrapPredicate::IncrementAnyWrap;
15781}
15782
15783const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(
15784 Value *V, SmallVectorImpl<const SCEVPredicate *> *ExtraPreds) {
15785 const SCEV *Expr = this->getSCEV(V);
15786 SmallVector<const SCEVPredicate *, 4> NewPreds;
15787 auto *New = SE.convertSCEVToAddRecWithPredicates(S: Expr, L: &L, Preds&: NewPreds);
15788
15789 if (!New)
15790 return nullptr;
15791
15792 if (ExtraPreds) {
15793 ExtraPreds->append(RHS: NewPreds);
15794 return New;
15795 }
15796
15797 addPredicates(Preds: NewPreds);
15798
15799 RewriteMap[SE.getSCEV(V)] = {Generation, New};
15800 return New;
15801}
15802
15803PredicatedScalarEvolution::PredicatedScalarEvolution(
15804 const PredicatedScalarEvolution &Init)
15805 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L),
15806 Preds(std::make_unique<SCEVUnionPredicate>(args: Init.Preds->getPredicates(),
15807 args&: SE)),
15808 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {}
15809
15810void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const {
15811 // For each block.
15812 for (auto *BB : L.getBlocks())
15813 for (auto &I : *BB) {
15814 if (!SE.isSCEVable(Ty: I.getType()))
15815 continue;
15816
15817 auto *Expr = SE.getSCEV(V: &I);
15818 auto II = RewriteMap.find(Val: Expr);
15819
15820 if (II == RewriteMap.end())
15821 continue;
15822
15823 // Don't print things that are not interesting.
15824 if (II->second.second == Expr)
15825 continue;
15826
15827 OS.indent(NumSpaces: Depth) << "[PSE]" << I << ":\n";
15828 OS.indent(NumSpaces: Depth + 2) << *Expr << "\n";
15829 OS.indent(NumSpaces: Depth + 2) << "--> " << *II->second.second << "\n";
15830 }
15831}
15832
15833ScalarEvolution::LoopGuards
15834ScalarEvolution::LoopGuards::collect(const Loop *L, ScalarEvolution &SE) {
15835 BasicBlock *Header = L->getHeader();
15836 BasicBlock *Pred = L->getLoopPredecessor();
15837 LoopGuards Guards(SE);
15838 if (!Pred)
15839 return Guards;
15840 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
15841 collectFromBlock(SE, Guards, Block: Header, Pred, VisitedBlocks);
15842 return Guards;
15843}
15844
15845void ScalarEvolution::LoopGuards::collectFromPHI(
15846 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15847 const PHINode &Phi, SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks,
15848 SmallDenseMap<const BasicBlock *, LoopGuards> &IncomingGuards,
15849 unsigned Depth) {
15850 if (!SE.isSCEVable(Ty: Phi.getType()))
15851 return;
15852
15853 using MinMaxPattern = std::pair<const SCEVConstant *, SCEVTypes>;
15854 auto GetMinMaxConst = [&](unsigned IncomingIdx) -> MinMaxPattern {
15855 const BasicBlock *InBlock = Phi.getIncomingBlock(i: IncomingIdx);
15856 if (!VisitedBlocks.insert(Ptr: InBlock).second)
15857 return {nullptr, scCouldNotCompute};
15858
15859 // Avoid analyzing unreachable blocks so that we don't get trapped
15860 // traversing cycles with ill-formed dominance or infinite cycles
15861 if (!SE.DT.isReachableFromEntry(A: InBlock))
15862 return {nullptr, scCouldNotCompute};
15863
15864 auto [G, Inserted] = IncomingGuards.try_emplace(Key: InBlock, Args: LoopGuards(SE));
15865 if (Inserted)
15866 collectFromBlock(SE, Guards&: G->second, Block: Phi.getParent(), Pred: InBlock, VisitedBlocks,
15867 Depth: Depth + 1);
15868 auto &RewriteMap = G->second.RewriteMap;
15869 if (RewriteMap.empty())
15870 return {nullptr, scCouldNotCompute};
15871 auto S = RewriteMap.find(Val: SE.getSCEV(V: Phi.getIncomingValue(i: IncomingIdx)));
15872 if (S == RewriteMap.end())
15873 return {nullptr, scCouldNotCompute};
15874 auto *SM = dyn_cast_if_present<SCEVMinMaxExpr>(Val: S->second);
15875 if (!SM)
15876 return {nullptr, scCouldNotCompute};
15877 if (const SCEVConstant *C0 = dyn_cast<SCEVConstant>(Val: SM->getOperand(i: 0)))
15878 return {C0, SM->getSCEVType()};
15879 return {nullptr, scCouldNotCompute};
15880 };
15881 auto MergeMinMaxConst = [](MinMaxPattern P1,
15882 MinMaxPattern P2) -> MinMaxPattern {
15883 auto [C1, T1] = P1;
15884 auto [C2, T2] = P2;
15885 if (!C1 || !C2 || T1 != T2)
15886 return {nullptr, scCouldNotCompute};
15887 switch (T1) {
15888 case scUMaxExpr:
15889 return {C1->getAPInt().ult(RHS: C2->getAPInt()) ? C1 : C2, T1};
15890 case scSMaxExpr:
15891 return {C1->getAPInt().slt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15892 case scUMinExpr:
15893 return {C1->getAPInt().ugt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15894 case scSMinExpr:
15895 return {C1->getAPInt().sgt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15896 default:
15897 llvm_unreachable("Trying to merge non-MinMaxExpr SCEVs.");
15898 }
15899 };
15900 auto P = GetMinMaxConst(0);
15901 for (unsigned int In = 1; In < Phi.getNumIncomingValues(); In++) {
15902 if (!P.first)
15903 break;
15904 P = MergeMinMaxConst(P, GetMinMaxConst(In));
15905 }
15906 if (P.first) {
15907 const SCEV *LHS = SE.getSCEV(V: const_cast<PHINode *>(&Phi));
15908 SmallVector<SCEVUse, 2> Ops({P.first, LHS});
15909 const SCEV *RHS = SE.getMinMaxExpr(Kind: P.second, Ops);
15910 Guards.RewriteMap.insert(KV: {LHS, RHS});
15911 }
15912}
15913
15914// Return a new SCEV that modifies \p Expr to the closest number divides by
15915// \p Divisor and less or equal than Expr. For now, only handle constant
15916// Expr.
15917static const SCEV *getPreviousSCEVDivisibleByDivisor(const SCEV *Expr,
15918 const APInt &DivisorVal,
15919 ScalarEvolution &SE) {
15920 const APInt *ExprVal;
15921 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15922 DivisorVal.isNonPositive())
15923 return Expr;
15924 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15925 // return the SCEV: Expr - Expr % Divisor
15926 return SE.getConstant(Val: *ExprVal - Rem);
15927}
15928
15929// Return a new SCEV that modifies \p Expr to the closest number divides by
15930// \p Divisor and greater or equal than Expr. For now, only handle constant
15931// Expr.
15932static const SCEV *getNextSCEVDivisibleByDivisor(const SCEV *Expr,
15933 const APInt &DivisorVal,
15934 ScalarEvolution &SE) {
15935 const APInt *ExprVal;
15936 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15937 DivisorVal.isNonPositive())
15938 return Expr;
15939 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15940 if (Rem.isZero())
15941 return Expr;
15942 // return the SCEV: Expr + Divisor - Expr % Divisor
15943 return SE.getConstant(Val: *ExprVal + DivisorVal - Rem);
15944}
15945
15946static bool collectDivisibilityInformation(
15947 ICmpInst::Predicate Predicate, const SCEV *LHS, const SCEV *RHS,
15948 DenseMap<const SCEV *, const SCEV *> &DivInfo,
15949 DenseMap<const SCEV *, APInt> &Multiples, ScalarEvolution &SE) {
15950 // If we have LHS == 0, check if LHS is computing a property of some unknown
15951 // SCEV %v which we can rewrite %v to express explicitly.
15952 if (Predicate != CmpInst::ICMP_EQ || !match(S: RHS, P: m_scev_Zero()))
15953 return false;
15954 // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to
15955 // explicitly express that.
15956 const SCEVUnknown *URemLHS = nullptr;
15957 const SCEV *URemRHS = nullptr;
15958 if (!match(S: LHS, P: m_scev_URem(LHS: m_SCEVUnknown(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE)))
15959 return false;
15960
15961 const SCEV *Multiple =
15962 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: URemLHS, RHS: URemRHS), RHS: URemRHS);
15963 DivInfo[URemLHS] = Multiple;
15964 if (auto *C = dyn_cast<SCEVConstant>(Val: URemRHS))
15965 Multiples[URemLHS] = C->getAPInt();
15966 return true;
15967}
15968
15969// Check if the condition is a divisibility guard (A % B == 0).
15970static bool isDivisibilityGuard(const SCEV *LHS, const SCEV *RHS,
15971 ScalarEvolution &SE) {
15972 const SCEV *X, *Y;
15973 return match(S: LHS, P: m_scev_URem(LHS: m_SCEV(V&: X), RHS: m_SCEV(V&: Y), SE)) && RHS->isZero();
15974}
15975
15976// Apply divisibility by \p Divisor on MinMaxExpr with constant values,
15977// recursively. This is done by aligning up/down the constant value to the
15978// Divisor.
15979static const SCEV *applyDivisibilityOnMinMaxExpr(const SCEV *MinMaxExpr,
15980 APInt Divisor,
15981 ScalarEvolution &SE) {
15982 // Return true if \p Expr is a MinMax SCEV expression with a non-negative
15983 // constant operand. If so, return in \p SCTy the SCEV type and in \p RHS
15984 // the non-constant operand and in \p LHS the constant operand.
15985 auto IsMinMaxSCEVWithNonNegativeConstant =
15986 [&](const SCEV *Expr, SCEVTypes &SCTy, const SCEV *&LHS,
15987 const SCEV *&RHS) {
15988 if (auto *MinMax = dyn_cast<SCEVMinMaxExpr>(Val: Expr)) {
15989 if (MinMax->getNumOperands() != 2)
15990 return false;
15991 if (auto *C = dyn_cast<SCEVConstant>(Val: MinMax->getOperand(i: 0))) {
15992 if (C->getAPInt().isNegative())
15993 return false;
15994 SCTy = MinMax->getSCEVType();
15995 LHS = MinMax->getOperand(i: 0);
15996 RHS = MinMax->getOperand(i: 1);
15997 return true;
15998 }
15999 }
16000 return false;
16001 };
16002
16003 const SCEV *MinMaxLHS = nullptr, *MinMaxRHS = nullptr;
16004 SCEVTypes SCTy;
16005 if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS,
16006 MinMaxRHS))
16007 return MinMaxExpr;
16008 auto IsMin = isa<SCEVSMinExpr>(Val: MinMaxExpr) || isa<SCEVUMinExpr>(Val: MinMaxExpr);
16009 assert(SE.isKnownNonNegative(MinMaxLHS) && "Expected non-negative operand!");
16010 auto *DivisibleExpr =
16011 IsMin ? getPreviousSCEVDivisibleByDivisor(Expr: MinMaxLHS, DivisorVal: Divisor, SE)
16012 : getNextSCEVDivisibleByDivisor(Expr: MinMaxLHS, DivisorVal: Divisor, SE);
16013 SmallVector<SCEVUse> Ops = {
16014 applyDivisibilityOnMinMaxExpr(MinMaxExpr: MinMaxRHS, Divisor, SE), DivisibleExpr};
16015 return SE.getMinMaxExpr(Kind: SCTy, Ops);
16016}
16017
16018void ScalarEvolution::LoopGuards::collectFromBlock(
16019 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
16020 const BasicBlock *Block, const BasicBlock *Pred,
16021 SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks, unsigned Depth) {
16022
16023 assert(SE.DT.isReachableFromEntry(Block) && SE.DT.isReachableFromEntry(Pred));
16024
16025 SmallVector<SCEVUse> ExprsToRewrite;
16026 auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS,
16027 const SCEV *RHS,
16028 DenseMap<const SCEV *, const SCEV *> &RewriteMap,
16029 const LoopGuards &DivGuards) {
16030 // WARNING: It is generally unsound to apply any wrap flags to the proposed
16031 // replacement SCEV which isn't directly implied by the structure of that
16032 // SCEV. In particular, using contextual facts to imply flags is *NOT*
16033 // legal. See the scoping rules for flags in the header to understand why.
16034
16035 // Puts rewrite rule \p From -> \p To into the rewrite map. Also if \p From
16036 // and \p FromRewritten are the same (i.e. there has been no rewrite
16037 // registered for \p From), then puts this value in the list of rewritten
16038 // expressions.
16039 auto AddRewrite = [&](const SCEV *From, const SCEV *FromRewritten,
16040 const SCEV *To) {
16041 if (From == FromRewritten)
16042 ExprsToRewrite.push_back(Elt: From);
16043 RewriteMap[From] = To;
16044 };
16045
16046 // Checks whether \p S has already been rewritten. In that case returns the
16047 // existing rewrite because we want to chain further rewrites onto the
16048 // already rewritten value. Otherwise returns \p S.
16049 auto GetMaybeRewritten = [&](const SCEV *S) {
16050 return RewriteMap.lookup_or(Val: S, Default&: S);
16051 };
16052
16053 // Check for a condition of the form (-C1 + X < C2). InstCombine will
16054 // create this form when combining two checks of the form (X u< C2 + C1) and
16055 // (X >=u C1).
16056 auto MatchRangeCheckIdiom = [&](ICmpInst::Predicate Pred,
16057 const SCEV *MatchLHS,
16058 const SCEV *MatchRHS) {
16059 const SCEVConstant *C1;
16060 const SCEVUnknown *LHSUnknown;
16061 auto *C2 = dyn_cast<SCEVConstant>(Val: MatchRHS);
16062 if (!match(S: MatchLHS,
16063 P: m_scev_Add(Op0: m_SCEVConstant(V&: C1), Op1: m_SCEVUnknown(V&: LHSUnknown))) ||
16064 !C2)
16065 return false;
16066
16067 auto ExactRegion =
16068 ConstantRange::makeExactICmpRegion(Pred, Other: C2->getAPInt())
16069 .sub(Other: C1->getAPInt());
16070
16071 // Tighten the raw range with what we already know about LHSUnknown
16072 // from prior guards recorded in RewriteMap, or from SCEV's own range
16073 // analysis.
16074 const SCEV *RewrittenLHS = GetMaybeRewritten(LHSUnknown);
16075 ExactRegion = ExactRegion.intersectWith(CR: SE.getUnsignedRange(S: RewrittenLHS),
16076 Type: ConstantRange::Unsigned);
16077
16078 // Bail if the guard is inconsistent with prior facts, or if the range
16079 // is still not a monotonic non-wrapping interval after tightening.
16080 if (ExactRegion.isEmptySet() || ExactRegion.isWrappedSet() ||
16081 ExactRegion.isFullSet())
16082 return false;
16083
16084 const SCEV *RegionMin = SE.getConstant(Val: ExactRegion.getUnsignedMin());
16085 const SCEV *RegionMax = SE.getConstant(Val: ExactRegion.getUnsignedMax());
16086 const SCEV *ClampedLHS =
16087 SE.getUMaxExpr(LHS: RegionMin, RHS: SE.getUMinExpr(LHS: RewrittenLHS, RHS: RegionMax));
16088 AddRewrite(LHSUnknown, RewrittenLHS, ClampedLHS);
16089 return true;
16090 };
16091 if (MatchRangeCheckIdiom(Predicate, LHS, RHS))
16092 return;
16093
16094 // Do not apply information for constants or if RHS contains an AddRec.
16095 if (isa<SCEVConstant>(Val: LHS) || SE.containsAddRecurrence(S: RHS))
16096 return;
16097
16098 // If RHS is SCEVUnknown, make sure the information is applied to it.
16099 if (!isa<SCEVUnknown>(Val: LHS) && isa<SCEVUnknown>(Val: RHS)) {
16100 std::swap(a&: LHS, b&: RHS);
16101 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
16102 }
16103
16104 const SCEV *RewrittenLHS = GetMaybeRewritten(LHS);
16105 // Apply divisibility information when computing the constant multiple.
16106 const APInt &DividesBy =
16107 SE.getConstantMultiple(S: DivGuards.rewrite(Expr: RewrittenLHS));
16108
16109 // Collect rewrites for LHS and its transitive operands based on the
16110 // condition.
16111 // For min/max expressions, also apply the guard to its operands:
16112 // 'min(a, b) >= c' -> '(a >= c) and (b >= c)',
16113 // 'min(a, b) > c' -> '(a > c) and (b > c)',
16114 // 'max(a, b) <= c' -> '(a <= c) and (b <= c)',
16115 // 'max(a, b) < c' -> '(a < c) and (b < c)'.
16116
16117 // We cannot express strict predicates in SCEV, so instead we replace them
16118 // with non-strict ones against plus or minus one of RHS depending on the
16119 // predicate.
16120 const SCEV *One = SE.getOne(Ty: RHS->getType());
16121 switch (Predicate) {
16122 case CmpInst::ICMP_ULT:
16123 if (RHS->getType()->isPointerTy())
16124 return;
16125 RHS = SE.getUMaxExpr(LHS: RHS, RHS: One);
16126 [[fallthrough]];
16127 case CmpInst::ICMP_SLT: {
16128 RHS = SE.getMinusSCEV(LHS: RHS, RHS: One);
16129 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16130 break;
16131 }
16132 case CmpInst::ICMP_UGT:
16133 case CmpInst::ICMP_SGT:
16134 RHS = SE.getAddExpr(LHS: RHS, RHS: One);
16135 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16136 break;
16137 case CmpInst::ICMP_ULE:
16138 case CmpInst::ICMP_SLE:
16139 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16140 break;
16141 case CmpInst::ICMP_UGE:
16142 case CmpInst::ICMP_SGE:
16143 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16144 break;
16145 default:
16146 break;
16147 }
16148
16149 SmallVector<SCEVUse, 16> Worklist(1, LHS);
16150 SmallPtrSet<const SCEV *, 16> Visited;
16151
16152 auto EnqueueOperands = [&Worklist](const SCEVNAryExpr *S) {
16153 append_range(C&: Worklist, R: S->operands());
16154 };
16155
16156 while (!Worklist.empty()) {
16157 const SCEV *From = Worklist.pop_back_val();
16158 if (isa<SCEVConstant>(Val: From))
16159 continue;
16160 if (!Visited.insert(Ptr: From).second)
16161 continue;
16162 const SCEV *FromRewritten = GetMaybeRewritten(From);
16163 const SCEV *To = nullptr;
16164
16165 switch (Predicate) {
16166 case CmpInst::ICMP_ULT:
16167 case CmpInst::ICMP_ULE:
16168 To = SE.getUMinExpr(LHS: FromRewritten, RHS);
16169 if (auto *UMax = dyn_cast<SCEVUMaxExpr>(Val: FromRewritten))
16170 EnqueueOperands(UMax);
16171 break;
16172 case CmpInst::ICMP_SLT:
16173 case CmpInst::ICMP_SLE:
16174 To = SE.getSMinExpr(LHS: FromRewritten, RHS);
16175 if (auto *SMax = dyn_cast<SCEVSMaxExpr>(Val: FromRewritten))
16176 EnqueueOperands(SMax);
16177 break;
16178 case CmpInst::ICMP_UGT:
16179 case CmpInst::ICMP_UGE:
16180 To = SE.getUMaxExpr(LHS: FromRewritten, RHS);
16181 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: FromRewritten))
16182 EnqueueOperands(UMin);
16183 break;
16184 case CmpInst::ICMP_SGT:
16185 case CmpInst::ICMP_SGE:
16186 To = SE.getSMaxExpr(LHS: FromRewritten, RHS);
16187 if (auto *SMin = dyn_cast<SCEVSMinExpr>(Val: FromRewritten))
16188 EnqueueOperands(SMin);
16189 break;
16190 case CmpInst::ICMP_EQ:
16191 if (isa<SCEVConstant>(Val: RHS))
16192 To = RHS;
16193 break;
16194 case CmpInst::ICMP_NE:
16195 if (match(S: RHS, P: m_scev_Zero())) {
16196 const SCEV *OneAlignedUp =
16197 getNextSCEVDivisibleByDivisor(Expr: One, DivisorVal: DividesBy, SE);
16198 To = SE.getUMaxExpr(LHS: FromRewritten, RHS: OneAlignedUp);
16199 } else {
16200 // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS),
16201 // but creating the subtraction eagerly is expensive. Track the
16202 // inequalities in a separate map, and materialize the rewrite lazily
16203 // when encountering a suitable subtraction while re-writing.
16204 if (LHS->getType()->isPointerTy()) {
16205 LHS = SE.getLosslessPtrToIntExpr(Op: LHS);
16206 RHS = SE.getLosslessPtrToIntExpr(Op: RHS);
16207 if (isa<SCEVCouldNotCompute>(Val: LHS) || isa<SCEVCouldNotCompute>(Val: RHS))
16208 break;
16209 }
16210 const SCEVConstant *C;
16211 const SCEV *A, *B;
16212 if (match(S: RHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: A))) &&
16213 match(S: LHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: B)))) {
16214 RHS = A;
16215 LHS = B;
16216 }
16217 if (LHS > RHS)
16218 std::swap(a&: LHS, b&: RHS);
16219 Guards.NotEqual.insert(V: {LHS, RHS});
16220 continue;
16221 }
16222 break;
16223 default:
16224 break;
16225 }
16226
16227 if (To)
16228 AddRewrite(From, FromRewritten, To);
16229 }
16230 };
16231
16232 SmallVector<PointerIntPair<Value *, 1, bool>> Terms;
16233 // First, collect information from assumptions dominating the loop.
16234 for (auto &AssumeVH : SE.AC.assumptions()) {
16235 if (!AssumeVH)
16236 continue;
16237 auto *AssumeI = cast<CallInst>(Val&: AssumeVH);
16238 if (!SE.DT.dominates(Def: AssumeI, BB: Block))
16239 continue;
16240 Terms.emplace_back(Args: AssumeI->getOperand(i_nocapture: 0), Args: true);
16241 }
16242
16243 // Second, collect information from llvm.experimental.guards dominating the loop.
16244 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
16245 M: SE.F.getParent(), id: Intrinsic::experimental_guard);
16246 if (GuardDecl)
16247 for (const auto *GU : GuardDecl->users())
16248 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
16249 if (Guard->getFunction() == Block->getParent() &&
16250 SE.DT.dominates(Def: Guard, BB: Block))
16251 Terms.emplace_back(Args: Guard->getArgOperand(i: 0), Args: true);
16252
16253 // Third, collect conditions from dominating branches. Starting at the loop
16254 // predecessor, climb up the predecessor chain, as long as there are
16255 // predecessors that can be found that have unique successors leading to the
16256 // original header.
16257 // TODO: share this logic with isLoopEntryGuardedByCond.
16258 unsigned NumCollectedConditions = 0;
16259 VisitedBlocks.insert(Ptr: Block);
16260 std::pair<const BasicBlock *, const BasicBlock *> Pair(Pred, Block);
16261 for (; Pair.first;
16262 Pair = SE.getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
16263 VisitedBlocks.insert(Ptr: Pair.second);
16264 const CondBrInst *LoopEntryPredicate =
16265 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
16266 if (!LoopEntryPredicate)
16267 continue;
16268
16269 Terms.emplace_back(Args: LoopEntryPredicate->getCondition(),
16270 Args: LoopEntryPredicate->getSuccessor(i: 0) == Pair.second);
16271 NumCollectedConditions++;
16272
16273 // If we are recursively collecting guards stop after 2
16274 // conditions to limit compile-time impact for now.
16275 if (Depth > 0 && NumCollectedConditions == 2)
16276 break;
16277 }
16278 // Finally, if we stopped climbing the predecessor chain because
16279 // there wasn't a unique one to continue, try to collect conditions
16280 // for PHINodes by recursively following all of their incoming
16281 // blocks and try to merge the found conditions to build a new one
16282 // for the Phi.
16283 if (Pair.second->hasNPredecessorsOrMore(N: 2) &&
16284 Depth < MaxLoopGuardCollectionDepth) {
16285 SmallDenseMap<const BasicBlock *, LoopGuards> IncomingGuards;
16286 for (auto &Phi : Pair.second->phis())
16287 collectFromPHI(SE, Guards, Phi, VisitedBlocks, IncomingGuards, Depth);
16288 }
16289
16290 // Now apply the information from the collected conditions to
16291 // Guards.RewriteMap. Conditions are processed in reverse order, so the
16292 // earliest conditions is processed first, except guards with divisibility
16293 // information, which are moved to the back. This ensures the SCEVs with the
16294 // shortest dependency chains are constructed first.
16295 SmallVector<std::tuple<CmpInst::Predicate, const SCEV *, const SCEV *>>
16296 GuardsToProcess;
16297 for (auto [Term, EnterIfTrue] : reverse(C&: Terms)) {
16298 SmallVector<Value *, 8> Worklist;
16299 SmallPtrSet<Value *, 8> Visited;
16300 Worklist.push_back(Elt: Term);
16301 while (!Worklist.empty()) {
16302 Value *Cond = Worklist.pop_back_val();
16303 if (!Visited.insert(Ptr: Cond).second)
16304 continue;
16305
16306 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
16307 auto Predicate =
16308 EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate();
16309 const auto *LHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 0));
16310 const auto *RHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 1));
16311 // If LHS is a constant, apply information to the other expression.
16312 // TODO: If LHS is not a constant, check if using CompareSCEVComplexity
16313 // can improve results.
16314 if (isa<SCEVConstant>(Val: LHS)) {
16315 std::swap(a&: LHS, b&: RHS);
16316 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
16317 }
16318 GuardsToProcess.emplace_back(Args&: Predicate, Args&: LHS, Args&: RHS);
16319 continue;
16320 }
16321
16322 Value *L, *R;
16323 if (EnterIfTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: L), R: m_Value(V&: R)))
16324 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: L), R: m_Value(V&: R)))) {
16325 Worklist.push_back(Elt: L);
16326 Worklist.push_back(Elt: R);
16327 }
16328 }
16329 }
16330
16331 // Process divisibility guards in reverse order to populate DivGuards early.
16332 DenseMap<const SCEV *, APInt> Multiples;
16333 LoopGuards DivGuards(SE);
16334 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess) {
16335 if (!isDivisibilityGuard(LHS, RHS, SE))
16336 continue;
16337 collectDivisibilityInformation(Predicate, LHS, RHS, DivInfo&: DivGuards.RewriteMap,
16338 Multiples, SE);
16339 }
16340
16341 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess)
16342 CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap, DivGuards);
16343
16344 // Apply divisibility information last. This ensures it is applied to the
16345 // outermost expression after other rewrites for the given value.
16346 for (const auto &[K, Divisor] : Multiples) {
16347 const SCEV *DivisorSCEV = SE.getConstant(Val: Divisor);
16348 Guards.RewriteMap[K] =
16349 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: applyDivisibilityOnMinMaxExpr(
16350 MinMaxExpr: Guards.rewrite(Expr: K), Divisor, SE),
16351 RHS: DivisorSCEV),
16352 RHS: DivisorSCEV);
16353 ExprsToRewrite.push_back(Elt: K);
16354 }
16355
16356 // Let the rewriter preserve NUW/NSW flags if the unsigned/signed ranges of
16357 // the replacement expressions are contained in the ranges of the replaced
16358 // expressions.
16359 Guards.PreserveNUW = true;
16360 Guards.PreserveNSW = true;
16361 for (const SCEV *Expr : ExprsToRewrite) {
16362 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16363 Guards.PreserveNUW &=
16364 SE.getUnsignedRange(S: Expr).contains(CR: SE.getUnsignedRange(S: RewriteTo));
16365 Guards.PreserveNSW &=
16366 SE.getSignedRange(S: Expr).contains(CR: SE.getSignedRange(S: RewriteTo));
16367 }
16368
16369 // Now that all rewrite information is collect, rewrite the collected
16370 // expressions with the information in the map. This applies information to
16371 // sub-expressions.
16372 if (ExprsToRewrite.size() > 1) {
16373 for (const SCEV *Expr : ExprsToRewrite) {
16374 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16375 Guards.RewriteMap.erase(Val: Expr);
16376 Guards.RewriteMap.insert(KV: {Expr, Guards.rewrite(Expr: RewriteTo)});
16377 }
16378 }
16379}
16380
16381const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
16382 /// A rewriter to replace SCEV expressions in Map with the corresponding entry
16383 /// in the map. It skips AddRecExpr because we cannot guarantee that the
16384 /// replacement is loop invariant in the loop of the AddRec.
16385 class SCEVLoopGuardRewriter
16386 : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> {
16387 const DenseMap<const SCEV *, const SCEV *> &Map;
16388 const SmallDenseSet<std::pair<const SCEV *, const SCEV *>> &NotEqual;
16389
16390 SCEV::NoWrapFlags FlagMask = SCEV::FlagAnyWrap;
16391
16392 public:
16393 SCEVLoopGuardRewriter(ScalarEvolution &SE,
16394 const ScalarEvolution::LoopGuards &Guards)
16395 : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap),
16396 NotEqual(Guards.NotEqual) {
16397 if (Guards.PreserveNUW)
16398 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNUW);
16399 if (Guards.PreserveNSW)
16400 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNSW);
16401 }
16402
16403 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
16404
16405 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
16406 return Map.lookup_or(Val: Expr, Default&: Expr);
16407 }
16408
16409 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
16410 if (const SCEV *S = Map.lookup(Val: Expr))
16411 return S;
16412
16413 // If we didn't find the extact ZExt expr in the map, check if there's
16414 // an entry for a smaller ZExt we can use instead.
16415 Type *Ty = Expr->getType();
16416 const SCEV *Op = Expr->getOperand(i: 0);
16417 unsigned Bitwidth = Ty->getScalarSizeInBits() / 2;
16418 while (Bitwidth % 8 == 0 && Bitwidth >= 8 &&
16419 Bitwidth > Op->getType()->getScalarSizeInBits()) {
16420 Type *NarrowTy = IntegerType::get(C&: SE.getContext(), NumBits: Bitwidth);
16421 auto *NarrowExt = SE.getZeroExtendExpr(Op, Ty: NarrowTy);
16422 if (const SCEV *S = Map.lookup(Val: NarrowExt))
16423 return SE.getZeroExtendExpr(Op: S, Ty);
16424 Bitwidth = Bitwidth / 2;
16425 }
16426
16427 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitZeroExtendExpr(
16428 Expr);
16429 }
16430
16431 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
16432 if (const SCEV *S = Map.lookup(Val: Expr))
16433 return S;
16434 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSignExtendExpr(
16435 Expr);
16436 }
16437
16438 const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) {
16439 if (const SCEV *S = Map.lookup(Val: Expr))
16440 return S;
16441 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitUMinExpr(Expr);
16442 }
16443
16444 const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) {
16445 if (const SCEV *S = Map.lookup(Val: Expr))
16446 return S;
16447 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSMinExpr(Expr);
16448 }
16449
16450 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
16451 // Helper to check if S is a subtraction (A - B) where A != B, and if so,
16452 // return UMax(S, 1).
16453 auto RewriteSubtraction = [&](const SCEV *S) -> const SCEV * {
16454 SCEVUse LHS, RHS;
16455 if (MatchBinarySub(S, LHS, RHS)) {
16456 if (LHS > RHS)
16457 std::swap(a&: LHS, b&: RHS);
16458 if (NotEqual.contains(V: {LHS, RHS})) {
16459 const SCEV *OneAlignedUp = getNextSCEVDivisibleByDivisor(
16460 Expr: SE.getOne(Ty: S->getType()), DivisorVal: SE.getConstantMultiple(S), SE);
16461 return SE.getUMaxExpr(LHS: OneAlignedUp, RHS: S);
16462 }
16463 }
16464 return nullptr;
16465 };
16466
16467 // Check if Expr itself is a subtraction pattern with guard info.
16468 if (const SCEV *Rewritten = RewriteSubtraction(Expr))
16469 return Rewritten;
16470
16471 // Trip count expressions sometimes consist of adding 3 operands, i.e.
16472 // (Const + A + B). There may be guard info for A + B, and if so, apply
16473 // it.
16474 // TODO: Could more generally apply guards to Add sub-expressions.
16475 if (isa<SCEVConstant>(Val: Expr->getOperand(i: 0)) &&
16476 Expr->getNumOperands() == 3) {
16477 const SCEV *Add =
16478 SE.getAddExpr(LHS: Expr->getOperand(i: 1), RHS: Expr->getOperand(i: 2));
16479 if (const SCEV *Rewritten = RewriteSubtraction(Add))
16480 return SE.getAddExpr(
16481 LHS: Expr->getOperand(i: 0), RHS: Rewritten,
16482 Flags: ScalarEvolution::maskFlags(Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16483 if (const SCEV *S = Map.lookup(Val: Add))
16484 return SE.getAddExpr(LHS: Expr->getOperand(i: 0), RHS: S);
16485 }
16486 SmallVector<SCEVUse, 2> Operands;
16487 bool Changed = false;
16488 for (SCEVUse Op : Expr->operands()) {
16489 Operands.push_back(
16490 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
16491 Changed |= Op != Operands.back();
16492 }
16493 // We are only replacing operands with equivalent values, so transfer the
16494 // flags from the original expression.
16495 return !Changed ? Expr
16496 : SE.getAddExpr(Ops&: Operands,
16497 OrigFlags: ScalarEvolution::maskFlags(
16498 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16499 }
16500
16501 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
16502 SmallVector<SCEVUse, 2> Operands;
16503 bool Changed = false;
16504 for (SCEVUse Op : Expr->operands()) {
16505 Operands.push_back(
16506 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
16507 Changed |= Op != Operands.back();
16508 }
16509 // We are only replacing operands with equivalent values, so transfer the
16510 // flags from the original expression.
16511 return !Changed ? Expr
16512 : SE.getMulExpr(Ops&: Operands,
16513 OrigFlags: ScalarEvolution::maskFlags(
16514 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16515 }
16516 };
16517
16518 if (RewriteMap.empty() && NotEqual.empty())
16519 return Expr;
16520
16521 SCEVLoopGuardRewriter Rewriter(SE, *this);
16522 return Rewriter.visit(S: Expr);
16523}
16524
16525const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
16526 return applyLoopGuards(Expr, Guards: LoopGuards::collect(L, SE&: *this));
16527}
16528
16529const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr,
16530 const LoopGuards &Guards) {
16531 return Guards.rewrite(Expr);
16532}
16533